etcd-dump-logs: add decoder support

tools/etcd-dump-logs/README.md

@@ -0,0 +1,129 @@
+### etcd-dump-logs
+
+etcd-dump-logs inspects etcd db files.
+
+```
+Usage:
+
+  etcd-dump-logs [data dir]
+    * Data dir is where the snapshots and WAL logs are located. The structure of the data dir should look like this:
+        - data_dir/member
+            - data_dir/member/snap
+            - data_dir/member/wal
+                - data_dir/member/wal/0000000000000000-0000000000000000.wal
+
+Flags:
+
+  -entry-type string
+    	If set, filters output by entry type. Must be one or more than one of:
+	    ConfigChange, Normal, Request, InternalRaftRequest,
+	    IRRRange, IRRPut, IRRDeleteRange, IRRTxn,
+	    IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke
+  -start-index uint
+    	The index to start dumping
+  -start-snap string
+    	The base name of snapshot file to start dumping
+  -stream-decoder string
+    	The name and arguments of an executable decoding tool, the executable
+    	must process hex encoded lines of binary input (from etcd-dump-logs)
+	    and output a hex encoded line of binary for each input line
+```
+#### etcd-dump-logs -entry-type <ENTRY_TYPE_NAME(S)> [data dir]
+
+Lists all the interested entries from WAL log.
+
+```
+$ etcd-dump-logs -entry-type IRRTxn /tmp/datadir
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"k8s\000\n\025\n\002v1\022\017RangeAllocation\022#\n\022\n\000\022\000\032\000\"\000*\0002\0008\000B\000z\000\022\01310.0.0.0/16\032\000\032\000\"\000" > > failure:<request_delete_range:<key:"a" range_end:"k8s\000\n\025\n\002v1\022\017RangeAllocation\022#\n\022\n\000\022\000\032\000\"\000*\0002\0008\000B\000z\000\022\01310.0.0.0/16\032\000\032\000\"\000" > > >
+
+Entry types (IRRTxn) count is : 1
+
+$ etcd-dump-logs -entry-type ConfigChange,IRRCompaction /tmp/datadir
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data
+   1	         1	conf	method=ConfChangeAddNode id=2
+   2	         2	conf	method=ConfChangeRemoveNode id=2
+   2	         3	conf	method=ConfChangeUpdateNode id=2
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3
+   8	        14	norm	ID:9 compaction:<physical:true >
+
+Entry types (ConfigChange,IRRCompaction) count is : 5
+```
+#### etcd-dump-logs -stream-decoder <EXECUTABLE_DECODER> [data dir]
+
+Decode each entry based on logic in the passed decoder. Decoded status and decoded data are listed in separated tab/columns in the ouput. For parsing purpose, the output from decoder are expected to be in format of "<DECODING_STATUS>|<DECODED_DATA". Please refer to [decoder_correctoutputformat.sh] as an example.
+
+However, if the decoder output format is not as expected, "decoder_status" will be "decoder output format is not right, print output anyway", and all output from decoder will be considered as "decoded_data"
+
+```
+$ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh  /tmp/datadir
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data	decoder_status	decoded_data
+   1	         1	conf	method=ConfChangeAddNode id=2	decoder output format is not right, print output anyway	jhjaajjjahjbbbjj
+   3	         2	norm	noop	decoder output format is not right, print output anyway	jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         3	norm	method=QGET path="/path1"	decoder output format is not right, print output anyway	jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   7	         4	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 	decoder output format is not right, print output anyway	jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
+   8	         5	norm	ID:9 compaction:<physical:true > 	decoder output format is not right, print output anyway	jhjicajbajja
+   9	         6	norm	ID:10 lease_grant:<TTL:1 ID:1 > 	decoder output format is not right, print output anyway	jhjadbjdjhjaajja
+  12	         7	norm	ID:13 auth_enable:<> 	decoder output format is not right, print output anyway	jhjdcbcejj
+  27	         8	norm	???	decoder output format is not right, print output anyway	cf
+Entry types () count is : 8
+
+$ etcd-dump-logs -stream-decoder decoder_wrongoutputformat.sh  /tmp/datadir
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data	decoder_status	decoded_data
+   1	         1	conf	method=ConfChangeAddNode id=2	ERROR	jhjaajjjahjbbbjj
+   3	         2	norm	noop	OK	jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         3	norm	method=QGET path="/path1"	OK	jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   7	         4	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 	OK	jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
+   8	         5	norm	ID:9 compaction:<physical:true > 	ERROR	jhjicajbajja
+   9	         6	norm	ID:10 lease_grant:<TTL:1 ID:1 > 	ERROR	jhjadbjdjhjaajja
+  12	         7	norm	ID:13 auth_enable:<> 	ERROR	jhjdcbcejj
+  27	         8	norm	???	ERROR	cf
+Entry types () count is : 8
+```
+####  etcd-dump-logs -start-index <INDEX NUMBER> [data dir]
+
+Only shows WAL log entries after the specified start-index number, exclusively.
+
+```
+$ etcd-dump-logs -start-index 30  /tmp/datadir
+Start dumping log entries from index 30.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data
+  25	        31	norm	ID:26 auth_role_get:<role:"role3" >
+  26	        32	norm	ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > >
+  27	        33	norm	ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" >
+  27	        34	norm	???
+Entry types () count is : 4
+```
+[decoder_correctoutputformat.sh]: ./testdecoder/decoder_correctoutputformat.sh

tools/etcd-dump-logs/etcd-dump-log_test.go

@@ -45,6 +45,9 @@ func TestEtcdDumpLogEntryType(t *testing.T) {
 		t.Skipf("%q does not exist", dumpLogsBinary)
 	}
 
+	decoder_correctoutputformat := filepath.Join(binDir, "/testdecoder/decoder_correctoutputformat.sh")
+	decoder_wrongoutputformat := filepath.Join(binDir, "/testdecoder/decoder_wrongoutputformat.sh")
+
 	p, err := ioutil.TempDir(os.TempDir(), "etcddumplogstest")
 	if err != nil {
 		t.Fatal(err)
@@ -102,6 +105,8 @@ func TestEtcdDumpLogEntryType(t *testing.T) {
 		{"lease grant entry-type", []string{"-entry-type", "IRRLeaseGrant", p}, "expectedoutput/listIRRLeaseGrant.output"},
 		{"lease revoke entry-type", []string{"-entry-type", "IRRLeaseRevoke", p}, "expectedoutput/listIRRLeaseRevoke.output"},
 		{"confchange and txn entry-type", []string{"-entry-type", "ConfigChange,IRRCompaction", p}, "expectedoutput/listConfigChangeIRRCompaction.output"},
+		{"decoder_correctoutputformat", []string{"-stream-decoder", decoder_correctoutputformat, p}, "expectedoutput/decoder_correctoutputformat.output"},
+		{"decoder_wrongoutputformat", []string{"-stream-decoder", decoder_wrongoutputformat, p}, "expectedoutput/decoder_wrongoutputformat.output"},
 	}
 
 	for _, argtest := range argtests {

tools/etcd-dump-logs/expectedoutput/decoder_correctoutputformat.output

@@ -0,0 +1,44 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data	decoder_status	decoded_data
+   1	         1	conf	method=ConfChangeAddNode id=2	ERROR	jhjaajjjahjbbbjj
+   2	         2	conf	method=ConfChangeRemoveNode id=2	ERROR	jhjbajjaahjbbbjj
+   2	         3	conf	method=ConfChangeUpdateNode id=2	ERROR	jhjcajjbahjbbbjj
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3	ERROR	jhjdajjcahjcbbjj
+   3	         5	norm	noop	OK	jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         6	norm	method=QGET path="/path1"	OK	jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         7	norm	method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"	OK	jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         8	norm	method=DELETE path="/path3"	OK	jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         9	norm	method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"	OK	jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 	OK	jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 	OK	jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 	OK	jhjgbajhjajacjabjaciahja
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 	OK	jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
+   8	        14	norm	ID:9 compaction:<physical:true > 	ERROR	jhjicajbajja
+   9	        15	norm	ID:10 lease_grant:<TTL:1 ID:1 > 	ERROR	jhjadbjdjhjaajja
+  10	        16	norm	ID:11 lease_revoke:<ID:2 > 	ERROR	jhjbdajbjhjb
+  11	        17	norm	ID:12 alarm:<action:3 memberID:4 alarm:5 > 	OK	jhjcebjfjhjcajjdahje
+  12	        18	norm	ID:13 auth_enable:<> 	ERROR	jhjdcbcejj
+  13	        19	norm	ID:14 auth_disable:<> 	ERROR	jhjeiacfjj
+  14	        20	norm	ID:15 authenticate:<name:"myname" password:"password" simple_token:"token" > 	OK	jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe
+  15	        21	norm	ID:16 auth_user_add:<name:"name1" password:"pass1" > 	OK	jhajebddjejajefefafdfecaabjegjfagcgcca
+  16	        22	norm	ID:17 auth_user_delete:<name:"name1" > 	OK	jhaaeaddjgjajefefafdfeca
+  17	        23	norm	ID:18 auth_user_get:<name:"name1" > 	OK	jhabfbddjgjajefefafdfeca
+  18	        24	norm	ID:19 auth_user_change_password:<name:"name1" password:"pass2" > 	OK	jhacfaddjejajefefafdfecaabjegjfagcgccb
+  19	        25	norm	ID:20 auth_user_grant_role:<user:"user1" role:"role1" > 	OK	jhadhbdejejajegegcfegbcaabjegbfffcfeca
+  20	        26	norm	ID:21 auth_user_revoke_role:<name:"user2" role:"role2" > 	OK	jhaehadejejajegegcfegbcbabjegbfffcfecb
+  21	        27	norm	ID:22 auth_user_list:<> 	ERROR	jhafibdejj
+  22	        28	norm	ID:23 auth_role_list:<> 	ERROR	jhagiadejj
+  23	        29	norm	ID:24 auth_role_add:<name:"role2" > 	OK	jhahhbdbjgjajegbfffcfecb
+  24	        30	norm	ID:25 auth_role_delete:<role:"role1" > 	OK	jhaihadbjgjajegbfffcfeca
+  25	        31	norm	ID:26 auth_role_get:<role:"role3" > 	OK	jhaaibdbjgjajegbfffcfecc
+  26	        32	norm	ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > > 	OK	jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd
+  27	        33	norm	ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" > 	OK	jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd
+  27	        34	norm	???	ERROR	cf
+
+Entry types () count is : 34

tools/etcd-dump-logs/expectedoutput/decoder_wrongoutputformat.output

@@ -0,0 +1,44 @@
+Snapshot:
+empty
+Start dupmping log entries from snapshot.
+WAL metadata:
+nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
+WAL entries:
+lastIndex=34
+term	     index	type	data	decoder_status	decoded_data
+   1	         1	conf	method=ConfChangeAddNode id=2	decoder output format is not right, print output anyway	jhjaajjjahjbbbjj
+   2	         2	conf	method=ConfChangeRemoveNode id=2	decoder output format is not right, print output anyway	jhjbajjaahjbbbjj
+   2	         3	conf	method=ConfChangeUpdateNode id=2	decoder output format is not right, print output anyway	jhjcajjbahjbbbjj
+   2	         4	conf	method=ConfChangeAddLearnerNode id=3	decoder output format is not right, print output anyway	jhjdajjcahjcbbjj
+   3	         5	norm	noop	decoder output format is not right, print output anyway	jhjjabjjaajfbfgjfagdfhcjbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjacbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         6	norm	method=QGET path="/path1"	decoder output format is not right, print output anyway	jhjaabjdeadgdeedaajfbfgjfagdfhcabbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjiejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         7	norm	method=SYNC time="1969-12-31 16:00:00.000000001 -0800 PST"	decoder output format is not right, print output anyway	jhjbabjdeceidedcaajfbfgjfagdfhcbbbacgbbbcjbbcabbcabbbcbbcbbbcaebbbccbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         8	norm	method=DELETE path="/path3"	decoder output format is not right, print output anyway	jhjcabjfdddedcdeeddeaajfbfgjfagdfhccbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjadhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   3	         9	norm	method=RANDOM path="/path4/superlong/path/path/path/path/path/path/path/path/path/pa"..."path/path/path/path/path/path/path/path/path/path/path/path/path" val="{\"hey\":\"ho\",\"hi\":[\"yo\"]}"	decoder output format is not right, print output anyway	jhjdabjfebdadedddfddaaafjabfgjfagdfhcdbfgcgegjfegbfcfffefgbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbfgjfagdfhbbahgbbbfhfegibbcabbfhffbbbcbbfhfibbcaebbbgiffbbedgdbhjjcbjjchjjdjjjdhjbejjjehjafjjjfhjjgjjjghjahjjajjhhjajj
+   4	        10	norm	ID:5 range:<key:"1" range_end:"hi" limit:6 revision:1 sort_order:ASCEND max_mod_revision:20000 max_create_revision:20000 > 	decoder output format is not right, print output anyway	jhjeaaaejajacaabjbfhfiahjfbjjabhjaehajicjafhajicja
+   5	        11	norm	ID:6 put:<key:"foo1" value:"bar1" lease:1 ignore_lease:true > 	decoder output format is not right, print output anyway	jhjfbbajjajdffffffcaabjdfbfagbcaahjacjja
+   6	        12	norm	ID:7 delete_range:<key:"0" range_end:"9" prev_kv:true > 	decoder output format is not right, print output anyway	jhjgbajhjajacjabjaciahja
+   7	        13	norm	ID:8 txn:<success:<request_delete_range:<key:"a" range_end:"b" > > failure:<request_delete_range:<key:"a" range_end:"b" > > > 	decoder output format is not right, print output anyway	jhjhcbadabjhaajfjajafaabjafbaajhaajfjajafaabjafb
+   8	        14	norm	ID:9 compaction:<physical:true > 	decoder output format is not right, print output anyway	jhjicajbajja
+   9	        15	norm	ID:10 lease_grant:<TTL:1 ID:1 > 	decoder output format is not right, print output anyway	jhjadbjdjhjaajja
+  10	        16	norm	ID:11 lease_revoke:<ID:2 > 	decoder output format is not right, print output anyway	jhjbdajbjhjb
+  11	        17	norm	ID:12 alarm:<action:3 memberID:4 alarm:5 > 	decoder output format is not right, print output anyway	jhjcebjfjhjcajjdahje
+  12	        18	norm	ID:13 auth_enable:<> 	decoder output format is not right, print output anyway	jhjdcbcejj
+  13	        19	norm	ID:14 auth_disable:<> 	decoder output format is not right, print output anyway	jhjeiacfjj
+  14	        20	norm	ID:15 authenticate:<name:"myname" password:"password" simple_token:"token" > 	decoder output format is not right, print output anyway	jhjfabcfaijajffdgifefafdfeabjhgjfagcgcggffgbfdaajegdfffbfefe
+  15	        21	norm	ID:16 auth_user_add:<name:"name1" password:"pass1" > 	decoder output format is not right, print output anyway	jhajebddjejajefefafdfecaabjegjfagcgcca
+  16	        22	norm	ID:17 auth_user_delete:<name:"name1" > 	decoder output format is not right, print output anyway	jhaaeaddjgjajefefafdfeca
+  17	        23	norm	ID:18 auth_user_get:<name:"name1" > 	decoder output format is not right, print output anyway	jhabfbddjgjajefefafdfeca
+  18	        24	norm	ID:19 auth_user_change_password:<name:"name1" password:"pass2" > 	decoder output format is not right, print output anyway	jhacfaddjejajefefafdfecaabjegjfagcgccb
+  19	        25	norm	ID:20 auth_user_grant_role:<user:"user1" role:"role1" > 	decoder output format is not right, print output anyway	jhadhbdejejajegegcfegbcaabjegbfffcfeca
+  20	        26	norm	ID:21 auth_user_revoke_role:<name:"user2" role:"role2" > 	decoder output format is not right, print output anyway	jhaehadejejajegegcfegbcbabjegbfffcfecb
+  21	        27	norm	ID:22 auth_user_list:<> 	decoder output format is not right, print output anyway	jhafibdejj
+  22	        28	norm	ID:23 auth_role_list:<> 	decoder output format is not right, print output anyway	jhagiadejj
+  23	        29	norm	ID:24 auth_role_add:<name:"role2" > 	decoder output format is not right, print output anyway	jhahhbdbjgjajegbfffcfecb
+  24	        30	norm	ID:25 auth_role_delete:<role:"role1" > 	decoder output format is not right, print output anyway	jhaihadbjgjajegbfffcfeca
+  25	        31	norm	ID:26 auth_role_get:<role:"role3" > 	decoder output format is not right, print output anyway	jhaaibdbjgjajegbfffcfecc
+  26	        32	norm	ID:27 auth_role_grant_permission:<name:"role3" perm:<permType:WRITE key:"Keys" range_end:"RangeEnd" > > 	decoder output format is not right, print output anyway	jhabiadbabjajegbfffcfeccababjhjaabjddbfegigcaajhebfafefgfedefefd
+  27	        33	norm	ID:28 auth_role_revoke_permission:<role:"role3" key:"key" range_end:"rangeend" > 	decoder output format is not right, print output anyway	jhacabdbafjajegbfffcfeccabjcfbfegiaajhgbfafefgfefefefd
+  27	        34	norm	???	decoder output format is not right, print output anyway	cf
+
+Entry types () count is : 34

tools/etcd-dump-logs/expectedoutput/listConfigChangeIRRLeaseRevoke.output

@@ -1,11 +0,0 @@
-Snapshot:
-empty
-Start dupmping log entries from snapshot.
-WAL metadata:
-nodeID=0 clusterID=0 term=0 commitIndex=0 vote=0
-WAL entries:
-lastIndex=34
-term	     index	type	data
-  10	        16	norm	ID:11 lease_revoke:<ID:2 > 
-
-Entry types (IRRLeaseRevoke) count is : 1

tools/etcd-dump-logs/main.go

@@ -15,9 +15,15 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"encoding/hex"
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
@@ -40,6 +46,9 @@ func main() {
 	ConfigChange, Normal, Request, InternalRaftRequest,
 	IRRRange, IRRPut, IRRDeleteRange, IRRTxn,
 	IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`)
+	streamdecoder := flag.String("stream-decoder", "", `The name of an executable decoding tool, the executable must process 
+	hex encoded lines of binary input (from etcd-dump-logs) 
+	and output a hex encoded line of binary for each input line`)
 
 	flag.Parse()
 
@@ -101,8 +110,14 @@ func main() {
 
 	fmt.Printf("WAL entries:\n")
 	fmt.Printf("lastIndex=%d\n", ents[len(ents)-1].Index)
-	fmt.Printf("%4s\t%10s\ttype\tdata\n", "term", "index")
-	listEntriesType(*entrytype, ents)
+
+	fmt.Printf("%4s\t%10s\ttype\tdata", "term", "index")
+	if *streamdecoder != "" {
+		fmt.Printf("\tdecoder_status\tdecoded_data")
+	}
+	fmt.Println()
+
+	listEntriesType(*entrytype, *streamdecoder, ents)
 }
 
 func walDir(dataDir string) string { return filepath.Join(dataDir, "member", "wal") }
@@ -203,12 +218,12 @@ type EntryPrinter func(e raftpb.Entry)
 func printInternalRaftRequest(entry raftpb.Entry) {
 	var rr etcdserverpb.InternalRaftRequest
 	if err := rr.Unmarshal(entry.Data); err == nil {
-		fmt.Printf("%4d\t%10d\tnorm\t%s\n", entry.Term, entry.Index, rr.String())
+		fmt.Printf("%4d\t%10d\tnorm\t%s", entry.Term, entry.Index, rr.String())
 	}
 }
 
 func printUnknownNormal(entry raftpb.Entry) {
-	fmt.Printf("%4d\t%10d\tnorm\t???\n", entry.Term, entry.Index)
+	fmt.Printf("%4d\t%10d\tnorm\t???", entry.Term, entry.Index)
 }
 
 func printConfChange(entry raftpb.Entry) {
@@ -216,9 +231,9 @@ func printConfChange(entry raftpb.Entry) {
 	fmt.Printf("\tconf")
 	var r raftpb.ConfChange
 	if err := r.Unmarshal(entry.Data); err != nil {
-		fmt.Printf("\t???\n")
+		fmt.Printf("\t???")
 	} else {
-		fmt.Printf("\tmethod=%s id=%s\n", r.Type, types.ID(r.NodeID))
+		fmt.Printf("\tmethod=%s id=%s", r.Type, types.ID(r.NodeID))
 	}
 }
 
@@ -228,13 +243,13 @@ func printRequest(entry raftpb.Entry) {
 		fmt.Printf("%4d\t%10d\tnorm", entry.Term, entry.Index)
 		switch r.Method {
 		case "":
-			fmt.Printf("\tnoop\n")
+			fmt.Printf("\tnoop")
 		case "SYNC":
-			fmt.Printf("\tmethod=SYNC time=%q\n", time.Unix(0, r.Time))
+			fmt.Printf("\tmethod=SYNC time=%q", time.Unix(0, r.Time))
 		case "QGET", "DELETE":
-			fmt.Printf("\tmethod=%s path=%s\n", r.Method, excerpt(r.Path, 64, 64))
+			fmt.Printf("\tmethod=%s path=%s", r.Method, excerpt(r.Path, 64, 64))
 		default:
-			fmt.Printf("\tmethod=%s path=%s val=%s\n", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0))
+			fmt.Printf("\tmethod=%s path=%s val=%s", r.Method, excerpt(r.Path, 64, 64), excerpt(r.Val, 128, 0))
 		}
 	}
 }
@@ -281,13 +296,33 @@ IRRCompaction, IRRLeaseGrant, IRRLeaseRevoke`, et)
 }
 
 //  listEntriesType filters and prints entries based on the entry-type flag,
-func listEntriesType(entrytype string, ents []raftpb.Entry) {
+func listEntriesType(entrytype string, streamdecoder string, ents []raftpb.Entry) {
 	entryFilters := evaluateEntrytypeFlag(entrytype)
 	printerMap := map[string]EntryPrinter{"InternalRaftRequest": printInternalRaftRequest,
 		"Request":       printRequest,
 		"ConfigChange":  printConfChange,
 		"UnknownNormal": printUnknownNormal}
+	var stderr bytes.Buffer
+	args := strings.Split(streamdecoder, " ")
+	cmd := exec.Command(args[0], args[1:]...)
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		log.Panic(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		log.Panic(err)
+	}
+	cmd.Stderr = &stderr
+	if streamdecoder != "" {
+		err = cmd.Start()
+		if err != nil {
+			log.Panic(err)
+		}
+	}
+
 	cnt := 0
+
 	for _, e := range ents {
 		passed := false
 		currtype := ""
@@ -301,7 +336,55 @@ func listEntriesType(entrytype string, ents []raftpb.Entry) {
 		if passed {
 			printer := printerMap[currtype]
 			printer(e)
+			if streamdecoder == "" {
+				fmt.Println()
+				continue
+			}
+
+			// if decoder is set, pass the e.Data to stdin and read the stdout from decoder
+			io.WriteString(stdin, hex.EncodeToString(e.Data))
+			io.WriteString(stdin, "\n")
+			outputReader := bufio.NewReader(stdout)
+			decoderoutput, currerr := outputReader.ReadString('\n')
+			if currerr != nil {
+				fmt.Println(currerr)
+				return
+			}
+
+			decoder_status, decoded_data := parseDecoderOutput(decoderoutput)
+
+			fmt.Printf("\t%s\t%s", decoder_status, decoded_data)
+		}
+	}
+
+	stdin.Close()
+	err = cmd.Wait()
+	if streamdecoder != "" {
+		if err != nil {
+			log.Panic(err)
+		}
+		if stderr.String() != "" {
+			os.Stderr.WriteString("decoder stderr: " + stderr.String())
 		}
 	}
+
 	fmt.Printf("\nEntry types (%s) count is : %d", entrytype, cnt)
 }
+
+func parseDecoderOutput(decoderoutput string) (string, string) {
+	var decoder_status string
+	var decoded_data string
+	output := strings.Split(decoderoutput, "|")
+	switch len(output) {
+	case 1:
+		decoder_status = "decoder output format is not right, print output anyway"
+		decoded_data = decoderoutput
+	case 2:
+		decoder_status = output[0]
+		decoded_data = output[1]
+	default:
+		decoder_status = output[0] + "(*WARNING: data might contain deliminator used by etcd-dump-logs)"
+		decoded_data = strings.Join(output[1:], "")
+	}
+	return decoder_status, decoded_data
+}

tools/etcd-dump-logs/testdecoder/decoder_correctoutputformat.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+while read line
+do
+  LEN=$(echo ${#line})
+  if [ $LEN -ge 20 ]; then
+  	echo "OK|$line" | tr 1234567890 abcdefghij
+  else
+  	echo "ERROR|$line" | tr 1234567890 abcdefghij
+  fi
+done < "${1:-/dev/stdin}"

tools/etcd-dump-logs/testdecoder/decoder_wrongoutputformat.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+while read line
+do
+  echo "$line" | tr 1234567890 abcdefghij
+done < "${1:-/dev/stdin}"