aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Kacer <kacer.martin@gmail.com>2017-11-04 08:49:54 +0100
committerAnders Broman <a.broman58@gmail.com>2017-11-06 06:17:49 +0000
commit156a0b62eb9dee2423b6d7d7f5aed1a5b6cb33ae (patch)
tree09f817dd9fad7c9913f4ab2e9d5b0ace45e0bed9
parentba6dd97081c7490db0d6e98c9e8cd3b819b9dd6f (diff)
Tools: json2pcap script update
Updated script to support duplicated json fields and --no-duplicate-keys tshark flag. Bug: 12958 Change-Id: Id20885baacbfd26124cb82716df6829225be6de8 Reviewed-on: https://code.wireshark.org/review/24236 Reviewed-by: Anders Broman <a.broman58@gmail.com>
-rw-r--r--tools/json2pcap/json2pcap.py276
1 files changed, 183 insertions, 93 deletions
diff --git a/tools/json2pcap/json2pcap.py b/tools/json2pcap/json2pcap.py
index 50d0ab5..0256698 100644
--- a/tools/json2pcap/json2pcap.py
+++ b/tools/json2pcap/json2pcap.py
@@ -33,6 +33,25 @@ import argparse
import subprocess
from collections import OrderedDict
+def make_unique(key, dct):
+ counter = 0
+ unique_key = key
+
+ while unique_key in dct:
+ counter += 1
+ unique_key = '{}_{}'.format(key, counter)
+ return unique_key
+
+
+def parse_object_pairs(pairs):
+ dct = OrderedDict()
+ for key, value in pairs:
+ if key in dct:
+ key = make_unique(key, dct)
+ dct[key] = value
+
+ return dct
+
#
# ********* PY TEMPLATES *********
#
@@ -92,73 +111,10 @@ py_footer = py_footer + read_py_function("hex_to_txt")
py_footer = py_footer + read_py_function("to_bytes")
py_footer = py_footer + read_py_function("lsb")
py_footer = py_footer + read_py_function("rewrite_frame")
+py_footer = py_footer + read_py_function("assemble_frame")
+py_footer = py_footer + read_py_function("generate_pcap")
py_footer = py_footer + """
-def assemble_frame(d):
- input = d['frame_raw'][1]
- isFlat = False
- linux_cooked_header = False;
- while(isFlat == False):
- isFlat = True
- for key, val in d.items():
- h = str(val[1]) # hex
- p = val[2] * 2 # position
- l = val[3] * 2 # length
- b = val[4] # bitmask
- t = val[5] # type
-
- if (key == "sll_raw"):
- linux_cooked_header = True;
-
- # only if the node is not parent
- isParent = False
- for k, v in d.items():
- if (v[0] == key):
- isParent = True
- isFlat = False
- break
-
- if (isParent == False and val[0] is not None):
- d[val[0]][1] = rewrite_frame(d[val[0]][1], h, p, l, b, t)
- del d[key]
-
- output = d['frame_raw'][1]
-
- # for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap
- if (linux_cooked_header):
- output = "000000000000" + output[6*2:] # replce dest MAC
- output = output[:12*2] + "" + output[14*2:] # remove two bytes before Protocol
-
- return output
-
-def generate_pcap(d):
- # 1. Assemble frame
- input = d['frame_raw'][1]
- output = assemble_frame(d)
- print(input)
- print(output)
-
- # 2. Testing: compare input and output for not modified json
- if (input != output):
- print("Modified frames: ")
- s1 = input
- s2 = output
- print(s1)
- print(s2)
- if (len(s1) == len(s2)):
- d = [i for i in xrange(len(s1)) if s1[i] != s2[i]]
- print(d)
-
- # 3. Open TMP file used by text2pcap
- file = sys.argv[0] + '.tmp'
- f = open(file,'w')
- hex_to_txt(output, file)
- f.close()
-
- # 4. Generate pcap
- to_pcap_file(sys.argv[0] + '.tmp', sys.argv[0] + '.pcap')
- print("Generated " + sys.argv[0] + ".tmp")
- print("Generated " + sys.argv[0] + ".pcap")
if __name__ == '__main__':
main()
@@ -212,25 +168,54 @@ def py_generator(d, r, frame_name='frame_raw', frame_position=0):
if hasattr(d, 'items'):
for k, v in d.items():
+
# no recursion
- if k.endswith("_raw"):
- h = v[0]
- p = v[1]
- l = v[2] * 2
- b = v[3]
- t = v[4]
- if (len(h) != l):
- l = len(h)
-
- p = p - frame_position
-
- # Add into result dictionary
- key = str(k).replace('.', '_')
- fn = frame_name.replace('.', '_')
- if (fn == key):
- fn = None
- value = [fn , h, p, l, b, t]
- r[key] = value
+ if ( k.endswith("_raw") or ("_raw_" in k) ):
+ if (isinstance(v[1], (list, tuple)) or isinstance(v[2], (list, tuple)) ):
+ #i = 1;
+ for _v in v:
+ h = _v[0]
+ p = _v[1]
+ l = _v[2] * 2
+ b = _v[3]
+ t = _v[4]
+ if (len(h) != l):
+ l = len(h)
+
+ p = p - frame_position
+
+ # Add into result dictionary
+ key = str(k).replace('.', '_')
+ key = make_unique(key, r)
+
+ fn = frame_name.replace('.', '_')
+ if (fn == key):
+ fn = None
+ value = [fn , h, p, l, b, t]
+
+ r[key] = value
+
+ else:
+ h = v[0]
+ p = v[1]
+ l = v[2] * 2
+ b = v[3]
+ t = v[4]
+ if (len(h) != l):
+ l = len(h)
+
+ p = p - frame_position
+
+ # Add into result dictionary
+ key = str(k).replace('.', '_')
+ key = make_unique(key, r)
+
+ fn = frame_name.replace('.', '_')
+ if (fn == key):
+ fn = None
+ value = [fn , h, p, l, b, t]
+
+ r[key] = value
# recursion
else:
@@ -241,16 +226,40 @@ def py_generator(d, r, frame_name='frame_raw', frame_position=0):
# if there is also preceding raw protocol frame use it
# remove tree suffix
key = k
- if key.endswith("_tree"):
- key = key[:-5]
+ if (key.endswith("_tree") or ("_tree_" in key)):
+ key = key.replace('_tree', '')
+
raw_key = key + "_raw"
if (raw_key in d):
# f = d[raw_key][0]
fn = raw_key
fp = d[raw_key][1]
+
py_generator(v, r, fn, fp)
+ elif isinstance(v, (list, tuple)):
+
+ fn = frame_name
+ fp = frame_position
+
+ # if there is also preceding raw protocol frame use it
+ # remove tree suffix
+ key = k
+ if (key.endswith("_tree") or ("_tree_" in key)):
+ key = key.replace('_tree', '')
+
+ raw_key = key + "_raw"
+ if (raw_key in d):
+ fn = raw_key
+ fp = d[raw_key][1]
+
+ for _v in v:
+ py_generator(_v, r, frame_name, frame_position)
+
+
+
+
# To emulate Python 3.2
def to_bytes(n, length, endianess='big'):
h = '%x' % n
@@ -325,6 +334,73 @@ def rewrite_frame(frame_raw, h, p, l, b, t):
return frame_raw[:p] + masked_h + frame_raw[p + l:]
+
+def assemble_frame(d):
+ input = d['frame_raw'][1]
+ isFlat = False
+ linux_cooked_header = False;
+ while(isFlat == False):
+ isFlat = True
+ for key, val in d.items():
+ h = str(val[1]) # hex
+ p = val[2] * 2 # position
+ l = val[3] * 2 # length
+ b = val[4] # bitmask
+ t = val[5] # type
+
+ if (key == "sll_raw"):
+ linux_cooked_header = True;
+
+ # only if the node is not parent
+ isParent = False
+ for k, v in d.items():
+ if (v[0] == key):
+ isParent = True
+ isFlat = False
+ break
+
+ if (isParent == False and val[0] is not None):
+ d[val[0]][1] = rewrite_frame(d[val[0]][1], h, p, l, b, t)
+ del d[key]
+
+ output = d['frame_raw'][1]
+
+ # for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap
+ if (linux_cooked_header):
+ output = "000000000000" + output[6*2:] # replce dest MAC
+ output = output[:12*2] + "" + output[14*2:] # remove two bytes before Protocol
+
+ return output
+
+def generate_pcap(d):
+ # 1. Assemble frame
+ input = d['frame_raw'][1]
+ output = assemble_frame(d)
+ print(input)
+ print(output)
+
+ # 2. Testing: compare input and output for not modified json
+ if (input != output):
+ print("Modified frames: ")
+ s1 = input
+ s2 = output
+ print(s1)
+ print(s2)
+ if (len(s1) == len(s2)):
+ d = [i for i in xrange(len(s1)) if s1[i] != s2[i]]
+ print(d)
+
+ # 3. Open TMP file used by text2pcap
+ file = sys.argv[0] + '.tmp'
+ f = open(file,'w')
+ hex_to_txt(output, file)
+ f.close()
+
+ # 4. Generate pcap
+ to_pcap_file(sys.argv[0] + '.tmp', sys.argv[0] + '.pcap')
+ print("Generated " + sys.argv[0] + ".tmp")
+ print("Generated " + sys.argv[0] + ".pcap")
+
#
# ************ MAIN **************
#
@@ -361,7 +437,8 @@ args = parser.parse_args()
infile = args.infile[0]
with open(infile) as data_file:
- json = json.load(data_file, object_pairs_hook=OrderedDict)
+ #json = json.load(data_file, object_pairs_hook=OrderedDict)
+ json = json.load(data_file, object_pairs_hook=parse_object_pairs)
input_frame_raw = ''
frame_raw = ''
@@ -374,21 +451,21 @@ if args.python == False:
# Iterate over packets in JSON
for packet in json:
- list = []
+ _list = []
linux_cooked_header = False;
- # get flat raw fields into list
+ # get flat raw fields into _list
for raw in raw_flat_collector(packet['_source']['layers']):
if (raw[0] == "frame_raw"):
frame_raw = raw[1][0]
input_frame_raw = copy.copy(frame_raw)
else:
- list.append(raw[1])
+ _list.append(raw[1])
if (raw[0] == "sll_raw"):
linux_cooked_header = True
- # sort list
- sorted_list = sorted(list, key=operator.itemgetter(1), reverse=False)
+ # sort _list
+ sorted_list = sorted(_list, key=operator.itemgetter(1), reverse=False)
sorted_list = sorted(sorted_list, key=operator.itemgetter(2), reverse=True)
# print("Debug: " + str(sorted_list))
@@ -400,8 +477,19 @@ if args.python == False:
b = raw[3] # bitmask
t = raw[4] # type
- # print("Debug: " + str(raw))
- frame_raw = rewrite_frame(frame_raw, h, p, l, b, t)
+ if (isinstance(p, (list, tuple)) or isinstance(l, (list, tuple))):
+ for r in raw:
+ _h = str(r[0]) # hex
+ _p = r[1] * 2 # position
+ _l = r[2] * 2 # length
+ _b = r[3] # bitmask
+ _t = r[4] # type
+ # print("Debug: " + str(raw))
+ frame_raw = rewrite_frame(frame_raw, _h, _p, _l, _b, _t)
+
+ else:
+ # print("Debug: " + str(raw))
+ frame_raw = rewrite_frame(frame_raw, h, p, l, b, t)
# for Linux cooked header replace dest MAC and remove two bytes to reconstruct normal frame using text2pcap
if (linux_cooked_header):
@@ -434,6 +522,8 @@ else:
f.write(py_header)
r = OrderedDict({})
+
+ #print "packet = " + str(packet['_source']['layers'])
py_generator(packet['_source']['layers'], r)
for key, value in r.iteritems() :