diff --git a/dataFilter.py b/dataFilter.py index 1653a760bd9067696dd26b1a857f6398c6b10d24..249fce9fd35c50f5c1cb850cd9602c271440b9c0 100755 --- a/dataFilter.py +++ b/dataFilter.py @@ -41,8 +41,8 @@ def squash(data): for line in data: if line[0] == '#': if line.startswith("#timestamp"): - header="#begin, end, dur," + DELIM.join(line.split(',')[OMIT_COL_BEGIN:]) - filtered.append(header) + global header + header="#begin, end, dur, last detect, PXE," + DELIM.join(line.split(',')[OMIT_COL_BEGIN:]) continue l = line.split(DELIM) @@ -91,7 +91,7 @@ def squash(data): r[1] = r[0] ncut += r[2] r[2] = 0 - + filtered.append(DELIM.join(map(str,r))) if cutOff: @@ -99,11 +99,42 @@ def squash(data): return filtered +def add_lastDetect_and_PXE( data ): + "add 'time since last detection' and 'patch-Cross-Event'" + + OUT_COL_END=1 + OUT_COL_LASTDETECT=3 + OUT_COL_PXE=4 + + + known = {} #{tag: line} + + for i,line in enumerate(data): + + l = line.split(DELIM) + tag = l[ OUT_COL_TAG ] + + if tag not in known: + l.insert(OUT_COL_LASTDETECT,"NA") + # l.insert(OUT_COL_PXE,"TODO") + + else: + last_end = int(known[tag][OUT_COL_END]) + this_start = int(l[OUT_COL_END]) + l.insert(OUT_COL_LASTDETECT,this_start - last_end) + + known[tag] = l + + + data[i] = DELIM.join(map(str,l)) + + return data def write(outFile, data): print(f"writing to {outFile}") with open(outFile, "w") as f: + f.writelines(header) f.writelines(data) # cmd_compress=f"tar -zcvf {outFile}.tgz {outFile}" # ">/dev/null 2>/dev/null" @@ -115,8 +146,11 @@ def main(filename): data = f.readlines() data = squash( data ) + write(filename + "-filtered-squashed", data) + + data = add_lastDetect_and_PXE( data ) - write(filename + "-filtered", data) + write(filename + "-filtered-added", data) if __name__ == "__main__" :