add time since last detection col

dataFilter.py

@@ -41,8 +41,8 @@ def squash(data):
 	for line in data:
 		if line[0] == '#':
 			if line.startswith("#timestamp"):
-				header="#begin, end, dur," + DELIM.join(line.split(',')[OMIT_COL_BEGIN:])
-				filtered.append(header)
+				global header
+				header="#begin, end, dur, last detect, PXE," + DELIM.join(line.split(',')[OMIT_COL_BEGIN:])
 			continue
 
 		l = line.split(DELIM)
@@ -99,11 +99,42 @@ def squash(data):
 
 	return filtered
 
+def add_lastDetect_and_PXE( data ):
+	"add 'time since last detection' and 'patch-Cross-Event'"
+
+	OUT_COL_END=1
+	OUT_COL_LASTDETECT=3
+	OUT_COL_PXE=4
+	
+
+	known = {} #{tag: line}
+
+	for i,line in enumerate(data):
+
+		l = line.split(DELIM)
+		tag = l[ OUT_COL_TAG ]
+
+		if tag not in known:
+			l.insert(OUT_COL_LASTDETECT,"NA")
+			# l.insert(OUT_COL_PXE,"TODO")
+
+		else:
+			last_end = int(known[tag][OUT_COL_END]) 
+			this_start = int(l[OUT_COL_END])
+			l.insert(OUT_COL_LASTDETECT,this_start - last_end)
+
+		known[tag] = l
+
+			
+		data[i] = DELIM.join(map(str,l))
+
+	return data
 
 def write(outFile, data):
 
 	print(f"writing to {outFile}")
 	with open(outFile, "w") as f:
+		f.writelines(header)
 		f.writelines(data)
 
 	# cmd_compress=f"tar -zcvf {outFile}.tgz {outFile}" # ">/dev/null 2>/dev/null"
@@ -115,8 +146,11 @@ def main(filename):
 		data = f.readlines()
 
 	data = squash( data )
+	write(filename + "-filtered-squashed", data)
+
+	data = add_lastDetect_and_PXE( data )
 
-	write(filename + "-filtered", data)
+	write(filename + "-filtered-added", data)
 
 if __name__ == "__main__" :