dataFilter: Squash Plateaus. trackevent: start,end,dur (end = no trackevent in...

dataFilter: Squash Plateaus. trackevent: start,end,dur (end = no trackevent in start+coolOff). Check same position to detect really quick beetles. Optional CutOff from end. RM some cols (ms,date,time)

dataFilter.py

 #!/usr/bin/python3
-
-#todo 
 #
-#	define trackevent: start,end,dur (end = no trackevent in start+coolOff)
-#	
+# Todo 
+#
 #	time since last detection
-
+#
 #	sensor-crossing. SxE
-#	> patch-crossing? PxE
+#	>> patch-crossing? PxE
 #
 #	dont cutOff first trackEvent from plateau to end
 #
@@ -16,96 +14,122 @@
 #		nLines after coolOff
 #		nLines after cutOff (total nDel lines)
 #
-#	def coolOff() and def cutOff()
-#	think about plateaus? eg. when a beetle chills on sensor for a day? (no patchcrossevent - pce)
-#
 import sys, os
 
+cutOff=True
+
 LEN_TIMESTAMP=len("1602626336")
 COL_TIMESTAMP=0
-COL_TAG=8
-
-def cutOff(data):
-	print("cutOff from End")
-	# print(recently)
-	# for ts, tag in recently:
-	# 	print(f"cutoff tag {tag} ... ",end='')
-		
-	# 	i = len(filtered)-1
-	# 	ndel=0
-	# 	while i:
-	# 		line=filtered[i]
-	# 		timestamp=int(line[:LEN_TIMESTAMP])
-	# 		if timestamp - ts <= -coolOff:
-	# 			#assert no more deadbeetle before ts...
-	# 			break
-	# 		if tag == line.split(',')[COL_TAG]:
-	# 				del filtered[i]
-	# 				ndel += 1
-	# 				ts = timestamp - coolOff
-
-	# 		i -= 1
-
-	# 	print(f"done. Deleted {ndel} lines up to ts: {ts} idx: {i}")
-	return data
-
+COL_X=6
+COL_Y=COL_X+1
+COL_TAG=COL_Y+1
+DELIM=","
 
+OUT_COL_X=5
+OUT_COL_Y=OUT_COL_X+1
+OUT_COL_TAG=OUT_COL_Y+1
 
 def squash(data):
+
 	print("squash plateaus")
 
-	coolOff = 3 #also try 2
-	begin = end = dur = 0
-	filtered, recently = [], []
+	coolOff = 2
+	recent = []
+	filtered = []
 	for line in data:
 		if line[0] == '#':
 			if line.startswith("#timestamp"):
 				HEADER = line
-				#prepend to data
+				filtered += HEADER
 			continue
-		l = line.split(',')
 
+		l = line.split(DELIM)
 		timestamp = int(l[COL_TIMESTAMP])
 		tag = l[ COL_TAG ]
+		x = l[ COL_X ]
+		y = l[ COL_Y ]
 
-		# count as track if tag hasnt been detected for coolOff seconds
-		if len(recently):
-			while timestamp - recently[0][0] >= coolOff:
-				end = recently[0]
-				del recently[0]
+		tagFound = False
+		i=0
+		while i < len(recent):
+			
+			# print(i,":",recent[i])
+			samePos=(recent[i][OUT_COL_X] == x and recent[i][OUT_COL_Y] == y)
+			recentBeetle=recent[i][OUT_COL_TAG] == tag
+			fastBeetle = not samePos and recentBeetle
+			ago = timestamp - recent[i][1]
 
-		# add tag if its not in list
-		if all(tag != recent_tag for recent_ts,recent_tag in recently):
-			recently.append((timestamp,tag))
-			filtered.append(line)
+			if ago > coolOff or fastBeetle:
+				
+				## add new squashed / trackevent
+				filtered.append(DELIM.join(map(str,recent[i])))
+
+				del recent[i]	# del old
+				continue		# dont incr idx!
+
+			if recentBeetle:
+				
+				## update
+				tagFound = True
+				dur = timestamp-recent[i][0]
+				recent[i][1] = timestamp
+				recent[i][2] = dur
+
+			i +=1
+
+		if not tagFound:
+
+			## add recent
+			recent.append([timestamp,timestamp,0]+l[4:]) #omit ts,ms,date,time
+
+	## cutOff
+	ncut = 0
+	for r in recent:
+		if cutOff:
+			r[1] = r[0]
+			ncut += r[2]
+			r[2] = 0
+		filtered.append(DELIM.join(map(str,r)))
+
+	if cutOff:
+		print(f"cutOff {ncut} lines")
+
+	return filtered
 
-	return data
 
 def write(outFile, data):
+
 	print(f"writing to {outFile}")
 	with open(outFile, "w") as f:
 		f.writelines(data)
 
-	cmd_compress=f"tar -zcvf {outFile}.tgz {outFile}" # ">/dev/null 2>/dev/null"
-	os.system(cmd_compress)
+	# cmd_compress=f"tar -zcvf {outFile}.tgz {outFile}" # ">/dev/null 2>/dev/null"
+	# os.system(cmd_compress)
 
 def main(filename):
+
 	with open(filename) as f:
 		data = f.readlines()
 
+	print(len(data))
 	data = squash( data )
-	data = cutOff( data )
+	print(len(data))
 
-	# write(filename + "-filtered", data)
+	write(filename + "-filtered", data)
 
 if __name__ == "__main__" :
-	main("ecolux-data-20200915-20200916")
+
+	# main("ecolux-data-20200915-20200916")
+	main("test")
 	exit()
 	if len(sys.argv) == 1:
 		this = os.path.basename(__file__)
 		
 		sys.exit(	f'Usage:	{this} dataFile\n\n'
-					f'	Squash plateaus\n'
-					f'	Cut off tags at end. (eg tag lying on sensor)')
+					f'	Squash plateaus (start/end/dur)\n'
+					f'	CutOff tags at end. (eg tag lying on sensor)'
+					f'	Time since last detection\n'
+					f'	Sensor Cross Event?\n'
+					f'	Patch Cross Event\n'	)
 
 	main(sys.argv[1])