recalc data, format times human-readable, always add cut to filename. Make it...

recalc data, format times human-readable, always add cut to filename. Make it optional argument, sort by end-time, update updateGitignore to not ignore actual raw-data

.gitignore

 block-II/02-inspect/ecolux-data-20200915-20201013
-block-II/01-raw/backup-ecoTron-2020-10-28-06-00-1603861202.tgz
 block-I/02-inspect/ecolux-data-20200721-20200818
 *.txt
 test/*

dataFilter.py

 #!/usr/bin/python3
 #
 # Todo:
-# 	compat /w ecoTrack ? 1nt3gr4t3.
+#	> use global vars for Col-indices. each function that changes em updates em
+# 	> compat /w ecoTrack ? 1nt3gr4t3.
 
 import sys, os
+from datetime import datetime as dt
 
 cutOff = True
 ncut = 0
 coolOff = 5
 
 DELIM=","
+OUT_FMT="%d-%m-%y %T"
 
 def squash(data):
 
@@ -22,7 +25,8 @@ def squash(data):
 	col_tag=8
 
 	omit_col_begin=4 #omit ts,ms,date,time
-
+	
+	nfast=0
 	recent = []
 	filtered = []
 	for line in data:
@@ -39,18 +43,22 @@ def squash(data):
 		x = l[ col_x ]
 		y = l[ col_y ]
 
+
+
 		tagFound = False
 		i=0
 		while i < len(recent):
 
 			samePos=(recent[i][5] == x and recent[i][6] == y)
 			recentBeetle=recent[i][7] == tag
-			fastBeetle = not samePos and recentBeetle
+			fastBeetle = recentBeetle and not samePos
 			ago = timestamp - recent[i][1]
 
 			# print(i,":",recent[i])
 			# print(f"tag: {tag}| timestamp: {timestamp}| ago:{ago}| coolOff:{coolOff}| fastBeetle:{fastBeetle}| {x},{y}:{recent[i][5]},{recent[i][6]}|recentBeetle:{recentBeetle}")
 			if ago > coolOff or fastBeetle:
+				if fastBeetle:
+					nfast += 1
 
 				## add new squashed / trackevent
 				filtered.append(DELIM.join(map(str,recent[i])))
@@ -85,9 +93,8 @@ def squash(data):
 
 		filtered.append(DELIM.join(map(str,r)))
 
-	if cutOff:
-		print(f"	cutOff {ncut} lines")
-	
+	print(f"	cutOff: {ncut}")
+	print(f"	nfast: {nfast}")
 	return filtered
 
 def add_cols( data ):
@@ -138,19 +145,22 @@ def add_cols( data ):
 			l.insert(col_sxe, sxe)
 			l.insert(col_pxe, pxe)
 
-
 		known[tag] = l
 		data[i] = DELIM.join(map(str,l))
 
 	print(f"	nsxe: {nsxe}")
 	print(f"	npxe: {npxe}")
-	
 
 	return data
 
-def pad(data):
 
-	print("pad cols")
+def format(data):
+	"sort, pad, human-readable times"
+	print("format")
+
+	#sort by end-time
+	data.sort(key=lambda l: l.split(DELIM)[1])
+
 	#TODO: get max len of each col and use that!
 	#	f'{last_detect:03}'
 	#	lines = ['cat', 'dog', 'elephant', 'horse']
@@ -159,12 +169,14 @@ def pad(data):
 	#	[x.rjust(len('elephant')) for x in lines]
 	for i,d in enumerate(data):
 		d=d.split(DELIM)
+		d[0]= dt.fromtimestamp(int( d[0] )).strftime(OUT_FMT)
+		d[1]= dt.fromtimestamp(int( d[1] )).strftime(OUT_FMT)
 		d[2]=d[2].rjust(4)	# dur
 		d[3]=d[3].rjust(6)	# lastdetect
 		d[4]=d[4].rjust(5)	# "FALSE"
 		d[5]=d[5].rjust(5)	# "FALSE"
 		d[6]=d[6].rjust(2)	# unit '13'
-		# d[7]=d[7].rjust(5)	# "FALS"
+		# d[7]=d[7].rjust(5)	# "FALSE"
 		d[8]=d[8].rjust(1)	# x
 		d[9]=d[9].rjust(1)	# y
 		d[11]=d[11].rjust(2) # species
@@ -177,9 +189,7 @@ def pad(data):
 
 def write(outFile, data):
 
-	outFile += "-filtered-" + str(coolOff)
-	if cutOff:
-		outFile += "-co-" + str(ncut)
+	outFile += f"-cool-{coolOff}-cut-{ncut}"
 
 	print(f"writing to {outFile}")
 	with open(outFile, "w") as f:
@@ -196,8 +206,7 @@ def main(filename):
 
 	data = squash( data )
 	data = add_cols( data )
-	data = pad( data )
-	data.sort()
+	data = format( data )
 
 	write(filename, data)
 
@@ -206,16 +215,29 @@ if __name__ == "__main__" :
 	if len(sys.argv) == 1:
 		this = os.path.basename(__file__)
 		
-		sys.exit(	f'Usage:	{this} dataFile coolOff\n\n'
-					f'	new detection after Cooloff Seconds will be counted new trackevent\n'
+		sys.exit(	f'Usage:	{this} dataFile coolOff cutOff\n\n'
+					f'	another detection for a given tag and position '
+					f'		* WITHIN coolOff seconds will be added to the last plateau\n'
+					f'		* AFTER coolOff seconds will be used for a new plateau\n'
+					f'		default: {coolOff}\n'
 					f''
-					f'	> Squash plateaus (start/end/dur)\n'
-					f'	> CutOff tags from end\n'
+					f'	cutOff plateau which goes all the way to the end.\n'
+					f'		eg beetle lost tag on sensor\n'
+					f'		default: {cutOff}\n'
+					f''
+					f'	> squash plateaus (start/end/dur)\n'
+					f'	> cutOff tags from end [optional]\n'
 					f'	> add time-since-last-detection\n'
 					f'	> add Patch Cross Event (PXE)\n'
-					f'	> add Sensor Cross Event (SXE)\n')
+					f'	> add Sensor Cross Event (SXE)\n'
+					f'	> format: time,pad\n')
 
-	if len(sys.argv) == 3:
+	if len(sys.argv) >= 3:
 		coolOff = int(sys.argv[2])
 
+	if len(sys.argv) >= 4:
+		if sys.argv[3] in ('0','False'):
+			cutOff = False
+
 	main(sys.argv[1])
+

dataSync.py

@@ -299,7 +299,7 @@ class Data:
 		with open(file) as f:
 			lines = f.readlines()
 
-		lines = self.clean(lines, A_MINLEN, A_MAXLEN, A_NCOLS, A_DELIM)		
+		lines = self.clean(lines, A_MINLEN, A_MAXLEN, A_NCOLS, A_DELIM)
 		#TODO: check for duplicate tags and print warning / ignore?
 		
 		assert (lines), "tags: Not valid. Check times / formatting."

updateGitIgnore.sh

@@ -16,8 +16,9 @@ cd "${projdir}"
 
 echo "update gitignore for project in ${projdir}"
 
-#ignore .txt files in find, since they are ignored anyway. Also ignore .git folder
-find . -size +"${maxSize}"M ! -name '*.txt' ! -wholename './.git*' ! -wholename './test/*'| sed 's/\.\///g' > "${gitignore}"
+#ignore .txt files in find, since they are ignored anyway. Also ignore .git folder and raw-folder of all blox!
+
+find . -size +"${maxSize}"M ! -name '*.txt' ! -wholename './.git*' ! -wholename './test/*' ! -wholename './*/01-raw/*' | sed 's/\.\///g' > "${gitignore}"
 
 for i in "${ignore[@]}"
 do