updateGitignore uses git to find top-level dir and find ignore .git and test...

updateGitignore uses git to find top-level dir and find ignore .git and test dirs. dataFilter grabs and uses header from file. dataSync: chdir and zip files to avoid zipping full dir structure, add PROJ to blox and depend fileConfiguration from it, add support for "ecotrack": diff filestructures

dataFilter.py

@@ -16,13 +16,12 @@ import sys, os
 LEN_TIMESTAMP=len("1602626336")
 COL_TIMESTAMP=0
 COL_TAG=8
-HEADER="#timestamp, milliseconds, date, time, unit, x, y, tagID, species, speciesnumber, weight_without_tag_mg, moon_real_mLux, moon_eco_mLux, skyglow_Lux\n"
-
 
 def main(arg1,arg2=10):
-	global inFile, coolOff
-	inFile=arg1
-	coolOff=arg2
+	global inFile, coolOff, HEADER
+	inFile = arg1
+	coolOff = arg2
+
 	print(f"start filtering data with {coolOff} second coolOff")
 
 	with open(inFile) as f:
@@ -32,6 +31,8 @@ def main(arg1,arg2=10):
 	filtered, recently = [], []
 	for line in lines:
 		if line[0] == '#':
+			if line.startswith("#timestamp"):
+				HEADER = line
 			continue
 		l = line.split(',')
 
@@ -50,7 +51,7 @@ def main(arg1,arg2=10):
 	#cutOff from End
 	print(recently)
 	for ts, tag in recently:
-		print(f"cutoff tag {tag}",end='')
+		print(f"cutoff tag {tag} ... ",end='')
 		
 		i = len(filtered)-1
 		ndel=0

dataSync.py

@@ -43,14 +43,15 @@ import dataFilter
 blox=[
 ("ecotron","./block-I/01-raw","./block-I/02-inspect","2020-07-21 00:00:00","2020-08-18 23:59:00"), 		# Block1: 21.7 - 18.8
 ("ecotron","./block-II/01-raw","./block-II/02-inspect","2020-09-15 00:00:00","2020-10-13 23:59:00"),	# Block2: 15.9 - 13.10
-("ecotrack","../camtrondata/test/in","../camtrondata/test/out","2019-06-28 00:00:00","2019-10-16 23:59:00"),
-# ("test","./test","./test","2020-07-21 00:00:00","2020-08-18 23:59:00"),
+# ("ecotrack","~/code/idiv/camtrondata/test/in","~/code/idiv/camtrondata/test/in","2019-06-28 00:00:00","2019-10-16 23:59:00"),
+("ecotrack","~/code/idiv/camtrondata/test/in","~/code/idiv/camtrondata/test/out","2019-10-14 00:00:00","2019-10-16 23:59:00"),
 ]
 
 TIME_FMT='%Y-%m-%d %H:%M:00'
 noTime = dt.fromtimestamp(0)
 TAG_LEN=len("04B94A7F7288588022")
 SCRIPTPATH=os.path.dirname(os.path.abspath(__file__))+os.sep
+RULER=f"{'#'*100}\n"
 
 def initFileStructure():
 
@@ -231,7 +232,7 @@ class Data:
 			self.endLight = self.getLightTime( self.lightData[-1] )
 
 		global OUT_FILE
-		OUT_FILE = f"{OUTPUTDIR}{os.sep}{PROJ}-data-{self.startTrack.strftime(OUT_FMT)}-{self.endTrack.strftime(OUT_FMT)}"
+		OUT_FILE = f"{PROJ}-data-{self.startTrack.strftime(OUT_FMT)}-{self.endTrack.strftime(OUT_FMT)}"
 
 		pp(self.header())
 
@@ -430,7 +431,7 @@ class Data:
 
 	def header(self):
 		h= f"# {PROJ} data\n" \
-		f"#################################################################################################\n" \
+		f"{RULER}" \
 		f"#     len(Track): {len(self.trackData)}\n" \
 		f"#     startTrack: {self.startTrack}\n" \
 		f"#       endTrack: {self.endTrack}\n" \
@@ -439,30 +440,30 @@ class Data:
 			h += 	f"#     len(Light): {len(self.lightData)}\n" \
 					f"#     startLight: {self.startLight}\n" \
 					f"#       endLight: {self.endLight}\n"
-		h+= f"#################################################################################################\n" \
-			f"#timestamp, ms, date, time, unit, habitat, x, y, tag, species, speciesnumber, weight_without_tag[mg]{', moon_real[mLux], moon_eco[mLux], skyglow[Lux]' if hasLight() else ''}\n" \
-			f"#################################################################################################\n"
-		return  h
+		h += RULER
+		h += f"#timestamp, ms, date, time, unit, habitat, x, y, tag, species, speciesnumber, weight_without_tag[mg]{', moon_real[mLux], moon_eco[mLux], skyglow[Lux]' if hasLight() else ''}\n"
+		h += RULER
+		return h
 
 	def write(self):
-
+		# os.makedirs(os.path.dirname(OUTPUTDIR), exist_ok=True)
+		os.makedirs(OUTPUTDIR, exist_ok=True)
+		os.chdir(OUTPUTDIR)
 		pp("write to " + OUT_FILE)
-		os.makedirs(os.path.dirname(OUT_FILE), exist_ok=True)
 		with open(OUT_FILE, "w") as f:
 			f.writelines(self.header())
 			self.trackData=map(lambda x:x+'\n', self.trackData)
 			f.writelines(self.trackData)
 
-		cmd_compress=f"tar -zcvf {OUT_FILE}.tgz {OUT_FILE}" # ">/dev/null 2>/dev/null"
+		cmd_compress=f"tar -C {OUTPUTDIR} -zcvf {OUT_FILE}.tgz {OUT_FILE}" # ">/dev/null 2>/dev/null"
 		os.system(cmd_compress)
 
 def xtract():
 	for root, dirs, files in os.walk(INPUTDIR): #walk recursively
 		for file in files:
 			if file.endswith(".tgz") or file.endswith(".tar.gz"):
-				file=INPUTDIR + os.sep + file
 				print(f"xtracting {file}\n")
-				cmd_xtract=f"tar -zxvf {file} --directory {INPUTDIR}"
+				cmd_xtract=f"tar -zxvf {file} -C {INPUTDIR}"
 				os.system(cmd_xtract)
 				return
 
@@ -514,12 +515,12 @@ def main():
 
 	dataFilter.main(OUT_FILE)
 
-	print("update git to ignore big files (>50MB)")
+	print("update git to ignore large files")
 	cmd=SCRIPTPATH+"updateGitIgnore.sh"
 	os.system(cmd)
 
 if __name__ == "__main__" :
-	print(f"	Process raw data for which block?\n {50*'#'}\n")
+	print(f"	Process raw data for which block?\n {RULER}")
 	for idx, b in enumerate(blox):
 		print(f"    {idx+1}) Project {b[0]}")
 		print(f"        from: {b[3]}")

updateGitIgnore.sh

 #!/bin/bash
 
+maxSize=50
+
 ignore=(
 "*.txt"
 "test/*"
@@ -8,16 +10,16 @@ ignore=(
 __pycache__
 )
 
-projdir=`dirname "$(realpath $0)"`
-ignorefile="${projdir}/.gitignore"
+projdir=`git rev-parse --show-toplevel`
+gitignore="${projdir}/.gitignore"
 cd "${projdir}"
 
-#ignore .txt files in find, since they are ignored anyway
-find . -size +50M ! -name '*.txt' | sed 's/\.\///g' > "${ignorefile}"
+echo "update gitignore for project in ${projdir}"
+
+#ignore .txt files in find, since they are ignored anyway. Also ignore .git folder
+find . -size +"${maxSize}"M ! -name '*.txt' ! -wholename './.git*' ! -wholename './test/*'| sed 's/\.\///g' > "${gitignore}"
 
 for i in "${ignore[@]}"
 do
-	echo "$i" >> "${ignorefile}"
+	echo "$i" >> "${gitignore}"
 done
-
-#cd -
\ No newline at end of file