diff --git a/.gitignore b/.gitignore index c11786d300a0fbf52bb8105ae6b47740c185db73..840a9726bdb5fa1e291393c7b86a61ff5a8bc980 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -block-II/02-inspect/ecotron-data-20200915-20201013 -block-II/01-raw/backup-ecoTron-2020-10-28-06-00-1603861202.tgz -block-I/02-inspect/ecotron-data-20200721-20200818 +/home/sugu/code/idiv/ecotrondata/block-II/02-inspect/ecotron-data-20200915-20201013 +/home/sugu/code/idiv/ecotrondata/block-II/01-raw/backup-ecoTron-2020-10-28-06-00-1603861202.tgz +/home/sugu/code/idiv/ecotrondata/block-I/02-inspect/ecotron-data-20200721-20200818 *.txt test/* *# diff --git a/block-I/02-inspect/ecotron-data-20200721-20200818 b/block-I/02-inspect/ecotron-data-20200721-20200818 new file mode 100644 index 0000000000000000000000000000000000000000..ef2566f68bbd784d619da0984ff67069bf3f02aa Binary files /dev/null and b/block-I/02-inspect/ecotron-data-20200721-20200818 differ diff --git a/dataFilterCoolOff.py b/dataFilter.py similarity index 93% rename from dataFilterCoolOff.py rename to dataFilter.py index 048a6c81e00bcd381776f1943f4841b2ba03da6b..0a31f81c86dcb0bddd6ec160e0c92eaaf5bd7e3b 100755 --- a/dataFilterCoolOff.py +++ b/dataFilter.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 #todo +# dont cutOff first trackEvent from plateau to end +# # print more info: # nLines before coolOff # nLines after coolOff @@ -21,10 +23,12 @@ def main(arg1,arg2=10): global inFile, coolOff inFile=arg1 coolOff=arg2 + print(f"start filtering data with {coolOff} second coolOff") with open(inFile) as f: lines = f.readlines() + #filter coolOff filtered, recently = [], [] for line in lines: if line[0] == '#': @@ -43,7 +47,7 @@ def main(arg1,arg2=10): recently.append((timestamp,tag)) filtered.append(line) - #cutOffEnd + #cutOff from End print(recently) for ts, tag in recently: print(f"cutoff tag {tag}",end='') diff --git a/dataSync.py b/dataSync.py index c6548b2ee29981ff5713d579f2b03c394843f436..7fbbdf4503e2c09e47033bbc384b56dd35cb9da3 100755 --- a/dataSync.py +++ b/dataSync.py @@ -7,13 +7,15 @@ ############################ # TODO ############################ -# > could add COOLOFF to blox-config. Now defaulting to 10 -# > parse moonmap and skyglowdict from skyglow Project sources (submodule ... ?) +# > parse moonmap and skyglowdict from skyglow Project sources +# -> maybe integrate skyglow project as submodule # > integrate with ST # > make Skyglow optional -> TEST # > ST-Filenames should include 'unit-X'! +# # > how to process skylog error in merge() # > include temperature and humidity data? +# > could add COOLOFF to blox-config. Now defaulting to 10 # # > Speed Up! (takes 16minutes for 8mio lines...) # 1. Threads @@ -32,7 +34,7 @@ import sys, getopt, os, re, datetime, string, time from pprint import pprint as pp from datetime import datetime as dt -import dataFilterCoolOff +import dataFilter PROJ = "ecotron" # PROJ = "schrebatron" @@ -45,7 +47,6 @@ blox=[ ("./test","./test","2020-07-21 00:00:00","2020-08-18 23:59:00"), # test ] - TIME_FMT='%Y-%m-%d %H:%M:00' noTime = dt.fromtimestamp(0) TAG_LEN=len("04B94A7F7288588022") @@ -84,7 +85,6 @@ A_COL_TAG=3 # out file OUT_FMT='%Y%m%d' -OUT_FILE='default' DELIM=',' COL_UID=4 COL_TAG=8 @@ -467,29 +467,30 @@ def getFileList(): return animalFile, skyglowFile, trackFiles def make_rel_abs_path(path): - if path.startswith('.'): - path=os.path.dirname(os.path.abspath(__file__))+os.sep+path + if path.startswith('../'): + path= scriptPath()+path[3:] + elif path.startswith('./'): + path= scriptPath()+path[2:] return path +def scriptPath(): + return os.path.dirname(os.path.abspath(__file__))+os.sep + def main(): - + xtract() animalFile, skyglowFile, trackFiles = getFileList() data = Data(animalFile,skyglowFile,trackFiles) data.merge() data.write() - dataFilterCoolOff.main(OUT_FILE) - #call updateGitIgnore.sh? + dataFilter.main(OUT_FILE) -if __name__ == "__main__" : - - print( os.path.dirname(os.path.abspath(__file__)) ) - cmd_compress=f"updateGitIgnore.sh" - os.system(cmd_compress) - exit() - + print("update git to ignore big files (>50MB)") + cmd=scriptPath()+"updateGitIgnore.sh" + os.system(cmd) +if __name__ == "__main__" : print(" Process raw data for which block?") for idx, b in enumerate(blox): print(f" {idx+1}) {b[2]} >> {b[3]}") diff --git a/updateGitIgnore.sh b/updateGitIgnore.sh index 1af2ebd2ebfc1eca0aecd856b169e1743337badd..ccdc13a344b7a60df98973543c700d715be904a6 100755 --- a/updateGitIgnore.sh +++ b/updateGitIgnore.sh @@ -1,7 +1,10 @@ #!/bin/bash -#ignore .txt files in find -find . -size +50M ! -name '*.txt' | sed 's/\.\///g' > .gitignore +dir=`dirname "$(realpath $0)"` +ignorefile="${dir}/.gitignore" + +#ignore .txt files in find, since they are ignored anyway +find "${dir}" -size +50M ! -name '*.txt' | sed 's/\.\///g' > "${ignorefile}" list=( "*.txt" @@ -13,5 +16,5 @@ __pycache__ for i in "${list[@]}" do - echo "$i" >> .gitignore + echo "$i" >> "${ignorefile}" done