diff --git a/.gitignore b/.gitignore index 48dd5e1d0aa5f9d9d2ab98d886521c68ec3ae340..6ec99f70bd38b3c98efe53bf79f2c9141259a144 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,3 @@ test/* *skyglow.log __pycache__ temp -skyglow.log \ No newline at end of file diff --git a/conf.json b/conf.json index 7a1b3543e20df303898baf0fe2b60066266e9a56..c5f1b3d3167f9e3e79c5c55dabf9efa8b1f98df6 100644 --- a/conf.json +++ b/conf.json @@ -7,7 +7,8 @@ ["ecotrack","./ecotrack/in","./ecotrack/out/II","2019-08-05","2019-09-27"], ["ecotrack","./ecotrack/in","./ecotrack/out/III","2019-09-27","2019-11-20"], ["ecotrack","./ecotrack/in","./ecotrack/out/all","2019-06-29","2019-11-01"], - ["ecotrack","./ecotrack/in","./ecotrack/out/test","2019-09-17","2019-09-30"] + ["ecotrack","./ecotrack/in","./blatrack","2019-10-12","2019-10-15"], + ["ecolux","./ecolux/I/01-raw","./blalux","2020-09-01","2020-09-02"] ], "blox" : { diff --git a/dataSync.py b/dataSync.py index b8e48993118add5bc7a0dcb7f58eeee71c9297ad..88b2d30a8262ef58b635a1664f99de5f8d148e46 100755 --- a/dataSync.py +++ b/dataSync.py @@ -24,22 +24,19 @@ # 3. Consider using Pandas, NumPy, Cython, Pypy, Koalas, inline C++/C/Bash, sql (SQLAlchemy) #Q: wieviel track-events wurden von clean rausgefiltert? + import sys, getopt, os, re, string, platform, json, datetime, time from pprint import pp as p from datetime import datetime as dt import dataFilter - -skyglow_submodule_path=os.path.dirname(os.path.realpath(__file__)) + os.sep + "skyglow" -sys.path.append(skyglow_submodule_path) -# print(sys.path) - try: + scriptpath=os.path.dirname(os.path.realpath(__file__)) + sys.path.append(scriptpath + os.sep + "skyglow") from skyglow import moon as moonMap + from skyglow import skyglowDict except ModuleNotFoundError as e: print(f"caught: {e}") -print(moonMap) -exit(1223) TIME_FMT='%Y-%m-%d %H:%M:%S' noTime = dt.fromtimestamp(0) @@ -130,9 +127,9 @@ def initFileStructure(): A_DELIM='\t' A_MINLEN = 25 A_MAXLEN = 45 - A_NCOLS = 4 - A_COL_SPECIES=0 - A_COL_SPECIES_IND=1 + A_NCOLS = 4 #/wo comments + A_COL_SPECIES=1 + A_COL_SPECIES_IND=0 A_COL_WEIGHT_WO_TAG_MG=2 A_COL_TAG=3 elif PROJ == "ecotrack": @@ -140,8 +137,8 @@ def initFileStructure(): A_MINLEN = 30 A_MAXLEN = 45 A_NCOLS = 7 #/w comments - A_COL_SPECIES=1 A_COL_SPECIES_IND=0 + A_COL_SPECIES=1 A_COL_WEIGHT_WO_TAG_MG=2 A_COL_TAG=5 @@ -378,10 +375,10 @@ class Data: else: temps = self.meanTemps[0][uid][block] + self.meanTemps[1][uid][block] - if isDay(time): - temps += "day" - else: - temps += "night" + # if isDay(time): + # temps += "day" + # else: + # temps += "night" self.tempData[uid][idx].extend( temps ) @@ -402,14 +399,20 @@ class Data: self.meanTemps[day][uid][block] = [mean] - def initAnimalData(self,file): + def initAnimalData(self,file): + with open(file) as f: lines = f.readlines() lines = list(filter(lambda l: not l.split(A_DELIM)[A_COL_TAG].startswith("XXX"), lines)) #filter "XXX" tags lines = list(filter(lambda l: l.split(A_DELIM)[A_COL_TAG] != "049E4C5EF2F44880CE", lines)) #filter thomas control tag - lines = [ l[:l.rindex(A_DELIM)+1] for l in lines ] #remove comments + if PROJ == "ecotrack": + lines = [ l[:l.rindex(A_DELIM)+1] for l in lines ] #remove comments + + removedTags = len(lines) lines = self.clean(lines, A_MINLEN, A_MAXLEN, A_NCOLS, A_DELIM) + removedTags -= len(lines) + print(f">> removed {removedTags} Tags") #TODO: check for duplicate tags and pp warning / ignore? assert (lines), "tags: Not valid. Check times / formatting." @@ -418,6 +421,7 @@ class Data: def clean(self, lines, minLen, maxLen, nCols, sep, timeFunc=None, delta=datetime.timedelta(minutes=0)): """ remove trailing newline, empty, comments and all-digits and remove lines before startTime or after endTime""" + lines = [l.strip() for l in lines] lines = [l for l in lines if minLen <= len(l) <= maxLen] lines = list(filter(lambda q: q and q[0] != '#', lines)) @@ -472,7 +476,7 @@ class Data: if ttag == atag: found=True - aLine = DELIM + aLine[A_COL_SPECIES_IND] + DELIM + aLine[A_COL_SPECIES] + DELIM + aLine[A_COL_WEIGHT_WO_TAG_MG] + aLine = DELIM + aLine[A_COL_SPECIES] + DELIM + aLine[A_COL_SPECIES_IND] + DELIM + aLine[A_COL_WEIGHT_WO_TAG_MG] self.trackData[idx] += aLine break; @@ -613,8 +617,7 @@ class Data: h+=', moon_real[mLux], moon_eco[mLux], skyglow[Lux]' if hasTemp(): h+=', temp[C°], mean_night_temp[C°], min_night_temp[C°], max_night_temp[C°], mean_day_temp[C°], min_day_temp[C°], max_day_temp[C°], is_day' - h +="\n" - h += RULER + "\n" + h +="\n" + RULER + "\n" return h def write(self):