From 83780bbec348688d53efa5ff135110a4a4da0d84 Mon Sep 17 00:00:00 2001
From: am0ebe <am0ebe@gmx.de>
Date: Thu, 10 Jun 2021 17:07:53 +0200
Subject: [PATCH] fix bugs. rearrange species cols for ecolux, filter comments
from tags file for ecotrack only, import correct skyglow vars
---
.gitignore | 1 -
conf.json | 3 ++-
dataSync.py | 43 +++++++++++++++++++++++--------------------
3 files changed, 25 insertions(+), 22 deletions(-)
diff --git a/.gitignore b/.gitignore
index 48dd5e1..6ec99f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,4 +5,3 @@ test/*
*skyglow.log
__pycache__
temp
-skyglow.log
\ No newline at end of file
diff --git a/conf.json b/conf.json
index 7a1b354..c5f1b3d 100644
--- a/conf.json
+++ b/conf.json
@@ -7,7 +7,8 @@
["ecotrack","./ecotrack/in","./ecotrack/out/II","2019-08-05","2019-09-27"],
["ecotrack","./ecotrack/in","./ecotrack/out/III","2019-09-27","2019-11-20"],
["ecotrack","./ecotrack/in","./ecotrack/out/all","2019-06-29","2019-11-01"],
- ["ecotrack","./ecotrack/in","./ecotrack/out/test","2019-09-17","2019-09-30"]
+ ["ecotrack","./ecotrack/in","./blatrack","2019-10-12","2019-10-15"],
+ ["ecolux","./ecolux/I/01-raw","./blalux","2020-09-01","2020-09-02"]
],
"blox" :
{
diff --git a/dataSync.py b/dataSync.py
index b8e4899..88b2d30 100755
--- a/dataSync.py
+++ b/dataSync.py
@@ -24,22 +24,19 @@
# 3. Consider using Pandas, NumPy, Cython, Pypy, Koalas, inline C++/C/Bash, sql (SQLAlchemy)
#Q: wieviel track-events wurden von clean rausgefiltert?
+
import sys, getopt, os, re, string, platform, json, datetime, time
from pprint import pp as p
from datetime import datetime as dt
import dataFilter
-
-skyglow_submodule_path=os.path.dirname(os.path.realpath(__file__)) + os.sep + "skyglow"
-sys.path.append(skyglow_submodule_path)
-# print(sys.path)
-
try:
+ scriptpath=os.path.dirname(os.path.realpath(__file__))
+ sys.path.append(scriptpath + os.sep + "skyglow")
from skyglow import moon as moonMap
+ from skyglow import skyglowDict
except ModuleNotFoundError as e:
print(f"caught: {e}")
-print(moonMap)
-exit(1223)
TIME_FMT='%Y-%m-%d %H:%M:%S'
noTime = dt.fromtimestamp(0)
@@ -130,9 +127,9 @@ def initFileStructure():
A_DELIM='\t'
A_MINLEN = 25
A_MAXLEN = 45
- A_NCOLS = 4
- A_COL_SPECIES=0
- A_COL_SPECIES_IND=1
+ A_NCOLS = 4 #/wo comments
+ A_COL_SPECIES=1
+ A_COL_SPECIES_IND=0
A_COL_WEIGHT_WO_TAG_MG=2
A_COL_TAG=3
elif PROJ == "ecotrack":
@@ -140,8 +137,8 @@ def initFileStructure():
A_MINLEN = 30
A_MAXLEN = 45
A_NCOLS = 7 #/w comments
- A_COL_SPECIES=1
A_COL_SPECIES_IND=0
+ A_COL_SPECIES=1
A_COL_WEIGHT_WO_TAG_MG=2
A_COL_TAG=5
@@ -378,10 +375,10 @@ class Data:
else:
temps = self.meanTemps[0][uid][block] + self.meanTemps[1][uid][block]
- if isDay(time):
- temps += "day"
- else:
- temps += "night"
+ # if isDay(time):
+ # temps += "day"
+ # else:
+ # temps += "night"
self.tempData[uid][idx].extend( temps )
@@ -402,14 +399,20 @@ class Data:
self.meanTemps[day][uid][block] = [mean]
- def initAnimalData(self,file):
+ def initAnimalData(self,file):
+
with open(file) as f:
lines = f.readlines()
lines = list(filter(lambda l: not l.split(A_DELIM)[A_COL_TAG].startswith("XXX"), lines)) #filter "XXX" tags
lines = list(filter(lambda l: l.split(A_DELIM)[A_COL_TAG] != "049E4C5EF2F44880CE", lines)) #filter thomas control tag
- lines = [ l[:l.rindex(A_DELIM)+1] for l in lines ] #remove comments
+ if PROJ == "ecotrack":
+ lines = [ l[:l.rindex(A_DELIM)+1] for l in lines ] #remove comments
+
+ removedTags = len(lines)
lines = self.clean(lines, A_MINLEN, A_MAXLEN, A_NCOLS, A_DELIM)
+ removedTags -= len(lines)
+ print(f">> removed {removedTags} Tags")
#TODO: check for duplicate tags and pp warning / ignore?
assert (lines), "tags: Not valid. Check times / formatting."
@@ -418,6 +421,7 @@ class Data:
def clean(self, lines, minLen, maxLen, nCols, sep, timeFunc=None, delta=datetime.timedelta(minutes=0)):
""" remove trailing newline, empty, comments and all-digits and remove lines before startTime or after endTime"""
+
lines = [l.strip() for l in lines]
lines = [l for l in lines if minLen <= len(l) <= maxLen]
lines = list(filter(lambda q: q and q[0] != '#', lines))
@@ -472,7 +476,7 @@ class Data:
if ttag == atag:
found=True
- aLine = DELIM + aLine[A_COL_SPECIES_IND] + DELIM + aLine[A_COL_SPECIES] + DELIM + aLine[A_COL_WEIGHT_WO_TAG_MG]
+ aLine = DELIM + aLine[A_COL_SPECIES] + DELIM + aLine[A_COL_SPECIES_IND] + DELIM + aLine[A_COL_WEIGHT_WO_TAG_MG]
self.trackData[idx] += aLine
break;
@@ -613,8 +617,7 @@ class Data:
h+=', moon_real[mLux], moon_eco[mLux], skyglow[Lux]'
if hasTemp():
h+=', temp[C°], mean_night_temp[C°], min_night_temp[C°], max_night_temp[C°], mean_day_temp[C°], min_day_temp[C°], max_day_temp[C°], is_day'
- h +="\n"
- h += RULER + "\n"
+ h +="\n" + RULER + "\n"
return h
def write(self):
--
GitLab