Skip to content
Snippets Groups Projects
Commit e3573d01 authored by am0ebe's avatar am0ebe
Browse files

rename project ecotron to ecolux, normalize paths for use with windows and... 

 rename project ecotron to ecolux, normalize paths for use with windows and linux, xtract all archives in dir, give helpful error msg how to adapt blox
parent 5ab716f6
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 35 additions and 34 deletions
dataSync.py
+ 35
− 34
View file @ e3573d01
......@@ -36,15 +36,15 @@ from pprint import pprint as pp
from datetime import datetime as dt
import dataFilter
# Experiment Config! Time and paths.
# Configuration of Time and paths
#####################################
# ecotron -- Track-start: 7.7.20 // Light start: 20.7.20 // Dark: 10.7.20 - 20.7.20
# Project, in, out, start, end
# ecolux -- Track-start: 7.7.20 // Light start: 20.7.20 // Dark: 10.7.20 - 20.7.20
blox=[
("ecotron","./block-I/01-raw","./block-I/02-inspect","2020-07-21 00:00:00","2020-08-18 23:59:00"), # Block1: 21.7 - 18.8
("ecotron","./block-II/01-raw","./block-II/02-inspect","2020-09-15 00:00:00","2020-10-13 23:59:00"), # Block2: 15.9 - 13.10
# ("ecotrack","~/code/idiv/camtrondata/test/in","~/code/idiv/camtrondata/test/in","2019-06-28 00:00:00","2019-10-16 23:59:00"),
("ecotrack","~/code/idiv/camtrondata/test/in","~/code/idiv/camtrondata/test/out","2019-10-14 00:00:00","2019-10-16 23:59:00"),
("ecolux","./block-I/01-raw","./block-I/02-inspect","2020-07-21 00:00:00","2020-08-18 23:59:00"), # Block1: 21.7 - 18.8
("ecolux","./block-II/01-raw","./block-II/02-inspect","2020-09-15 00:00:00","2020-10-13 23:59:00"), # Block2: 15.9 - 13.10
("foo","~/block-I/01-raw/../..","~/block-I/02-inspect/","2020-07-21 00:00:00","2020-08-18 23:59:00"),
("ecotrack","/opt/virtualBoxVMs/share/block-I/01-raw","/opt/virtualBoxVMs/share/block-I/02-inspect","2019-10-14 00:00:00","2019-10-16 23:59:00"),
## -- add here -- ##
]
TIME_FMT='%Y-%m-%d %H:%M:00'
......@@ -66,7 +66,7 @@ def initFileStructure():
T_COL_Y=3
T_COL_TAG=4
#t_col_signalstrength=5
if PROJ == "ecotron":
if PROJ == "ecolux":
T_MINLEN = 38 # without newline.
T_MAXLEN = 40 # ms column varies 1-3. Will be padded later for uniform len
T_NCOLS = 6
......@@ -90,7 +90,7 @@ def initFileStructure():
# animalfile
###########
global A_DELIM, A_MINLEN, A_MAXLEN, A_NCOLS, A_COL_SPECIES, A_COL_SPECIES_IND, A_COL_WEIGHT_WO_TAG_MG, A_COL_TAG
if PROJ == "ecotron":
if PROJ == "ecolux":
A_DELIM='\t'
A_MINLEN = 25
A_MAXLEN = 30
......@@ -109,7 +109,7 @@ def initFileStructure():
A_COL_SPECIES=1
A_COL_SPECIES_IND=0
A_COL_WEIGHT_WO_TAG_MG=2
A_COL_TAG=5
A_COL_TAG=5w
# out file
......@@ -202,7 +202,7 @@ skyglowDict = {
}
def hasLight():
return PROJ == "ecotron"
return PROJ == "ecolux"
def getUID(fileName):
""" extract and return unit-id from fileName """
......@@ -240,7 +240,7 @@ class Data:
""" get lines from all tracklogs, clean, merge and add/rm columns"""
pp(f"processing: {fileName}")
with open(INPUTDIR+'/'+fileName) as f:
with open(INPUTDIR+os.sep+fileName) as f:
lines = f.readlines()
lines = self.clean(lines, T_MINLEN, T_MAXLEN, T_NCOLS, T_DELIM, self.getTrackTime)
......@@ -253,7 +253,7 @@ class Data:
idx_after_ms = TIMESTAMP_LEN + len('999') + len(DELIM)
for idx, line in enumerate(lines):
if PROJ in ["ecotron","ecotrack"]:
if PROJ in ["ecolux","ecotrack"]:
# pad ms column with zeros to fit three digits
cols=line.split(DELIM)
ms_digits=len(cols[T_COL_MS])
......@@ -271,7 +271,7 @@ class Data:
# add columns (date,time,uid,isHabitat)
line = line[:idx_after_ms] + DELIM + date + DELIM + time + DELIM + str(uid) + DELIM + str(isHabitat) + line[idx_after_ms:]
if PROJ == "ecotron":
if PROJ == "ecolux":
line = line[:line.rindex(DELIM)] # remove signalstrength column
lines[idx] = line
......@@ -304,7 +304,7 @@ class Data:
def clean(self, lines, minLen, maxLen, nCols, sep, timeFunc=None):
""" remove trailing newline, empty and comments and remove lines before startTime or after endTime"""
lines = [l.strip() for l in lines]
lines = [l for l in lines if minLen <= len(l) <= maxLen] # EcoTron: ms are variable in length (0-999), SchrebaTron: ms is padded with zeros
lines = [l for l in lines if minLen <= len(l) <= maxLen] # ecolux: ms are variable in length (0-999), SchrebaTron: ms is padded with zeros
lines = [l for l in lines if len(l.split(sep)) == nCols]
lines = [l.replace(sep,DELIM) for l in lines]
lines = list(filter(lambda q: q and q[0] != '#', lines))
......@@ -459,27 +459,28 @@ class Data:
os.system(cmd_compress)
def xtract():
# os.chdir(INPUTDIR)
for root, dirs, files in os.walk(INPUTDIR): #walk recursively
for file in files:
if file.endswith(".tgz") or file.endswith(".tar.gz"):
print(f"xtracting {file}\n")
cmd_xtract=f"tar -zxvf {file} -C {INPUTDIR}"
cmd_xtract=f"tar -zxvf {INPUTDIR}{os.sep}{file} -C {INPUTDIR}"
os.system(cmd_xtract)
return
return
def getFileList():
animalFile, skyglowFile, trackFiles = '','',[]
print( "Looking for 'skyglow.log', 'tags' and files containing 'unit-X'" )
print( "Get files." )
for root, dirs, files in os.walk(INPUTDIR): #walk recursively
for file in files:
if "unit" in file and not file.endswith(".tgz") and not file.endswith(".tar.gz"):
trackFiles.append(file)
elif file == "tags":
animalFile = INPUTDIR+'/'+file
animalFile = INPUTDIR+os.sep+file
print(f"Found animalFile: {animalFile}")
elif file == "skyglow.log":
skyglowFile = INPUTDIR+'/'+file
elif hasLight() and file == "skyglow.log":
skyglowFile = INPUTDIR+os.sep+file
print(f"Found logfile: {skyglowFile}")
else:
pp(f"Ignoring ({file})")
......@@ -494,13 +495,9 @@ def getFileList():
return animalFile, skyglowFile, trackFiles
def make_rel_abs_path(path):
if path.startswith('../'):
path = os.path.abspath(SCRIPTPATH+path)
elif path.startswith('./'):
path = SCRIPTPATH+path[2:]
elif path.startswith("~/"):
path = os.path.expanduser("~")+path[1:]
path = os.path.normpath(path)
path = os.path.expanduser(path)
path = os.path.abspath(path)
return path
def main():
......@@ -520,6 +517,7 @@ def main():
os.system(cmd)
if __name__ == "__main__" :
known_projects=["ecotrack","ecolux","schrebatron"]
print(f" Process raw data for which block?\n {RULER}")
for idx, b in enumerate(blox):
print(f" {idx+1}) Project {b[0]}")
......@@ -535,13 +533,16 @@ if __name__ == "__main__" :
startTime = dt.strptime(blox[n][3], TIME_FMT)
endTime = dt.strptime(blox[n][4], TIME_FMT)
known_projects=["ecotrack","ecotron","schrebatron"]
if PROJ not in known_projects:
print(f"Unknown Project '{PROJ}'. Choose one of {known_projects}")
exit()
except:
print("Insert Block Number..")
raise Exception(f"Unknown Project '{PROJ}'")
except Exception as e:
print(f"Error: {e}\n")
print("You can add a configuration by adding a line to the 'blox' structure. It has following form:\n")
print(" (Project, inDir, outDir, start, end)\n" )
print(f"> Project must be in {known_projects}" )
print(f"> times must have Format like '{dt.now().strftime(TIME_FMT)}'" )
print(f"> dirs can be relative. eg: ('./data','../data','~/data')")
exit()
start = time.time()
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment