diff --git a/dataFilterCooloff.py b/dataFilterCooloff.py index c20286876223b9270bdc07ba854a4971d0e68dba..550e507ea52df75ee8cdb9cc122fd60521d9b7d7 100755 --- a/dataFilterCooloff.py +++ b/dataFilterCooloff.py @@ -5,7 +5,7 @@ import sys, os COL_TIMESTAMP=0 COL_TAG=7 COL_INDIV=9 -HEADER="#timestamp, milliseconds, date, time, unit, x, y, tagID, species, speciesnumber, weight_without_tag[mg], moon_real[mLux], moon_eco[mLux], skyglow[Lux])\n" +HEADER="#timestamp, milliseconds, date, time, unit, x, y, tagID, species, speciesnumber, weight_without_tag_mg, moon_real_mLux, moon_eco_mLux, skyglow_Lux\n" def main(): diff --git a/dataSync.py b/dataSync.py index a234ddc65abf46a0715de0d7691879c28114eb38..7dee1aa27dc024cc905093ff5dfb3d5178a4605b 100755 --- a/dataSync.py +++ b/dataSync.py @@ -3,7 +3,7 @@ # output should be one huge table with all necessary fields for further processing/vizualization # ############################ -# TODO *most important +# TODO ############################ # parse moonmap and skyglowdict from skyglow Project sources (submodule ... ?) # @@ -25,9 +25,7 @@ # * calc indices of data dependent on numThreads # # 2. rewrite animalmerge for loops -# 3. Consider using Pandas, NumPy, Cython, Pypy, Koalas, ... -# 4. inline C++/C/Bash -# 5. sql (SQLAlchemy) +# 3. Consider using Pandas, NumPy, Cython, Pypy, Koalas, inline C++/C/Bash, sql (SQLAlchemy) # # how to profile to find bottlenecks? # @@ -42,10 +40,9 @@ # Block2: 15.9 - 13.10 # -import sys, getopt, os, re, datetime, string +import sys, getopt, os, re, datetime, string, time from pprint import pprint as pp from datetime import datetime as dt -import time PROJ = "ecotron" # PROJ = "schrebatron" @@ -138,7 +135,7 @@ moonMap = [ 1.409, 187, 192.067, 196.433, - 201.467, #42 > max-moon is 201.9mlx + 201.467, 206.5, 211.733, 216.4, @@ -297,7 +294,7 @@ class Data: """ merge AnimalData into TrackData on TagID """ pp("merging animal data into track data (this might take a while)") - #TODO SPEED UP!!! 8Mio lines take 16Minutes... 32mins + #TODO SPEED UP!!! # ttags_not_found=[] dflt=f'{DELIM}X{DELIM}X00{DELIM}0' @@ -472,12 +469,12 @@ def main(): if __name__ == "__main__" : if len(sys.argv) == 1: this = os.path.basename(__file__) - sys.exit(f' {this} <dir> - search for "skyglow.log"/"tags" and trackfiles in dir and write processed file to dir' - f'or\n {this} <indir> <outdir> - search files in <indir> and write to <outdir>') + sys.exit(f' {this} <dir> - search for "skyglow.log"/"tags" and trackfiles in dir and write processed file to dir\n' + f'{this} <indir> <outdir> - search files in <indir> and write to <outdir>') elif len(sys.argv) == 2: INPUTDIR = sys.argv[1] OUTPUTDIR = INPUTDIR - elif len(sys.argv) >= 3: + else: INPUTDIR = sys.argv[1] OUTPUTDIR = sys.argv[2]