Various improvements

Minor changes

Author: RasmusOrsoe

I3ToSQLite/I3ToSQLite.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.1
 Name: I3ToSQLite
-Version: 0.1.53
+Version: 0.1.63
 Summary: An unoffical I3-file to SQLite database converter.
 Home-page: https://github.com/RasmusOrsoe/I3ToSQLite
 Author: Rasmus F. Ørsøe

I3ToSQLite/I3ToSQLite/ __init__.py

@@ -2,6 +2,6 @@
 from .tools import *
 from .CreateTemporaryDatabases import *
 from .MergeTemporaryDatabases import *
-__version__ = "0.1.53"
+__version__ = "0.1.62"
 __author__ = 'Rasmus F. Ørsøe'
 __credits__ = 'Niels Bohr Instutite, IceCube Collaboration'

I3ToSQLite/I3ToSQLite/CreateTemporaryDatabases.py

@@ -11,7 +11,14 @@
 import time
 from multiprocessing import Pool
 import pickle
-#icetray.I3Logger.global_logger = icetray.I3NullLogger()
+import argparse
+import sys
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-config", "--config", type=str, required=True)
+args = parser.parse_args()
+
+
 
 def Contains_RetroReco(frame):
     try:
@@ -248,6 +255,7 @@ def WriteDicts(settings):
         if verbose > 0:
             print('Worker %s Reading %s'%(id,input_file.split('/')[-1]))
             print(input_file)
+            sys.stdout.flush()
         gcd_count  +=1
 
         while i3_file.more() :
@@ -281,6 +289,7 @@ def WriteDicts(settings):
                 if len(retros)>0 :
                     retro_big   = retro_big.append(retro, ignore_index = True, sort = True)
                 if len(truth_big) >= max_dict_size:
+                    print('saving')
                     engine = sqlalchemy.create_engine('sqlite:///'+outdir + '/%s/tmp/worker-%s-%s.db'%(db_name,id,output_count))
                     truth_big.to_sql('truth',engine,index= False, if_exists = 'append')
                     if len(retro_big)> 0:
@@ -296,6 +305,7 @@ def WriteDicts(settings):
         if verbose > 0:
             print('Worker %s has finished %s/%s I3 files.'%(id, file_counter, len(input_files)))
     if (len(feature_big) > 0):
+        print('saving eof')
         engine = sqlalchemy.create_engine('sqlite:///'+outdir +  '/%s/tmp/worker-%s-%s.db'%(db_name,id,output_count))
         truth_big.to_sql('truth',engine,index= False, if_exists = 'append')
         if len(retro_big)> 0:
@@ -363,8 +373,18 @@ def WalkDirectory(dir, extensions):
             gcds_folder.append(gcd_folder)
         files_list.extend(i3files_folder)
         GCD_list.extend(gcds_folder)
+    
+    files_list, GCD_list = PairWiseShuffle(files_list, GCD_list)
     return files_list, GCD_list
 
+def PairWiseShuffle(files_list, gcd_list):
+    df = pd.DataFrame({'i3': files_list, 'gcd': gcd_list})
+    df_shuffled = df.sample(frac = 1)
+    i3_shuffled = df_shuffled['i3'].tolist()
+    gcd_shuffled = df_shuffled['gcd'].tolist()
+    return i3_shuffled, gcd_shuffled
+
+
 def FindFiles(paths,outdir,db_name,gcd_rescue, extensions = None):
     print('Counting files in: \n%s\n This might take a few minutes...'%paths)
     if extensions == None:
@@ -379,10 +399,10 @@ def FindFiles(paths,outdir,db_name,gcd_rescue, extensions = None):
         input_files.extend(input_files_mid)
         gcd_files.extend(gcd_files_mid)
 
-    
+    input_files, gcd_files = PairWiseShuffle(input_files, gcd_files)
     Save_Filenames(input_files, outdir, db_name)
-
     return input_files, gcd_files
+    
 def Save_Filenames(input_files,outdir, db_name):
     input_files = pd.DataFrame(input_files)
     input_files.columns = ['filename']
@@ -405,8 +425,8 @@ def PickleCleaner(List):
         clean_list.append(str(element))
     return clean_list
 
-def Extract_Config():
-    with open('tmp/config/config.pkl', 'rb') as handle:
+def Extract_Config(config_path):
+    with open('%s/config.pkl'%config_path, 'rb') as handle:
         config = pickle.load(handle)  
     paths = PickleCleaner(config['paths'])
 
@@ -429,10 +449,10 @@ def Extract_Config():
         custom_truth    = None
     return paths, outdir, workers, pulse_keys, db_name, max_dictionary_size, custom_truth, gcd_rescue, verbose
 def Transmit_Start_Time(start_time,config_path):
-    with open(config_path, 'rb') as handle:
+    with open('%s/config.pkl'%config_path, 'rb') as handle:
         config = pickle.load(handle)
     config['start_time'] = start_time
-    with open(config_path , 'wb') as handle:
+    with open('%s/config.pkl'%config_path, 'wb') as handle:
         pickle.dump(config, handle, protocol=2)  
     return
 
@@ -443,7 +463,9 @@ def PrintMessage(workers, input_files):
     print('----------------------')
 def CreateTemporaryDatabases(paths, outdir, workers, pulse_keys,config_path, start_time,db_name,gcd_rescue,verbose,max_dictionary_size = 10000, custom_truth = None):
     if __name__ == "__main__" :
-        start_time = time.time()    
+        start_time = time.time()
+        if verbose == 0:
+            icetray.I3Logger.global_logger = icetray.I3NullLogger()    
         directory_exists = CreateOutDirectory(outdir + '/%s/tmp'%db_name)
         input_files, gcd_files = FindFiles(paths, outdir,db_name,gcd_rescue)
         if workers > len(input_files):
@@ -463,13 +485,15 @@ def CreateTemporaryDatabases(paths, outdir, workers, pulse_keys,config_path, sta
         p.map_async(WriteDicts, settings)
         p.close()
         p.join()
+        print('transmitting..')
         Transmit_Start_Time(start_time, config_path)
 
 
 start_time = time.time()
 
-paths, outdir, workers, pulse_keys, db_name, max_dictionary_size, custom_truth, gcd_rescue, verbose = Extract_Config()
-CreateTemporaryDatabases(paths, outdir, workers, pulse_keys,'tmp/config/config.pkl', start_time,db_name,gcd_rescue,verbose, max_dictionary_size, custom_truth)
+paths, outdir, workers, pulse_keys, db_name, max_dictionary_size, custom_truth, gcd_rescue, verbose = Extract_Config(args.config)
+print(args.config)
+CreateTemporaryDatabases(paths, outdir, workers, pulse_keys,args.config, start_time,db_name,gcd_rescue,verbose, max_dictionary_size, custom_truth)
 
 
 

I3ToSQLite/I3ToSQLite/MergeTemporaryDatabases.py

@@ -6,6 +6,13 @@
 import time
 import pickle
 from tqdm import tqdm
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-config", "--config", type=str, required=True)
+args = parser.parse_args()
+
+
 def fetch_temps(path):
     out = []
     files = os.listdir(path)
@@ -67,7 +74,7 @@ def Run_SQL_Code(database, CODE):
     return
 
 def Attach_Index(database, table_name):
-    CODE = "PRAGMA foreign_keys=off;\nBEGIN TRANSACTION;\nCREATE INDEX event_no ON {} (event_no);\nCOMMIT TRANSACTION;\nPRAGMA foreign_keys=on;".format(table_name)
+    CODE = "PRAGMA foreign_keys=off;\nBEGIN TRANSACTION;\nCREATE INDEX event_no_{} ON {} (event_no);\nCOMMIT TRANSACTION;\nPRAGMA foreign_keys=on;".format(table_name,table_name)
     Run_SQL_Code(database,CODE)
     return
 
@@ -95,21 +102,25 @@ def CreateTable(database,table_name, columns, is_pulse_map = False):
     CODE = "PRAGMA foreign_keys=off;\nCREATE TABLE {} ({});\nPRAGMA foreign_keys=on;".format(table_name,query_columns) 
     Run_SQL_Code(database, CODE)
     if is_pulse_map:
-        try:
-            Attach_Index(database,table_name)
-        except:
-            notimportant = 0
+        #try:
+        print(table_name)
+        print('attaching indexs')
+        Attach_Index(database,table_name)
+        #except:
+        #    notimportant = 0
     return
 
 def Create_Empty_Tables(database,pulse_map_keys,truth_columns, pulse_map_columns, retro_columns):
-    for pulse_map_key in pulse_map_keys:
-        # Creates the pulse map tables
-        print('Creating Empty %s Table'%pulse_map_key)
-        CreateTable(database, pulse_map_key,pulse_map_columns[pulse_map_key], is_pulse_map = True)
+    
     print('Creating Empty Truth Table')
     CreateTable(database, 'truth', truth_columns, is_pulse_map = False) # Creates the truth table containing primary particle attributes and RetroReco reconstructions
     print('Creating Empty RetroReco Table')
     CreateTable(database, 'RetroReco',retro_columns, is_pulse_map = False) # Creates the RetroReco Table with reconstuctions and associated values.
+    
+    for pulse_map_key in pulse_map_keys:
+        # Creates the pulse map tables
+        print('Creating Empty %s Table'%pulse_map_key)
+        CreateTable(database, pulse_map_key,pulse_map_columns[pulse_map_key], is_pulse_map = True)
     return
 
 def Submit_Truth(database, truth):
@@ -150,8 +161,8 @@ def PickleCleaner(List):
         clean_list.append(str(element))
     return clean_list
 
-def Extract_Config():
-    with open('tmp/config/config.pkl', 'rb') as handle:
+def Extract_Config(config_path):
+    with open('%s/config.pkl'%config_path, 'rb') as handle:
         config = pickle.load(handle)  
     paths = PickleCleaner(config['paths'])
 
@@ -198,7 +209,7 @@ def CreateDatabase(database_name,outdir, pulse_map_keys):
     return
 
 
-paths, outdir, workers, pulse_keys, db_name, max_dictionary_size, custom_truth, start_time = Extract_Config()
+paths, outdir, workers, pulse_keys, db_name, max_dictionary_size, custom_truth, start_time = Extract_Config(args.config)
 CreateDatabase(db_name, outdir, pulse_keys)
 
 print('Database Creation Successful!')

I3ToSQLite/I3ToSQLite/Tools.py

@@ -16,42 +16,40 @@ def Build_Configuration(paths, outdir, workers, pulse_keys, db_name, gcd_rescue,
         os.makedirs(dictionary_path)
     except:
         notimportant = 0
-    try:
-        os.makedirs('tmp/config')
-    except:
-        notimportant = 0
 
     with open(dictionary_path + '/config.pkl', 'wb') as handle:
         pickle.dump(dictionary, handle, protocol=2) 
-    with open('tmp/config/config.pkl', 'wb') as handle:
-        pickle.dump(dictionary, handle, protocol=2)              
-    return dictionary_path + '/config.pkl'
-def MakeDir():
+    
+    print(dictionary_path)              
+    return dictionary_path
+def MakeDir(coms_path):
     try:
-        os.makedirs('tmp/coms')
+        os.makedirs(coms_path)
     except:
         notimportant = 0
-def Write_Handler(cvmfs_setup_path, cvmfs_shell_path):
-    CODE = "eval `%s` \n%s ./tmp/coms/run_extraction.sh"%(cvmfs_setup_path,cvmfs_shell_path)
-    text_file = open("tmp/coms/handler.sh", "w")
+    return coms_path
+
+def Write_Handler(cvmfs_setup_path, cvmfs_shell_path, coms_path):
+    CODE = "eval `%s` \n%s %s/run_extraction.sh"%(cvmfs_setup_path,cvmfs_shell_path, coms_path)
+    text_file = open("%s/handler.sh"%coms_path, "w")
     text_file.write(CODE)
     text_file.close()
-    os.system("chmod 755 tmp/coms/handler.sh")
+    os.system("chmod 755 %s/handler.sh"%coms_path)
 
-def Write_Executer():
+def Write_Executer(config_path, coms_path):
     path = (anchor.__file__).split('anchor.py')[0]
-    CODE = "python %sCreateTemporaryDatabases.py && python %sMergeTemporaryDatabases.py && exit"%(path,path)
-    text_file = open("tmp/coms/run_extraction.sh", "w")
+    CODE = "python %sCreateTemporaryDatabases.py --config %s && python %sMergeTemporaryDatabases.py --config %s && exit"%(path,config_path,path,config_path)
+    text_file = open("%s/run_extraction.sh"%coms_path, "w")
     text_file.write(CODE)
     text_file.close()
-    os.system("chmod 755 tmp/coms/run_extraction.sh")
+    os.system("chmod 755 %s/run_extraction.sh"%coms_path)
 
 def CreateDatabase(paths, outdir, workers, cvmfs_setup_path, cvmfs_shell_path, db_name, pulse_keys, gcd_rescue, verbose = 1):
-    configuration_path = Build_Configuration(paths, outdir, workers, pulse_keys,db_name, gcd_rescue, verbose)
-    MakeDir()
-    Write_Executer()
-    Write_Handler(cvmfs_setup_path, cvmfs_shell_path)
-    os.system('./tmp/coms/handler.sh')
+    config_path = Build_Configuration(paths, outdir, workers, pulse_keys,db_name, gcd_rescue, verbose)
+    coms_path = MakeDir(outdir + '/%s/config'%db_name)
+    Write_Executer(config_path, coms_path)
+    Write_Handler(cvmfs_setup_path, cvmfs_shell_path, coms_path)
+    os.system('%s/handler.sh'%coms_path)