Added option to avoid data/raw to be removed, need to be explicitly set

Author: jwackito

.env_example

@@ -11,6 +11,16 @@
 #
 # DO NOT ADD THE REAL FILE TO VERSION CONTROL!
 
+###### [config] ######
+# Uncomment if you want raw data to cleaned after processed.
+# Note that if you mount the zeek output folder as a volume inside the docker
+# image instead of manually copy those directories to the data/raw folder,
+# and if you enable this option, then AIP  WILL REMOVE YOUR ZEEK FILES!!!
+# Use with care, and only if the data/raw/ folder contains a copy of the files
+# to be processed.
+# True, TRUE, true, TrUe are equivalent
+#remove_raw_data = 'true'
+
 ###### [secret] ######
 salt = 'mysecretsalt'
 

lib/aip/data/functions.py

@@ -65,4 +65,7 @@ def getrawdata(date):
 def removerawdata(date):
     dt.datetime.strptime(date, '%Y-%m-%d')
     p = path.join(_project_dir,'data','raw', date)
-    shutil.rmtree(p)
+    # Only delete raw data if explicitly allowed in the configuration file
+    if _config['remove_raw_data'].lower() == 'true':
+        shutil.rmtree(p)
+