Curate outputs and logging

Author: csala

.dockerignore

@@ -1,4 +1,5 @@
 input/
+input_dir/
 output/
 docs/
 tests/

.gitignore

@@ -105,7 +105,7 @@ ENV/
 # vim
 .*.swp
 
-input
+input*
 output/
 notebooks/
 templates.bak/

run_docker.sh

@@ -2,6 +2,7 @@
 
 # docker build -t mit-d3m-ta2 .
 
+rm -r output
 mkdir -p output
 chown $USER output
 

ta2/__main__.py

@@ -153,6 +153,12 @@ def process_dataset(dataset_name, dataset_root, problem, args):
 
 def _ta2_test(args):
 
+    # Cleanup output dir
+    shutil.rmtree(os.path.join(args.output, 'pipelines_ranked'), ignore_errors=True)
+    shutil.rmtree(os.path.join(args.output, 'pipelines_scored'), ignore_errors=True)
+    shutil.rmtree(os.path.join(args.output, 'pipelines_searched'), ignore_errors=True)
+    shutil.rmtree(os.path.join(args.output, 'predictions'), ignore_errors=True)
+
     results = list()
     if args.all:
         args.dataset = os.listdir(args.input)
@@ -365,12 +371,12 @@ def parse_args():
 
     args = parser.parse_args()
 
-    # Cleanup output dir
-    shutil.rmtree(args.output)
     os.makedirs(args.output, exist_ok=True)
 
-    if not args.logfile:
+    if args.mode is _ta2_test and not args.logfile:
         args.logfile = os.path.join(args.output, 'ta2.log')
+        if os.path.exists(args.logfile):
+            os.remove(args.logfile)
 
     logging_setup(args.verbose, args.logfile, stdout=args.stdout)
     logging.getLogger("d3m.metadata.pipeline_run").setLevel(logging.ERROR)