Merge branch 'master' of github.com:covidgraph/motherlode

motherlode/config.py

@@ -26,6 +26,8 @@ class DEFAULT(ConfigBase):
     # similar to DATALOADER_LIST but only to run a single dataloader. overrides DATALOADER_LIST
     DATALOADER_SINGLE = None
 
+    SKIP_DATALOADER_LIST = []
+
     # DOCKER_DEAMON_BASE_URL = "tcp://127.0.0.1:1234"
     DOCKER_DEAMON_BASE_URL = "unix://var/run/docker.sock"
 

motherlode/main.py

@@ -50,6 +50,7 @@ def create_log_node(dataloader_name, image):
 
 
 def get_log_nodes(dataloader_name, image):
+    # NodeMatcher will fail with neo4j 4.x and py2neo 4.*
     return list(
         py2neo.NodeMatcher(get_graph()).match(
             "LoadingLog",
@@ -186,9 +187,13 @@ def run_datasource_containers():
 
     for datasource in sorted_datasources:
         envs = env_vars.copy()
+
         if "envs" in datasource:
             envs.update(datasource["envs"])
-        log.info("###########################".format(datasource["dockerimage"]))
+        log.info("###########################{}".format(datasource["dockerimage"]))
+        if datasource["name"] in config.SKIP_DATALOADER_LIST:
+            log.info("Skip because of value in config.SKIP_DATALOADER_LIST")
+            continue
         container_name = "ML_{}".format(datasource["name"])
         log.info(
             "[{}]: checking dependencies: {}".format(
@@ -249,7 +254,11 @@ def run_datasource_containers():
                 log_file = open(log_file_path, "a")
                 log_file.write(l.decode())
                 log_file.close()
-            res = container.wait()
+            try:
+                res = container.wait()
+            except:
+                # container allready shut down. this can happen when exiting happens very fast
+                pass
 
         log_file = open(log_file_path, "a")
         log_file.write("================================================")