Copy in

vmms/ec2SSH.py

@@ -16,6 +16,7 @@
 import time
 import logging
 import threading
+from concurrent.futures import ThreadPoolExecutor
 import config
 import backoff
 import boto3
@@ -510,18 +511,19 @@ def waitVM(self, vm, max_secs):
             # Sleep a bit before trying again
             time.sleep(config.Config.TIMER_POLL_INTERVAL)
 
-    def copyIn(self, vm, inputFiles):
+    def copyIn(self, vm, inputFiles, job_id):
         """copyIn - Copy input files to VM"""
         self.log.info("copyIn %s - writing files" % self.instanceName(vm.id, vm.name))
 
         domain_name = self.domainName(vm)
 
+        # Creates directory and add permissions
         result = subprocess.run(
             ["ssh"]
             + self.ssh_flags
             + [
                 "%s@%s" % (self.ec2User, domain_name),
-                "(mkdir autolab)",
+                "(mkdir -p autolab && chmod 775 autolab)",
             ],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
@@ -530,11 +532,15 @@ def copyIn(self, vm, inputFiles):
 
         # Print the output and error
         for line in result.stdout:
-            self.log.info("%s" % line)
-        self.log.info("Standard Error: %s" % result.stderr)
-        self.log.info("Return Code: %s" % result.returncode)
+            self.log.info("%s for job $s" % line, job_id)
+        self.log.info("Return Code: %s, job: %s" % result.returncode, job_id)
+        if result.stderr != 0:
+            self.log.info("Standard Error: %s, job: %s" % result.stderr, job_id)
+
+        # Validate inputFiles structure
+        if not inputFiles or not all(hasattr(file, 'localFile') and hasattr(file, 'destFile') for file in inputFiles):
+            self.log.info("Error: Invalid inputFiles Structure, job: %s", job_id)
 
-        # Copy the input files to the input directory
         for file in inputFiles:
             self.log.info("%s - %s" % (file.localFile, file.destFile))
             ret = timeout_with_retries(
@@ -547,6 +553,7 @@ def copyIn(self, vm, inputFiles):
                 config.Config.COPYIN_TIMEOUT,
             )
             if ret != 0:
+                self.log.info("Copy-in Error: SCP failure, job: %s", job_id)
                 return ret
 
         return 0

worker.py

@@ -288,7 +288,10 @@ def run(self):
             )
 
             # Copy input files to VM
-            ret["copyin"] = self.vmms.copyIn(vm, self.job.input)
+            self.log.debug(f"Before copyIn: ret[copyin] = {ret['copyin']}, job_id: {str(self.job.id)}")
+            ret["copyin"] = self.vmms.copyIn(vm, self.job.input, self.job.id)
+            self.log.debug(f"After copyIn: ret[copyin] = {ret['copyin']}, job_id: {str(self.job.id)}")
+
             if ret["copyin"] != 0:
                 Config.copyin_errors += 1
                 msg = "Error: Copy in to VM failed (status=%d)" % (ret["copyin"])