s3-sidecar: add lifetime timer

WIPACrepo · Feb 6, 2025 · b8ba6fa · b8ba6fa
1 parent af17b26
commit b8ba6fa
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 2 deletions.
diff --git a/s3_sidecar/__main__.py b/s3_sidecar/__main__.py
@@ -24,6 +24,7 @@ class EnvConfig:
     S3_SECRET_KEY: str
     S3_BUCKET: str
     S3_OBJECT_KEY: str
+    K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS: int
 
 
 ENV = from_environment_as_dataclass(EnvConfig)
@@ -84,13 +85,26 @@ def main() -> None:
     args = parser.parse_args()
     logging_tools.log_argparse_args(args)
 
-    logger_timer = IntervalTimer(5, LOGGER)
+    housekeeping_timer = IntervalTimer(
+        5,
+        logging.getLogger(f"{LOGGER.name}.housekeeping"),
+    )
+    lifetime_timer = IntervalTimer(
+        ENV.K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS,
+        logging.getLogger(f"{LOGGER.name}.lifetime_timer"),
+    )
 
     if args.wait_indefinitely:
         LOGGER.info("Waiting for file to exist...")
         while not args.fpath.exists():
-            if logger_timer.has_interval_elapsed():
+            if housekeeping_timer.has_interval_elapsed():
+                # log
                 LOGGER.info("still waiting...")
+                # has it been too long?
+                if lifetime_timer.has_interval_elapsed():
+                    raise RuntimeError(
+                        f"lifetime timer has expired: {lifetime_timer} seconds"
+                    )
             time.sleep(1)
 
     post(args.fpath)

diff --git a/skydriver/config.py b/skydriver/config.py
@@ -91,6 +91,7 @@ class EnvConfig:
     K8S_SCANNER_SIDECAR_S3_CPU_LIMIT: float = 0.10
     K8S_SCANNER_SIDECAR_S3_MEM_REQUEST: str = K8S_MIN_MEM_REQUEST
     K8S_SCANNER_SIDECAR_S3_CPU_REQUEST: float = 0.05
+    K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS: int = 15 * 60  # 15 mins
 
     # EWMS optional config
     EWMS_WORKER_MEMORY__DEFAULT: str = "8GB"

diff --git a/skydriver/k8s/scanner_instance.py b/skydriver/k8s/scanner_instance.py
@@ -199,6 +199,8 @@ def _make_job(
                           value: "{ENV.S3_BUCKET}"
                         - name: S3_OBJECT_KEY
                           value: "{s3.make_object_key(scan_id)}"
+                        - name: K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS
+                          value: {ENV.K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS}
                       resources:
                         limits:
                           memory: "{ENV.K8S_SCANNER_SIDECAR_S3_MEM_LIMIT}"

diff --git a/tests/integration/test_rest_routes.py b/tests/integration/test_rest_routes.py
@@ -315,6 +315,10 @@ async def _assert_db_skyscank8sjobs_coll(  # noqa: MFL000
                                         "name": "S3_OBJECT_KEY",
                                         "value": f"{post_resp['scan_id']}-s3-object",
                                     },
+                                    {
+                                        "name": "K8S_SCANNER_SIDECAR_S3_LIFETIME_SECONDS",
+                                        "value": 15 * 60,
+                                    },
                                 ],
                                 "image": os.environ["THIS_IMAGE_WITH_TAG"],
                                 "name": f"sidecar-s3-{post_resp['scan_id']}",