add auto pause expiry support:

- stop crawls that have been paused for too long
- add 'paused_crawl_limit_minutes' to Helm chart
- add paused time and expiry to crawlconfig API response
- set to 'stopped_pause_expired' state
- ui: add support for 'Stopped: Paused Too Long' for stopped_pause_expired
- use 'paused_at' in CrawlJob to indicate crawl is paused and when

Author: tw4l

backend/btrixcloud/crawlconfigs.py

@@ -11,7 +11,7 @@
 import re
 import os
 import traceback
-from datetime import datetime
+from datetime import datetime, timedelta
 from uuid import UUID, uuid4
 import urllib.parse
 
@@ -89,6 +89,8 @@ class CrawlConfigOps:
     crawler_images_map: dict[str, str]
     crawler_image_pull_policy_map: dict[str, str]
 
+    paused_expiry_delta: timedelta
+
     def __init__(
         self,
         dbclient,
@@ -115,6 +117,10 @@ def __init__(
             "DEFAULT_CRAWLER_IMAGE_PULL_POLICY", "IfNotPresent"
         )
 
+        self.paused_expiry_delta = timedelta(
+            minutes=int(os.environ.get("PAUSED_CRAWL_LIMIT_MINUTES", "10080"))
+        )
+
         self.router = APIRouter(
             prefix="/crawlconfigs",
             tags=["crawlconfigs"],
@@ -750,6 +756,12 @@ async def _add_running_curr_crawl_stats(self, crawlconfig: CrawlConfigOut):
         crawlconfig.lastCrawlSize = crawl.stats.size if crawl.stats else 0
         crawlconfig.lastCrawlStopping = crawl.stopping
         crawlconfig.lastCrawlPausing = crawl.pausing
+        crawlconfig.lastCrawlPausedAt = crawl.pausedAt
+        crawlconfig.lastCrawlPausedExpiry = None
+        if crawl.pausedAt:
+            crawlconfig.lastCrawlPausedExpiry = (
+                crawl.pausedAt + self.paused_expiry_delta
+            )
         crawlconfig.isCrawlRunning = True
 
     async def get_crawl_config_out(self, cid: UUID, org: Organization):

backend/btrixcloud/crawlmanager.py

@@ -4,7 +4,7 @@
 import secrets
 
 from typing import Optional, Dict, Tuple
-from datetime import timedelta
+from datetime import datetime, timedelta
 
 from fastapi import HTTPException
 
@@ -386,9 +386,13 @@ async def shutdown_crawl(self, crawl_id: str, graceful=True) -> dict:
 
         return await self.delete_crawl_job(crawl_id)
 
-    async def pause_resume_crawl(self, crawl_id: str, pause: bool) -> dict:
+    async def pause_resume_crawl(
+        self, crawl_id: str, paused_at: Optional[datetime] = None
+    ) -> dict:
         """pause or unpause a crawl"""
-        return await self._patch_job(crawl_id, {"paused": int(pause)})
+        return await self._patch_job(
+            crawl_id, {"pausedAt": date_to_str(paused_at) if paused_at else ""}
+        )
 
     async def delete_crawl_configs_for_org(self, org: str) -> None:
         """Delete all crawl configs for given org"""

backend/btrixcloud/crawls.py

@@ -778,13 +778,21 @@ async def pause_crawl(
             raise HTTPException(status_code=400, detail="not_a_crawl")
 
         result = None
+
+        if pause:
+            paused_at = dt_now()
+        else:
+            paused_at = None
+
         try:
-            result = await self.crawl_manager.pause_resume_crawl(crawl_id, pause=pause)
+            result = await self.crawl_manager.pause_resume_crawl(
+                crawl_id, paused_at=paused_at
+            )
 
             if result.get("success"):
                 await self.crawls.find_one_and_update(
                     {"_id": crawl_id, "type": "crawl", "oid": org.id},
-                    {"$set": {"pausing": pause}},
+                    {"$set": {"pausing": pause, "pausedAt": paused_at}},
                 )
 
                 return {"success": True}

backend/btrixcloud/models.py

@@ -238,6 +238,7 @@ class UserOrgInfoOut(BaseModel):
 TYPE_SUCCESSFUL_STATES = Literal[
     "complete",
     "stopped_by_user",
+    "stopped_pause_expired",
     "stopped_storage_quota_reached",
     "stopped_time_quota_reached",
     "stopped_org_readonly",
@@ -481,6 +482,8 @@ class CrawlConfigOut(CrawlConfigCore, CrawlConfigAdditional):
 
     lastCrawlStopping: Optional[bool] = False
     lastCrawlPausing: Optional[bool] = False
+    lastCrawlPausedAt: Optional[datetime] = None
+    lastCrawlPausedExpiry: Optional[datetime] = None
     profileName: Optional[str] = None
     firstSeed: Optional[str] = None
     seedCount: int = 0
@@ -867,6 +870,7 @@ class CrawlOut(BaseMongoModel):
     profileName: Optional[str] = None
     stopping: Optional[bool] = False
     pausing: Optional[bool] = False
+    pausedAt: Optional[datetime] = None
     manual: bool = False
     cid_rev: Optional[int] = None
     scale: Scale = 1

backend/btrixcloud/operator/crawls.py

@@ -4,8 +4,8 @@
 import os
 import math
 from pprint import pprint
-from typing import Optional, Any, Sequence
-from datetime import datetime
+from typing import Optional, Any, Sequence, cast
+from datetime import datetime, timedelta
 from uuid import UUID
 
 import json
@@ -79,6 +79,7 @@
 
 # pylint: disable=too-many-public-methods, too-many-locals, too-many-branches, too-many-statements
 # pylint: disable=invalid-name, too-many-lines, too-many-return-statements
+# pylint: disable=too-many-instance-attributes
 # ============================================================================
 class CrawlOperator(BaseOperator):
     """CrawlOperator Handler"""
@@ -93,6 +94,8 @@ class CrawlOperator(BaseOperator):
 
     min_avail_storage_ratio: float
 
+    paused_expires_delta: timedelta
+
     def __init__(self, *args):
         super().__init__(*args)
 
@@ -110,6 +113,13 @@ def __init__(self, *args):
             os.environ.get("CRAWLER_MIN_AVAIL_STORAGE_RATIO") or 0
         )
 
+        # time in minutes before paused crawl is stopped - default is 7 days
+        paused_crawl_limit_minutes = int(
+            os.environ.get("PAUSED_CRAWL_LIMIT_MINUTES", "10080")
+        )
+
+        self.paused_expires_delta = timedelta(minutes=paused_crawl_limit_minutes)
+
     def init_routes(self, app):
         """init routes for this operator"""
 
@@ -160,7 +170,7 @@ async def sync_crawls(self, data: MCSyncData):
             scale=spec.get("scale", 1),
             started=data.parent["metadata"]["creationTimestamp"],
             stopping=spec.get("stopping", False),
-            paused=spec.get("paused", False),
+            paused_at=str_to_date(spec.get("pausedAt")),
             timeout=spec.get("timeout") or 0,
             max_crawl_size=int(spec.get("maxCrawlSize") or 0),
             scheduled=spec.get("manual") != "1",
@@ -265,11 +275,25 @@ async def sync_crawls(self, data: MCSyncData):
             status.scale = 1
 
         # stopping paused crawls
-        if crawl.paused and crawl.stopping:
-            status.stopReason = "stopped_by_user"
-            status.stopping = True
-            print(f"Paused crawl stopped by user, id: {crawl.id}")
-            await self.mark_finished(crawl, status, "stopped_by_user")
+        if crawl.paused_at:
+            stop_reason: Optional[StopReason] = None
+            state: Optional[TYPE_NON_RUNNING_STATES] = None
+            # Check if pause expiry limit is reached and if so, stop crawl
+            if dt_now() >= (crawl.paused_at + self.paused_expires_delta):
+                print(f"Paused crawl expiry reached, stopping crawl, id: {crawl.id}")
+                stop_reason = "stopped_pause_expired"
+                state = "stopped_pause_expired"
+
+            # Check if paused crawl was stopped manually
+            elif crawl.stopping:
+                print(f"Paused crawl stopped by user, id: {crawl.id}")
+                stop_reason = "stopped_by_user"
+                state = "stopped_by_user"
+
+            if stop_reason and state:
+                status.stopping = True
+                status.stopReason = stop_reason
+                await self.mark_finished(crawl, status, state)
 
         children = self._load_redis(params, status, data.children)
 
@@ -335,7 +359,9 @@ async def sync_crawls(self, data: MCSyncData):
 
         for i in range(0, status.scale):
             children.extend(
-                self._load_crawler(params, i, status, data.children, crawl.paused)
+                self._load_crawler(
+                    params, i, status, data.children, bool(crawl.paused_at)
+                )
             )
 
         return {
@@ -858,12 +884,12 @@ async def sync_crawl_state(
                 ):
                     # mark as waiting (if already running)
                     await self.set_state(
-                        "waiting_capacity" if not crawl.paused else "paused",
+                        "waiting_capacity" if not crawl.paused_at else "paused",
                         status,
                         crawl,
                         allowed_from=(
                             RUNNING_AND_STARTING_ONLY
-                            if not crawl.paused
+                            if not crawl.paused_at
                             else RUNNING_AND_WAITING_STATES
                         ),
                     )
@@ -884,7 +910,7 @@ async def sync_crawl_state(
 
                     # crawler pods already shut down, remove redis pause key
                     # for simple resume later
-                    if crawl.paused:
+                    if crawl.paused_at:
                         await redis.delete(f"{crawl.id}:paused")
 
                 elif crawler_running and not redis:
@@ -896,7 +922,7 @@ async def sync_crawl_state(
                 return status
 
             # only get here if at least one crawler pod is running
-            if crawl.paused:
+            if crawl.paused_at:
                 await redis.set(f"{crawl.id}:paused", "1")
 
             # update lastActiveTime if crawler is running

backend/btrixcloud/operator/models.py

@@ -1,6 +1,7 @@
 """Operator Models"""
 
 from collections import defaultdict
+from datetime import datetime
 from uuid import UUID
 from typing import Optional, DefaultDict, Literal, Annotated, Any
 from pydantic import BaseModel, Field
@@ -17,6 +18,7 @@
 
 StopReason = Literal[
     "stopped_by_user",
+    "stopped_pause_expired",
     "time-limit",
     "size-limit",
     "stopped_storage_quota_reached",
@@ -76,7 +78,7 @@ class CrawlSpec(BaseModel):
     started: str
     crawler_channel: str
     stopping: bool = False
-    paused: bool = False
+    paused_at: Optional[datetime] = None
     scheduled: bool = False
     timeout: int = 0
     max_crawl_size: int = 0

chart/app-templates/crawl_job.yaml

@@ -36,5 +36,5 @@ spec:
 
   proxyId: "{{ proxy_id }}"
 
-  paused: {{ paused or 0 }}
+  pausedAt: "{{ pausedAt }}"
 

chart/templates/configmap.yaml

@@ -20,6 +20,8 @@ data:
 
   INVITE_EXPIRE_SECONDS: "{{ .Values.invite_expire_seconds }}"
 
+  PAUSED_CRAWL_LIMIT_MINUTES: "{{ .Values.paused_crawl_limit_minutes }}"
+
   REGISTRATION_ENABLED: "{{ .Values.registration_enabled | default 0 }}"
 
   REGISTER_TO_ORG_ID: "{{ .Values.registration_org_id }}"

chart/values.yaml

@@ -82,6 +82,9 @@ allow_dupe_invites: "0"
 # number of seconds before pending invites expire - default is 7 days
 invite_expire_seconds: 604800
 
+# number of minutes before paused crawls are stopped - default is 7 days
+paused_crawl_limit_minutes: 10080
+
 # base url for replayweb.page
 rwp_base_url: "https://cdn.jsdelivr.net/npm/[email protected]/"
 

frontend/src/features/archived-items/crawl-status.ts

@@ -241,6 +241,16 @@ export class CrawlStatus extends TailwindElement {
         label = msg("Stopped");
         break;
 
+      case "stopped_pause_expired":
+        color = "var(--warning)";
+        icon = html`<sl-icon
+          name="dash-square-fill"
+          slot="prefix"
+          style="color: ${color}"
+        ></sl-icon>`;
+        label = msg("Stopped: Paused Too Long");
+        break;
+
       case "stopped_storage_quota_reached":
         color = "var(--warning)";
         icon = html`<sl-icon