Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option to force analysis even if resource has not changed #205

Merged
merged 7 commits into from
Nov 6, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: force analysis by default for CLI and API call
  • Loading branch information
Pierlou committed Oct 29, 2024
commit d5538dd962eaa3b4e70a87dd29bba861e365f763
2 changes: 1 addition & 1 deletion udata_hydra/cli.py
Original file line number Diff line number Diff line change
@@ -137,7 +137,7 @@ async def crawl_url(url: str, method: str = "get"):


@cli
async def check_resource(resource_id: str, method: str = "get", force_analysis: bool = False):
async def check_resource(resource_id: str, method: str = "get", force_analysis: bool = True):
"""Trigger a complete check for a given resource_id"""
resource: asyncpg.Record | None = await Resource.get(resource_id)
if not resource:
4 changes: 2 additions & 2 deletions udata_hydra/crawl/check_resources.py
Original file line number Diff line number Diff line change
@@ -58,10 +58,10 @@ async def check_resource(
url: str,
resource_id: str,
session,
force_analysis: bool = False,
sleep: float = 0,
method: str = "head",
worker_priority: str = "default",
force_analysis: bool = False,
) -> str:
log.debug(f"check {url}, sleep {sleep}, method {method}")

@@ -128,7 +128,7 @@ async def check_resource(
await Resource.update(resource_id, data={"status": "TO_ANALYSE_RESOURCE"})

# Enqueue the resource for analysis
queue.enqueue(analyse_resource, check["id"], is_first_check, _priority=worker_priority)
queue.enqueue(analyse_resource, check["id"], is_first_check, force_analysis, _priority=worker_priority)

return RESOURCE_RESPONSE_STATUSES["OK"]

2 changes: 1 addition & 1 deletion udata_hydra/routes/checks.py
Original file line number Diff line number Diff line change
@@ -64,7 +64,7 @@ async def create_check(request: web.Request) -> web.Response:
try:
payload: dict = await request.json()
resource_id: str = payload["resource_id"]
force_analysis: bool = payload.get("force_analysis", False)
force_analysis: bool = payload.get("force_analysis", True)
except Exception as err:
raise web.HTTPBadRequest(text=json.dumps({"error": str(err)}))