Wip crawler logic

wip crawl moodle

Wip iteration 2

wip

wip

bin/runner.py

@@ -0,0 +1,72 @@
+"""Run Experience Index indexers"""
+
+import logging
+
+import argparse
+
+import asyncio
+
+from httpx import HTTPError
+from pydantic import ValidationError
+
+
+from warren.xi.client import ExperienceIndex
+
+from warren.xi.indexers.moodle.client import (
+    Moodle
+)
+
+from warren.xi.indexers.moodle.etl import (
+    Courses,
+    CourseContent,
+)
+
+from warren.xi.indexers.factories import IndexerFactory, SourceFactory
+
+
+logger = logging.getLogger(__name__)
+
+
+source_factory = SourceFactory()
+source_factory.register(Moodle)
+
+indexer_factory = IndexerFactory()
+indexer_factory.register(Moodle, 'courses', Courses)
+indexer_factory.register(Moodle, 'modules', CourseContent)
+
+
+async def run_indexer(args):
+    """Run an XI indexer based on provided arguments."""
+
+    target = ExperienceIndex()
+    source = source_factory.create(args.source_key)
+
+    try:
+        indexer = await indexer_factory.create(source, target, **vars(args))
+    except Exception:
+        logger.exception("Could not instantiate indexer")
+
+    try:
+        logger.debug("Running indexer: %s", indexer)
+        await indexer.execute()
+
+    except (HTTPError, ValidationError, ValueError):
+        logger.exception("Runner has stopped")
+
+    finally:
+        logger.debug("Closing clients session")
+        await source.close()
+        await target.close()
+
+
+if __name__ == "__main__":
+    """Main function to run the XI indexer."""
+
+    parser = argparse.ArgumentParser(description="Wip.")
+    parser.add_argument('source_key', type=str, help='Wip.')
+    parser.add_argument('indexer_key', type=str, help='Wip.',)
+    parser.add_argument('-ie', '--ignore_errors', action='store_true', default=False)
+    parser.add_argument('--course_iri', type=str, nargs='?')
+
+    args = parser.parse_args()
+    asyncio.run(run_indexer(args))

src/api/core/warren/conf.py

@@ -65,6 +65,10 @@ class Settings(BaseSettings):
     EXECUTION_ENVIRONMENT: str = "development"
     SENTRY_API_TRACES_SAMPLE_RATE: float = 1.0
 
+    # Experience Index
+    MOODLE_BASE_URL: str = "https://moodle.preprod-fun.apps.openfun.fr"
+    MOODLE_WS_TOKEN: str = "87e89ec22a9c96965677d9e86288b98d"
+
     @property
     def DATABASE_URL(self) -> str:
         """Get the database URL as required by SQLAlchemy."""

src/api/core/warren/tests/xi/client/__init__.py

@@ -0,0 +1 @@
+"""Experience Index client tests package."""

src/api/core/warren/tests/xi/client/test_crud_experience.py

@@ -0,0 +1,108 @@
+"""Tests for XI experiences client."""
+
+import re
+import uuid
+from unittest.mock import AsyncMock
+
+import pytest
+from httpx import AsyncClient, HTTPError
+from pydantic.main import BaseModel
+from pytest_httpx import HTTPXMock
+from sqlmodel import Session
+
+from warren.xi.client import CRUDExperience
+from warren.xi.factories import ExperienceFactory
+from warren.xi.models import ExperienceCreate, ExperienceRead
+
+
+@pytest.mark.anyio
+async def test_crud_experience_raise_status(
+    httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+    """Test that each operation raises an HTTP error in case of failure."""
+    crud_instance = CRUDExperience(client=http_client)
+
+    # Mock each request to the XI by returning a 422 status
+    httpx_mock.add_response(url=re.compile(r".*experiences.*"), status_code=422)
+
+    class WrongData(BaseModel):
+        name: str
+
+    # Assert 'create' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.create(data=WrongData(name="foo"))
+
+    # Assert 'update' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.update(object_id=uuid.uuid4(), data=WrongData(name="foo"))
+
+    # Assert 'read' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.read()
+
+    # Assert 'get' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.get(object_id="foo.")
+
+
+@pytest.mark.anyio
+async def test_crud_experience_get_not_found(
+    httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+    """Test getting an unknown experience."""
+    crud_instance = CRUDExperience(client=http_client)
+
+    # Mock GET request to the XI by returning a 404 status
+    httpx_mock.add_response(
+        method="GET", url=re.compile(r".*experiences.*"), status_code=404
+    )
+
+    # Assert 'get' return 'None' without raising any HTTP errors
+    response = await crud_instance.get(object_id=uuid.uuid4())
+    assert response is None
+
+
+@pytest.mark.anyio
+async def test_crud_experience_create_or_update_new(
+    http_client: AsyncClient, db_session: Session
+):
+    """Test creating an experience using 'create_or_update'."""
+    crud_instance = CRUDExperience(client=http_client)
+
+    # Get random experience data
+    data = ExperienceFactory.build_dict()
+
+    # Simulate a 'Not Found' experience by mocking the 'get' method
+    crud_instance.get = AsyncMock(return_value=None)
+
+    crud_instance.create = AsyncMock()
+    crud_instance.update = AsyncMock()
+
+    # Attempt creating an experience
+    await crud_instance.create_or_update(ExperienceCreate(**data))
+
+    crud_instance.create.assert_awaited_once()
+    crud_instance.update.assert_not_awaited()
+
+
+@pytest.mark.anyio
+async def test_crud_experience_create_or_update_existing(
+    http_client: AsyncClient, db_session: Session
+):
+    """Test updating an experience using 'create_or_update'."""
+    crud_instance = CRUDExperience(client=http_client)
+
+    # Get random experience data
+    data = ExperienceFactory.build_dict(exclude={})
+
+    # Simulate an existing experience by mocking the 'get' method
+    crud_instance.get = AsyncMock(return_value=ExperienceRead(**data))
+
+    crud_instance.create = AsyncMock()
+    crud_instance.update = AsyncMock()
+
+    # Attempt updating an experience
+    await crud_instance.create_or_update(ExperienceCreate(**data))
+
+    crud_instance.create.assert_not_awaited()
+    crud_instance.update.assert_awaited_once()

src/api/core/warren/tests/xi/client/test_crud_relation.py

@@ -0,0 +1,106 @@
+"""Tests for XI relations client."""
+
+import re
+from unittest.mock import AsyncMock, call
+from uuid import uuid4
+
+import pytest
+from httpx import AsyncClient, HTTPError
+from pydantic.main import BaseModel
+from pytest_httpx import HTTPXMock
+from sqlmodel import Session
+
+from warren.xi.client import CRUDRelation
+from warren.xi.enums import RelationType
+from warren.xi.models import RelationCreate
+
+
+@pytest.mark.anyio
+async def test_crud_relation_raise_status(
+    httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+    """Test that each operation raises an HTTP error in case of failure."""
+    crud_instance = CRUDRelation(client=http_client)
+
+    # Mock each request to the XI by returning a 422 status
+    httpx_mock.add_response(url=re.compile(r".*relations.*"), status_code=422)
+
+    class WrongData(BaseModel):
+        name: str
+
+    # Assert 'create' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.create(data=WrongData(name="foo"))
+
+    # Assert 'update' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.update(object_id=uuid4(), data=WrongData(name="foo"))
+
+    # Assert 'read' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.read()
+
+    # Assert 'get' raises an HTTP error
+    with pytest.raises(HTTPError):
+        await crud_instance.get(object_id="foo.")
+
+
+@pytest.mark.anyio
+async def test_crud_relation_get_not_found(
+    httpx_mock: HTTPXMock, http_client: AsyncClient
+):
+    """Test getting an unknown relation."""
+    crud_instance = CRUDRelation(client=http_client)
+
+    # Mock GET request to the XI by returning a 404 status
+    httpx_mock.add_response(
+        method="GET", url=re.compile(r".*relations.*"), status_code=404
+    )
+
+    # Assert 'get' return 'None' without raising any HTTP errors
+    response = await crud_instance.get(object_id=uuid4())
+    assert response is None
+
+
+@pytest.mark.anyio
+async def test_crud_relation_create_bidirectional(
+    http_client: AsyncClient, db_session: Session
+):
+    """Test creating bidirectional relations."""
+    crud_instance = CRUDRelation(client=http_client)
+
+    # Get two inverse relation types
+    relation_type = RelationType.HASPART
+    inverted_relation_type = RelationType.ISPARTOF
+
+    # Get two random UUIDs
+    source_id = uuid4()
+    target_id = uuid4()
+
+    # Mock 'create' method by returning a random UUID
+    crud_instance.create = AsyncMock(return_value=uuid4())
+
+    # Attempt creating bidirectional relations
+    _ = await crud_instance.create_bidirectional(
+        source_id=source_id,
+        target_id=target_id,
+        kinds=[relation_type, inverted_relation_type],
+    )
+
+    # Assert 'create' has been called twice, with inverted arguments
+    crud_instance.create.assert_has_awaits(
+        [
+            call(
+                RelationCreate(
+                    source_id=source_id, target_id=target_id, kind=relation_type
+                )
+            ),
+            call(
+                RelationCreate(
+                    source_id=target_id,
+                    target_id=source_id,
+                    kind=inverted_relation_type,
+                )
+            ),
+        ]
+    )

src/api/core/warren/tests/xi/indexers/__init__.py

@@ -0,0 +1 @@
+"""Experience Index indexers tests package."""

src/api/core/warren/tests/xi/indexers/moodle/__init__.py

@@ -0,0 +1 @@
+"""Experience Index Moodle tests package."""