Merge branch 'main' into better_exceptions

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 100
+extend-ignore = E203,E701
+exclude = */proto/*_pb2*.py

.github/workflows/docker-publish.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     permissions:
       contents: read
@@ -30,39 +30,41 @@ jobs:
       id-token: write
 
     steps:
+      # The following steps are mostly inspired by the example found in 
+      # the README.md at https://github.com/docker/build-push-action
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+        uses: docker/setup-buildx-action@v3
 
       # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=edge,suffix=-py${{ matrix.python-version }},branch=main
             type=edge,enable=${{ matrix.python-version == '3.10' }},branch=main
 
       # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.github/workflows/tests.yml

@@ -5,24 +5,25 @@ on:
   - pull_request
 
 jobs:
-  build:
+  generate-jobs:
+    runs-on: ubuntu-latest
+    outputs:
+      session: ${{ steps.set-matrix.outputs.session }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wntrblm/nox@main
+      - id: set-matrix
+        shell: bash
+        run: echo session=$(nox --json -l -s tests | jq -c '[.[].session]') | tee --append $GITHUB_OUTPUT
+  checks:
+    name: Session ${{ matrix.session }}
+    needs: [generate-jobs]
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
+        session: ${{ fromJson(needs.generate-jobs.outputs.session) }}
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
-          # TODO: when calling pytest, it build a new dev image, but it should build the image
-          # using the specific python version. Before it was working cause the compose was not
-          # building the image, rather the image was built here with the {{ matrix.python-version}}
-      - name: Test with tox
-        run: tox
+      - uses: actions/checkout@v4
+      - uses: wntrblm/nox@main
+      - run: nox -s "${{ matrix.session }}"

.gitignore

@@ -15,6 +15,8 @@ tests/mock/client.properties
 /setup.py.orig
 /.project
 .tox
+.nox
+.mypy_cache
 *__pycache__*
 /.idea/
 /venv*/

addons/storage/README.md

@@ -23,9 +23,13 @@ enqueue_compss \
 --storage_home=$DATACLAY_HOME/storage/ \
 --storage_props=$DATACLAY_HOME/storage/storage_props_example \
 --classpath=$DATACLAY_HOME/storage/StorageItf-1.0.jar \
---pythonpath=$DATACLAY_PYTHONPATH \
+--pythonpath=$DATACLAY_PYTHONPATH:$PWD \
 --project_name="bsc19" \
 --qos=gp_debug \
+--num_nodes=4 \
+--exec_time=30 \
+--worker_working_dir=local_disk \
+--scheduler=es.bsc.compss.scheduler.orderstrict.fifo.FifoTS \
 client.py
 ```
 

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.9'
 services:
 
   redis:

docs/contributing.rst

@@ -1,7 +1,8 @@
 Contributing
 ============
 
-dataClay is a `BSC <https://www.bsc.es/research-and-development/software-and-apps/software-list/dataclay>`_
+The dataClay distributed data store is a 
+`BSC <https://www.bsc.es/research-and-development/software-and-apps/software-list/dataclay>`_
 project under the `BSD License <https://github.com/bsc-dom/dataclay/blob/main/LICENSE.txt>`_
 and we happily accept contributions.
 
@@ -16,7 +17,7 @@ If you wish to add a new feature or fix a bug:
 #. Write a test which shows that the bug was fixed or that the feature works
    as expected.
 #. Format your changes with `Black <https://black.readthedocs.io/en/stable/>`_ using the
-   command `tox -e format` and lint your changes using the command `tox -e lint`.
+   command `nox -s format` and lint your changes using the command `nox -s lint`.
 #. Send a pull request and follow up with the maintainer until it gets merged and published.
 
 .. #. Add a `changelog entry
@@ -25,14 +26,14 @@ If you wish to add a new feature or fix a bug:
 Setting up your development environment
 ---------------------------------------
 
-To set up your development environment, you will need `tox`_ installed on your machine:
+To set up your development environment, you will need `nox`_ installed on your machine:
 
 .. code-block:: console
 
-   $ python -m pip install --user --upgrade tox
+   $ python -m pip install --user --upgrade nox
 
 You wll also need to have `docker engine <https://docs.docker.com/engine/install/ubuntu/>`_ installed 
-for `tox`_ to use `pytest-docker <https://pypi.org/project/pytest-docker/>`_.
+for `nox`_ to use `pytest-docker <https://pypi.org/project/pytest-docker/>`_.
 
 Install dataClay in editable mode with the ``dev`` extra requirement:
 
@@ -44,11 +45,11 @@ Running the tests
 -----------------
 
 When running the test suite, we use external dependencies, multiple interpreters, and code coverage analysis. 
-Our `tox.ini <https://github.com/bsc-dom/dataclay/blob/main/tox.ini>`_ file handles much of this for you:
+Our `noxfile.py <https://github.com/bsc-dom/dataclay/blob/main/noxfile.py>`_ file handles much of this for you:
 
 .. code-block:: console
 
-   $ tox
+   $ nox
   
 
-.. _tox: https://tox.wiki/en/stable/
+.. _nox: https://nox.thea.codes/en/stable/

docs/index.rst

@@ -10,7 +10,7 @@ dataClay
    main-concepts
    alien-objects
    advanced-usage
-   telemetry
+   logging
    examples/index
 
 .. toctree::
@@ -24,12 +24,22 @@ dataClay
    deployment/hpc-manual-deployment
    deployment/compile-redis
 
+.. toctree::
+   :hidden:
+   :caption: Telemetry
+
+   telemetry/configuration
+   telemetry/offline
+   telemetry/real-time
+   telemetry/prometheus
+   telemetry/hpc-tracing
+
 .. toctree::
    :hidden:
    :caption: Release Notes
 
-   releasenotes/3-x
    releasenotes/4-x
+   releasenotes/3-x
 
 .. toctree::
    :hidden:

docs/logging.rst

@@ -0,0 +1,43 @@
+=======
+Logging
+=======
+
+DataClay offers logging to allow code debugging and information collection. 
+
+When dataclay is imported, the logging is first initialized with a basic configuration in the config.py file:
+
+.. code-block:: python
+    :caption: config.py
+
+    ...
+
+    class Settings(BaseSettings):
+
+    ...
+
+        loglevel: Annotated[str, StringConstraints(strip_whitespace=True, to_upper=True)] = "INFO"
+
+    ...
+
+    settings = Settings()
+
+    ...
+
+
+    def logger_config(**kwargs):
+        logging.basicConfig(**kwargs)
+
+
+    logger_config(level=settings.loglevel)
+
+
+
+If the user wants to configure its own logging, it can be done by importing the logging library and modifying 
+the basicConfig. When client.start() function is called, then logger_config() is executed again, and if the argument
+"force" is True then the logging configuration is overwritten.
+
+.. warning::
+    When modifying the basicConfig remember that the **force=True** parameter is mandatory. Otherwise, this new 
+    configuration will be obviated.
+
+An example is available in `GitHub <https://github.com/bsc-dom/dataclay/tree/main/examples/client-logger-config>`_

docs/telemetry/configuration.rst

@@ -0,0 +1,41 @@
+Telemetry Configuration
+=======================
+
+dataClay is instrumented with `OpenTelemetry <https://opentelemetry.io/>`_ to allow observability of
+distributed traces, metrics, and logs. You can configure tracing to export telemetry data either in real-time or for post-mortem analysis. Visualizations can be performed in Grafana.
+
+Configuration
+-------------
+
+To activate tracing in dataClay, the following environment variables need to be set:
+
+- **`DATACLAY_TRACING`**: Set to `true` to enable tracing.
+- **`DATACLAY_TRACING_EXPORTER`**: Export traces to the OpenTelemetry Collector (`otlp`) or print traces to the console (`console`). The default is `otlp`.
+- **`DATACLAY_TRACING_HOST`**: Host of the OpenTelemetry Collector (default: `localhost`).
+- **`DATACLAY_TRACING_PORT`**: Port of the OpenTelemetry Collector (default: `4317`).
+- **`DATACLAY_SERVICE_NAME`**: The service name, which identifies dataClay components in trace data.
+
+Metrics
+-------
+
+.. list-table::
+   :header-rows: 1
+
+   * - Metric
+     - Description
+     - Service
+   * - dataclay_inmemory_objects
+     - Number of objects in memory
+     - backend, client
+   * - dataclay_loaded_objects
+     - Number of loaded objects
+     - backend
+   * - dataclay_stored_objects
+     - Number of stored objects
+     - backend
+   * - dataclay_inmemory_misses_total
+     - Number of inmemory misses
+     - backend, client
+   * - dataclay_inmemory_hits_total
+     - Number of inmemory hits
+     - backend, client