Merge branch 'main' into proxy-improvements

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 100
+extend-ignore = E203,E701
+exclude = */proto/*_pb2*.py

.github/workflows/docker-publish.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
 
     permissions:
       contents: read
@@ -30,39 +30,41 @@ jobs:
       id-token: write
 
     steps:
+      # The following steps are mostly inspired by the example found in 
+      # the README.md at https://github.com/docker/build-push-action
+
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-      # Workaround: https://github.com/docker/build-push-action/issues/461
       - name: Setup Docker buildx
-        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+        uses: docker/setup-buildx-action@v3
 
       # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
       - name: Extract Docker metadata
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=edge,suffix=-py${{ matrix.python-version }},branch=main
             type=edge,enable=${{ matrix.python-version == '3.10' }},branch=main
 
       # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
       - name: Build and push Docker image
         id: build-and-push
-        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64

.github/workflows/tests.yml

@@ -5,24 +5,25 @@ on:
   - pull_request
 
 jobs:
-  build:
+  generate-jobs:
+    runs-on: ubuntu-latest
+    outputs:
+      session: ${{ steps.set-matrix.outputs.session }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: wntrblm/nox@main
+      - id: set-matrix
+        shell: bash
+        run: echo session=$(nox --json -l -s tests | jq -c '[.[].session]') | tee --append $GITHUB_OUTPUT
+  checks:
+    name: Session ${{ matrix.session }}
+    needs: [generate-jobs]
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
-
+        session: ${{ fromJson(needs.generate-jobs.outputs.session) }}
     steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install tox tox-gh-actions
-          # TODO: when calling pytest, it build a new dev image, but it should build the image
-          # using the specific python version. Before it was working cause the compose was not
-          # building the image, rather the image was built here with the {{ matrix.python-version}}
-      - name: Test with tox
-        run: tox
+      - uses: actions/checkout@v4
+      - uses: wntrblm/nox@main
+      - run: nox -s "${{ matrix.session }}"

.gitignore

@@ -15,8 +15,12 @@ tests/mock/client.properties
 /setup.py.orig
 /.project
 .tox
+.nox
+.mypy_cache
 *__pycache__*
 /.idea/
 /venv*/
 /pyclay.iml
 /.vscode
+.coverage*
+coverage.xml

ACKNOWLEDGMENTS.md

@@ -0,0 +1,14 @@
+The development of this software has been supported by the following institutions, projects and grants:
+
+  - Spanish Government under contracts PID2019-107255 GB and TIN2015-65316-P
+  - Generalitat de Catalunya under contract 2017-SGR-01414
+  - European Commission's through the following:
+    - H2020 BigStorage European Training Network (Contract 642963)
+    - H2020 NextGenIO (Contract 671951)
+    - H2020 EXPERTISE European Training Network (Contract 721865)
+    - H2020 mF2C Project (Contract 730929)
+    - H2020 CLASS Project (Contract 780622)
+    - H2020 ELASTIC Project (Contract 825473)
+    - H2020 EuroHPC Joint Undertaking ADMIRE Project (Contract 956748)
+    - H2020 EuroHPC Joint Undertaking eFlows4HPC Project (Contract 955558)
+    - Horizon Europe ICOS project (Contract 101070177)

CHANGES.rst

@@ -1,3 +1,8 @@
+Version 4.0.0
+-------------
+
+Released 2024-06-21
+
 Version 3.1.0
 -------------
 

Dockerfile

@@ -9,8 +9,7 @@ RUN python -m pip install --upgrade pip \
   && python -m pip install /app[telemetry]
 
 # prepare dataclay storage dir
-RUN mkdir -p /data/storage; \
-  mkdir -p /data/metadata
+RUN mkdir -p /data/storage; 
 
 # set workdir and entrypoint
 WORKDIR /workdir

Dockerfile.dev

@@ -6,11 +6,10 @@ ARG PYTHON_VERSION=3.10-bullseye
 FROM python:$PYTHON_VERSION
 COPY . /app
 RUN python -m pip install --upgrade pip \
-  && python -m pip install -e /app[telemetry]
+  && python -m pip install -e /app[telemetry,dev]
 
 # prepare dataclay storage dir
-RUN mkdir -p /data/storage; \
-  mkdir -p /data/metadata
+RUN mkdir -p /data/storage; 
 
 # set workdir and entrypoint
 WORKDIR /workdir

PUBLISH.md

@@ -10,31 +10,39 @@
    - Add `Released YYYY-MM-DD` to `CHANGES.rst`
 
 3. Create and push a new tag:
+
    ```bash
    git tag -a <VERSION> -m "Release <VERSION>"
    git push origin <VERSION>
    ```
-4. Follow the instructions to create a [new release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) in GitHub.
 
+4. Follow the instructions to create a [new release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) in GitHub.
 
 5. Publish the `dataclay:latest` docker image:
     1. First, set up your personal access token and log in to GitHub Packages (ghcr.io):
+
         ```bash
         export CR_PAT=<YOUR_TOKEN>
         echo $CR_PAT | docker login ghcr.io -u USERNAME --password-stdin
 
         # Or just log if the token is already stored
         docker login ghcr.io
         ```
+
     2. Build and publish the docker image. To use `buildx` you may need to `sudo apt install qemu-user-static`:
+
         ```bash
         # Set the version variable
         VERSION=<VERSION>
 
+        # Create a new builder instance
+        export DOCKER_BUILDKIT=1
+        docker buildx create --use
+
         # Build and push Python 3.10 bullseye
         docker buildx build --platform linux/amd64,linux/arm64 \
         -t ghcr.io/bsc-dom/dataclay:$VERSION-py3.10-bullseye \
-        -t ghcr.io/bsc-dom/dataclay:$VERSION \
+        -t ghcr.io/bsc-dom/dataclay:$`VERSION` \
         -t ghcr.io/bsc-dom/dataclay:latest \
         --build-arg PYTHON_VERSION=3.10-bullseye --push .
         ```
@@ -69,7 +77,6 @@
 
 8. Update the _active versions_ on ReadTheDocs, i.e. go to the [versions page](https://readthedocs.org/projects/dataclay/versions/) and activate/deactivate versions accordingly. You probably must add the newly added release, and maybe you will need to deactivate patch versions that are irrelevant.
 
-
 ## Publish a development distribution to TestPyPI
 
 To build and publish a development distribution:
@@ -98,7 +105,6 @@ python3 -m pip install --index-url https://test.pypi.org/simple/ --extra-index-u
 - `--index-url` tells pip to download the package form TestPyPI instead of PyPI
 - `--extra-index-url` is used to install the package dependencies from PyPI
 
-
 ## Build the dataclay dev image for testing
 
 ```bash

README.md

@@ -46,7 +46,12 @@ We welcome contributions to dataClay. Please see the [Contributing](https://data
 - [@alexbarcelo](https://github.com/alexbarcelo) (Alex Barcelo)
 - [@aqueralt](https://github.com/aqueralt) (Anna Queralt)
 - [@marcmonfort](https://github.com/marcmonfort) (Marc Monfort)
+- [@Canicula98](https://github.com/Canicula98) (David Cano)
 
 ## Other resources
 
 [BSC official website](https://www.bsc.es/dataclay)
+
+## Acknolwedgments
+
+See the [full list of acknowledgments](ACKNOWLEDGMENTS.md)

addons/storage/README.md

@@ -12,7 +12,7 @@ java -cp target/StorageItf-1.0-SNAPSHOT.jar storage.StorageItf
 First load dataclay
 
 ```bash
-module load DATACLAY
+module load dataclay
 module load COMPSs
 ```
 
@@ -23,8 +23,13 @@ enqueue_compss \
 --storage_home=$DATACLAY_HOME/storage/ \
 --storage_props=$DATACLAY_HOME/storage/storage_props_example \
 --classpath=$DATACLAY_HOME/storage/StorageItf-1.0.jar \
---pythonpath=$DATACLAY_PYTHONPATH \
---qos=debug \
+--pythonpath=$DATACLAY_PYTHONPATH:$PWD \
+--project_name="bsc19" \
+--qos=gp_debug \
+--num_nodes=4 \
+--exec_time=30 \
+--worker_working_dir=local_disk \
+--scheduler=es.bsc.compss.scheduler.orderstrict.fifo.FifoTS \
 client.py
 ```
 

addons/storage/dependency-reduced-pom.xml

@@ -113,10 +113,10 @@
     </dependency>
   </dependencies>
   <properties>
-    <maven.compiler.target>1.8</maven.compiler.target>
-    <maven.compiler.source>1.8</maven.compiler.source>
     <protoc.version>3.22.3</protoc.version>
-    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <grpc.version>1.56.0</grpc.version>
+    <maven.compiler.source>1.8</maven.compiler.source>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <maven.compiler.target>1.8</maven.compiler.target>
   </properties>
 </project>

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3.9'
 services:
 
   redis:

docs/README.md

@@ -0,0 +1,23 @@
+# dataClay Documentation
+
+The dataClay documentation is generated using [Sphinx](https://www.sphinx-doc.org/en/master/). The documentation is written in reStructuredText format and is located in the `dataclay/docs` directory.
+
+## Manual Deployment
+
+In order to generate the documentation, you need to install the following dependencies (note that the command must be run from the root of the repository):
+
+```bash
+pip install -e .[docs]
+```
+
+Then, you can generate the documentation by running (from the `dataclay/docs` directory):
+
+```bash
+make html
+```
+
+The documentation will be generated in the `dataclay/docs/_build/html` directory. You can open the `index.html` file in your browser to view the documentation.
+
+## Automatic Deployment
+
+The documentation is automatically deployed using the `.readthedocs.yml` configuration file. The deployment is triggered whenever a new commit is pushed to the `main` branch. The deployed documentation is available at [https://dataclay.readthedocs.io/en/latest/](https://dataclay.readthedocs.io/en/latest/).

docs/advanced-usage.rst

@@ -112,6 +112,42 @@ It is also possible to create versions from other versions::
 .. example with compss @task
 
 
+
+Asynchronous Operations
+-----------------------
+
+The dataClay runtime is asynchronous by default. This means that all dataClay operations are 
+executed as tasks in an event loop which is running in a separate thread. 
+In the previous examples, we have seen the synchronous version of the dataClay operations,
+which block the current thread until the operation is completed. This may be sufficient for
+most use cases, but in some cases, it may be necessary to execute multiple operations concurrently
+to improve performance for I/O-bound applications.
+
+All DataClayObject methods have an asynchronous version that can be called using the ``await`` keyword::
+
+    async def main():
+        client = Client()
+        client.start()
+
+        # Print the available backends using the asynchronous version of the method
+        print(await client.a_get_backends())
+
+        employee_1 = Employee("John", 1000.0)
+        employee_2 = Employee("Jane", 2000.0)
+
+        # Use gather to execute multiple make_persistent concurrently
+        await asyncio.gather(
+            employee_1.a_make_persistent(),
+            employee_2.a_make_persistent()
+        )
+
+    if __name__ == "__main__":
+        asyncio.run(main())
+
+.. example for async attribute access
+.. example for async activemethod
+
+
 Multithreading
 --------------