Merge pull request #2 from interTwin-eu/cleanup-plugin

Fix issue with installation of plugin subpackages

Author: pagecp

.github/linters/.ruff.toml

@@ -0,0 +1,11 @@
+line-length = 95
+
+[lint]
+select = ["E", "F", "I"]
+ignore = ["E203"]
+
+[format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"

.github/workflows/lint.yml

@@ -10,6 +10,14 @@ jobs:
     name: Lint with Super-Linter
     runs-on: ubuntu-latest
 
+    ############################################
+    # Grant status permission for MULTI_STATUS #
+    ############################################
+    permissions:
+      contents: read
+      packages: read
+      statuses: write
+
     steps:
       # Checks out a copy of your repository on the ubuntu-latest machine
       - name: Checkout code
@@ -22,12 +30,20 @@ jobs:
 
       # Runs the Super-Linter action
       - name: Run Super-Linter on new changes
-        uses: docker://ghcr.io/github/super-linter:slim-v4
+        uses: github/super-linter/slim@v7
         env:
           DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           MARKDOWN_CONFIG_FILE: .markdownlint.json
+          PYTHON_RUFF_CONFIG_FILE: .ruff.toml
           # Only check new or edited files
           VALIDATE_ALL_CODEBASE: false
           # Fail on errors
           DISABLE_ERRORS: false
+          # Both options below should be already covered by ruff
+          VALIDATE_PYTHON_ISORT: false
+          VALIDATE_PYTHON_FLAKE8: false
+          VALIDATE_PYTHON_BLACK: false
+          VALIDATE_PYTHON_MYPY: false
+          VALIDATE_PYTHON_PYLINT: false
+          VALIDATE_PYTHON_PYINK: false

.gitignore

@@ -1,3 +1,5 @@
+uv.lock
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

.python-version

@@ -0,0 +1 @@
+3.11

.vscode/extensions.json

@@ -0,0 +1,7 @@
+{
+    "recommendations": [
+        "charliermarsh.ruff",
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+    ]
+}

.vscode/settings.json

@@ -1,14 +1,23 @@
 {
-    "markdownlint.config": {
-        "MD013": {
-            "line_length": 120,
-            "code_blocks": false,
-            "tables": false
-          },
-          "MD014": false,
-          "MD024": false,
-          "MD026": {
-            "punctuation": ".,:;!"
-          }
+  "markdownlint.config": {
+    "MD013": {
+      "line_length": 120,
+      "code_blocks": false,
+      "tables": false
+    },
+    "MD014": false,
+    "MD024": false,
+    "MD026": {
+      "punctuation": ".,:;!"
     }
+  },
+  "[python]": {
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.fixAll": "explicit"
+    }
+  },
+  "ruff.organizeImports": true,
+  "ruff.fixAll": true,
+  "editor.formatOnSave": true,
 }

src/itwinai/plugins/xtclim/preprocess_3d_seasons.ipynb renamed to notebooks/preprocess_3d_seasons.ipynb

@@ -12,17 +12,17 @@ pipeline:
    init_args:
      steps:
          preprocessing-step:
-           class_path: itwinai.plugins.xtclim.preprocess_functions_2d_ssp.PreprocessData
+           class_path: itwinai.plugins.xtclim.preprocessing.preprocess_functions_2d_ssp.PreprocessData
            init_args:
                dataset_root: ${dataset_root}
                scenario: ${scenario}
          preprocessing-split-step:
-           class_path: itwinai.plugins.xtclim.preprocess_2d_seasons.SplitPreprocessedData
+           class_path: itwinai.plugins.xtclim.preprocessing.preprocess_2d_seasons.SplitPreprocessedData
            init_args:
                scenario: ${scenario} 
          training-step:
              #class_path: src.trainer_dist.XTClimTrainer
-           class_path: itwinai.plugins.xtclim.trainer.TorchTrainer
+           class_path: itwinai.plugins.xtclim.src.trainer.TorchTrainer
            init_args:
                epochs: ${epochs}
                batch_size: ${batch_size}

src/itwinai/plugins/xtclim/preprocess_functions_1d_temp_members.ipynb renamed to notebooks/preprocess_functions_1d_temp_members.ipynb

@@ -5,6 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "itwinai-xtclim-plugin"
 version = "0.1.0"
+requires-python = ">=3.10"
 description = "xtclim plugin for itwinai"
 authors = [{ name = "Christian Page", email = "[email protected]" }]
 dependencies = [
@@ -26,12 +27,33 @@ dependencies = [
     "xarray",
     "netCDF4",
     "h5netcdf",
+    "ruff>=0.9.6",
 ]
 
 [tool.setuptools.packages.find]
 # Declare this package as part of the `itwinai.plugins` namespace
 where = ["src"]
-include = ["itwinai.plugins.xtclim"]
+# Add plugin subfolders explicitly here
+include = [
+    "itwinai.plugins.xtclim",
+    "itwinai.plugins.xtclim.src",
+    "itwinai.plugins.xtclim.preprocessing",
+]
 
 [tool.setuptools.package-data]
 "*" = ["*.json", "*.yaml", "*.ipynb"]
+
+# Ruff configuration: https://docs.astral.sh/ruff/configuration/
+[tool.ruff]
+line-length = 95
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W"]
+ignore = ["E203"]
+fixable = ["ALL"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+skip-magic-trailing-comma = false
+line-ending = "auto"

src/itwinai/plugins/xtclim/scenario_season_comparison.ipynb renamed to notebooks/scenario_season_comparison.ipynb

@@ -0,0 +1,4 @@
+[flake8]
+extend-ignore = E203,W503
+max-line-length = 95
+exclude = .git,__pycache__,docs/conf.py,use-cases,tutorials,tests,old,build,dist,.venv*,envAI*,env-files,.vscode

src/itwinai/plugins/xtclim/pipeline.yaml renamed to pipeline.yaml

@@ -10,21 +10,20 @@
 
 # #### 0. Libraries
 
+import configparser as cp
+
 import numpy as np
 import pandas as pd
-import configparser as cp
 
 from itwinai.components import DataGetter, monitor_exec
 
+
 class SplitPreprocessedData(DataGetter):
-    def __init__(
-            self,
-            scenario: str
-    ):
+    def __init__(self, scenario: str):
         super().__init__()
         self.scenario = scenario
 
-    ##### 2. Split Yearly Data into Four Seasonal Datasets
+    # #### 2. Split Yearly Data into Four Seasonal Datasets
     # split daily data into seasons
     def season_split(
         self,
@@ -103,17 +102,17 @@ def season_split(
 
     @monitor_exec
     def execute(self):
-        #### Configuration file
+        # ### Configuration file
         config = cp.ConfigParser()
-        config.read('xtclim.json')
+        config.read("xtclim.json")
 
-        ##### 1. Load Data to xarray
+        # #### 1. Load Data to xarray
 
         # choose the needed number of members
-        n_memb = config.get('TRAIN', 'n_memb')
+        n_memb = config.get("TRAIN", "n_memb")
 
         # define relevant scenarios
-        #scenarios = ["126", "245", "370", "585"]
+        # scenarios = ["126", "245", "370", "585"]
         scenarios = [self.scenario]
         # Load preprocessed "daily temperature images" and time series
 
@@ -122,8 +121,7 @@ def execute(self):
         train_time = pd.read_csv("input/dates_train_data.csv")
         test_time = pd.read_csv("input/dates_test_data.csv")
 
-
-        ##### 3. Apply to Train and Test Datasets
+        # #### 3. Apply to Train and Test Datasets
         train_season_images, train_season_time = self.season_split(
             train_images, train_time, "train", n_memb
         )
@@ -132,8 +130,7 @@ def execute(self):
             test_images, test_time, "test", n_memb
         )
 
-
-        ##### 4. Apply to Projection Datasets
+        # #### 4. Apply to Projection Datasets
 
         for scenario in scenarios:
             proj_images = np.load(f"input/preprocessed_2d_proj{scenario}_data_allssp.npy")

pyproject.toml

@@ -10,22 +10,24 @@
 
 # #### 0. Libraries
 
+import configparser as cp
+import json
+
 import numpy as np
 import pandas as pd
-import configparser as cp
 
-#### Configuration file
+# #### Configuration file
 config = cp.ConfigParser()
-config.read('xtclim.json')
+config.read("xtclim.json")
 
 # #### 1. Load Data to xarray
 
 # choose the needed number of members
-n_memb = config.getint('TRAIN', 'n_memb')
+n_memb = config.getint("TRAIN", "n_memb")
 
 # define relevant scenarios
-#scenarios = ["126", "245", "370", "585"]
-scenarios = json.loads(config.getint('GENERAL', 'scenarios'))
+# scenarios = ["126", "245", "370", "585"]
+scenarios = json.loads(config.getint("GENERAL", "scenarios"))
 
 # Load preprocessed "daily temperature images" and time series
 
@@ -122,9 +124,7 @@ def season_split(
 )
 
 
-test_season_images, test_season_time = season_split(
-    test_images, test_time, "test", n_memb
-)
+test_season_images, test_season_time = season_split(test_images, test_time, "test", n_memb)
 
 
 # #### 4. Apply to Projection Datasets