feat: export job metadata along with values

.pre-commit-config.yaml

@@ -61,7 +61,7 @@ repos:
     exclude: '(tests/\*|ci/conda/kiara/meta.yaml)'
   - id: debug-statements
   - id: end-of-file-fixer
-    exclude: '.*.json'
+    exclude: '(.*.json|.*.j2)'
   - id: requirements-txt-fixer
   - id: fix-encoding-pragma
   - id: mixed-line-ending

CHANGELOG.md

@@ -10,26 +10,28 @@
     - `kiara data import`
     - `kiara data export`
   - new api endpoints:
-    - `register_archive`
-    - `set_archive_metadata_value`
     - `retrieve_archive_info`
     - `export_archive`
     - `import_archive`
-    - `copy_archive`
     - `export_values`
     - `import_values`
+- allow a 'comment' to be associated with a job:
+  - require a 'comment' for every `run_job`/`queue_job` call
+  - new api endpoints:
     - `list_all_job_record_ids`
     - `list_job_record_ids`
     - `list_all_job_records`
     - `list_job_records`
     - `get_job_record`
     - `get_job_comment`
+    - `set_job_comment`
 - add convenience api endpoint `get_values`
 - improved input options for 'store_values' API endpoint
-- 'beta' implementation of 'value_create' property on 'Value' instances
-- fix: plugin info for plugins with '-'
-- add '--runtime-info' cli flag
-- require a 'comment' for every `run_job`/`queue_job` call
+- added 'value_created' property on 'Value' instances
+- add '--runtime-info' cli flag to display runtime folders/files used by *kiara*
+- fix: plugin info for plugins with '-' in name
+- moved `KiaraAPI` class to `kiara.interfaces.python_api.kiara_api` module (the 'offical' import path `kiara.api.KiaraAPI` is still available, and should be used)
+- have `KiaraAPI` proxy a `BaseAPI` class, to make it easier to extend the API and keep it stable
 
 ## Version 0.5.9
 

Makefile

@@ -75,6 +75,9 @@ coverage: ## check code coverage quickly with the default Python
 
 check: black flake mypy test ## run dev-related checks
 
+render-api:
+	kiara render --source-type base_api --target-type kiara_api item kiara_api template_file=src/kiara/interfaces/python_api/kiara_api.py target_file=src/kiara/interfaces/python_api/kiara_api.py
+
 pre-commit: ## run pre-commit on all files
 	pre-commit run --all-files
 

dev/script.py

@@ -2,7 +2,6 @@
 # %%
 
 
-from kiara.interfaces.python_api import Step
 from kiara.utils.cli import terminal_print
 
 step_read_files_in_folder = Step(

pyproject.toml

@@ -400,7 +400,8 @@ exclude = [
     "dist",
     "node_modules",
     "venv",
-    "examples/"
+    "examples/",
+    "dev/"
 ]
 
 # Assume Python 3.10.

src/kiara/api.py

@@ -13,11 +13,13 @@
     "ValueMap",
     "ValueMapSchema",
     "ValueSchema",
+    "KiArchive",
 ]
 
 from .context import Kiara
 from .context.config import KiaraConfig
-from .interfaces.python_api import KiaraAPI
+from .interfaces.python_api.kiara_api import KiaraAPI
+from .interfaces.python_api.models.archive import KiArchive
 from .interfaces.python_api.models.job import JobDesc, RunSpec
 from .models.module.pipeline.pipeline import Pipeline, PipelineStructure
 from .models.values.value import Value, ValueMap

src/kiara/context/__init__.py

@@ -104,9 +104,9 @@ class Kiara(object):
     @classmethod
     def instance(cls) -> "Kiara":
         """The default *kiara* context. In most cases, it's recommended you create and manage your own, though."""
-        from kiara.interfaces.python_api import KiaraAPI
 
-        return KiaraAPI.instance().context
+        raise NotImplementedError("Kiara.instance() is not implemented yet.")
+        # return BaseAPI.instance().context
 
     def __init__(
         self,
@@ -131,7 +131,7 @@ def __init__(
         self._config: KiaraContextConfig = config
         self._runtime_config: KiaraRuntimeConfig = runtime_config
 
-        self._env_mgmt: EnvironmentRegistry = EnvironmentRegistry.instance()
+        self._env_mgmt: EnvironmentRegistry = EnvironmentRegistry()
 
         self._event_registry: EventRegistry = EventRegistry(kiara=self)
         self._type_registry: TypeRegistry = TypeRegistry(self)
@@ -179,7 +179,9 @@ def __init__(
                 file_name = f"{archive_path.name}.kiarchive"
 
                 js_config = SqliteArchiveConfig.create_new_store_config(
-                    store_base_path=archive.config["archive_path"], file_name=file_name
+                    store_base_path=archive.config["archive_path"],
+                    file_name=file_name,
+                    use_wal_mode=True,
                 )
                 archive = KiaraArchiveConfig(
                     archive_type="sqlite_job_store", config=js_config.model_dump()
@@ -264,6 +266,7 @@ def context_info(self) -> "KiaraContextInfo":
 
     @property
     def environment_registry(self) -> EnvironmentRegistry:
+
         return self._env_mgmt
 
     @property
@@ -486,7 +489,9 @@ def save_values(
             value = _values[field_name]
             try:
                 if field_aliases:
-                    self.alias_registry.register_aliases(value.value_id, *field_aliases)
+                    self.alias_registry.register_aliases(
+                        value_id=value.value_id, aliases=field_aliases
+                    )
 
                 stored[field_name] = StoreValueResult(
                     value=value,

src/kiara/context/config.py

@@ -40,15 +40,13 @@
     kiara_app_dirs,
 )
 from kiara.exceptions import KiaraException
-from kiara.registries.environment import EnvironmentRegistry
 from kiara.registries.ids import ID_REGISTRY
 from kiara.utils import log_message
 from kiara.utils.files import get_data_from_file
 
 if TYPE_CHECKING:
     from kiara.context import Kiara
     from kiara.models.context import ContextInfo
-    from kiara.models.runtime_environment.kiara import KiaraTypesRuntimeEnvironment
     from kiara.registries import BaseArchive, KiaraArchive
 
 logger = structlog.getLogger()
@@ -358,12 +356,15 @@ class KiaraSettings(BaseSettings):
 
 
 def create_default_store_config(
-    store_type: str, stores_base_path: str
+    store_type: str, stores_base_path: str, use_wal_mode: bool = False
 ) -> KiaraArchiveConfig:
 
-    env_registry = EnvironmentRegistry.instance()
-    kiara_types: "KiaraTypesRuntimeEnvironment" = env_registry.environments["kiara_types"]  # type: ignore
-    available_archives = kiara_types.archive_types
+    from kiara.utils.archives import find_archive_types
+
+    # env_registry = EnvironmentRegistry.instance()
+    # find_archive_types = find_archive_types()
+    # kiara_types: "KiaraTypesRuntimeEnvironment" = env_registry.environments["kiara_types"]  # type: ignore
+    available_archives = find_archive_types()
 
     assert store_type in available_archives.item_infos.keys()
 
@@ -378,7 +379,9 @@ def create_default_store_config(
         store_type=cls.__name__,
     )
 
-    config = cls._config_cls.create_new_store_config(store_base_path=stores_base_path)
+    config = cls._config_cls.create_new_store_config(
+        store_base_path=stores_base_path, use_wal_mode=use_wal_mode
+    )
 
     # store_id: uuid.UUID = config.get_archive_id()
 
@@ -389,7 +392,7 @@ def create_default_store_config(
     return data_store
 
 
-DEFAULT_STORE_TYPE: Literal["auto"] = "auto"
+DEFAULT_STORE_TYPE: Literal["sqlite"] = "sqlite"
 
 
 class KiaraConfig(BaseSettings):
@@ -458,10 +461,10 @@ def load_from_file(cls, path: Union[Path, str, None] = None) -> "KiaraConfig":
         description="The name of the default context to use if none is provided.",
         default=DEFAULT_CONTEXT_NAME,
     )
-    default_store_type: Literal["auto", "sqlite", "filesystem"] = Field(
-        description="The default store type to use when creating new stores.",
-        default=DEFAULT_STORE_TYPE,
-    )
+    # default_store_type: Literal["sqlite", "filesystem"] = Field(
+    #     description="The default store type to use when creating new stores.",
+    #     default=DEFAULT_STORE_TYPE,
+    # )
     auto_generate_contexts: bool = Field(
         description="Whether to auto-generate requested contexts if they don't exist yet.",
         default=True,
@@ -595,7 +598,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool:
         sqlite_base_path = os.path.join(self.stores_base_path, "sqlite_stores")
         filesystem_base_path = os.path.join(self.stores_base_path, "filesystem_stores")
 
-        def create_default_sqlite_archive_config() -> Dict[str, Any]:
+        def create_default_sqlite_archive_config(use_wal_mode: bool) -> Dict[str, Any]:
 
             store_id = str(uuid.uuid4())
             file_name = f"{store_id}.karchive"
@@ -628,11 +631,17 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]:
             conn.commit()
             conn.close()
 
-            return {"sqlite_db_path": archive_path.as_posix()}
+            return {
+                "sqlite_db_path": archive_path.as_posix(),
+                "use_wal_mode": use_wal_mode,
+            }
 
         default_sqlite_config: Union[Dict[str, Any], None] = None
 
-        if self.default_store_type == "auto":
+        use_wal_mode: bool = True
+        default_store_type = "sqlite"
+
+        if default_store_type == "auto":
 
             # if windows, we want sqlite as default, because although it's slower, it does not
             # need the user to enable developer mode
@@ -645,30 +654,34 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]:
             alias_store_type = "sqlite"
             job_store_type = "sqlite"
             workflow_store_type = "sqlite"
-        elif self.default_store_type == "filesystem":
+        elif default_store_type == "filesystem":
             metadata_store_type = "filesystem"
             data_store_type = "filesystem"
             alias_store_type = "filesystem"
             job_store_type = "filesystem"
             workflow_store_type = "filesystem"
-        elif self.default_store_type == "sqlite":
+        elif default_store_type == "sqlite":
             metadata_store_type = "sqlite"
             data_store_type = "sqlite"
             alias_store_type = "sqlite"
             job_store_type = "sqlite"
             workflow_store_type = "sqlite"
         else:
-            raise Exception(f"Unknown store type: {self.default_store_type}")
+            raise Exception(f"Unknown store type: {default_store_type}")
 
         if DEFAULT_METADATA_STORE_MARKER not in context_config.archives.keys():
 
             if metadata_store_type == "sqlite":
-                default_sqlite_config = create_default_sqlite_archive_config()
+                default_sqlite_config = create_default_sqlite_archive_config(
+                    use_wal_mode=use_wal_mode
+                )
                 metaddata_store = KiaraArchiveConfig(
                     archive_type="sqlite_metadata_store", config=default_sqlite_config
                 )
             elif metadata_store_type == "filesystem":
-                default_sqlite_config = create_default_sqlite_archive_config()
+                default_sqlite_config = create_default_sqlite_archive_config(
+                    use_wal_mode=use_wal_mode
+                )
                 metaddata_store = KiaraArchiveConfig(
                     archive_type="sqlite_metadata_store", config=default_sqlite_config
                 )
@@ -684,7 +697,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]:
 
             if data_store_type == "sqlite":
                 if default_sqlite_config is None:
-                    default_sqlite_config = create_default_sqlite_archive_config()
+                    default_sqlite_config = create_default_sqlite_archive_config(
+                        use_wal_mode=use_wal_mode
+                    )
 
                 data_store = KiaraArchiveConfig(
                     archive_type="sqlite_data_store", config=default_sqlite_config
@@ -708,7 +723,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]:
             if job_store_type == "sqlite":
 
                 if default_sqlite_config is None:
-                    default_sqlite_config = create_default_sqlite_archive_config()
+                    default_sqlite_config = create_default_sqlite_archive_config(
+                        use_wal_mode=use_wal_mode
+                    )
 
                 job_store = KiaraArchiveConfig(
                     archive_type="sqlite_job_store", config=default_sqlite_config
@@ -732,7 +749,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]:
             if alias_store_type == "sqlite":
 
                 if default_sqlite_config is None:
-                    default_sqlite_config = create_default_sqlite_archive_config()
+                    default_sqlite_config = create_default_sqlite_archive_config(
+                        use_wal_mode=use_wal_mode
+                    )
 
                 alias_store = KiaraArchiveConfig(
                     archive_type="sqlite_alias_store", config=default_sqlite_config
@@ -902,9 +921,6 @@ def save(self, path: Union[Path, None] = None):
             }
         )
 
-        if data["default_store_type"] == DEFAULT_STORE_TYPE:
-            data.pop("default_store_type")
-
         with path.open("wt") as f:
             yaml.dump(
                 data,
@@ -916,6 +932,7 @@ def save(self, path: Union[Path, None] = None):
     def delete(
         self, context_name: Union[str, None] = None, dry_run: bool = True
     ) -> Union["ContextInfo", None]:
+        """Deletes the context with the specified name."""
 
         if context_name is None:
             context_name = self.default_context

src/kiara/defaults.py

@@ -9,6 +9,7 @@
 import typing
 import uuid
 from enum import Enum
+from pathlib import Path
 
 from appdirs import AppDirs
 
@@ -139,6 +140,10 @@
 VALUE_ATTR_DELIMITER = "::"
 VALID_VALUE_QUERY_CATEGORIES = ["data", "properties"]
 
+CHUNK_CACHE_BASE_DIR = Path(kiara_app_dirs.user_cache_dir) / "data" / "chunks"
+CHUNK_CACHE_DIR_DEPTH = 2
+CHUNK_CACHE_DIR_WIDTH = 1
+
 
 class SpecialValue(Enum):
 
@@ -266,6 +271,9 @@ class SpecialValue(Enum):
 KIARA_MODEL_DATA_KEY = "data"
 KIARA_MODEL_SCHEMA_KEY = "schema"
 
+ENVIRONMENT_MARKER_KEY = "environment"
+"""Constant string to indicate this is a metadata entry of type 'environment'."""
+
 SYMLINK_ISSUE_MSG = """Your operating system does not support symlinks, which is a requirement for kiara to work.
 
 You can enable developer mode to fix this issue: