From 5dde2af4287ef4464cdb260f4d9c4bc149e4d25d Mon Sep 17 00:00:00 2001 From: Markus Binsteiner Date: Tue, 2 Apr 2024 10:33:06 +0200 Subject: [PATCH] feat: export job metadata along with values --- .pre-commit-config.yaml | 2 +- CHANGELOG.md | 16 +- Makefile | 3 + dev/script.py | 1 - pyproject.toml | 3 +- src/kiara/api.py | 4 +- src/kiara/context/__init__.py | 15 +- src/kiara/context/config.py | 69 +- src/kiara/defaults.py | 8 + src/kiara/interfaces/__init__.py | 67 +- src/kiara/interfaces/cli/archive/commands.py | 12 +- src/kiara/interfaces/cli/context/commands.py | 4 +- src/kiara/interfaces/cli/data/commands.py | 51 +- src/kiara/interfaces/cli/info/commands.py | 8 +- src/kiara/interfaces/cli/module/commands.py | 8 +- .../interfaces/cli/operation/commands.py | 10 +- src/kiara/interfaces/cli/pipeline/commands.py | 2 +- src/kiara/interfaces/cli/render/commands.py | 6 +- src/kiara/interfaces/cli/run.py | 4 +- src/kiara/interfaces/cli/type/commands.py | 5 +- src/kiara/interfaces/cli/workflow/commands.py | 11 +- src/kiara/interfaces/python_api/__init__.py | 3484 +--------------- src/kiara/interfaces/python_api/base_api.py | 3573 +++++++++++++++++ src/kiara/interfaces/python_api/kiara_api.py | 1168 ++++++ .../interfaces/python_api/models/archive.py | 109 +- src/kiara/interfaces/python_api/models/doc.py | 2 +- .../interfaces/python_api/models/info.py | 175 +- src/kiara/interfaces/python_api/models/job.py | 12 +- src/kiara/interfaces/python_api/proxy.py | 55 +- src/kiara/models/archives.py | 69 +- src/kiara/models/module/jobs.py | 17 +- src/kiara/models/module/operation.py | 2 +- src/kiara/models/module/pipeline/pipeline.py | 8 +- .../models/runtime_environment/__init__.py | 4 +- src/kiara/models/runtime_environment/kiara.py | 97 +- src/kiara/models/values/value.py | 8 +- .../models/values/value_metadata/__init__.py | 103 - src/kiara/processing/__init__.py | 4 +- src/kiara/registries/__init__.py | 12 +- src/kiara/registries/aliases/__init__.py | 37 +- src/kiara/registries/aliases/sqlite_store.py | 39 +- src/kiara/registries/data/__init__.py | 52 +- .../registries/data/data_store/__init__.py | 75 +- .../data/data_store/filesystem_store.py | 18 +- .../data/data_store/sqlite_store.py | 83 +- src/kiara/registries/environment/__init__.py | 27 +- src/kiara/registries/jobs/__init__.py | 92 +- .../registries/jobs/job_store/sqlite_store.py | 46 +- src/kiara/registries/metadata/__init__.py | 217 +- .../metadata/metadata_store/__init__.py | 155 +- .../metadata/metadata_store/sqlite_store.py | 381 +- .../included_renderers/api/__init__.py | 0 .../included_renderers/api/base_api.py | 335 ++ .../{api.py => api/kiara_api.py} | 33 +- src/kiara/renderers/included_renderers/job.py | 9 +- .../renderers/included_renderers/pipeline.py | 41 - .../render/kiara_api/kiara_api_endpoint.py.j2 | 11 + .../render/pipeline/python_script.py.j2 | 5 +- src/kiara/utils/archives.py | 36 + src/kiara/utils/cli/rich_click.py | 7 +- src/kiara/utils/cli/run.py | 13 +- src/kiara/utils/config.py | 63 + src/kiara/utils/db.py | 62 +- src/kiara/utils/introspection.py | 167 + src/kiara/utils/metadata.py | 61 +- src/kiara/utils/stores.py | 13 +- src/kiara/utils/testing/__init__.py | 2 +- src/kiara/utils/values.py | 4 +- src/kiara/zmq/service/__init__.py | 9 +- tests/conftest.py | 6 +- .../resources/archives/export_test.kiarchive | Bin 135168 -> 159744 bytes tests/test_api/test_data_types.py | 8 +- tests/test_api/test_misc.py | 10 +- tests/test_api/test_module_types.py | 6 +- tests/test_api/test_operations.py | 6 +- tests/test_archives/test_archive_export.py | 40 +- tests/test_archives/test_archive_import.py | 24 +- tests/test_included_data_types/test_string.py | 8 +- tests/test_pipelines/test_pipeline_configs.py | 10 +- tests/test_rendering.py | 4 +- 80 files changed, 7174 insertions(+), 4252 deletions(-) create mode 100644 src/kiara/interfaces/python_api/base_api.py create mode 100644 src/kiara/interfaces/python_api/kiara_api.py create mode 100644 src/kiara/renderers/included_renderers/api/__init__.py create mode 100644 src/kiara/renderers/included_renderers/api/base_api.py rename src/kiara/renderers/included_renderers/{api.py => api/kiara_api.py} (84%) create mode 100644 src/kiara/resources/templates/render/kiara_api/kiara_api_endpoint.py.j2 create mode 100644 src/kiara/utils/archives.py create mode 100644 src/kiara/utils/config.py create mode 100644 src/kiara/utils/introspection.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index df8ad4ec1..47e316c58 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -61,7 +61,7 @@ repos: exclude: '(tests/\*|ci/conda/kiara/meta.yaml)' - id: debug-statements - id: end-of-file-fixer - exclude: '.*.json' + exclude: '(.*.json|.*.j2)' - id: requirements-txt-fixer - id: fix-encoding-pragma - id: mixed-line-ending diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f50ea3fd..59c15b7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,26 +10,28 @@ - `kiara data import` - `kiara data export` - new api endpoints: - - `register_archive` - - `set_archive_metadata_value` - `retrieve_archive_info` - `export_archive` - `import_archive` - - `copy_archive` - `export_values` - `import_values` +- allow a 'comment' to be associated with a job: + - require a 'comment' for every `run_job`/`queue_job` call + - new api endpoints: - `list_all_job_record_ids` - `list_job_record_ids` - `list_all_job_records` - `list_job_records` - `get_job_record` - `get_job_comment` + - `set_job_comment` - add convenience api endpoint `get_values` - improved input options for 'store_values' API endpoint -- 'beta' implementation of 'value_create' property on 'Value' instances -- fix: plugin info for plugins with '-' -- add '--runtime-info' cli flag -- require a 'comment' for every `run_job`/`queue_job` call +- added 'value_created' property on 'Value' instances +- add '--runtime-info' cli flag to display runtime folders/files used by *kiara* +- fix: plugin info for plugins with '-' in name +- moved `KiaraAPI` class to `kiara.interfaces.python_api.kiara_api` module (the 'offical' import path `kiara.api.KiaraAPI` is still available, and should be used) +- have `KiaraAPI` proxy a `BaseAPI` class, to make it easier to extend the API and keep it stable ## Version 0.5.9 diff --git a/Makefile b/Makefile index e909ab82f..3e51b2b30 100644 --- a/Makefile +++ b/Makefile @@ -75,6 +75,9 @@ coverage: ## check code coverage quickly with the default Python check: black flake mypy test ## run dev-related checks +render-api: + kiara render --source-type base_api --target-type kiara_api item kiara_api template_file=src/kiara/interfaces/python_api/kiara_api.py target_file=src/kiara/interfaces/python_api/kiara_api.py + pre-commit: ## run pre-commit on all files pre-commit run --all-files diff --git a/dev/script.py b/dev/script.py index 7dae04175..b6ccd24ad 100644 --- a/dev/script.py +++ b/dev/script.py @@ -2,7 +2,6 @@ # %% -from kiara.interfaces.python_api import Step from kiara.utils.cli import terminal_print step_read_files_in_folder = Step( diff --git a/pyproject.toml b/pyproject.toml index a6d883311..5d6478173 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -400,7 +400,8 @@ exclude = [ "dist", "node_modules", "venv", - "examples/" + "examples/", + "dev/" ] # Assume Python 3.10. diff --git a/src/kiara/api.py b/src/kiara/api.py index 561cf6b3f..2e74e7479 100644 --- a/src/kiara/api.py +++ b/src/kiara/api.py @@ -13,11 +13,13 @@ "ValueMap", "ValueMapSchema", "ValueSchema", + "KiArchive", ] from .context import Kiara from .context.config import KiaraConfig -from .interfaces.python_api import KiaraAPI +from .interfaces.python_api.kiara_api import KiaraAPI +from .interfaces.python_api.models.archive import KiArchive from .interfaces.python_api.models.job import JobDesc, RunSpec from .models.module.pipeline.pipeline import Pipeline, PipelineStructure from .models.values.value import Value, ValueMap diff --git a/src/kiara/context/__init__.py b/src/kiara/context/__init__.py index 53eaa3038..8c3ec3e9c 100644 --- a/src/kiara/context/__init__.py +++ b/src/kiara/context/__init__.py @@ -104,9 +104,9 @@ class Kiara(object): @classmethod def instance(cls) -> "Kiara": """The default *kiara* context. In most cases, it's recommended you create and manage your own, though.""" - from kiara.interfaces.python_api import KiaraAPI - return KiaraAPI.instance().context + raise NotImplementedError("Kiara.instance() is not implemented yet.") + # return BaseAPI.instance().context def __init__( self, @@ -131,7 +131,7 @@ def __init__( self._config: KiaraContextConfig = config self._runtime_config: KiaraRuntimeConfig = runtime_config - self._env_mgmt: EnvironmentRegistry = EnvironmentRegistry.instance() + self._env_mgmt: EnvironmentRegistry = EnvironmentRegistry() self._event_registry: EventRegistry = EventRegistry(kiara=self) self._type_registry: TypeRegistry = TypeRegistry(self) @@ -179,7 +179,9 @@ def __init__( file_name = f"{archive_path.name}.kiarchive" js_config = SqliteArchiveConfig.create_new_store_config( - store_base_path=archive.config["archive_path"], file_name=file_name + store_base_path=archive.config["archive_path"], + file_name=file_name, + use_wal_mode=True, ) archive = KiaraArchiveConfig( archive_type="sqlite_job_store", config=js_config.model_dump() @@ -264,6 +266,7 @@ def context_info(self) -> "KiaraContextInfo": @property def environment_registry(self) -> EnvironmentRegistry: + return self._env_mgmt @property @@ -486,7 +489,9 @@ def save_values( value = _values[field_name] try: if field_aliases: - self.alias_registry.register_aliases(value.value_id, *field_aliases) + self.alias_registry.register_aliases( + value_id=value.value_id, aliases=field_aliases + ) stored[field_name] = StoreValueResult( value=value, diff --git a/src/kiara/context/config.py b/src/kiara/context/config.py index 97e2618cc..9bfd9206e 100644 --- a/src/kiara/context/config.py +++ b/src/kiara/context/config.py @@ -40,7 +40,6 @@ kiara_app_dirs, ) from kiara.exceptions import KiaraException -from kiara.registries.environment import EnvironmentRegistry from kiara.registries.ids import ID_REGISTRY from kiara.utils import log_message from kiara.utils.files import get_data_from_file @@ -48,7 +47,6 @@ if TYPE_CHECKING: from kiara.context import Kiara from kiara.models.context import ContextInfo - from kiara.models.runtime_environment.kiara import KiaraTypesRuntimeEnvironment from kiara.registries import BaseArchive, KiaraArchive logger = structlog.getLogger() @@ -358,12 +356,15 @@ class KiaraSettings(BaseSettings): def create_default_store_config( - store_type: str, stores_base_path: str + store_type: str, stores_base_path: str, use_wal_mode: bool = False ) -> KiaraArchiveConfig: - env_registry = EnvironmentRegistry.instance() - kiara_types: "KiaraTypesRuntimeEnvironment" = env_registry.environments["kiara_types"] # type: ignore - available_archives = kiara_types.archive_types + from kiara.utils.archives import find_archive_types + + # env_registry = EnvironmentRegistry.instance() + # find_archive_types = find_archive_types() + # kiara_types: "KiaraTypesRuntimeEnvironment" = env_registry.environments["kiara_types"] # type: ignore + available_archives = find_archive_types() assert store_type in available_archives.item_infos.keys() @@ -378,7 +379,9 @@ def create_default_store_config( store_type=cls.__name__, ) - config = cls._config_cls.create_new_store_config(store_base_path=stores_base_path) + config = cls._config_cls.create_new_store_config( + store_base_path=stores_base_path, use_wal_mode=use_wal_mode + ) # store_id: uuid.UUID = config.get_archive_id() @@ -389,7 +392,7 @@ def create_default_store_config( return data_store -DEFAULT_STORE_TYPE: Literal["auto"] = "auto" +DEFAULT_STORE_TYPE: Literal["sqlite"] = "sqlite" class KiaraConfig(BaseSettings): @@ -458,10 +461,10 @@ def load_from_file(cls, path: Union[Path, str, None] = None) -> "KiaraConfig": description="The name of the default context to use if none is provided.", default=DEFAULT_CONTEXT_NAME, ) - default_store_type: Literal["auto", "sqlite", "filesystem"] = Field( - description="The default store type to use when creating new stores.", - default=DEFAULT_STORE_TYPE, - ) + # default_store_type: Literal["sqlite", "filesystem"] = Field( + # description="The default store type to use when creating new stores.", + # default=DEFAULT_STORE_TYPE, + # ) auto_generate_contexts: bool = Field( description="Whether to auto-generate requested contexts if they don't exist yet.", default=True, @@ -595,7 +598,7 @@ def _validate_context(self, context_config: KiaraContextConfig) -> bool: sqlite_base_path = os.path.join(self.stores_base_path, "sqlite_stores") filesystem_base_path = os.path.join(self.stores_base_path, "filesystem_stores") - def create_default_sqlite_archive_config() -> Dict[str, Any]: + def create_default_sqlite_archive_config(use_wal_mode: bool) -> Dict[str, Any]: store_id = str(uuid.uuid4()) file_name = f"{store_id}.karchive" @@ -628,11 +631,17 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]: conn.commit() conn.close() - return {"sqlite_db_path": archive_path.as_posix()} + return { + "sqlite_db_path": archive_path.as_posix(), + "use_wal_mode": use_wal_mode, + } default_sqlite_config: Union[Dict[str, Any], None] = None - if self.default_store_type == "auto": + use_wal_mode: bool = True + default_store_type = "sqlite" + + if default_store_type == "auto": # if windows, we want sqlite as default, because although it's slower, it does not # need the user to enable developer mode @@ -645,30 +654,34 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]: alias_store_type = "sqlite" job_store_type = "sqlite" workflow_store_type = "sqlite" - elif self.default_store_type == "filesystem": + elif default_store_type == "filesystem": metadata_store_type = "filesystem" data_store_type = "filesystem" alias_store_type = "filesystem" job_store_type = "filesystem" workflow_store_type = "filesystem" - elif self.default_store_type == "sqlite": + elif default_store_type == "sqlite": metadata_store_type = "sqlite" data_store_type = "sqlite" alias_store_type = "sqlite" job_store_type = "sqlite" workflow_store_type = "sqlite" else: - raise Exception(f"Unknown store type: {self.default_store_type}") + raise Exception(f"Unknown store type: {default_store_type}") if DEFAULT_METADATA_STORE_MARKER not in context_config.archives.keys(): if metadata_store_type == "sqlite": - default_sqlite_config = create_default_sqlite_archive_config() + default_sqlite_config = create_default_sqlite_archive_config( + use_wal_mode=use_wal_mode + ) metaddata_store = KiaraArchiveConfig( archive_type="sqlite_metadata_store", config=default_sqlite_config ) elif metadata_store_type == "filesystem": - default_sqlite_config = create_default_sqlite_archive_config() + default_sqlite_config = create_default_sqlite_archive_config( + use_wal_mode=use_wal_mode + ) metaddata_store = KiaraArchiveConfig( archive_type="sqlite_metadata_store", config=default_sqlite_config ) @@ -684,7 +697,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]: if data_store_type == "sqlite": if default_sqlite_config is None: - default_sqlite_config = create_default_sqlite_archive_config() + default_sqlite_config = create_default_sqlite_archive_config( + use_wal_mode=use_wal_mode + ) data_store = KiaraArchiveConfig( archive_type="sqlite_data_store", config=default_sqlite_config @@ -708,7 +723,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]: if job_store_type == "sqlite": if default_sqlite_config is None: - default_sqlite_config = create_default_sqlite_archive_config() + default_sqlite_config = create_default_sqlite_archive_config( + use_wal_mode=use_wal_mode + ) job_store = KiaraArchiveConfig( archive_type="sqlite_job_store", config=default_sqlite_config @@ -732,7 +749,9 @@ def create_default_sqlite_archive_config() -> Dict[str, Any]: if alias_store_type == "sqlite": if default_sqlite_config is None: - default_sqlite_config = create_default_sqlite_archive_config() + default_sqlite_config = create_default_sqlite_archive_config( + use_wal_mode=use_wal_mode + ) alias_store = KiaraArchiveConfig( archive_type="sqlite_alias_store", config=default_sqlite_config @@ -902,9 +921,6 @@ def save(self, path: Union[Path, None] = None): } ) - if data["default_store_type"] == DEFAULT_STORE_TYPE: - data.pop("default_store_type") - with path.open("wt") as f: yaml.dump( data, @@ -916,6 +932,7 @@ def save(self, path: Union[Path, None] = None): def delete( self, context_name: Union[str, None] = None, dry_run: bool = True ) -> Union["ContextInfo", None]: + """Deletes the context with the specified name.""" if context_name is None: context_name = self.default_context diff --git a/src/kiara/defaults.py b/src/kiara/defaults.py index 5c8d223b8..59ca1d10e 100644 --- a/src/kiara/defaults.py +++ b/src/kiara/defaults.py @@ -9,6 +9,7 @@ import typing import uuid from enum import Enum +from pathlib import Path from appdirs import AppDirs @@ -139,6 +140,10 @@ VALUE_ATTR_DELIMITER = "::" VALID_VALUE_QUERY_CATEGORIES = ["data", "properties"] +CHUNK_CACHE_BASE_DIR = Path(kiara_app_dirs.user_cache_dir) / "data" / "chunks" +CHUNK_CACHE_DIR_DEPTH = 2 +CHUNK_CACHE_DIR_WIDTH = 1 + class SpecialValue(Enum): @@ -266,6 +271,9 @@ class SpecialValue(Enum): KIARA_MODEL_DATA_KEY = "data" KIARA_MODEL_SCHEMA_KEY = "schema" +ENVIRONMENT_MARKER_KEY = "environment" +"""Constant string to indicate this is a metadata entry of type 'environment'.""" + SYMLINK_ISSUE_MSG = """Your operating system does not support symlinks, which is a requirement for kiara to work. You can enable developer mode to fix this issue: diff --git a/src/kiara/interfaces/__init__.py b/src/kiara/interfaces/__init__.py index 079ad4080..5076eb6f9 100644 --- a/src/kiara/interfaces/__init__.py +++ b/src/kiara/interfaces/__init__.py @@ -9,7 +9,6 @@ import os import sys import uuid -from pathlib import Path from typing import ( TYPE_CHECKING, Any, @@ -21,7 +20,6 @@ Union, ) -from kiara.defaults import KIARA_CONFIG_FILE_NAME, KIARA_MAIN_CONFIG_FILE from kiara.exceptions import KiaraException from kiara.utils.cli import terminal_print @@ -30,8 +28,7 @@ from kiara.context import Kiara from kiara.context.config import KiaraConfig - from kiara.interfaces.python_api import KiaraAPI - + from kiara.interfaces.python_api.base_api import BaseAPI # log = structlog.getLogger() @@ -222,6 +219,12 @@ def set_console_width(width: Union[int, None] = None, prefer_env: bool = True): class KiaraAPIWrap(object): + """A wrapper class to help with lazy loading. + + This is mostly relevant in terms of Python imports and the cli, because that allows + to avoid importing lots of Python modules if only `--help` is called. + """ + def __init__( self, config: Union[str, None], @@ -244,7 +247,7 @@ def __init__( self._ensure_plugins: Union[str, Iterable[str], None] = ensure_plugins self._kiara_config: Union["KiaraConfig", None] = None - self._api: Union[KiaraAPI, None] = None + self._api: Union[BaseAPI, None] = None self._reload_process_if_plugins_installed = True @@ -292,49 +295,17 @@ def kiara_config(self) -> "KiaraConfig": if self._kiara_config is not None: return self._kiara_config - from kiara.context.config import KiaraConfig - - # kiara_config: Optional[KiaraConfig] = None - exists = False - create = False - if self._config: - config_path = Path(self._config) - if config_path.exists(): - if config_path.is_file(): - config_file_path = config_path - exists = True - else: - config_file_path = config_path / KIARA_CONFIG_FILE_NAME - if config_file_path.exists(): - exists = True - else: - config_path.parent.mkdir(parents=True, exist_ok=True) - config_file_path = config_path - else: - config_file_path = Path(KIARA_MAIN_CONFIG_FILE) - if not config_file_path.exists(): - create = True - exists = False - else: - exists = True - - if not exists: - if not create: - from kiara.utils.cli import terminal_print - - terminal_print() - terminal_print( - f"Can't create kiara context, specified config file does not exist: {self._config}." - ) - sys.exit(1) + try: + from kiara.utils.config import assemble_kiara_config - kiara_config = KiaraConfig() - kiara_config.save(config_file_path) - else: - kiara_config = KiaraConfig.load_from_file(config_file_path) + kiara_config = assemble_kiara_config(config_file=self._config) + except Exception as e: + terminal_print() + terminal_print(f"Error loading kiara config: {e}") + sys.exit(1) - kiara_config.runtime_config.runtime_profile = "default" + # kiara_config.runtime_config.runtime_profile = "default" self._kiara_config = kiara_config return self._kiara_config @@ -344,7 +315,7 @@ def lock_file(self, context: str) -> str: return "asdf" @property - def kiara_api(self) -> "KiaraAPI": + def kiara_api(self) -> "BaseAPI": if self._api is not None: return self._api @@ -355,9 +326,9 @@ def kiara_api(self) -> "KiaraAPI": if not context: context = self.kiara_config.default_context - from kiara.interfaces.python_api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI - api = KiaraAPI(kiara_config=self.kiara_config) + api = BaseAPI(kiara_config=self.kiara_config) if self._ensure_plugins: installed = api.ensure_plugin_packages(self._ensure_plugins, update=False) if installed and self._reload_process_if_plugins_installed: diff --git a/src/kiara/interfaces/cli/archive/commands.py b/src/kiara/interfaces/cli/archive/commands.py index e353ba48d..572c49e5d 100644 --- a/src/kiara/interfaces/cli/archive/commands.py +++ b/src/kiara/interfaces/cli/archive/commands.py @@ -34,9 +34,9 @@ def explain_archive( ): """Print details of an archive file.""" - from kiara.api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api info = kiara_api.retrieve_archive_info(archive) @@ -58,9 +58,9 @@ def explain_archive( @handle_exception() def export_archive(ctx, path: str, compression: str, append: bool, no_aliases: bool): - from kiara.api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI - api: KiaraAPI = ctx.obj.kiara_api + api: BaseAPI = ctx.obj.kiara_api target_store_params = {"compression": CHUNK_COMPRESSION_TYPE[compression.upper()]} result = api.export_archive( @@ -88,9 +88,9 @@ def export_archive(ctx, path: str, compression: str, append: bool, no_aliases: b def import_archive(ctx, path: str, no_aliases: bool): """Import an archive file.""" - from kiara.interfaces.python_api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api result = kiara_api.import_archive(source_archive=path, no_aliases=no_aliases) diff --git a/src/kiara/interfaces/cli/context/commands.py b/src/kiara/interfaces/cli/context/commands.py index cae31b1af..82c855f5c 100644 --- a/src/kiara/interfaces/cli/context/commands.py +++ b/src/kiara/interfaces/cli/context/commands.py @@ -20,7 +20,7 @@ ) if TYPE_CHECKING: - from kiara.api import Kiara, KiaraAPI, KiaraConfig + from kiara.interfaces.python_api.base_api import BaseAPI, Kiara, KiaraConfig @click.group("context") @@ -33,7 +33,7 @@ def context(ctx): @click.pass_context def list_contexts(ctx) -> None: """List existing contexts.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api summaries = kiara_api.retrieve_context_infos() diff --git a/src/kiara/interfaces/cli/data/commands.py b/src/kiara/interfaces/cli/data/commands.py index b6f7e301d..08347c9e3 100644 --- a/src/kiara/interfaces/cli/data/commands.py +++ b/src/kiara/interfaces/cli/data/commands.py @@ -15,6 +15,7 @@ import rich_click as click import structlog +from kiara.defaults import DATA_ARCHIVE_DEFAULT_VALUE_MARKER from kiara.exceptions import InvalidCommandLineInvocation from kiara.utils import is_develop, log_exception, log_message from kiara.utils.cli import output_format_option, terminal_print, terminal_print_model @@ -37,7 +38,7 @@ if TYPE_CHECKING: - from kiara.api import Kiara, KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI, Kiara from kiara.operations.included_core_operations.filter import FilterOperationType logger = structlog.getLogger() @@ -146,10 +147,9 @@ def list_values( ) -> None: """List all data items that are stored in kiara.""" - from kiara.interfaces.python_api import ValuesInfo - from kiara.interfaces.python_api.models.info import RENDER_FIELDS + from kiara.interfaces.python_api.models.info import RENDER_FIELDS, ValuesInfo - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api if include_internal: all_values = True @@ -269,7 +269,7 @@ def explain_value( All of the 'show-additional-information' flags are only applied when the 'terminal' output format is selected. This might change in the future. """ - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api render_config = { "show_pedigree": pedigree, @@ -321,7 +321,7 @@ def explain_value( def load_value(ctx, value: str): """Load a stored value and print it in a format suitable for the terminal.""" # kiara_obj: Kiara = ctx.obj["kiara"] - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api try: _value = kiara_api.get_value(value=value) @@ -412,7 +412,7 @@ def filter_value( silent = True kiara_obj: Kiara = ctx.obj.kiara - api: KiaraAPI = ctx.obj.kiara_api + api: BaseAPI = ctx.obj.kiara_api cmd_help = "[yellow bold]Usage: [/yellow bold][bold]kiara data filter VALUE FILTER_1:FILTER_2 [FILTER ARGS...][/bold]" @@ -548,6 +548,7 @@ def filter_value( default="zstd", ) @click.option("--append", "-a", help="Append data to existing archive.", is_flag=True) +@click.option("--replace", help="Replace existing archive.", is_flag=True) # @click.option( # "--no-default-value", "-nd", help="Do not set a default value.", is_flag=True # ) @@ -562,6 +563,7 @@ def export_data_archive( path: Union[str, None], compression: str, append: bool, + replace: bool, no_default_value: bool = False, no_aliases: bool = False, ): @@ -570,7 +572,7 @@ def export_data_archive( Aliases that already exist in the target archve will be overwritten. """ - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api values = [] for idx, alias in enumerate(aliases, start=1): @@ -608,13 +610,24 @@ def export_data_archive( full_path = Path(base_path) / file_name - if full_path.exists() and not append: + delete = False + + if full_path.exists() and (not append and not replace): terminal_print( - f"[red]Error[/red]: File '{full_path}' already exists and '--append' not specified." + f"[red]Error[/red]: File '{full_path}' already exists and '--append' or '--replace' not specified." ) sys.exit(1) elif full_path.exists(): - terminal_print(f"Appending to existing data_store '{file_name}'...") + if append and replace: + terminal_print( + "[red]Error[/red]: Can't specify both '--append' and '--replace'." + ) + sys.exit(1) + if append: + terminal_print(f"Appending to existing data_store '{file_name}'...") + else: + terminal_print(f"Replacing existing data_store '{file_name}'...") + delete = True else: terminal_print(f"Creating new data_store '{file_name}'...") @@ -645,6 +658,17 @@ def export_data_archive( "compression": compression, } try: + no_default_value = False + if not no_default_value: + metadata_to_add = { + DATA_ARCHIVE_DEFAULT_VALUE_MARKER: str(values[0][0].value_id) + } + else: + metadata_to_add = None + + if delete: + os.unlink(full_path) + store_result = kiara_api.export_values( target_archive=full_path, values=values_to_store, @@ -653,6 +677,7 @@ def export_data_archive( target_registered_name=archive_name, append=append, target_store_params=target_store_params, + additional_archive_metadata=metadata_to_add, ) render_config = {"add_field_column": False} terminal_print_model( @@ -679,7 +704,7 @@ def export_data_archive( def import_data_store(ctx, archive: str, values: Tuple[str], no_aliases: bool = False): """Import one or several values from a kiara archive.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api archive_path = Path(archive) if not archive_path.exists(): @@ -713,7 +738,7 @@ def import_data_store(ctx, archive: str, values: Tuple[str], no_aliases: bool = def write_serialized(ctx, value_id_or_alias: str, directory: str, force: bool): """Write the serialized form of a value to a directory""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api value = kiara_api.get_value(value_id_or_alias) serialized = value.serialized_data diff --git a/src/kiara/interfaces/cli/info/commands.py b/src/kiara/interfaces/cli/info/commands.py index 77d7e152e..0cfc046d2 100644 --- a/src/kiara/interfaces/cli/info/commands.py +++ b/src/kiara/interfaces/cli/info/commands.py @@ -11,7 +11,7 @@ from kiara.utils.cli.exceptions import handle_exception if TYPE_CHECKING: - from kiara.interfaces import KiaraAPI, KiaraAPIWrap + from kiara.interfaces import BaseAPI, KiaraAPIWrap @click.group("info") @@ -55,9 +55,9 @@ def plugin(ctx): def list_plugins(ctx, filter_regex: str, format): """List installed kiara plugins.""" - from kiara.interfaces.python_api import KiaraPluginInfos + from kiara.interfaces.python_api.models.info import KiaraPluginInfos - api: KiaraAPI = ctx.obj.kiara_api + api: BaseAPI = ctx.obj.kiara_api title = "All available plugins" if filter_regex: @@ -75,7 +75,7 @@ def list_plugins(ctx, filter_regex: str, format): @click.pass_context def explain_plugin_info(ctx, plugin_name: str, format: str): - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api plugin_info = kiara_api.retrieve_plugin_info(plugin_name) title = f"Info for plugin: [i]{plugin_name}[/i]" diff --git a/src/kiara/interfaces/cli/module/commands.py b/src/kiara/interfaces/cli/module/commands.py index 0bc920fa7..7dcec9b81 100644 --- a/src/kiara/interfaces/cli/module/commands.py +++ b/src/kiara/interfaces/cli/module/commands.py @@ -18,7 +18,7 @@ ) if TYPE_CHECKING: - from kiara.api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI @click.group() @@ -51,7 +51,7 @@ def list_modules( python_package: Union[str, None], ): """List available module data_types.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api module_types_info = kiara_api.retrieve_module_types_info( filter=filter, python_package=python_package @@ -79,7 +79,7 @@ def explain_module_type(ctx, module_type: str, format: str): instantiated with configuration, before we can query all their properties (like input/output data_types). """ - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api info = kiara_api.retrieve_module_type_info(module_type=module_type) terminal_print_model( @@ -106,7 +106,7 @@ def explain_module(ctx, module_type: str, module_config: Iterable[Any], format: else: module_config = {} - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api operation = kiara_api.create_operation( module_type=module_type, module_config=module_config diff --git a/src/kiara/interfaces/cli/operation/commands.py b/src/kiara/interfaces/cli/operation/commands.py index 5a785ff17..1e4552d70 100644 --- a/src/kiara/interfaces/cli/operation/commands.py +++ b/src/kiara/interfaces/cli/operation/commands.py @@ -15,7 +15,7 @@ from kiara.utils.cli.exceptions import handle_exception if TYPE_CHECKING: - from kiara.api import Kiara, KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI, Kiara @click.group() @@ -72,7 +72,7 @@ def list_types(ctx, full_doc: bool, format: str, filter: Iterable[str]): @handle_exception() def explain_type(ctx, operation_type: str, format: str): - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api op_type = kiara_api.retrieve_operation_type_info(operation_type) @@ -120,7 +120,7 @@ def list_operations( ): kiara_obj: Kiara = ctx.obj.kiara - api: KiaraAPI = ctx.obj.kiara_api + api: BaseAPI = ctx.obj.kiara_api operations = api.list_operations( filter=filter, include_internal=include_internal, python_packages=python_package @@ -149,7 +149,7 @@ def list_operations( # # operations = temp - from kiara.interfaces.python_api import OperationGroupInfo + from kiara.interfaces.python_api.models.info import OperationGroupInfo ops_info = OperationGroupInfo.create_from_operations( kiara=kiara_obj, group_title=title, **operations.root @@ -182,7 +182,7 @@ def list_operations( def explain(ctx, operation_id: str, source: bool, format: str, module_info: bool): kiara_obj: Kiara = ctx.obj.kiara - api: KiaraAPI = ctx.obj.kiara_api + api: BaseAPI = ctx.obj.kiara_api if os.path.isfile(os.path.realpath(operation_id)): operation = api.get_operation(operation_id) diff --git a/src/kiara/interfaces/cli/pipeline/commands.py b/src/kiara/interfaces/cli/pipeline/commands.py index b05fe8b95..a605f71ad 100644 --- a/src/kiara/interfaces/cli/pipeline/commands.py +++ b/src/kiara/interfaces/cli/pipeline/commands.py @@ -68,7 +68,7 @@ def list_pipelines(ctx, full_doc: bool, filter: typing.Iterable[str], format: st op_id: kiara_obj.operation_registry.get_operation(op_id) for op_id in op_ids } - from kiara.interfaces.python_api import OperationGroupInfo + from kiara.interfaces.python_api.models.info import OperationGroupInfo ops_info = OperationGroupInfo.create_from_operations( kiara=kiara_obj, group_title=title, **operations diff --git a/src/kiara/interfaces/cli/render/commands.py b/src/kiara/interfaces/cli/render/commands.py index bb7bdec26..9cd79bc46 100644 --- a/src/kiara/interfaces/cli/render/commands.py +++ b/src/kiara/interfaces/cli/render/commands.py @@ -23,8 +23,8 @@ from kiara.utils.cli.exceptions import handle_exception if typing.TYPE_CHECKING: - from kiara.api import KiaraAPI from kiara.interfaces import KiaraAPIWrap + from kiara.interfaces.python_api.base_api import BaseAPI # def list_renderers(ctx, param, value) -> None: @@ -63,7 +63,7 @@ def render( def list_render_combinations(ctx, format: str): api_wrap: KiaraAPIWrap = ctx.obj - kiara_api: KiaraAPI = api_wrap.kiara_api + kiara_api: BaseAPI = api_wrap.kiara_api source_type = api_wrap.get_item("source_type") target_type = api_wrap.get_item("target_type") @@ -104,7 +104,7 @@ def render_item( """Render an internal kiara item.""" api_wrap: KiaraAPIWrap = ctx.obj - kiara_api: KiaraAPI = api_wrap.kiara_api + kiara_api: BaseAPI = api_wrap.kiara_api source_type = api_wrap.get_item("source_type") target_type = api_wrap.get_item("target_type") diff --git a/src/kiara/interfaces/cli/run.py b/src/kiara/interfaces/cli/run.py index a06313b87..326d4ea43 100644 --- a/src/kiara/interfaces/cli/run.py +++ b/src/kiara/interfaces/cli/run.py @@ -22,7 +22,7 @@ from kiara.utils.files import get_data_from_file if typing.TYPE_CHECKING: - from kiara.api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI @click.command() @@ -121,7 +121,7 @@ def run( sys.exit(1) - api: KiaraAPI = ctx.obj.kiara_api # type: ignore + api: BaseAPI = ctx.obj.kiara_api # type: ignore cmd_arg = ctx.params["module_or_operation"] cmd_help = f"[yellow bold]Usage: [/yellow bold][bold]kiara run [OPTIONS] [i]{cmd_arg}[/i] [INPUTS][/bold]" diff --git a/src/kiara/interfaces/cli/type/commands.py b/src/kiara/interfaces/cli/type/commands.py index 2b0cc5fda..a41e09e57 100644 --- a/src/kiara/interfaces/cli/type/commands.py +++ b/src/kiara/interfaces/cli/type/commands.py @@ -44,7 +44,7 @@ def list_types( ): """List available data_types.""" from kiara.data_types import DataType - from kiara.interfaces.python_api import DataTypeClassesInfo + from kiara.interfaces.python_api.models.info import DataTypeClassesInfo kiara_obj: Kiara = ctx.obj.kiara @@ -110,7 +110,8 @@ def hierarchy(ctx, include_internal) -> None: @click.pass_context def explain_data_type(ctx, type_name: str, format: str): """Print details of a data type.""" - from kiara.interfaces.python_api import DataTypeClassInfo + + from kiara.interfaces.python_api.models.info import DataTypeClassInfo kiara_obj: Kiara = ctx.obj.kiara diff --git a/src/kiara/interfaces/cli/workflow/commands.py b/src/kiara/interfaces/cli/workflow/commands.py index a5523b902..0e6553fd4 100644 --- a/src/kiara/interfaces/cli/workflow/commands.py +++ b/src/kiara/interfaces/cli/workflow/commands.py @@ -15,7 +15,7 @@ from kiara.utils.cli import dict_from_cli_args, terminal_print, terminal_print_model if typing.TYPE_CHECKING: - from kiara.api import Kiara, KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI, Kiara logger = structlog.getLogger() @@ -36,7 +36,7 @@ def workflow(ctx): @click.pass_context def list(ctx, all) -> None: """List existing workflows.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api if all: workflows = kiara_api.retrieve_workflows_info() @@ -69,7 +69,7 @@ def create( force_alias: bool = False, ): """Create a new workflow.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api inputs_dict: Union[None, Dict[str, Any]] = None if inputs: @@ -99,7 +99,7 @@ def create( @click.pass_context def explain(ctx, workflow: str): """Explain the workflow with the specified id/alias.""" - kiara_api: KiaraAPI = ctx.obj.kiara_api + kiara_api: BaseAPI = ctx.obj.kiara_api workflow_info = kiara_api.retrieve_workflow_info(workflow=workflow) terminal_print( workflow_info.create_renderable(), @@ -120,7 +120,8 @@ def explain(ctx, workflow: str): @click.pass_context def set_input(ctx, workflow: str, inputs: Tuple[str], process: bool): """Set one or several inputs on the specified workflow.""" - from kiara.interfaces.python_api import Workflow + + from kiara.interfaces.python_api.workflow import Workflow kiara: Kiara = ctx.obj.kiara diff --git a/src/kiara/interfaces/python_api/__init__.py b/src/kiara/interfaces/python_api/__init__.py index f9691260a..15a2ef137 100644 --- a/src/kiara/interfaces/python_api/__init__.py +++ b/src/kiara/interfaces/python_api/__init__.py @@ -1,3486 +1,4 @@ # -*- coding: utf-8 -*- # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -import inspect -import json -import os.path -import sys -import textwrap -import uuid -from functools import cached_property -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - ClassVar, - Dict, - Iterable, - List, - Literal, - Mapping, - MutableMapping, - Set, - Type, - Union, -) - -import dpath -import structlog -from ruamel.yaml import YAML - -from kiara.defaults import ( - DATA_ARCHIVE_DEFAULT_VALUE_MARKER, - DEFAULT_CHUNK_COMPRESSION, - DEFAULT_STORE_MARKER, - OFFICIAL_KIARA_PLUGINS, - VALID_VALUE_QUERY_CATEGORIES, - VALUE_ATTR_DELIMITER, -) -from kiara.exceptions import ( - DataTypeUnknownException, - KiaraException, - NoSuchExecutionTargetException, - NoSuchWorkflowException, -) -from kiara.interfaces.python_api.models.info import ( - DataTypeClassesInfo, - DataTypeClassInfo, - KiaraPluginInfo, - KiaraPluginInfos, - ModuleTypeInfo, - ModuleTypesInfo, - OperationGroupInfo, - OperationInfo, - OperationTypeInfo, - RendererInfos, - ValueInfo, - ValuesInfo, -) -from kiara.interfaces.python_api.models.job import JobDesc -from kiara.interfaces.python_api.value import StoreValueResult, StoreValuesResult -from kiara.models.context import ContextInfo, ContextInfos -from kiara.models.module.manifest import Manifest -from kiara.models.module.operation import Operation -from kiara.models.rendering import RenderValueResult -from kiara.models.runtime_environment.python import PythonRuntimeEnvironment -from kiara.models.values.matchers import ValueMatcher -from kiara.models.values.value import ( - PersistedData, - Value, - ValueMapReadOnly, - ValueSchema, -) -from kiara.models.workflow import WorkflowGroupInfo, WorkflowInfo, WorkflowMetadata -from kiara.operations import OperationType -from kiara.operations.included_core_operations.filter import FilterOperationType -from kiara.operations.included_core_operations.pipeline import PipelineOperationDetails -from kiara.operations.included_core_operations.pretty_print import ( - PrettyPrintOperationType, -) -from kiara.operations.included_core_operations.render_value import ( - RenderValueOperationType, -) -from kiara.registries.environment import EnvironmentRegistry -from kiara.registries.ids import ID_REGISTRY -from kiara.registries.operations import OP_TYPE -from kiara.renderers import KiaraRenderer -from kiara.utils import log_exception, log_message -from kiara.utils.downloads import get_data_from_url -from kiara.utils.files import get_data_from_file -from kiara.utils.operations import create_operation -from kiara.utils.string_vars import replace_var_names_in_obj - -if TYPE_CHECKING: - from kiara.context import Kiara, KiaraConfig, KiaraRuntimeConfig - from kiara.interfaces.python_api.models.archive import KiArchive - from kiara.interfaces.python_api.models.doc import ( - OperationsMap, - PipelinesMap, - WorkflowsMap, - ) - from kiara.interfaces.python_api.workflow import Workflow - from kiara.models.archives import KiArchiveInfo - from kiara.models.module.jobs import ActiveJob, JobRecord - from kiara.models.module.pipeline import PipelineConfig, PipelineStructure - from kiara.models.module.pipeline.pipeline import PipelineGroupInfo, PipelineInfo - from kiara.registries import KiaraArchive - -logger = structlog.getLogger() -yaml = YAML(typ="safe") - - -class KiaraAPI(object): - """ - Public API for clients. - - This class wraps a [Kiara][kiara.context.kiara.Kiara] instance, and allows easy a access to tasks that are - typically done by a frontend. The return types of each method are json seriable in most cases. - - Can be extended for special scenarios and augmented with scenario-specific methdos (Jupyter, web-frontend, ...) - - The naming of the API endpoints follows a (loose-ish) convention: - - list_*: return a list of ids or items, if items, filtering is supported - - get_*: get specific instances of a type (operation, value, etc.) - - retrieve_*: get augmented information about an instance or type of something. This usually implies that there is some overhead, - so before you use this, make sure that there is not 'get_*' or 'list_*' endpoint that could give you what you need. - . - """ - - _default_instance: ClassVar[Union["KiaraAPI", None]] = None - _context_instances: ClassVar[Dict[str, "KiaraAPI"]] = {} - - @classmethod - def instance(cls, context_name: Union[str, None] = None) -> "KiaraAPI": - - if context_name is None: - - if cls._default_instance is not None: - return cls._default_instance - - from kiara.context import KiaraConfig - - config = KiaraConfig() - - api = KiaraAPI(kiara_config=config) - cls._default_instance = api - return api - else: - raise NotImplementedError() - - def __init__(self, kiara_config: Union["KiaraConfig", None] = None): - - if kiara_config is None: - from kiara.context import Kiara, KiaraConfig - - kiara_config = KiaraConfig() - - self._kiara_config: KiaraConfig = kiara_config - self._contexts: Dict[str, Kiara] = {} - self._workflow_cache: Dict[uuid.UUID, Workflow] = {} - - self._current_context: Union[None, Kiara] = None - self._current_context_alias: Union[None, str] = None - - @cached_property - def doc(self) -> Dict[str, str]: - """Get the documentation for this API.""" - result = {} - for method_name in dir(self): - if method_name.startswith("_"): - continue - - method = getattr(self.__class__, method_name) - doc = inspect.getdoc(method) - if doc is None: - doc = "-- n/a --" - else: - doc = textwrap.dedent(doc) - - result[method_name] = doc - - return result - - def list_available_plugin_names( - self, regex: str = "^kiara[-_]plugin\\..*" - ) -> List[str]: - """ - Get a list of all available plugins. - - Arguments: - regex: an optional regex to indicate the plugin naming scheme (default: /$kiara[_-]plugin\\..*/) - - Returns: - a list of plugin names - """ - - if not regex: - regex = "^kiara[-_]plugin\\..*" - - return KiaraPluginInfos.get_available_plugin_names( - kiara=self.context, regex=regex - ) - - def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo: - """ - Get information about a plugin. - - This contains information about included data-types, modules, operations, pipelines, as well as metadata - about author(s), etc. - - Arguments: - plugin_name: the name of the plugin - - Returns: - a dictionary with information about the plugin - """ - - info = KiaraPluginInfo.create_from_instance( - kiara=self.context, instance=plugin_name - ) - return info - - def retrieve_plugin_infos(self, plugin_name_regex: str = "^kiara[-_]plugin\\..*"): - """Get information about multiple plugins. - - This is just a convenience method to get information about multiple plugins at once. - """ - - if not plugin_name_regex: - plugin_name_regex = "^kiara[-_]plugin\\..*" - - plugin_infos = KiaraPluginInfos.create_group( - self.context, None, plugin_name_regex - ) - return plugin_infos - - @property - def context(self) -> "Kiara": - """ - Return the kiara context. - - DON"T USE THIS! This is going away in the production release. - """ - if self._current_context is None: - self._current_context = self._kiara_config.create_context( - extra_pipelines=None - ) - self._current_context_alias = self._kiara_config.default_context - - return self._current_context - - def get_runtime_config(self) -> "KiaraRuntimeConfig": - """Retrieve the current runtime configuration. - - Check the 'KiaraRuntimeConfig' class for more information about the available options. - """ - return self.context.runtime_config - - def get_context_info(self) -> ContextInfo: - """Retrieve information about the current kiara context. - - This contains information about the context, like its name/alias, the values & aliases it contains, and which archives are connected to it. - - """ - context_config = self._kiara_config.get_context_config( - self.get_current_context_name() - ) - info = ContextInfo.create_from_context_config( - context_config, - context_name=self.get_current_context_name(), - runtime_config=self._kiara_config.runtime_config, - ) - - return info - - def ensure_plugin_packages( - self, package_names: Union[str, Iterable[str]], update: bool = False - ) -> Union[bool, None]: - """ - Ensure that the specified packages are installed. - - - NOTE: this is not tested, and it might go away in the future, so don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment. - - Arguments: - package_names: The names of the packages to install. - update: If True, update the packages if they are already installed - - Returns: - 'None' if run in jupyter, 'True' if any packages were installed, 'False' otherwise. - """ - if isinstance(package_names, str): - package_names = [package_names] - - env_reg = EnvironmentRegistry.instance() - python_env: PythonRuntimeEnvironment = env_reg.environments[ # type: ignore - "python" - ] # type: ignore - - if not package_names: - package_names = OFFICIAL_KIARA_PLUGINS # type: ignore - - if not update: - plugin_packages: List[str] = [] - pkgs = [p.name.replace("_", "-") for p in python_env.packages] - for package_name in package_names: - if package_name.startswith("git:"): - package_name = package_name.replace("git:", "") - git = True - else: - git = False - package_name = package_name.replace("_", "-") - if not package_name.startswith("kiara-plugin."): - package_name = f"kiara-plugin.{package_name}" - - if git or package_name.replace("_", "-") not in pkgs: - if git: - package_name = package_name.replace("-", "_") - plugin_packages.append( - f"git+https://x:x@github.com/DHARPA-project/{package_name}@develop" - ) - else: - plugin_packages.append(package_name) - else: - plugin_packages = package_names # type: ignore - - in_jupyter = "google.colab" in sys.modules or "jupyter_client" in sys.modules - - if not plugin_packages: - if in_jupyter: - return None - else: - # nothing to do - return False - - class DummyContext(object): - def __getattribute__(self, item): - raise Exception( - "Currently installing plugins, no other operations are allowed." - ) - - current_context_name = self._current_context_alias - for k in self._contexts.keys(): - self._contexts[k] = DummyContext() # type: ignore - self._current_context = DummyContext() # type: ignore - - cmd = ["-q", "--isolated", "install"] - if update: - cmd.append("--upgrade") - cmd.extend(plugin_packages) - - if in_jupyter: - from IPython import get_ipython - - ipython = get_ipython() - cmd_str = f"sc -l stdout = {sys.executable} -m pip {' '.join(cmd)}" - ipython.magic(cmd_str) - exit_code = 100 - else: - import pip._internal.cli.main as pip - - log_message( - "install.python_packages", packages=plugin_packages, update=update - ) - exit_code = pip.main(cmd) - - self._contexts.clear() - self._current_context = None - self._current_context_alias = None - - EnvironmentRegistry._instance = None - if current_context_name: - self.set_active_context(context_name=current_context_name) - - if exit_code == 100: - raise SystemExit( - f"Please manually re-run all cells. Updated or newly installed plugin packages: {', '.join(plugin_packages)}." - ) - elif exit_code != 0: - raise Exception( - f"Failed to install plugin packages: {', '.join(plugin_packages)}" - ) - - return True - - # ================================================================================================================== - # context-management related functions - def list_context_names(self) -> List[str]: - """list the names of all available/registered contexts. - - NOTE: this functionality might be changed in the future, depending on requirements and feedback and - whether we want to support single-file contexts in the future. - """ - return list(self._kiara_config.available_context_names) - - def retrieve_context_infos(self) -> ContextInfos: - """Retrieve information about the available/registered contexts. - - NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. - """ - return ContextInfos.create_context_infos(self._kiara_config.context_configs) - - def get_current_context_name(self) -> str: - """Retrieve the name of the current context. - - NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. - """ - if self._current_context_alias is None: - self.context - return self._current_context_alias # type: ignore - - def create_new_context(self, context_name: str, set_active: bool = True) -> None: - """ - Create a new context. - - NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. So if you need something like this, please let me know. - - Arguments: - context_name: the name of the new context - set_active: set the newly created context as the active one - """ - if context_name in self.list_context_names(): - raise Exception( - f"Can't create context with name '{context_name}': context already exists." - ) - - ctx = self._kiara_config.create_context(context_name, extra_pipelines=None) - if set_active: - self._current_context = ctx - self._current_context_alias = context_name - - # return ctx - - def set_active_context(self, context_name: str, create: bool = False) -> None: - """Set the currently active context for this KiarAPI instance. - - NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. - """ - - if not context_name: - raise Exception("No context name provided.") - - if context_name == self._current_context_alias: - return - if context_name not in self.list_context_names(): - if create: - self._current_context = self._kiara_config.create_context( - context=context_name, extra_pipelines=None - ) - self._current_context_alias = context_name - return - else: - raise Exception(f"No context with name '{context_name}' available.") - - self._current_context = self._kiara_config.create_context( - context=context_name, extra_pipelines=None - ) - self._current_context_alias = context_name - - # ================================================================================================================== - # methods for data_types - - def list_data_type_names(self, include_profiles: bool = False) -> List[str]: - """Get a list of all registered data types. - - Arguments: - include_profiles: if True, also include the names of all registered data type profiles - """ - - return self.context.type_registry.get_data_type_names( - include_profiles=include_profiles - ) - - def is_internal_data_type(self, data_type_name: str) -> bool: - """Checks if the data type is prepdominantly used internally by kiara, or whether it should be exposed to the user.""" - - return self.context.type_registry.is_internal_type( - data_type_name=data_type_name - ) - - def retrieve_data_types_info( - self, - filter: Union[str, Iterable[str], None] = None, - include_data_type_profiles: bool = False, - python_package: Union[None, str] = None, - ) -> DataTypeClassesInfo: - """ - Retrieve information about all data types. - - A data type is a Python class that inherits from [DataType[kiara.data_types.DataType], and it wraps a specific - Python class that holds the actual data and provides metadata and convenience methods for managing the data internally. Data types are not directly used by users, but they are exposed in the input/output schemas of moudles and other data-related features. - - Arguments: - filter: an optional string or (list of strings) the returned datatype ids have to match (all filters in the case of a list) - include_data_type_profiles: if True, also include the names of all registered data type profiles - python_package: if provided, only return data types that are defined in the given python package - - Returns: - an object containing all information about all data types - """ - - if python_package: - data_type_info = self.context.type_registry.get_context_metadata( - only_for_package=python_package - ) - - if filter: - title = f"Filtered data types in package '{python_package}'" - - if isinstance(filter, str): - filter = [filter] - - filtered_types: Dict[str, DataTypeClassInfo] = {} - - for dt in data_type_info.item_infos.keys(): - match = True - - for f in filter: - if f.lower() not in dt.lower(): - match = False - break - if match: - filtered_types[dt] = data_type_info.item_infos[dt] - - data_types_info = DataTypeClassesInfo( - group_title=title, item_infos=filtered_types - ) - data_types_info._kiara = self.context - - else: - title = f"All data types in package '{python_package}'" - data_types_info = data_type_info - data_types_info.group_title = title - else: - if filter: - if isinstance(filter, str): - filter = [filter] - - title = f"Filtered data_types: {filter}" - data_type_names: Iterable[str] = [] - - for m in self.context.type_registry.get_data_type_names( - include_profiles=include_data_type_profiles - ): - match = True - - for f in filter: - - if f.lower() not in m.lower(): - match = False - break - - if match: - data_type_names.append(m) # type: ignore - else: - title = "All data types" - data_type_names = self.context.type_registry.get_data_type_names( - include_profiles=include_data_type_profiles - ) - - data_types = { - d: self.context.type_registry.get_data_type_cls(d) - for d in data_type_names - } - data_types_info = DataTypeClassesInfo.create_from_type_items( # type: ignore - kiara=self.context, group_title=title, **data_types - ) - - return data_types_info # type: ignore - - def retrieve_data_type_info(self, data_type_name: str) -> DataTypeClassInfo: - """ - Retrieve information about a specific data type. - - Arguments: - data_type: the registered name of the data type - - Returns: - an object containing all information about a data type - """ - dt_cls = self.context.type_registry.get_data_type_cls(data_type_name) - info = DataTypeClassInfo.create_from_type_class( - kiara=self.context, type_cls=dt_cls - ) - return info - - # ================================================================================================================== - # methods for module and operations info - - def list_module_type_names(self) -> List[str]: - """Get a list of all registered module types.""" - return list(self.context.module_registry.get_module_type_names()) - - def retrieve_module_types_info( - self, - filter: Union[None, str, Iterable[str]] = None, - python_package: Union[str, None] = None, - ) -> ModuleTypesInfo: - """ - Retrieve information for all available module types (or a filtered subset thereof). - - A module type is Python class that inherits from [KiaraModule][kiara.modules.KiaraModule], and is the basic - building block for processing pipelines. Module types are not used directly by users, Operations are. Operations - are instantiated modules (meaning: the module & some (optional) configuration). - - Arguments: - filter: an optional string (or list of string) the returned module names have to match (all filters in case of list) - python_package: an optional string, if provided, only modules from the specified python package are returned - - Returns: - a mapping object containing module names as keys, and information about the modules as values - """ - - if python_package: - - modules_type_info = self.context.module_registry.get_context_metadata( - only_for_package=python_package - ) - - if filter: - title = f"Filtered modules: {filter} (in package '{python_package}')" - if isinstance(filter, str): - filter = [filter] - - filtered_types: Dict[str, ModuleTypeInfo] = {} - - for m in modules_type_info.item_infos.keys(): - match = True - - for f in filter: - - if f.lower() not in m.lower(): - match = False - break - - if match: - filtered_types[m] = modules_type_info.item_infos[m] - - module_types_info = ModuleTypesInfo( - group_title=title, item_infos=filtered_types - ) - module_types_info._kiara = self.context - else: - title = f"All modules in package '{python_package}'" - module_types_info = modules_type_info - module_types_info.group_title = title - - else: - - if filter: - - if isinstance(filter, str): - filter = [filter] - title = f"Filtered modules: {filter}" - module_types_names: Iterable[str] = [] - - for m in self.context.module_registry.get_module_type_names(): - match = True - - for f in filter: - - if f.lower() not in m.lower(): - match = False - break - - if match: - module_types_names.append(m) # type: ignore - else: - title = "All modules" - module_types_names = ( - self.context.module_registry.get_module_type_names() - ) - - module_types = { - n: self.context.module_registry.get_module_class(n) - for n in module_types_names - } - - module_types_info = ModuleTypesInfo.create_from_type_items( # type: ignore - kiara=self.context, group_title=title, **module_types - ) - - return module_types_info # type: ignore - - def retrieve_module_type_info(self, module_type: str) -> ModuleTypeInfo: - """ - Retrieve information about a specific module type. - - This can be used to retrieve information like module documentation and configuration options. - - Arguments: - module_type: the registered name of the module - - Returns: - an object containing all information about a module type - """ - m_cls = self.context.module_registry.get_module_class(module_type) - info = ModuleTypeInfo.create_from_type_class(kiara=self.context, type_cls=m_cls) - return info - - def create_operation( - self, - module_type: str, - module_config: Union[Mapping[str, Any], str, None] = None, - ) -> Operation: - """ - Create an [Operation][kiara.models.module.operation.Operation] instance for the specified module type and (optional) config. - - An operation is defined as a specific module type, and a specific configuration. - - This endpoint can be used to get information about the operation itself, it's inputs & outputs schemas, documentation etc. - - Arguments: - module_type: the registered name of the module - module_config: (Optional) configuration for the module instance. - - Returns: - an Operation instance (which contains all the available information about an instantiated module) - """ - if module_config is None: - module_config = {} - elif isinstance(module_config, str): - try: - module_config = json.load(module_config) # type: ignore - except Exception: - try: - module_config = yaml.load(module_config) # type: ignore - except Exception: - raise Exception( - f"Can't parse module config string: {module_config}." - ) - - if module_type == "pipeline": - if not module_config: - raise Exception("Pipeline configuration can't be empty.") - assert module_config is None or isinstance(module_config, Mapping) - operation = create_operation( - "pipeline", operation_config=module_config, kiara=self.context - ) - return operation - else: - mc = Manifest(module_type=module_type, module_config=module_config) - module_obj = self.context.module_registry.create_module(mc) - - return module_obj.operation - - def list_operation_ids( - self, - filter: Union[str, None, Iterable[str]] = None, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - operation_types: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - python_packages: Union[str, None, Iterable[str]] = None, - ) -> List[str]: - """ - Get a list of all operation ids that match the specified filter. - - Arguments: - filter: the (optional) filter string(s), an operation must match all of them to be included in the result - input_types: each operation must have at least one input that matches one of the specified types - output_types: each operation must have at least one output that matches one of the specified types - operation_types: only include operations of the specified type(s) - include_internal: whether to include operations that are predominantly used internally in kiara. - python_packages: only include operations that are contained in one of the provided python packages - """ - if not filter and include_internal and not python_packages: - return sorted(self.context.operation_registry.operation_ids) - - else: - return sorted( - self.list_operations( - filter=filter, - input_types=input_types, - output_types=output_types, - operation_types=operation_types, - include_internal=include_internal, - python_packages=python_packages, - ).keys() - ) - - def get_operation( - self, - operation: Union[Mapping[str, Any], str, Path], - allow_external: Union[bool, None] = None, - ) -> Operation: - """ - Return the operation instance with the specified id. - - The difference to the 'create_operation' endpoint is slight, in most cases you could use either of them, but this one is a bit more convenient in most cases, as it tries to do the right thing with whatever 'operation' argument you use it. The 'create_opearation' endpoint will always create a new 'Operation' instance, while this may or may not return a re-used one. - - This endpoint can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc. - - The order in which the operation argument is resolved: - - if it's a string, and an existing, registered operation_id, the associated operation is returned - - if it's a path to an existing file, the content of the file is loaded into a dict and depending on the content a pipeline module will be created, or a 'normal' manifest (if module_type is a key in the dict) - - Arguments: - operation: the operation id, module_type_name, path to a file, or url - allow_external: if True, allow loading operations from external sources (e.g. a URL), if 'None' is provided, the configured value in the runtime configuration is used. - - Returns: - operation instance data - """ - _module_type = None - _module_config: Any = None - - if allow_external is None: - allow_external = self.get_runtime_config().allow_external - - if isinstance(operation, Path): - operation = operation.as_posix() - - if ( - isinstance(operation, Mapping) - and "module_type" in operation.keys() - and "module_config" in operation.keys() - and not operation["module_config"] - ): - operation = operation["module_type"] - - if isinstance(operation, str): - - if operation in self.list_operation_ids(include_internal=True): - _operation = self.context.operation_registry.get_operation(operation) - return _operation - - if not allow_external: - raise NoSuchExecutionTargetException( - selected_target=operation, - available_targets=self.context.operation_registry.operation_ids, - msg=f"Can't find operation with id '{operation}', and external operations are not allowed.", - ) - - if os.path.isfile(operation): - try: - from kiara.models.module.pipeline import PipelineConfig - - # we use the 'from_file' here, because that will resolve any relative paths in the pipeline - # if this doesn't work, we just assume the file is not a pipeline configuration but - # a manifest file with 'module_type' and optional 'module_config' keys - pipeline_conf = PipelineConfig.from_file( - path=operation, kiara=self.context - ) - _module_config = pipeline_conf.model_dump() - except Exception as e: - log_exception(e) - _module_config = get_data_from_file(operation) - elif operation.startswith("http"): - _module_config = get_data_from_url(operation) - else: - try: - _module_config = json.load(operation) # type: ignore - except Exception: - try: - _module_config = yaml.load(operation) # type: ignore - except Exception: - raise Exception( - f"Can't parse configuration string: {operation}." - ) - if not isinstance(_module_config, Mapping): - raise NoSuchExecutionTargetException( - selected_target=operation, - available_targets=self.context.operation_registry.operation_ids, - msg=f"Can't find operation or execution target for string '{operation}'.", - ) - - else: - _module_config = dict(operation) # type: ignore - - if "module_type" in _module_config.keys(): - _module_type = _module_config["module_type"] - _module_config = _module_config.get("module_config", {}) - else: - _module_type = "pipeline" - - op = self.create_operation( - module_type=_module_type, module_config=_module_config - ) - return op - - def list_operations( - self, - filter: Union[str, None, Iterable[str]] = None, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - operation_types: Union[str, Iterable[str], None] = None, - python_packages: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - ) -> "OperationsMap": - """ - List all available operations, optionally filter. - - Arguments: - filter: the (optional) filter string(s), an operation must match all of them to be included in the result - input_types: each operation must have at least one input that matches one of the specified types - output_types: each operation must have at least one output that matches one of the specified types - operation_types: only include operations of the specified type(s) - include_internal: whether to include operations that are predominantly used internally in kiara. - python_packages: only include operations that are contained in one of the provided python packages - - Returns: - a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value - """ - if operation_types: - if isinstance(operation_types, str): - operation_types = [operation_types] - temp: Dict[str, Operation] = {} - for op_type_name in operation_types: - op_type = self.context.operation_registry.operation_types.get( - op_type_name, None - ) - if op_type is None: - raise Exception(f"Operation type not registered: {op_type_name}") - - temp.update(op_type.operations) - - operations: Mapping[str, Operation] = temp - else: - operations = self.context.operation_registry.operations - - if filter: - if isinstance(filter, str): - filter = [filter] - temp = {} - for op_id, op in operations.items(): - match = True - for f in filter: - if not f: - continue - if f.lower() not in op_id.lower(): - match = False - break - if match: - temp[op_id] = op - operations = temp - - if not include_internal: - temp = {} - for op_id, op in operations.items(): - if not op.operation_details.is_internal_operation: - temp[op_id] = op - - operations = temp - - if input_types: - if isinstance(input_types, str): - input_types = [input_types] - temp = {} - for op_id, op in operations.items(): - for input_type in input_types: - match = False - for schema in op.inputs_schema.values(): - if schema.type == input_type: - temp[op_id] = op - match = True - break - if match: - break - - operations = temp - - if output_types: - if isinstance(output_types, str): - output_types = [output_types] - temp = {} - for op_id, op in operations.items(): - for output_type in output_types: - match = False - for schema in op.outputs_schema.values(): - if schema.type == output_type: - temp[op_id] = op - match = True - break - if match: - break - - operations = temp - - if python_packages: - temp = {} - if isinstance(python_packages, str): - python_packages = [python_packages] - for op_id, op in operations.items(): - info = OperationInfo.create_from_instance( - kiara=self.context, instance=op - ) - pkg = info.context.labels.get("package", None) - if pkg in python_packages: - temp[op_id] = op - operations = temp - - from kiara.interfaces.python_api.models.doc import OperationsMap - - return OperationsMap.model_construct(root=operations) # type: ignore - - def retrieve_operation_info( - self, operation: str, allow_external: bool = False - ) -> OperationInfo: - """ - Return the full information for the specified operation id. - - This is similar to the 'get_operation' method, but returns additional information. Only use this instead of - 'get_operation' if you need the additional info, as it's more expensive to get. - - Arguments: - operation: the operation id - - Returns: - augmented operation instance data - """ - if not allow_external: - op = self.context.operation_registry.get_operation(operation_id=operation) - else: - op = create_operation(module_or_operation=operation) - op_info = OperationInfo.create_from_operation(kiara=self.context, operation=op) - return op_info - - def retrieve_operations_info( - self, - *filters, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - operation_types: Union[str, Iterable[str], None] = None, - python_packages: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - ) -> OperationGroupInfo: - """ - Retrieve information about the matching operations. - - This retrieves the same list of operations as [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations], - but augments each result instance with additional information that might be useful in frontends. - - 'OperationInfo' objects contains augmented information on top of what 'normal' [Operation][kiara.models.module.operation.Operation] objects - hold, but they can take longer to create/resolve. If you don't need any - of the augmented information, just use the [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations] method - instead. - - Arguments: - filters: the (optional) filter strings, an operation must match all of them to be included in the result - include_internal: whether to include operations that are predominantly used internally in kiara. - input_types: each operation must have at least one input that matches one of the specified types - output_types: each operation must have at least one output that matches one of the specified types - operation_types: only include operations of the specified type(s) - include_internal: whether to include operations that are predominantly used internally in kiara. - python_packages: only include operations that are contained in one of the provided python packages - Returns: - a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value - """ - title = "Available operations" - if filters: - title = "Filtered operations" - - operations = self.list_operations( - filters, - input_types=input_types, - output_types=output_types, - include_internal=include_internal, - operation_types=operation_types, - python_packages=python_packages, - ) - - ops_info = OperationGroupInfo.create_from_operations( - kiara=self.context, group_title=title, **operations - ) - return ops_info - - # ================================================================================================================== - # methods relating to pipelines - - def list_pipeline_ids( - self, - filter: Union[str, None, Iterable[str]] = None, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - python_packages: Union[str, None, Iterable[str]] = None, - ) -> List[str]: - """ - Get a list of all pipeline (operation) ids that match the specified filter. - - Arguments: - filter: an optional single or list of filters (all filters must match the operation id for the operation to be included) - include_internal: also return internal pipelines - """ - - return self.list_operation_ids( - filter=filter, - input_types=input_types, - output_types=output_types, - operation_types=["pipeline"], - include_internal=include_internal, - python_packages=python_packages, - ) - - def list_pipelines( - self, - filter: Union[str, None, Iterable[str]] = None, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - python_packages: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - ) -> "PipelinesMap": - """List all available pipelines, optionally filter. - - Arguments: - filter: the (optional) filter string(s), an operation must match all of them to be included in the result - input_types: each operation must have at least one input that matches one of the specified types - output_types: each operation must have at least one output that matches one of the specified types - operation_types: only include operations of the specified type(s) - include_internal: whether to include operations that are predominantly used internally in kiara. - python_packages: only include operations that are contained in one of the provided python packages - - Returns: - a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value - """ - from kiara.interfaces.python_api.models.doc import PipelinesMap - - ops = self.list_operations( - filter=filter, - input_types=input_types, - output_types=output_types, - operation_types=["pipeline"], - python_packages=python_packages, - include_internal=include_internal, - ) - - result: Dict[str, PipelineStructure] = {} - for op in ops.values(): - details: PipelineOperationDetails = op.operation_details - config: "PipelineConfig" = details.pipeline_config - structure = config.structure - result[op.operation_id] = structure - - return PipelinesMap.model_construct(root=result) - - def get_pipeline_structure( - self, - pipeline: Union[Mapping[str, Any], str, Path], - allow_external: Union[bool, None] = None, - ) -> "PipelineStructure": - """ - Return the pipeline (Structure) instance with the specified id. - - This can be used to get information about a pipeline, like inputs/outputs scheman, documentation, included steps, stages, etc. - - The order in which the operation argument is resolved: - - if it's a string, and an existing, registered operation_id, the associated operation is returned - - if it's a path to an existing file, the content of the file is loaded into a dict and a pipeline operation will be created - - Arguments: - pipeline: the pipeline id, module_type_name, path to a file, or url - allow_external: if True, allow loading operations from external sources (e.g. a URL), if 'None' is provided, the configured value in the runtime configuration is used. - - Returns: - pipeline structure data - """ - - op = self.get_operation(operation=pipeline, allow_external=allow_external) - if op.module_type != "pipeline": - raise KiaraException( - f"Operation '{op.operation_id}' is not a pipeline, but a '{op.module_type}'" - ) - details: PipelineOperationDetails = op.operation_details # type: ignore - config: "PipelineConfig" = details.pipeline_config - - return config.structure - - def retrieve_pipeline_info( - self, pipeline: str, allow_external: bool = False - ) -> "PipelineInfo": - """ - Return the full information for the specified pipeline id. - - This is similar to the 'get_pipeline' method, but returns additional information. Only use this instead of - 'get_pipeline' if you need the additional info, as it's more expensive to get. - - Arguments: - pipeline: the pipeline (operation) id - - Returns: - augmented pipeline instance data - """ - if not allow_external: - op = self.context.operation_registry.get_operation(operation_id=pipeline) - else: - op = create_operation(module_or_operation=pipeline) - - if op.module_type != "pipeline": - raise KiaraException( - f"Operation '{op.operation_id}' is not a pipeline, but a '{op.module_type}'" - ) - - from kiara.models.module.pipeline.pipeline import Pipeline, PipelineInfo - - details: PipelineOperationDetails = op.operation_details # type: ignore - config: "PipelineConfig" = details.pipeline_config - pipeline_instance = Pipeline(structure=config.structure, kiara=self.context) - - p_info: PipelineInfo = PipelineInfo.create_from_instance( - kiara=self.context, instance=pipeline_instance - ) - return p_info - - def retrieve_pipelines_info( - self, - *filters, - input_types: Union[str, Iterable[str], None] = None, - output_types: Union[str, Iterable[str], None] = None, - python_packages: Union[str, Iterable[str], None] = None, - include_internal: bool = False, - ) -> "PipelineGroupInfo": - """ - Retrieve information about the matching pipelines. - - This retrieves the same list of pipelines as [list_pipelines][kiara.interfaces.python_api.KiaraAPI.list_pipelines], - but augments each result instance with additional information that might be useful in frontends. - - 'PipelineInfo' objects contains augmented information on top of what 'normal' [PipelineStructure][kiara.models.module.pipeline.PipelineStructure] objects - hold, but they can take longer to create/resolve. If you don't need any - of the augmented information, just use the [list_pipelines][kiara.interfaces.python_api.KiaraAPI.list_pipelines] method - instead. - - Arguments: - filters: the (optional) filter strings, an operation must match all of them to be included in the result - include_internal: whether to include operations that are predominantly used internally in kiara. - input_types: each operation must have at least one input that matches one of the specified types - output_types: each operation must have at least one output that matches one of the specified types - include_internal: whether to include operations that are predominantly used internally in kiara. - python_packages: only include operations that are contained in one of the provided python packages - Returns: - a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value - """ - - title = "Available pipelines" - if filters: - title = "Filtered pipelines" - - operations = self.list_operations( - filters, - input_types=input_types, - output_types=output_types, - include_internal=include_internal, - operation_types=["pipeline"], - python_packages=python_packages, - ) - - from kiara.models.module.pipeline.pipeline import Pipeline, PipelineGroupInfo - - pipelines = {} - for op_id, op in operations.items(): - details: PipelineOperationDetails = op.operation_details # type: ignore - config: "PipelineConfig" = details.pipeline_config - pipeline = Pipeline(structure=config.structure, kiara=self.context) - pipelines[op_id] = pipeline - - ps_info = PipelineGroupInfo.create_from_pipelines( - kiara=self.context, group_title=title, **pipelines - ) - return ps_info - - def register_pipeline( - self, - data: Union[Path, str, Mapping[str, Any]], - operation_id: Union[str, None] = None, - ) -> Operation: - """ - Register a pipelne as new operation into this context. - - If 'operation_id' is not provided, the id will be auto-determined (in most cases using the pipeline name). - - Arguments: - data: a dict or a path to a json/yaml file containing the definition - operation_id: the id to use for the operation (if not specified, the id will be auto-determined) - - Returns: - the assembled operation - """ - return self.context.operation_registry.register_pipeline( - data=data, operation_id=operation_id - ) - - def register_pipelines( - self, *pipeline_paths: Union[str, Path] - ) -> Dict[str, Operation]: - """Register all pipelines found in the specified paths.""" - return self.context.operation_registry.register_pipelines(*pipeline_paths) - - # ================================================================================================================== - # methods relating to values and data - - def register_data( - self, - data: Any, - data_type: Union[None, str, ValueSchema, Mapping[str, Any]] = None, - reuse_existing: bool = False, - ) -> Value: - """ - Register data with kiara. - - This will create a new value instance from the data and return it. The data/value itself won't be stored - in a store, you have to use the 'store_value' function for that. - - Arguments: - data: the data to register - data_type: (optional) the data type of the data. If not provided, kiara will try to infer the data type. - reuse_existing: whether to re-use an existing value that is already registered and has the same hash. - - Returns: - a [kiara.models.values.value.Value] instance - """ - if data_type is None: - raise NotImplementedError( - "Infering data types not implemented yet. Please provide one manually." - ) - - value = self.context.data_registry.register_data( - data=data, schema=data_type, reuse_existing=reuse_existing - ) - return value - - def list_all_value_ids(self) -> List[uuid.UUID]: - """List all value ids in the current context. - - This returns everything, even internal values. It should be faster than using - `list_value_ids` with equivalent parameters, because no filtering has to happen. - - Returns: - all value_ids in the current context, using every registered store - """ - - _values = self.context.data_registry.retrieve_all_available_value_ids() - return sorted(_values) - - def list_value_ids(self, **matcher_params) -> List[uuid.UUID]: - """ - List all available value ids for this kiara context. - - By default, this also includes internal values. - - This method exists mainly so frontends can retrieve a list of all value_ids that exists on the backend without - having to look up the details of each value (like [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] - does). This method can also be used with a matcher, but in this case the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] - would be preferable in most cases, because it is called under the hood, and the performance advantage of not - having to look up value details is gone. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters and defaults - - Returns: - a list of value ids - """ - - values = self.list_values(**matcher_params) - return sorted((v.value_id for v in values.values())) - - def list_all_values(self) -> ValueMapReadOnly: - """List all values in the current context, incl. internal ones. - - This should be faster than `list_values` with equivalent matcher params, because no - filtering has to happen. - """ - - # TODO: make that parallel? - values = { - k: self.context.data_registry.get_value(k) - for k in self.context.data_registry.retrieve_all_available_value_ids() - } - result = ValueMapReadOnly.create_from_values( - **{str(k): v for k, v in values.items()} - ) - return result - - def list_values(self, **matcher_params: Any) -> ValueMapReadOnly: - """ - List all available (relevant) values, optionally filter. - - Retrieve information about all values that are available in the current kiara context session (both stored and non-stored). - - Check the `ValueMatcher` class for available parameters and defaults, for example this excludes - internal values by default. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters - - Returns: - a dictionary with value_id as key, and [kiara.models.values.value.Value] as value - """ - - matcher = ValueMatcher.create_matcher(**matcher_params) - values = self.context.data_registry.find_values(matcher=matcher) - - result = ValueMapReadOnly.create_from_values( - **{str(k): v for k, v in values.items()} - ) - return result - - def get_value(self, value: Union[str, Value, uuid.UUID, Path]) -> Value: - """ - Retrieve a value instance with the specified id or alias. - - Basically a convenience method to convert any possible Python type into - a 'Value' instance. Raises an exception if no value could be found. - - Arguments: - value: a value id, alias or object that has a 'value_id' attribute. - - Returns: - the Value instance - """ - return self.context.data_registry.get_value(value=value) - - def get_values(self, **values: Union[str, Value, uuid.UUID]) -> ValueMapReadOnly: - """Retrieve Value instances for the specified value ids or aliases. - - This is a convenience method to get fully 'hydrated' `Value` objects from references to them. - - Arguments: - values: a dictionary with value ids or aliases as keys, and value instances as values - - Returns: - a mapping with value_id as key, and [kiara.models.values.value.Value] as value - """ - - return self.context.data_registry.load_values(values=values) - - def query_value( - self, - value_or_path: Union[str, Value, uuid.UUID], - query_path: Union[str, None] = None, - ) -> Any: - """ - Retrieve a value attribute with the specified id or alias. - - NOTE: This is a provisional endpoint, don't use for now, if you have a requirement that would - be covered by this, please let me know. - - A query path is delimited by "::", and has the following format: - - ``` - ::[]::[]::[...] - ``` - - Currently supported categories: - - "data": the data of the value - - "properties: the properties of the value - - If no category is specified, the value instance itself is returned. - - Raises an exception if no value could be found. - - Arguments: - value_or_path: a value or value reference, or a query path containing the value id or alias as first token - query_path: a query path which will be appended a potential query path computed from the first argument - - Returns: - the attribute value - """ - - if isinstance(value_or_path, str): - tokens = value_or_path.split(VALUE_ATTR_DELIMITER) - value_id = tokens.pop(0) - _value = self.get_value(value=value_id) - else: - tokens = [] - _value = self.get_value(value=value_or_path) - - if query_path: - tokens.extend(query_path.split(VALUE_ATTR_DELIMITER)) - - if not tokens: - return _value - - current_result: Any = _value - category = tokens.pop(0) - if category == "properties": - current_result = current_result.get_all_property_data(flatten_models=True) - elif category == "data": - current_result = current_result.data - else: - raise KiaraException( - f"Invalid query path category: {category}. Valid categories are: {', '.join(VALID_VALUE_QUERY_CATEGORIES)}" - ) - - if tokens: - try: - path = VALUE_ATTR_DELIMITER.join(tokens) - current_result = dpath.get( - current_result, path, separator=VALUE_ATTR_DELIMITER - ) - - except Exception: - - def dict_path(path, my_dict, all_paths): - for k, v in my_dict.items(): - if isinstance(v, dict): - dict_path(path + "::" + k, v, all_paths) - else: - all_paths.append(path[2:] + "::" + k) - - valid_base_keys = list(current_result.keys()) - details = "Valid (base) sub-keys are:\n\n" - for k in valid_base_keys: - details += f" - {k}\n" - - all_paths: List[str] = [] - dict_path("", current_result, all_paths) - - details += "\nValid (full) sub-paths are:\n\n" - for k in all_paths: - details += f" - {k}\n" - - raise KiaraException( - msg=f"Failed to retrieve value attribute using query sub-path: {path}", - details=details, - ) - - return current_result - - def retrieve_value_info( - self, value: Union[str, uuid.UUID, Value, Path] - ) -> ValueInfo: - """ - Retrieve an info object for a value. - - Companion method to 'get_value', 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects - hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any - of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.get_value] method - instead. - - Arguments: - value: a value id, alias or object that has a 'value_id' attribute. - - Returns: - the ValueInfo instance - - """ - _value = self.get_value(value=value) - return ValueInfo.create_from_instance(kiara=self.context, instance=_value) - - def retrieve_values_info(self, **matcher_params) -> ValuesInfo: - """ - Retrieve information about the matching values. - - This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], - but augments each result value instance with additional information that might be useful in frontends. - - 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects - hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any - of the augmented information, just use the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] method - instead. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters - - Returns: - a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value - """ - values: MutableMapping[str, Value] = self.list_values(**matcher_params) - - infos = ValuesInfo.create_from_instances( - kiara=self.context, instances={str(k): v for k, v in values.items()} - ) - return infos # type: ignore - - def list_alias_names(self, **matcher_params) -> List[str]: - """ - List all available alias keys. - - This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without - having to look up the details of each value (like [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] - does). This method can also be used with a matcher, but in this case the [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] - would be preferrable in most cases, because it is called under the hood, and the performance advantage of not - having to look up value details is gone. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters - - Returns: - a list of value ids - """ - if matcher_params: - values = self.list_aliases(**matcher_params) - return list(values.keys()) - else: - _values = self.context.alias_registry.all_aliases - return list(_values) - - def list_aliases(self, **matcher_params) -> ValueMapReadOnly: - """ - List all available values that have an alias assigned, optionally filter. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters - - Returns: - a dictionary with value_id as key, and [kiara.models.values.value.Value] as value - """ - if matcher_params: - matcher_params["has_alias"] = True - all_values = self.list_values(**matcher_params) - - result: Dict[str, Value] = {} - for value in all_values.values(): - aliases = self.context.alias_registry.find_aliases_for_value_id( - value_id=value.value_id - ) - for a in aliases: - if a in result.keys(): - raise Exception( - f"Duplicate value alias '{a}': this is most likely a bug." - ) - result[a] = value - - result = {k: result[k] for k in sorted(result.keys())} - else: - # faster if not other matcher params - all_aliases = self.context.alias_registry.all_aliases - result = { - k: self.context.data_registry.get_value(f"alias:{k}") - for k in all_aliases - } - - return ValueMapReadOnly.create_from_values(**result) - - def retrieve_aliases_info(self, **matcher_params) -> ValuesInfo: - """ - Retrieve information about the matching values. - - This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], - but augments each result value instance with additional information that might be useful in frontends. - - 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects - hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any - of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.list_aliases] method - instead. - - Arguments: - matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters - - Returns: - a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value - """ - values = self.list_aliases(**matcher_params) - - infos = ValuesInfo.create_from_instances( - kiara=self.context, instances={str(k): v for k, v in values.items()} - ) - return infos # type: ignore - - def assemble_value_map( - self, - values: Mapping[str, Union[uuid.UUID, None, str, Value, Any]], - values_schema: Union[None, Mapping[str, ValueSchema]] = None, - register_data: bool = False, - reuse_existing_data: bool = False, - ) -> ValueMapReadOnly: - """ - Retrive a [ValueMap][kiara.models.values.value.ValueMap] object from the provided value ids or value links. - - In most cases, this endpoint won't be used by front-ends, it's a fairly low-level method that is - mainly used for internal purposes. If you have a use-case, let me know and I'll improve the docs - if insufficient. - - By default, this method can only use values/datasets that are already registered in *kiara*. If you want to - auto-register 'raw' data, you need to set the 'register_data' flag to 'True', and provide a schema for each of the fields that are not yet registered. - - Arguments: - values: a dictionary with the values in question - values_schema: an optional dictionary with the schema for each of the values that are not yet registered - register_data: whether to allow auto-registration of 'raw' data - reuse_existing_data: whether to reuse existing data with the same hash as the 'raw' data that is being registered - - Returns: - a value map instance - """ - - if register_data: - temp: Dict[str, Union[str, Value, uuid.UUID, None]] = {} - for k, v in values.items(): - - if isinstance(v, (Value, uuid.UUID)): - temp[k] = v - continue - - if not values_schema: - details = "No schema provided." - raise KiaraException( - f"Invalid field name: '{k}' (value: {v}).", details=details - ) - - if k not in values_schema.keys(): - details = "Valid field names: " + ", ".join(values_schema.keys()) - raise KiaraException( - f"Invalid field name: '{k}' (value: {v}).", details=details - ) - - if isinstance(v, str): - - if v.startswith("alias:"): - temp[k] = v - continue - elif v.startswith("archive:"): - temp[k] = v - continue - - try: - v = uuid.UUID(v) - temp[k] = v - continue - except Exception: - if v.startswith("alias:"): # type: ignore - _v = v.replace("alias:", "") # type: ignore - else: - _v = v - - data_type = values_schema[k].type - if data_type != "string" and _v in self.list_aliases(): - temp[k] = f"alias:{_v}" - continue - - if v is None: - temp[k] = None - else: - _v = self.register_data( - data=v, - # data_type=values_schema[k].type, - data_type=values_schema[k], - reuse_existing=reuse_existing_data, - ) - temp[k] = _v - values = temp - return self.context.data_registry.load_values( - values=values, values_schema=values_schema - ) - - def store_value( - self, - value: Union[str, uuid.UUID, Value], - alias: Union[str, Iterable[str], None], - allow_overwrite: bool = True, - store: Union[str, None] = None, - data_store: Union[str, None] = None, - alias_store: Union[str, None] = None, - set_as_store_default: bool = False, - ) -> StoreValueResult: - """ - Store the specified value in a value store. - - If you provide values for the 'data_store' and/or 'alias_store' other than 'default', you need - to make sure those stores are registered with the current context. In most cases, the 'export' endpoint (to be done) will probably be an easier way to export values, which I suspect will - be the main use-case for this endpoint if any of the 'store' arguments where needed. Otherwise, this endpoint is useful to persist values for use in later seperate sessions. - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValueResult' instance that is returned to see if the storing was successful. - - Arguments: - value: the value (or a reference to it) - alias: (Optional) one or several aliases for the value - allow_overwrite: whether to allow overwriting existing aliases - store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten - data_store: the registered name (or archive id as string) of the store to write the data - alias_store: the registered name (or archive id as string) of the store to persist the alias(es)/value_id mapping - set_as_store_default: whether to set the specified store as the default store for the value - """ - if isinstance(alias, str): - alias = [alias] - - value_obj = self.get_value(value) - persisted_data: Union[None, PersistedData] = None - - if not data_store: - data_store = store - if not alias_store: - alias_store = store - - try: - persisted_data = self.context.data_registry.store_value( - value=value_obj, data_store=data_store - ) - if alias: - self.context.alias_registry.register_aliases( - value_obj.value_id, - *alias, - allow_overwrite=allow_overwrite, - alias_store=alias_store, - ) - - if set_as_store_default: - store_instance = self.context.data_registry.get_archive(data_store) - store_instance.set_archive_metadata_value( - DATA_ARCHIVE_DEFAULT_VALUE_MARKER, str(value_obj.value_id) - ) - result = StoreValueResult( - value=value_obj, - aliases=sorted(alias) if alias else [], - error=None, - persisted_data=persisted_data, - ) - except Exception as e: - log_exception(e) - result = StoreValueResult( - value=value_obj, - aliases=sorted(alias) if alias else [], - error=( - str(e) if str(e) else f"Unknown error (type '{type(e).__name__}')." - ), - persisted_data=persisted_data, - ) - - return result - - def store_values( - self, - values: Union[ - str, - Mapping[str, Union[str, uuid.UUID, Value]], - Iterable[Union[str, uuid.UUID, Value]], - ], - alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, - allow_alias_overwrite: bool = True, - store: Union[str, None] = None, - data_store: Union[str, None] = None, - alias_store: Union[str, None] = None, - ) -> StoreValuesResult: - """ - Store multiple values into the (default) kiara value store. - - Convenience method to store multiple values. In a lot of cases you can be more flexible if you - loop over the values on the frontend side, and call the 'store_value' method for each value. But this might be meaningfully slower. This method has the potential to be optimized in the future. - - You have several options to provide the values and aliases you want to store: - - - as a string, in which case the item will be wrapped in a list (see non-mapping iterable below) - - - as a (non-mapping) iterable of value items, those can either be: - - - a value id (as string or uuid) - - a value alias (as string) - - a value instance - - If you do that, then the 'alias_map' argument can either be: - - - 'False', in which case no aliases will be registered - - 'True', in which case all items in the 'values' iterable must be a valid alias, and the alias will be copied without change to the new store - - a 'string', in which case all items in the 'values' iterable also must be a valid alias, and the alias that will be registered in the new store will use the string value as prefix (e.g. 'alias_map' = 'experiment1' and 'values' = ['a', 'b'] will result in the aliases 'experiment1.a' and 'experiment1.b') - - a map that uses the stringi-fied uuid of the value that should get one or several aliases as key, and a list of aliases as values - - You can also use a mapping type (like a dict) for the 'values' argument. In this case, the key is a string, and the value can be: - - - a value id (as string or uuid) - - a value alias (as string) - - a value instance - - In this case, the meaning of the 'alias_map' is as follows: - - - 'False': no aliases will be registered - - 'True': the key in the 'values' argument will be used as alias - - a string: all keys from the 'values' map will be used as alias, prefixed with the value of 'alias_map' - - another map, with a string referring to the key in the 'values' argument as key, and a list of aliases (strings) as value - - Sorry, this is all a bit convoluted, but it's the only way I could think of to make this work for all the requirements I had. In most keases, you'll only have to use 'True' or 'False' here, hopefully. - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful. - - Arguments: - values: an iterable/map of value keys/values - alias_map: a map of value keys aliases - allow_alias_overwrite: whether to allow overwriting existing aliases - store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten - data_store: the registered name (or archive id as string) of the store to write the data - alias_store: the registered name (or archive id as string) of the store to persist the alias(es)/value_id mapping - - Returns: - an object outlining which values (identified by the specified value key or an enumerated index) where stored and how - - """ - - if not data_store: - data_store = store - - if not alias_store: - alias_store = store - - if isinstance(values, str): - values = [values] - - result = {} - if not isinstance(values, Mapping): - if not alias_map: - use_aliases = False - elif alias_map and (alias_map is True or isinstance(alias_map, str)): - - invalid: List[Union[str, uuid.UUID, Value]] = [] - valid: Dict[str, List[str]] = {} - for value in values: - if not isinstance(value, str): - invalid.append(value) - continue - value_id = self.context.alias_registry.find_value_id_for_alias( - alias=value - ) - if value_id is None: - invalid.append(value) - else: - if alias_map is True: - if "#" in value: - new_alias = value.split("#")[1] - else: - new_alias = value - valid.setdefault(str(value_id), []).append(new_alias) - else: - if "#" in value: - new_alias = value.split("#")[1] - else: - new_alias = value - new_alias = f"{alias_map}{new_alias}" - valid.setdefault(str(value_id), []).append(new_alias) - if invalid: - invalid_str = ", ".join((str(x) for x in invalid)) - raise KiaraException( - msg=f"Cannot use auto-aliases with non-mapping iterable, some items are not valid aliases: {invalid_str}" - ) - else: - alias_map = valid - use_aliases = True - else: - use_aliases = True - - for value in values: - - aliases: Set[str] = set() - - value_obj = self.get_value(value) - if use_aliases: - alias_key = str(value_obj.value_id) - alias: Union[str, None] = alias_map.get(alias_key, None) # type: ignore - if alias: - aliases.update(alias) - - store_result = self.store_value( - value=value_obj, - alias=aliases, - allow_overwrite=allow_alias_overwrite, - data_store=data_store, - alias_store=alias_store, - ) - result[str(value_obj.value_id)] = store_result - else: - - for field_name, value in values.items(): - if alias_map is False: - aliases_map: Union[None, Iterable[str]] = None - elif alias_map is True: - aliases_map = [field_name] - elif isinstance(alias_map, str): - aliases_map = [f"{alias_map}.{field_name}"] - else: - # means it's a mapping - _aliases = alias_map.get(field_name) - if _aliases: - aliases_map = list(_aliases) - else: - aliases_map = None - - value_obj = self.get_value(value) - store_result = self.store_value( - value=value_obj, - alias=aliases_map, - allow_overwrite=allow_alias_overwrite, - data_store=data_store, - alias_store=alias_store, - ) - result[field_name] = store_result - - return StoreValuesResult(root=result) - - # ------------------------------------------------------------------------------------------------------------------ - # archive-related methods - - def import_values( - self, - source_archive: Union[str, Path], - values: Union[ - str, - Mapping[str, Union[str, uuid.UUID, Value]], - Iterable[Union[str, uuid.UUID, Value]], - ], - alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, - allow_alias_overwrite: bool = True, - source_registered_name: Union[str, None] = None, - ) -> StoreValuesResult: - """Import one or several values from an external kiara archive, along with their aliases (optional). - - For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, - and there are several ways to use them which is information I don't want to duplicate. - - If you provide aliases in the 'values' parameter, the aliases must be available in the external archive. - - Currently, this only works with an external archive file, not with an archive that is registered into the context. - This will probably be added later on, let me know if there is demand, then I'll prioritize. - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful. - - # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. - - Arguments: - source_archive: the name of the archive to store the values into - values: an iterable/map of value keys/values - alias_map: a map of value keys aliases - allow_alias_overwrite: whether to allow overwriting existing aliases - source_registered_name: the name to register the archive under in the context - """ - - if source_archive in [None, DEFAULT_STORE_MARKER]: - raise KiaraException( - "You cannot use the default store as source for this operation." - ) - - if alias_map is True: - pass - elif alias_map is False: - pass - elif isinstance(alias_map, str): - pass - elif isinstance(alias_map, Mapping): - pass - else: - raise KiaraException( - f"Invalid type for 'alias_map' argument: {type(alias_map)}." - ) - - source_archive_ref = self.register_archive( - archive=source_archive, # type: ignore - registered_name=source_registered_name, - create_if_not_exists=False, - allow_write_access=False, - existing_ok=True, - ) - - value_ids: Set[uuid.UUID] = set() - aliases: Set[str] = set() - - if isinstance(values, str): - values = [values] - - if not isinstance(values, Mapping): - # means we have a list of value ids/aliases - for value in values: - if isinstance(value, uuid.UUID): - value_ids.add(value) - elif isinstance(value, str): - try: - _value = uuid.UUID(value) - value_ids.add(_value) - except Exception: - aliases.add(value) - else: - raise NotImplementedError("Not implemented yet.") - - new_values: Dict[str, Union[uuid.UUID, str]] = {} - idx = 0 - for value_id in value_ids: - field = f"field_{idx}" - idx += 1 - new_values[field] = value_id - - new_alias_map = {} - for alias in aliases: - field = f"field_{idx}" - idx += 1 - new_values[field] = f"{source_archive_ref}#{alias}" - if alias_map is False: - pass - elif alias_map is True: - new_alias_map[field] = [f"{alias}"] - elif isinstance(alias_map, str): - new_alias_map[field] = [f"{alias_map}{alias}"] - else: - # means its a dict - if alias in alias_map.keys(): - for a in alias_map[alias]: - new_alias_map.setdefault(field, []).append(a) - - result = self.store_values( - values=new_values, - alias_map=new_alias_map, - allow_alias_overwrite=allow_alias_overwrite, - ) - return result - - def export_values( - self, - target_archive: Union[str, Path], - values: Union[ - str, - Mapping[str, Union[str, uuid.UUID, Value]], - Iterable[Union[str, uuid.UUID, Value]], - ], - alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, - allow_alias_overwrite: bool = True, - target_registered_name: Union[str, None] = None, - append: bool = False, - target_store_params: Union[None, Mapping[str, Any]] = None, - ) -> StoreValuesResult: - """Store one or several values along with (optional) aliases into a kiara archive. - - For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, - and there are several ways to use them which is information I don't want to duplicate. - - Currently, this only works with an external archive file, not with an archive that is registered into the context. - This will probably be added later on, let me know if there is demand, then I'll prioritize. - - 'target_store_params' is used if the archive does not exist yet. The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: - - - zstd: zstd compression (default) -- fairly fast, and good compression - - none: no compression - - LZMA: LZMA compression -- very slow, but very good compression - - LZ4: LZ4 compression -- very fast, but not as good compression as zstd - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful. - - # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. - - Arguments: - target_store: the name of the archive to store the values into - values: an iterable/map of value keys/values - alias_map: a map of value keys aliases - allow_alias_overwrite: whether to allow overwriting existing aliases - target_registered_name: the name to register the archive under in the context - append: whether to append to an existing archive - target_store_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet - - """ - - if target_archive in [None, DEFAULT_STORE_MARKER]: - raise KiaraException( - "You cannot use the default store as target for this operation." - ) - - if target_store_params is None: - target_store_params = {} - - target_archive_ref = self.register_archive( - archive=target_archive, # type: ignore - registered_name=target_registered_name, - create_if_not_exists=True, - allow_write_access=True, - existing_ok=True if append else False, - **target_store_params, - ) - - result = self.store_values( - values=values, - alias_map=alias_map, - allow_alias_overwrite=allow_alias_overwrite, - store=target_archive_ref, - ) - return result - - def register_archive( - self, - archive: Union[str, Path, "KiArchive"], - allow_write_access: bool = False, - registered_name: Union[str, None] = None, - create_if_not_exists: bool = True, - existing_ok: bool = True, - **create_params: Any, - ) -> str: - """Register a kiarchive with the current context. - - In most cases, this will be used to 'load' an existing kiarchive file and attach it to the current context. - If the file does not exist, one will be created, with the filename (without '.kiarchive' suffix) as the archive name if not specified. - - In the future this might also take a URL, but for now only local files are supported. - - # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. - - Arguments: - archive: the uri of the archive (file path), or a [Kiarchive][kiara.interfaces.python_api.models.archive.Kiarchive] instance - allow_write_access: whether to allow write access to the archive - registered_name: the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint, if not provided, it will be auto-determined from the file name - create_if_not_exists: if the file does not exist, create it. If this is 'False', an exception will be raised if the file does not exist. - existing_ok: whether the file is allowed to exist already, if 'False', an exception will be raised if the file exists - create_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet - - Returns: - the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint - """ - from kiara.interfaces.python_api.models.archive import KiArchive - - if not existing_ok and not create_if_not_exists: - raise KiaraException( - "Both 'existing_ok' and 'create_if_not_exists' cannot be 'False' at the same time." - ) - - if isinstance(archive, str): - archive = Path(archive) - - if isinstance(archive, Path): - - if not archive.name.endswith(".kiarchive"): - archive = archive.parent / f"{archive.name}.kiarchive" - - if archive.exists(): - if not existing_ok: - raise KiaraException( - f"Archive file '{archive.as_posix()}' already exists." - ) - archive = KiArchive.load_kiarchive( - kiara=self.context, - path=archive, - archive_name=registered_name, - allow_write_access=allow_write_access, - ) - log_message("archive.loaded", archive_name=archive.archive_name) - else: - if not create_if_not_exists: - raise KiaraException( - f"Archive file '{archive.as_posix()}' does not exist." - ) - kiarchive_alias = archive.name - if kiarchive_alias.endswith(".kiarchive"): - kiarchive_alias = kiarchive_alias[:-10] - - if "compression" not in create_params.keys(): - create_params["compression"] = DEFAULT_CHUNK_COMPRESSION - - archive = KiArchive.create_kiarchive( - kiara=self.context, - kiarchive_uri=archive.as_posix(), - allow_existing=False, - archive_name=kiarchive_alias, - allow_write_access=allow_write_access, - **create_params, - ) - log_message("archive.created", archive_name=archive.archive_name) - - else: - raise NotImplementedError("Only local files are supported for now.") - - data_alias = self.context.register_external_archive( - archive.data_archive, - allow_write_access=allow_write_access, - ) - - alias_alias = self.context.register_external_archive( - archive.alias_archive, allow_write_access=allow_write_access - ) - assert data_alias["data"] == alias_alias["alias"] - assert archive.archive_name == data_alias["data"] - - return archive.archive_name - - def set_archive_metadata_value( - self, - archive: Union[str, uuid.UUID], - key: str, - value: Any, - archive_type: Literal["data", "alias"] = "data", - ) -> None: - """Add metadata to an archive. - - Note that this is different to adding metadata to a context, since it is attached directly - to a special section of the archive itself. - """ - - if archive_type == "data": - _archive: Union[ - None, KiaraArchive - ] = self.context.data_registry.get_archive(archive) - if _archive is None: - raise KiaraException(f"Archive '{archive}' does not exist.") - _archive.set_archive_metadata_value(key, value) - elif archive_type == "alias": - _archive = self.context.alias_registry.get_archive(archive) - if _archive is None: - raise KiaraException(f"Archive '{archive}' does not exist.") - _archive.set_archive_metadata_value(key, value) - else: - raise KiaraException( - f"Invalid archive type: {archive_type}. Valid types are: 'data', 'alias'." - ) - - def retrieve_archive_info( - self, archive: Union[str, "KiArchive"] - ) -> "KiArchiveInfo": - """Retrieve information about an archive at the specified local path - - Currently, this only works with an external archive file, not with an archive that is registered into the context. - This will probably be added later on, let me know if there is demand, then I'll prioritize. - - # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. - - Arguments: - archive: the uri of the archive (file path) - - Returns: - a [KiarchiveInfo][kiara.interfaces.python_api.models.archive.KiarchiveInfo] instance, containing details about the archive - """ - - from kiara.interfaces.python_api.models.archive import KiArchive - from kiara.models.archives import KiArchiveInfo - - if not isinstance(archive, KiArchive): - archive = KiArchive.load_kiarchive(kiara=self.context, path=archive) - - kiarchive_info = KiArchiveInfo.create_from_instance( - kiara=self.context, instance=archive - ) - return kiarchive_info - - def export_archive( - self, - target_archive: Union[str, Path], - target_registered_name: Union[str, None] = None, - append: bool = False, - no_aliases: bool = False, - target_store_params: Union[None, Mapping[str, Any]] = None, - ) -> StoreValuesResult: - """Export all data from the default store in your context into the specfied archive path. - - The target archives will be registered into the context, either using the provided registered_name, or the name - will be auto-determined from the archive metadata. - - Currently, this only works with an external archive file, not with an archive that is already registered into the context. - This will be added later on. - - Also, currently you can only export all data from the default store, there is no way to select only a sub-set. This will - also be supported later on. - - The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: - - - zstd: zstd compression (default) -- fairly fast, and good compression - - none: no compression - - LZMA: LZMA compression -- very slow, but very good compression - - LZ4: LZ4 compression -- very fast, but not as good compression as zstd - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful - - Arguments: - target_archive: the registered_name or uri of the target archive - target_registered_name: the name/alias that the archive should be registered in the context (if necessary) - append: whether to append to an existing archive or error out if the target already exists - no_aliases: whether to skip importing aliases - target_store_params: additional parameters to pass to the 'create_kiarchive' method if the target file does not exist yet - - Returns: - an object outlining which values (identified by the specified value key or an enumerated index) where stored and how - """ - - result = self.copy_archive( - source_archive=DEFAULT_STORE_MARKER, - target_archive=target_archive, - target_registered_name=target_registered_name, - append=append, - target_store_params=target_store_params, - no_aliases=no_aliases, - ) - return result - - def import_archive( - self, - source_archive: Union[str, Path], - source_registered_name: Union[str, None] = None, - no_aliases: bool = False, - ) -> StoreValuesResult: - """Import all data from the specified archive into the current contexts default data & alias store. - - The source target will be registered into the context, either using the provided registered_name, otherwise the name - will be auto-determined from the archive metadata. - - Currently, this only works with an external archive file, not with an archive that is registered into the context. - This will be added later on. - - Also, currently you can only import all data into the default store, there is no way to select only a sub-set. This will - also be supported later on. - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful - - Arguments: - source_archive: the registered_name or uri of the source archive - source_registered_name: the name/alias that the archive should be registered in the context (if necessary) - no_aliases: whether to skip importing aliases - - Returns: - an object outlining which values (identified by the specified value key or an enumerated index) where stored and how - - """ - - result = self.copy_archive( - source_archive=source_archive, - target_archive=DEFAULT_STORE_MARKER, - source_registered_name=source_registered_name, - no_aliases=no_aliases, - ) - return result - - def copy_archive( - self, - source_archive: Union[None, str, Path], - target_archive: Union[None, str, Path] = None, - source_registered_name: Union[str, None] = None, - target_registered_name: Union[str, None] = None, - append: bool = False, - no_aliases: bool = False, - target_store_params: Union[None, Mapping[str, Any]] = None, - ) -> StoreValuesResult: - """Import all data from the specified archive into the current context. - - The archives will be registered into the context, either using the provided registered_name, otherwise the name - will be auto-determined from the archive metadata. - - Currently, this only works with an external archive file, not with an archive that is registered into the context. - This will be added later on. - - The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: - - - zstd: zstd compression (default) -- fairly fast, and good compression - - none: no compression - - LZMA: LZMA compression -- very slow, but very good compression - - LZ4: LZ4 compression -- very fast, but not as good compression as zstd - - This method does not raise an error if the storing of the value fails, so you have to investigate the - 'StoreValuesResult' instance that is returned to see if the storing was successful - - Arguments: - source_archive: the registered_name or uri of the source archive, if None, the context default data/alias store will be used - target_archive: the registered_name or uri of the target archive, defaults to the context default data/alias store - source_registered_name: the name/alias that the archive should be registered in the context (if necessary) - target_registered_name: the name/alias that the archive should be registered in the context (if necessary) - append: whether to append to an existing archive or error out if the target already exists - no_aliases: whether to skip importing aliases - target_store_params: additional parameters to pass to the 'create_kiarchive' method if the target file does not exist yet - - Returns: - an object outlining which values (identified by the specified value key or an enumerated index) where stored and how - - """ - - if source_archive in [None, DEFAULT_STORE_MARKER]: - source_archive_ref = DEFAULT_STORE_MARKER - else: - source_archive_ref = self.register_archive( - archive=source_archive, # type: ignore - registered_name=source_registered_name, - create_if_not_exists=False, - existing_ok=True, - ) - - if target_archive in [None, DEFAULT_STORE_MARKER]: - target_archive_ref = DEFAULT_STORE_MARKER - else: - if target_store_params is None: - target_store_params = {} - target_archive_ref = self.register_archive( - archive=target_archive, # type: ignore - registered_name=target_registered_name, - create_if_not_exists=True, - allow_write_access=True, - existing_ok=True if append else False, - **target_store_params, - ) - - if source_archive_ref == target_archive_ref: - raise KiaraException( - f"Source and target archive cannot be the same: {source_archive_ref} != {target_archive_ref}" - ) - - source_values = self.list_values( - in_data_archives=[source_archive_ref], allow_internal=True, has_alias=False - ).values() - - if not no_aliases: - aliases = self.list_aliases(in_data_archives=[source_archive_ref]) - alias_map: Union[bool, Dict[str, List[str]]] = {} - for alias, value in aliases.items(): - - if source_archive_ref != DEFAULT_STORE_MARKER: - # TODO: maybe add a matcher arg to the list_aliases endpoint - if not alias.startswith(f"{source_archive_ref}#"): - continue - alias_map.setdefault(str(value.value_id), []).append( # type: ignore - alias[len(source_archive_ref) + 1 :] - ) - else: - if "#" in alias: - continue - alias_map.setdefault(str(value.value_id), []).append(alias) # type: ignore - else: - alias_map = False - - result = self.store_values( - source_values, alias_map=alias_map, store=target_archive_ref - ) - return result - - # ------------------------------------------------------------------------------------------------------------------ - # operation-related methods - - def get_operation_type(self, op_type: Union[str, Type[OP_TYPE]]) -> OperationType: - """Get the management object for the specified operation type.""" - return self.context.operation_registry.get_operation_type(op_type=op_type) - - def retrieve_operation_type_info( - self, op_type: Union[str, Type[OP_TYPE]] - ) -> OperationTypeInfo: - """Get an info object for the specified operation type.""" - _op_type = self.get_operation_type(op_type=op_type) - return OperationTypeInfo.create_from_type_class( - kiara=self.context, type_cls=_op_type.__class__ - ) - - def find_operation_id( - self, module_type: str, module_config: Union[None, Mapping[str, Any]] = None - ) -> Union[None, str]: - """ - Try to find the registered operation id for the specified module type and configuration. - - Arguments: - module_type: the module type - module_config: the module configuration - - Returns: - the registered operation id, if found, or None - """ - manifest = self.context.create_manifest( - module_or_operation=module_type, config=module_config - ) - return self.context.operation_registry.find_operation_id(manifest=manifest) - - def assemble_filter_pipeline_config( - self, - data_type: str, - filters: Union[str, Iterable[str], Mapping[str, str]], - endpoint: Union[None, Manifest, str] = None, - endpoint_input_field: Union[str, None] = None, - endpoint_step_id: Union[str, None] = None, - extra_input_aliases: Union[None, Mapping[str, str]] = None, - extra_output_aliases: Union[None, Mapping[str, str]] = None, - ) -> "PipelineConfig": - """ - Assemble a (pipeline) module config to filter values of a specific data type. - - NOTE: this is a preliminary endpoint, and might go away in the future. If you have a need for this - functionality, please let me know your requirements and we can work on fleshing this out. - - Optionally, a module that uses the filtered dataset as input can be specified. - - # TODO: document filter names - For the 'filters' argument, the accepted inputs are: - - a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter - - a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically - - a map of string pairs: the keys are step ids, the values operation ids or filter names - - Arguments: - data_type: the type of the data to filter - filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping) - endpoint: optional module to put as last step in the created pipeline - endpoing_input_field: field name of the input that will receive the filtered value - endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided) - extra_input_aliases: extra output aliases to add to the pipeline config - extra_output_aliases: extra output aliases to add to the pipeline config - - Returns: - the (pipeline) module configuration of the filter pipeline - """ - filter_op_type: FilterOperationType = self.context.operation_registry.get_operation_type("filter") # type: ignore - pipeline_config = filter_op_type.assemble_filter_pipeline_config( - data_type=data_type, - filters=filters, - endpoint=endpoint, - endpoint_input_field=endpoint_input_field, - endpoint_step_id=endpoint_step_id, - extra_input_aliases=extra_input_aliases, - extra_output_aliases=extra_output_aliases, - ) - - return pipeline_config - - # ------------------------------------------------------------------------------------------------------------------ - # metadata-related methods - - def register_metadata( - self, key: str, value: str, force: bool = False, store: Union[str, None] = None - ) -> uuid.UUID: - """Register a comment into the specified metadata store. - - Currently, this allows you to store comments within the default kiara context. You can use any string, - as key, for example a stringified `job_id`, or `value_id`, or any other string that makes sense in - the context you are using this in. - - If you use the store argument, the store needs to be mounted into the current *kiara* context. For now, - you can ignore this and not provide any value here, since this area is still in flux. If you need - to store a metadata item into an external context, and you can't figure out how to do it, - let me know. - - Note: this is preliminary and subject to change based on your input, so please provide your thoughts - - Arguments: - key: the key under which to store the metadata (can be anything you can think of) - value: the comment you want to store - force: overwrite the existing value if its key already exists in the store - store: the store to use, by default the context default is used - - Returns: - a globally unique identifier for the metadata item - """ - - if not value: - raise KiaraException("Cannot store empty metadata item.") - - from kiara.models.metadata import CommentMetadata - - item = CommentMetadata(comment=value) - - return self.context.metadata_registry.register_metadata_item( - key=key, item=item, force=force, store=store - ) - - # ------------------------------------------------------------------------------------------------------------------ - # render-related methods - - def retrieve_renderer_infos( - self, source_type: Union[str, None] = None, target_type: Union[str, None] = None - ) -> RendererInfos: - """Retrieve information about the available renderers. - - Note: this is preliminary and mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. - - Arguments: - source_type: the type of the item to render (optional filter) - target_type: the type/profile of the rendered result (optional filter) - - Returns: - a wrapper object containing the items as dictionary with renderer alias as key, and [kiara.interfaces.python_api.models.info.RendererInfo] as value - - """ - - if not source_type and not target_type: - renderers = self.context.render_registry.registered_renderers - elif source_type and not target_type: - renderers = self.context.render_registry.retrieve_renderers_for_source_type( - source_type=source_type - ) - elif target_type and not source_type: - raise KiaraException(msg="Cannot retrieve renderers for target type only.") - else: - renderers = self.context.render_registry.retrieve_renderers_for_source_target_combination( - source_type=source_type, target_type=target_type # type: ignore - ) - - group = {k.get_renderer_alias(): k for k in renderers} - infos = RendererInfos.create_from_instances(kiara=self.context, instances=group) - return infos # type: ignore - - def retrieve_renderers_for(self, source_type: str) -> List[KiaraRenderer]: - """Retrieve available renderer instances for a specific data type. - - Note: this is not preliminary, and, mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. - """ - - return self.context.render_registry.retrieve_renderers_for_source_type( - source_type=source_type - ) - - def render( - self, - item: Any, - source_type: str, - target_type: str, - render_config: Union[Mapping[str, Any], None] = None, - ) -> Any: - """Render an internal instance of a supported source type into one of the supported target types. - - Note: this is not preliminary, and, mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. - - To find out the supported source/target combinations, you can use the kiara cli: - - ``` - kiara render list-renderers - ``` - or, for a filtered list: - ```` - kiara render --source-type pipeline list-renderers - ``` - - What Python types are actually supported for the 'item' argument depends on the source_type of the renderer you are calling, for example if that is a pipeline, most of the ways to specify a pipeline would be supported (operation_id, pipeline file, etc.). This might need more documentation, let me know what exactly is needed in a support ticket and I'll add that information. - - Arguments: - item: the item to render - source_type: the type of the item to render - target_type: the type/profile of the rendered result - render_config: optional configuration, depends on the renderer that is called - - """ - - registry = self.context.render_registry - result = registry.render( - item=item, - source_type=source_type, - target_type=target_type, - render_config=render_config, - ) - return result - - def assemble_render_pipeline( - self, - data_type: str, - target_format: Union[str, Iterable[str]] = "string", - filters: Union[None, str, Iterable[str], Mapping[str, str]] = None, - use_pretty_print: bool = False, - ) -> Operation: - """ - Create a manifest describing a transformation that renders a value of the specified data type in the target format. - - NOTE: this is a preliminary endpoint, don't use in anger yet. - - If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches - the value type of the source value, and the provided target format. - - Arguments: - value: the value (or value id) - target_format: the format into which to render the value - filters: a list of filters to apply to the value before rendering it - use_pretty_print: if True, use a 'pretty_print' operation instead of 'render_value' - - Returns: - the manifest for the transformation - """ - if data_type not in self.context.data_type_names: - raise DataTypeUnknownException(data_type=data_type) - - if use_pretty_print: - pretty_print_op_type: PrettyPrintOperationType = ( - self.context.operation_registry.get_operation_type("pretty_print") - ) # type: ignore - ops = pretty_print_op_type.get_target_types_for(data_type) - else: - render_op_type: ( - RenderValueOperationType - ) = self.context.operation_registry.get_operation_type( - # type: ignore - "render_value" - ) # type: ignore - ops = render_op_type.get_render_operations_for_source_type(data_type) - - if isinstance(target_format, str): - target_format = [target_format] - - match = None - for _target_type in target_format: - if _target_type not in ops.keys(): - continue - match = ops[_target_type] - break - - if not match: - if not ops: - msg = f"No render operations registered for source type '{data_type}'." - else: - msg = f"Registered target types for source type '{data_type}': {', '.join(ops.keys())}." - raise Exception( - f"No render operation for source type '{data_type}' to target type(s) registered: '{', '.join(target_format)}'. {msg}" - ) - - if filters: - # filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore - endpoint = Manifest( - module_type=match.module_type, module_config=match.module_config - ) - extra_input_aliases = {"render_value.render_config": "render_config"} - extra_output_aliases = { - "render_value.render_value_result": "render_value_result" - } - pipeline_config = self.assemble_filter_pipeline_config( - data_type=data_type, - filters=filters, - endpoint=endpoint, - endpoint_input_field="value", - endpoint_step_id="render_value", - extra_input_aliases=extra_input_aliases, - extra_output_aliases=extra_output_aliases, - ) - manifest = Manifest( - module_type="pipeline", module_config=pipeline_config.model_dump() - ) - module = self.context.module_registry.create_module(manifest=manifest) - operation = Operation.create_from_module(module, doc=pipeline_config.doc) - else: - operation = match - - return operation - - # ------------------------------------------------------------------------------------------------------------------ - # job-related methods - def queue_manifest( - self, - manifest: Manifest, - inputs: Union[None, Mapping[str, Any]] = None, - **job_metadata: Any, - ) -> uuid.UUID: - """ - Queue a job using the provided manifest to describe the module and config that should be executed. - - You probably want to use 'queue_job' instead. - - Arguments: - manifest: the manifest - inputs: the job inputs (can be either references to values, or raw inputs - - Returns: - a result value map instance - """ - - if self.context.runtime_config.runtime_profile == "dharpa": - if not job_metadata: - raise Exception( - "No job metadata provided. You need to provide a 'comment' argument when running your job." - ) - - if "comment" not in job_metadata.keys(): - raise KiaraException(msg="You need to provide a 'comment' for the job.") - - save_values = True - else: - save_values = False - - if inputs is None: - inputs = {} - - job_config = self.context.job_registry.prepare_job_config( - manifest=manifest, inputs=inputs - ) - - job_id = self.context.job_registry.execute_job( - job_config=job_config, wait=False, auto_save_result=save_values - ) - - if job_metadata: - self.context.metadata_registry.register_job_metadata_items( - job_id=job_id, items=job_metadata - ) - - return job_id - - def run_manifest( - self, - manifest: Manifest, - inputs: Union[None, Mapping[str, Any]] = None, - **job_metadata: Any, - ) -> ValueMapReadOnly: - """ - Run a job using the provided manifest to describe the module and config that should be executed. - - You probably want to use 'run_job' instead. - - Arguments: - manifest: the manifest - inputs: the job inputs (can be either references to values, or raw inputs - job_metadata: additional metadata to store with the job - - Returns: - a result value map instance - """ - job_id = self.queue_manifest(manifest=manifest, inputs=inputs, **job_metadata) - return self.context.job_registry.retrieve_result(job_id=job_id) - - def queue_job( - self, - operation: Union[str, Path, Manifest, OperationInfo, JobDesc], - inputs: Union[Mapping[str, Any], None], - operation_config: Union[None, Mapping[str, Any]] = None, - **job_metadata: Any, - ) -> uuid.UUID: - """ - Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. - - This is a convenience method that auto-detects what is meant by the 'operation' string input argument. - - If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute - set, it will be ignored, so you'll have to store any results manually. - - Arguments: - operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. - inputs: the operation inputs - operation_config: the (optional) module config in case 'operation' is a module name - job_metadata: additional metadata to store with the job - - Returns: - the queued job id - """ - - if inputs is None: - inputs = {} - - if isinstance(operation, str): - if os.path.isfile(operation): - job_path = Path(operation) - if not job_path.is_file(): - raise Exception( - f"Can't queue job from file '{job_path.as_posix()}': file does not exist/not a file." - ) - - op_data = get_data_from_file(job_path) - if isinstance(op_data, Mapping) and "operation" in op_data.keys(): - try: - repl_dict: Dict[str, Any] = { - "this_dir": job_path.parent.as_posix() - } - job_data = replace_var_names_in_obj( - op_data, repl_dict=repl_dict - ) - job_data["job_alias"] = job_path.stem - job_desc = JobDesc(**job_data) - _operation: Union[Manifest, str] = job_desc.get_operation( - kiara_api=self - ) - if job_desc.inputs: - _inputs = dict(job_desc.inputs) - _inputs.update(inputs) - inputs = _inputs - except Exception as e: - raise KiaraException( - f"Failed to parse job description file: {operation}", - parent=e, - ) - else: - _operation = job_path.as_posix() - else: - _operation = operation - elif isinstance(operation, Path): - if not operation.is_file(): - raise Exception( - f"Can't queue job from file '{operation.as_posix()}': file does not exist/not a file." - ) - _operation = operation.as_posix() - elif isinstance(operation, OperationInfo): - _operation = operation.operation - elif isinstance(operation, JobDesc): - if operation_config: - raise KiaraException( - "Specifying 'operation_config' when operation is a job_desc is invalid." - ) - _operation = operation.get_operation(kiara_api=self) - if operation.inputs: - _inputs = dict(operation.inputs) - _inputs.update(inputs) - inputs = _inputs - else: - _operation = operation - - if not isinstance(_operation, Manifest): - manifest: Manifest = create_operation( - module_or_operation=_operation, - operation_config=operation_config, - kiara=self.context, - ) - else: - manifest = _operation - - job_id = self.queue_manifest(manifest=manifest, inputs=inputs, **job_metadata) - - return job_id - - def run_job( - self, - operation: Union[str, Path, Manifest, OperationInfo, JobDesc], - inputs: Union[None, Mapping[str, Any]] = None, - operation_config: Union[None, Mapping[str, Any]] = None, - **job_metadata, - ) -> ValueMapReadOnly: - """ - Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. - - This is a convenience method that auto-detects what is meant by the 'operation' string input argument. - - In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead, - since this is a blocking operation. - - If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute - set, it will be ignored, so you'll have to store any results manually. - - Arguments: - operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. - inputs: the operation inputs - operation_config: the (optional) module config in case 'operation' is a module name - **job_metadata: additional metadata to store with the job - - Returns: - the job result value map - - """ - if inputs is None: - inputs = {} - - job_id = self.queue_job( - operation=operation, - inputs=inputs, - operation_config=operation_config, - **job_metadata, - ) - return self.context.job_registry.retrieve_result(job_id=job_id) - - def get_job(self, job_id: Union[str, uuid.UUID]) -> "ActiveJob": - """Retrieve the status of the job with the provided id.""" - if isinstance(job_id, str): - job_id = uuid.UUID(job_id) - - job_status = self.context.job_registry.get_job(job_id=job_id) - return job_status - - def get_job_result(self, job_id: Union[str, uuid.UUID]) -> ValueMapReadOnly: - """Retrieve the result(s) of the specified job.""" - if isinstance(job_id, str): - job_id = uuid.UUID(job_id) - - result = self.context.job_registry.retrieve_result(job_id=job_id) - return result - - def list_all_job_record_ids(self) -> List[uuid.UUID]: - """List all available job ids in this kiara context, ordered from newest to oldest, including internal jobs. - - This should be faster than `list_job_record_ids` with equivalent parameters, because no filtering - needs to be done. - """ - - job_ids = self.context.job_registry.retrieve_all_job_record_ids() - return job_ids - - def list_job_record_ids(self, **matcher_params) -> List[uuid.UUID]: - """List all available job ids in this kiara context, ordered from newest to oldest. - - You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example - does not return jobs marked as 'internal'. - - Arguments: - matcher_params: additional parameters to pass to the job matcher - - Returns: - a list of job ids, ordered from latest to earliest - """ - - job_ids = list(self.list_job_records(**matcher_params).keys()) - return job_ids - - def list_all_job_records(self) -> Mapping[uuid.UUID, "JobRecord"]: - """List all available job records in this kiara context, ordered from newest to oldest, including internal jobs. - - This should be faster than `list_job_records` with equivalent parameters, because no filtering - needs to be done. - """ - - job_records = self.context.job_registry.retrieve_all_job_records() - return job_records - - def list_job_records(self, **matcher_params) -> Mapping[uuid.UUID, "JobRecord"]: - """List all available job ids in this kiara context, ordered from newest to oldest. - - You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example - does not return jobs marked as 'internal'. - - You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. - - Arguments: - matcher_params: additional parameters to pass to the job matcher - - Returns: - a list of job details, ordered from latest to earliest - - """ - - from kiara.models.module.jobs import JobMatcher - - matcher = JobMatcher(**matcher_params) - job_records = self.context.job_registry.find_job_records(matcher=matcher) - - return job_records - - def get_job_record(self, job_id: Union[str, uuid.UUID]) -> Union["JobRecord", None]: - """Retrieve the detailed job record for the specified job id. - - If no job can be found, 'None' is returned. - """ - - if isinstance(job_id, str): - job_id = uuid.UUID(job_id) - - job_record = self.context.job_registry.get_job_record(job_id=job_id) - return job_record - - def get_job_comment(self, job_id: Union[str, uuid.UUID]) -> Union[str, None]: - """Retrieve the comment for the specified job. - - Returns 'None' if the job_id does not exist, or the job does not have a comment attached to it. - - Arguments: - job_id: the job id - - Returns: - the comment as string, or None - """ - - from kiara.models.metadata import CommentMetadata - - if isinstance(job_id, str): - job_id = uuid.UUID(job_id) - - metadata: Union[ - None, CommentMetadata - ] = self.context.metadata_registry.retrieve_job_metadata_item( # type: ignore - job_id=job_id, key="comment" - ) - - if not metadata: - return None - - if not isinstance(metadata, CommentMetadata): - raise KiaraException( - msg=f"Metadata item 'comment' for job '{job_id}' is not a comment." - ) - return metadata.comment - - def render_value( - self, - value: Union[str, uuid.UUID, Value], - target_format: Union[str, Iterable[str]] = "string", - filters: Union[None, Iterable[str], Mapping[str, str]] = None, - render_config: Union[Mapping[str, str], None] = None, - add_root_scenes: bool = True, - use_pretty_print: bool = False, - ) -> RenderValueResult: - """ - Render a value in the specified target format. - - NOTE: this is a preliminary endpoint, don't use in anger yet. - - If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches - the value type of the source value, and the provided target format. - - Arguments: - value: the value (or value id) - target_format: the format into which to render the value - filters: an (optional) list of filters - render_config: manifest specific render configuration - add_root_scenes: add root scenes to the result - use_pretty_print: use 'pretty_print' operation instead of 'render_value' - - Returns: - the rendered value data, and any related scenes, if applicable - """ - _value = self.get_value(value) - try: - render_operation: Union[None, Operation] = self.assemble_render_pipeline( - data_type=_value.data_type_name, - target_format=target_format, - filters=filters, - use_pretty_print=use_pretty_print, - ) - - except Exception as e: - - log_message( - "create_render_pipeline.failure", - source_type=_value.data_type_name, - target_format=target_format, - error=e, - ) - - if use_pretty_print: - pretty_print_ops: PrettyPrintOperationType = self.context.operation_registry.get_operation_type("pretty_print") # type: ignore - if not isinstance(target_format, str): - raise NotImplementedError( - "Can't handle multiple target formats for 'render_value' yet." - ) - render_operation = ( - pretty_print_ops.get_operation_for_render_combination( - source_type="any", target_type=target_format - ) - ) - else: - render_ops: RenderValueOperationType = self.context.operation_registry.get_operation_type("render_value") # type: ignore - if not isinstance(target_format, str): - raise NotImplementedError( - "Can't handle multiple target formats for 'render_value' yet." - ) - render_operation = render_ops.get_render_operation( - source_type="any", target_type=target_format - ) - - if render_operation is None: - raise Exception( - f"Could not find render operation for value '{_value.value_id}', type: {_value.value_schema.type}" - ) - - if render_config and "render_config" in render_config.keys(): - # raise NotImplementedError() - # TODO: is this necessary? - render_config = render_config["render_config"] # type: ignore - # manifest_hash = render_config["manifest_hash"] - # if manifest_hash != render_operation.manifest_hash: - # raise NotImplementedError( - # "Using a non-default render operation is not supported (yet)." - # ) - # render_config = render_config["render_config"] - - if render_config is None: - render_config = {} - else: - render_config = dict(render_config) - - # render_type = render_config.pop("render_type", None) - # if not render_type or render_type == "data": - # pass - # elif render_type == "metadata": - # pass - # elif render_type == "properties": - # pass - # elif render_type == "lineage": - # pass - - result = render_operation.run( - kiara=self.context, - inputs={"value": _value, "render_config": render_config}, - ) - - if use_pretty_print: - render_result: Value = result["rendered_value"] - value_render_data: RenderValueResult = render_result.data - else: - render_result = result["render_value_result"] - - if render_result.data_type_name != "render_value_result": - raise Exception( - f"Invalid result type for render operation: {render_result.data_type_name}" - ) - - value_render_data = render_result.data # type: ignore - - return value_render_data - - # ------------------------------------------------------------------------------------------------------------------ - # workflow-related methods - # all of the workflow-related methods are provisional experiments, so don't rely on them to be availale long term - - def list_workflow_ids(self) -> List[uuid.UUID]: - """List all available workflow ids. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - return list(self.context.workflow_registry.all_workflow_ids) - - def list_workflow_alias_names(self) -> List[str]: - """ "List all available workflow aliases. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - return list(self.context.workflow_registry.workflow_aliases.keys()) - - def get_workflow( - self, workflow: Union[str, uuid.UUID], create_if_necessary: bool = True - ) -> "Workflow": - """Retrieve the workflow instance with the specified id or alias. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - no_such_alias: bool = False - workflow_id: Union[uuid.UUID, None] = None - workflow_alias: Union[str, None] = None - - if isinstance(workflow, str): - try: - workflow_id = uuid.UUID(workflow) - except Exception: - workflow_alias = workflow - try: - workflow_id = self.context.workflow_registry.get_workflow_id( - workflow_alias=workflow - ) - except NoSuchWorkflowException: - no_such_alias = True - else: - workflow_id = workflow - - if workflow_id is None: - raise Exception(f"Can't retrieve workflow for: {workflow}") - - if workflow_id in self._workflow_cache.keys(): - return self._workflow_cache[workflow_id] - - if workflow_id is None and not create_if_necessary: - if not no_such_alias: - msg = f"No workflow with id '{workflow}' registered." - else: - msg = f"No workflow with alias '{workflow}' registered." - - raise NoSuchWorkflowException(workflow=workflow, msg=msg) - - if workflow_id: - # workflow_metadata = self.context.workflow_registry.get_workflow_metadata( - # workflow=workflow_id - # ) - workflow_obj = Workflow(kiara=self.context, workflow=workflow_id) - self._workflow_cache[workflow_obj.workflow_id] = workflow_obj - else: - # means we need to create it - workflow_obj = self.create_workflow(workflow_alias=workflow_alias) - - return workflow_obj - - def retrieve_workflow_info( - self, workflow: Union[str, uuid.UUID, "Workflow"] - ) -> WorkflowInfo: - """Retrieve information about the specified workflow. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - - from kiara.interfaces.python_api.workflow import Workflow - - if isinstance(workflow, Workflow): - _workflow: Workflow = workflow - else: - _workflow = self.get_workflow(workflow) - - return WorkflowInfo.create_from_workflow(workflow=_workflow) - - def list_workflows(self, **matcher_params) -> "WorkflowsMap": - """List all available workflow sessions, indexed by their unique id.""" - - from kiara.interfaces.python_api.models.doc import WorkflowsMap - from kiara.interfaces.python_api.models.workflow import WorkflowMatcher - - workflows = {} - - matcher = WorkflowMatcher(**matcher_params) - if matcher.has_alias: - for ( - alias, - workflow_id, - ) in self.context.workflow_registry.workflow_aliases.items(): - - workflow = self.get_workflow(workflow=workflow_id) - workflows[workflow.workflow_id] = workflow - return WorkflowsMap(root={str(k): v for k, v in workflows.items()}) - else: - for workflow_id in self.context.workflow_registry.all_workflow_ids: - workflow = self.get_workflow(workflow=workflow_id) - workflows[workflow_id] = workflow - return WorkflowsMap(root={str(k): v for k, v in workflows.items()}) - - def list_workflow_aliases(self, **matcher_params) -> "WorkflowsMap": - """List all available workflow sessions that have an alias, indexed by alias. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - - from kiara.interfaces.python_api.models.doc import WorkflowsMap - from kiara.interfaces.python_api.workflow import Workflow - - if matcher_params: - matcher_params["has_alias"] = True - workflows = self.list_workflows(**matcher_params) - result: Dict[str, Workflow] = {} - for workflow in workflows.values(): - aliases = self.context.workflow_registry.get_aliases( - workflow_id=workflow.workflow_id - ) - for a in aliases: - if a in result.keys(): - raise Exception( - f"Duplicate workflow alias '{a}': this is most likely a bug." - ) - result[a] = workflow - result = {k: result[k] for k in sorted(result.keys())} - else: - # faster if not other matcher params - all_aliases = self.context.workflow_registry.workflow_aliases - result = { - a: self.get_workflow(workflow=all_aliases[a]) - for a in sorted(all_aliases.keys()) - } - - return WorkflowsMap(root=result) - - def retrieve_workflows_info(self, **matcher_params: Any) -> WorkflowGroupInfo: - """Get a map info instances for all available workflows, indexed by (stringified) workflow-id. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - workflows = self.list_workflows(**matcher_params) - - workflow_infos = WorkflowGroupInfo.create_from_workflows( - *workflows.values(), - group_title=None, - alias_map=self.context.workflow_registry.workflow_aliases, - ) - return workflow_infos - - def retrieve_workflow_aliases_info( - self, **matcher_params: Any - ) -> WorkflowGroupInfo: - """Get a map info instances for all available workflows, indexed by alias. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - workflows = self.list_workflow_aliases(**matcher_params) - workflow_infos = WorkflowGroupInfo.create_from_workflows( - *workflows.values(), - group_title=None, - alias_map=self.context.workflow_registry.workflow_aliases, - ) - return workflow_infos - - def create_workflow( - self, - workflow_alias: Union[None, str] = None, - initial_pipeline: Union[None, Path, str, Mapping[str, Any]] = None, - initial_inputs: Union[None, Mapping[str, Any]] = None, - documentation: Union[Any, None] = None, - save: bool = False, - force_alias: bool = False, - ) -> "Workflow": - """Create a workflow instance. - - NOTE: this is a provisional endpoint, don't use in anger yet - """ - - from kiara.interfaces.python_api.workflow import Workflow - - if workflow_alias is not None: - try: - uuid.UUID(workflow_alias) - raise Exception( - f"Can't create workflow, provided alias can't be a uuid: {workflow_alias}." - ) - except Exception: - pass - - workflow_id = ID_REGISTRY.generate() - metadata = WorkflowMetadata( - workflow_id=workflow_id, documentation=documentation - ) - - workflow_obj = Workflow(kiara=self.context, workflow=metadata) - if workflow_alias: - workflow_obj._pending_aliases.add(workflow_alias) - - if initial_pipeline: - operation = self.get_operation(operation=initial_pipeline) - if operation.module_type == "pipeline": - pipeline_details: PipelineOperationDetails = operation.operation_details # type: ignore - workflow_obj.add_steps(*pipeline_details.pipeline_config.steps) - input_aliases = pipeline_details.pipeline_config.input_aliases - for k, v in input_aliases.items(): - workflow_obj.set_input_alias(input_field=k, alias=v) - output_aliases = pipeline_details.pipeline_config.output_aliases - for k, v in output_aliases.items(): - workflow_obj.set_output_alias(output_field=k, alias=v) - else: - raise NotImplementedError() - - workflow_obj.set_inputs(**operation.module.config.defaults) - - if initial_inputs: - workflow_obj.set_inputs(**initial_inputs) - - self._workflow_cache[workflow_obj.workflow_id] = workflow_obj - - if save: - if force_alias and workflow_alias: - self.context.workflow_registry.unregister_alias(workflow_alias) - workflow_obj.save() - - return workflow_obj - - def _repr_html_(self): - - info = self.get_context_info() - r = info.create_renderable() - mime_bundle = r._repr_mimebundle_(include=[], exclude=[]) # type: ignore - return mime_bundle["text/html"] +# If you are looking for the `KiaraAPI` class, this has moved into the `kiara_api` sub-module. diff --git a/src/kiara/interfaces/python_api/base_api.py b/src/kiara/interfaces/python_api/base_api.py new file mode 100644 index 000000000..ded3796f4 --- /dev/null +++ b/src/kiara/interfaces/python_api/base_api.py @@ -0,0 +1,3573 @@ +# -*- coding: utf-8 -*- +# Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) + +import inspect +import json +import os.path +import sys +import textwrap +import uuid +from functools import cached_property +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + MutableMapping, + Set, + Type, + Union, +) + +import dpath +import structlog +from ruamel.yaml import YAML + +from kiara.defaults import ( + CHUNK_COMPRESSION_TYPE, + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, + DEFAULT_STORE_MARKER, + OFFICIAL_KIARA_PLUGINS, + VALID_VALUE_QUERY_CATEGORIES, + VALUE_ATTR_DELIMITER, +) +from kiara.exceptions import ( + DataTypeUnknownException, + KiaraException, + NoSuchExecutionTargetException, + NoSuchWorkflowException, +) +from kiara.interfaces.python_api.models.info import ( + DataTypeClassesInfo, + DataTypeClassInfo, + KiaraPluginInfo, + KiaraPluginInfos, + ModuleTypeInfo, + ModuleTypesInfo, + OperationGroupInfo, + OperationInfo, + OperationTypeInfo, + RendererInfos, + ValueInfo, + ValuesInfo, +) +from kiara.interfaces.python_api.models.job import JobDesc +from kiara.interfaces.python_api.value import StoreValueResult, StoreValuesResult +from kiara.models.context import ContextInfo, ContextInfos +from kiara.models.module.manifest import Manifest +from kiara.models.module.operation import Operation +from kiara.models.rendering import RenderValueResult +from kiara.models.runtime_environment.python import PythonRuntimeEnvironment +from kiara.models.values.matchers import ValueMatcher +from kiara.models.values.value import ( + PersistedData, + Value, + ValueMapReadOnly, + ValueSchema, +) +from kiara.models.workflow import WorkflowGroupInfo, WorkflowInfo, WorkflowMetadata +from kiara.operations import OperationType +from kiara.operations.included_core_operations.filter import FilterOperationType +from kiara.operations.included_core_operations.pipeline import PipelineOperationDetails +from kiara.operations.included_core_operations.pretty_print import ( + PrettyPrintOperationType, +) +from kiara.operations.included_core_operations.render_value import ( + RenderValueOperationType, +) +from kiara.registries.environment import EnvironmentRegistry +from kiara.registries.ids import ID_REGISTRY +from kiara.renderers import KiaraRenderer +from kiara.utils import log_exception, log_message +from kiara.utils.downloads import get_data_from_url +from kiara.utils.files import get_data_from_file +from kiara.utils.operations import create_operation +from kiara.utils.string_vars import replace_var_names_in_obj + +if TYPE_CHECKING: + from kiara.context import Kiara, KiaraConfig, KiaraRuntimeConfig + from kiara.interfaces.python_api.models.archive import KiArchive + from kiara.interfaces.python_api.models.doc import ( + OperationsMap, + PipelinesMap, + WorkflowsMap, + ) + from kiara.interfaces.python_api.workflow import Workflow + from kiara.models.archives import KiArchiveInfo + from kiara.models.module.jobs import ActiveJob, JobRecord + from kiara.models.module.pipeline import PipelineConfig, PipelineStructure + from kiara.models.module.pipeline.pipeline import PipelineGroupInfo, PipelineInfo + from kiara.registries import KiaraArchive + from kiara.registries.metadata import MetadataStore + +logger = structlog.getLogger() +yaml = YAML(typ="safe") + + +def tag(*tags: str): + def decorator(func): + func._tags = tags + return func + + return decorator + + +def find_base_api_endpoints(cls, label): + """Return all endpoints that are tagged with the provided label.""" + + # for func in dir(cls): + # if not func.startswith("_") and "_tags" not in dir(getattr(cls, func)): + # print(dir(getattr(cls, func))) + return [ + getattr(cls, func) + for func in dir(cls) + if "_tags" in dir(getattr(cls, func)) and label in getattr(cls, func)._tags + ] + + +class BaseAPI(object): + """Kiara base API. + + This class wraps a [Kiara][kiara.context.kiara.Kiara] instance, and allows easy a access to tasks that are + typically done by a frontend. The return types of each method are json seriable in most cases. + + Can be extended for special scenarios and augmented with scenario-specific methdos (Jupyter, web-frontend, ...) + + The naming of the API endpoints follows a (loose-ish) convention: + - list_*: return a list of ids or items, if items, filtering is supported + - get_*: get specific instances of a type (operation, value, etc.) + - retrieve_*: get augmented information about an instance or type of something. This usually implies that there is some overhead, + so before you use this, make sure that there is not 'get_*' or 'list_*' endpoint that could give you what you need. + . + """ + + def __init__(self, kiara_config: Union["KiaraConfig", None] = None): + + if kiara_config is None: + from kiara.context import Kiara, KiaraConfig + + kiara_config = KiaraConfig() + + self._kiara_config: KiaraConfig = kiara_config + self._contexts: Dict[str, Kiara] = {} + self._workflow_cache: Dict[uuid.UUID, Workflow] = {} + + self._current_context: Union[None, Kiara] = None + self._current_context_alias: Union[None, str] = None + + @cached_property + def doc(self) -> Dict[str, str]: + """Get the documentation for this API.""" + + result = {} + for method_name in dir(self): + if method_name.startswith("_"): + continue + + method = getattr(self.__class__, method_name) + doc = inspect.getdoc(method) + if doc is None: + doc = "-- n/a --" + else: + doc = textwrap.dedent(doc) + + result[method_name] = doc + + return result + + @tag("kiara_api") + def list_available_plugin_names( + self, regex: str = "^kiara[-_]plugin\\..*" + ) -> List[str]: + r""" + Get a list of all available plugins. + + Arguments: + regex: an optional regex to indicate the plugin naming scheme (default: /$kiara[_-]plugin\..*/) + + Returns: + a list of plugin names + """ + + if not regex: + regex = "^kiara[-_]plugin\\..*" + + return KiaraPluginInfos.get_available_plugin_names( + kiara=self.context, regex=regex + ) + + @tag("kiara_api") + def retrieve_plugin_info(self, plugin_name: str) -> KiaraPluginInfo: + """ + Get information about a plugin. + + This contains information about included data-types, modules, operations, pipelines, as well as metadata + about author(s), etc. + + Arguments: + plugin_name: the name of the plugin + + Returns: + a dictionary with information about the plugin + """ + + info = KiaraPluginInfo.create_from_instance( + kiara=self.context, instance=plugin_name + ) + return info + + @tag("kiara_api") + def retrieve_plugin_infos( + self, plugin_name_regex: str = "^kiara[-_]plugin\\..*" + ) -> KiaraPluginInfos: + """Get information about multiple plugins. + + This is just a convenience method to get information about multiple plugins at once. + """ + + if not plugin_name_regex: + plugin_name_regex = "^kiara[-_]plugin\\..*" + + plugin_infos = KiaraPluginInfos.create_group( + self.context, None, plugin_name_regex + ) + return plugin_infos + + @property + def context(self) -> "Kiara": + """ + Return the kiara context. + + DON"T USE THIS! This is going away in the production release. + """ + if self._current_context is None: + self._current_context = self._kiara_config.create_context( + extra_pipelines=None + ) + self._current_context_alias = self._kiara_config.default_context + + return self._current_context + + def get_runtime_config(self) -> "KiaraRuntimeConfig": + """Retrieve the current runtime configuration. + + Check the 'KiaraRuntimeConfig' class for more information about the available options. + """ + return self.context.runtime_config + + @tag("kiara_api") + def get_context_info(self) -> ContextInfo: + """Retrieve information about the current kiara context. + + This contains information about the context, like its name/alias, the values & aliases it contains, and which archives are connected to it. + + """ + context_config = self._kiara_config.get_context_config( + self.get_current_context_name() + ) + info = ContextInfo.create_from_context_config( + context_config, + context_name=self.get_current_context_name(), + runtime_config=self._kiara_config.runtime_config, + ) + + return info + + def ensure_plugin_packages( + self, package_names: Union[str, Iterable[str]], update: bool = False + ) -> Union[bool, None]: + """ + Ensure that the specified packages are installed. + + + NOTE: this is not tested, and it might go away in the future, so don't rely on it being available long-term. Ideally, we'll have other, external ways to manage the environment. + + Arguments: + package_names: The names of the packages to install. + update: If True, update the packages if they are already installed + + Returns: + 'None' if run in jupyter, 'True' if any packages were installed, 'False' otherwise. + """ + if isinstance(package_names, str): + package_names = [package_names] + + env_reg = EnvironmentRegistry.instance() + python_env: PythonRuntimeEnvironment = env_reg.environments[ # type: ignore + "python" + ] # type: ignore + + if not package_names: + package_names = OFFICIAL_KIARA_PLUGINS # type: ignore + + if not update: + plugin_packages: List[str] = [] + pkgs = [p.name.replace("_", "-") for p in python_env.packages] + for package_name in package_names: + if package_name.startswith("git:"): + package_name = package_name.replace("git:", "") + git = True + else: + git = False + package_name = package_name.replace("_", "-") + if not package_name.startswith("kiara-plugin."): + package_name = f"kiara-plugin.{package_name}" + + if git or package_name.replace("_", "-") not in pkgs: + if git: + package_name = package_name.replace("-", "_") + plugin_packages.append( + f"git+https://x:x@github.com/DHARPA-project/{package_name}@develop" + ) + else: + plugin_packages.append(package_name) + else: + plugin_packages = package_names # type: ignore + + in_jupyter = "google.colab" in sys.modules or "jupyter_client" in sys.modules + + if not plugin_packages: + if in_jupyter: + return None + else: + # nothing to do + return False + + class DummyContext(object): + def __getattribute__(self, item): + raise Exception( + "Currently installing plugins, no other operations are allowed." + ) + + current_context_name = self._current_context_alias + for k in self._contexts.keys(): + self._contexts[k] = DummyContext() # type: ignore + self._current_context = DummyContext() # type: ignore + + cmd = ["-q", "--isolated", "install"] + if update: + cmd.append("--upgrade") + cmd.extend(plugin_packages) + + if in_jupyter: + from IPython import get_ipython + + ipython = get_ipython() + cmd_str = f"sc -l stdout = {sys.executable} -m pip {' '.join(cmd)}" + ipython.magic(cmd_str) + exit_code = 100 + else: + import pip._internal.cli.main as pip + + log_message( + "install.python_packages", packages=plugin_packages, update=update + ) + exit_code = pip.main(cmd) + + self._contexts.clear() + self._current_context = None + self._current_context_alias = None + + EnvironmentRegistry._instance = None + if current_context_name: + self.set_active_context(context_name=current_context_name) + + if exit_code == 100: + raise SystemExit( + f"Please manually re-run all cells. Updated or newly installed plugin packages: {', '.join(plugin_packages)}." + ) + elif exit_code != 0: + raise Exception( + f"Failed to install plugin packages: {', '.join(plugin_packages)}" + ) + + return True + + # ================================================================================================================== + # context-management related functions + @tag("kiara_api") + def list_context_names(self) -> List[str]: + """list the names of all available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and + whether we want to support single-file contexts in the future. + """ + return list(self._kiara_config.available_context_names) + + @tag("kiara_api") + def retrieve_context_infos(self) -> ContextInfos: + """Retrieve information about the available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + return ContextInfos.create_context_infos(self._kiara_config.context_configs) + + @tag("kiara_api") + def get_current_context_name(self) -> str: + """Retrieve the name of the current context. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + if self._current_context_alias is None: + self.context + return self._current_context_alias # type: ignore + + def create_new_context(self, context_name: str, set_active: bool = True) -> None: + """ + Create a new context. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. So if you need something like this, please let me know. + + Arguments: + context_name: the name of the new context + set_active: set the newly created context as the active one + """ + if context_name in self.list_context_names(): + raise Exception( + f"Can't create context with name '{context_name}': context already exists." + ) + + ctx = self._kiara_config.create_context(context_name, extra_pipelines=None) + if set_active: + self._current_context = ctx + self._current_context_alias = context_name + + # return ctx + + @tag("kiara_api") + def set_active_context(self, context_name: str, create: bool = False) -> None: + """Set the currently active context for this KiarAPI instance. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + + if not context_name: + raise Exception("No context name provided.") + + if context_name == self._current_context_alias: + return + if context_name not in self.list_context_names(): + if create: + self._current_context = self._kiara_config.create_context( + context=context_name, extra_pipelines=None + ) + self._current_context_alias = context_name + return + else: + raise Exception(f"No context with name '{context_name}' available.") + + self._current_context = self._kiara_config.create_context( + context=context_name, extra_pipelines=None + ) + self._current_context_alias = context_name + + # ================================================================================================================== + # methods for data_types + + @tag("kiara_api") + def list_data_type_names(self, include_profiles: bool = False) -> List[str]: + """Get a list of all registered data types. + + Arguments: + include_profiles: if True, also include the names of all registered data type profiles + """ + + return self.context.type_registry.get_data_type_names( + include_profiles=include_profiles + ) + + def is_internal_data_type(self, data_type_name: str) -> bool: + """Checks if the data type is prepdominantly used internally by kiara, or whether it should be exposed to the user.""" + + return self.context.type_registry.is_internal_type( + data_type_name=data_type_name + ) + + @tag("kiara_api") + def retrieve_data_types_info( + self, + filter: Union[str, Iterable[str], None] = None, + include_data_type_profiles: bool = False, + python_package: Union[None, str] = None, + ) -> DataTypeClassesInfo: + """ + Retrieve information about all data types. + + A data type is a Python class that inherits from [DataType[kiara.data_types.DataType], and it wraps a specific + Python class that holds the actual data and provides metadata and convenience methods for managing the data internally. Data types are not directly used by users, but they are exposed in the input/output schemas of moudles and other data-related features. + + Arguments: + filter: an optional string or (list of strings) the returned datatype ids have to match (all filters in the case of a list) + include_data_type_profiles: if True, also include the names of all registered data type profiles + python_package: if provided, only return data types that are defined in the given python package + + Returns: + an object containing all information about all data types + """ + + kiara = self.context + + if python_package: + data_type_info = kiara.type_registry.get_context_metadata( + only_for_package=python_package + ) + + if filter: + title = f"Filtered data types in package '{python_package}'" + + if isinstance(filter, str): + filter = [filter] + + filtered_types: Dict[str, DataTypeClassInfo] = {} + + for dt in data_type_info.item_infos.keys(): + match = True + + for f in filter: + if f.lower() not in dt.lower(): + match = False + break + if match: + filtered_types[dt] = data_type_info.item_infos[dt] + + data_types_info = DataTypeClassesInfo( + group_title=title, item_infos=filtered_types + ) + # data_types_info._kiara = kiara + + else: + title = f"All data types in package '{python_package}'" + data_types_info = data_type_info + data_types_info.group_title = title + else: + if filter: + if isinstance(filter, str): + filter = [filter] + + title = f"Filtered data_types: {filter}" + data_type_names: Iterable[str] = [] + + for m in kiara.type_registry.get_data_type_names( + include_profiles=include_data_type_profiles + ): + match = True + + for f in filter: + + if f.lower() not in m.lower(): + match = False + break + + if match: + data_type_names.append(m) # type: ignore + else: + title = "All data types" + data_type_names = kiara.type_registry.get_data_type_names( + include_profiles=include_data_type_profiles + ) + + data_types = { + d: kiara.type_registry.get_data_type_cls(d) for d in data_type_names + } + data_types_info = DataTypeClassesInfo.create_from_type_items( # type: ignore + kiara=kiara, group_title=title, **data_types + ) + + return data_types_info # type: ignore + + @tag("kiara_api") + def retrieve_data_type_info(self, data_type_name: str) -> DataTypeClassInfo: + """ + Retrieve information about a specific data type. + + Arguments: + data_type: the registered name of the data type + + Returns: + an object containing all information about a data type + """ + dt_cls = self.context.type_registry.get_data_type_cls(data_type_name) + info = DataTypeClassInfo.create_from_type_class( + kiara=self.context, type_cls=dt_cls + ) + return info + + # ================================================================================================================== + # methods for module and operations info + + @tag("kiara_api") + def list_module_type_names(self) -> List[str]: + """Get a list of all registered module types.""" + return list(self.context.module_registry.get_module_type_names()) + + @tag("kiara_api") + def retrieve_module_types_info( + self, + filter: Union[None, str, Iterable[str]] = None, + python_package: Union[str, None] = None, + ) -> ModuleTypesInfo: + """ + Retrieve information for all available module types (or a filtered subset thereof). + + A module type is Python class that inherits from [KiaraModule][kiara.modules.KiaraModule], and is the basic + building block for processing pipelines. Module types are not used directly by users, Operations are. Operations + are instantiated modules (meaning: the module & some (optional) configuration). + + Arguments: + filter: an optional string (or list of string) the returned module names have to match (all filters in case of list) + python_package: an optional string, if provided, only modules from the specified python package are returned + + Returns: + a mapping object containing module names as keys, and information about the modules as values + """ + + if python_package: + + modules_type_info = self.context.module_registry.get_context_metadata( + only_for_package=python_package + ) + + if filter: + title = f"Filtered modules: {filter} (in package '{python_package}')" + if isinstance(filter, str): + filter = [filter] + + filtered_types: Dict[str, ModuleTypeInfo] = {} + + for m in modules_type_info.item_infos.keys(): + match = True + + for f in filter: + + if f.lower() not in m.lower(): + match = False + break + + if match: + filtered_types[m] = modules_type_info.item_infos[m] + + module_types_info = ModuleTypesInfo( + group_title=title, item_infos=filtered_types + ) + module_types_info._kiara = self.context + else: + title = f"All modules in package '{python_package}'" + module_types_info = modules_type_info + module_types_info.group_title = title + + else: + + if filter: + + if isinstance(filter, str): + filter = [filter] + title = f"Filtered modules: {filter}" + module_types_names: Iterable[str] = [] + + for m in self.context.module_registry.get_module_type_names(): + match = True + + for f in filter: + + if f.lower() not in m.lower(): + match = False + break + + if match: + module_types_names.append(m) # type: ignore + else: + title = "All modules" + module_types_names = ( + self.context.module_registry.get_module_type_names() + ) + + module_types = { + n: self.context.module_registry.get_module_class(n) + for n in module_types_names + } + + module_types_info = ModuleTypesInfo.create_from_type_items( # type: ignore + kiara=self.context, group_title=title, **module_types + ) + + return module_types_info # type: ignore + + @tag("kiara_api") + def retrieve_module_type_info(self, module_type: str) -> ModuleTypeInfo: + """ + Retrieve information about a specific module type. + + This can be used to retrieve information like module documentation and configuration options. + + Arguments: + module_type: the registered name of the module + + Returns: + an object containing all information about a module type + """ + m_cls = self.context.module_registry.get_module_class(module_type) + info = ModuleTypeInfo.create_from_type_class(kiara=self.context, type_cls=m_cls) + return info + + def create_operation( + self, + module_type: str, + module_config: Union[Mapping[str, Any], str, None] = None, + ) -> Operation: + """ + Create an [Operation][kiara.models.module.operation.Operation] instance for the specified module type and (optional) config. + + An operation is defined as a specific module type, and a specific configuration. + + This endpoint can be used to get information about the operation itself, it's inputs & outputs schemas, documentation etc. + + Arguments: + module_type: the registered name of the module + module_config: (Optional) configuration for the module instance. + + Returns: + an Operation instance (which contains all the available information about an instantiated module) + """ + if module_config is None: + module_config = {} + elif isinstance(module_config, str): + try: + module_config = json.load(module_config) # type: ignore + except Exception: + try: + module_config = yaml.load(module_config) # type: ignore + except Exception: + raise Exception( + f"Can't parse module config string: {module_config}." + ) + + if module_type == "pipeline": + if not module_config: + raise Exception("Pipeline configuration can't be empty.") + assert module_config is None or isinstance(module_config, Mapping) + operation = create_operation( + "pipeline", operation_config=module_config, kiara=self.context + ) + return operation + else: + mc = Manifest(module_type=module_type, module_config=module_config) + module_obj = self.context.module_registry.create_module(mc) + + return module_obj.operation + + @tag("kiara_api") + def list_operation_ids( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + python_packages: Union[str, None, Iterable[str]] = None, + ) -> List[str]: + """ + Get a list of all operation ids that match the specified filter. + + Arguments: + filter: the (optional) filter string(s), an operation must match all of them to be included in the result + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + """ + if not filter and include_internal and not python_packages: + return sorted(self.context.operation_registry.operation_ids) + + else: + return sorted( + self.list_operations( + filter=filter, + input_types=input_types, + output_types=output_types, + operation_types=operation_types, + include_internal=include_internal, + python_packages=python_packages, + ).keys() + ) + + @tag("kiara_api") + def get_operation( + self, + operation: Union[Mapping[str, Any], str, Path], + allow_external: Union[bool, None] = None, + ) -> Operation: + """ + Return the operation instance with the specified id. + + The difference to the 'create_operation' endpoint is slight, in most cases you could use either of them, but this one is a bit more convenient in most cases, as it tries to do the right thing with whatever 'operation' argument you use it. The 'create_opearation' endpoint will always create a new 'Operation' instance, while this may or may not return a re-used one. + + This endpoint can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc. + + The order in which the operation argument is resolved: + - if it's a string, and an existing, registered operation_id, the associated operation is returned + - if it's a path to an existing file, the content of the file is loaded into a dict and depending on the content a pipeline module will be created, or a 'normal' manifest (if module_type is a key in the dict) + + Arguments: + operation: the operation id, module_type_name, path to a file, or url + allow_external: if True, allow loading operations from external sources (e.g. a URL), if 'None' is provided, the configured value in the runtime configuration is used. + + Returns: + operation instance data + """ + _module_type = None + _module_config: Any = None + + if allow_external is None: + allow_external = self.get_runtime_config().allow_external + + if isinstance(operation, Path): + operation = operation.as_posix() + + if ( + isinstance(operation, Mapping) + and "module_type" in operation.keys() + and "module_config" in operation.keys() + and not operation["module_config"] + ): + operation = operation["module_type"] + + if isinstance(operation, str): + + if operation in self.list_operation_ids(include_internal=True): + _operation = self.context.operation_registry.get_operation(operation) + return _operation + + if not allow_external: + raise NoSuchExecutionTargetException( + selected_target=operation, + available_targets=self.context.operation_registry.operation_ids, + msg=f"Can't find operation with id '{operation}', and external operations are not allowed.", + ) + + if os.path.isfile(operation): + try: + from kiara.models.module.pipeline import PipelineConfig + + # we use the 'from_file' here, because that will resolve any relative paths in the pipeline + # if this doesn't work, we just assume the file is not a pipeline configuration but + # a manifest file with 'module_type' and optional 'module_config' keys + pipeline_conf = PipelineConfig.from_file( + path=operation, kiara=self.context + ) + _module_config = pipeline_conf.model_dump() + except Exception as e: + log_exception(e) + _module_config = get_data_from_file(operation) + elif operation.startswith("http"): + _module_config = get_data_from_url(operation) + else: + try: + _module_config = json.load(operation) # type: ignore + except Exception: + try: + _module_config = yaml.load(operation) # type: ignore + except Exception: + raise Exception( + f"Can't parse configuration string: {operation}." + ) + if not isinstance(_module_config, Mapping): + raise NoSuchExecutionTargetException( + selected_target=operation, + available_targets=self.context.operation_registry.operation_ids, + msg=f"Can't find operation or execution target for string '{operation}'.", + ) + + else: + _module_config = dict(operation) # type: ignore + + if "module_type" in _module_config.keys(): + _module_type = _module_config["module_type"] + _module_config = _module_config.get("module_config", {}) + else: + _module_type = "pipeline" + + op = self.create_operation( + module_type=_module_type, module_config=_module_config + ) + return op + + @tag("kiara_api") + def list_operations( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> "OperationsMap": + """ + List all available operations, optionally filter. + + Arguments: + filter: the (optional) filter string(s), an operation must match all of them to be included in the result + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + + Returns: + a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value + """ + if operation_types: + if isinstance(operation_types, str): + operation_types = [operation_types] + temp: Dict[str, Operation] = {} + for op_type_name in operation_types: + op_type = self.context.operation_registry.operation_types.get( + op_type_name, None + ) + if op_type is None: + raise Exception(f"Operation type not registered: {op_type_name}") + + temp.update(op_type.operations) + + operations: Mapping[str, Operation] = temp + else: + operations = self.context.operation_registry.operations + + if filter: + if isinstance(filter, str): + filter = [filter] + temp = {} + for op_id, op in operations.items(): + match = True + for f in filter: + if not f: + continue + if f.lower() not in op_id.lower(): + match = False + break + if match: + temp[op_id] = op + operations = temp + + if not include_internal: + temp = {} + for op_id, op in operations.items(): + if not op.operation_details.is_internal_operation: + temp[op_id] = op + + operations = temp + + if input_types: + if isinstance(input_types, str): + input_types = [input_types] + temp = {} + for op_id, op in operations.items(): + for input_type in input_types: + match = False + for schema in op.inputs_schema.values(): + if schema.type == input_type: + temp[op_id] = op + match = True + break + if match: + break + + operations = temp + + if output_types: + if isinstance(output_types, str): + output_types = [output_types] + temp = {} + for op_id, op in operations.items(): + for output_type in output_types: + match = False + for schema in op.outputs_schema.values(): + if schema.type == output_type: + temp[op_id] = op + match = True + break + if match: + break + + operations = temp + + if python_packages: + temp = {} + if isinstance(python_packages, str): + python_packages = [python_packages] + for op_id, op in operations.items(): + info = OperationInfo.create_from_instance( + kiara=self.context, instance=op + ) + pkg = info.context.labels.get("package", None) + if pkg in python_packages: + temp[op_id] = op + operations = temp + + from kiara.interfaces.python_api.models.doc import OperationsMap + + return OperationsMap.model_construct(root=operations) # type: ignore + + @tag("kiara_api") + def retrieve_operation_info( + self, operation: str, allow_external: bool = False + ) -> OperationInfo: + """ + Return the full information for the specified operation id. + + This is similar to the 'get_operation' method, but returns additional information. Only use this instead of + 'get_operation' if you need the additional info, as it's more expensive to get. + + Arguments: + operation: the operation id + + Returns: + augmented operation instance data + """ + if not allow_external: + op = self.context.operation_registry.get_operation(operation_id=operation) + else: + op = create_operation(module_or_operation=operation) + op_info = OperationInfo.create_from_operation(kiara=self.context, operation=op) + return op_info + + @tag("kiara_api") + def retrieve_operations_info( + self, + *filters: str, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> OperationGroupInfo: + """ + Retrieve information about the matching operations. + + This retrieves the same list of operations as [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations], + but augments each result instance with additional information that might be useful in frontends. + + 'OperationInfo' objects contains augmented information on top of what 'normal' [Operation][kiara.models.module.operation.Operation] objects + hold, but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations] method + instead. + + Arguments: + filters: the (optional) filter strings, an operation must match all of them to be included in the result + include_internal: whether to include operations that are predominantly used internally in kiara. + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + Returns: + a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value + """ + title = "Available operations" + if filters: + title = "Filtered operations" + + operations = self.list_operations( + filters, + input_types=input_types, + output_types=output_types, + include_internal=include_internal, + operation_types=operation_types, + python_packages=python_packages, + ) + + ops_info = OperationGroupInfo.create_from_operations( + kiara=self.context, group_title=title, **operations + ) + return ops_info + + # ================================================================================================================== + # methods relating to pipelines + + def list_pipeline_ids( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + python_packages: Union[str, None, Iterable[str]] = None, + ) -> List[str]: + """ + Get a list of all pipeline (operation) ids that match the specified filter. + + Arguments: + filter: an optional single or list of filters (all filters must match the operation id for the operation to be included) + include_internal: also return internal pipelines + """ + + result: List[str] = self.list_operation_ids( + filter=filter, + input_types=input_types, + output_types=output_types, + operation_types=["pipeline"], + include_internal=include_internal, + python_packages=python_packages, + ) + return result + + def list_pipelines( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> "PipelinesMap": + """List all available pipelines, optionally filter. + + Arguments: + filter: the (optional) filter string(s), an operation must match all of them to be included in the result + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + + Returns: + a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value + """ + from kiara.interfaces.python_api.models.doc import PipelinesMap + + ops = self.list_operations( + filter=filter, + input_types=input_types, + output_types=output_types, + operation_types=["pipeline"], + python_packages=python_packages, + include_internal=include_internal, + ) + + result: Dict[str, PipelineStructure] = {} + for op in ops.values(): + details: PipelineOperationDetails = op.operation_details + config: "PipelineConfig" = details.pipeline_config + structure = config.structure + result[op.operation_id] = structure + + return PipelinesMap.model_construct(root=result) + + def get_pipeline_structure( + self, + pipeline: Union[Mapping[str, Any], str, Path], + allow_external: Union[bool, None] = None, + ) -> "PipelineStructure": + """ + Return the pipeline (Structure) instance with the specified id. + + This can be used to get information about a pipeline, like inputs/outputs scheman, documentation, included steps, stages, etc. + + The order in which the operation argument is resolved: + - if it's a string, and an existing, registered operation_id, the associated operation is returned + - if it's a path to an existing file, the content of the file is loaded into a dict and a pipeline operation will be created + + Arguments: + pipeline: the pipeline id, module_type_name, path to a file, or url + allow_external: if True, allow loading operations from external sources (e.g. a URL), if 'None' is provided, the configured value in the runtime configuration is used. + + Returns: + pipeline structure data + """ + + op = self.get_operation(operation=pipeline, allow_external=allow_external) + if op.module_type != "pipeline": + raise KiaraException( + f"Operation '{op.operation_id}' is not a pipeline, but a '{op.module_type}'" + ) + details: PipelineOperationDetails = op.operation_details # type: ignore + config: "PipelineConfig" = details.pipeline_config + + return config.structure + + def retrieve_pipeline_info( + self, pipeline: str, allow_external: bool = False + ) -> "PipelineInfo": + """ + Return the full information for the specified pipeline id. + + This is similar to the 'get_pipeline' method, but returns additional information. Only use this instead of + 'get_pipeline' if you need the additional info, as it's more expensive to get. + + Arguments: + pipeline: the pipeline (operation) id + + Returns: + augmented pipeline instance data + """ + if not allow_external: + op = self.context.operation_registry.get_operation(operation_id=pipeline) + else: + op = create_operation(module_or_operation=pipeline) + + if op.module_type != "pipeline": + raise KiaraException( + f"Operation '{op.operation_id}' is not a pipeline, but a '{op.module_type}'" + ) + + from kiara.models.module.pipeline.pipeline import Pipeline, PipelineInfo + + details: PipelineOperationDetails = op.operation_details # type: ignore + config: "PipelineConfig" = details.pipeline_config + pipeline_instance = Pipeline(structure=config.structure, kiara=self.context) + + p_info: PipelineInfo = PipelineInfo.create_from_instance( + kiara=self.context, instance=pipeline_instance + ) + return p_info + + def retrieve_pipelines_info( + self, + *filters, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> "PipelineGroupInfo": + """ + Retrieve information about the matching pipelines. + + This retrieves the same list of pipelines as [list_pipelines][kiara.interfaces.python_api.KiaraAPI.list_pipelines], + but augments each result instance with additional information that might be useful in frontends. + + 'PipelineInfo' objects contains augmented information on top of what 'normal' [PipelineStructure][kiara.models.module.pipeline.PipelineStructure] objects + hold, but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [list_pipelines][kiara.interfaces.python_api.KiaraAPI.list_pipelines] method + instead. + + Arguments: + filters: the (optional) filter strings, an operation must match all of them to be included in the result + include_internal: whether to include operations that are predominantly used internally in kiara. + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + Returns: + a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value + """ + + title = "Available pipelines" + if filters: + title = "Filtered pipelines" + + operations = self.list_operations( + filters, + input_types=input_types, + output_types=output_types, + include_internal=include_internal, + operation_types=["pipeline"], + python_packages=python_packages, + ) + + from kiara.models.module.pipeline.pipeline import Pipeline, PipelineGroupInfo + + pipelines = {} + for op_id, op in operations.items(): + details: PipelineOperationDetails = op.operation_details # type: ignore + config: "PipelineConfig" = details.pipeline_config + pipeline = Pipeline(structure=config.structure, kiara=self.context) + pipelines[op_id] = pipeline + + ps_info = PipelineGroupInfo.create_from_pipelines( + kiara=self.context, group_title=title, **pipelines + ) + return ps_info + + def register_pipeline( + self, + data: Union[Path, str, Mapping[str, Any]], + operation_id: Union[str, None] = None, + ) -> Operation: + """ + Register a pipelne as new operation into this context. + + If 'operation_id' is not provided, the id will be auto-determined (in most cases using the pipeline name). + + Arguments: + data: a dict or a path to a json/yaml file containing the definition + operation_id: the id to use for the operation (if not specified, the id will be auto-determined) + + Returns: + the assembled operation + """ + return self.context.operation_registry.register_pipeline( + data=data, operation_id=operation_id + ) + + def register_pipelines( + self, *pipeline_paths: Union[str, Path] + ) -> Dict[str, Operation]: + """Register all pipelines found in the specified paths.""" + return self.context.operation_registry.register_pipelines(*pipeline_paths) + + # ================================================================================================================== + # methods relating to values and data + + def register_data( + self, + data: Any, + data_type: Union[None, str, ValueSchema, Mapping[str, Any]] = None, + reuse_existing: bool = False, + ) -> Value: + """ + Register data with kiara. + + This will create a new value instance from the data and return it. The data/value itself won't be stored + in a store, you have to use the 'store_value' function for that. + + Arguments: + data: the data to register + data_type: (optional) the data type of the data. If not provided, kiara will try to infer the data type. + reuse_existing: whether to re-use an existing value that is already registered and has the same hash. + + Returns: + a [kiara.models.values.value.Value] instance + """ + if data_type is None: + raise NotImplementedError( + "Infering data types not implemented yet. Please provide one manually." + ) + + value = self.context.data_registry.register_data( + data=data, schema=data_type, reuse_existing=reuse_existing + ) + return value + + @tag("kiara_api") + def list_all_value_ids(self) -> List[uuid.UUID]: + """List all value ids in the current context. + + This returns everything, even internal values. It should be faster than using + `list_value_ids` with equivalent parameters, because no filtering has to happen. + + Returns: + all value_ids in the current context, using every registered store + """ + + _values = self.context.data_registry.retrieve_all_available_value_ids() + return sorted(_values) + + @tag("kiara_api") + def list_value_ids(self, **matcher_params: Any) -> List[uuid.UUID]: + """ + List all available value ids for this kiara context. + + By default, this also includes internal values. + + This method exists mainly so frontends can retrieve a list of all value_ids that exists on the backend without + having to look up the details of each value (like [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] + does). This method can also be used with a matcher, but in this case the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] + would be preferable in most cases, because it is called under the hood, and the performance advantage of not + having to look up value details is gone. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters and defaults + + Returns: + a list of value ids + """ + + values = self.list_values(**matcher_params) + return sorted((v.value_id for v in values.values())) + + @tag("kiara_api") + def list_all_values(self) -> ValueMapReadOnly: + """List all values in the current context, incl. internal ones. + + This should be faster than `list_values` with equivalent matcher params, because no + filtering has to happen. + """ + + # TODO: make that parallel? + values = { + k: self.context.data_registry.get_value(k) + for k in self.context.data_registry.retrieve_all_available_value_ids() + } + result = ValueMapReadOnly.create_from_values( + **{str(k): v for k, v in values.items()} + ) + return result + + @tag("kiara_api") + def list_values(self, **matcher_params: Any) -> ValueMapReadOnly: + """ + List all available (relevant) values, optionally filter. + + Retrieve information about all values that are available in the current kiara context session (both stored and non-stored). + + Check the `ValueMatcher` class for available parameters and defaults, for example this excludes + internal values by default. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with value_id as key, and [kiara.models.values.value.Value] as value + """ + + matcher = ValueMatcher.create_matcher(**matcher_params) + values = self.context.data_registry.find_values(matcher=matcher) + + result = ValueMapReadOnly.create_from_values( + **{str(k): v for k, v in values.items()} + ) + return result + + @tag("kiara_api") + def get_value(self, value: Union[str, Value, uuid.UUID, Path]) -> Value: + """ + Retrieve a value instance with the specified id or alias. + + Basically a convenience method to convert any possible Python type into + a 'Value' instance. Raises an exception if no value could be found. + + Arguments: + value: a value id, alias or object that has a 'value_id' attribute. + + Returns: + the Value instance + """ + + return self.context.data_registry.get_value(value=value) + + @tag("kiara_api") + def get_values(self, **values: Union[str, Value, uuid.UUID]) -> ValueMapReadOnly: + """Retrieve Value instances for the specified value ids or aliases. + + This is a convenience method to get fully 'hydrated' `Value` objects from references to them. + + Arguments: + values: a dictionary with value ids or aliases as keys, and value instances as values + + Returns: + a mapping with value_id as key, and [kiara.models.values.value.Value] as value + """ + + return self.context.data_registry.load_values(values=values) + + def query_value( + self, + value_or_path: Union[str, Value, uuid.UUID], + query_path: Union[str, None] = None, + ) -> Any: + """ + Retrieve a value attribute with the specified id or alias. + + NOTE: This is a provisional endpoint, don't use for now, if you have a requirement that would + be covered by this, please let me know. + + A query path is delimited by "::", and has the following format: + + ``` + ::[]::[]::[...] + ``` + + Currently supported categories: + - "data": the data of the value + - "properties: the properties of the value + + If no category is specified, the value instance itself is returned. + + Raises an exception if no value could be found. + + Arguments: + value_or_path: a value or value reference, or a query path containing the value id or alias as first token + query_path: a query path which will be appended a potential query path computed from the first argument + + Returns: + the attribute value + """ + + if isinstance(value_or_path, str): + tokens = value_or_path.split(VALUE_ATTR_DELIMITER) + value_id = tokens.pop(0) + _value = self.get_value(value=value_id) + else: + tokens = [] + _value = self.get_value(value=value_or_path) + + if query_path: + tokens.extend(query_path.split(VALUE_ATTR_DELIMITER)) + + if not tokens: + return _value + + current_result: Any = _value + category = tokens.pop(0) + if category == "properties": + current_result = current_result.get_all_property_data(flatten_models=True) + elif category == "data": + current_result = current_result.data + else: + raise KiaraException( + f"Invalid query path category: {category}. Valid categories are: {', '.join(VALID_VALUE_QUERY_CATEGORIES)}" + ) + + if tokens: + try: + path = VALUE_ATTR_DELIMITER.join(tokens) + current_result = dpath.get( + current_result, path, separator=VALUE_ATTR_DELIMITER + ) + + except Exception: + + def dict_path(path, my_dict, all_paths): + for k, v in my_dict.items(): + if isinstance(v, dict): + dict_path(path + "::" + k, v, all_paths) + else: + all_paths.append(path[2:] + "::" + k) + + valid_base_keys = list(current_result.keys()) + details = "Valid (base) sub-keys are:\n\n" + for k in valid_base_keys: + details += f" - {k}\n" + + all_paths: List[str] = [] + dict_path("", current_result, all_paths) + + details += "\nValid (full) sub-paths are:\n\n" + for k in all_paths: + details += f" - {k}\n" + + raise KiaraException( + msg=f"Failed to retrieve value attribute using query sub-path: {path}", + details=details, + ) + + return current_result + + @tag("kiara_api") + def retrieve_value_info( + self, value: Union[str, uuid.UUID, Value, Path] + ) -> ValueInfo: + """ + Retrieve an info object for a value. + + Companion method to 'get_value', 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.get_value] method + instead. + + Arguments: + value: a value id, alias or object that has a 'value_id' attribute. + + Returns: + the ValueInfo instance + + """ + _value = self.get_value(value=value) + return ValueInfo.create_from_instance(kiara=self.context, instance=_value) + + @tag("kiara_api") + def retrieve_values_info(self, **matcher_params: Any) -> ValuesInfo: + """ + Retrieve information about the matching values. + + This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], + but augments each result value instance with additional information that might be useful in frontends. + + 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] method + instead. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value + """ + values: MutableMapping[str, Value] = self.list_values(**matcher_params) + + infos = ValuesInfo.create_from_instances( + kiara=self.context, instances={str(k): v for k, v in values.items()} + ) + return infos # type: ignore + + @tag("kiara_api") + def list_alias_names(self, **matcher_params: Any) -> List[str]: + """ + List all available alias keys. + + This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without + having to look up the details of each value (like [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] + does). This method can also be used with a matcher, but in this case the [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] + would be preferrable in most cases, because it is called under the hood, and the performance advantage of not + having to look up value details is gone. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a list of value ids + """ + if matcher_params: + values = self.list_aliases(**matcher_params) + return list(values.keys()) + else: + _values = self.context.alias_registry.all_aliases + return list(_values) + + @tag("kiara_api") + def list_aliases(self, **matcher_params: Any) -> ValueMapReadOnly: + """ + List all available values that have an alias assigned, optionally filter. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with value_id as key, and [kiara.models.values.value.Value] as value + """ + if matcher_params: + matcher_params["has_alias"] = True + all_values = self.list_values(**matcher_params) + + result: Dict[str, Value] = {} + for value in all_values.values(): + aliases = self.context.alias_registry.find_aliases_for_value_id( + value_id=value.value_id + ) + for a in aliases: + if a in result.keys(): + raise Exception( + f"Duplicate value alias '{a}': this is most likely a bug." + ) + result[a] = value + + result = {k: result[k] for k in sorted(result.keys())} + else: + # faster if not other matcher params + all_aliases = self.context.alias_registry.all_aliases + result = { + k: self.context.data_registry.get_value(f"alias:{k}") + for k in all_aliases + } + + return ValueMapReadOnly.create_from_values(**result) + + @tag("kiara_api") + def retrieve_aliases_info(self, **matcher_params: Any) -> ValuesInfo: + """ + Retrieve information about the matching values. + + This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], + but augments each result value instance with additional information that might be useful in frontends. + + 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.list_aliases] method + instead. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value + """ + values = self.list_aliases(**matcher_params) + + infos = ValuesInfo.create_from_instances( + kiara=self.context, instances={str(k): v for k, v in values.items()} + ) + return infos # type: ignore + + def register_value_alias( + self, + value: Union[str, Value, uuid.UUID], + alias: Union[str, Iterable[str]], + allow_overwrite: bool = False, + alias_store: Union[str, None] = None, + ) -> None: + + self.context.alias_registry.register_aliases( + value_id=value, + aliases=alias, + allow_overwrite=allow_overwrite, + alias_store=alias_store, + ) + + def assemble_value_map( + self, + values: Mapping[str, Union[uuid.UUID, None, str, Value, Any]], + values_schema: Union[None, Mapping[str, ValueSchema]] = None, + register_data: bool = False, + reuse_existing_data: bool = False, + ) -> ValueMapReadOnly: + """ + Retrive a [ValueMap][kiara.models.values.value.ValueMap] object from the provided value ids or value links. + + In most cases, this endpoint won't be used by front-ends, it's a fairly low-level method that is + mainly used for internal purposes. If you have a use-case, let me know and I'll improve the docs + if insufficient. + + By default, this method can only use values/datasets that are already registered in *kiara*. If you want to + auto-register 'raw' data, you need to set the 'register_data' flag to 'True', and provide a schema for each of the fields that are not yet registered. + + Arguments: + values: a dictionary with the values in question + values_schema: an optional dictionary with the schema for each of the values that are not yet registered + register_data: whether to allow auto-registration of 'raw' data + reuse_existing_data: whether to reuse existing data with the same hash as the 'raw' data that is being registered + + Returns: + a value map instance + """ + + if register_data: + temp: Dict[str, Union[str, Value, uuid.UUID, None]] = {} + for k, v in values.items(): + + if isinstance(v, (Value, uuid.UUID)): + temp[k] = v + continue + + if not values_schema: + details = "No schema provided." + raise KiaraException( + f"Invalid field name: '{k}' (value: {v}).", details=details + ) + + if k not in values_schema.keys(): + details = "Valid field names: " + ", ".join(values_schema.keys()) + raise KiaraException( + f"Invalid field name: '{k}' (value: {v}).", details=details + ) + + if isinstance(v, str): + + if v.startswith("alias:"): + temp[k] = v + continue + elif v.startswith("archive:"): + temp[k] = v + continue + + try: + v = uuid.UUID(v) + temp[k] = v + continue + except Exception: + if v.startswith("alias:"): # type: ignore + _v = v.replace("alias:", "") # type: ignore + else: + _v = v + + data_type = values_schema[k].type + if data_type != "string" and _v in self.list_aliases(): + temp[k] = f"alias:{_v}" + continue + + if v is None: + temp[k] = None + else: + _v = self.register_data( + data=v, + # data_type=values_schema[k].type, + data_type=values_schema[k], + reuse_existing=reuse_existing_data, + ) + temp[k] = _v + values = temp + return self.context.data_registry.load_values( + values=values, values_schema=values_schema + ) + + @tag("kiara_api") + def store_value( + self, + value: Union[str, uuid.UUID, Value], + alias: Union[str, Iterable[str], None], + allow_overwrite: bool = True, + store: Union[str, None] = None, + store_related_metadata: bool = True, + set_as_store_default: bool = False, + ) -> StoreValueResult: + """ + Store the specified value in a value store. + + If you provide values for the 'data_store' and/or 'alias_store' other than 'default', you need + to make sure those stores are registered with the current context. In most cases, the 'export' endpoint (to be done) will probably be an easier way to export values, which I suspect will + be the main use-case for this endpoint if any of the 'store' arguments where needed. Otherwise, this endpoint is useful to persist values for use in later seperate sessions. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValueResult' instance that is returned to see if the storing was successful. + + Arguments: + value: the value (or a reference to it) + alias: (Optional) one or several aliases for the value + allow_overwrite: whether to allow overwriting existing aliases + store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten + store_related_metadata: whether to store related metadata (comments, etc.) in the same store as the data + set_as_store_default: whether to set the specified store as the default store for the value + """ + # if isinstance(alias, str): + # alias = [alias] + + value_obj = self.get_value(value) + persisted_data: Union[None, PersistedData] = None + + try: + persisted_data = self.context.data_registry.store_value( + value=value_obj, data_store=store + ) + if alias: + self.context.alias_registry.register_aliases( + value_obj, + alias, + allow_overwrite=allow_overwrite, + alias_store=store, + ) + + if store_related_metadata: + + from kiara.registries.metadata import MetadataMatcher + + matcher = MetadataMatcher.create_matcher( + reference_item_ids=[value_obj.job_id, value_obj.value_id] + ) + + target_store: MetadataStore = self.context.metadata_registry.get_archive(store) # type: ignore + matching_metadata = self.context.metadata_registry.find_metadata_items( + matcher=matcher + ) + target_store.store_metadata_and_ref_items(matching_metadata) + + if set_as_store_default: + store_instance = self.context.data_registry.get_archive(store) + store_instance.set_archive_metadata_value( + DATA_ARCHIVE_DEFAULT_VALUE_MARKER, str(value_obj.value_id) + ) + result = StoreValueResult( + value=value_obj, + aliases=sorted(alias) if alias else [], + error=None, + persisted_data=persisted_data, + ) + except Exception as e: + log_exception(e) + result = StoreValueResult( + value=value_obj, + aliases=sorted(alias) if alias else [], + error=( + str(e) if str(e) else f"Unknown error (type '{type(e).__name__}')." + ), + persisted_data=persisted_data, + ) + + return result + + @tag("kiara_api") + def store_values( + self, + values: Union[ + str, + Value, + uuid.UUID, + Mapping[str, Union[str, uuid.UUID, Value]], + Iterable[Union[str, uuid.UUID, Value]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + store: Union[str, None] = None, + store_related_metadata: bool = True, + ) -> StoreValuesResult: + """ + Store multiple values into the (default) kiara value store. + + Convenience method to store multiple values. In a lot of cases you can be more flexible if you + loop over the values on the frontend side, and call the 'store_value' method for each value. But this might be meaningfully slower. This method has the potential to be optimized in the future. + + You have several options to provide the values and aliases you want to store: + + - as a string, in which case the item will be wrapped in a list (see non-mapping iterable below) + + - as a (non-mapping) iterable of value items, those can either be: + + - a value id (as string or uuid) + - a value alias (as string) + - a value instance + + If you do that, then the 'alias_map' argument can either be: + + - 'False', in which case no aliases will be registered + - 'True', in which case all items in the 'values' iterable must be a valid alias, and the alias will be copied without change to the new store + - a 'string', in which case all items in the 'values' iterable also must be a valid alias, and the alias that will be registered in the new store will use the string value as prefix (e.g. 'alias_map' = 'experiment1' and 'values' = ['a', 'b'] will result in the aliases 'experiment1.a' and 'experiment1.b') + - a map that uses the stringi-fied uuid of the value that should get one or several aliases as key, and a list of aliases as values + + You can also use a mapping type (like a dict) for the 'values' argument. In this case, the key is a string, and the value can be: + + - a value id (as string or uuid) + - a value alias (as string) + - a value instance + + In this case, the meaning of the 'alias_map' is as follows: + + - 'False': no aliases will be registered + - 'True': the key in the 'values' argument will be used as alias + - a string: all keys from the 'values' map will be used as alias, prefixed with the value of 'alias_map' + - another map, with a string referring to the key in the 'values' argument as key, and a list of aliases (strings) as value + + Sorry, this is all a bit convoluted, but it's the only way I could think of to make this work for all the requirements I had. In most keases, you'll only have to use 'True' or 'False' here, hopefully. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + Arguments: + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten + data_store: the registered name (or archive id as string) of the store to write the data + alias_store: the registered name (or archive id as string) of the store to persist the alias(es)/value_id mapping + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + + """ + + if isinstance(values, (str, uuid.UUID, Value)): + values = [values] + + result = {} + if not isinstance(values, Mapping): + if not alias_map: + use_aliases = False + elif alias_map and (alias_map is True or isinstance(alias_map, str)): + + invalid: List[Union[str, uuid.UUID, Value]] = [] + valid: Dict[str, List[str]] = {} + for value in values: + if not isinstance(value, str): + invalid.append(value) + continue + value_id = self.context.alias_registry.find_value_id_for_alias( + alias=value + ) + if value_id is None: + invalid.append(value) + else: + if alias_map is True: + if "#" in value: + new_alias = value.split("#")[1] + else: + new_alias = value + valid.setdefault(str(value_id), []).append(new_alias) + else: + if "#" in value: + new_alias = value.split("#")[1] + else: + new_alias = value + new_alias = f"{alias_map}{new_alias}" + valid.setdefault(str(value_id), []).append(new_alias) + if invalid: + invalid_str = ", ".join((str(x) for x in invalid)) + raise KiaraException( + msg=f"Cannot use auto-aliases with non-mapping iterable, some items are not valid aliases: {invalid_str}" + ) + else: + alias_map = valid + use_aliases = True + else: + use_aliases = True + + for value in values: + + aliases: Set[str] = set() + + value_obj = self.get_value(value) + if use_aliases: + alias_key = str(value_obj.value_id) + alias: Union[str, None] = alias_map.get(alias_key, None) # type: ignore + if alias: + aliases.update(alias) + + store_result = self.store_value( + value=value_obj, + alias=aliases, + allow_overwrite=allow_alias_overwrite, + store=store, + store_related_metadata=store_related_metadata, + ) + result[str(value_obj.value_id)] = store_result + else: + + for field_name, value in values.items(): + if alias_map is False: + aliases_map: Union[None, Iterable[str]] = None + elif alias_map is True: + aliases_map = [field_name] + elif isinstance(alias_map, str): + aliases_map = [f"{alias_map}.{field_name}"] + else: + # means it's a mapping + _aliases = alias_map.get(field_name) + if _aliases: + aliases_map = list(_aliases) + else: + aliases_map = None + + value_obj = self.get_value(value) + store_result = self.store_value( + value=value_obj, + alias=aliases_map, + allow_overwrite=allow_alias_overwrite, + store=store, + store_related_metadata=store_related_metadata, + ) + result[field_name] = store_result + + return StoreValuesResult(root=result) + + # ------------------------------------------------------------------------------------------------------------------ + # archive-related methods + @tag("kiara_api") + def import_values( + self, + source_archive: Union[str, Path], + values: Union[ + str, + Mapping[str, Union[str, uuid.UUID, Value]], + Iterable[Union[str, uuid.UUID, Value]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + source_registered_name: Union[str, None] = None, + ) -> StoreValuesResult: + """Import one or several values from an external kiara archive, along with their aliases (optional). + + For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, + and there are several ways to use them which is information I don't want to duplicate. + + If you provide aliases in the 'values' parameter, the aliases must be available in the external archive. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + source_archive: the name of the archive to store the values into + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + source_registered_name: the name to register the archive under in the context + """ + + if source_archive in [None, DEFAULT_STORE_MARKER]: + raise KiaraException( + "You cannot use the default store as source for this operation." + ) + + if alias_map is True: + pass + elif alias_map is False: + pass + elif isinstance(alias_map, str): + pass + elif isinstance(alias_map, Mapping): + pass + else: + raise KiaraException( + f"Invalid type for 'alias_map' argument: {type(alias_map)}." + ) + + source_archive_ref = self.register_archive( + archive=source_archive, # type: ignore + registered_name=source_registered_name, + create_if_not_exists=False, + allow_write_access=False, + existing_ok=True, + ) + + value_ids: Set[uuid.UUID] = set() + aliases: Set[str] = set() + + if isinstance(values, str): + values = [values] + + if not isinstance(values, Mapping): + # means we have a list of value ids/aliases + for value in values: + if isinstance(value, uuid.UUID): + value_ids.add(value) + elif isinstance(value, str): + try: + _value = uuid.UUID(value) + value_ids.add(_value) + except Exception: + aliases.add(value) + else: + raise NotImplementedError("Not implemented yet.") + + new_values: Dict[str, Union[uuid.UUID, str]] = {} + idx = 0 + for value_id in value_ids: + field = f"field_{idx}" + idx += 1 + new_values[field] = value_id + + new_alias_map = {} + for alias in aliases: + field = f"field_{idx}" + idx += 1 + new_values[field] = f"{source_archive_ref}#{alias}" + if alias_map is False: + pass + elif alias_map is True: + new_alias_map[field] = [f"{alias}"] + elif isinstance(alias_map, str): + new_alias_map[field] = [f"{alias_map}{alias}"] + else: + # means its a dict + if alias in alias_map.keys(): + for a in alias_map[alias]: + new_alias_map.setdefault(field, []).append(a) + + result: StoreValuesResult = self.store_values( + values=new_values, + alias_map=new_alias_map, + allow_alias_overwrite=allow_alias_overwrite, + ) + return result + + @tag("kiara_api") + def export_values( + self, + target_archive: Union[str, Path], + values: Union[ + str, + Mapping[str, Union[str, uuid.UUID, Value]], + Iterable[Union[str, uuid.UUID, Value]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + target_registered_name: Union[str, None] = None, + append: bool = False, + target_store_params: Union[None, Mapping[str, Any]] = None, + export_related_metadata: bool = True, + additional_archive_metadata: Union[None, Mapping[str, Any]] = None, + ) -> StoreValuesResult: + """Store one or several values along with (optional) aliases into a kiara archive. + + For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, + and there are several ways to use them which is information I don't want to duplicate. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + 'target_store_params' is used if the archive does not exist yet. The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: + + - zstd: zstd compression (default) -- fairly fast, and good compression + - none: no compression + - LZMA: LZMA compression -- very slow, but very good compression + - LZ4: LZ4 compression -- very fast, but not as good compression as zstd + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + target_store: the name of the archive to store the values into + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + target_registered_name: the name to register the archive under in the context + append: whether to append to an existing archive + target_store_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet + export_related_metadata: whether to export related metadata (e.g. job info, comments, ..) to the new archive or not + additional_archive_metadata: (optional) additional metadata to add to the archive + + """ + + if target_archive in [None, DEFAULT_STORE_MARKER]: + raise KiaraException( + "You cannot use the default store as target for this operation." + ) + + if target_store_params is None: + target_store_params = {} + + target_archive_ref = self.register_archive( + archive=target_archive, # type: ignore + registered_name=target_registered_name, + create_if_not_exists=True, + allow_write_access=True, + existing_ok=True if append else False, + **target_store_params, + ) + + result: StoreValuesResult = self.store_values( + values=values, + alias_map=alias_map, + allow_alias_overwrite=allow_alias_overwrite, + store=target_archive_ref, + store_related_metadata=export_related_metadata, + ) + + if additional_archive_metadata: + for k, v in additional_archive_metadata.items(): + self.set_archive_metadata_value(target_archive_ref, k, v) + + return result + + def register_archive( + self, + archive: Union[str, Path, "KiArchive"], + allow_write_access: bool = False, + registered_name: Union[str, None] = None, + create_if_not_exists: bool = True, + existing_ok: bool = True, + **create_params: Any, + ) -> str: + """Register a kiarchive with the current context. + + In most cases, this will be used to 'load' an existing kiarchive file and attach it to the current context. + If the file does not exist, one will be created, with the filename (without '.kiarchive' suffix) as the archive name if not specified. + + In the future this might also take a URL, but for now only local files are supported. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + archive: the uri of the archive (file path), or a [Kiarchive][kiara.interfaces.python_api.models.archive.Kiarchive] instance + allow_write_access: whether to allow write access to the archive + registered_name: the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint, if not provided, it will be auto-determined from the file name + create_if_not_exists: if the file does not exist, create it. If this is 'False', an exception will be raised if the file does not exist. + existing_ok: whether the file is allowed to exist already, if 'False', an exception will be raised if the file exists + create_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet + + Returns: + the name/alias that the archive is registered in the context, and which can be used in the 'store_value(s)' endpoint + """ + from kiara.interfaces.python_api.models.archive import KiArchive + + if not existing_ok and not create_if_not_exists: + raise KiaraException( + "Both 'existing_ok' and 'create_if_not_exists' cannot be 'False' at the same time." + ) + + if isinstance(archive, str): + archive = Path(archive) + + if isinstance(archive, Path): + + if not archive.name.endswith(".kiarchive"): + archive = archive.parent / f"{archive.name}.kiarchive" + + if archive.exists(): + if not existing_ok: + raise KiaraException( + f"Archive file '{archive.as_posix()}' already exists." + ) + archive = KiArchive.load_kiarchive( + kiara=self.context, + path=archive, + archive_name=registered_name, + allow_write_access=allow_write_access, + ) + log_message("archive.loaded", archive_name=archive.archive_name) + else: + if not create_if_not_exists: + raise KiaraException( + f"Archive file '{archive.as_posix()}' does not exist." + ) + kiarchive_alias = archive.name + if kiarchive_alias.endswith(".kiarchive"): + kiarchive_alias = kiarchive_alias[:-10] + + compression: Union[None, CHUNK_COMPRESSION_TYPE, str] = None + for k, v in create_params.items(): + if k == "compression": + compression = v + else: + raise KiaraException( + msg=f"Invalid archive creation parameter: '{k}'." + ) + + archive = KiArchive.create_kiarchive( + kiara=self.context, + kiarchive_uri=archive.as_posix(), + allow_existing=False, + archive_name=kiarchive_alias, + allow_write_access=allow_write_access, + compression=compression, + ) + log_message("archive.created", archive_name=archive.archive_name) + + else: + raise NotImplementedError("Only local files are supported for now.") + + data_archive = archive.data_archive + assert data_archive is not None + data_alias = self.context.register_external_archive( + data_archive, + allow_write_access=allow_write_access, + ) + + alias_archive = archive.alias_archive + assert alias_archive is not None + alias_alias = self.context.register_external_archive( + alias_archive, allow_write_access=allow_write_access + ) + + job_archive = archive.job_archive + assert job_archive is not None + job_alias = self.context.register_external_archive( + job_archive, allow_write_access=allow_write_access + ) + + metadata_archive = archive.metadata_archive + assert metadata_archive is not None + metadata_alias = self.context.register_external_archive( + metadata_archive, allow_write_access=allow_write_access + ) + assert data_alias["data"] == alias_alias["alias"] + assert data_alias["data"] == job_alias["job_record"] + assert data_alias["data"] == metadata_alias["metadata"] + assert archive.archive_name == data_alias["data"] + + return archive.archive_name + + def set_archive_metadata_value( + self, + archive: Union[str, uuid.UUID], + key: str, + value: Any, + archive_type: Literal["data", "alias", "job_record", "metadata"] = "data", + ) -> None: + """Add metadata to an archive. + + Note that this is different to adding metadata to a context, since it is attached directly + to a special section of the archive itself. + """ + + if archive_type == "data": + _archive: Union[ + None, KiaraArchive + ] = self.context.data_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) + elif archive_type == "alias": + _archive = self.context.alias_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) + elif archive_type == "metadata": + _archive = self.context.metadata_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) + elif archive_type == "job_record": + _archive = self.context.job_registry.get_archive(archive) + if _archive is None: + raise KiaraException(f"Archive '{archive}' does not exist.") + _archive.set_archive_metadata_value(key, value) + else: + raise KiaraException( + f"Invalid archive type: {archive_type}. Valid types are: 'data', 'alias'." + ) + + @tag("kiara_api") + def retrieve_archive_info( + self, archive: Union[str, "KiArchive"] + ) -> "KiArchiveInfo": + """Retrieve information about an archive at the specified local path + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + archive: the uri of the archive (file path) + + Returns: + a [KiarchiveInfo][kiara.interfaces.python_api.models.archive.KiarchiveInfo] instance, containing details about the archive + """ + + from kiara.interfaces.python_api.models.archive import KiArchive + from kiara.models.archives import KiArchiveInfo + + if not isinstance(archive, KiArchive): + archive = KiArchive.load_kiarchive(kiara=self.context, path=archive) + + kiarchive_info = KiArchiveInfo.create_from_instance( + kiara=self.context, instance=archive + ) + return kiarchive_info + + @tag("kiara_api") + def export_archive( + self, + target_archive: Union[str, Path], + target_registered_name: Union[str, None] = None, + append: bool = False, + no_aliases: bool = False, + target_store_params: Union[None, Mapping[str, Any]] = None, + ) -> StoreValuesResult: + """Export all data from the default store in your context into the specfied archive path. + + The target archives will be registered into the context, either using the provided registered_name, or the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is already registered into the context. + This will be added later on. + + Also, currently you can only export all data from the default store, there is no way to select only a sub-set. This will + also be supported later on. + + The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: + + - zstd: zstd compression (default) -- fairly fast, and good compression + - none: no compression + - LZMA: LZMA compression -- very slow, but very good compression + - LZ4: LZ4 compression -- very fast, but not as good compression as zstd + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + target_archive: the registered_name or uri of the target archive + target_registered_name: the name/alias that the archive should be registered in the context (if necessary) + append: whether to append to an existing archive or error out if the target already exists + no_aliases: whether to skip importing aliases + target_store_params: additional parameters to pass to the 'create_kiarchive' method if the target file does not exist yet + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + """ + + result = self.copy_archive( + source_archive=DEFAULT_STORE_MARKER, + target_archive=target_archive, + target_registered_name=target_registered_name, + append=append, + target_store_params=target_store_params, + no_aliases=no_aliases, + ) + return result + + @tag("kiara_api") + def import_archive( + self, + source_archive: Union[str, Path], + source_registered_name: Union[str, None] = None, + no_aliases: bool = False, + ) -> StoreValuesResult: + """Import all data from the specified archive into the current contexts default data & alias store. + + The source target will be registered into the context, either using the provided registered_name, otherwise the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will be added later on. + + Also, currently you can only import all data into the default store, there is no way to select only a sub-set. This will + also be supported later on. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + source_archive: the registered_name or uri of the source archive + source_registered_name: the name/alias that the archive should be registered in the context (if necessary) + no_aliases: whether to skip importing aliases + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + + """ + + result = self.copy_archive( + source_archive=source_archive, + target_archive=DEFAULT_STORE_MARKER, + source_registered_name=source_registered_name, + no_aliases=no_aliases, + ) + return result + + def copy_archive( + self, + source_archive: Union[None, str, Path], + target_archive: Union[None, str, Path] = None, + source_registered_name: Union[str, None] = None, + target_registered_name: Union[str, None] = None, + append: bool = False, + no_aliases: bool = False, + target_store_params: Union[None, Mapping[str, Any]] = None, + ) -> StoreValuesResult: + """Import all data from the specified archive into the current context. + + The archives will be registered into the context, either using the provided registered_name, otherwise the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will be added later on. + + The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: + + - zstd: zstd compression (default) -- fairly fast, and good compression + - none: no compression + - LZMA: LZMA compression -- very slow, but very good compression + - LZ4: LZ4 compression -- very fast, but not as good compression as zstd + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + source_archive: the registered_name or uri of the source archive, if None, the context default data/alias store will be used + target_archive: the registered_name or uri of the target archive, defaults to the context default data/alias store + source_registered_name: the name/alias that the archive should be registered in the context (if necessary) + target_registered_name: the name/alias that the archive should be registered in the context (if necessary) + append: whether to append to an existing archive or error out if the target already exists + no_aliases: whether to skip importing aliases + target_store_params: additional parameters to pass to the 'create_kiarchive' method if the target file does not exist yet + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + + """ + + if source_archive in [None, DEFAULT_STORE_MARKER]: + source_archive_ref = DEFAULT_STORE_MARKER + else: + source_archive_ref = self.register_archive( + archive=source_archive, # type: ignore + registered_name=source_registered_name, + create_if_not_exists=False, + existing_ok=True, + ) + + if target_archive in [None, DEFAULT_STORE_MARKER]: + target_archive_ref = DEFAULT_STORE_MARKER + else: + if target_store_params is None: + target_store_params = {} + target_archive_ref = self.register_archive( + archive=target_archive, # type: ignore + registered_name=target_registered_name, + create_if_not_exists=True, + allow_write_access=True, + existing_ok=True if append else False, + **target_store_params, + ) + + if source_archive_ref == target_archive_ref: + raise KiaraException( + f"Source and target archive cannot be the same: {source_archive_ref} != {target_archive_ref}" + ) + + source_values = self.list_values( + in_data_archives=[source_archive_ref], allow_internal=True, has_alias=False + ).values() + + if not no_aliases: + aliases = self.list_aliases(in_data_archives=[source_archive_ref]) + alias_map: Union[bool, Dict[str, List[str]]] = {} + for alias, value in aliases.items(): + + if source_archive_ref != DEFAULT_STORE_MARKER: + # TODO: maybe add a matcher arg to the list_aliases endpoint + if not alias.startswith(f"{source_archive_ref}#"): + continue + alias_map.setdefault(str(value.value_id), []).append( # type: ignore + alias[len(source_archive_ref) + 1 :] + ) + else: + if "#" in alias: + continue + alias_map.setdefault(str(value.value_id), []).append(alias) # type: ignore + else: + alias_map = False + + result: StoreValuesResult = self.store_values( + source_values, alias_map=alias_map, store=target_archive_ref + ) + return result + + # ------------------------------------------------------------------------------------------------------------------ + # operation-related methods + + def get_operation_type( + self, op_type: Union[str, Type[OperationType]] + ) -> OperationType: + """Get the management object for the specified operation type.""" + return self.context.operation_registry.get_operation_type(op_type=op_type) + + def retrieve_operation_type_info( + self, op_type: Union[str, Type[OperationType]] + ) -> OperationTypeInfo: + """Get an info object for the specified operation type.""" + _op_type = self.get_operation_type(op_type=op_type) + return OperationTypeInfo.create_from_type_class( + kiara=self.context, type_cls=_op_type.__class__ + ) + + def find_operation_id( + self, module_type: str, module_config: Union[None, Mapping[str, Any]] = None + ) -> Union[None, str]: + """ + Try to find the registered operation id for the specified module type and configuration. + + Arguments: + module_type: the module type + module_config: the module configuration + + Returns: + the registered operation id, if found, or None + """ + manifest = self.context.create_manifest( + module_or_operation=module_type, config=module_config + ) + return self.context.operation_registry.find_operation_id(manifest=manifest) + + def assemble_filter_pipeline_config( + self, + data_type: str, + filters: Union[str, Iterable[str], Mapping[str, str]], + endpoint: Union[None, Manifest, str] = None, + endpoint_input_field: Union[str, None] = None, + endpoint_step_id: Union[str, None] = None, + extra_input_aliases: Union[None, Mapping[str, str]] = None, + extra_output_aliases: Union[None, Mapping[str, str]] = None, + ) -> "PipelineConfig": + """ + Assemble a (pipeline) module config to filter values of a specific data type. + + NOTE: this is a preliminary endpoint, and might go away in the future. If you have a need for this + functionality, please let me know your requirements and we can work on fleshing this out. + + Optionally, a module that uses the filtered dataset as input can be specified. + + # TODO: document filter names + For the 'filters' argument, the accepted inputs are: + - a string, in which case a single-step pipeline will be created, with the string referencing the operation id or filter + - a list of strings: in which case a multi-step pipeline will be created, the step_ids will be calculated automatically + - a map of string pairs: the keys are step ids, the values operation ids or filter names + + Arguments: + data_type: the type of the data to filter + filters: a list of operation ids or filter names (and potentiall step_ids if type is a mapping) + endpoint: optional module to put as last step in the created pipeline + endpoing_input_field: field name of the input that will receive the filtered value + endpoint_step_id: id to use for the endpoint step (module type name will be used if not provided) + extra_input_aliases: extra output aliases to add to the pipeline config + extra_output_aliases: extra output aliases to add to the pipeline config + + Returns: + the (pipeline) module configuration of the filter pipeline + """ + filter_op_type: FilterOperationType = self.context.operation_registry.get_operation_type("filter") # type: ignore + pipeline_config = filter_op_type.assemble_filter_pipeline_config( + data_type=data_type, + filters=filters, + endpoint=endpoint, + endpoint_input_field=endpoint_input_field, + endpoint_step_id=endpoint_step_id, + extra_input_aliases=extra_input_aliases, + extra_output_aliases=extra_output_aliases, + ) + + return pipeline_config + + # ------------------------------------------------------------------------------------------------------------------ + # metadata-related methods + + def register_metadata_item( + self, key: str, value: str, store: Union[str, None] = None + ) -> uuid.UUID: + """Register a metadata item into the specified metadata store. + + Currently, this allows you to store comments within the default kiara context. You can use any string, + as key, for example a stringified `job_id`, or `value_id`, or any other string that makes sense in + the context you are using this in. + + If you use the store argument, the store needs to be mounted into the current *kiara* context. For now, + you can ignore this and not provide any value here, since this area is still in flux. If you need + to store a metadata item into an external context, and you can't figure out how to do it, + let me know. + + Note: this is preliminary and subject to change based on your input, so please provide your thoughts + + Arguments: + key: the key under which to store the metadata (can be anything you can think of) + value: the comment you want to store + store: the store to use, by default the context default is used + + Returns: + a globally unique identifier for the metadata item + """ + + if not value: + raise KiaraException("Cannot store empty metadata item.") + + from kiara.models.metadata import CommentMetadata + + item = CommentMetadata(comment=value) + + return self.context.metadata_registry.register_metadata_item( + key=key, item=item, store=store + ) + + def find_metadata_items(self, **matcher_params: Any): + + from kiara.registries.metadata import MetadataMatcher + + matcher = MetadataMatcher.create_matcher(**matcher_params) + + return self.context.metadata_registry.find_metadata_items(matcher=matcher) + + # ------------------------------------------------------------------------------------------------------------------ + # render-related methods + + def retrieve_renderer_infos( + self, source_type: Union[str, None] = None, target_type: Union[str, None] = None + ) -> RendererInfos: + """Retrieve information about the available renderers. + + Note: this is preliminary and mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. + + Arguments: + source_type: the type of the item to render (optional filter) + target_type: the type/profile of the rendered result (optional filter) + + Returns: + a wrapper object containing the items as dictionary with renderer alias as key, and [kiara.interfaces.python_api.models.info.RendererInfo] as value + + """ + + if not source_type and not target_type: + renderers = self.context.render_registry.registered_renderers + elif source_type and not target_type: + renderers = self.context.render_registry.retrieve_renderers_for_source_type( + source_type=source_type + ) + elif target_type and not source_type: + raise KiaraException(msg="Cannot retrieve renderers for target type only.") + else: + renderers = self.context.render_registry.retrieve_renderers_for_source_target_combination( + source_type=source_type, target_type=target_type # type: ignore + ) + + group = {k.get_renderer_alias(): k for k in renderers} + infos = RendererInfos.create_from_instances(kiara=self.context, instances=group) + return infos # type: ignore + + def retrieve_renderers_for(self, source_type: str) -> List[KiaraRenderer]: + """Retrieve available renderer instances for a specific data type. + + Note: this is not preliminary, and, mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. + """ + + return self.context.render_registry.retrieve_renderers_for_source_type( + source_type=source_type + ) + + def render( + self, + item: Any, + source_type: str, + target_type: str, + render_config: Union[Mapping[str, Any], None] = None, + ) -> Any: + """Render an internal instance of a supported source type into one of the supported target types. + + Note: this is not preliminary, and, mainly used in the cli, if another use-case comes up let me know and I'll make this more generic, and an 'official' endpoint. + + To find out the supported source/target combinations, you can use the kiara cli: + + ``` + kiara render list-renderers + ``` + or, for a filtered list: + ```` + kiara render --source-type pipeline list-renderers + ``` + + What Python types are actually supported for the 'item' argument depends on the source_type of the renderer you are calling, for example if that is a pipeline, most of the ways to specify a pipeline would be supported (operation_id, pipeline file, etc.). This might need more documentation, let me know what exactly is needed in a support ticket and I'll add that information. + + Arguments: + item: the item to render + source_type: the type of the item to render + target_type: the type/profile of the rendered result + render_config: optional configuration, depends on the renderer that is called + + """ + + registry = self.context.render_registry + result = registry.render( + item=item, + source_type=source_type, + target_type=target_type, + render_config=render_config, + ) + return result + + def assemble_render_pipeline( + self, + data_type: str, + target_format: Union[str, Iterable[str]] = "string", + filters: Union[None, str, Iterable[str], Mapping[str, str]] = None, + use_pretty_print: bool = False, + ) -> Operation: + """ + Create a manifest describing a transformation that renders a value of the specified data type in the target format. + + NOTE: this is a preliminary endpoint, don't use in anger yet. + + If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches + the value type of the source value, and the provided target format. + + Arguments: + value: the value (or value id) + target_format: the format into which to render the value + filters: a list of filters to apply to the value before rendering it + use_pretty_print: if True, use a 'pretty_print' operation instead of 'render_value' + + Returns: + the manifest for the transformation + """ + if data_type not in self.context.data_type_names: + raise DataTypeUnknownException(data_type=data_type) + + if use_pretty_print: + pretty_print_op_type: PrettyPrintOperationType = ( + self.context.operation_registry.get_operation_type("pretty_print") + ) # type: ignore + ops = pretty_print_op_type.get_target_types_for(data_type) + else: + render_op_type: ( + RenderValueOperationType + ) = self.context.operation_registry.get_operation_type( + # type: ignore + "render_value" + ) # type: ignore + ops = render_op_type.get_render_operations_for_source_type(data_type) + + if isinstance(target_format, str): + target_format = [target_format] + + match = None + for _target_type in target_format: + if _target_type not in ops.keys(): + continue + match = ops[_target_type] + break + + if not match: + if not ops: + msg = f"No render operations registered for source type '{data_type}'." + else: + msg = f"Registered target types for source type '{data_type}': {', '.join(ops.keys())}." + raise Exception( + f"No render operation for source type '{data_type}' to target type(s) registered: '{', '.join(target_format)}'. {msg}" + ) + + if filters: + # filter_op_type: FilterOperationType = self._kiara.operation_registry.get_operation_type("filter") # type: ignore + endpoint = Manifest( + module_type=match.module_type, module_config=match.module_config + ) + extra_input_aliases = {"render_value.render_config": "render_config"} + extra_output_aliases = { + "render_value.render_value_result": "render_value_result" + } + pipeline_config = self.assemble_filter_pipeline_config( + data_type=data_type, + filters=filters, + endpoint=endpoint, + endpoint_input_field="value", + endpoint_step_id="render_value", + extra_input_aliases=extra_input_aliases, + extra_output_aliases=extra_output_aliases, + ) + manifest = Manifest( + module_type="pipeline", module_config=pipeline_config.model_dump() + ) + module = self.context.module_registry.create_module(manifest=manifest) + operation = Operation.create_from_module(module, doc=pipeline_config.doc) + else: + operation = match + + return operation + + # ------------------------------------------------------------------------------------------------------------------ + # job-related methods + def queue_manifest( + self, + manifest: Manifest, + inputs: Union[None, Mapping[str, Any]] = None, + **job_metadata: Any, + ) -> uuid.UUID: + """ + Queue a job using the provided manifest to describe the module and config that should be executed. + + You probably want to use 'queue_job' instead. + + Arguments: + manifest: the manifest + inputs: the job inputs (can be either references to values, or raw inputs + + Returns: + a result value map instance + """ + + if self.context.runtime_config.runtime_profile == "dharpa": + if not job_metadata: + raise Exception( + "No job metadata provided. You need to provide a 'comment' argument when running your job." + ) + + if "comment" not in job_metadata.keys(): + raise KiaraException(msg="You need to provide a 'comment' for the job.") + + save_values = True + else: + save_values = False + + if inputs is None: + inputs = {} + + job_config = self.context.job_registry.prepare_job_config( + manifest=manifest, inputs=inputs + ) + + job_id = self.context.job_registry.execute_job( + job_config=job_config, wait=False, auto_save_result=save_values + ) + + if job_metadata: + self.context.metadata_registry.register_job_metadata_items( + job_id=job_id, items=job_metadata + ) + + return job_id + + def run_manifest( + self, + manifest: Manifest, + inputs: Union[None, Mapping[str, Any]] = None, + **job_metadata: Any, + ) -> ValueMapReadOnly: + """ + Run a job using the provided manifest to describe the module and config that should be executed. + + You probably want to use 'run_job' instead. + + Arguments: + manifest: the manifest + inputs: the job inputs (can be either references to values, or raw inputs + job_metadata: additional metadata to store with the job + + Returns: + a result value map instance + """ + job_id = self.queue_manifest(manifest=manifest, inputs=inputs, **job_metadata) + return self.context.job_registry.retrieve_result(job_id=job_id) + + def queue_job( + self, + operation: Union[str, Path, Manifest, OperationInfo, JobDesc], + inputs: Union[Mapping[str, Any], None], + operation_config: Union[None, Mapping[str, Any]] = None, + **job_metadata: Any, + ) -> uuid.UUID: + """ + Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. + + This is a convenience method that auto-detects what is meant by the 'operation' string input argument. + + If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute + set, it will be ignored, so you'll have to store any results manually. + + Arguments: + operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. + inputs: the operation inputs + operation_config: the (optional) module config in case 'operation' is a module name + job_metadata: additional metadata to store with the job + + Returns: + the queued job id + """ + + if inputs is None: + inputs = {} + + if isinstance(operation, str): + if os.path.isfile(operation): + job_path = Path(operation) + if not job_path.is_file(): + raise Exception( + f"Can't queue job from file '{job_path.as_posix()}': file does not exist/not a file." + ) + + op_data = get_data_from_file(job_path) + if isinstance(op_data, Mapping) and "operation" in op_data.keys(): + try: + repl_dict: Dict[str, Any] = { + "this_dir": job_path.parent.as_posix() + } + job_data = replace_var_names_in_obj( + op_data, repl_dict=repl_dict + ) + job_data["job_alias"] = job_path.stem + job_desc = JobDesc(**job_data) + _operation: Union[Manifest, str] = job_desc.get_operation( + kiara_api=self + ) + if job_desc.inputs: + _inputs = dict(job_desc.inputs) + _inputs.update(inputs) + inputs = _inputs + except Exception as e: + raise KiaraException( + f"Failed to parse job description file: {operation}", + parent=e, + ) + else: + _operation = job_path.as_posix() + else: + _operation = operation + elif isinstance(operation, Path): + if not operation.is_file(): + raise Exception( + f"Can't queue job from file '{operation.as_posix()}': file does not exist/not a file." + ) + _operation = operation.as_posix() + elif isinstance(operation, OperationInfo): + _operation = operation.operation + elif isinstance(operation, JobDesc): + if operation_config: + raise KiaraException( + "Specifying 'operation_config' when operation is a job_desc is invalid." + ) + _operation = operation.get_operation(kiara_api=self) + if operation.inputs: + _inputs = dict(operation.inputs) + _inputs.update(inputs) + inputs = _inputs + else: + _operation = operation + + if not isinstance(_operation, Manifest): + manifest: Manifest = create_operation( + module_or_operation=_operation, + operation_config=operation_config, + kiara=self.context, + ) + else: + manifest = _operation + + job_id = self.queue_manifest(manifest=manifest, inputs=inputs, **job_metadata) + + return job_id + + def run_job( + self, + operation: Union[str, Path, Manifest, OperationInfo, JobDesc], + inputs: Union[None, Mapping[str, Any]] = None, + operation_config: Union[None, Mapping[str, Any]] = None, + **job_metadata: Any, + ) -> ValueMapReadOnly: + """ + Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. + + This is a convenience method that auto-detects what is meant by the 'operation' string input argument. + + In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead, + since this is a blocking operation. + + If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute + set, it will be ignored, so you'll have to store any results manually. + + Arguments: + operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. + inputs: the operation inputs + operation_config: the (optional) module config in case 'operation' is a module name + **job_metadata: additional metadata to store with the job + + Returns: + the job result value map + + """ + if inputs is None: + inputs = {} + + job_id = self.queue_job( + operation=operation, + inputs=inputs, + operation_config=operation_config, + **job_metadata, + ) + return self.context.job_registry.retrieve_result(job_id=job_id) + + @tag("kiara_api") + def get_job(self, job_id: Union[str, uuid.UUID]) -> "ActiveJob": + """Retrieve the status of the job with the provided id.""" + if isinstance(job_id, str): + job_id = uuid.UUID(job_id) + + job_status = self.context.job_registry.get_job(job_id=job_id) + return job_status + + @tag("kiara_api") + def get_job_result(self, job_id: Union[str, uuid.UUID]) -> ValueMapReadOnly: + """Retrieve the result(s) of the specified job.""" + if isinstance(job_id, str): + job_id = uuid.UUID(job_id) + + result = self.context.job_registry.retrieve_result(job_id=job_id) + return result + + @tag("kiara_api") + def list_all_job_record_ids(self) -> List[uuid.UUID]: + """List all available job ids in this kiara context, ordered from newest to oldest, including internal jobs. + + This should be faster than `list_job_record_ids` with equivalent parameters, because no filtering + needs to be done. + """ + + job_ids = self.context.job_registry.retrieve_all_job_record_ids() + return job_ids + + @tag("kiara_api") + def list_job_record_ids(self, **matcher_params: Any) -> List[uuid.UUID]: + """List all available job ids in this kiara context, ordered from newest to oldest. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example + does not return jobs marked as 'internal'. + + Arguments: + matcher_params: additional parameters to pass to the job matcher + + Returns: + a list of job ids, ordered from latest to earliest + """ + + job_ids = list(self.list_job_records(**matcher_params).keys()) + return job_ids + + @tag("kiara_api") + def list_all_job_records(self) -> Mapping[uuid.UUID, "JobRecord"]: + """List all available job records in this kiara context, ordered from newest to oldest, including internal jobs. + + This should be faster than `list_job_records` with equivalent parameters, because no filtering + needs to be done. + """ + + job_records = self.context.job_registry.retrieve_all_job_records() + return job_records + + @tag("kiara_api") + def list_job_records( + self, **matcher_params: Any + ) -> Mapping[uuid.UUID, "JobRecord"]: + """List all available job ids in this kiara context, ordered from newest to oldest. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example + does not return jobs marked as 'internal'. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. + + Arguments: + matcher_params: additional parameters to pass to the job matcher + + Returns: + a list of job details, ordered from latest to earliest + + """ + + from kiara.models.module.jobs import JobMatcher + + matcher = JobMatcher(**matcher_params) + job_records = self.context.job_registry.find_job_records(matcher=matcher) + + return job_records + + @tag("kiara_api") + def get_job_record(self, job_id: Union[str, uuid.UUID]) -> Union["JobRecord", None]: + """Retrieve the detailed job record for the specified job id. + + If no job can be found, 'None' is returned. + """ + + if isinstance(job_id, str): + job_id = uuid.UUID(job_id) + + job_record = self.context.job_registry.get_job_record(job_id=job_id) + return job_record + + def render_value( + self, + value: Union[str, uuid.UUID, Value], + target_format: Union[str, Iterable[str]] = "string", + filters: Union[None, Iterable[str], Mapping[str, str]] = None, + render_config: Union[Mapping[str, str], None] = None, + add_root_scenes: bool = True, + use_pretty_print: bool = False, + ) -> RenderValueResult: + """ + Render a value in the specified target format. + + NOTE: this is a preliminary endpoint, don't use in anger yet. + + If a list is provided as value for 'target_format', all items are tried until a 'render_value' operation is found that matches + the value type of the source value, and the provided target format. + + Arguments: + value: the value (or value id) + target_format: the format into which to render the value + filters: an (optional) list of filters + render_config: manifest specific render configuration + add_root_scenes: add root scenes to the result + use_pretty_print: use 'pretty_print' operation instead of 'render_value' + + Returns: + the rendered value data, and any related scenes, if applicable + """ + _value = self.get_value(value) + try: + render_operation: Union[None, Operation] = self.assemble_render_pipeline( + data_type=_value.data_type_name, + target_format=target_format, + filters=filters, + use_pretty_print=use_pretty_print, + ) + + except Exception as e: + + log_message( + "create_render_pipeline.failure", + source_type=_value.data_type_name, + target_format=target_format, + error=e, + ) + + if use_pretty_print: + pretty_print_ops: PrettyPrintOperationType = self.context.operation_registry.get_operation_type("pretty_print") # type: ignore + if not isinstance(target_format, str): + raise NotImplementedError( + "Can't handle multiple target formats for 'render_value' yet." + ) + render_operation = ( + pretty_print_ops.get_operation_for_render_combination( + source_type="any", target_type=target_format + ) + ) + else: + render_ops: RenderValueOperationType = self.context.operation_registry.get_operation_type("render_value") # type: ignore + if not isinstance(target_format, str): + raise NotImplementedError( + "Can't handle multiple target formats for 'render_value' yet." + ) + render_operation = render_ops.get_render_operation( + source_type="any", target_type=target_format + ) + + if render_operation is None: + raise Exception( + f"Could not find render operation for value '{_value.value_id}', type: {_value.value_schema.type}" + ) + + if render_config and "render_config" in render_config.keys(): + # raise NotImplementedError() + # TODO: is this necessary? + render_config = render_config["render_config"] # type: ignore + # manifest_hash = render_config["manifest_hash"] + # if manifest_hash != render_operation.manifest_hash: + # raise NotImplementedError( + # "Using a non-default render operation is not supported (yet)." + # ) + # render_config = render_config["render_config"] + + if render_config is None: + render_config = {} + else: + render_config = dict(render_config) + + # render_type = render_config.pop("render_type", None) + # if not render_type or render_type == "data": + # pass + # elif render_type == "metadata": + # pass + # elif render_type == "properties": + # pass + # elif render_type == "lineage": + # pass + + result = render_operation.run( + kiara=self.context, + inputs={"value": _value, "render_config": render_config}, + ) + + if use_pretty_print: + render_result: Value = result["rendered_value"] + value_render_data: RenderValueResult = render_result.data + else: + render_result = result["render_value_result"] + + if render_result.data_type_name != "render_value_result": + raise Exception( + f"Invalid result type for render operation: {render_result.data_type_name}" + ) + + value_render_data = render_result.data # type: ignore + + return value_render_data + + # ------------------------------------------------------------------------------------------------------------------ + # workflow-related methods + # all of the workflow-related methods are provisional experiments, so don't rely on them to be availale long term + + def list_workflow_ids(self) -> List[uuid.UUID]: + """List all available workflow ids. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + return list(self.context.workflow_registry.all_workflow_ids) + + def list_workflow_alias_names(self) -> List[str]: + """ "List all available workflow aliases. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + return list(self.context.workflow_registry.workflow_aliases.keys()) + + def get_workflow( + self, workflow: Union[str, uuid.UUID], create_if_necessary: bool = True + ) -> "Workflow": + """Retrieve the workflow instance with the specified id or alias. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + no_such_alias: bool = False + workflow_id: Union[uuid.UUID, None] = None + workflow_alias: Union[str, None] = None + + if isinstance(workflow, str): + try: + workflow_id = uuid.UUID(workflow) + except Exception: + workflow_alias = workflow + try: + workflow_id = self.context.workflow_registry.get_workflow_id( + workflow_alias=workflow + ) + except NoSuchWorkflowException: + no_such_alias = True + else: + workflow_id = workflow + + if workflow_id is None: + raise Exception(f"Can't retrieve workflow for: {workflow}") + + if workflow_id in self._workflow_cache.keys(): + return self._workflow_cache[workflow_id] + + if workflow_id is None and not create_if_necessary: + if not no_such_alias: + msg = f"No workflow with id '{workflow}' registered." + else: + msg = f"No workflow with alias '{workflow}' registered." + + raise NoSuchWorkflowException(workflow=workflow, msg=msg) + + if workflow_id: + # workflow_metadata = self.context.workflow_registry.get_workflow_metadata( + # workflow=workflow_id + # ) + workflow_obj = Workflow(kiara=self.context, workflow=workflow_id) + self._workflow_cache[workflow_obj.workflow_id] = workflow_obj + else: + # means we need to create it + workflow_obj = self.create_workflow(workflow_alias=workflow_alias) + + return workflow_obj + + def retrieve_workflow_info( + self, workflow: Union[str, uuid.UUID, "Workflow"] + ) -> WorkflowInfo: + """Retrieve information about the specified workflow. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + + from kiara.interfaces.python_api.workflow import Workflow + + if isinstance(workflow, Workflow): + _workflow: Workflow = workflow + else: + _workflow = self.get_workflow(workflow) + + return WorkflowInfo.create_from_workflow(workflow=_workflow) + + def list_workflows(self, **matcher_params) -> "WorkflowsMap": + """List all available workflow sessions, indexed by their unique id.""" + + from kiara.interfaces.python_api.models.doc import WorkflowsMap + from kiara.interfaces.python_api.models.workflow import WorkflowMatcher + + workflows = {} + + matcher = WorkflowMatcher(**matcher_params) + if matcher.has_alias: + for ( + alias, + workflow_id, + ) in self.context.workflow_registry.workflow_aliases.items(): + + workflow = self.get_workflow(workflow=workflow_id) + workflows[workflow.workflow_id] = workflow + return WorkflowsMap(root={str(k): v for k, v in workflows.items()}) + else: + for workflow_id in self.context.workflow_registry.all_workflow_ids: + workflow = self.get_workflow(workflow=workflow_id) + workflows[workflow_id] = workflow + return WorkflowsMap(root={str(k): v for k, v in workflows.items()}) + + def list_workflow_aliases(self, **matcher_params) -> "WorkflowsMap": + """List all available workflow sessions that have an alias, indexed by alias. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + + from kiara.interfaces.python_api.models.doc import WorkflowsMap + from kiara.interfaces.python_api.workflow import Workflow + + if matcher_params: + matcher_params["has_alias"] = True + workflows = self.list_workflows(**matcher_params) + result: Dict[str, Workflow] = {} + for workflow in workflows.values(): + aliases = self.context.workflow_registry.get_aliases( + workflow_id=workflow.workflow_id + ) + for a in aliases: + if a in result.keys(): + raise Exception( + f"Duplicate workflow alias '{a}': this is most likely a bug." + ) + result[a] = workflow + result = {k: result[k] for k in sorted(result.keys())} + else: + # faster if not other matcher params + all_aliases = self.context.workflow_registry.workflow_aliases + result = { + a: self.get_workflow(workflow=all_aliases[a]) + for a in sorted(all_aliases.keys()) + } + + return WorkflowsMap(root=result) + + def retrieve_workflows_info(self, **matcher_params: Any) -> WorkflowGroupInfo: + """Get a map info instances for all available workflows, indexed by (stringified) workflow-id. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + workflows = self.list_workflows(**matcher_params) + + workflow_infos = WorkflowGroupInfo.create_from_workflows( + *workflows.values(), + group_title=None, + alias_map=self.context.workflow_registry.workflow_aliases, + ) + return workflow_infos + + def retrieve_workflow_aliases_info( + self, **matcher_params: Any + ) -> WorkflowGroupInfo: + """Get a map info instances for all available workflows, indexed by alias. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + workflows = self.list_workflow_aliases(**matcher_params) + workflow_infos = WorkflowGroupInfo.create_from_workflows( + *workflows.values(), + group_title=None, + alias_map=self.context.workflow_registry.workflow_aliases, + ) + return workflow_infos + + def create_workflow( + self, + workflow_alias: Union[None, str] = None, + initial_pipeline: Union[None, Path, str, Mapping[str, Any]] = None, + initial_inputs: Union[None, Mapping[str, Any]] = None, + documentation: Union[Any, None] = None, + save: bool = False, + force_alias: bool = False, + ) -> "Workflow": + """Create a workflow instance. + + NOTE: this is a provisional endpoint, don't use in anger yet + """ + + from kiara.interfaces.python_api.workflow import Workflow + + if workflow_alias is not None: + try: + uuid.UUID(workflow_alias) + raise Exception( + f"Can't create workflow, provided alias can't be a uuid: {workflow_alias}." + ) + except Exception: + pass + + workflow_id = ID_REGISTRY.generate() + metadata = WorkflowMetadata( + workflow_id=workflow_id, documentation=documentation + ) + + workflow_obj = Workflow(kiara=self.context, workflow=metadata) + if workflow_alias: + workflow_obj._pending_aliases.add(workflow_alias) + + if initial_pipeline: + operation = self.get_operation(operation=initial_pipeline) + if operation.module_type == "pipeline": + pipeline_details: PipelineOperationDetails = operation.operation_details # type: ignore + workflow_obj.add_steps(*pipeline_details.pipeline_config.steps) + input_aliases = pipeline_details.pipeline_config.input_aliases + for k, v in input_aliases.items(): + workflow_obj.set_input_alias(input_field=k, alias=v) + output_aliases = pipeline_details.pipeline_config.output_aliases + for k, v in output_aliases.items(): + workflow_obj.set_output_alias(output_field=k, alias=v) + else: + raise NotImplementedError() + + workflow_obj.set_inputs(**operation.module.config.defaults) + + if initial_inputs: + workflow_obj.set_inputs(**initial_inputs) + + self._workflow_cache[workflow_obj.workflow_id] = workflow_obj + + if save: + if force_alias and workflow_alias: + self.context.workflow_registry.unregister_alias(workflow_alias) + workflow_obj.save() + + return workflow_obj + + def _repr_html_(self): + + info = self.get_context_info() + r = info.create_renderable() + mime_bundle = r._repr_mimebundle_(include=[], exclude=[]) # type: ignore + return mime_bundle["text/html"] diff --git a/src/kiara/interfaces/python_api/kiara_api.py b/src/kiara/interfaces/python_api/kiara_api.py new file mode 100644 index 000000000..b8e0481f4 --- /dev/null +++ b/src/kiara/interfaces/python_api/kiara_api.py @@ -0,0 +1,1168 @@ +# -*- coding: utf-8 -*- +import uuid +from pathlib import Path + +# BEGIN AUTO-GENERATED-IMPORTS +from typing import TYPE_CHECKING, Any, ClassVar, Iterable, List, Mapping, Union +from uuid import UUID + +if TYPE_CHECKING: + from kiara.interfaces.python_api.models.info import ( + DataTypeClassesInfo, + DataTypeClassInfo, + KiaraPluginInfo, + KiaraPluginInfos, + ModuleTypeInfo, + ModuleTypesInfo, + OperationGroupInfo, + OperationInfo, + ValueInfo, + ValuesInfo, + ) + from kiara.interfaces.python_api.value import StoreValueResult, StoreValuesResult + from kiara.models.context import ContextInfo, ContextInfos + from kiara.models.module.operation import Operation + from kiara.models.values.value import Value, ValueMapReadOnly + +# END AUTO-GENERATED-IMPORTS + +if TYPE_CHECKING: + from kiara.context import KiaraConfig + from kiara.interfaces.python_api.models.archive import KiArchive + from kiara.interfaces.python_api.models.doc import OperationsMap + from kiara.interfaces.python_api.models.info import ( + KiaraPluginInfo, + KiaraPluginInfos, + ) + from kiara.interfaces.python_api.models.job import JobDesc + from kiara.models.archives import KiArchiveInfo + from kiara.models.metadata import KiaraMetadata + from kiara.models.module.jobs import ActiveJob, JobRecord + from kiara.models.module.manifest import Manifest + + +class KiaraAPI(object): + """Kiara API for clients. + + This class wraps a [Kiara][kiara.context.kiara.Kiara] instance, and allows easy a access to tasks that are + typically done by a frontend. The return types of each method are json seriable in most cases. + + The naming of the API endpoints follows a (loose-ish) convention: + - list_*: return a list of ids or items, if items, filtering is supported + - get_*: get specific instances of a type (operation, value, etc.) + - retrieve_*: get augmented information about an instance or type of something. This usually implies that there is some overhead, + so before you use this, make sure that there is not 'get_*' or 'list_*' endpoint that could give you what you need. + + Some methods of this class are copied (automatically) from the [BaseAPI][kiara.interfaces.python_api.base_api.BaseAPI] class, which is the actual implementation of the API. + This is done for different reasons: + - to keep the 'BaseAPI' class flexible, as it is used internally, so the `KiaraAPI` class can serve as a sort of 'stable' frontend, even if the underlying BaseAPI changes + - to avoid having to write the same documentation / code twice + - to be able to postpone the imports that are in the `base_api` module + - to be able to add instrumentation, logging, etc. to the API calls later on + + Re-generating those copied methods can be done like so: + + ``` + kiara render --source-type base_api --target-type kiara_api item kiara_api template_file=kiara/src/kiara/interfaces/python_api/kiara_api.py target_file=kiara/src/kiara/interfaces/python_api/kiara_api.py + ``` + + All endpoints that have the 'tag' annotation `kiara_api` will then be copied. + + """ + + _default_instance: ClassVar[Union["KiaraAPI", None]] = None + + @classmethod + def instance(cls) -> "KiaraAPI": + """Retrieve the default KiaraAPI instance. + + This is a convenience method to get a singleton KiaraAPI instance. If this is the first time this method is called, it loads the default *kiara* context. If this is called subsequently, it will return + the same instance, so if you or some-one (or -thing) switched that context, this might not be the case. + + So make sure you understand the implications, and if in doubt, it might be safer to create your own `KiaraAPI` instance manually. + """ + + if cls._default_instance is not None: + return cls._default_instance + + from kiara.utils.config import assemble_kiara_config + + config = assemble_kiara_config() + + api = KiaraAPI(kiara_config=config) + cls._default_instance = api + return api + + def __init__(self, kiara_config: Union["KiaraConfig", None] = None): + + from kiara.interfaces.python_api.base_api import BaseAPI + + self._api: BaseAPI = BaseAPI(kiara_config=kiara_config) + + def run_job( + self, + operation: Union[str, Path, "Manifest", "OperationInfo", "JobDesc"], + inputs: Mapping[str, Any], + comment: str, + operation_config: Union[None, Mapping[str, Any]] = None, + ) -> "ValueMapReadOnly": + """ + Run a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. + + This is a convenience method that auto-detects what is meant by the 'operation' string input argument. + + In general, try to avoid this method and use 'queue_job', 'get_job' and 'retrieve_job_result' manually instead, + since this is a blocking operation. + + If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute + set, it will be ignored, so you'll have to store any results manually. + + Arguments: + operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. + inputs: the operation inputs + comment: a (required) comment to attach to the job + operation_config: the (optional) module config in case 'operation' is a module name + + Returns: + the job result value map + + """ + + if not comment and comment != "": + from kiara.exceptions import KiaraException + + raise KiaraException(msg="Can't submit job: no comment provided.") + + return self._api.run_job( + operation=operation, + inputs=inputs, + operation_config=operation_config, + comment=comment, + ) + + def queue_job( + self, + operation: Union[str, Path, "Manifest", "OperationInfo", "JobDesc"], + inputs: Mapping[str, Any], + comment: str, + operation_config: Union[None, Mapping[str, Any]] = None, + ) -> uuid.UUID: + """ + Queue a job from a operation id, module_name (and config), or pipeline file, wait for the job to finish and retrieve the result. + + This is a convenience method that auto-detects what is meant by the 'operation' string input argument. + + If the 'operation' is a JobDesc instance, and that JobDesc instance has the 'save' attribute + set, it will be ignored, so you'll have to store any results manually. + + Arguments: + operation: a module name, operation id, or a path to a pipeline file (resolved in this order, until a match is found).. + inputs: the operation inputs + comment: a (required) comment to attach to the job + operation_config: the (optional) module config in case 'operation' is a module name + + Returns: + the queued job id + """ + + if not comment and comment != "": + from kiara.exceptions import KiaraException + + raise KiaraException(msg="Can't submit job: no comment provided.") + + return self._api.queue_job( + operation=operation, + inputs=inputs, + operation_config=operation_config, + comment=comment, + ) + + def set_job_comment( + self, job_id: Union[str, uuid.UUID], comment: str, force: bool = True + ): + """Set a comment for the specified job. + + Arguments: + job_id: the job id + comment: the comment to set + force: whether to overwrite an existing comment + """ + + from kiara.models.metadata import CommentMetadata + + if isinstance(job_id, str): + job_id = uuid.UUID(job_id) + + comment_metadata = CommentMetadata(comment=comment) + items = {"comment": comment_metadata} + + self._api.context.metadata_registry.register_job_metadata_items( + job_id=job_id, items=items + ) + + def get_job_comment(self, job_id: Union[str, uuid.UUID]) -> Union[str, None]: + """Retrieve the comment for the specified job. + + Returns 'None' if the job_id does not exist, or the job does not have a comment attached to it. + + Arguments: + job_id: the job id + + Returns: + the comment as string, or None + """ + + from kiara.models.metadata import CommentMetadata + + if isinstance(job_id, str): + job_id = uuid.UUID(job_id) + + metadata: Union[ + None, "KiaraMetadata" + ] = self._api.context.metadata_registry.retrieve_job_metadata_item( + job_id=job_id, key="comment" + ) + + if not metadata: + return None + + if not isinstance(metadata, CommentMetadata): + from kiara.exceptions import KiaraException + + raise KiaraException( + msg=f"Metadata item 'comment' for job '{job_id}' is not a comment." + ) + return metadata.comment + + # BEGIN IMPORTED-ENDPOINTS + def list_available_plugin_names( + self, regex: str = r"^kiara[-_]plugin\..*" + ) -> List[str]: + r"""Get a list of all available plugins. + + Arguments: + regex: an optional regex to indicate the plugin naming scheme (default: /$kiara[_-]plugin\..*/) + + Returns: + a list of plugin names + """ + + result: List[str] = self._api.list_available_plugin_names(regex=regex) + return result + + def retrieve_plugin_info(self, plugin_name: str) -> "KiaraPluginInfo": + """Get information about a plugin. + + This contains information about included data-types, modules, operations, pipelines, as well as metadata + about author(s), etc. + + Arguments: + plugin_name: the name of the plugin + + Returns: + a dictionary with information about the plugin + """ + + result: "KiaraPluginInfo" = self._api.retrieve_plugin_info( + plugin_name=plugin_name + ) + return result + + def retrieve_plugin_infos( + self, plugin_name_regex: str = r"^kiara[-_]plugin\..*" + ) -> "KiaraPluginInfos": + """Get information about multiple plugins. + + This is just a convenience method to get information about multiple plugins at once. + """ + + result: "KiaraPluginInfos" = self._api.retrieve_plugin_infos( + plugin_name_regex=plugin_name_regex + ) + return result + + def get_context_info(self) -> "ContextInfo": + """Retrieve information about the current kiara context. + + This contains information about the context, like its name/alias, the values & aliases it contains, and which archives are connected to it. + """ + + result: "ContextInfo" = self._api.get_context_info() + return result + + def list_context_names(self) -> List[str]: + """list the names of all available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and + whether we want to support single-file contexts in the future. + """ + + result: List[str] = self._api.list_context_names() + return result + + def retrieve_context_infos(self) -> "ContextInfos": + """Retrieve information about the available/registered contexts. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + + result: "ContextInfos" = self._api.retrieve_context_infos() + return result + + def get_current_context_name(self) -> str: + """Retrieve the name of the current context. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + + result: str = self._api.get_current_context_name() + return result + + def set_active_context(self, context_name: str, create: bool = False): + """Set the currently active context for this KiarAPI instance. + + NOTE: this functionality might be changed in the future, depending on requirements and feedback and whether we want to support single-file contexts in the future. + """ + + self._api.set_active_context(context_name=context_name, create=create) + + def list_data_type_names(self, include_profiles: bool = False) -> List[str]: + """Get a list of all registered data types. + + Arguments: + include_profiles: if True, also include the names of all registered data type profiles + """ + + result: List[str] = self._api.list_data_type_names( + include_profiles=include_profiles + ) + return result + + def retrieve_data_types_info( + self, + filter: Union[str, Iterable[str], None] = None, + include_data_type_profiles: bool = False, + python_package: Union[None, str] = None, + ) -> "DataTypeClassesInfo": + """Retrieve information about all data types. + + A data type is a Python class that inherits from [DataType[kiara.data_types.DataType], and it wraps a specific + Python class that holds the actual data and provides metadata and convenience methods for managing the data internally. Data types are not directly used by users, but they are exposed in the input/output schemas of moudles and other data-related features. + + Arguments: + filter: an optional string or (list of strings) the returned datatype ids have to match (all filters in the case of a list) + include_data_type_profiles: if True, also include the names of all registered data type profiles + python_package: if provided, only return data types that are defined in the given python package + + Returns: + an object containing all information about all data types + """ + + result: "DataTypeClassesInfo" = self._api.retrieve_data_types_info( + filter=filter, + include_data_type_profiles=include_data_type_profiles, + python_package=python_package, + ) + return result + + def retrieve_data_type_info(self, data_type_name: str) -> "DataTypeClassInfo": + """Retrieve information about a specific data type. + + Arguments: + data_type: the registered name of the data type + + Returns: + an object containing all information about a data type + """ + + result: "DataTypeClassInfo" = self._api.retrieve_data_type_info( + data_type_name=data_type_name + ) + return result + + def list_module_type_names(self) -> List[str]: + """Get a list of all registered module types.""" + + result: List[str] = self._api.list_module_type_names() + return result + + def retrieve_module_types_info( + self, + filter: Union[None, str, Iterable[str]] = None, + python_package: Union[str, None] = None, + ) -> "ModuleTypesInfo": + """Retrieve information for all available module types (or a filtered subset thereof). + + A module type is Python class that inherits from [KiaraModule][kiara.modules.KiaraModule], and is the basic + building block for processing pipelines. Module types are not used directly by users, Operations are. Operations + are instantiated modules (meaning: the module & some (optional) configuration). + + Arguments: + filter: an optional string (or list of string) the returned module names have to match (all filters in case of list) + python_package: an optional string, if provided, only modules from the specified python package are returned + + Returns: + a mapping object containing module names as keys, and information about the modules as values + """ + + result: "ModuleTypesInfo" = self._api.retrieve_module_types_info( + filter=filter, python_package=python_package + ) + return result + + def retrieve_module_type_info(self, module_type: str) -> "ModuleTypeInfo": + """Retrieve information about a specific module type. + + This can be used to retrieve information like module documentation and configuration options. + + Arguments: + module_type: the registered name of the module + + Returns: + an object containing all information about a module type + """ + + result: "ModuleTypeInfo" = self._api.retrieve_module_type_info( + module_type=module_type + ) + return result + + def list_operation_ids( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + python_packages: Union[str, None, Iterable[str]] = None, + ) -> List[str]: + """Get a list of all operation ids that match the specified filter. + + Arguments: + filter: the (optional) filter string(s), an operation must match all of them to be included in the result + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + """ + + result: List[str] = self._api.list_operation_ids( + filter=filter, + input_types=input_types, + output_types=output_types, + operation_types=operation_types, + include_internal=include_internal, + python_packages=python_packages, + ) + return result + + def get_operation( + self, + operation: Union[Mapping[str, Any], str, "Path"], + allow_external: Union[bool, None] = None, + ) -> "Operation": + """Return the operation instance with the specified id. + + The difference to the 'create_operation' endpoint is slight, in most cases you could use either of them, but this one is a bit more convenient in most cases, as it tries to do the right thing with whatever 'operation' argument you use it. The 'create_opearation' endpoint will always create a new 'Operation' instance, while this may or may not return a re-used one. + + This endpoint can be used to get information about a specific operation, like inputs/outputs scheman, documentation, etc. + + The order in which the operation argument is resolved: + - if it's a string, and an existing, registered operation_id, the associated operation is returned + - if it's a path to an existing file, the content of the file is loaded into a dict and depending on the content a pipeline module will be created, or a 'normal' manifest (if module_type is a key in the dict) + + Arguments: + operation: the operation id, module_type_name, path to a file, or url + allow_external: if True, allow loading operations from external sources (e.g. a URL), if 'None' is provided, the configured value in the runtime configuration is used. + + Returns: + operation instance data + """ + + result: "Operation" = self._api.get_operation( + operation=operation, allow_external=allow_external + ) + return result + + def list_operations( + self, + filter: Union[str, None, Iterable[str]] = None, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> "OperationsMap": + """List all available operations, optionally filter. + + Arguments: + filter: the (optional) filter string(s), an operation must match all of them to be included in the result + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + + Returns: + a dictionary with the operation id as key, and [kiara.models.module.operation.Operation] instance data as value + """ + + result: "OperationsMap" = self._api.list_operations( + filter=filter, + input_types=input_types, + output_types=output_types, + operation_types=operation_types, + python_packages=python_packages, + include_internal=include_internal, + ) + return result + + def retrieve_operation_info( + self, operation: str, allow_external: bool = False + ) -> "OperationInfo": + """Return the full information for the specified operation id. + + This is similar to the 'get_operation' method, but returns additional information. Only use this instead of + 'get_operation' if you need the additional info, as it's more expensive to get. + + Arguments: + operation: the operation id + + Returns: + augmented operation instance data + """ + + result: "OperationInfo" = self._api.retrieve_operation_info( + operation=operation, allow_external=allow_external + ) + return result + + def retrieve_operations_info( + self, + *filters: str, + input_types: Union[str, Iterable[str], None] = None, + output_types: Union[str, Iterable[str], None] = None, + operation_types: Union[str, Iterable[str], None] = None, + python_packages: Union[str, Iterable[str], None] = None, + include_internal: bool = False, + ) -> "OperationGroupInfo": + """Retrieve information about the matching operations. + + This retrieves the same list of operations as [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations], + but augments each result instance with additional information that might be useful in frontends. + + 'OperationInfo' objects contains augmented information on top of what 'normal' [Operation][kiara.models.module.operation.Operation] objects + hold, but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [list_operations][kiara.interfaces.python_api.KiaraAPI.list_operations] method + instead. + + Arguments: + filters: the (optional) filter strings, an operation must match all of them to be included in the result + include_internal: whether to include operations that are predominantly used internally in kiara. + input_types: each operation must have at least one input that matches one of the specified types + output_types: each operation must have at least one output that matches one of the specified types + operation_types: only include operations of the specified type(s) + include_internal: whether to include operations that are predominantly used internally in kiara. + python_packages: only include operations that are contained in one of the provided python packages + Returns: + a wrapper object containing a dictionary of items with value_id as key, and [kiara.interfaces.python_api.models.info.OperationInfo] as value + """ + + result: "OperationGroupInfo" = self._api.retrieve_operations_info( + *filters, + input_types=input_types, + output_types=output_types, + operation_types=operation_types, + python_packages=python_packages, + include_internal=include_internal, + ) + return result + + def list_all_value_ids(self) -> List["UUID"]: + """List all value ids in the current context. + + This returns everything, even internal values. It should be faster than using + `list_value_ids` with equivalent parameters, because no filtering has to happen. + + Returns: + all value_ids in the current context, using every registered store + """ + + result: List["UUID"] = self._api.list_all_value_ids() + return result + + def list_value_ids(self, **matcher_params: Any) -> List["UUID"]: + """List all available value ids for this kiara context. + + By default, this also includes internal values. + + This method exists mainly so frontends can retrieve a list of all value_ids that exists on the backend without + having to look up the details of each value (like [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] + does). This method can also be used with a matcher, but in this case the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] + would be preferable in most cases, because it is called under the hood, and the performance advantage of not + having to look up value details is gone. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters and defaults + + Returns: + a list of value ids + """ + + result: List["UUID"] = self._api.list_value_ids(**matcher_params) + return result + + def list_all_values(self) -> "ValueMapReadOnly": + """List all values in the current context, incl. internal ones. + + This should be faster than `list_values` with equivalent matcher params, because no + filtering has to happen. + """ + + result: "ValueMapReadOnly" = self._api.list_all_values() + return result + + def list_values(self, **matcher_params: Any) -> "ValueMapReadOnly": + """List all available (relevant) values, optionally filter. + + Retrieve information about all values that are available in the current kiara context session (both stored and non-stored). + + Check the `ValueMatcher` class for available parameters and defaults, for example this excludes + internal values by default. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with value_id as key, and [kiara.models.values.value.Value] as value + """ + + result: "ValueMapReadOnly" = self._api.list_values(**matcher_params) + return result + + def get_value(self, value: Union[str, "Value", "UUID", "Path"]) -> "Value": + """Retrieve a value instance with the specified id or alias. + + Basically a convenience method to convert any possible Python type into + a 'Value' instance. Raises an exception if no value could be found. + + Arguments: + value: a value id, alias or object that has a 'value_id' attribute. + + Returns: + the Value instance + """ + + result: "Value" = self._api.get_value(value=value) + return result + + def get_values(self, **values: Union[str, "Value", "UUID"]) -> "ValueMapReadOnly": + """Retrieve Value instances for the specified value ids or aliases. + + This is a convenience method to get fully 'hydrated' `Value` objects from references to them. + + Arguments: + values: a dictionary with value ids or aliases as keys, and value instances as values + + Returns: + a mapping with value_id as key, and [kiara.models.values.value.Value] as value + """ + + result: "ValueMapReadOnly" = self._api.get_values(**values) + return result + + def retrieve_value_info( + self, value: Union[str, "UUID", "Value", "Path"] + ) -> "ValueInfo": + """Retrieve an info object for a value. + + Companion method to 'get_value', 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.get_value] method + instead. + + Arguments: + value: a value id, alias or object that has a 'value_id' attribute. + + Returns: + the ValueInfo instance + """ + + result: "ValueInfo" = self._api.retrieve_value_info(value=value) + return result + + def retrieve_values_info(self, **matcher_params: Any) -> "ValuesInfo": + """Retrieve information about the matching values. + + This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], + but augments each result value instance with additional information that might be useful in frontends. + + 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [list_values][kiara.interfaces.python_api.KiaraAPI.list_values] method + instead. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a wrapper object containing the items as dictionary with value_id as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value + """ + + result: "ValuesInfo" = self._api.retrieve_values_info(**matcher_params) + return result + + def list_alias_names(self, **matcher_params: Any) -> List[str]: + """List all available alias keys. + + This method exists mainly so frontend can retrieve a list of all value_ids that exists on the backend without + having to look up the details of each value (like [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] + does). This method can also be used with a matcher, but in this case the [list_aliases][kiara.interfaces.python_api.KiaraAPI.list_aliases] + would be preferrable in most cases, because it is called under the hood, and the performance advantage of not + having to look up value details is gone. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a list of value ids + """ + + result: List[str] = self._api.list_alias_names(**matcher_params) + return result + + def list_aliases(self, **matcher_params: Any) -> "ValueMapReadOnly": + """List all available values that have an alias assigned, optionally filter. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with value_id as key, and [kiara.models.values.value.Value] as value + """ + + result: "ValueMapReadOnly" = self._api.list_aliases(**matcher_params) + return result + + def retrieve_aliases_info(self, **matcher_params: Any) -> "ValuesInfo": + """Retrieve information about the matching values. + + This retrieves the same list of values as [list_values][kiara.interfaces.python_api.KiaraAPI.list_values], + but augments each result value instance with additional information that might be useful in frontends. + + 'ValueInfo' objects contains augmented information on top of what 'normal' [Value][kiara.models.values.value.Value] objects + hold (like resolved properties for example), but they can take longer to create/resolve. If you don't need any + of the augmented information, just use the [get_value][kiara.interfaces.python_api.KiaraAPI.list_aliases] method + instead. + + Arguments: + matcher_params: the (optional) filter parameters, check the [ValueMatcher][kiara.models.values.matchers.ValueMatcher] class for available parameters + + Returns: + a dictionary with a value alias as key, and [kiara.interfaces.python_api.models.values.ValueInfo] as value + """ + + result: "ValuesInfo" = self._api.retrieve_aliases_info(**matcher_params) + return result + + def store_value( + self, + value: Union[str, "UUID", "Value"], + alias: Union[str, Iterable[str], None], + allow_overwrite: bool = True, + store: Union[str, None] = None, + store_related_metadata: bool = True, + set_as_store_default: bool = False, + ) -> "StoreValueResult": + """Store the specified value in a value store. + + If you provide values for the 'data_store' and/or 'alias_store' other than 'default', you need + to make sure those stores are registered with the current context. In most cases, the 'export' endpoint (to be done) will probably be an easier way to export values, which I suspect will + be the main use-case for this endpoint if any of the 'store' arguments where needed. Otherwise, this endpoint is useful to persist values for use in later seperate sessions. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValueResult' instance that is returned to see if the storing was successful. + + Arguments: + value: the value (or a reference to it) + alias: (Optional) one or several aliases for the value + allow_overwrite: whether to allow overwriting existing aliases + store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten + store_related_metadata: whether to store related metadata (comments, etc.) in the same store as the data + set_as_store_default: whether to set the specified store as the default store for the value + """ + + result: "StoreValueResult" = self._api.store_value( + value=value, + alias=alias, + allow_overwrite=allow_overwrite, + store=store, + store_related_metadata=store_related_metadata, + set_as_store_default=set_as_store_default, + ) + return result + + def store_values( + self, + values: Union[ + str, + "Value", + "UUID", + Mapping[str, Union[str, "UUID", "Value"]], + Iterable[Union[str, "UUID", "Value"]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + store: Union[str, None] = None, + store_related_metadata: bool = True, + ) -> "StoreValuesResult": + """Store multiple values into the (default) kiara value store. + + Convenience method to store multiple values. In a lot of cases you can be more flexible if you + loop over the values on the frontend side, and call the 'store_value' method for each value. But this might be meaningfully slower. This method has the potential to be optimized in the future. + + You have several options to provide the values and aliases you want to store: + + - as a string, in which case the item will be wrapped in a list (see non-mapping iterable below) + + - as a (non-mapping) iterable of value items, those can either be: + + - a value id (as string or uuid) + - a value alias (as string) + - a value instance + + If you do that, then the 'alias_map' argument can either be: + + - 'False', in which case no aliases will be registered + - 'True', in which case all items in the 'values' iterable must be a valid alias, and the alias will be copied without change to the new store + - a 'string', in which case all items in the 'values' iterable also must be a valid alias, and the alias that will be registered in the new store will use the string value as prefix (e.g. 'alias_map' = 'experiment1' and 'values' = ['a', 'b'] will result in the aliases 'experiment1.a' and 'experiment1.b') + - a map that uses the stringi-fied uuid of the value that should get one or several aliases as key, and a list of aliases as values + + You can also use a mapping type (like a dict) for the 'values' argument. In this case, the key is a string, and the value can be: + + - a value id (as string or uuid) + - a value alias (as string) + - a value instance + + In this case, the meaning of the 'alias_map' is as follows: + + - 'False': no aliases will be registered + - 'True': the key in the 'values' argument will be used as alias + - a string: all keys from the 'values' map will be used as alias, prefixed with the value of 'alias_map' + - another map, with a string referring to the key in the 'values' argument as key, and a list of aliases (strings) as value + + Sorry, this is all a bit convoluted, but it's the only way I could think of to make this work for all the requirements I had. In most keases, you'll only have to use 'True' or 'False' here, hopefully. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + Arguments: + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + store: in case data and alias store names are the same, you can use this, if you specify one or both of the others, this will be overwritten + data_store: the registered name (or archive id as string) of the store to write the data + alias_store: the registered name (or archive id as string) of the store to persist the alias(es)/value_id mapping + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + """ + + result: "StoreValuesResult" = self._api.store_values( + values=values, + alias_map=alias_map, + allow_alias_overwrite=allow_alias_overwrite, + store=store, + store_related_metadata=store_related_metadata, + ) + return result + + def import_values( + self, + source_archive: Union[str, "Path"], + values: Union[ + str, + Mapping[str, Union[str, "UUID", "Value"]], + Iterable[Union[str, "UUID", "Value"]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + source_registered_name: Union[str, None] = None, + ) -> "StoreValuesResult": + """Import one or several values from an external kiara archive, along with their aliases (optional). + + For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, + and there are several ways to use them which is information I don't want to duplicate. + + If you provide aliases in the 'values' parameter, the aliases must be available in the external archive. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + source_archive: the name of the archive to store the values into + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + source_registered_name: the name to register the archive under in the context + """ + + result: "StoreValuesResult" = self._api.import_values( + source_archive=source_archive, + values=values, + alias_map=alias_map, + allow_alias_overwrite=allow_alias_overwrite, + source_registered_name=source_registered_name, + ) + return result + + def export_values( + self, + target_archive: Union[str, "Path"], + values: Union[ + str, + Mapping[str, Union[str, "UUID", "Value"]], + Iterable[Union[str, "UUID", "Value"]], + ], + alias_map: Union[Mapping[str, Iterable[str]], bool, str] = False, + allow_alias_overwrite: bool = True, + target_registered_name: Union[str, None] = None, + append: bool = False, + target_store_params: Union[None, Mapping[str, Any]] = None, + export_related_metadata: bool = True, + additional_archive_metadata: Union[None, Mapping[str, Any]] = None, + ) -> "StoreValuesResult": + """Store one or several values along with (optional) aliases into a kiara archive. + + For the 'values' & 'alias_map' arguments, see the 'store_values' endpoint, as they will be forwarded to that endpoint as is, + and there are several ways to use them which is information I don't want to duplicate. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + 'target_store_params' is used if the archive does not exist yet. The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: + + - zstd: zstd compression (default) -- fairly fast, and good compression + - none: no compression + - LZMA: LZMA compression -- very slow, but very good compression + - LZ4: LZ4 compression -- very fast, but not as good compression as zstd + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + target_store: the name of the archive to store the values into + values: an iterable/map of value keys/values + alias_map: a map of value keys aliases + allow_alias_overwrite: whether to allow overwriting existing aliases + target_registered_name: the name to register the archive under in the context + append: whether to append to an existing archive + target_store_params: additional parameters to pass to the 'create_kiarchive' method if the file does not exist yet + export_related_metadata: whether to export related metadata (e.g. job info, comments, ..) to the new archive or not + additional_archive_metadata: (optional) additional metadata to add to the archive + """ + + result: "StoreValuesResult" = self._api.export_values( + target_archive=target_archive, + values=values, + alias_map=alias_map, + allow_alias_overwrite=allow_alias_overwrite, + target_registered_name=target_registered_name, + append=append, + target_store_params=target_store_params, + export_related_metadata=export_related_metadata, + additional_archive_metadata=additional_archive_metadata, + ) + return result + + def retrieve_archive_info( + self, archive: Union[str, "KiArchive"] + ) -> "KiArchiveInfo": + """Retrieve information about an archive at the specified local path + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will probably be added later on, let me know if there is demand, then I'll prioritize. + + # NOTE: this is a preliminary endpoint, and might be changed in the future. If you have a use-case for this, please let me know. + + Arguments: + archive: the uri of the archive (file path) + + Returns: + a [KiarchiveInfo][kiara.interfaces.python_api.models.archive.KiarchiveInfo] instance, containing details about the archive + """ + + result: "KiArchiveInfo" = self._api.retrieve_archive_info(archive=archive) + return result + + def export_archive( + self, + target_archive: Union[str, "Path"], + target_registered_name: Union[str, None] = None, + append: bool = False, + no_aliases: bool = False, + target_store_params: Union[None, Mapping[str, Any]] = None, + ) -> "StoreValuesResult": + """Export all data from the default store in your context into the specfied archive path. + + The target archives will be registered into the context, either using the provided registered_name, or the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is already registered into the context. + This will be added later on. + + Also, currently you can only export all data from the default store, there is no way to select only a sub-set. This will + also be supported later on. + + The one supported value for the 'target_store_params' argument currently is 'compression', which can be one of: + + - zstd: zstd compression (default) -- fairly fast, and good compression + - none: no compression + - LZMA: LZMA compression -- very slow, but very good compression + - LZ4: LZ4 compression -- very fast, but not as good compression as zstd + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + target_archive: the registered_name or uri of the target archive + target_registered_name: the name/alias that the archive should be registered in the context (if necessary) + append: whether to append to an existing archive or error out if the target already exists + no_aliases: whether to skip importing aliases + target_store_params: additional parameters to pass to the 'create_kiarchive' method if the target file does not exist yet + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + """ + + result: "StoreValuesResult" = self._api.export_archive( + target_archive=target_archive, + target_registered_name=target_registered_name, + append=append, + no_aliases=no_aliases, + target_store_params=target_store_params, + ) + return result + + def import_archive( + self, + source_archive: Union[str, "Path"], + source_registered_name: Union[str, None] = None, + no_aliases: bool = False, + ) -> "StoreValuesResult": + """Import all data from the specified archive into the current contexts default data & alias store. + + The source target will be registered into the context, either using the provided registered_name, otherwise the name + will be auto-determined from the archive metadata. + + Currently, this only works with an external archive file, not with an archive that is registered into the context. + This will be added later on. + + Also, currently you can only import all data into the default store, there is no way to select only a sub-set. This will + also be supported later on. + + This method does not raise an error if the storing of the value fails, so you have to investigate the + 'StoreValuesResult' instance that is returned to see if the storing was successful + + Arguments: + source_archive: the registered_name or uri of the source archive + source_registered_name: the name/alias that the archive should be registered in the context (if necessary) + no_aliases: whether to skip importing aliases + + Returns: + an object outlining which values (identified by the specified value key or an enumerated index) where stored and how + """ + + result: "StoreValuesResult" = self._api.import_archive( + source_archive=source_archive, + source_registered_name=source_registered_name, + no_aliases=no_aliases, + ) + return result + + def get_job(self, job_id: Union[str, "UUID"]) -> "ActiveJob": + """Retrieve the status of the job with the provided id.""" + + result: "ActiveJob" = self._api.get_job(job_id=job_id) + return result + + def get_job_result(self, job_id: Union[str, "UUID"]) -> "ValueMapReadOnly": + """Retrieve the result(s) of the specified job.""" + + result: "ValueMapReadOnly" = self._api.get_job_result(job_id=job_id) + return result + + def list_all_job_record_ids(self) -> List["UUID"]: + """List all available job ids in this kiara context, ordered from newest to oldest, including internal jobs. + + This should be faster than `list_job_record_ids` with equivalent parameters, because no filtering + needs to be done. + """ + + result: List["UUID"] = self._api.list_all_job_record_ids() + return result + + def list_job_record_ids(self, **matcher_params: Any) -> List["UUID"]: + """List all available job ids in this kiara context, ordered from newest to oldest. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example + does not return jobs marked as 'internal'. + + Arguments: + matcher_params: additional parameters to pass to the job matcher + + Returns: + a list of job ids, ordered from latest to earliest + """ + + result: List["UUID"] = self._api.list_job_record_ids(**matcher_params) + return result + + def list_all_job_records(self) -> Mapping["UUID", "JobRecord"]: + """List all available job records in this kiara context, ordered from newest to oldest, including internal jobs. + + This should be faster than `list_job_records` with equivalent parameters, because no filtering + needs to be done. + """ + + result: Mapping["UUID", "JobRecord"] = self._api.list_all_job_records() + return result + + def list_job_records(self, **matcher_params: Any) -> Mapping["UUID", "JobRecord"]: + """List all available job ids in this kiara context, ordered from newest to oldest. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. By default, this method for example + does not return jobs marked as 'internal'. + + You can look up the supported matcher parameter arguments via the [JobMatcher][kiara.models.module.jobs.JobMatcher] class. + + Arguments: + matcher_params: additional parameters to pass to the job matcher + + Returns: + a list of job details, ordered from latest to earliest + """ + + result: Mapping["UUID", "JobRecord"] = self._api.list_job_records( + **matcher_params + ) + return result + + def get_job_record(self, job_id: Union[str, "UUID"]) -> Union["JobRecord", None]: + """Retrieve the detailed job record for the specified job id. + + If no job can be found, 'None' is returned. + """ + + result: Union["JobRecord", None] = self._api.get_job_record(job_id=job_id) + return result + + # END IMPORTED-ENDPOINTS diff --git a/src/kiara/interfaces/python_api/models/archive.py b/src/kiara/interfaces/python_api/models/archive.py index 5f964a1de..f78ef041e 100644 --- a/src/kiara/interfaces/python_api/models/archive.py +++ b/src/kiara/interfaces/python_api/models/archive.py @@ -5,13 +5,14 @@ from pydantic import Field, PrivateAttr -from kiara.defaults import CHUNK_COMPRESSION_TYPE +from kiara.defaults import CHUNK_COMPRESSION_TYPE, DEFAULT_CHUNK_COMPRESSION from kiara.models import KiaraModel if TYPE_CHECKING: from kiara.context import Kiara from kiara.registries.aliases import AliasArchive, AliasStore from kiara.registries.data import DataArchive, DataStore + from kiara.registries.jobs import JobArchive, JobStore from kiara.registries.metadata import MetadataArchive, MetadataStore @@ -62,36 +63,40 @@ def load_kiarchive( alias_archive_config = None alias_archive = None - if data_archive is None and alias_archive is None: - raise Exception(f"No data archive found in file: {path}") - elif data_archive: - if alias_archive is not None: - if data_archive.archive_id != alias_archive.archive_id: + if "job_record" in archives.keys(): + jobs_archive: Union[JobArchive, None] = archives["job_record"] # type: ignore + jobs_archive_config: Union[Mapping[str, Any], None] = jobs_archive.config.model_dump() # type: ignore + else: + jobs_archive_config = None + jobs_archive = None + + _archives = [ + x + for x in (data_archive, alias_archive, metadata_archive, jobs_archive) + if x is not None + ] + if not _archives: + raise Exception(f"No archive found in file: {path}") + else: + archive_id = _archives[0].archive_id + archive_alias = _archives[0].archive_name + for archive in _archives: + if archive.archive_id != archive_id: raise Exception( - f"Data and alias archives in file '{path}' have different IDs." + f"Multiple different archive ids found in file: {path}" ) - if data_archive.archive_name != alias_archive.archive_name: + if archive.archive_name != archive_alias: raise Exception( - f"Data and alias archives in file '{path}' have different aliases." + f"Multiple different archive aliases found in file: {path}" ) - archive_id = data_archive.archive_id - archive_alias = data_archive.archive_name - elif alias_archive: - # we can assume data archive is None here - archive_id = alias_archive.archive_id - archive_alias = alias_archive.archive_name - else: - raise Exception( - "This should never happen, but we need to handle it anyway. Bug in code." - ) - kiarchive = KiArchive( archive_id=archive_id, archive_name=archive_alias, metadata_archive_config=metadata_archive_config, data_archive_config=data_archive_config, alias_archive_config=alias_archive_config, + job_archive_config=jobs_archive_config, archive_base_path=archive_path.parent.as_posix(), archive_file_name=archive_path.name, allow_write_access=allow_write_access, @@ -100,6 +105,7 @@ def load_kiarchive( kiarchive._metadata_archive = metadata_archive kiarchive._data_archive = data_archive kiarchive._alias_archive = alias_archive + kiarchive._jobs_archive = jobs_archive kiarchive._kiara = kiara return kiarchive @@ -110,11 +116,14 @@ def create_kiarchive( kiara: "Kiara", kiarchive_uri: Union[str, Path], archive_name: Union[str, None] = None, - compression: Union[CHUNK_COMPRESSION_TYPE, str] = CHUNK_COMPRESSION_TYPE.ZSTD, + compression: Union[None, CHUNK_COMPRESSION_TYPE, str] = None, allow_write_access: bool = True, allow_existing: bool = False, ) -> "KiArchive": + if compression is None: + compression = DEFAULT_CHUNK_COMPRESSION + if isinstance(kiarchive_uri, str): kiarchive_uri = Path(kiarchive_uri) @@ -154,6 +163,7 @@ def create_kiarchive( store_type="sqlite_metadata_store", file_name=archive_file_name, allow_write_access=True, + set_archive_name_metadata=False, ) metadata_store_config = metadata_store.config @@ -163,12 +173,24 @@ def create_kiarchive( store_type="sqlite_alias_store", file_name=archive_file_name, allow_write_access=allow_write_access, + set_archive_name_metadata=False, ) alias_store_config = alias_store.config + job_store: JobStore = create_new_archive( # type: ignore + archive_name=archive_name, + store_base_path=archive_base_path, + store_type="sqlite_job_store", + file_name=archive_file_name, + allow_write_access=allow_write_access, + set_archive_name_metadata=False, + ) + job_store_config = job_store.config + kiarchive_id = data_store.archive_id assert alias_store.archive_id == kiarchive_id assert metadata_store.archive_id == kiarchive_id + assert job_store.archive_id == kiarchive_id kiarchive = KiArchive( archive_id=kiarchive_id, @@ -178,11 +200,13 @@ def create_kiarchive( metadata_archive_config=metadata_store_config.model_dump(), data_archive_config=data_store_config.model_dump(), alias_archive_config=alias_store_config.model_dump(), + job_archive_config=job_store_config.model_dump(), allow_write_access=allow_write_access, ) kiarchive._metadata_archive = metadata_store kiarchive._data_archive = data_store kiarchive._alias_archive = alias_store + kiarchive._jobs_archive = job_store kiarchive._kiara = kiara return kiarchive @@ -205,18 +229,26 @@ def create_kiarchive( alias_archive_config: Union[Mapping[str, Any], None] = Field( description="The archive to store aliases in.", default=None ) + job_archive_config: Union[Mapping[str, Any], None] = Field( + description="The archive to store jobs in.", default=None + ) _metadata_archive: Union["MetadataArchive", None] = PrivateAttr(default=None) _data_archive: Union["DataArchive", None] = PrivateAttr(default=None) _alias_archive: Union["AliasArchive", None] = PrivateAttr(default=None) + _jobs_archive: Union["JobArchive", None] = PrivateAttr(default=None) + _kiara: Union["Kiara", None] = PrivateAttr(default=None) @property - def metadata_archive(self) -> "MetadataArchive": + def metadata_archive(self) -> Union["MetadataArchive", None]: if self._metadata_archive: return self._metadata_archive + if self.metadata_archive_config is None: + return None + from kiara.utils.stores import create_new_archive metadata_archive: MetadataArchive = create_new_archive( # type: ignore @@ -231,11 +263,14 @@ def metadata_archive(self) -> "MetadataArchive": return self._metadata_archive @property - def data_archive(self) -> "DataArchive": + def data_archive(self) -> Union["DataArchive", None]: if self._data_archive: return self._data_archive + if self.data_archive_config is None: + return None + from kiara.utils.stores import create_new_archive data_archive: DataArchive = create_new_archive( # type: ignore @@ -250,15 +285,18 @@ def data_archive(self) -> "DataArchive": return self._data_archive @property - def alias_archive(self) -> "AliasArchive": + def alias_archive(self) -> Union["AliasArchive", None]: if self._alias_archive is not None: return self._alias_archive + if self.alias_archive_config is None: + return None + from kiara.utils.stores import create_new_archive alias_archive: AliasStore = create_new_archive( # type: ignore - archive_alias=self.archive_name, + archive_name=self.archive_name, store_base_path=self.archive_base_path, store_type="sqlite_alias_store", file_name=self.archive_file_name, @@ -266,3 +304,24 @@ def alias_archive(self) -> "AliasArchive": ) self._alias_archive = alias_archive return self._alias_archive + + @property + def job_archive(self) -> Union["JobArchive", None]: + + if self._jobs_archive is not None: + return self._jobs_archive + + if self.job_archive_config is None: + return None + + from kiara.utils.stores import create_new_archive + + jobs_archive: JobStore = create_new_archive( # type: ignore + archive_name=self.archive_name, + store_base_path=self.archive_base_path, + store_type="sqlite_job_store", + file_name=self.archive_file_name, + allow_write_access=True, + ) + self._jobs_archive = jobs_archive + return self._jobs_archive diff --git a/src/kiara/interfaces/python_api/models/doc.py b/src/kiara/interfaces/python_api/models/doc.py index 6bf6fa5bd..41e6697ec 100644 --- a/src/kiara/interfaces/python_api/models/doc.py +++ b/src/kiara/interfaces/python_api/models/doc.py @@ -6,7 +6,7 @@ if TYPE_CHECKING: # we don't want those imports (yet), since they take a while to load - from kiara.interfaces.python_api import Workflow + from kiara.interfaces.python_api.workflow import Workflow from kiara.models.module.operation import Operation from kiara.models.module.pipeline import PipelineStructure diff --git a/src/kiara/interfaces/python_api/models/info.py b/src/kiara/interfaces/python_api/models/info.py index 65a084a58..0a6b5c594 100644 --- a/src/kiara/interfaces/python_api/models/info.py +++ b/src/kiara/interfaces/python_api/models/info.py @@ -73,10 +73,17 @@ from kiara.utils.json import orjson_dumps from kiara.utils.output import extract_renderable +try: + from typing import Self # type: ignore +except ImportError: + from typing_extensions import Self # type: ignore + + if TYPE_CHECKING: from kiara.context import Kiara from kiara.data_types import DataType from kiara.models.runtime_environment.python import PythonRuntimeEnvironment + from kiara.models.values.value_metadata import ValueMetadata from kiara.operations import OperationType from kiara.registries.aliases import AliasRegistry from kiara.registries.data import DataRegistry @@ -326,7 +333,7 @@ def base_info_class(cls) -> Type[TypeInfo]: @classmethod def create_from_type_items( cls, kiara: "Kiara", group_title: Union[str, None] = None, **items: Type - ) -> "TypeInfoItemGroup": + ) -> Self: type_infos: Mapping[str, TypeInfo[Any]] = { k: cls.base_info_class().create_from_type_class(type_cls=v, kiara=kiara) @@ -790,6 +797,90 @@ def _retrieve_data_to_hash(self) -> Any: return self.python_class.full_name +class MetadataTypeInfo(TypeInfo): + + _kiara_model_id: ClassVar = "info.metadata_type" + + @classmethod + def create_from_type_class( + self, type_cls: Type["ValueMetadata"], kiara: "Kiara" + ) -> "MetadataTypeInfo": + + authors_md = AuthorsMetadataModel.from_class(type_cls) + doc = DocumentationMetadataModel.from_class_doc(type_cls) + python_class = PythonClass.from_class(type_cls) + properties_md = ContextMetadataModel.from_class(type_cls) + type_name = type_cls._metadata_key # type: ignore + schema = type_cls.model_json_schema() + + return MetadataTypeInfo( + type_name=type_name, + documentation=doc, + authors=authors_md, + context=properties_md, + python_class=python_class, + metadata_schema=schema, + ) + + @classmethod + def base_class(self) -> Type["ValueMetadata"]: + from kiara.models.values.value_metadata import ValueMetadata + + return ValueMetadata + + @classmethod + def category_name(cls) -> str: + return "value_metadata" + + metadata_schema: Dict[str, Any] = Field( + description="The (json) schema for this metadata value." + ) + + def create_renderable(self, **config: Any) -> RenderableType: + + include_doc = config.get("include_doc", True) + include_schema = config.get("include_schema", True) + + table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) + table.add_column("property", style="i") + table.add_column("value") + + if include_doc: + table.add_row( + "Documentation", + Panel(self.documentation.create_renderable(), box=box.SIMPLE), + ) + table.add_row("Author(s)", self.authors.create_renderable()) + table.add_row("Context", self.context.create_renderable()) + + if hasattr(self, "python_class"): + table.add_row("Python class", self.python_class.create_renderable()) + + if include_schema: + schema = Syntax( + orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2), + "json", + background_color="default", + ) + table.add_row("metadata_schema", schema) + + return table + + +class MetadataTypeClassesInfo(TypeInfoItemGroup): + + _kiara_model_id: ClassVar = "info.metadata_types" + + @classmethod + def base_info_class(cls) -> Type[TypeInfo]: + return MetadataTypeInfo + + type_name: Literal["value_metadata"] = "value_metadata" + item_infos: Mapping[str, MetadataTypeInfo] = Field( # type: ignore + description="The value metadata info instances for each type." + ) + + class DataTypeClassInfo(TypeInfo[Type["DataType"]]): _kiara_model_id: ClassVar = "info.data_type" @@ -799,10 +890,26 @@ def create_from_type_class( self, type_cls: Type["DataType"], kiara: Union["Kiara", None] = None ) -> "DataTypeClassInfo": + from kiara.utils.metadata import get_metadata_model_for_data_type + authors = AuthorsMetadataModel.from_class(type_cls) doc = DocumentationMetadataModel.from_class_doc(type_cls) properties_md = ContextMetadataModel.from_class(type_cls) + if kiara is None: + raise NotImplementedError( + "Kiara instance is required to create DataTypeClassInfo." + ) + else: + data_type_name = getattr(type_cls, "_data_type_name", None) + if not data_type_name: + raise KiaraException( + f"Data type class '{type_cls.__name__}' does not have a '_data_type_name' attribute." + ) + metadata_models = get_metadata_model_for_data_type( + kiara=kiara, data_type=data_type_name + ) + if kiara is not None: qual_profiles = kiara.type_registry.get_associated_profiles(type_cls._data_type_name) # type: ignore lineage = kiara.type_registry.get_type_lineage(type_cls._data_type_name) # type: ignore @@ -823,6 +930,7 @@ def create_from_type_class( documentation=doc, authors=authors, context=properties_md, + supported_properties=metadata_models, ) except Exception as e: if isinstance( @@ -854,6 +962,9 @@ def category_name(cls) -> str: qualifier_profiles: Union[Mapping[str, Mapping[str, Any]], None] = Field( description="A map of qualifier profiles for this data types." ) + supported_properties: MetadataTypeClassesInfo = Field( + description="The supported property types for this data type." + ) _kiara: Union["Kiara", None] = PrivateAttr(default=None) def _retrieve_id(self) -> str: @@ -865,28 +976,32 @@ def _retrieve_data_to_hash(self) -> Any: def create_renderable(self, **config: Any) -> RenderableType: include_doc = config.get("include_doc", True) + include_lineage = config.get("include_lineage", True) + include_qualifer_profiles = config.get("include_qualifier_profiles", True) table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) table.add_column("property", style="i") table.add_column("value") - if self.lineage: - table.add_row("lineage", "\n".join(self.lineage[0:])) - else: - table.add_row("lineage", "-- n/a --") - - if self.qualifier_profiles: - qual_table = Table(show_header=False, box=box.SIMPLE) - qual_table.add_column("name") - qual_table.add_column("config") - for name, details in self.qualifier_profiles.items(): - json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2) - qual_table.add_row( - name, Syntax(json_details, "json", background_color="default") - ) - table.add_row("qualifier profile(s)", qual_table) - else: - table.add_row("qualifier profile(s)", "-- n/a --") + if include_lineage: + if self.lineage: + table.add_row("lineage", "\n".join(self.lineage[0:])) + else: + table.add_row("lineage", "-- n/a --") + + if include_qualifer_profiles: + if self.qualifier_profiles: + qual_table = Table(show_header=False, box=box.SIMPLE) + qual_table.add_column("name") + qual_table.add_column("config") + for name, details in self.qualifier_profiles.items(): + json_details = orjson_dumps(details, option=orjson.OPT_INDENT_2) + qual_table.add_row( + name, Syntax(json_details, "json", background_color="default") + ) + table.add_row("qualifier profile(s)", qual_table) + else: + table.add_row("qualifier profile(s)", "-- n/a --") if include_doc: table.add_row( @@ -949,7 +1064,7 @@ def base_info_class(cls) -> Type[DataTypeClassInfo]: def create_renderable(self, **config: Any) -> RenderableType: full_doc = config.get("full_doc", False) - show_subtypes_inline = config.get("show_qualifier_profiles_inline", True) + show_subtypes_inline = config.get("show_qualifier_profiles_inline", False) show_lineage = config.get("show_type_lineage", True) show_lines = full_doc or show_subtypes_inline or show_lineage @@ -974,28 +1089,20 @@ def create_renderable(self, **config: Any) -> RenderableType: t_md = self.item_infos[type_name] # type: ignore row: List[Any] = [type_name] - if show_lineage: - if self._kiara is None: + lineage = t_md.lineage + if lineage is None: lineage_str = "-- n/a --" else: - lineage = list( - self._kiara.type_registry.get_type_lineage(type_name) - ) lineage_str = ", ".join(reversed(lineage[1:])) row.append(lineage_str) if show_subtypes_inline: - if self._kiara is None: - qual_profiles = "-- n/a --" + qual_profiles = t_md.qualifier_profiles + if not qual_profiles: + qual_profiles_str = "-- n/a --" else: - qual_p = self._kiara.type_registry.get_associated_profiles( - data_type_name=type_name - ).keys() - if qual_p: - qual_profiles = "\n".join(qual_p) - else: - qual_profiles = "-- n/a --" - row.append(qual_profiles) + qual_profiles_str = "\n".join(qual_profiles) + row.append(qual_profiles_str) if full_doc: md = Markdown(t_md.documentation.full_doc) diff --git a/src/kiara/interfaces/python_api/models/job.py b/src/kiara/interfaces/python_api/models/job.py index bb97fac4b..15f571f54 100644 --- a/src/kiara/interfaces/python_api/models/job.py +++ b/src/kiara/interfaces/python_api/models/job.py @@ -15,7 +15,7 @@ from kiara.utils.string_vars import replace_var_names_in_obj if TYPE_CHECKING: - from kiara.interfaces.python_api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI from kiara.models.module.operation import Operation from kiara.models.values.value import ValueMap @@ -149,10 +149,12 @@ def validate_inputs(cls, values): def validate_doc(cls, value): return DocumentationMetadataModel.create(value) - def get_operation(self, kiara_api: "KiaraAPI") -> "Operation": + def get_operation(self, kiara_api: "BaseAPI") -> "Operation": if not self.module_config: - operation = kiara_api.get_operation(self.operation, allow_external=True) + operation: Operation = kiara_api.get_operation( + self.operation, allow_external=True + ) else: data = { "module_type": self.operation, @@ -255,12 +257,12 @@ def validate_doc(cls, value): class JobTest(object): def __init__( self, - kiara_api: "KiaraAPI", + kiara_api: "BaseAPI", job_desc: JobDesc, tests: Union[Mapping[str, Mapping[str, Any]], None] = None, ): - self._kiara_api: KiaraAPI = kiara_api + self._kiara_api: BaseAPI = kiara_api self._job_desc = job_desc if tests is None: tests = {} diff --git a/src/kiara/interfaces/python_api/proxy.py b/src/kiara/interfaces/python_api/proxy.py index 01dde0641..7ba4b2dd8 100644 --- a/src/kiara/interfaces/python_api/proxy.py +++ b/src/kiara/interfaces/python_api/proxy.py @@ -26,6 +26,7 @@ def __init__(self, func: Callable): self._wrapped: Union[None, ValidatedFunction] = None self._arg_names: Union[None, List[str]] = None self._param_details: Union[None, Dict[str, Any]] = None + self._raw_doc: Union[None, str] = None self._doc_string: Union[None, str] = None self._parsed_doc: Union[Docstring, None] = None self._doc: Union[DocumentationMetadataModel, None] = None @@ -38,12 +39,25 @@ def doc_string(self): if self._doc_string is not None: return self._doc_string + _doc_string = self.raw_doc + self._doc_string = inspect.cleandoc(_doc_string) + return self._doc_string + + @property + def func(self) -> Callable: + return self._func + + @property + def raw_doc(self) -> str: + + if self._raw_doc is not None: + return self._raw_doc + _doc_string = self._func.__doc__ if _doc_string is None: _doc_string = "" - - self._doc_string = inspect.cleandoc(_doc_string) - return self._doc_string + self._raw_doc = _doc_string + return self._raw_doc @property def doc(self) -> DocumentationMetadataModel: @@ -224,6 +238,7 @@ def __init__( api_cls: Type, filters: Union[None, Iterable[str], str] = None, exclude: Union[None, Iterable[str], str] = None, + include_tags: Union[None, Iterable[str], str] = None, ): if filters is None: @@ -236,9 +251,16 @@ def __init__( elif isinstance(exclude, str): exclude = [exclude] + if include_tags is None: + include_tags = [] + elif isinstance(include_tags, str): + include_tags = [include_tags] + self._api_cls = api_cls self._filters: Iterable[str] = filters self._exclude: Iterable[str] = exclude + self._include_tags: Iterable[str] = include_tags + self._api_endpoint_names: Union[None, List[str]] = None self._endpoint_details: Dict[str, ApiEndpoint] = {} @@ -249,16 +271,37 @@ def api_endpint_names(self) -> List[str]: return self._api_endpoint_names temp = [] - for func_name in dir(self._api_cls): + + avail_methods = list( + inspect.getmembers(self._api_cls, predicate=inspect.isfunction) + ) + + avail_methods.sort(key=lambda x: inspect.getsourcelines(x[1])[1]) + + method_names = [x[0] for x in avail_methods] + for func_name in method_names: if func_name.startswith("_"): continue if func_name in self._exclude: continue - if not callable(getattr(self._api_cls, func_name)): + func = getattr(self._api_cls, func_name) + if not callable(func): continue + if self._include_tags: + if not hasattr(func, "_tags"): + continue + tags = getattr(func, "_tags") + match = False + for t in tags: + if t in self._include_tags: + match = True + break + if not match: + continue + if self._filters: match = True for f in self._filters: @@ -271,7 +314,7 @@ def api_endpint_names(self) -> List[str]: else: temp.append(func_name) - self._api_endpoint_names = sorted(temp) + self._api_endpoint_names = temp return self._api_endpoint_names def get_api_endpoint(self, endpoint_name: str) -> ApiEndpoint: diff --git a/src/kiara/models/archives.py b/src/kiara/models/archives.py index 51d47ab47..e83ac1402 100644 --- a/src/kiara/models/archives.py +++ b/src/kiara/models/archives.py @@ -36,8 +36,8 @@ from kiara.utils.json import orjson_dumps if TYPE_CHECKING: + from kiara.api import KiArchive from kiara.context import Kiara - from kiara.interfaces.python_api import KiArchive class ArchiveTypeInfo(TypeInfo): @@ -327,7 +327,7 @@ def create_renderable(self, **config: Any) -> RenderableType: class KiArchiveInfo(ItemInfo): @classmethod def base_instance_class(cls) -> Type["KiArchive"]: - from kiara.interfaces.python_api import KiArchive + from kiara.api import KiArchive return KiArchive @@ -343,9 +343,13 @@ def create_from_kiarchive(cls, kiarchive: "KiArchive") -> "KiArchiveInfo": data_archive = kiarchive.data_archive alias_archive = kiarchive.alias_archive + job_archive = kiarchive.job_archive + metadata_archive = kiarchive.metadata_archive data_archive_info = None alias_archive_info = None + job_archive_info = None + metadata_archive_info = None documentation: Union[DocumentationMetadataModel, None] = None authors: Union[AuthorsMetadataModel, None] = None @@ -372,6 +376,22 @@ def create_from_kiarchive(cls, kiarchive: "KiArchive") -> "KiArchiveInfo": authors = alias_archive_info.authors context = alias_archive_info.context + if metadata_archive: + metadata_archive_info = ArchiveInfo.create_from_archive( + kiara=_kiara, archive=metadata_archive + ) + documentation = metadata_archive_info.documentation + authors = metadata_archive_info.authors + context = metadata_archive_info.context + + if job_archive: + job_archive_info = ArchiveInfo.create_from_archive( + kiara=_kiara, archive=job_archive + ) + documentation = job_archive_info.documentation + authors = job_archive_info.authors + context = job_archive_info.context + if documentation is None or authors is None or context is None: raise ValueError("No documentation, authors or context found.") @@ -379,14 +399,24 @@ def create_from_kiarchive(cls, kiarchive: "KiArchive") -> "KiArchiveInfo": type_name=kiarchive.archive_file_name, data_archive_info=data_archive_info, alias_archive_info=alias_archive_info, + metadata_archive_info=metadata_archive_info, + job_archive_info=job_archive_info, documentation=documentation, authors=authors, context=context, ) - data_archive_info: ArchiveInfo = Field(description="The info for the data archive.") - alias_archive_info: ArchiveInfo = Field( - description="The info for the alias archive." + data_archive_info: Union[ArchiveInfo, None] = Field( + description="The info for the included data archive." + ) + alias_archive_info: Union[ArchiveInfo, None] = Field( + description="The info for the included alias archive." + ) + metadata_archive_info: Union[ArchiveInfo, None] = Field( + description="The info for the included metadata archive." + ) + job_archive_info: Union[ArchiveInfo, None] = Field( + description="The info for the included job archive." ) def create_renderable(self, **config: Any) -> RenderableType: @@ -395,11 +425,28 @@ def create_renderable(self, **config: Any) -> RenderableType: table.add_column("property", style="i") table.add_column("value") - table.add_row( - "data archive", self.data_archive_info.create_renderable(**config) - ) - table.add_row( - "alias archive", self.alias_archive_info.create_renderable(**config) - ) + if self.data_archive_info: + content = self.data_archive_info.create_renderable(**config) + else: + content = "-- no data archive --" + table.add_row("data archive", content) + + if self.alias_archive_info: + content = self.alias_archive_info.create_renderable(**config) + else: + content = "-- no alias archive --" + table.add_row("alias archive", content) + + if self.metadata_archive_info: + content = self.metadata_archive_info.create_renderable(**config) + else: + content = "-- no metadata archive --" + table.add_row("metadata archive", content) + + if self.job_archive_info: + content = self.job_archive_info.create_renderable(**config) + else: + content = "-- no job archive --" + table.add_row("job archive", content) return table diff --git a/src/kiara/models/module/jobs.py b/src/kiara/models/module/jobs.py index c5fb82812..909b5d9ad 100644 --- a/src/kiara/models/module/jobs.py +++ b/src/kiara/models/module/jobs.py @@ -254,6 +254,11 @@ def from_active_job(self, kiara: "Kiara", active_job: ActiveJob): module = kiara.module_registry.create_module(active_job.job_config) is_internal = module.characteristics.is_internal + env_hashes = { + env.model_type_id: str(env.instance_cid) + for env in kiara.current_environments.values() + } + job_record = JobRecord( job_id=active_job.job_id, job_submitted=active_job.submitted, @@ -264,7 +269,7 @@ def from_active_job(self, kiara: "Kiara", active_job: ActiveJob): inputs=active_job.job_config.inputs, outputs=active_job.results, runtime_details=job_details, - environment_hashes=kiara.environment_registry.environment_hashes, + environment_hashes=env_hashes, # input_ids_hash=active_job.job_config.input_ids_hash, inputs_data_hash=inputs_data_hash, ) @@ -276,13 +281,13 @@ def from_active_job(self, kiara: "Kiara", active_job: ActiveJob): job_id: uuid.UUID = Field(description="The globally unique id for this job.") job_submitted: datetime = Field(description="When the job was submitted.") - environment_hashes: Mapping[str, Mapping[str, str]] = Field( + environment_hashes: Mapping[str, str] = Field( description="Hashes for the environments this value was created in." ) - enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field( - description="Information about the environments this value was created in.", - default=None, - ) + # enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field( + # description="Information about the environments this value was created in.", + # default=None, + # ) is_internal: bool = Field(description="Whether this job was created by the system.") # job_hash: str = Field(description="The hash of the job. Calculated from manifest & input_ids hashes.") # manifest_hash: str = Field(description="The hash of the manifest.") diff --git a/src/kiara/models/module/operation.py b/src/kiara/models/module/operation.py index aa9af02b2..6be5238de 100644 --- a/src/kiara/models/module/operation.py +++ b/src/kiara/models/module/operation.py @@ -419,7 +419,7 @@ def create_renderable(self, **config: Any) -> RenderableType: ) table.add_row("Outputs", outputs_table) - from kiara.interfaces.python_api import ModuleTypeInfo + from kiara.interfaces.python_api.models.info import ModuleTypeInfo module_type_md: Union[ModuleTypeInfo, None] = None diff --git a/src/kiara/models/module/pipeline/pipeline.py b/src/kiara/models/module/pipeline/pipeline.py index 8e0b1d8d4..6336a5224 100644 --- a/src/kiara/models/module/pipeline/pipeline.py +++ b/src/kiara/models/module/pipeline/pipeline.py @@ -64,7 +64,7 @@ if TYPE_CHECKING: from kiara.context import Kiara - from kiara.interfaces.python_api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI yaml = StringYAML() @@ -81,13 +81,13 @@ class Pipeline(object): @classmethod def create_pipeline( cls, - kiara: Union["Kiara", "KiaraAPI"], + kiara: Union["Kiara", "BaseAPI"], pipeline: Union[PipelineConfig, PipelineStructure, Mapping, str], ) -> "Pipeline": - from kiara.api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI - if isinstance(kiara, KiaraAPI): + if isinstance(kiara, BaseAPI): kiara = kiara.context if isinstance(pipeline, Mapping): diff --git a/src/kiara/models/runtime_environment/__init__.py b/src/kiara/models/runtime_environment/__init__.py index fe1f607bc..6eac5292d 100644 --- a/src/kiara/models/runtime_environment/__init__.py +++ b/src/kiara/models/runtime_environment/__init__.py @@ -15,7 +15,7 @@ from rich.table import Table from kiara.defaults import DEFAULT_ENV_HASH_KEY, ENVIRONMENT_TYPE_CATEGORY_ID -from kiara.models import KiaraModel +from kiara.models.metadata import KiaraMetadata from kiara.utils.hashing import compute_cid from kiara.utils.json import orjson_dumps from kiara.utils.output import extract_renderable @@ -23,7 +23,7 @@ logger = structlog.get_logger() -class RuntimeEnvironment(KiaraModel): +class RuntimeEnvironment(KiaraMetadata): model_config = ConfigDict(frozen=True) @classmethod diff --git a/src/kiara/models/runtime_environment/kiara.py b/src/kiara/models/runtime_environment/kiara.py index 2f66e12ce..293ea3a03 100644 --- a/src/kiara/models/runtime_environment/kiara.py +++ b/src/kiara/models/runtime_environment/kiara.py @@ -4,62 +4,53 @@ # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -from typing import TYPE_CHECKING, Any, ClassVar, Dict, Literal, Union - -from pydantic import Field - -from kiara.interfaces.python_api.models.info import TypeInfo -from kiara.models.archives import ArchiveTypeClassesInfo -from kiara.models.runtime_environment import RuntimeEnvironment -from kiara.models.values.value_metadata import MetadataTypeClassesInfo -from kiara.utils.class_loading import find_all_archive_types -from kiara.utils.metadata import find_metadata_models +from typing import TYPE_CHECKING if TYPE_CHECKING: - from kiara.context import Kiara - - -def find_archive_types( - alias: Union[str, None] = None, only_for_package: Union[str, None] = None -) -> ArchiveTypeClassesInfo: - - archive_types = find_all_archive_types() - - kiara: Kiara = None # type: ignore - group: ArchiveTypeClassesInfo = ArchiveTypeClassesInfo.create_from_type_items( # type: ignore - kiara=kiara, group_title=alias, **archive_types - ) - - if only_for_package: - temp: Dict[str, TypeInfo] = {} - for key, info in group.item_infos.items(): - if info.context.labels.get("package") == only_for_package: - temp[key] = info # type: ignore + pass - group = ArchiveTypeClassesInfo( - group_id=group.group_id, group_title=group.group_alias, item_infos=temp # type: ignore - ) - return group - - -class KiaraTypesRuntimeEnvironment(RuntimeEnvironment): - - _kiara_model_id: ClassVar = "info.runtime.kiara_types" - - environment_type: Literal["kiara_types"] - archive_types: ArchiveTypeClassesInfo = Field( - description="The available implemented store types." - ) - metadata_types: MetadataTypeClassesInfo = Field( - description="The available metadata types." - ) - - @classmethod - def retrieve_environment_data(cls) -> Dict[str, Any]: +# class KiaraDataTypesRuntimeEnvironment(RuntimeEnvironment): +# +# _kiara_model_id: ClassVar = "info.runtime.kiara_data_types" +# +# environment_type: Literal["kiara_data_types"] +# data_types: DataTypeClassesInfo = Field( +# description="The available data types and their metadata." +# ) +# +# @classmethod +# def retrieve_environment_data(cls) -> Dict[str, Any]: +# +# from kiara.api import KiaraAPI +# +# kiara_api = KiaraAPI.instance() +# +# data_types_infos: DataTypeClassesInfo = kiara_api.retrieve_data_types_info() +# data_types = data_types_infos.model_dump() +# +# return {"data_types": data_types} - result: Dict[str, Any] = {} - result["metadata_types"] = find_metadata_models() - result["archive_types"] = find_archive_types() - return result +# class KiaraTypesRuntimeEnvironment(RuntimeEnvironment): +# +# _kiara_model_id: ClassVar = "info.runtime.kiara_types" +# +# environment_type: Literal["kiara_types"] +# archive_types: ArchiveTypeClassesInfo = Field( +# description="The available implemented store types." +# ) +# metadata_types: MetadataTypeClassesInfo = Field( +# description="The available metadata types." +# ) +# +# @classmethod +# def retrieve_environment_data(cls) -> Dict[str, Any]: +# +# from kiara.utils.archives import find_archive_types +# +# result: Dict[str, Any] = {} +# result["metadata_types"] = find_metadata_models() +# result["archive_types"] = find_archive_types() +# +# return result diff --git a/src/kiara/models/values/value.py b/src/kiara/models/values/value.py index c349a501d..01fa5fd5e 100644 --- a/src/kiara/models/values/value.py +++ b/src/kiara/models/values/value.py @@ -780,10 +780,10 @@ class Value(ValueDetails): environment_hashes: Mapping[str, Mapping[str, str]] = Field( description="Hashes for the environments this value was created in." ) - enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field( - description="Information about the environments this value was created in.", - default=None, - ) + # enviroments: Union[Mapping[str, Mapping[str, Any]], None] = Field( + # description="Information about the environments this value was created in.", + # default=None, + # ) property_links: Mapping[str, uuid.UUID] = Field( description="Links to values that are properties of this value.", default_factory=dict, diff --git a/src/kiara/models/values/value_metadata/__init__.py b/src/kiara/models/values/value_metadata/__init__.py index 28af83bdc..493d5319c 100644 --- a/src/kiara/models/values/value_metadata/__init__.py +++ b/src/kiara/models/values/value_metadata/__init__.py @@ -9,37 +9,16 @@ from typing import ( TYPE_CHECKING, Any, - ClassVar, Dict, Iterable, - Literal, - Mapping, - Type, Union, ) -import orjson -from pydantic import Field -from rich import box -from rich.console import RenderableType -from rich.panel import Panel -from rich.syntax import Syntax -from rich.table import Table - -from kiara.interfaces.python_api.models.info import TypeInfo, TypeInfoItemGroup from kiara.models import KiaraModel -from kiara.models.documentation import ( - AuthorsMetadataModel, - ContextMetadataModel, - DocumentationMetadataModel, -) # from kiara.models.info import TypeInfo -from kiara.models.python_class import PythonClass -from kiara.utils.json import orjson_dumps if TYPE_CHECKING: - from kiara.context import Kiara from kiara.models.values.value import Value @@ -65,85 +44,3 @@ def _retrieve_id(self) -> str: def _retrieve_data_to_hash(self) -> Any: return {"metadata": self.model_dump(), "schema": self.schema_json()} - - -class MetadataTypeInfo(TypeInfo): - - _kiara_model_id: ClassVar = "info.metadata_type" - - @classmethod - def create_from_type_class( - self, type_cls: Type[ValueMetadata], kiara: "Kiara" - ) -> "MetadataTypeInfo": - - authors_md = AuthorsMetadataModel.from_class(type_cls) - doc = DocumentationMetadataModel.from_class_doc(type_cls) - python_class = PythonClass.from_class(type_cls) - properties_md = ContextMetadataModel.from_class(type_cls) - type_name = type_cls._metadata_key # type: ignore - schema = type_cls.model_json_schema() - - return MetadataTypeInfo( - type_name=type_name, - documentation=doc, - authors=authors_md, - context=properties_md, - python_class=python_class, - metadata_schema=schema, - ) - - @classmethod - def base_class(self) -> Type[ValueMetadata]: - return ValueMetadata - - @classmethod - def category_name(cls) -> str: - return "value_metadata" - - metadata_schema: Dict[str, Any] = Field( - description="The (json) schema for this metadata value." - ) - - def create_renderable(self, **config: Any) -> RenderableType: - - include_doc = config.get("include_doc", True) - include_schema = config.get("include_schema", True) - - table = Table(box=box.SIMPLE, show_header=False, padding=(0, 0, 0, 0)) - table.add_column("property", style="i") - table.add_column("value") - - if include_doc: - table.add_row( - "Documentation", - Panel(self.documentation.create_renderable(), box=box.SIMPLE), - ) - table.add_row("Author(s)", self.authors.create_renderable()) - table.add_row("Context", self.context.create_renderable()) - - if hasattr(self, "python_class"): - table.add_row("Python class", self.python_class.create_renderable()) - - if include_schema: - schema = Syntax( - orjson_dumps(self.metadata_schema, option=orjson.OPT_INDENT_2), - "json", - background_color="default", - ) - table.add_row("metadata_schema", schema) - - return table - - -class MetadataTypeClassesInfo(TypeInfoItemGroup): - - _kiara_model_id: ClassVar = "info.metadata_types" - - @classmethod - def base_info_class(cls) -> Type[TypeInfo]: - return MetadataTypeInfo - - type_name: Literal["value_metadata"] = "value_metadata" - item_infos: Mapping[str, MetadataTypeInfo] = Field( # type: ignore - description="The value metadata info instances for each type." - ) diff --git a/src/kiara/processing/__init__.py b/src/kiara/processing/__init__.py index 10d6da1ba..a36ef16eb 100644 --- a/src/kiara/processing/__init__.py +++ b/src/kiara/processing/__init__.py @@ -111,8 +111,8 @@ def create_job( ) -> uuid.UUID: environments = { - env_name: env.instance_id - for env_name, env in self._kiara.current_environments.items() + env.model_type_id: str(env.instance_cid) + for env in self._kiara.current_environments.values() } result_pedigree = ValuePedigree( diff --git a/src/kiara/registries/__init__.py b/src/kiara/registries/__init__.py index 94dc7aa7c..4a8b1ad91 100644 --- a/src/kiara/registries/__init__.py +++ b/src/kiara/registries/__init__.py @@ -420,11 +420,18 @@ def create_new_store_config( # Close the connection conn.close() - return SqliteArchiveConfig(sqlite_db_path=archive_path) + use_wal_mode = kwargs.get("wal_mode", False) + + return SqliteArchiveConfig( + sqlite_db_path=archive_path, use_wal_mode=use_wal_mode + ) sqlite_db_path: str = Field( description="The path where the data for this archive is stored." ) + use_wal_mode: bool = Field( + description="Whether to use WAL mode for the SQLite database.", default=False + ) class SqliteDataStoreConfig(SqliteArchiveConfig): @@ -474,9 +481,12 @@ def create_new_store_config( # Close the connection conn.close() + use_wal_mode = kwargs.get("wal_mode", False) + return SqliteDataStoreConfig( sqlite_db_path=archive_path, default_chunk_compression=default_chunk_compression, + use_wal_mode=use_wal_mode, ) default_chunk_compression: Literal["none", "lz4", "zstd", "lzma"] = Field( # type: ignore diff --git a/src/kiara/registries/aliases/__init__.py b/src/kiara/registries/aliases/__init__.py index a238ef6e8..4793ddf83 100644 --- a/src/kiara/registries/aliases/__init__.py +++ b/src/kiara/registries/aliases/__init__.py @@ -356,9 +356,12 @@ def _get_value_id(self, value_id: Union[uuid.UUID, ValueLink, str]) -> uuid.UUID if isinstance(_value_id, str): _value_id = uuid.UUID(_value_id) else: - _value_id = uuid.UUID( - value_id # type: ignore - ) # this should fail if not string or wrong string format + try: + _value_id = uuid.UUID( + value_id # type: ignore + ) # this should fail if not string or wrong string format + except ValueError: + raise KiaraException(f"Could not resolve value id for: {value_id}") else: _value_id = value_id @@ -401,14 +404,34 @@ def find_aliases_for_value_id( def register_aliases( self, value_id: Union[uuid.UUID, ValueLink, str], - *aliases: str, + aliases: Union[str, Iterable[str]], allow_overwrite: bool = False, alias_store: Union[str, None] = None, ): + value = self._kiara.data_registry.get_value(value=value_id) + if alias_store in [DEFAULT_STORE_MARKER, DEFAULT_ALIAS_STORE_MARKER, None]: alias_store = self.default_alias_store + if isinstance(aliases, str): + aliases = [aliases] + else: + for alias in aliases: + if not isinstance(alias, str): + raise KiaraException( + msg=f"Invalid alias: {alias}.", + details="Alias must be a string.", + ) + try: + uuid.UUID(alias) + raise KiaraException( + msg=f"Invalid alias name: {alias}.", + details="Alias can't be a UUID.", + ) + except Exception: + pass + aliases_to_store: Dict[str, List[str]] = {} for alias in aliases: if "#" in alias: @@ -464,8 +487,6 @@ def register_aliases( if duplicates: raise Exception(f"Aliases already registered: {duplicates}") - value_id = self._get_value_id(value_id=value_id) - for store_alias, aliases_for_store in aliases_to_store.items(): store: AliasStore = self.get_archive(archive_alias=store_alias) # type: ignore @@ -479,13 +500,13 @@ def register_aliases( for store_alias, aliases_for_store in aliases_to_store.items(): store = self.get_archive(archive_alias=store_alias) # type: ignore - store.register_aliases(value_id, *aliases_for_store) + store.register_aliases(value.value_id, *aliases_for_store) for alias in aliases: alias_item = AliasItem( full_alias=alias, rel_alias=alias, - value_id=value_id, + value_id=value.value_id, alias_archive=store_alias, alias_archive_id=store.archive_id, ) diff --git a/src/kiara/registries/aliases/sqlite_store.py b/src/kiara/registries/aliases/sqlite_store.py index a193a28bc..b938f3bdd 100644 --- a/src/kiara/registries/aliases/sqlite_store.py +++ b/src/kiara/registries/aliases/sqlite_store.py @@ -3,11 +3,13 @@ from pathlib import Path from typing import Any, Dict, Mapping, Set, Union -from sqlalchemy import create_engine, text +from sqlalchemy import text from sqlalchemy.engine import Engine from kiara.registries import SqliteArchiveConfig from kiara.registries.aliases import AliasArchive, AliasStore +from kiara.utils.dates import get_current_time_incl_timezone +from kiara.utils.db import create_archive_engine, delete_archive_db class SqliteAliasArchive(AliasArchive): @@ -59,6 +61,7 @@ def __init__( ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None + self._use_wal_mode: bool = archive_config.use_wal_mode # self._lock: bool = True def _retrieve_archive_metadata(self) -> Mapping[str, Any]: @@ -85,9 +88,9 @@ def sqlite_path(self): self._db_path.parent.mkdir(parents=True, exist_ok=True) return self._db_path - @property - def db_url(self) -> str: - return f"sqlite:///{self.sqlite_path}" + # @property + # def db_url(self) -> str: + # return f"sqlite:///{self.sqlite_path}" @property def sqlite_engine(self) -> "Engine": @@ -95,14 +98,17 @@ def sqlite_engine(self) -> "Engine": if self._cached_engine is not None: return self._cached_engine - # def _pragma_on_connect(dbapi_con, con_record): - # dbapi_con.execute("PRAGMA query_only = ON") + self._cached_engine = create_archive_engine( + db_path=self.sqlite_path, + force_read_only=self.is_force_read_only(), + use_wal_mode=self._use_wal_mode, + ) - self._cached_engine = create_engine(self.db_url, future=True) create_table_sql = """ CREATE TABLE IF NOT EXISTS aliases ( alias TEXT PRIMARY KEY, - value_id TEXT NOT NULL + value_id TEXT NOT NULL, + alias_created TEXT NOT NULL ); """ with self._cached_engine.begin() as connection: @@ -138,6 +144,10 @@ def retrieve_all_aliases(self) -> Union[Mapping[str, uuid.UUID], None]: result = connection.execute(sql) return {row[0]: uuid.UUID(row[1]) for row in result} + def _delete_archive(self): + + delete_archive_db(db_path=self.sqlite_path) + class SqliteAliasStore(SqliteAliasArchive, AliasStore): @@ -186,12 +196,21 @@ def _set_archive_metadata_value(self, key: str, value: Any): def register_aliases(self, value_id: uuid.UUID, *aliases: str): + alias_created = get_current_time_incl_timezone().isoformat() + sql = text( - "INSERT OR REPLACE INTO aliases (alias, value_id) VALUES (:alias, :value_id)" + "INSERT OR REPLACE INTO aliases (alias, value_id, alias_created) VALUES (:alias, :value_id, :alias_created)" ) with self.sqlite_engine.connect() as connection: - params = [{"alias": alias, "value_id": str(value_id)} for alias in aliases] + params = [ + { + "alias": alias, + "value_id": str(value_id), + "alias_created": alias_created, + } + for alias in aliases + ] for param in params: connection.execute(sql, param) diff --git a/src/kiara/registries/data/__init__.py b/src/kiara/registries/data/__init__.py index fff8cd988..1318a97cd 100644 --- a/src/kiara/registries/data/__init__.py +++ b/src/kiara/registries/data/__init__.py @@ -33,6 +33,7 @@ DATA_ARCHIVE_DEFAULT_VALUE_MARKER, DEFAULT_DATA_STORE_MARKER, DEFAULT_STORE_MARKER, + ENVIRONMENT_MARKER_KEY, INVALID_HASH_MARKER, NO_SERIALIZATION_MARKER, NONE_STORE_ID, @@ -44,6 +45,7 @@ ) from kiara.exceptions import ( InvalidValuesException, + KiaraException, NoSuchValueAliasException, NoSuchValueException, NoSuchValueIdException, @@ -145,6 +147,7 @@ def resolve_alias(self, alias: str) -> uuid.UUID: msg=f"Can't retrive value for alias '{rest}': no such alias registered.", ) elif ref_type == ARCHIVE_REF_TYPE_NAME: + if "#" in rest: archive_ref, path_in_archive = rest.split("#", maxsplit=1) else: @@ -164,6 +167,10 @@ def resolve_alias(self, alias: str) -> uuid.UUID: default_value = data_archive.get_archive_metadata( DATA_ARCHIVE_DEFAULT_VALUE_MARKER ) + if default_value is None: + raise NoSuchValueException( + f"No default value found for uri: {alias}" + ) _value_id = uuid.UUID(default_value) else: from kiara.registries.aliases import AliasArchive @@ -254,6 +261,7 @@ def __init__(self, kiara: "Kiara"): self._cached_data[NOT_SET_VALUE_ID] = SpecialValue.NOT_SET self._registered_values[NOT_SET_VALUE_ID] = self._not_set_value self._persisted_value_descs[NOT_SET_VALUE_ID] = NONE_PERSISTED_DATA + # self._env_cache: Dict[str, Dict[str, RuntimeEnvironment]] = {} self._none_value: Value = Value( value_id=NONE_VALUE_ID, @@ -446,7 +454,11 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str, Path]) -> Value: raise Exception( f"Can't retrieve value for '{value}': invalid type '{type(value)}'." ) - _value_id = self._alias_resolver.resolve_alias(value) + try: + _value_id = self._alias_resolver.resolve_alias(value) + except Exception as e: + log_exception(e) + raise e else: _value_id = value @@ -491,20 +503,44 @@ def get_value(self, value: Union[uuid.UUID, ValueLink, str, Path]) -> Value: self._registered_values[_value_id] = stored_value return self._registered_values[_value_id] + def _persist_environment(self, env_hash: str, store: Union[str, None]): + + # cached = self._env_cache.get(env_type, {}).get(env_hash, None) + # if cached is not None: + # return + + environment = self._kiara.metadata_registry.retrieve_environment_item(env_hash) + + if not environment: + raise KiaraException( + f"Can't persist data environment with hash '{env_hash}': no such environment registered." + ) + + self._kiara.metadata_registry.register_metadata_item( + key=ENVIRONMENT_MARKER_KEY, item=environment, store=store + ) + # self._env_cache.setdefault(env_type, {})[env_hash] = environment + def store_value( self, value: Union[ValueLink, uuid.UUID, str], - data_store: Union[str, uuid.UUID, None] = None, + data_store: Union[str, None] = None, ) -> Union[PersistedData, None]: """Store a value into a data store. If 'data_store' is not provided, the default data store is used. If the 'data_store' argument is of type uuid, the archive_id is used, if string, first it will be converted to an uuid, if that works, again, the archive_id is used, if not, the string is used as the archive alias. + """ _value = self.get_value(value) + # first, persist environment information + for env_hash in _value.pedigree.environments.values(): + + self._persist_environment(env_hash, store=data_store) + store: DataStore = self.get_archive(archive_id_or_alias=data_store) # type: ignore if not store.is_writeable(): if data_store: @@ -544,7 +580,9 @@ def store_value( self._event_callback(store_event) if _value.job_id: - self._kiara.job_registry.store_job_record(job_id=_value.job_id) + self._kiara.job_registry.store_job_record( + job_id=_value.job_id, store=data_store + ) return persisted_value @@ -1136,9 +1174,11 @@ def retrieve_chunks( archive = self.get_archive(archive_id) - return archive.retrieve_chunks( + chunks = archive.retrieve_chunks( chunk_ids, as_files=as_files, symlink_ok=symlink_ok ) + + return chunks # for chunk_id in chunk_ids: # yield archive.retrieve_chunk(chunk_id) @@ -1301,8 +1341,8 @@ def create_valuemap( else: try: _d = self._alias_resolver.resolve_alias(_d) - except Exception: - pass + except Exception as e: + log_exception(e) if isinstance(_d, Value): _resolved[input_name] = _d diff --git a/src/kiara/registries/data/data_store/__init__.py b/src/kiara/registries/data/data_store/__init__.py index 52fe8280c..172be3975 100644 --- a/src/kiara/registries/data/data_store/__init__.py +++ b/src/kiara/registries/data/data_store/__init__.py @@ -24,7 +24,6 @@ import structlog from rich.console import RenderableType -from kiara.models.runtime_environment import RuntimeEnvironment from kiara.models.values.matchers import ValueMatcher from kiara.models.values.value import ( SERIALIZE_TYPES, @@ -373,14 +372,14 @@ def _persist_value_pedigree(self, value: Value): to the job that produced it is preserved. """ - @abc.abstractmethod - def _persist_environment_details( - self, env_type: str, env_hash: str, env_data: Mapping[str, Any] - ): - """Persist the environment details. - - Each store type needs to store this for lookup purposes. - """ + # @abc.abstractmethod + # def _persist_environment_details( + # self, env_type: str, env_hash: str, env_data: Mapping[str, Any] + # ): + # """Persist the environment details. + # + # Each store type needs to store this for lookup purposes. + # """ @abc.abstractmethod def _persist_destiny_backlinks(self, value: Value): @@ -395,16 +394,16 @@ def store_value(self, value: Value) -> PersistedData: value_hash=value.value_hash, ) - # first, persist environment information - for env_type, env_hash in value.pedigree.environments.items(): - cached = self._env_cache.get(env_type, {}).get(env_hash, None) - if cached is not None: - continue - - env = self.kiara_context.environment_registry.get_environment_for_cid( - env_hash - ) - self.persist_environment(env) + # # first, persist environment information + # for env_type, env_hash in value.pedigree.environments.items(): + # cached = self._env_cache.get(env_type, {}).get(env_hash, None) + # if cached is not None: + # continue + # + # env = self.kiara_context.environment_registry.get_environment_for_cid( + # env_hash + # ) + # self.persist_environment(env) # save the value data and metadata persisted_value = self._persist_value(value) @@ -544,25 +543,25 @@ def _persist_value(self, value: Value) -> PersistedData: return persisted_value_info - def persist_environment(self, environment: RuntimeEnvironment): - """ - Persist the specified environment. - - The environment is stored as a dictionary, including it's schema, not as the actual Python model. - This is to make sure it can still be loaded later on, in case the Python model has changed in later versions. - """ - env_type = environment.get_environment_type_name() - env_hash = str(environment.instance_cid) - - env = self._env_cache.get(env_type, {}).get(env_hash, None) - if env is not None: - return - - env_data = environment.as_dict_with_schema() - self._persist_environment_details( - env_type=env_type, env_hash=env_hash, env_data=env_data - ) - self._env_cache.setdefault(env_type, {})[env_hash] = env_data + # def persist_environment(self, environment: RuntimeEnvironment): + # """ + # Persist the specified environment. + # + # The environment is stored as a dictionary, including it's schema, not as the actual Python model. + # This is to make sure it can still be loaded later on, in case the Python model has changed in later versions. + # """ + # env_type = environment.get_environment_type_name() + # env_hash = str(environment.instance_cid) + # + # env = self._env_cache.get(env_type, {}).get(env_hash, None) + # if env is not None: + # return + # + # env_data = environment.as_dict_with_schema() + # self._persist_environment_details( + # env_type=env_type, env_hash=env_hash, env_data=env_data + # ) + # self._env_cache.setdefault(env_type, {})[env_hash] = env_data def create_renderable(self, **config: Any) -> RenderableType: """Create a renderable for this module configuration.""" diff --git a/src/kiara/registries/data/data_store/filesystem_store.py b/src/kiara/registries/data/data_store/filesystem_store.py index f7c0fd28a..9acba8a6a 100644 --- a/src/kiara/registries/data/data_store/filesystem_store.py +++ b/src/kiara/registries/data/data_store/filesystem_store.py @@ -411,15 +411,15 @@ class FilesystemDataStore(FileSystemDataArchive, BaseDataStore): _archive_type_name = "filesystem_data_store" - def _persist_environment_details( - self, env_type: str, env_hash: str, env_data: Mapping[str, Any] - ): - - base_path = self.get_path(entity_type=EntityType.ENVIRONMENT) - env_details_file = base_path / f"{env_type}_{env_hash}.json" - - if not env_details_file.exists(): - env_details_file.write_text(orjson_dumps(env_data)) + # def _persist_environment_details( + # self, env_type: str, env_hash: str, env_data: Mapping[str, Any] + # ): + # + # base_path = self.get_path(entity_type=EntityType.ENVIRONMENT) + # env_details_file = base_path / f"{env_type}_{env_hash}.json" + # + # if not env_details_file.exists(): + # env_details_file.write_text(orjson_dumps(env_data)) def _persist_stored_value_info(self, value: Value, persisted_value: PersistedData): diff --git a/src/kiara/registries/data/data_store/sqlite_store.py b/src/kiara/registries/data/data_store/sqlite_store.py index edc4c37bd..279bc199a 100644 --- a/src/kiara/registries/data/data_store/sqlite_store.py +++ b/src/kiara/registries/data/data_store/sqlite_store.py @@ -18,10 +18,15 @@ ) import orjson -from sqlalchemy import create_engine, text +from sqlalchemy import text from sqlalchemy.engine import Connection, Engine -from kiara.defaults import CHUNK_COMPRESSION_TYPE, kiara_app_dirs +from kiara.defaults import ( + CHUNK_CACHE_BASE_DIR, + CHUNK_CACHE_DIR_DEPTH, + CHUNK_CACHE_DIR_WIDTH, + CHUNK_COMPRESSION_TYPE, +) from kiara.models.values.value import PersistedData, Value from kiara.registries import ( ARCHIVE_CONFIG_CLS, @@ -31,8 +36,8 @@ ) from kiara.registries.data import DataArchive from kiara.registries.data.data_store import BaseDataStore +from kiara.utils.db import create_archive_engine, delete_archive_db from kiara.utils.hashfs import shard -from kiara.utils.json import orjson_dumps if TYPE_CHECKING: from multiformats import CID @@ -94,11 +99,13 @@ def __init__( ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None - self._data_cache_dir = Path(kiara_app_dirs.user_cache_dir) / "data" / "chunks" + self._data_cache_dir = CHUNK_CACHE_BASE_DIR self._data_cache_dir.mkdir(parents=True, exist_ok=True, mode=0o700) - self._cache_dir_depth = 2 - self._cache_dir_width = 1 + + self._cache_dir_depth = CHUNK_CACHE_DIR_DEPTH + self._cache_dir_width = CHUNK_CACHE_DIR_WIDTH self._value_id_cache: Union[Iterable[uuid.UUID], None] = None + self._use_wal_mode: bool = archive_config.use_wal_mode # self._lock: bool = True def _retrieve_archive_metadata(self) -> Mapping[str, Any]: @@ -135,9 +142,9 @@ def sqlite_path(self): self._db_path.parent.mkdir(parents=True, exist_ok=True) return self._db_path - @property - def db_url(self) -> str: - return f"sqlite:///{self.sqlite_path}" + # @property + # def db_url(self) -> str: + # return f"sqlite:///{self.sqlite_path}" def get_chunk_path(self, chunk_id: str) -> Path: @@ -158,14 +165,18 @@ def sqlite_engine(self) -> "Engine": if self._cached_engine is not None: return self._cached_engine - # def _pragma_on_connect(dbapi_con, con_record): - # dbapi_con.execute("PRAGMA query_only = ON") - self._cached_engine = create_engine(self.db_url, future=True) + self._cached_engine = create_archive_engine( + db_path=self.sqlite_path, + force_read_only=self.is_force_read_only(), + use_wal_mode=self._use_wal_mode, + ) + create_table_sql = """ CREATE TABLE IF NOT EXISTS values_metadata ( value_id TEXT PRIMARY KEY, value_hash TEXT NOT NULL, value_size INTEGER NOT NULL, + value_created TEXT NOT NULL, data_type_name TEXT NOT NULL, value_metadata TEXT NOT NULL ); @@ -477,7 +488,8 @@ def retrieve_missing_chunks( assert not missing_chunk_ids def _delete_archive(self): - os.unlink(self.sqlite_path) + + delete_archive_db(db_path=self.sqlite_path) def get_archive_details(self) -> ArchiveDetails: @@ -552,26 +564,26 @@ def _set_archive_metadata_value(self, key: str, value: Any): conn.execute(sql, params) conn.commit() - def _persist_environment_details( - self, env_type: str, env_hash: str, env_data: Mapping[str, Any] - ): - - sql = text( - "INSERT OR IGNORE INTO environments (environment_type, environment_hash, environment_data) VALUES (:environment_type, :environment_hash, :environment_data)" - ) - env_data_json = orjson_dumps(env_data) - with self.sqlite_engine.connect() as conn: - params = { - "environment_type": env_type, - "environment_hash": env_hash, - "environment_data": env_data_json, - } - conn.execute(sql, params) - conn.commit() - # print(env_type) - # print(env_hash) - # print(env_data_json) - # raise NotImplementedError() + # def _persist_environment_details( + # self, env_type: str, env_hash: str, env_data: Mapping[str, Any] + # ): + # + # sql = text( + # "INSERT OR IGNORE INTO environments (environment_type, environment_hash, environment_data) VALUES (:environment_type, :environment_hash, :environment_data)" + # ) + # env_data_json = orjson_dumps(env_data) + # with self.sqlite_engine.connect() as conn: + # params = { + # "environment_type": env_type, + # "environment_hash": env_hash, + # "environment_data": env_data_json, + # } + # conn.execute(sql, params) + # conn.commit() + # # print(env_type) + # # print(env_hash) + # # print(env_data_json) + # # raise NotImplementedError() # def _persist_value_data(self, value: Value) -> PersistedData: # @@ -692,16 +704,19 @@ def _persist_value_details(self, value: Value): value_size = value.value_size data_type_name = value.data_type_name + value_created = value.value_created.isoformat() + metadata = value.model_dump_json() sql = text( - "INSERT INTO values_metadata (value_id, value_hash, value_size, data_type_name, value_metadata) VALUES (:value_id, :value_hash, :value_size, :data_type_name, :metadata)" + "INSERT INTO values_metadata (value_id, value_hash, value_size, value_created, data_type_name, value_metadata) VALUES (:value_id, :value_hash, :value_size, :value_created, :data_type_name, :metadata)" ) with self.sqlite_engine.connect() as conn: params = { "value_id": value_id, "value_hash": value_hash, "value_size": value_size, + "value_created": value_created, "data_type_name": data_type_name, "metadata": metadata, } diff --git a/src/kiara/registries/environment/__init__.py b/src/kiara/registries/environment/__init__.py index b28c018b3..bd6899509 100644 --- a/src/kiara/registries/environment/__init__.py +++ b/src/kiara/registries/environment/__init__.py @@ -7,7 +7,7 @@ import inspect -from typing import Any, Dict, Iterable, Mapping, Type, Union +from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Type, Union from pydantic import BaseModel, Field, create_model from rich import box @@ -16,6 +16,9 @@ from kiara.models.runtime_environment import RuntimeEnvironment, logger from kiara.utils import _get_all_subclasses, is_debug, to_camel_case +if TYPE_CHECKING: + pass + class EnvironmentRegistry(object): @@ -28,17 +31,22 @@ def instance(cls) -> "EnvironmentRegistry": cls._instance = EnvironmentRegistry() return cls._instance - def __init__( - self, - ) -> None: + def __init__(self) -> None: + self._environments: Union[Dict[str, RuntimeEnvironment], None] = None self._environment_hashes: Union[Dict[str, Mapping[str, str]], None] = None self._full_env_model: Union[BaseModel, None] = None + # self._kiara: Kiara = kiara + def get_environment_for_cid(self, env_cid: str) -> RuntimeEnvironment: - envs = [env for env in self.environments.values() if env.instance_id == env_cid] + envs = [ + env + for env in self.environments.values() + if str(env.instance_cid) == env_cid + ] if len(envs) == 0: raise Exception(f"No environment with id '{env_cid}' available.") elif len(envs) > 1: @@ -47,6 +55,13 @@ def get_environment_for_cid(self, env_cid: str) -> RuntimeEnvironment: ) return envs[0] + def has_environment(self, env_cid: str) -> bool: + + for env in self.environments.values(): + if str(env.instance_cid) == env_cid: + return True + return False + @property def environment_hashes(self) -> Mapping[str, Mapping[str, str]]: @@ -67,7 +82,7 @@ def environments(self) -> Mapping[str, RuntimeEnvironment]: return self._environments import kiara.models.runtime_environment.kiara - import kiara.models.runtime_environment.operating_system + import kiara.models.runtime_environment.operating_system # nowa import kiara.models.runtime_environment.python # noqa subclasses: Iterable[Type[RuntimeEnvironment]] = _get_all_subclasses( diff --git a/src/kiara/registries/jobs/__init__.py b/src/kiara/registries/jobs/__init__.py index be7942715..b36b3b83d 100644 --- a/src/kiara/registries/jobs/__init__.py +++ b/src/kiara/registries/jobs/__init__.py @@ -14,7 +14,13 @@ from bidict import bidict from rich.console import Group -from kiara.exceptions import FailedJobException +from kiara.defaults import ( + DEFAULT_DATA_STORE_MARKER, + DEFAULT_JOB_STORE_MARKER, + DEFAULT_STORE_MARKER, + ENVIRONMENT_MARKER_KEY, +) +from kiara.exceptions import FailedJobException, KiaraException from kiara.models.events import KiaraEvent from kiara.models.events.job_registry import ( JobArchiveAddedEvent, @@ -38,6 +44,7 @@ if TYPE_CHECKING: from kiara.context import Kiara from kiara.context.runtime_config import JobCacheStrategy + from kiara.models.runtime_environment import RuntimeEnvironment logger = structlog.getLogger() @@ -171,6 +178,8 @@ def __init__(self, kiara: "Kiara"): self._event_callback = self._kiara.event_registry.add_producer(self) + self._env_cache: Dict[str, Dict[str, RuntimeEnvironment]] = {} + # default_archive = FileSystemJobStore.create_from_kiara_context(self._kiara) # self.register_job_archive(default_archive, store_alias=DEFAULT_STORE_MARKER) @@ -250,14 +259,27 @@ def default_job_store(self) -> str: raise Exception("No default job store set (yet).") return self._default_job_store # type: ignore - def get_archive(self, store_id: Union[str, None] = None) -> JobArchive: + def get_archive(self, store_id: Union[str, None, uuid.UUID] = None) -> JobArchive: - if store_id is None: - store_id = self.default_job_store - if store_id is None: + if store_id in [ + None, + "", + DEFAULT_DATA_STORE_MARKER, + DEFAULT_JOB_STORE_MARKER, + DEFAULT_STORE_MARKER, + ]: + if self.default_job_store is None: raise Exception("Can't retrieve deafult job archive, none set (yet).") + _store_id: str = self.default_job_store + + elif not isinstance(store_id, str): + raise NotImplementedError( + "Can't retrieve job archive by (uu)id or other type (yet)." + ) + else: + _store_id = store_id - return self._job_archives[store_id] + return self._job_archives[_store_id] @property def job_archives(self) -> Mapping[str, JobArchive]: @@ -282,14 +304,32 @@ def job_status_changed( self._finished_jobs[job_hash] = job_id self._archived_records[job_id] = job_record - def store_job_record(self, job_id: uuid.UUID): + def _persist_environment(self, env_type: str, env_hash: str): - if job_id not in self._archived_records.keys(): - raise Exception( - f"Can't store job with id '{job_id}': no job record with that id exists." + cached = self._env_cache.get(env_type, {}).get(env_hash, None) + if cached is not None: + return + + environment = self._kiara.metadata_registry.retrieve_environment_item(env_hash) + + if not environment: + raise KiaraException( + f"Can't persist job environment for with hash '{env_hash}': no such environment registered." ) - job_record = self._archived_records[job_id] + self._kiara.metadata_registry.register_metadata_item( + key=ENVIRONMENT_MARKER_KEY, item=environment + ) + self._env_cache.setdefault(env_type, {})[env_hash] = environment + + def store_job_record(self, job_id: uuid.UUID, store: Union[str, None] = None): + + # TODO: allow to store job record to external store + + job_record = self.get_job_record(job_id=job_id) + + for env_type, env_hash in job_record.environment_hashes.items(): + self._persist_environment(env_type, env_hash) if job_record._is_stored: logger.debug( @@ -297,18 +337,10 @@ def store_job_record(self, job_id: uuid.UUID): ) return - store: JobStore = self.get_archive() # type: ignore + store: JobStore = self.get_archive(store) # type: ignore if not isinstance(store, JobStore): raise Exception("Can't store job record to archive: not writable.") - # if job_record.job_id in self._finished_jobs.values(): - # logger.debug( - # "ignore.store.job_record", - # reason="already stored in store", - # job_id=str(job_id), - # ) - # return - logger.debug( "store.job_record", job_hash=job_record.job_hash, @@ -331,7 +363,7 @@ def get_job_record_in_session(self, job_id: uuid.UUID) -> JobRecord: return self._processor.get_job_record(job_id) - def get_job_record(self, job_id: uuid.UUID) -> Union[JobRecord, None]: + def get_job_record(self, job_id: uuid.UUID) -> JobRecord: if job_id in self._archived_records.keys(): return self._archived_records[job_id] @@ -342,13 +374,13 @@ def get_job_record(self, job_id: uuid.UUID) -> Union[JobRecord, None]: except Exception: pass - try: - job = self._processor.get_job(job_id=job_id) - if job is not None: - if job.status == JobStatus.FAILED: - return None - except Exception: - pass + # try: + # job = self._processor.get_job(job_id=job_id) + # if job is not None: + # if job.status == JobStatus.FAILED: + # return None + # except Exception: + # pass all_job_records = self.retrieve_all_job_records() for r in all_job_records.values(): @@ -406,7 +438,9 @@ def retrieve_all_job_records(self) -> Mapping[uuid.UUID, JobRecord]: for archive in self.job_archives.values(): all_record_ids = archive.retrieve_all_job_ids().keys() for r in all_record_ids: - assert r not in all_records.keys() + if r in all_records.keys(): + continue + job_record = archive.retrieve_record_for_job_id(r) assert job_record is not None all_records[r] = job_record diff --git a/src/kiara/registries/jobs/job_store/sqlite_store.py b/src/kiara/registries/jobs/job_store/sqlite_store.py index b6bff4e3f..e8aac33b8 100644 --- a/src/kiara/registries/jobs/job_store/sqlite_store.py +++ b/src/kiara/registries/jobs/job_store/sqlite_store.py @@ -5,12 +5,13 @@ from typing import Any, Dict, Generator, Iterable, Mapping, Union import orjson -from sqlalchemy import create_engine, text +from sqlalchemy import text from sqlalchemy.engine import Engine from kiara.models.module.jobs import JobMatcher, JobRecord -from kiara.registries import SqliteArchiveConfig +from kiara.registries import ArchiveDetails, SqliteArchiveConfig from kiara.registries.jobs import JobArchive, JobStore +from kiara.utils.db import create_archive_engine, delete_archive_db class SqliteJobArchive(JobArchive): @@ -62,6 +63,7 @@ def __init__( ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None + self._use_wal_mode: bool = archive_config.use_wal_mode # self._lock: bool = True # def _retrieve_archive_id(self) -> uuid.UUID: @@ -98,9 +100,9 @@ def sqlite_path(self): self._db_path.parent.mkdir(parents=True, exist_ok=True) return self._db_path - @property - def db_url(self) -> str: - return f"sqlite:///{self.sqlite_path}" + # @property + # def db_url(self) -> str: + # return f"sqlite:///{self.sqlite_path}" @property def sqlite_engine(self) -> "Engine": @@ -108,10 +110,12 @@ def sqlite_engine(self) -> "Engine": if self._cached_engine is not None: return self._cached_engine - # def _pragma_on_connect(dbapi_con, con_record): - # dbapi_con.execute("PRAGMA query_only = ON") + self._cached_engine = create_archive_engine( + db_path=self.sqlite_path, + force_read_only=self.is_force_read_only(), + use_wal_mode=self._use_wal_mode, + ) - self._cached_engine = create_engine(self.db_url, future=True) create_table_sql = """ CREATE TABLE IF NOT EXISTS job_records ( job_id TEXT PRIMARY KEY, @@ -266,6 +270,21 @@ def retrieve_all_job_hashes( result = connection.execute(sql, params) return {row[0] for row in result} + def _delete_archive(self): + + delete_archive_db(db_path=self.sqlite_path) + + def get_archive_details(self) -> ArchiveDetails: + + all_job_records_sql = text("SELECT COUNT(*) FROM job_records") + + with self.sqlite_engine.connect() as connection: + result = connection.execute(all_job_records_sql) + job_count = result.fetchone()[0] + + details = {"no_job_records": job_count, "dynamic_archive": False} + return ArchiveDetails(**details) + class SqliteJobStore(SqliteJobArchive, JobStore): @@ -329,3 +348,14 @@ def store_job_record(self, job_record: JobRecord): connection.execute(sql, params) connection.commit() + + def _set_archive_metadata_value(self, key: str, value: Any): + """Set custom metadata for the archive.""" + + sql = text( + "INSERT OR REPLACE INTO archive_metadata (key, value) VALUES (:key, :value)" + ) + with self.sqlite_engine.connect() as conn: + params = {"key": key, "value": value} + conn.execute(sql, params) + conn.commit() diff --git a/src/kiara/registries/metadata/__init__.py b/src/kiara/registries/metadata/__init__.py index dc802f1aa..4959abf49 100644 --- a/src/kiara/registries/metadata/__init__.py +++ b/src/kiara/registries/metadata/__init__.py @@ -1,17 +1,79 @@ # -*- coding: utf-8 -*- import uuid -from typing import TYPE_CHECKING, Any, Callable, Dict, Literal, Mapping, Union - -from pydantic import Field - -from kiara.defaults import DEFAULT_METADATA_STORE_MARKER, DEFAULT_STORE_MARKER +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + Generator, + List, + Literal, + Mapping, + Tuple, + Union, +) + +from pydantic import Field, field_validator + +from kiara.defaults import ( + DEFAULT_DATA_STORE_MARKER, + DEFAULT_METADATA_STORE_MARKER, + DEFAULT_STORE_MARKER, +) +from kiara.exceptions import KiaraException +from kiara.models import KiaraModel from kiara.models.events import RegistryEvent from kiara.models.metadata import CommentMetadata, KiaraMetadata from kiara.registries.metadata.metadata_store import MetadataArchive, MetadataStore if TYPE_CHECKING: from kiara.context import Kiara - from kiara.registries.environment import EnvironmentRegistry + from kiara.models.runtime_environment import RuntimeEnvironment + + +class MetadataMatcher(KiaraModel): + """An object describing requirements metadata items should satisfy in order to be included in a query result.""" + + @classmethod + def create_matcher(cls, **match_options: Any): + m = MetadataMatcher(**match_options) + return m + + # metadata_item_keys: Union[None, List[str]] = Field( + # description="The metadata item key to match (if provided).", default=None + # ) + reference_item_types: Union[None, List[str]] = Field( + description="A 'reference_item_type' a metadata item is referenced from.", + default=None, + ) + reference_item_keys: Union[None, List[str]] = Field( + description="A 'reference_item_key' a metadata item is referenced from.", + default=None, + ) + reference_item_ids: Union[None, List[str]] = Field( + description="An list of ids that a metadata item is referenced from.", + default=None, + ) + + @field_validator( + "reference_item_types", + "reference_item_keys", + "reference_item_ids", + mode="before", + ) + @classmethod + def validate_reference_item_ids(cls, v): + + if v is None: + return None + elif isinstance(v, str): + return [v] + elif isinstance(v, uuid.UUID): + return [str(v)] + else: + v = set(v) + result = [str(x) for x in v] + return result class MetadataArchiveAddedEvent(RegistryEvent): @@ -38,9 +100,9 @@ def __init__(self, kiara: "Kiara"): self._event_callback: Callable = self._kiara.event_registry.add_producer(self) self._metadata_archives: Dict[str, MetadataArchive] = {} - self._default_data_store: Union[str, None] = None + self._default_metadata_store: Union[str, None] = None - self._env_registry: EnvironmentRegistry = self._kiara.environment_registry + # self._env_registry: EnvironmentRegistry = self._kiara.environment_registry @property def kiara_id(self) -> uuid.UUID: @@ -70,14 +132,14 @@ def register_metadata_archive( if isinstance(archive, MetadataStore): is_store = True - if set_as_default_store and self._default_data_store is not None: + if set_as_default_store and self._default_metadata_store is not None: raise Exception( f"Can't set data store '{alias}' as default store: default store already set." ) - if self._default_data_store is None or set_as_default_store: + if self._default_metadata_store is None or set_as_default_store: is_default_store = True - self._default_data_store = alias + self._default_metadata_store = alias event = MetadataArchiveAddedEvent( kiara_id=self._kiara.id, @@ -91,10 +153,10 @@ def register_metadata_archive( return alias @property - def default_data_store(self) -> str: - if self._default_data_store is None: + def default_metadata_store(self) -> str: + if self._default_metadata_store is None: raise Exception("No default metadata store set.") - return self._default_data_store + return self._default_metadata_store @property def metadata_archives(self) -> Mapping[str, MetadataArchive]: @@ -108,8 +170,9 @@ def get_archive( None, DEFAULT_STORE_MARKER, DEFAULT_METADATA_STORE_MARKER, + DEFAULT_DATA_STORE_MARKER, ): - archive_id_or_alias = self.default_data_store + archive_id_or_alias = self.default_metadata_store if archive_id_or_alias is None: raise Exception( "Can't retrieve default metadata archive, none set (yet)." @@ -142,20 +205,104 @@ def get_archive( f"Can't retrieve archive with id '{archive_id_or_alias}': no archive with that id registered." ) + def find_metadata_items( + self, matcher: MetadataMatcher + ) -> Generator[Tuple[Any, ...], None, None]: + + mounted_store: MetadataArchive = self.get_archive() + + return mounted_store.find_matching_metadata_items(matcher=matcher) + + def retrieve_environment_item(self, env_cid: str) -> "RuntimeEnvironment": + + if self._kiara.environment_registry.has_environment(env_cid): + environment = self._kiara.environment_registry.get_environment_for_cid( + env_cid + ) + else: + _environment = self.retrieve_metadata_item_with_hash(item_hash=env_cid) + if _environment is None: + raise KiaraException( + f"No environment with id '{env_cid}' available in metadata store." + ) + + from kiara.models.runtime_environment import RuntimeEnvironment + + if isinstance(_environment, RuntimeEnvironment): + environment = _environment + else: + raise KiaraException( + f"Invalid environment item with id '{env_cid}' available in metadata store." + ) + + return environment + + def retrieve_metadata_item_with_hash( + self, item_hash: str, store: Union[str, uuid.UUID, None] = None + ) -> Union[KiaraMetadata, None]: + """Retrieves a metadata item by its hash.""" + + if store: + mounted_archive: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore + result = mounted_archive.find_metadata_item_with_hash(item_hash=item_hash) + else: + mounted_archive: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore + result = mounted_archive.find_metadata_item_with_hash(item_hash=item_hash) + if not result: + for archive in self.metadata_archives.values(): + + result = archive.find_metadata_item_with_hash(item_hash=item_hash) + if result: + break + + if result is None: + return None + + model_type_id, data = result + model_cls = self._kiara.kiara_model_registry.get_model_cls( + kiara_model_id=model_type_id, required_subclass=KiaraMetadata + ) + + model_instance = model_cls(**data) + return model_instance # type: ignore + def retrieve_metadata_item( self, key: str, reference_item_type: Union[str, None] = None, + reference_item_key: Union[str, None] = None, reference_item_id: Union[str, None] = None, store: Union[str, uuid.UUID, None] = None, ) -> Union[KiaraMetadata, None]: """Retrieves a metadata item.""" - mounted_store: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore - - result = mounted_store.retrieve_metadata_item( - key=key, reference_type=reference_item_type, reference_id=reference_item_id - ) + if store: + mounted_store: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore + result = mounted_store.retrieve_metadata_item( + metadata_item_key=key, + reference_type=reference_item_type, + reference_key=reference_item_key, + reference_id=reference_item_id, + ) + else: + mounted_store: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore + result = mounted_store.retrieve_metadata_item( + metadata_item_key=key, + reference_type=reference_item_type, + reference_key=reference_item_key, + reference_id=reference_item_id, + ) + if not result: + + for archive in self.metadata_archives.values(): + result = archive.retrieve_metadata_item( + metadata_item_key=key, + reference_type=reference_item_type, + reference_key=reference_item_key, + reference_id=reference_item_id, + ) + if result: + break if result is None: return None @@ -173,39 +320,62 @@ def register_metadata_item( key: str, item: KiaraMetadata, reference_item_type: Union[str, None] = None, + reference_item_key: Union[str, None] = None, reference_item_id: Union[str, None] = None, - force: bool = False, + replace_existing_references: bool = False, + allow_multiple_references: bool = False, store: Union[str, uuid.UUID, None] = None, ) -> uuid.UUID: mounted_store: MetadataStore = self.get_archive(archive_id_or_alias=store) # type: ignore - return mounted_store.store_metadata_item( + result = mounted_store.store_metadata_item( key=key, item=item, reference_item_type=reference_item_type, + reference_item_key=reference_item_key, reference_item_id=reference_item_id, - force=force, + replace_existing_references=replace_existing_references, + allow_multiple_references=allow_multiple_references, ) + return result def register_job_metadata_items( self, job_id: uuid.UUID, items: Mapping[str, Any], store: Union[str, uuid.UUID, None] = None, + reference_item_key: Union[str, None] = None, + replace_existing_references: bool = True, + allow_multiple_references: bool = False, ) -> None: for key, value in items.items(): + + _reference_item_key = None if isinstance(value, str): value = CommentMetadata(comment=value) + if not reference_item_key: + _reference_item_key = "comment" + else: + _reference_item_key = reference_item_key + elif isinstance(value, CommentMetadata): + _reference_item_key = "comment" elif not isinstance(value, KiaraMetadata): raise Exception(f"Invalid metadata value for key '{key}': {value}") + + if not _reference_item_key: + _reference_item_key = value._kiara_model_id + self.register_metadata_item( key=key, item=value, reference_item_type="job", + reference_item_key=_reference_item_key, reference_item_id=str(job_id), store=store, + replace_existing_references=replace_existing_references, + allow_multiple_references=allow_multiple_references, ) def retrieve_job_metadata_items(self, job_id: uuid.UUID): @@ -219,6 +389,7 @@ def retrieve_job_metadata_item( return self.retrieve_metadata_item( key=key, reference_item_type="job", + reference_item_key="comment", reference_item_id=str(job_id), store=store, ) diff --git a/src/kiara/registries/metadata/metadata_store/__init__.py b/src/kiara/registries/metadata/metadata_store/__init__.py index 6f32bf233..1d23587c0 100644 --- a/src/kiara/registries/metadata/metadata_store/__init__.py +++ b/src/kiara/registries/metadata/metadata_store/__init__.py @@ -2,12 +2,25 @@ import abc import json import uuid -from typing import Any, Dict, Generic, Iterable, Mapping, Tuple, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Generic, + Iterable, + Mapping, + Tuple, + Union, +) from kiara.exceptions import KiaraException from kiara.models.metadata import KiaraMetadata from kiara.registries import ARCHIVE_CONFIG_CLS, BaseArchive +if TYPE_CHECKING: + from kiara.registries.metadata import MetadataMatcher + class MetadataArchive(BaseArchive[ARCHIVE_CONFIG_CLS], Generic[ARCHIVE_CONFIG_CLS]): """Base class for data archiv implementationss.""" @@ -30,25 +43,76 @@ def __init__( archive_config=archive_config, force_read_only=force_read_only, ) + self._schema_stored_cache: Dict[str, Any] = {} + self._schema_stored_item: Dict[str, uuid.UUID] = {} + + def find_metadata_item_with_hash( + self, item_hash: str, key: Union[str, None] = None + ) -> Union[Tuple[str, Mapping[str, Any]], None]: + """Return the key of the metadata item with the specified hash.""" + + return self._retrieve_metadata_item_with_hash(item_hash=item_hash, key=key) + + @abc.abstractmethod + def _retrieve_metadata_item_with_hash( + self, item_hash: str, key: Union[str, None] = None + ) -> Union[Tuple[str, Mapping[str, Any]], None]: + pass + + def find_matching_metadata_items( + self, + matcher: "MetadataMatcher", + metadata_item_result_fields: Union[Iterable[str], None] = None, + reference_item_result_fields: Union[Iterable[str], None] = None, + ) -> Generator[Tuple[Any, ...], None, None]: + + return self._find_matching_metadata_and_ref_items( + matcher=matcher, + metadata_item_result_fields=metadata_item_result_fields, + reference_item_result_fields=reference_item_result_fields, + ) + + @abc.abstractmethod + def _find_matching_metadata_and_ref_items( + self, + matcher: "MetadataMatcher", + metadata_item_result_fields: Union[Iterable[str], None] = None, + reference_item_result_fields: Union[Iterable[str], None] = None, + ) -> Generator[Tuple[Any, ...], None, None]: + pass def retrieve_metadata_item( self, - key: str, + metadata_item_key: str, reference_type: Union[str, None] = None, + reference_key: Union[str, None] = None, reference_id: Union[str, None] = None, ) -> Union[Tuple[str, Mapping[str, Any]], None]: + """Return the model type and model data for the specified metadata item. + + If more than one item matches, an exception is raised. - if reference_id and not reference_type: + Arguments: + metadata_item_key: The key of the metadata item to retrieve. + reference_type: The type of the referenced item. + reference_key: The key of the referenced item. + reference_id: The id of the referenced item. + """ + + if reference_id and (not reference_type or not reference_key): raise ValueError( - "If reference_id is set, reference_type must be set as well." + "If reference_id is set, reference_key & reference_type must be set as well." ) - if reference_type: + if reference_type and reference_key: if reference_id is None: raise KiaraException( msg="reference_id must set also if reference_type is set." ) result = self._retrieve_referenced_metadata_item_data( - key=key, reference_type=reference_type, reference_id=reference_id + key=metadata_item_key, + reference_type=reference_type, + reference_key=reference_key, + reference_id=reference_id, ) if result is None: return None @@ -61,7 +125,7 @@ def retrieve_metadata_item( @abc.abstractmethod def _retrieve_referenced_metadata_item_data( - self, key: str, reference_type: str, reference_id: str + self, key: str, reference_type: str, reference_key: str, reference_id: str ) -> Union[Tuple[str, Mapping[str, Any]], None]: """Return the model type id and model data for the specified referenced metadata item.""" @@ -79,7 +143,6 @@ def __init__( archive_config=archive_config, force_read_only=force_read_only, ) - self._schema_stored_cache: Dict[str, Any] = {} @classmethod def _is_writeable(cls) -> bool: @@ -96,10 +159,17 @@ def store_metadata_item( key: str, item: KiaraMetadata, reference_item_type: Union[str, None] = None, + reference_item_key: Union[str, None] = None, reference_item_id: Union[str, None] = None, - force: bool = False, + replace_existing_references: bool = False, + allow_multiple_references: bool = False, store: Union[str, uuid.UUID, None] = None, ) -> uuid.UUID: + """Store a metadata item into the store. + + If `reference_item_type` and `reference_item_id` are set, the stored metadata item will + be linked to the stored metadata item, to enable lokoups later on. + """ if store: raise NotImplementedError( @@ -109,27 +179,34 @@ def store_metadata_item( # TODO: check if already stored model_type = item.model_type_id model_schema_hash = str(item.get_schema_cid()) - model_item_schema = item.model_json_schema() - model_item_schema_str = json.dumps(model_item_schema) - self._store_metadata_schema( - model_schema_hash=model_schema_hash, - model_type_id=model_type, - model_schema=model_item_schema_str, - ) + if model_schema_hash not in self._schema_stored_cache.keys(): + + model_item_schema = item.model_json_schema() + model_item_schema_str = json.dumps(model_item_schema) + + self._store_metadata_schema( + model_schema_hash=model_schema_hash, + model_type_id=model_type, + model_schema=model_item_schema_str, + ) + self._schema_stored_cache[model_schema_hash] = model_item_schema # data = item.model_dump() data_json = item.model_dump_json() data_hash = str(item.instance_cid) - metadata_item_id = self._store_metadata_item( - key=key, - value_json=data_json, - value_hash=data_hash, - model_type_id=model_type, - model_schema_hash=model_schema_hash, - force=force, - ) + metadata_item_id = self._schema_stored_item.get(data_hash, None) + if not metadata_item_id: + + metadata_item_id = self._store_metadata_item( + key=key, + value_json=data_json, + value_hash=data_hash, + model_type_id=model_type, + model_schema_hash=model_schema_hash, + ) + self._schema_stored_item[data_hash] = metadata_item_id if (reference_item_id and not reference_item_type) or ( reference_item_type and not reference_item_id @@ -140,15 +217,27 @@ def store_metadata_item( if reference_item_type: assert reference_item_id is not None + assert reference_item_key is not None self._store_metadata_reference( - reference_item_type, reference_item_id, str(metadata_item_id) + reference_item_type=reference_item_type, + reference_item_key=reference_item_key, + reference_item_id=reference_item_id, + metadata_item_id=str(metadata_item_id), + replace_existing_references=replace_existing_references, + allow_multiple_references=allow_multiple_references, ) return metadata_item_id @abc.abstractmethod def _store_metadata_reference( - self, reference_item_type: str, reference_item_id: str, metadata_item_id: str + self, + reference_item_type: str, + reference_item_key: str, + reference_item_id: str, + metadata_item_id: str, + replace_existing_references: bool = False, + allow_multiple_references: bool = False, ) -> None: pass @@ -160,6 +249,18 @@ def _store_metadata_item( value_hash: str, model_type_id: str, model_schema_hash: str, - force: bool = False, ) -> uuid.UUID: pass + + def store_metadata_and_ref_items( + self, items: Generator[Tuple[Any, ...], None, None] + ): + + return self._store_metadata_and_ref_items(items) + + @abc.abstractmethod + def _store_metadata_and_ref_items( + self, items: Generator[Tuple[Any, ...], None, None] + ): + + pass diff --git a/src/kiara/registries/metadata/metadata_store/sqlite_store.py b/src/kiara/registries/metadata/metadata_store/sqlite_store.py index 18ffa389d..39358c1dc 100644 --- a/src/kiara/registries/metadata/metadata_store/sqlite_store.py +++ b/src/kiara/registries/metadata/metadata_store/sqlite_store.py @@ -1,14 +1,25 @@ # -*- coding: utf-8 -*- import uuid from pathlib import Path -from typing import Any, Dict, Mapping, Tuple, Union +from typing import ( + Any, + Dict, + Generator, + Iterable, + Mapping, + Tuple, + Union, +) import orjson from sqlalchemy import text -from sqlalchemy.engine import Engine, create_engine +from sqlalchemy.engine import Engine -from kiara.registries import SqliteArchiveConfig -from kiara.registries.metadata import MetadataArchive, MetadataStore +from kiara.exceptions import KiaraException +from kiara.registries import ArchiveDetails, SqliteArchiveConfig +from kiara.registries.metadata import MetadataArchive, MetadataMatcher, MetadataStore +from kiara.utils.dates import get_current_time_incl_timezone +from kiara.utils.db import create_archive_engine, delete_archive_db REQUIRED_METADATA_TABLES = { "metadata", @@ -60,6 +71,8 @@ def __init__( ) self._db_path: Union[Path, None] = None self._cached_engine: Union[Engine, None] = None + self._use_wal_mode: bool = archive_config.use_wal_mode + # self._lock: bool = True # def _retrieve_archive_id(self) -> uuid.UUID: @@ -96,9 +109,9 @@ def sqlite_path(self): self._db_path.parent.mkdir(parents=True, exist_ok=True) return self._db_path - @property - def db_url(self) -> str: - return f"sqlite:///{self.sqlite_path}" + # @property + # def db_url(self) -> str: + # return f"sqlite:///{self.sqlite_path}" @property def sqlite_engine(self) -> "Engine": @@ -106,10 +119,12 @@ def sqlite_engine(self) -> "Engine": if self._cached_engine is not None: return self._cached_engine - # def _pragma_on_connect(dbapi_con, con_record): - # dbapi_con.execute("PRAGMA query_only = ON") + self._cached_engine = create_archive_engine( + db_path=self.sqlite_path, + force_read_only=self.is_force_read_only(), + use_wal_mode=self._use_wal_mode, + ) - self._cached_engine = create_engine(self.db_url, future=True) create_table_sql = """ CREATE TABLE IF NOT EXISTS metadata_schemas ( model_schema_hash TEXT PRIMARY KEY, @@ -118,18 +133,23 @@ def sqlite_engine(self) -> "Engine": ); CREATE TABLE IF NOT EXISTS metadata ( metadata_item_id TEXT PRIMARY KEY, + metadata_item_created TEXT NOT NULL, metadata_item_key TEXT NOT NULL, metadata_item_hash TEXT NOT NULL, model_type_id TEXT NOT NULL, model_schema_hash TEXT NOT NULL, metadata_value TEXT NOT NULL, - FOREIGN KEY (model_schema_hash) REFERENCES metadata_schemas (model_schema_hash) + FOREIGN KEY (model_schema_hash) REFERENCES metadata_schemas (model_schema_hash), + UNIQUE (metadata_item_key, metadata_item_hash) ); CREATE TABLE IF NOT EXISTS metadata_references ( reference_item_type TEXT NOT NULL, + reference_item_key TEXT NOT NULL, reference_item_id TEXT NOT NULL, + reference_created TEXT NOT NULL, metadata_item_id TEXT NOT NULL, - FOREIGN KEY (metadata_item_id) REFERENCES metadata (metadata_item_id) + FOREIGN KEY (metadata_item_id) REFERENCES metadata (metadata_item_id), + UNIQUE (reference_item_type, reference_item_key, reference_item_id, metadata_item_id, reference_created) ); """ @@ -142,8 +162,186 @@ def sqlite_engine(self) -> "Engine": # event.listen(self._cached_engine, "connect", _pragma_on_connect) return self._cached_engine + def _retrieve_metadata_item_with_hash( + self, item_hash: str, key: Union[str, None] = None + ) -> Union[Tuple[str, Mapping[str, Any]], None]: + + if not key: + sql = text( + """ + SELECT m.model_type_id, m.metadata_value + FROM metadata m + WHERE m.metadata_item_hash = :item_hash + """ + ) + else: + sql = text( + """ + SELECT m.model_type_id, m.metadata_value + FROM metadata m + WHERE m.metadata_item_hash = :item_hash AND m.metadata_item_key = :key + """ + ) + + with self.sqlite_engine.connect() as connection: + params = {"item_hash": item_hash} + if key: + params["key"] = key + result = connection.execute(sql, params) + row = result.fetchall() + if not row: + return None + + if len(row) > 1: + msg = ( + f"Multiple ({len(row)}) metadata items found for hash '{item_hash}'" + ) + if key: + msg += f" and key '{key}'" + msg += "." + raise KiaraException(msg) + + data_str = row[0][1] + data = orjson.loads(data_str) + + return (row[0][0], data) + + def _find_matching_metadata_and_ref_items( + self, + matcher: "MetadataMatcher", + metadata_item_result_fields: Union[Iterable[str], None] = None, + reference_item_result_fields: Union[Iterable[str], None] = None, + ) -> Generator[Tuple[Any, ...], None, None]: + + # find all metadata items first + + if not metadata_item_result_fields: + metadata_fields_str = "m.*" + else: + metadata_fields_str = ", ".join( + (f"m.{x}" for x in metadata_item_result_fields) + ) + + metadata_fields_str += ", :result_type as result_type" + + sql_string = f"SELECT {metadata_fields_str} FROM metadata m " # noqa + conditions = [] + params = {"result_type": "metadata_item"} + + ref_query = False + if ( + matcher.reference_item_types + or matcher.reference_item_keys + or matcher.reference_item_ids + ): + ref_query = True + sql_string += ( + "JOIN metadata_references r ON m.metadata_item_id = r.metadata_item_id" + ) + + # if matcher.metadata_item_keys: + # conditions.append("m.metadata_item_key in :metadata_item_keys") + # params["metadata_item_key"] = matcher.metadata_item_keys + + if matcher.reference_item_ids: + assert ref_query + in_clause = [] + for idx, item_id in enumerate(matcher.reference_item_ids): + params[f"ri_id_{idx}"] = item_id + in_clause.append(f":ri_id_{idx}") + in_clause_str = ", ".join(in_clause) + conditions.append(f"r.reference_item_id IN ({in_clause_str})") + # params["reference_item_ids"] = tuple(matcher.reference_item_ids) + + if matcher.reference_item_types: + assert ref_query + in_clause = [] + for idx, item_type in enumerate(matcher.reference_item_types): + params[f"ri_type_{idx}"] = item_type + in_clause.append(f":ri_type_{idx}") + in_clause_str = ", ".join(in_clause) + conditions.append(f"r.reference_item_type IN ({in_clause_str})") + # params["reference_item_types"] = tuple(matcher.reference_item_types) + + if matcher.reference_item_keys: + assert ref_query + in_clause = [] + for idx, item_key in enumerate(matcher.reference_item_keys): + params[f"ri_key_{idx}"] = item_key + in_clause.append(f":ri_key_{idx}") + in_clause_str = ", ".join(in_clause) + conditions.append(f"r.reference_item_key IN ({in_clause_str})") + # params["reference_item_keys"] = tuple(matcher.reference_item_keys) + + if conditions: + sql_string += " WHERE" + for cond in conditions: + sql_string += f" {cond} AND" + + sql_string = sql_string[:-4] + sql = text(sql_string) + + # ... now construct the query to find the reference items (if applicable) + if not reference_item_result_fields: + reference_fields_str = "r.*" + else: + reference_fields_str = ", ".join( + (f"r.{x}" for x in reference_item_result_fields) + ) + + ref_sql_string = f"SELECT {reference_fields_str}, :result_type as result_type FROM metadata_references r" # noqa + ref_params = {"result_type": "metadata_ref_item"} + ref_conditions = [] + + if matcher.reference_item_ids: + assert ref_query + in_clause = [] + for idx, item_id in enumerate(matcher.reference_item_ids): + ref_params[f"ri_id_{idx}"] = item_id + in_clause.append(f":ri_id_{idx}") + ref_conditions.append(f"r.reference_item_id IN ({in_clause_str})") + # ref_params["reference_item_ids"] = tuple(matcher.reference_item_ids) + + if matcher.reference_item_types: + assert ref_query + in_clause = [] + for idx, item_type in enumerate(matcher.reference_item_types): + ref_params[f"ri_type_{idx}"] = item_type + in_clause.append(f":ri_type_{idx}") + in_clause_str = ", ".join(in_clause) + ref_conditions.append(f"r.reference_item_type IN ({in_clause_str})") + # ref_params["reference_item_types"] = tuple(matcher.reference_item_types) + + if matcher.reference_item_keys: + assert ref_query + in_clause = [] + for idx, item_key in enumerate(matcher.reference_item_keys): + ref_params[f"ri_key_{idx}"] = item_key + in_clause.append(f":ri_key_{idx}") + in_clause_str = ", ".join(in_clause) + ref_conditions.append(f"r.reference_item_key IN ({in_clause_str})") + # ref_params["reference_item_keys"] = tuple(matcher.reference_item_keys) + + if ref_conditions: + ref_sql_string += " WHERE" + for cond in ref_conditions: + ref_sql_string += f" {cond} AND" + + ref_sql_string = ref_sql_string[:-4] + + ref_sql = text(ref_sql_string) + + with self.sqlite_engine.connect() as connection: + result = connection.execute(sql, params) + for row in result: + yield row + + result = connection.execute(ref_sql, ref_params) + for row in result: + yield row + def _retrieve_referenced_metadata_item_data( - self, key: str, reference_type: str, reference_id: str + self, key: str, reference_type: str, reference_key: str, reference_id: str ) -> Union[Tuple[str, Mapping[str, Any]], None]: sql = text( @@ -151,25 +349,58 @@ def _retrieve_referenced_metadata_item_data( SELECT m.model_type_id, m.metadata_value FROM metadata m JOIN metadata_references r ON m.metadata_item_id = r.metadata_item_id - WHERE r.reference_item_type = :reference_type AND r.reference_item_id = :reference_id and m.metadata_item_key = :key + WHERE r.reference_item_type = :reference_type AND r.reference_item_key = :reference_key AND r.reference_item_id = :reference_id and m.metadata_item_key = :key """ ) with self.sqlite_engine.connect() as connection: parmas = { "reference_type": reference_type, + "reference_key": reference_key, "reference_id": reference_id, "key": key, } result = connection.execute(sql, parmas) - row = result.fetchone() - if row is None: + row = result.fetchall() + if not row: return None - data_str = row[1] + if len(row) > 1: + msg = f"Multiple ({len(row)}) metadata items found for key '{key}'" + if reference_type: + msg += f" and reference type '{reference_type}'" + if reference_id: + msg += f" and reference id '{reference_id}'" + msg += "." + raise KiaraException(msg) + + data_str = row[0][1] data = orjson.loads(data_str) - return (row[0], data) + return (row[0][0], data) + + def _delete_archive(self): + + delete_archive_db(db_path=self.sqlite_path) + + def get_archive_details(self) -> ArchiveDetails: + + all_metadata_items_sql = text("SELECT COUNT(*) FROM metadata") + all_references_sql = text("SELECT COUNT(*) FROM metadata_references") + + with self.sqlite_engine.connect() as connection: + result = connection.execute(all_metadata_items_sql) + metadata_count = result.fetchone()[0] + + result = connection.execute(all_references_sql) + reference_count = result.fetchone()[0] + + details = { + "no_metadata_items": metadata_count, + "no_references": reference_count, + "dynamic_archive": False, + } + return ArchiveDetails(**details) class SqliteMetadataStore(SqliteMetadataArchive, MetadataStore): @@ -236,24 +467,23 @@ def _store_metadata_item( value_hash: str, model_type_id: str, model_schema_hash: str, - force: bool = False, ) -> uuid.UUID: from kiara.registries.ids import ID_REGISTRY - if force: - sql = text( - "INSERT OR REPLACE INTO metadata (metadata_item_id, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" - ) - else: - sql = text( - "INSERT INTO metadata (metadata_item_id, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" - ) + metadata_item_created = get_current_time_incl_timezone().isoformat() - metadata_item_id = ID_REGISTRY.generate(comment="new metadata item id") + sql = text( + "INSERT OR IGNORE INTO metadata (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" + ) + + metadata_item_id = ID_REGISTRY.generate( + comment="new provisional metadata item id" + ) params = { "metadata_item_id": str(metadata_item_id), + "metadata_item_created": metadata_item_created, "metadata_item_key": key, "metadata_item_hash": value_hash, "model_type_id": model_type_id, @@ -261,24 +491,101 @@ def _store_metadata_item( "metadata_value": value_json, } + query_metadata_id = text( + "SELECT metadata_item_id FROM metadata WHERE metadata_item_key = :metadata_item_key AND metadata_item_hash = :metadata_item_hash" + ) + query_metadata_params = { + "metadata_item_key": key, + "metadata_item_hash": value_hash, + } + with self.sqlite_engine.connect() as conn: conn.execute(sql, params) + result = conn.execute(query_metadata_id, query_metadata_params) + metadata_item_id = uuid.UUID(result.fetchone()[0]) conn.commit() return metadata_item_id def _store_metadata_reference( - self, reference_item_type: str, reference_item_id: str, metadata_item_id: str + self, + reference_item_type: str, + reference_item_key: str, + reference_item_id: str, + metadata_item_id: str, + replace_existing_references: bool = False, + allow_multiple_references: bool = False, ) -> None: - sql = text( - "INSERT INTO metadata_references (reference_item_type, reference_item_id, metadata_item_id) VALUES (:reference_item_type, :reference_item_id, :metadata_item_id)" + if not replace_existing_references: + raise NotImplementedError( + "not replacing existing metadata references is not yet supported" + ) + + else: + + sql_replace = text( + "DELETE FROM metadata_references WHERE reference_item_type = :reference_item_type AND reference_item_key = :reference_item_key AND reference_item_id = :reference_item_id" + ) + sql_replace_params = { + "reference_item_type": reference_item_type, + "reference_item_key": reference_item_key, + "reference_item_id": reference_item_id, + } + + metadata_reference_created = get_current_time_incl_timezone().isoformat() + sql_insert = text( + "INSERT INTO metadata_references (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" + ) + sql_insert_params = { + "reference_item_type": reference_item_type, + "reference_item_key": reference_item_key, + "reference_item_id": reference_item_id, + "reference_created": metadata_reference_created, + "metadata_item_id": metadata_item_id, + } + with self.sqlite_engine.connect() as conn: + conn.execute(sql_replace, sql_replace_params) + conn.execute(sql_insert, sql_insert_params) + conn.commit() + + def _store_metadata_and_ref_items( + self, items: Generator[Tuple[Any, ...], None, None] + ): + + insert_metadata_sql = text( + "INSERT OR IGNORE INTO metadata (metadata_item_id, metadata_item_created, metadata_item_key, metadata_item_hash, model_type_id, model_schema_hash, metadata_value) VALUES (:metadata_item_id, :metadata_item_created, :metadata_item_key, :metadata_item_hash, :model_type_id, :model_schema_hash, :metadata_value)" ) - params = { - "reference_item_type": reference_item_type, - "reference_item_id": reference_item_id, - "metadata_item_id": metadata_item_id, - } + + insert_ref_sql = text( + "INSERT OR IGNORE INTO metadata_references (reference_item_type, reference_item_key, reference_item_id, reference_created, metadata_item_id) VALUES (:reference_item_type, :reference_item_key, :reference_item_id, :reference_created, :metadata_item_id)" + ) + + batch_size = 100 + with self.sqlite_engine.connect() as conn: - conn.execute(sql, params) + + metadata_items = [] + ref_items = [] + + for item in items: + if item.result_type == "metadata_item": # type: ignore + metadata_items.append(item._asdict()) # type: ignore + elif item.result_type == "metadata_ref_item": # type: ignore + ref_items.append(item._asdict()) # type: ignore + else: + raise KiaraException(f"Unknown result type '{item.result_type}'") # type: ignore + + if len(metadata_items) >= batch_size: + conn.execute(insert_metadata_sql, metadata_items) + metadata_items.clear() + if len(ref_items) >= batch_size: + conn.execute(insert_ref_sql, ref_items) + ref_items.clear() + + if metadata_items: + conn.execute(insert_metadata_sql, metadata_items) + if ref_items: + conn.execute(insert_ref_sql, ref_items) + conn.commit() diff --git a/src/kiara/renderers/included_renderers/api/__init__.py b/src/kiara/renderers/included_renderers/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/kiara/renderers/included_renderers/api/base_api.py b/src/kiara/renderers/included_renderers/api/base_api.py new file mode 100644 index 000000000..a910d4417 --- /dev/null +++ b/src/kiara/renderers/included_renderers/api/base_api.py @@ -0,0 +1,335 @@ +# -*- coding: utf-8 -*- +import abc +import re +from pathlib import Path +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + Mapping, + Set, + Union, +) + +from pydantic.fields import Field + +from kiara.interfaces.python_api.base_api import BaseAPI +from kiara.interfaces.python_api.proxy import ApiEndpoints +from kiara.models.rendering import RenderValueResult +from kiara.renderers import ( + KiaraRenderer, + KiaraRendererConfig, + RenderInputsSchema, +) +from kiara.utils.cli import terminal_print +from kiara.utils.introspection import ( + create_signature_string, + extract_arg_names, + extract_proxy_arg_str, +) +from kiara.utils.output import ( + create_table_from_base_model_v1_cls, +) + +if TYPE_CHECKING: + from kiara.context import Kiara + + +class BaseApiRenderInputsSchema(RenderInputsSchema): + + pass + + +class BaseApiRendererConfig(KiaraRendererConfig): + + tags: Union[None, str, Iterable[str]] = Field( + description="The tag to filter the api endpoints by (if any tag matches, the endpoint will be included.", + default="kiara_api", + ) + filter: Union[str, Iterable[str], None] = Field( + description="One or a list of filter tokens -- if provided -- all of which must match for the api endpoing to be in the render result.", + default=None, + ) + + +class BaseApiRenderer( + KiaraRenderer[ + BaseAPI, BaseApiRenderInputsSchema, RenderValueResult, BaseApiRendererConfig + ] +): + _inputs_schema = BaseApiRenderInputsSchema + _renderer_config_cls = BaseApiRendererConfig + + def __init__( + self, + kiara: "Kiara", + renderer_config: Union[None, Mapping[str, Any], KiaraRendererConfig] = None, + ): + + super().__init__(kiara=kiara, renderer_config=renderer_config) + + filters = self.renderer_config.filter + tags = self.renderer_config.tags + + self._api_endpoints: ApiEndpoints = ApiEndpoints( + api_cls=BaseAPI, filters=filters, include_tags=tags + ) + + @property + def api_endpoints(self) -> ApiEndpoints: + return self._api_endpoints + + def get_renderer_alias(self) -> str: + return f"api_to_{self.get_target_type()}" + + def retrieve_supported_render_sources(self) -> str: + return "base_api" + + def retrieve_doc(self) -> Union[str, None]: + + return f"Render the kiara base API to: '{self.get_target_type()}'." + + @abc.abstractmethod + def get_target_type(self) -> str: + pass + + +class BaseApiDocRenderer(BaseApiRenderer): + + _renderer_name = "base_api_doc_renderer" + + def get_target_type(self) -> str: + return "html" + + def retrieve_supported_render_targets(self) -> Union[Iterable[str], str]: + return "html" + + def _render( + self, instance: BaseAPI, render_config: BaseApiRenderInputsSchema + ) -> Any: + + # details = self.api_endpoints.get_api_endpoint("get_value") + details = self.api_endpoints.get_api_endpoint("retrieve_aliases_info") + + # for k, v in details.arg_schema.items(): + # print(k) + # print(type(v)) + + terminal_print(create_table_from_base_model_v1_cls(details.arg_model)) + + return "xxx" + + +class BaseApiDocTextRenderer(BaseApiRenderer): + + _renderer_name = "base_api_doc_markdown_renderer" + + def get_target_type(self) -> str: + return "markdown" + + def retrieve_supported_render_targets(self) -> Union[Iterable[str], str]: + return "markdown" + + def _render( + self, instance: BaseAPI, render_config: BaseApiRenderInputsSchema + ) -> Any: + + template = self._kiara.render_registry.get_template( + "kiara_api/api_doc.md.j2", "kiara" + ) + + result = "" + for ep in self.api_endpoints.api_endpint_names: + doc = self.api_endpoints.get_api_endpoint(ep).doc + rendered = template.render(endpoint_name=ep, doc=doc) + result += f"{rendered}\n" + + # details = self.api_endpoints.get_api_endpoint("get_value") + # dbg(details.validated_func.__dict__) + + # for k, v in details.arg_schema.items(): + # print(k) + # print(type(v)) + + # inputs = { + # "value": "tm.date_array" + # } + # result = details.execute(instance, **inputs) + # print(result) + + return result + + +class BaseApiRenderKiaraApiInputsSchema(BaseApiRenderInputsSchema): + + template_file: str = Field( + description="The file that should contain the rendered code." + ) + target_file: Union[str, None] = Field( + description="The file to write the rendered code to.", default=None + ) + + +class BaseToKiaraApiRenderer(BaseApiRenderer): + + _renderer_name = "base_api_kiara_api_renderer" + _inputs_schema = BaseApiRenderKiaraApiInputsSchema # type: ignore + _renderer_config_cls = BaseApiRendererConfig + + def __init__( + self, + kiara: "Kiara", + renderer_config: Union[None, Mapping[str, Any], KiaraRendererConfig] = None, + ): + + self._api_endpoints: ApiEndpoints = ApiEndpoints(api_cls=BaseAPI) + super().__init__(kiara=kiara, renderer_config=renderer_config) + + def get_target_type(self) -> str: + return "kiara_api" + + def retrieve_supported_render_targets(self) -> Union[Iterable[str], str]: + return "kiara_api" + + def _render( + self, instance: BaseAPI, render_config: BaseApiRenderInputsSchema + ) -> Any: + + assert isinstance(render_config, BaseApiRenderKiaraApiInputsSchema) + + template_file = Path(render_config.template_file) + + if not template_file.is_file(): + raise ValueError(f"File '{template_file}' does not exist.") + + BEGIN_IMPORTS_MARKER = "# BEGIN AUTO-GENERATED-IMPORTS" + END_IMPORTS_MARKER = "# END AUTO-GENERATED-IMPORTS" + BEGIN_MARKER = "# BEGIN IMPORTED-ENDPOINTS" + END_MARKER = "# END IMPORTED-ENDPOINTS" + + template_file_content = template_file.read_text() + if BEGIN_IMPORTS_MARKER not in template_file_content: + raise ValueError( + f"File '{template_file}' does not contain BEGIN_IMPORTS_MARKER '{BEGIN_IMPORTS_MARKER}'." + ) + if END_IMPORTS_MARKER not in template_file_content: + raise ValueError( + f"File '{template_file}' does not contain END_IMPORTS_MARKER '{END_IMPORTS_MARKER}'." + ) + if BEGIN_MARKER not in template_file_content: + raise ValueError( + f"File '{template_file}' does not contain BEGIN_MARKER '{BEGIN_MARKER}'." + ) + if END_MARKER not in template_file_content: + raise ValueError( + f"File '{template_file}' does not contain END_MARKER '{END_MARKER}'." + ) + + endpoint_code_template = self._kiara.render_registry.get_template( + "kiara_api/kiara_api_endpoint.py.j2", "kiara" + ) + + # tag = render_config.tag + # endpoints = find_base_api_endpoints(BaseAPI, label=tag) + + endpoint_data = [] + imports: Dict[str, Set[str]] = {} + imports.setdefault("typing", set()).add("Dict") + imports.setdefault("typing", set()).add("Mapping") + imports.setdefault("typing", set()).add("ClassVar") + + for endpoint_name in self.api_endpoints.api_endpint_names: + endpoint_instance = self.api_endpoints.get_api_endpoint(endpoint_name) + + doc = endpoint_instance.raw_doc + + sig_args = extract_arg_names(endpoint_instance.func) + sig_args.remove("self") + + arg_names_str = extract_proxy_arg_str(endpoint_instance.func) + + sig_string, return_type = create_signature_string( + endpoint_instance.func, imports=imports + ) + regex_str = "" + if "\\" in doc: + regex_str = "r" + endpoint_data.append( + { + "endpoint_name": endpoint_name, + "doc": doc.strip(), + "signature_str": sig_string, + "arg_names_str": arg_names_str, + "result_type": return_type, + "regex_str": regex_str, + } + ) + + # remove abc modules + imports.pop("collections.abc", None) + + result = "" + for endpoint_item in endpoint_data: + + rendered = endpoint_code_template.render(**endpoint_item) + result += f"{rendered}\n" + + result = f"{BEGIN_MARKER}\n{result} {END_MARKER}" + + pattern = rf"{BEGIN_MARKER}.*?{END_MARKER}" + new_content = re.sub(pattern, result, template_file_content, flags=re.DOTALL) + + TYPE_CHECKING_FILTER = ["typing", "builtins", "collections", "uuid", "pathlib"] + + imports.setdefault("typing", set()).add("TYPE_CHECKING") + + imports_str = "" + for module, items in imports.items(): + first_token = module.split(".")[0] + if first_token in TYPE_CHECKING_FILTER: + items_str = ", ".join(sorted(items)) + imports_str += f"from {module} import {items_str}\n" + + match = False + imports_str += "if TYPE_CHECKING:\n" + for module, items in imports.items(): + first_token = module.split(".")[0] + if first_token not in TYPE_CHECKING_FILTER: + match = True + items_str = ", ".join(sorted(items)) + imports_str += f" from {module} import {items_str}\n" + + if not match: + imports_str += " pass\n" + + imports_pattern = rf"{BEGIN_IMPORTS_MARKER}\n.*?{END_IMPORTS_MARKER}" + new_content = re.sub( + imports_pattern, + f"{BEGIN_IMPORTS_MARKER}\n{imports_str}\n{END_IMPORTS_MARKER}", + new_content, + flags=re.DOTALL, + ) + + try_formatting = False + try: + import black + + try_formatting = True + except ImportError: + pass + + if try_formatting: + try: + new_content = black.format_str(new_content, mode=black.FileMode()) + except Exception as e: + raise ValueError(f"Failed to format code: {e}") + + if render_config.target_file: + target_file = Path(render_config.target_file) + target_file.parent.mkdir(parents=True, exist_ok=True) + target_file.write_text(new_content) + terminal_print() + terminal_print(f"Rendered api to file '{target_file}'.") + else: + return new_content diff --git a/src/kiara/renderers/included_renderers/api.py b/src/kiara/renderers/included_renderers/api/kiara_api.py similarity index 84% rename from src/kiara/renderers/included_renderers/api.py rename to src/kiara/renderers/included_renderers/api/kiara_api.py index 200249087..7f7dcd6cb 100644 --- a/src/kiara/renderers/included_renderers/api.py +++ b/src/kiara/renderers/included_renderers/api/kiara_api.py @@ -29,15 +29,15 @@ class ApiRenderInputsSchema(RenderInputsSchema): - filter: Union[str, Iterable[str]] = Field( - description="One or a list of filter tokens -- if provided -- all of which must match for the api endpoing to be in the render result.", - default_factory=list, - ) + pass class ApiRendererConfig(KiaraRendererConfig): - pass + filter: Union[str, Iterable[str]] = Field( + description="One or a list of filter tokens -- if provided -- all of which must match for the api endpoing to be in the render result.", + default_factory=list, + ) # target_type: str = Field(description="The target type to render the api as.") @@ -53,9 +53,18 @@ def __init__( renderer_config: Union[None, Mapping[str, Any], KiaraRendererConfig] = None, ): - self._api_endpoints: ApiEndpoints = ApiEndpoints(api_cls=KiaraAPI) super().__init__(kiara=kiara, renderer_config=renderer_config) + filters: Union[None, str, Iterable[str]] = self.renderer_config.filter + if not filters: + filters = None + elif isinstance(filters, str): + filters = [filters] + + self._api_endpoints: ApiEndpoints = ApiEndpoints( + api_cls=KiaraAPI, filters=filters + ) + @property def api_endpoints(self) -> ApiEndpoints: return self._api_endpoints @@ -68,16 +77,16 @@ def retrieve_supported_render_sources(self) -> str: def retrieve_doc(self) -> Union[str, None]: - return f"Render the kiara (of a supported type) to: '{self.get_target_type()}'." + return f"Render the kiara API endpoints to: '{self.get_target_type()}'." @abc.abstractmethod def get_target_type(self) -> str: pass -class ApiDocRenderer(ApiRenderer): +class KiaraApiDocRenderer(ApiRenderer): - _renderer_name = "api_doc_renderer" + _renderer_name = "kiara_api_doc_renderer" def get_target_type(self) -> str: return "html" @@ -99,9 +108,9 @@ def _render(self, instance: KiaraAPI, render_config: ApiRenderInputsSchema) -> A return "xxx" -class ApiDocTextRenderer(ApiRenderer): +class KiaraApiDocTextRenderer(ApiRenderer): - _renderer_name = "api_doc_markdown_renderer" + _renderer_name = "kiara_api_doc_markdown_renderer" def get_target_type(self) -> str: return "markdown" @@ -119,7 +128,7 @@ def _render(self, instance: KiaraAPI, render_config: ApiRenderInputsSchema) -> A for ep in self.api_endpoints.api_endpint_names: doc = self.api_endpoints.get_api_endpoint(ep).doc rendered = template.render(endpoint_name=ep, doc=doc) - result += rendered + result += f"{rendered}\n" # details = self.api_endpoints.get_api_endpoint("get_value") # dbg(details.validated_func.__dict__) diff --git a/src/kiara/renderers/included_renderers/job.py b/src/kiara/renderers/included_renderers/job.py index c174bab65..4e4eb4146 100644 --- a/src/kiara/renderers/included_renderers/job.py +++ b/src/kiara/renderers/included_renderers/job.py @@ -11,7 +11,6 @@ from jinja2 import Template -from kiara.interfaces.python_api import JobDesc from kiara.models.module.pipeline.pipeline import Pipeline from kiara.renderers import ( RenderInputsSchema, @@ -20,6 +19,7 @@ from kiara.renderers.jinja import BaseJinjaRenderer, JinjaEnv if TYPE_CHECKING: + from kiara.api import JobDesc from kiara.context import Kiara @@ -29,6 +29,8 @@ def __init__(self, kiara: "Kiara"): super().__init__() def retrieve_supported_python_classes(self) -> Iterable[Type]: + from kiara.api import JobDesc + return [JobDesc, str, Mapping, Path] def retrieve_supported_inputs_descs(self) -> Union[str, Iterable[str]]: @@ -38,7 +40,8 @@ def retrieve_supported_inputs_descs(self) -> Union[str, Iterable[str]]: "the job description as a Python dict", ] - def validate_and_transform(self, source: Any) -> Union[JobDesc, None]: + def validate_and_transform(self, source: Any) -> Union["JobDesc", None]: + from kiara.api import JobDesc if isinstance(source, JobDesc): return source @@ -50,7 +53,7 @@ def validate_and_transform(self, source: Any) -> Union[JobDesc, None]: return None -class JobDescPythonScriptRenderer(BaseJinjaRenderer[JobDesc, RenderInputsSchema]): +class JobDescPythonScriptRenderer(BaseJinjaRenderer["JobDesc", RenderInputsSchema]): """Renders a simple executable python script from a job description. ## Examples diff --git a/src/kiara/renderers/included_renderers/pipeline.py b/src/kiara/renderers/included_renderers/pipeline.py index b2fa81848..9c6cc40d2 100644 --- a/src/kiara/renderers/included_renderers/pipeline.py +++ b/src/kiara/renderers/included_renderers/pipeline.py @@ -318,44 +318,3 @@ def assemble_render_inputs( pipeline_user_inputs: Mapping[str, Any] = render_config.inputs result = create_pipeline_render_inputs(pipeline, pipeline_user_inputs) return result - - # from kiara.defaults import SpecialValue - # - # pipeline_inputs_user = render_config.inputs - # pipeline: Pipeline = instance - # - # invalid = [] - # for field_name in pipeline_inputs_user.keys(): - # if field_name not in pipeline.pipeline_inputs_schema.keys(): - # invalid.append(field_name) - # - # if invalid: - # msg = "Valid pipeline inputs:\n" - # for field_name, field in pipeline.pipeline_inputs_schema.items(): - # msg = f"{msg} - *{field_name}*: {field.doc.description}\n" - # raise KiaraException( - # msg=f"Invalid pipeline inputs: {', '.join(invalid)}.", details=msg - # ) - # - # pipeline_inputs = {} - # for field_name, schema in pipeline.pipeline_inputs_schema.items(): - # if field_name in pipeline_inputs_user.keys(): - # value = pipeline_inputs_user[field_name] - # elif schema.default not in [SpecialValue.NOT_SET]: - # if callable(schema.default): - # value = schema.default() - # else: - # value = schema.default - # elif not schema.is_required(): - # value = None - # else: - # value = "" - # - # if isinstance(value, str): - # value = f'"{value}"' - # pipeline_inputs[field_name] = value - # - # inputs: MutableMapping[str, Any] = render_config.model_dump() - # inputs["pipeline"] = pipeline - # inputs["pipeline_inputs"] = pipeline_inputs - # return inputs diff --git a/src/kiara/resources/templates/render/kiara_api/kiara_api_endpoint.py.j2 b/src/kiara/resources/templates/render/kiara_api/kiara_api_endpoint.py.j2 new file mode 100644 index 000000000..9fa5b3034 --- /dev/null +++ b/src/kiara/resources/templates/render/kiara_api/kiara_api_endpoint.py.j2 @@ -0,0 +1,11 @@ + {{ signature_str }} + {{ regex_str }}"""{{ doc }} + """ +{% if result_type == None %} + self._api.{{ endpoint_name }}({{ arg_names_str }}) +{% else %} + result: {{ result_type }} = self._api.{{ endpoint_name }}({{ arg_names_str }}) + return result +{% endif %} + + diff --git a/src/kiara/resources/templates/render/pipeline/python_script.py.j2 b/src/kiara/resources/templates/render/pipeline/python_script.py.j2 index 177f29ebe..914a1a6a5 100644 --- a/src/kiara/resources/templates/render/pipeline/python_script.py.j2 +++ b/src/kiara/resources/templates/render/pipeline/python_script.py.j2 @@ -44,6 +44,7 @@ results_{{ step_id }} = kiara.run_job('{{ step.manifest_src.module_type }}', ope {%- set pipeline_output_refs = pipeline.pipeline_output_refs -%} {% for field_name, output_ref in pipeline_output_refs.items() %} pipeline_result_{{ field_name }} = results_{{ output_ref.connected_output.step_id }}['{{ output_ref.connected_output.value_name }}'] -terminal_rendered_result_{{ field_name }} = kiara.render_value(pipeline_result_{{ field_name }}, target_format="terminal_renderable", use_pretty_print=True) -terminal_print(terminal_rendered_result_{{ field_name }}, in_panel='Pipeline result: [b]{{ field_name }}[/b]') + +terminal_print(pipeline_result_{{ field_name }}, in_panel='Pipeline result metadata: [b]{{ field_name }}[/b]') +terminal_print(pipeline_result_{{ field_name }}.data, in_panel='Pipeline result data: [b]{{ field_name }}[/b]') {% endfor %} diff --git a/src/kiara/utils/archives.py b/src/kiara/utils/archives.py new file mode 100644 index 000000000..9c4b0ee7b --- /dev/null +++ b/src/kiara/utils/archives.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +from functools import lru_cache +from typing import TYPE_CHECKING, Dict, Union + +if TYPE_CHECKING: + from kiara.context import Kiara + from kiara.interfaces.python_api.models.info import TypeInfo + from kiara.models.archives import ArchiveTypeClassesInfo + + +@lru_cache(maxsize=None) +def find_archive_types( + alias: Union[str, None] = None, only_for_package: Union[str, None] = None +) -> "ArchiveTypeClassesInfo": + + from kiara.models.archives import ArchiveTypeClassesInfo + from kiara.utils.class_loading import find_all_archive_types + + archive_types = find_all_archive_types() + + kiara: Kiara = None # type: ignore + group: ArchiveTypeClassesInfo = ArchiveTypeClassesInfo.create_from_type_items( # type: ignore + kiara=kiara, group_title=alias, **archive_types + ) + + if only_for_package: + temp: Dict[str, TypeInfo] = {} + for key, info in group.item_infos.items(): + if info.context.labels.get("package") == only_for_package: + temp[key] = info # type: ignore + + group = ArchiveTypeClassesInfo( + group_id=group.group_id, group_title=group.group_alias, item_infos=temp # type: ignore + ) + + return group diff --git a/src/kiara/utils/cli/rich_click.py b/src/kiara/utils/cli/rich_click.py index 8bcf77dc2..8c94e3baf 100644 --- a/src/kiara/utils/cli/rich_click.py +++ b/src/kiara/utils/cli/rich_click.py @@ -59,7 +59,7 @@ ) from kiara.api import ValueMap -from kiara.interfaces.python_api import KiaraAPI, OperationGroupInfo +from kiara.interfaces.python_api.base_api import BaseAPI from kiara.models.module.operation import Operation # from kiara.interfaces.python_api.operation import KiaraOperation @@ -73,7 +73,7 @@ def rich_format_filter_operation_help( - api: KiaraAPI, + api: BaseAPI, obj: Union[click.Command, click.Group], ctx: click.Context, cmd_help: str, @@ -107,6 +107,9 @@ def rich_format_filter_operation_help( filter_op_type: FilterOperationType = api.get_operation_type("filter") # type: ignore v = api.get_value(value) ops = filter_op_type.find_filter_operations_for_data_type(v.data_type_name) + + from kiara.interfaces.python_api.models.info import OperationGroupInfo + ops_info = OperationGroupInfo.create_from_operations( kiara=api.context, group_title=f"{v.data_type_name} filters", **ops ) diff --git a/src/kiara/utils/cli/run.py b/src/kiara/utils/cli/run.py index 58b5fa9c0..3b9ae4e1f 100644 --- a/src/kiara/utils/cli/run.py +++ b/src/kiara/utils/cli/run.py @@ -10,15 +10,16 @@ from rich.markdown import Markdown from rich.rule import Rule -from kiara.api import KiaraAPI, ValueMap from kiara.exceptions import ( FailedJobException, InvalidCommandLineInvocation, KiaraException, NoSuchExecutionTargetException, ) +from kiara.interfaces.python_api.base_api import BaseAPI from kiara.interfaces.python_api.utils import create_save_config from kiara.models.module.operation import Operation +from kiara.models.values.value import ValueMap # from kiara.interfaces.python_api.operation import KiaraOperation from kiara.utils import log_exception @@ -46,7 +47,7 @@ def _validate_save_option(save: Iterable[str]) -> bool: def validate_operation_in_terminal( - api: KiaraAPI, + api: BaseAPI, module_or_operation: Union[str, Path, Mapping[str, Any]], allow_external=False, ) -> Operation: @@ -57,7 +58,7 @@ def validate_operation_in_terminal( # operation_config=module_config, # ) try: - operation = api.get_operation(operation=module_or_operation) + operation: Operation = api.get_operation(operation=module_or_operation) # validate that operation config is valid, ignoring inputs for now # kiara_op.operation except NoSuchExecutionTargetException as nset: @@ -186,7 +187,7 @@ def calculate_aliases( def set_and_validate_inputs( - api: KiaraAPI, + api: BaseAPI, operation: Operation, inputs: Iterable[str], explain: bool, @@ -326,7 +327,7 @@ def set_and_validate_inputs( def execute_job( - api: KiaraAPI, + api: BaseAPI, operation: Operation, inputs: ValueMap, silent: bool, @@ -393,7 +394,7 @@ def execute_job( title = "Result details" format = "terminal" - from kiara.interfaces.python_api import ValueInfo + from kiara.interfaces.python_api.models.info import ValueInfo v_infos = ( ValueInfo.create_from_instance(kiara=api.context, instance=v) diff --git a/src/kiara/utils/config.py b/src/kiara/utils/config.py new file mode 100644 index 000000000..02be6df19 --- /dev/null +++ b/src/kiara/utils/config.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +from pathlib import Path +from typing import TYPE_CHECKING, Union + +from kiara.defaults import KIARA_CONFIG_FILE_NAME, KIARA_MAIN_CONFIG_FILE +from kiara.exceptions import KiaraException + +if TYPE_CHECKING: + from kiara.context import KiaraConfig + + +def assemble_kiara_config( + config_file: Union[str, None] = None, create_config_file: bool = False +) -> "KiaraConfig": + """Assemble a KiaraConfig object from a config file path or create a new one. + + + + Arguments: + config_file: The path to a Kiara config file or a folder containing one named 'kiara.config'. + create_config_file: If True, create a new config file if it does not exist. + + """ + + exists = False + if config_file: + config_path = Path(config_file) + if config_path.exists(): + if config_path.is_file(): + config_file_path = config_path + exists = True + else: + config_file_path = config_path / KIARA_CONFIG_FILE_NAME + if config_file_path.exists(): + exists = True + else: + config_path.parent.mkdir(parents=True, exist_ok=True) + config_file_path = config_path + + else: + config_file_path = Path(KIARA_MAIN_CONFIG_FILE) + if not config_file_path.exists(): + exists = False + else: + exists = True + + from kiara.context import KiaraConfig + + if not exists: + kiara_config = KiaraConfig() + + if config_file: + if not create_config_file: + raise KiaraException( + f"specified config file does not exist: {config_file}." + ) + else: + if create_config_file: + kiara_config.save(config_file_path) + else: + kiara_config = KiaraConfig.load_from_file(config_file_path) + + return kiara_config diff --git a/src/kiara/utils/db.py b/src/kiara/utils/db.py index be311739e..8df68a0f1 100644 --- a/src/kiara/utils/db.py +++ b/src/kiara/utils/db.py @@ -6,10 +6,17 @@ # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) import os -from typing import Any +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict import orjson +from kiara import is_debug +from kiara.utils import log_message + +if TYPE_CHECKING: + from sqlalchemy.engine import Engine + def get_kiara_db_url(base_path: str): @@ -35,6 +42,59 @@ def orm_json_deserialize(obj: str) -> Any: return orjson.loads(obj) +def create_archive_engine( + db_path: Path, force_read_only: bool, use_wal_mode: bool +) -> "Engine": + + from sqlalchemy import create_engine, text + + # if use_wal_mode: + # # TODO: not sure this does anything + # connect_args = {"check_same_thread": False, "isolation_level": "IMMEDIATE"} + # execution_options = {"sqlite_wal_mode": True} + # else: + + connect_args: Dict[str, Any] = {} + execution_options: Dict[str, Any] = {} + + # TODO: enable this for read-only mode? + # def _pragma_on_connect(dbapi_con, con_record): + # dbapi_con.execute("PRAGMA query_only = ON") + + db_url = f"sqlite+pysqlite:///{db_path.as_posix()}" + if force_read_only: + db_url = db_url + "?mode=ro&uri=true" + + db_engine = create_engine( + db_url, + future=True, + connect_args=connect_args, + execution_options=execution_options, + ) + + if use_wal_mode: + with db_engine.connect() as conn: + conn.execute(text("PRAGMA journal_mode=wal;")) + + if is_debug(): + with db_engine.connect() as conn: + wal_mode = conn.execute(text("PRAGMA journal_mode;")).fetchone() + log_message( + "detect.sqlite.journal_mode", result={wal_mode[0]}, db_url=db_url + ) + + return db_engine + + +def delete_archive_db(db_path: Path): + + db_path.unlink(missing_ok=True) + shm_file = db_path.parent / f"{db_path.name}-shm" + shm_file.unlink(missing_ok=True) + wal_file = db_path.parent / f"{db_path.name}-wal" + wal_file.unlink(missing_ok=True) + + # def ensure_current_environments_persisted( # engine: Engine, # ) -> Mapping[str, EnvironmentOrm]: diff --git a/src/kiara/utils/introspection.py b/src/kiara/utils/introspection.py new file mode 100644 index 000000000..c899685fb --- /dev/null +++ b/src/kiara/utils/introspection.py @@ -0,0 +1,167 @@ +# -*- coding: utf-8 -*- +import inspect +import typing +from typing import Any, Callable, Dict + + +def extract_cls(arg: Any, imports: Dict[str, typing.Set[str]]) -> str: + + if arg in (type(None), None): + return "None" + elif isinstance(arg, type): + name = arg.__name__ + module = arg.__module__ + if module == "typing": + imports.setdefault(module, set()).add(name) + return name + elif module != "builtins": + imports.setdefault(module, set()).add(name) + return f'"{name}"' + else: + return name + elif isinstance(arg, typing._UnionGenericAlias): # type: ignore + all_args = [] + for a in arg.__args__: + cls = extract_cls(a, imports=imports) + all_args.append(cls) + + imports.setdefault("typing", set()).add("Union") + return f"Union[{', '.join(all_args)}]" + elif isinstance(arg, typing._LiteralSpecialForm): # type: ignore + return "Literal" + + elif isinstance(arg, typing._GenericAlias): # type: ignore + + origin_cls = extract_cls(arg.__origin__, imports=imports) + if origin_cls == "Literal": + all_args_str = ", ".join((f'"{x}"' for x in arg.__args__)) + imports.setdefault("typing", set()).add("Literal") + return f"Literal[{all_args_str}]" + + all_args = [] + for a in arg.__args__: + cls = extract_cls(a, imports=imports) + all_args.append(cls) + + if origin_cls == '"Mapping"': + imports.setdefault("typing", set()).add("Mapping") + assert len(all_args) == 2 + return f"Mapping[{all_args[0]}, {all_args[1]}]" + elif origin_cls == '"Iterable"': + imports.setdefault("typing", set()).add("Iterable") + return f"Iterable[{', '.join(all_args)}]" + elif origin_cls in ('"List"', "list"): + imports.setdefault("typing", set()).add("List") + return f"List[{', '.join(all_args)}]" + elif origin_cls == "dict": + assert len(all_args) == 2 + imports.setdefault("typing", set()).add("Dict") + return f"Dict[{all_args[0]}, {all_args[1]}]" + elif origin_cls == "type": + imports.setdefault("typing", set()).add("Type") + result = f"Type[{', '.join(all_args)}]" + return result + else: + raise Exception(f"Unexpected generic alias: {origin_cls}") + elif isinstance(arg, typing.ForwardRef): + return f'"{arg.__forward_arg__}"' + else: + raise Exception(f"Unexpected type '{type(arg)}' for arg: {arg}") + + +def create_default_string(default: Any) -> str: + + if default is None: + return "None" + elif isinstance(default, bool): + return str(default) + elif isinstance(default, str): + if "\\" in default: + default = f'r"{default}"' + return default + else: + return f'"{default}"' + else: + raise Exception(f"Unexpected default value: {default}") + + +def parse_signature_args(func: Callable, imports: Dict[str, typing.Set[str]]) -> str: + sig = inspect.signature(func) + + all_tokens = [] + param: inspect.Parameter + for field_name, param in sig.parameters.items(): + if field_name == "self": + all_tokens.append("self") + else: + arg_str = extract_cls(arg=param.annotation, imports=imports) + + if param.kind == inspect.Parameter.VAR_POSITIONAL: + sig_token = f"*{field_name}: {arg_str}" + elif param.kind == inspect.Parameter.VAR_KEYWORD: + sig_token = f"**{field_name}: {arg_str}" + else: + sig_token = f"{field_name}: {arg_str}" + + if param.default != inspect.Parameter.empty: + default_str = create_default_string(default=param.default) + sig_token += f" = {default_str}" + + all_tokens.append(sig_token) + + return ", ".join(all_tokens) + + +def parse_signature_return(func: Callable, imports: Dict[str, typing.Set[str]]) -> str: + + sig = inspect.signature(func) + sig_return_type = sig.return_annotation + if isinstance(sig_return_type, str): + return f'"{sig_return_type}"' + elif sig_return_type == inspect.Parameter.empty: + return "None" + else: + sig_return_type_str = extract_cls(arg=sig_return_type, imports=imports) + return sig_return_type_str + + +def create_signature_string( + func: Callable, imports: Dict[str, typing.Set[str]] +) -> typing.Tuple[str, typing.Union[str, None]]: + + params = parse_signature_args(func=func, imports=imports) + return_type = parse_signature_return(func=func, imports=imports) + if return_type == "None": + sig_str = f"def {func.__name__}({params}):" + _return_type = None + else: + sig_str = f"def {func.__name__}({params}) -> {return_type}:" + _return_type = return_type + + return sig_str, _return_type + + +def extract_arg_names(func: Callable) -> typing.List[str]: + sig = inspect.signature(func) + return list(sig.parameters.keys()) + + +def extract_proxy_arg_str(func: Callable) -> str: + + sig = inspect.signature(func) + arg_str = "" + for field_name, param in sig.parameters.items(): + + if field_name == "self": + continue + + if param.kind == inspect.Parameter.VAR_POSITIONAL: + arg_str += f"*{field_name}, " + elif param.kind == inspect.Parameter.VAR_KEYWORD: + arg_str += f"**{field_name}, " + else: + arg_str += f"{field_name}={field_name}, " + + if arg_str.endswith(", "): + arg_str = arg_str[:-2] + return arg_str diff --git a/src/kiara/utils/metadata.py b/src/kiara/utils/metadata.py index 284ea9a5e..016f522ea 100644 --- a/src/kiara/utils/metadata.py +++ b/src/kiara/utils/metadata.py @@ -1,26 +1,36 @@ # -*- coding: utf-8 -*- +from functools import lru_cache # Copyright (c) 2021, University of Luxembourg / DHARPA project # Copyright (c) 2021, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) +from typing import TYPE_CHECKING, Dict, Type, Union -from typing import Dict, Type, Union - -from kiara.models.values.value_metadata import MetadataTypeClassesInfo, ValueMetadata +from kiara.models.values.value_metadata import ValueMetadata from kiara.registries.models import ModelRegistry +if TYPE_CHECKING: + from kiara.context import Kiara + from kiara.interfaces.python_api.models.info import ( + MetadataTypeClassesInfo, + ) + +@lru_cache() def find_metadata_models( alias: Union[str, None] = None, only_for_package: Union[str, None] = None -) -> MetadataTypeClassesInfo: +) -> "MetadataTypeClassesInfo": + + from kiara.interfaces.python_api.models.info import MetadataTypeClassesInfo model_registry = ModelRegistry.instance() _group = model_registry.get_models_of_type(ValueMetadata) # type: ignore classes: Dict[str, Type[ValueMetadata]] = {} for model_id, info in _group.item_infos.items(): - classes[model_id] = info.python_class.get_class() # type: ignore + model_cls = info.python_class.get_class() # type: ignore + classes[model_id] = model_cls group: MetadataTypeClassesInfo = MetadataTypeClassesInfo.create_from_type_items(group_title=alias, kiara=None, **classes) # type: ignore @@ -35,3 +45,44 @@ def find_metadata_models( ) return group + + +def get_metadata_model_for_data_type( + kiara: "Kiara", data_type: str +) -> "MetadataTypeClassesInfo": + """ + Return all available metadata extract operations for the provided type (and it's parent types). + + Arguments: + --------- + data_type: the value type + + Returns: + ------- + a mapping with the metadata type as key, and the operation as value + """ + + from kiara.interfaces.python_api.models.info import MetadataTypeClassesInfo + + # TODO: add models for parent types? + # lineage = set(kiara.type_registry.get_type_lineage(data_type_name=data_type)) + + model_registry = ModelRegistry.instance() + all_metadata_models = model_registry.get_models_of_type(ValueMetadata) + + matching_types = {} + + for name, model_info in all_metadata_models.item_infos.items(): + + metadata_cls: Type[ValueMetadata] = model_info.python_class.get_class() + supported = metadata_cls.retrieve_supported_data_types() + if data_type in supported: + matching_types[name] = metadata_cls + + result: MetadataTypeClassesInfo = MetadataTypeClassesInfo.create_from_type_items( + kiara=kiara, + group_title=f"Metadata models for type '{data_type}'", + **matching_types, + ) + + return result diff --git a/src/kiara/utils/stores.py b/src/kiara/utils/stores.py index 0a79628b9..b8c4bd55d 100644 --- a/src/kiara/utils/stores.py +++ b/src/kiara/utils/stores.py @@ -12,8 +12,19 @@ def create_new_archive( store_base_path: str, store_type: str, allow_write_access: bool = False, + set_archive_name_metadata: bool = True, **kwargs: Any, ) -> "KiaraArchive": + """Create a new archive instance of the specified type. + + Arguments: + archive_name: Name of the archive. + store_base_path: Base path for the archive. + store_type: Type of the archive. + allow_write_access: Whether write access should be allowed. + set_archive_name_metadata: Whether to set the archive name as metadata within the archive. + **kwargs: Additional arguments to pass to the archive config constructor. + """ from kiara.utils.class_loading import find_all_archive_types @@ -33,7 +44,7 @@ def create_new_archive( archive_instance = archive_cls(archive_name=archive_name, archive_config=config, force_read_only=force_read_only) # type: ignore - if not force_read_only: + if not force_read_only and set_archive_name_metadata: archive_instance.set_archive_metadata_value(ARCHIVE_NAME_MARKER, archive_name) return archive_instance diff --git a/src/kiara/utils/testing/__init__.py b/src/kiara/utils/testing/__init__.py index 0d8561f7c..6ed158be3 100644 --- a/src/kiara/utils/testing/__init__.py +++ b/src/kiara/utils/testing/__init__.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, Mapping, Union from kiara.defaults import INIT_EXAMPLE_NAME -from kiara.interfaces.python_api import JobDesc +from kiara.interfaces.python_api.models.job import JobDesc from kiara.utils import log_exception from kiara.utils.files import get_data_from_file diff --git a/src/kiara/utils/values.py b/src/kiara/utils/values.py index f974fb5b6..6a7d022d8 100644 --- a/src/kiara/utils/values.py +++ b/src/kiara/utils/values.py @@ -20,13 +20,13 @@ if TYPE_CHECKING: from kiara.context import Kiara - from kiara.interfaces.python_api import KiaraAPI + from kiara.interfaces.python_api.base_api import BaseAPI from kiara.models.values.value import ValueMapReadOnly from kiara.registries.data import ValueLink def construct_valuemap( - kiara_api: "KiaraAPI", + kiara_api: "BaseAPI", values: Mapping[str, Union[uuid.UUID, None, str, "ValueLink"]], ) -> "ValueMapReadOnly": diff --git a/src/kiara/zmq/service/__init__.py b/src/kiara/zmq/service/__init__.py index 07c69d75d..1dd33ac01 100644 --- a/src/kiara/zmq/service/__init__.py +++ b/src/kiara/zmq/service/__init__.py @@ -7,10 +7,9 @@ import orjson import zmq -from kiara.api import KiaraAPI from kiara.defaults import KIARA_MAIN_CONTEXT_LOCKS_PATH from kiara.exceptions import KiaraException -from kiara.interfaces import KiaraAPIWrap, get_console, get_proxy_console +from kiara.interfaces import BaseAPI, KiaraAPIWrap, get_console, get_proxy_console from kiara.interfaces.cli.proxy_cli import proxy_cli from kiara.interfaces.python_api.proxy import ApiEndpoints from kiara.zmq import ( @@ -56,7 +55,7 @@ def __init__( self._port: int = int(port) self._service_thread = None self._msg_builder = KiaraApiMsgBuilder() - self._api_endpoints: ApiEndpoints = ApiEndpoints(api_cls=KiaraAPI) + self._api_endpoints: ApiEndpoints = ApiEndpoints(api_cls=BaseAPI) self._initial_timeout = listen_timout_in_ms self._allow_timeout_change = False @@ -178,7 +177,7 @@ def service_loop(self): print(f"ERROR IN ZMQ SERVICE: {e}", file=self._stderr) print("Stopping...", file=self._stderr) - def call_cli(self, api: KiaraAPI, **kwargs) -> Mapping[str, str]: + def call_cli(self, api: BaseAPI, **kwargs) -> Mapping[str, str]: console = get_console() old_width = console.width @@ -216,7 +215,7 @@ def call_cli(self, api: KiaraAPI, **kwargs) -> Mapping[str, str]: return {"stdout": stdout, "stderr": stderr} - def call_endpoint(self, api: KiaraAPI, endpoint: str, **kwargs) -> Any: + def call_endpoint(self, api: BaseAPI, endpoint: str, **kwargs) -> Any: try: endpoint_proxy = self._api_endpoints.get_api_endpoint( diff --git a/tests/conftest.py b/tests/conftest.py index 2371f5051..41e72bc3f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,9 +20,9 @@ import pytest -from kiara.api import KiaraAPI from kiara.context import Kiara from kiara.context.config import KiaraConfig +from kiara.interfaces.python_api.base_api import BaseAPI from kiara.interfaces.python_api.batch import BatchOperation from .utils import INVALID_PIPELINES_FOLDER, MODULE_CONFIGS_FOLDER, PIPELINES_FOLDER @@ -97,12 +97,12 @@ def kiara() -> Kiara: @pytest.fixture -def api() -> KiaraAPI: +def api() -> BaseAPI: instance_path = create_temp_dir() kc = KiaraConfig.create_in_folder(instance_path) kc.runtime_config.runtime_profile = "default" - api = KiaraAPI(kc) + api = BaseAPI(kc) return api diff --git a/tests/resources/archives/export_test.kiarchive b/tests/resources/archives/export_test.kiarchive index b13428cd4befc2478e454439b95f2014509cd35f..fe2a2c66e302b665fbbaae750d48f29055a98a1f 100644 GIT binary patch literal 159744 zcmeHwX>24(b{^U6dk&7Yl6tN2O0_jR>S;_TC$GwbD?wuM9J0vaQN<>?8kF*gtV*&f zvnn&I$YQ%K^d2J#tA(@@3?DFT7_i}g7W_jnY#1;s*gtH;uwnRz{6~Nd13utCHVnbg z!iHbuQOwAyVkf(MW_xC$Fw-QnUPiq5;>C*>5ij1`+o{+ADp;=9)`EgC_14t%^wh5u z3R6>4--Q3~!~e3)#*Q*ntgBfAH4A&-u!Q` z|MfTi%4`4Zb#(WCzxKVm%{w37{owX*-}%{Fzju4%M(dV#i=8>R@$U4W&3t`Qq4fF7 zK;YZoxOeN`JMTh}oo;yAMiVy?O7$cc!Oo$3$m-w*{l6 zuJr;p{JWk)q|T-O_;cS`|LI%z1Y!DG) z>%pLdNVX$*Gh`=dkb^4e%>;pLQHF=KNxcO)Zs#f5x?TMwe}!B5@1xA?VZiUOa!{L>$NCD#i>f%&PY(p$IgG0gP0f1Z$% zq$x?k$=^RQR%()O6Y0seevg?~+GQYcOn1T%cEK6aGceZb7#ytnZ7q~^p7)VcVV zx1JiW-@5niyVKvf6EcsSzU{eA8##fW{{0Q28R>80wxk|SRP1S*Nux4NGdizD!kIJ{frT=>K>7Ca?seZZ~!V~ri%*rsg+m4O=?BA~w zFlE1up}MSVqml=8M{8%4$=wG}-@J3{-uv%QKiy07(m|%(@DPeCOa8n&j+5leOBoq8 zh=#8bw$R-NzjXKJy`A?bp_*Z+&Sf6F_Vnj&g_hj6XVUG#ogP!-UvH0VM<^JPC1*5y z&dG3`q$DE}mFVk`zFhgJTp%^P;kG*-@_pNNBBA|otGc|pyc^njU_;%$%!W!aIcyuQ z=+4u@jZkL3^KRI#4)T242iv0_y^;O=9$0{CS;#F7xGQf?Ndc zSXCSW_iSjRB9bO$XQyAfo@0z9FAwD~d2r+D>(i0`+6dv)Jfms%(Xjine_ta2&3+rM zj_~q{_`HX$umGI}X~j`^SP0t{eUXg12fuvnGV#dP2eVX$-x*jz-o~rQH{k!7*~e3} z|9bZC!w>qQ2v7tl0u%v?07ZZzKoOt_Py{Ff6ak6=Md0NkAkNI(ubIfwdaWSZ($XwN z5Y>{wsFrRqB3CLgnra}1S1k@HIucDy{n=|X@4k~P=kYERI_tPzP!C`$m7AXVdM)|R zHcJZ2AyL&CQxSMZlmwH}IaOnjWJxv|@ccD!)v;Ssl_J7U(NBBWM6ak6=MSvne z5ugZA1SkR&0g3=cfFeKis%{b^!LVdkxwcc+sr0lZys?TwkQ-$@?W<_&1m zjoJT#TlNdHKbrm1+5b8FKcYWh{l-!bDFPG$iU37`B0v$K2v7tl0u%v?07ZZzKoQ6y z@QqvW`hF_N|uBKR%K+Qv5aVG zJfrEX&SbNRW>x`dRU@r%Dl<) z46kq{Miikil~q(IR$@6t&_z)cWeUMy@qESVL`vfT$}pt%;REb5HulMrqR-Tzc?S}DJ{+yb=PebA40O?UJLc? z79>aYiw}!duhqg|Jb7~ct?Nhs{xAR09rT~yU7Eh`|3cCpYh&m5plt82YWwE4aB8fT zrOjHoQBiw?-rm}ymBsQ&$=#L9invk>*|`4osj0y0p(y_U;neKEnEm1G;pe=IDf<)w ziU37`B0v$K2v7tl0u%v?07ZZzKoR(?5s+v6>F3yNm`S~NmA!>)(v1H+n*sRz|MqM9 zQ?q~P&Hwu558n8%fm9?%a(fFeK(9o8-v7_3(kU55fFeKf^up!fe@O8kF({(pV;_SEbj z&3<=w1%9C)iU37`B0v$K2v7tl0u%v?07ZZzKoOt_e1Q@8#p^RQIgh{2Sg)C11jhwA zkHgCNx39t5c|118zYWd-avr~iv2Ww`|I}Mw;EttDqX$q|LNHuO~HTkLlK||Py{Ff6ak6=MSvne5ugZA1SkR&0gAvE z4}t45vv&sg{D0=g^{LnY>FwWvAM`^Jpa@U|zElYO#_gFKuJD7`&dpA*EG!CK_1rit z4VE|BhXJ=zK6tcI+bQiUOFgT%T=RDJ_SF6T17oMMVLO&P=k=VxZlgGh(&OScAQ2NJ zVJbW>ZV!T{>uhVriPnJBPVl7TxgF#MHi`%wtsOl#k#Bf*C$L=y=!(@QD&X%5u2l%& z_o$#kCn{;K_^?n6?4X6;EpFj5@PjaYRH5hxp6xV>PvB7>d45#lW5^CQQbYW8QpwoT z_v0eQRDf&nP`77$2(n2PzEuocldAS({8GXuiAAb7S3gEZP<(=m(@fLG7-+5SG*Fh- z@)3NUK@T~7+jE^Za)LP16(AfY#lpBmNv#N|mM>l?R54t*&WNs_AFHS(WG6(=3YVaT zYl^xr>iRMLk$hjUabB-h3z_g+y8qvU_69wz^`MYyddP?!pmtR5H#}tF=D+hnD2E?J zoNdQ4h#P#-B?&+HVJtnG=RqE>urr9hO;s3SJ&Ihqg=4yYHyS3XK1fS#A_c$s!L=Li z`@f^%p}64pg5F_sAU2#pUGxKeUEkQ@OsUrIRN5$K5#x<^V^BZi{uKYo@h zW$@qB92lcM0B*TnTf;(F&~&#Kg!q572S04T5T;Ozy2mk40iM9%cVM`|v$qxpgfY1{0UUx302E{tHW@X_9Vzwllg_-qXd<-H~U0dD@4BpRy@;LK^%{ecf1 z{%qNf-*BDe4XlGQEyucMNLPqT4wVln;U7hn11x(Nm5-{NPD=|gf*^upYf8Q!Q^R)h zYDyea%FIJ8r1=xo<*?wGZcGKcaS7wge})1WmLJoNDgW}(-`0$#?M&of^Wf_MG-1y} zB=Vc_i{sgL{kq>lhHcr#ILtwEldlWICJY0C5fb!~U)W1ZO4XH&OsqPdsS<{Qln5NW zlvIl=OF9vLP4I?a6KDvn@zGvBjq%AZPTz1BzK=qUG1yw#7f%Pf&7*yL@yOacE-!ND z3%iYF@!&Mzgx+9n`{=+@SIes#%Y?>=k|TCny#^>>I)@K?hv*V=E`g3qW{tFryv#YA z=oLa(Q`5-BR8Epl&_Iz6k9s<>sIcDuOs3oooP)El9f?`kpy!x($z~Eg6qX+spKti! z^Z%LI`qb=Sn*CSsgMKIi6ak6=MSvne5ugZA1SkR&0g3=cfFeK@$OB@Sr+C8l})pP4`+dj2|bK+>G6M9ixc@lkGOmPtV2MnD8Mo#^U z!W_rXv4y{o90`~OeNX^r0#jq)JhOq|oAC7ZVc{bqC=`SOv|o5wV3m1MnCJNig>S7c zE*78+VUGQ{_%L2G^x7QJg@_={f=yWS}TMy7#NArB-5u~_3h=%I6X@kED+4s0Jh~iq{n?ca< zVz!w&oQE4tf=J4CsqVG{Hw!)Pc(S&h-7<-SfPf@1u-8&ng5YW?FvX`C9}zPsCnMN~ z)#wwI2>=lxGDZ{Q7%&cY#c0{*8At`_DYDX*ok2?h{$+wMOu~R?mey+pOv8gNC3wJ+ zO$-~uZD<{m2J=baX1Hx|!XO5577lOF3EYOKb(#Z$QpA#7iaCU-=4Zdtymklc-}l|` zZt#MoB1M)Mg;PtA<(pL))f80*$t9Up1tdwLGEV)Lc!g6YsNaFzII~gLm%Mg+rMXddSlmsl=n(;#9Ls#*l~UP z43HW+4a^rSw$sCp65oOOq6E5<6B(Xm=PW}|=gt(VE|DYj9j>sqxn0=aU0&YYuGW_h zw-z=(Tr8}3c41v}3cLjRd!FOxIjK;qE@EYl-%lJ7=8MB33I7UkX|b_OiVlhad+H69 z^$IdwIM_{Cy-@3d3dI_(i9+YN`69ShJ^9|<>9DI8+)oA^TSB~{AjUK$0cKiNVsus) z8B2y~R^TjEGxTv?ErJBd-1wQ+*SqJ|BYka2YI}>D?%~Qp^+306^R(fK+UlcY-QIPU zl`~=2FsnpY69>yzUP+Gab)rasg8X~_Vs1=-F=x0Q3a9L>iY~#pHc?-Czai_ABI|@2 zAc_Dg)N;uI3?nXjpX1+eAOmqMuceg)QNVEz43CWF85RKVi>k_%bbZdm-NOnTF&z*k zxsL8?9-dACfD6&Ug%N6xcq!2IUP~kA68wf3x1W<0%h08X><8dLNva$+ZoBe+!xA~s zG}YphkDo;6|DR0F{_*S|%zpAEGkR%rDFPG$iU37`B0v$K2v7tl0u%v?07ZZz@R=g; z-b`&e&!NiH7b9(WCEC1SkR&0g3=cfFeK< zpa@U|C;}7#iU396^MSw(VDsASpJHb3&;BU-<@3Qg<&q*m5ugZA1SkR&0g3=cfFeK< zpa@U|C;}9LXChF*EB~LrhR^?R&c24@|Gfqg0L9slX%xU`Vn+!m0u%v?07ZZzKoOt_ zPy{Ff6ak6=Md0&{0FJFEaJ*VFO_tFt!Ge4Y63=KHuP`QW396#%2Co=5!)Se)%(5z9 zVvLdri5qy!VpNf18M!3#rl{zaELb-%p(#r|G9{By1k+?hQ_&brhJ+6a#|cUp%FmFl zV?xABf+%Yeqp`Zqh$Yctbk(pRbC06%qADXr*TeXKCB=XTP(+F6RYpWC0zyGzI7MR> zgXfT}s!{wup65kV)EP)nz(Zr8MT#UDjKPb7R?-Ew#BXh^z=wHpvm>Z0Y&PJZqCFeb_AB z66$!!z}c0E@^E4oj>zl+DLhUL$ee=+I!IoXX(?p-XxTbJgmVNbL`!ix5Hd>>H>Z+m z9ZbiMQdkj1@i>PGQIu=o)FMuU8Iwf_ayKN&VxrFKXaSk}2Un!#)+9_pJPeh zP+PNG^*$uoF-Q6fmq>7k&XD0CGIbG7V?!k2z(kN8C6ns{lM!WEnjatsWpX(I5hSI! z$bp5aj(tdA1ijVPGWc<7rbvX$G0;Kwp*fDM+ipQ>kW9uPNQ#un!DIm$&eKIyJCXs* zYH24(A!uZ%kx1-ebxnt$;BX?6bVC~+LJm|%P(dOsQY4$u4l;vCHmy}N2s$`<3PA>G zzR280mPD3J1M zw>z$fM;y~efd=A8Qe#jF$ZW@k^lX+*CRM?LMzq833_!Da$P_XS&g60ex)bC%nZ#kz z9Iujn0CYhUa{r{!Q|9&BnmRYhP$H*iT zxZ%MOp2@tBNK9hWfaE+A>((D z0eQqgPeYSQa3GHka0^-|ErxLx9tN`cI2kZ8-(zW8KAH9h`i7MMxX`-M0`VrtoRU?` zL)z@9D!~Lqq823#szU}i7+3_v(5OsSyrkHOxDH6SZAWX7h~RubWamXyv@}w$gD8(} zAkfu2gGPG{bmQ5}^cI#Xu`cDLtiICy8=AD@Csl(U- zlQAT**bq{)3ADY=o@S8~8n#S`xuvZIot7I;9=6l69g-Ymk(_GUh5;!e2{y4gNmSi7 zL;jdftJknInhK`VEOF1vnx`2}-v&#umthRImrx&y1Neu~AD5!uBB3FWA5*!2t>eoq+}CkEn5MZ4ktcK1R=C{V+o2 zI-od76#*GfVGd3=-_fJC6Ws^qR0c{1`2nNhq@tNDsvrxQicxUtdMBhogp-OApg54j z58|v5wT|Ed>0OcTx+itNX^&=xBznc^wMS6qB~r_}o{7M6VIUVKQ7_i_1jMd~r&xkU zz_TMUNL&Ed17nR88aXTY9mqVYfyig9i5!d@tT4p zfZVSrlhswsE(AMv#xBWd5%`{gG^H@8GJS+~C?PyacFd9NzavRvvHEI|ra8_?d%-;3 zuz}To$Ax5rB<9dVfEEd9he78VJhM=F2$W2_ZIdDk#y&ZWVX+b6D1wY}9AjW9)Aa)*n@ge&(~2OC4#r4kUIpU| z47@aN9%>-c7D*PyJTaYnsM~{nCWQgAl4nc<4>BYRg365FlGS8qkTo4s6;t7^_W=Exad26d97@d5Y{r z2`o2Kru{~{1x*mhl<*@XKT4f_v8xxZugMx6*l!;(h%@%T5BqS)gq1>|!BQ$?6~I)BR0! zNf6r^#G&9U;@y@VAjS%Ok!TmjoR|xNVN8g1BiW0v>?NTOz_7-8l-$>3vy&_Ai}!Ih zaT3o`)+B|qf}Rn8&Lr5w%8=}yX#F3MrQxkJg1LDA$;_J6FsO*}4zvBh@O^TR5j0|k z${ih@8Py7hLynfIF0=XxJk1VZPn?7m6l+G|<2`zxx#ai$ z?Rj9S5_MoDPsnWa!emP)Zu(fj;u(8_Jpkfz8QSpjxam8=nrFwdIetC|nIrUW)Tql; z>aa)0yce9qiBQIJ$8$Ykm)2uY3C!Th?3mrP4+lbI5M_l1-l|X7FadGSSoOjAh?*F+ zbH;N_G_7}X;yQ>FxRECHJevtn8!stIB>_T1Dhb%*!jljoTO-DP-}Zu@*2-FbFrcvQ zl*aN@3yfyNC68^X|Ju~gP2DNLKO>(X{`$4sHFtXR!Qb*()?hgq zAx5-x4gTRcMiX>!cu-l6D~T);H95?Ha8@fnI%pg(><9;@*sE+U9CcQYA2oM&O}@K3 zSYP)VJIjywttEB4WaT6qtXWwVJd6o?iq2OSo-iCE<>v0jx z*dD6eCait)Fa#J!2Lj<6rlPZ~j&w$`3}}ZaB1YBJ5;#tjkcK4ixX{DdKCB}?bDn=( z#F_=_BUyox@pD+7hG&pK7F87T88MigW6ni76Lv2&*g|;num~;<8f@Yg=bc`w^|0u6 z@L?DzpLq*TfY8ID4LcohItCpFH1RW-sESYtEcV0IGgdwG#cC53l3EI|En0wWk^(Fh z`>@St7DBA%fYI<>T-Ot*JVMa$Q0OW#Uwl>%U@?sV3ACW+L#?1=FCN38M0o!i zNUkghP+$iRIPp0+ZYH4EYoU6KeAEyaIGzSe;H1IuN=$aq2MKJz=Hm;XlfS_BN{ZAR zY>DRYGUyUKxwgPGE6#)Ig?q{A^uRa)ZbSZ{)qa%2LDIwa~UMulGtSUiQRg3||k- zL$z9|sXK?(V$C@*K%v*{Gq3D4blS~9w zlc?rs6kSRFvCu}&A-FroHSm0AUE8hL(q2>0&%53Ipj}>F5Kb2dC!OY*;T~?5cD6ef zayQR>e0-E`;7Px{)%Tli` z2TK>_URtEDJ1h3;j$l|zht6hOJREp-b#rx@Usw2&c4|r+o4Y%bx+<@`d(m+BaCZcQ zIvn4^4ri>sVSqV9z*dD$V{>3`UmrXFN)oFGkp%?j zi7li#gz1xq6X>XD)V&1LC*}c)ByAq7$&vx{qPEOJA+Dtm**H23Ot5EczYtEe1u%yK z^nBJ#3QcH2G=t#cCRj6KLX$FPuGFV^j7&6UmJh1SyITvD`qDym0VuOJ3}AYhu>oJF zx|NMT4E~wvh-bG&QGQhu4Nv$>gc^LNYFhhjS`abNO5u z8&YGZO8gxZkSaDbpC`f%Bp4 z{0Wbf88BQCK5PAdJz4+jR>?w&%rmmVNsOqLbVkz@xSn9Bl2R(EvY{zgUiOV&|HJJD zzVyuXe;$TlJg2lJUH?PxN7LUk*8iy<00^HsW774153fyG)s%S?uG;VlSAzW|R%cXJ zQ5j8_SWXdi*m1)9k;GIzHqZJ$ z&oxTC{+~(K|9NVDN$Ywmibj}99`%j#J> z>U90Tw!C3)?C?w2wx#R;Pk;TNYvl`lk$3$+ee3T|!GH8a5ugZA1SkR&0gAw@g~0Fb z--wo>fAa%YvLqz1DkCF}1xGv$LKNw+)u^$uZV7^JAT@NvyV8;6*kfoonl7o_CCktp zoP0r+q2Om1`q~sCZ=@GEjZTC5<6n)!1p?cc0}oMPC3_NGyxf}(wX{KT9$c5f(!A`% zx~Rd{cutQ6o$jJ%eDRX?RJ5!Avb^Z9^W(+I`b5XMi+!I9VtTr7d#;0Hs!;#7M#h1U z`nToxZ<~0!4-o=IJ&f9i6o}e~IF(Tam1B6`U=10mlC0rN0MA<87xTmuEJptS*SDtb z-iLqahax}`pa?uS0^i@icBkfUx~}{C_wt;tg$_f$et4*@uN*BrT3b9SJBO-UIoe-d zD6NQ#e%JN~2ZCHVLOVLgc`IYRWfh2|sz%F{@cfMnwijyF(IBWE@4KbFZB^YlcUQK9 zqm@N&wYy?<%iCzvIA3e;??hLY`sYXPw$VP7l)=gl=N@d8I%wy3y?U^+wkNjzjYp!! z_xH`+ttG(-Sw1zk#7;@5ZZ*!f+Q;X|%QaouZI%q*EmaJ)=XCjQXWw2v>U2-I&+BcL zT!)N!$KUQC9CPd9VorRoDe*Ha=8*(2NHM+N(;;*W4&plDW>TuJ;gytMOv)M~F0*_A zy<@`l*6<<+aUH218tN&><%O$=Dj*uIZ?Ttf_NvW%G4HuUd@-qRLLg6kEgs6keTy%v z!>{3wZ^5-XxVr>i*zL~bt`_0`=0p5Ky#<%z;es5;!ZOu|VsL?l#EA_@jLlJ+B;Yu$ z6H5bLaNGkht@4z{mss|)(kzHi3b$X(g5NW6=pgJ?Jb(+A*z+20Xv69up}?)eYv?PA ze!}j=aUzxCJCwlYMP1|X`@y9QF5c(?>bU>H#XZX^68GfE_xGoViapQ8iBL=S_a1eY z%O|Mo2{ozr$k$3I?foWqyuZI~txD$>zhYW@$k?>o=e@CtT^3pOg%rD^soH^aSUXZr z54S4bskmz$8B2pDNn2m6cw1`UJ1HIP$^6=pXn$!Gd#=5hT#7x9B!E2SQtYX|hF7R! z2W^+WXZoos_FPw~K_Xs~VwXX$;~2yfRq>S+yDZA3OQ(rMQ|40a5Urn+F5@jxr7uyj zlcJxnJ3+BSb7ej4am)P!Q^#8T-=xc;BH_V(1>_3K^uK|d6M1cAT3 zaU-7d|M|3%Bi%(ffqETP(xjsvAMClkCD&c#*N=8qEl<$;J6m0$(_3rTHhVQ)HX6>@ zIbTsFHFkXr3v_mm)a?^((RR99Xxl1nt?iz#mfI>qjq<^Pv;N4l5B50zyf*OL;R|lV zQ8m3L2J+r%b&b=G%x+m(bKisXDOI~TW(q3FQ-Kx1#2}-Ej&o1VY#SaX{s~w*tFD4hTK^MY7j1!&2z-{CX zn9X1$$Ax0IOBmu2(fHFEP|w6EOdK}?7Zu%ijbGCs1Q^)eb>9mEO1xkNxK4R+qSY8O4U`?UDAr?ns3rMXT9SDW zFJ6u%iMyJ|A$wkmB?+a;5RLGX(h}1wk0r^ok|JMf?iwsN7c3NnX^+`UEJv?`$naG zV78GgZX6wzJ)2Xv%Izh-RIxV3D|Sv4#I#~Jklj66bJXL*GxJepr?GahrBr=w>!7~@ zVTU`9%B7muF70kSYCH<{^k-@=TsDE{bGOJ&^XWAr-d+2YoGpgtN)lHi30{RYau{JE zPnUz>0JVD`}XHdl{to9+RC zkjA$mw~n)G$`GF`EEBFSlBp$L=q~a__uzq5{Dc5}kt_~Z*}2^GNhJaTJ;qx<))TNd zk6OF!%jnw}?zyCj|fN}Tp^Xhcty@%e3BDYz;oLCwdtC%Z!(P8d&cWK`^y3bDvd$jJ#= z1e$Phl`mmm8_hGCc0W$*P*PY9iK@<+iU8MNB*A2KPSqGBnQ-$())gt-pEH_0=L9CQ zdRqer(x~_F&$vzD*NkoAJRd{S7S47z|E=P=&{r0E+VT3Hvg@pu-6h`GYIiI8rr4;O z2Rr@cV;NZ+hEpyZr)TTE2Ba`6e*AoF`f+{)X#aOVx`E~S?cXr+Y`0>0zS<4m__7`; z$j`dLcWF<8;D84&9v2wd7nDp;c8MI%S&*VL439rfHX+XfB*&=Z^sOK!VeOwAk+L>+ zjt|QA{;IZbZVRWzT3Onxl^YecH|XuHJz7~TpOoBPxvYpQwHFg95hUt+o7ZCvna4Am zYRK4!hH6MW_q>uCawgG`IU=c24M{cRc-tVFbR!LUE!L2ET%4(fq#E*7w;`uv4VmW> z2i1^NLq1DGhVlRJ{z>Qtm|@hxxZf9tMTjFv1Dv4 zY}Fc#Zgu^z!r8%jMP4;d506$}>I^_V<%wc9K0mF<2oSYF+6mTE_*O{Huf zS2c5wX7_8thFc$bK5V>!*o3<(zV58pt2=^WEgd?WZSip6+11U}<^2BtL*LrDBsx}0 z^Fb{Y^XydSiT|G`WLxnGHgj;rNgT2T$J`7v-Qc-k5a6cA%05n}9jV<;tJkodIm7i( z9nY}-Ttwy1UCg{VJ6&TNBdf|%*rW?k-1DA9IVglg}br^TMDCl{PU$|fOdgy+^ zwhFo%G^76V3z~=GZ{}cRgatSt-%#3B>?ElB@*sUqWM1V6(ic;PW0kOPM=QX!5<&WK zlpqK$n2ik>8JO`n$0h=Vq^^e#qi8=*K8BLwQ8^~MUs{c9xxwa-QH`Xl|FWqO=%X*@ z0Np|jEkJ%DXrgFTW=7)2iI{6GTk}z9;zQ8OIS_Ubc(}g*q0K&i&I4i;C)$XNkFVT@w+%x z7UYQ$A$=4Cw$msiqamK$VZi4|`Jk$hBw-StA&*4SaLW_teSl|qix(ybMGuQFToifH z0{EtZ<_lczN+w4)yQ~t_nJ<9;%vB4|s**CXN0T^{>DFLFq;Ma~zn`xHj+OZJ;KCbaF@Xq_6eW6F~Y?7^_NwG`vhBod8C(zXrJuFxb~C#1j^6ZClG2*nHW5E zTm{)a$q5(KqfS&lm(`OOCvXdzG|C}(WnoywOHAx?5%L^r4!eAE0whqEN*o^@h!bwW zY8bmM4`WdnAOD}1L{X*j|30_JJd|;Y07ZZzKoNKW1b+DIx0CpPdA8lJenEKbq|7ON zl5LH25>9gNMFF*Qcr1ejaP?!*2}kB6D^N1I?;KYIyJ^o1AokKu*sHM9lY}5CSgsdt z{lF@yZ<~-=FKkC<4=6bSPcleNFpE+>01b@+nS8LWN(oW=Wc@0EAfcAYC+kr|Uj1|* zdAAv^E|RG&qA&T$d?bsB(1X}q+#0|^5%D?8#glleeq`icP+Ww3iaI6F@TKwpK7{RB z&(T_Ngn?T7TC0buH~|yhyiG}JYBMu^ycr5`M>9@f6Mzhl*E#JUFCF*YRnw~-`aRjI z7ZfJ&|0iHCx%oKYxbE} zb{e|x*Y=OjPnz0kv!iW`Ycec$i!Njuh1()dqwWu2hov3L4ze_eu9Orxup4Le{J&b; zY^cjpuPp~lDctL}W3=k=A^1x_|A!758~-ntlFbwUPvCg9WST6aSpwV;Vd2`7#_N!P2#N zA4z))!CGa{4LW-ZcJFlO_^`+M8(VvpSLL^*?Lm((ul9vbb^Jz|%$9_7a=&1Ep=KQo zg6i?UTiV-J)tz&9Wji=pS>#r`D^|C>jW&(*wf6o_beZ4jaZ9}JXh*Bs&d#~H>{!9( z{`z^K?aEtyvr=0wW&*?3v4FhkeKDYzNiT zqyE;~rXg(@yMv9ac<5?*qAvnIV^4N(b9pv!GOtc_ zJiw(T2kQAsQ+Q>fm(Qnj{*UoU)EsnF!tSK4YZ-!-L^=_OpLPBZ8{O)J{C}(}Lx|M> E57Zzd&;S4c literal 135168 zcmeHwTZ|mZnpU^l;~9G{ePB5ZX9t#?nlp^8an*EPyQ?{?-M+a!c6Yn4zRWqJI$4=n zRoPvYSFKFoH;k#N zsZZel58;3Q^A`ManSX$Pr-k1~{rbsNA^7|6&iyKn!aYrl2%@Y?oUe{}UPkE%j{uMyA)XaqC@8Uc-fMqn%ghwr`l{s$jS zKm9Z^n_b83x@PEv?4Q|{-L<9tHDiBid1K9BUmDl18Sq0s8T)Gw_l@n{&o`HL9~pPo z9^E9*j?M1CX}C6hymfEi*gDwQxOO8yd*%J552mMF&vs73$6c_a8s;GK$>#>Ezfokr zeCMh4&YSOl{PFZpuM*lE@7N7|ujhDCDE|JgM5XwIkt%%Eh=zSfCX13Rmg$&b=e#nu z8JSsS1T2i}Vx`#dO+)+!o4IlA#{9ci-oN|tC{TnK!bVa2^xo5}Z@>9|K0p0*jle@d z4;!`Gab*k(Rz=yY0Yn zSY!O-wJb{b!?A>Y& z=&U6E`c_sy2xHJGVs!@IQ9KO<&Y*bG$E5u7#=T{O6VIOdy?)??q3e5;X+PiEUt3?> zWmBp8)|K}^|LC0lVcl`x@Wa)o!#4=e{Pbheuf7w6ZWuXsgLX^(d`0S^`ZRN>Qr#pq zH%84V`ZRRU93$=^lNXHZun#zSGWd>`P{?ZZoX8}oDlGA9SFXIjmmdY0QVUEr^_w@J zc4r8a|8!Pl3f>%J4F2)51fBBmB|(mY&ncG0*|i(LdU=8=#9xltjDPXQ(|4w+c)Cl_ zG6SpQ9y{^qsGl!Mc&Sf`T0ylV{ainChDPL^MxtCWZW^Rt^Z}n#H-7EXxY1YJ10|%; zI|Eu2{rRbP;r~ms|8#2h-@;$|MLDYyKQd*pz`S5^0bM3iOt2#et*Yf50`AWW0w99#Ou5RV$N_Ml_ zbXp75=KMA2)uq}0GBx|Z;4l575zq)|1T+E~0gZr0KqH_L&VD2JM-aH{)KD50lk`;{h!#k4`=^m_J7k~U(K#+sx<-{0gZr0KqH_L&!#!6?YRZVbeePZW~=%J7PO14vTar7@}*J> z3Yz6wzPZq}^X2MXz1Aw3m6}t&j0LO3lG$1?ojl~_L%~|9o?j@poP4=ip0~>jm7+O6 zKSP?X)N5s{)XKM74y5(l3#GhSorls@%Yyv=def@bFMTp|aMdx(k6rw7Xq9<~pccFGU!%3xz_>5KmQ;e*c3u3dV(JG^rzXz#2&C~d9QxBJ15 zT`zJ1&+ImOzU_3s%8}zm@Mw9uXomgEn40BIz=0YcM;TWJZGs5#v`hnkayV%TO=zCB(e-A4a&cuO_@GhKF!`_BB zzRIzx{pO)#Mfe3!Kj=E-OcAKD>zj5VMLjE%kWP=v7}eRx4BAd4&?z^1PJe)AH*COR zFMjvf0a#jQH*}snfv@q@6u{4RETA;(n5BHFT7&*4*(mV+B%4;?9ex^3qi6OJ1{^*7 zm-UG}&bxLf7VW~}FuBDa=FZB;3rhp@@Xp@Cu6JkIUoBZ%y~i8P%}RUUezpaN~|y@CAv!C$lL6UYwa=jGDft-Gi9T`#!x z=yv7kY*aoXb7*g0Zu&l)c;(5A%)SH^V)jlr7>0!N z){Ey%fsYHV!0$CU^Hcs!coR5h&;{-@qYkhG8P`{arCN7)4j(SN_t(w)_ICNiy1iW6 zJXmgT)Ca@C-t7msR+f*d{_gzpLgm)M*JG&)aMW*XUS<+<5+$Q0BoXaeLQ)m?QcB1f zCLt%Wg|vj!5;9{$E2uiEbg6{A#BIn4jAkt%wS;`xZOCaRAt&+ew1m_W@;nKt*Z*IN zV236`BcKt`2xtT}0vZ90fJQ(gpb^jrXav4F1jyR@FXHl8CSWv(NsY@#FlZsh`cniQF_t ze34uS3F)qwubaC`^R)+V7w$gA4CdCL+rfxsDhdV;J^%mrre^=m?CGy)m{jetf#BcKt`2xtT}0vZ90fJWdO zLSTL-oPLJb!ll|w_$h}@yg|cpA;uK^-Deeikxdza89GZw4z3Qt`;1zrJgbb=M9CC=|v%I z_=n?#Y^PAg>|w?KhxL)3bkqJ^KsvpOKx^4>STA0gZr0 zKqH_L&`Mp1$n*D>>uVyRoi~i9FXaqC@8Uc-f zMnEH=5zq)|1T+E~0gb@xgTN0if6t%p52KFno!R}t(n-fWC>_+-+&d4-{mRLXf707N zTCd%zb<9Mn zaDV5EHQy{&Zmn!KS66P|Uwu^XJ4O5GWKiGUy=y<3S-Sj@KTQvoh=zS9BsCoGSZjU1 zUpgJ!?cA?NC+pF0>m)qdY}Yq}y63ePR(dOE)x-Aj7fZcM%a=d#uhJU&-9Z~J3m|no z_WFHy>CAS5J58@uFAqZBT6dRsto`DHho!aR!TQeX&eH1eYNM8oQuVc77@o|9Y5-EzCm z9?5K2>bhpQ6j&Yi*nzthSYdWW1uIRG5DJ0QcEc!eov>iyOZYE+FN7CcgKn4Ces1}~ zi1y*G4mWI^1a4$DyFh2l?1s+GTsY|W{UCDeh6}kEAzoURe2;DQj}8uT15f|L*6!?$cFv;wzJKj!jz!~4uM@_Fbd6wNJq z&{=5J96*XmfY*v7zy9#697Ao>42}k&u?&eDk>h$!0B<`z)9nKOJ@V|=TftGc;QF~I z*ax_@B!bILu(rTyIf3I@4yilz2Z7}@0;lie!$E*GbfTyqE`Ih|+l@MdX2J4%pRL|r z+TC8tZwLONV@02lQ9z2JFYv;C$1%uPMxPci;8g=F4P5|hSjYhlDIny=?2Z}q&4M4a zY3&WE{AY4w@T$@B1JY+F za{{#?mO6KKXd{SOT(5!=bjNRAR1p9(gazaXBK9t<2hMq2Py+&O7vyxLE^sVAuvJ|U z(>+ENBxN+OzfJ}}@YBVFIxxaF#_5!@=f>wJesI+4`X>_zcPtA2|1U|!HRr!hod1Av z!$n0s=c362IO#V8fv%EqL2l=Dt1_K`e{L*CDX4k;b>#7jiFnQu&zZxYNt$z72!Dr< zyAZ=F$@2tSW|mx*VQ zqzg)CtX;G;@N>2W&1e=1Xr6wZcpBs)`2T-~*a?#@>mw_O(e4+()z2iXRi4I3-gr?h zme5@Nx^Oi_`YtM{l~~_JIC{*hOiT;(Tu@3w-C`u0k%ixeFj%R?mDF?1)2|UvQ<*sF zlH?e+JkOFI^J>D;&m^I<`I$#P&JotBVw$^O7w&%WwIkFQk=UG(WdD+}wo1wcB{r{H zw5|DfV}-U-Nb~mV#@iPY-<%y}9z9QTXTPOz_A^OtUNi7D%m^{RkXhIy-4=60`3++{ z|Msg~JMahnM&w4AtBX0YknxBKx~|vqNj9hFL|A{L9hm)21M*JcShWZeh}BhksT^P- zC5vKVIe~GDR!dVBL&(|3v zB?-GeOWPG&9+9<+NuUZD(x^x&ygAy3FUMr^%DtU5t(T4TI`R+XHFNeEXDdk!6`%{4 zS)36m&nj0K(Voctq1AybZe&W+wq48+H@gC9JZF6pj+G~bH|RB;py9U~9uQ6Dh*AkD zVE8SA6fj(xipVC9K_(ea1U4aaTLbX_zl)U`_gcmlsgzD}NUwGw*}M%o>Cm;LVaINd zZX`Vpg1wHednEp;O%t;P; zgYZCuj$4CEF;xVmi=HosFzFl;sb843;&IHnFgEkOLrPz2*&RTR1^Fjd zBn8*Ax&s^b&~iSDjlOge5t%BihDj?bWRaUa?Z0n3E#lt@uUybxnUJ$(L#9urgR~Pm z%a+0YyWDUP+Y#*t_B|OKV+%J`$5T2nSy+33CMYSkQ+2`y={q2!(0W1+0E_7ugqG`N zP_aoG7)gdOzFDo~W7Y=tJOpLxh>hIB=hWB|au^a{Yicio$Xe3+$us znxPM{u%HH7=AaA5J&=Tn?90@O${uDdP6JO+V{2aepxqS~P00gNFl)B_8PB0FInHY$SHPS`oC zmh29*!%yT+5?+Wyjxz5e-;n7tN^vDrq}3L3H&&1j`;O(dT+854rIizDNa!6BwK;N- zg_%w?QKLEV>}-X^A5Es0Xg#lN02UYu$Ce3z*Vm0f?pP*6F7U`pt)vuc1qRlt$}zO6 zBIe33psKo=i!}(HB1GH7cENVCK%2b^x_Yj0#uUJK8;#0+!H^Qq>Owu1a40xfw6v(% zDeWUIs$?+|^<2g}>}~(V=$YP-od|;=L>^p;*EiCwZqe0SScye}D3}w1^~I}c#4Tr= z1}T>`9v5xKeVns>`0x_H|zY7=M-GoqWuZec1 z*y1mqEz2!4=E;nGnlt3bpKYCy_0qwaLu?0rzpnpSQMsMj2~&o3!t4i zK1#F7sv;qNa&)3Nk?H%dQs9QK&O=uQT@%ku6bxk+8GX@9eyr{1vfju00ZejkMglK8 zX*vQQdp^wptkJf$0xXXQ*-dy$h!V54cAh0hMA|^P;{JMjCqp{v)pvhCvKSBrpilNg9h(Fydwo zlSttJ&SeS(_D3c&87;IWLR#8j$kEUk32}qv1%V+54`KNIx=*YTkjq2J%Z8H2Hp ztR9ayYPRQPY@a`gDB}@jlwkv_;M39}pQNOE90AM+Ca*Jali(c(j4~vdS%Oh)27x(5 zW0sMTgov{tNLj?RS|)R#9+``gd|IU;JV;Tm(8y}b7%0ZhWcrUXK9u3kwmni43J++N z=F-@igB<6dj2f>ve-z6Q)1HhIPk2_S+JXCi3^ys@E0K@1c?tZYshPD4kK8>#`c+8L5zHLLNVUJd4;sfta3BQ2Zdn* zWboSdrFj+iQjnwBT$X8{a&0)$`ck9}woOJTyNwhl%fx1~G95Fl+17vXnAQ2SMdM;9ImwF_OKUioGkQD11|75Rcn3lQVlc#U(OwDAg=8n2=8DjQyj9z#!tYFRY(N4YeL zG9`w-7}G#Ei`2rLt#m2@Z9_24GYtgUGeKF3&9#PpbesSdxr;P4xaRQB>x<`ULe84RK0F}ao~=lHrQNy%WR(hwdp zzD7#Lr0bZX1vzrjk}n003uY+X`(oB-VO*q@w-{WuNSUl!ewi#+5)C8I#PoU`)Qn@n z7h2@qzvMI=x;}`Jop_rSJ5F^UUL+eYo$zzm8cvq6$7LEM_B$4pjseS97>?KcpJ82? z)D5YtaNn>v(Cc{en6Xb5hoR&0sbkrUpITQH`c66x^`zIO*&%4KCf)MkFMf7FysZc! z7YX|u@*NDUE@RbI8lbQ6zF#=MV=*V4vxYSAePPOjc<~yVq=`uCdbj|Cz&%1g#W7BC z(9N7s@)_7b;662J2;+eC`e?Y;*rvHx2qD=4i&V#nT#FC5X{Vsc@Wth2%lpQoikZR6ulz-UxkOONcHze}U~2W>-mf+Ht_w5ZCWLO=;Z1|4EZ&3x}XV$<%QV zIu2MJjOWZ0=# zq%(aM_bwJf&!EdlRR(A$=w2M|p;SUYIk0Qwig;DDtGR;|TYL?80mxKQrvYa0|EC~L zcEk{5NFjib5=7`TVa|^l2v!WoM35Ok2~K79@G~oLOvn&ron=A>M=Z2CbE$T!A#pJWq@qB-cj6D8N}UK)+*{C z!6Ye?#RVw3(_^zl&RruSAD*YGjz@(tRm|)$bM9lcG1D=@BWAUavb|C@%Kc3UquM4# z8Xx8jvq{hMRQMibGB3i0J0UaS1WCiox)S&yoAyj)B2~my_?X!u^;XbSIq6nqDIa{V z1NtE=I~7MslxnNOYw^$|LkF^-@T8iq8JD=AB%|k>fE6bx6>3I6?k4J|>)-?S%;3oO zPdq7y9$+JSiVu|pW7dErQzSKCmn3X3%N#BwCwzlqimCVA5FiO2&f3jn((Xl%qX=|(as_&nGL zO(Hr7=gkNi%U9G16gy9R9uOqbWF`8HIMd1A4EM!3gNoZOx`U}ld=}xrUBn?2`z#K4 zEIhJ#gTQk!ra&P*CU&G(isrdiqp7JlQ-xSy0SI5*UU;O5-jmUIDfpYxK%0-tmsz!1pj9 zLobB@=7QK131YH*Jm5;=AFv?7TJi=x6bl;+tPUI|klZ3MGNvVj7(_Ld9hi>~aWY&U zH3Azo1n-j`*aeXmXgr=-sF~=(@~D87IMXiDJa2-8G_h4^R+J4X6jex4RDPKVBD&U$ zwC@hQ2+kT45YCWF;QoI=7ZTn6@Wh3~_#+&iVeJp;oEvLID7iprDmD@JMQ@WOYm`3m03c zJ*;N-5PWv2`bq(?$RHZZHTuIgiRel%%mM9L(BTW2>4+p=N7@>y&SwHJyR4V=ng z7YoIjj5ZN#3aM47ssLBR%zxUh*tKT0IcJs&HZV(Nu2d|G-6n76**1C{usjtzB6H3a z6ilU;e6Q)7fsIGtPyQ0WP$rot(0fMlT+=e=ta(CjxmYSn!$g|qIj_(|r|Zj{i^P(e zigTZ~>ouo5Z?{q%<`u9pnSgNq2ISSU^BzRftAz>@U;MOf&lijHEhh%MICMpM+{;IEmjjAsjO3 z@0l%m(!wlM>x}wcKkB;8yz2q!a&a8Tn%L={WkEnpB6AKzmeh0)Zm9wK7$2%L8s@50 zAxM7=b|hWFHYF9&gA?Tk3Srn^xlpfThMlrh6PbllV__Q;eSo;A1L{a_wW{|_VKzdEKULitQ1$`uqiRXk--VBGH1;?|CC5tev7o_^= z6+5R=sj{lJ@U9-PW~4&JR2TfnfrT@SJS98@94e2l$4WKAj@y%p0$<591diuQ4Ogiq z_tX#Iv?vD-@^ED2z!H#w^8++3qz21qeMso~KzQJSrzDk|SBI45tf|rlObBTZ`ruZX za!n`%B*}wie5>VlK@H1Aaik>L`X=0o)pKE+y@)T^MQX7il*b%-O_FAKxEG%NDOCiM zUojM*7%YQIg-YrS=r?6^qn0Y*oI*?L*eJ{o68s2u%6H&=HK{oiaFXQjyRw!7$W_qY ze!lBkFnyIlOv&{AQF{=%UAUY#^t<4tkaaf^T9DTYd-UOSG@O36s|*$vRED8DXt(jM zw`3|{ee#IM#djAjApU@8_K}sUDmsay?*fz;S6kSBAM~fSMVy;k;r$vacOFcsQH7`*L)>Gia)g3B2 zio)x|xx`?gn{sssklZ8`>zQp==6J9HW#$^1J-H%auevgIhOiRmpGedf(Fm5?8(JVR z!)OQ@*&@&|!Jtv8qeH6?JdiTlOG+u*1Rqc)zXSq8X_b~jSr~zElCeY4BMVhZ_T=KBSBkbWF*GP>FEC^?%e;vY;pBKsLylJEa?Gefd+HUNe={T23h; zV!!1f{nJU!VuV^@NHq5HHfa2m?T|u;6h#%C@A_@Lu27yhpbF%96Ujo6c<|*y-wz`N z&qqs}8&YMcqogwZ;h6&#j$f9F;Rq_-IqN-^ihysLVu>^8Z6GUPVCR@<5%H?uB4-(UbG4y4W0T$Uh!Q%CW+L+=J zj*xEFk?;ZIRT0=&1a*ofTZzcw64pWA4pmPJWsy>903ok#Atkq=X=dD&d-|ES8AH}yMAV^VEq4FTs zphl(6cCQOvD60@EQz~?2z5z~E?19j8TdkCXpcpCW;5aroSL9kh#au~QpO8b;4_$~S zNU%U(#mT$3v$530la*!r2#w2<#MN*kxJe7VWI zuOU+bE@=S4Db=MbNlZs}tkf1F~F*2#7PDJ1*pTK>+WvNxHPMQN?iwvzW zEJ>vx(jW`r$XARnXiu5$SVWFPBg;r42t=A^Dumb6XioIlmOal(p{7Iwqrk+44Mm`$ zohlOaQ@!KDdFG{moEnheTZ=OksD4ZyTAZ&pRyw5S06~Sm zvs3H1DYHri{F@S(VPC>JYsLPc>3EX}j3W)FEbL`Jpu*$Qo{W@Ry12~PXzQVbSavfILvo~l6?a*|Jcn=1LN$6B+WG7nxe;54! znVI_3wLgCA$MBE-(FkY+CPCo0zd!Rv-v9aMxJw%{L%en)9EMAx5`%r^ zd#pi_9uU1EPCH*`fUh8xdw_mC+|1&@6%z1G4Sd96C10AGD`0r0aJo=y)MUT(qoT35 zxovFkuB~lu?>AN-Z7pqnzGB=8T;mSxzbe&?(%fRPv{1=2H7-DCBV ztfx0k541Yoic8l&fdb*))q8t%gDFXCh&S)zp1)w=c`&H(UExRGkGzkJ{d=qT7GXmH z-IE5D89uCZ1jfC+PeS85?8x%ELt}YwHM{||%N9g>T$8N0i~;6aVlB}!v6f5^ewOtR zmog|%5=%wYmG>xH0Tz3NX@O*>h!XoGVWzAm_|sI?+ia~XtIC)e$JWS7rE1<~tein> z%sOK$W~EXU-yda43A~+EG09VmAdU~qL?XJIY(iuZs!j}HW6oj@xRK8qBgU5}3yKZg z3ynQ?P*wu7V7HxOb}@sodWbm_`R{?Gn(gQgA< zKbFYI;~i_Q@ApfmgS(yk_2^_h8g8A0N1N^XMo{;>*1}3}<*a(xKK^2<2k4KMkwV3Q zMk;w!+IB+E=R||8ff9k3g0PD2(0S4zrwBAy6)35YTz+1{$a1g+L)=78B%U;|(iSd# zLvq3YR&PiK?f$2J`^xkipZGujB{82ZBy9QE>-XKIGsq6P)AU;P@&Nqjb$5Bk+AltM zSXwI{tnaMuEUgYtZm-;#fGufiA6oiSQ{>WcrrpQ`odYO@H?uPyU#1*D9ZK-Tjf1kM z8|ytmC5<)NCO4PlNdygz*+w9jMKe?q%INcxBA{yrZP<_c4F&-2x+WOD+oauRi2}%7 z6SOj^nAJOxen$U)`f_^;{^%c#fJQ(g@FEcSkJm0=`JVsuAAiy(v0!3popj8D(m{RA zz4M^lubk}oC%x^X_1djk$K37}_uB5Jz0__O?^r?S_E<}c)(IvS{UwPZ=U^EDmUWE3 z!ezS+*GA#yc-zc_2j6z!331(TlK4ikf+3CaO#?z4rTN9m+zsQ0>nke;)B(v2Ut$As z1o?+?6vOl&H5pfEGq-Li3mqn;hvR|>)D`=zIb17}79Qc^EG4}H7BJ+c8nfR|Ws^b{ zj#5w1_Pr9Mk+hWM?^HIOnUu`K^||Slr4lYVON&Jr35{x^su@}?M26(mA6ST%*WS%A zihQ_LQC`Zz6cK6hnWUxXyTL%h4v13v97W3N8Kg5PYkSLhM2J$2n(BcUugVl5;SvH( z2R2~4aH=Nd*5mUY9=q0sOXlPpw^YJ-IvEXjO3HL$ zzFMB0IhH4J2n9lAcOoWZOC@lCZa`@+!GncB#wz>su$aUxEzlu&ZlOw?ym=8zVWx1# z24Lv6!p$wm0k~5s_9O{dZX3dfkd-3~V~|pcmc!^FiLr|_H>FftpnG8-N;89XQtF3f z$uCIkkP`2$$F?VxAUVZihDbYr9a=z|R0G?flhQ=mVaFA0Z)nLft^|stETzl1wvufJJ6xn{dU((xpq}T!cB!wBh$5>B zI6i=t&rVmvQIJ@~_1uhIhRDUzjDQ{ZAdX>IT3Koz3>>Ap{_w~NJjH~8oI1tWg!)pu zCP7KTAKH5`^nV%s|4UQ9KXvs(`0M5W`Cb3gn+N{%jlcKZdIk68mRk8%%c)ZX{$K(J3*ws!q27i^KCn~u%EN)KHAdDLg&5$=LcB@{WtHLI$O3levth$|V z&I85ug;vd8s8!8!m1KdM-GRdoa$#!@(4I^R)J)bQ7#~3~+7O)Y;dC3MInKNV<_1|^ z6ZS#D!fAZ!DP8^GH*Mz@*}-*iNGeo|`BiY(Fh9(Wo(Y!@nekTQ^Y_4-JBEp9Cuib- zPNO%W;i@e~5Gs!bu-cLX6APJ}b{VX!++w}PKjVI2D01OZg9`g_(gaE}Tzg?ho)BTX zNSi7^vRYumh8s>jyuyklfN~6QQ7A!;e}eTVcBBd3YYgvCjz8ebHVXnDLB_QMDMMa= z0IC$IB~U=G9kUEuddkiGT(Q~ASE|4^^@Y-WzO_)QH><_U{Je$J_8Is~SbiogQo?D_ z6TV-TTYO^~Z4cbzEH`6qDYk}QDpg)2 z%Ll7uvtblm(ZQpK2LnHR>@IZNh4Ai9&t0~I_1*P5D_h>ayIpl2?Hp~{16aM@%z&%k z+&EdPE%~Qs>&?^s)-8LmTy+mCQT@~%?r&K2`%d%pLGgCm**QcrSsF2zYgQKyUO{s~ z2%k~lcKx$@`*?ZfVC~2)owT&;rUQhnU4Sr6TMx!>L#+S`}ek>Qf5F-c1RH5GBs!PMZ# zma8@fARn{**R>s%4Au%elog_kZFagp>Oyk0JISKCnDtP}Fdi;60LZcVrTrx+soD$x&^YA|kA*RsV?TNPSNV%SgEj<-OV}DJ;RGtWv(fN_e#i704R|fq z6QlKJGR#TgSYbjl;}sz@Tr%|_K7$H`YKiTMl8(w0q87OtT&Abwe_h=NoxZTj6Pr#1 z3q7z-t2bF=YW4OqskaddYC^vhCnxOlR+ZzEC$F6Uf0_CJcCl5qt;$@!RBE;I6|-E+ zHy4_AzFeKF*IFeLT<$WlU!Ir$KS_fik_3UT{eNIP=IU$z9}f!}d$`~P0=`ZF?k|s<$KMeTUQM^v{y+Ee71y79wyTE{xM#zaXJ7&deamIPs^hIm|6W`YYI_;}+0!a4ybd2^=?bO(;51;N;F2v4Q|HaAqBnu|7xuNJ2iXWwyb;|PiO~KJfAl! z1~rV_&mL9zHx=_w1b-ErDv9R1b(I7-j*bQqg%&ZZG;Ofb&_893T$tiD0uVwdFFb1 ao8I20x3>{RrMI`eknL>~3D(i+ZT}zfsBfnL diff --git a/tests/test_api/test_data_types.py b/tests/test_api/test_data_types.py index b2aaf6477..15fcad2f7 100644 --- a/tests/test_api/test_data_types.py +++ b/tests/test_api/test_data_types.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI # Copyright (c) 2023, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -def test_data_types(api: KiaraAPI): +def test_data_types(api: BaseAPI): # test 'boolean' and 'table' are in the available data type names assert "boolean" in api.list_data_type_names() assert "table" in api.list_data_type_names() -def test_internal_data_types(api: KiaraAPI): +def test_internal_data_types(api: BaseAPI): assert not api.is_internal_data_type("table") assert not api.is_internal_data_type("boolean") @@ -21,7 +21,7 @@ def test_internal_data_types(api: KiaraAPI): assert api.is_internal_data_type("render_value_result") -def test_data_type_info(api: KiaraAPI): +def test_data_type_info(api: BaseAPI): infos = api.retrieve_data_types_info(filter="table") assert len(infos.item_infos) == 2 diff --git a/tests/test_api/test_misc.py b/tests/test_api/test_misc.py index a7bf04936..243d56c65 100644 --- a/tests/test_api/test_misc.py +++ b/tests/test_api/test_misc.py @@ -1,29 +1,29 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI # Copyright (c) 2023, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -def test_api_instance(api: KiaraAPI): +def test_api_instance(api: BaseAPI): assert api.context.id -def test_api_doc(api: KiaraAPI): +def test_api_doc(api: BaseAPI): assert "doc" in api.doc.keys() assert "Get the documentation" in api.doc["doc"] -def test_runtime_config(api: KiaraAPI): +def test_runtime_config(api: BaseAPI): rtc = api.get_runtime_config() assert "job_cache" in rtc.model_dump().keys() -def test_context_names(api: KiaraAPI): +def test_context_names(api: BaseAPI): # this is specific to the test setup api context, usually there are names in there, at least 'default' assert not api.list_context_names() diff --git a/tests/test_api/test_module_types.py b/tests/test_api/test_module_types.py index eedaf6350..6337855ff 100644 --- a/tests/test_api/test_module_types.py +++ b/tests/test_api/test_module_types.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI # Copyright (c) 2023, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -def test_module_types(api: KiaraAPI): +def test_module_types(api: BaseAPI): # test 'boolean' and 'table' are in the available data type names assert "logic.and" in api.list_module_type_names() assert "query.database" in api.list_module_type_names() -def test_module_type_info(api: KiaraAPI): +def test_module_type_info(api: BaseAPI): infos = api.retrieve_module_types_info(filter="query.databas") assert len(infos.item_infos) == 1 diff --git a/tests/test_api/test_operations.py b/tests/test_api/test_operations.py index 0cbcaf032..552cd7704 100644 --- a/tests/test_api/test_operations.py +++ b/tests/test_api/test_operations.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI # Copyright (c) 2023, Markus Binsteiner # # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -def test_operation_list(api: KiaraAPI): +def test_operation_list(api: BaseAPI): op_list = api.list_operation_ids() @@ -22,7 +22,7 @@ def test_operation_list(api: KiaraAPI): assert "query.database" in op_list -def test_get_operation(api: KiaraAPI): +def test_get_operation(api: BaseAPI): op = api.get_operation("query.database") assert "Execute a sql query against a (sqlite) database." in op.doc.full_doc diff --git a/tests/test_archives/test_archive_export.py b/tests/test_archives/test_archive_export.py index 8044f27ad..35909738d 100644 --- a/tests/test_archives/test_archive_export.py +++ b/tests/test_archives/test_archive_export.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import datetime import os import sqlite3 import sys @@ -9,7 +10,7 @@ import pytest -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI from kiara.models.values.value import ValueMapReadOnly, Value @@ -91,7 +92,7 @@ def check_tables_are_not_empty(archive_file: Union[str, Path], *table_names: str sys.platform == "win32", reason="Does not run on Windows for some reason, need to investigate", ) -def test_archive_export_values_no_alias(api: KiaraAPI): +def test_archive_export_values_no_alias(api: BaseAPI): result: ValueMapReadOnly = api.run_job( operation="logic.and", inputs={"a": True, "b": True} @@ -101,9 +102,10 @@ def test_archive_export_values_no_alias(api: KiaraAPI): temp_file_path = Path(temp_dir) / "export_test_no_alias.kiarchive" temp_file_path = temp_file_path.resolve() - print("temp_file_path", temp_file_path.as_posix()) - store_result = api.export_values(temp_file_path, result, alias_map=False) + store_result = api.export_values( + temp_file_path, result, alias_map=False, export_related_metadata=False + ) if not temp_file_path.is_file(): raise Exception(f"Export file {temp_file_path.as_posix()} was not created") @@ -115,7 +117,6 @@ def test_archive_export_values_no_alias(api: KiaraAPI): required_tables = [ "values_pedigree", - "environments", "values_metadata", "archive_metadata", "aliases", @@ -130,7 +131,6 @@ def test_archive_export_values_no_alias(api: KiaraAPI): check_tables_are_not_empty( temp_file_path, "values_pedigree", - "environments", "values_metadata", "archive_metadata", "values_data", @@ -144,7 +144,7 @@ def test_archive_export_values_no_alias(api: KiaraAPI): sys.platform == "win32", reason="Does not run on Windows for some reason, need to investigate", ) -def test_archive_export_values_alias(api: KiaraAPI): +def test_archive_export_values_alias(api: BaseAPI): result: ValueMapReadOnly = api.run_job( operation="logic.and", inputs={"a": True, "b": True} @@ -154,9 +154,10 @@ def test_archive_export_values_alias(api: KiaraAPI): temp_file_path = Path(temp_dir) / "export_test_alias.kiarchive" temp_file_path = temp_file_path.resolve() - print("temp_file_path", temp_file_path.as_posix()) - store_result = api.export_values(temp_file_path, result, alias_map=True) + store_result = api.export_values( + temp_file_path, result, alias_map=True, export_related_metadata=False + ) if not temp_file_path.is_file(): raise Exception(f"Export file {temp_file_path.name} was not created") @@ -168,7 +169,6 @@ def test_archive_export_values_alias(api: KiaraAPI): required_tables = [ "values_pedigree", - "environments", "values_metadata", "archive_metadata", "aliases", @@ -181,7 +181,6 @@ def test_archive_export_values_alias(api: KiaraAPI): check_tables_are_not_empty( temp_file_path, "values_pedigree", - "environments", "values_metadata", "archive_metadata", "values_data", @@ -192,7 +191,7 @@ def test_archive_export_values_alias(api: KiaraAPI): result = run_sql_query('SELECT * FROM "aliases";', temp_file_path) assert len(result) == 1 - assert len(result[0]) == 2 + assert result[0][0] == "y" assert uuid.UUID(result[0][1]) @@ -202,7 +201,7 @@ def test_archive_export_values_alias(api: KiaraAPI): sys.platform == "win32", reason="Does not run on Windows for some reason, need to investigate", ) -def test_archive_export_values_alias_multipe_values(api: KiaraAPI): +def test_archive_export_values_alias_multipe_values(api: BaseAPI): result_1: Value = api.run_job(operation="logic.and", inputs={"a": True, "b": True})[ "y" @@ -220,9 +219,10 @@ def test_archive_export_values_alias_multipe_values(api: KiaraAPI): temp_file_path = Path(temp_dir) / "export_test_alias_multiple_values.kiarchive" temp_file_path = temp_file_path.resolve() - print("temp_file_path", temp_file_path.as_posix()) - store_result = api.export_values(temp_file_path, results, alias_map=True) + store_result = api.export_values( + temp_file_path, results, alias_map=True, export_related_metadata=False + ) if not temp_file_path.is_file(): raise Exception(f"Export file {temp_file_path.name} was not created") @@ -235,7 +235,6 @@ def test_archive_export_values_alias_multipe_values(api: KiaraAPI): required_tables = [ "values_pedigree", - "environments", "values_metadata", "archive_metadata", "aliases", @@ -248,7 +247,6 @@ def test_archive_export_values_alias_multipe_values(api: KiaraAPI): check_tables_are_not_empty( temp_file_path, "values_pedigree", - "environments", "values_metadata", "archive_metadata", "values_data", @@ -259,10 +257,14 @@ def test_archive_export_values_alias_multipe_values(api: KiaraAPI): result = run_sql_query('SELECT * FROM "aliases";', temp_file_path) - assert len(result[0]) == 2 + print(result) + assert len(result) == 2 + assert len(result[0]) == 3 assert result[0][0] in ["result_1", "result_2"] assert uuid.UUID(result[0][1]) + datetime.datetime.fromisoformat(result[0][2]) - assert len(result[1]) == 2 + assert len(result[1]) == 3 assert result[1][0] in ["result_1", "result_2"] assert uuid.UUID(result[1][1]) + datetime.datetime.fromisoformat(result[1][2]) diff --git a/tests/test_archives/test_archive_import.py b/tests/test_archives/test_archive_import.py index b358c7f42..ad1838e35 100644 --- a/tests/test_archives/test_archive_import.py +++ b/tests/test_archives/test_archive_import.py @@ -3,13 +3,13 @@ import uuid from pathlib import Path -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) TEST_RESOURCES_FOLDER = os.path.join(ROOT_DIR, "tests", "resources") -def test_archive_import_values_no_alias(api: KiaraAPI): +def test_archive_import_values_no_alias(api: BaseAPI): resources_folder = Path(TEST_RESOURCES_FOLDER) @@ -19,15 +19,17 @@ def test_archive_import_values_no_alias(api: KiaraAPI): result = api.import_archive(archive_file, no_aliases=True) - assert len(result) == 6 - assert "512af8ae-f85f-4629-83fe-3b37d3841a77" in result.keys() + assert not result.errors - assert uuid.UUID("512af8ae-f85f-4629-83fe-3b37d3841a77") in api.list_all_value_ids() + assert len(result) == 8 + assert "af83495c-9fbf-4155-a9ce-29f1e8be4da9" in result.keys() + + assert uuid.UUID("af83495c-9fbf-4155-a9ce-29f1e8be4da9") in api.list_all_value_ids() assert ["export_test#y"] == api.list_alias_names() -def test_archive_import_values_with_alias(api: KiaraAPI): +def test_archive_import_values_with_alias(api: BaseAPI): resources_folder = Path(TEST_RESOURCES_FOLDER) @@ -37,9 +39,11 @@ def test_archive_import_values_with_alias(api: KiaraAPI): result = api.import_archive(archive_file, no_aliases=False) - assert len(result) == 6 - assert "512af8ae-f85f-4629-83fe-3b37d3841a77" in result.keys() + assert not result.errors + + assert len(result) == 8 + assert "af83495c-9fbf-4155-a9ce-29f1e8be4da9" in result.keys() - assert uuid.UUID("512af8ae-f85f-4629-83fe-3b37d3841a77") in api.list_all_value_ids() + assert uuid.UUID("af83495c-9fbf-4155-a9ce-29f1e8be4da9") in api.list_all_value_ids() - assert ["export_test#y", "y"] == api.list_alias_names() + assert {"y", "export_test#y"} == set(api.list_alias_names()) diff --git a/tests/test_included_data_types/test_string.py b/tests/test_included_data_types/test_string.py index 0c1c22c1d..8a0ec8a4b 100644 --- a/tests/test_included_data_types/test_string.py +++ b/tests/test_included_data_types/test_string.py @@ -1,16 +1,16 @@ # -*- coding: utf-8 -*- import pytest -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI -def test_pure_string(api: KiaraAPI): +def test_pure_string(api: BaseAPI): value = api.register_data("test_string", "string") assert value.data == "test_string" -def test_string_with_config(api: KiaraAPI): +def test_string_with_config(api: BaseAPI): config = { "type": "string", "type_config": {"allowed_strings": ["x", "y", "z", "test_string"]}, @@ -19,7 +19,7 @@ def test_string_with_config(api: KiaraAPI): assert value.data == "test_string" -def test_invalid_string(api: KiaraAPI): +def test_invalid_string(api: BaseAPI): with pytest.raises(ValueError): config = {"type": "string", "type_config": {"allowed_strings": ["x", "y", "z"]}} diff --git a/tests/test_pipelines/test_pipeline_configs.py b/tests/test_pipelines/test_pipeline_configs.py index f6c884203..ad03a65f6 100644 --- a/tests/test_pipelines/test_pipeline_configs.py +++ b/tests/test_pipelines/test_pipeline_configs.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI -def test_pipeline_default_config_simple(api: KiaraAPI): +def test_pipeline_default_config_simple(api: BaseAPI): pipeline_config = """ pipeline_name: test_pipeline @@ -24,7 +24,7 @@ def test_pipeline_default_config_simple(api: KiaraAPI): assert outputs_schema["step_1__y"].type == "boolean" -def test_pipeline_config_aliases(api: KiaraAPI): +def test_pipeline_config_aliases(api: BaseAPI): pipeline_config = """ pipeline_name: test_pipeline @@ -53,7 +53,7 @@ def test_pipeline_config_aliases(api: KiaraAPI): assert inputs_schema["d"].type == "boolean" -def test_pipeline_config_aliases_2(api: KiaraAPI): +def test_pipeline_config_aliases_2(api: BaseAPI): pipeline_config = """ pipeline_name: test_pipeline @@ -80,7 +80,7 @@ def test_pipeline_config_aliases_2(api: KiaraAPI): assert inputs_schema["b"].type == "boolean" -def test_pipeline_module_config(api: KiaraAPI): +def test_pipeline_module_config(api: BaseAPI): pipeline_config = """ pipeline_name: test_pipeline diff --git a/tests/test_rendering.py b/tests/test_rendering.py index 74a9c2531..c6f39f16d 100644 --- a/tests/test_rendering.py +++ b/tests/test_rendering.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from kiara.api import KiaraAPI +from kiara.interfaces.python_api.base_api import BaseAPI # Copyright (c) 2021, University of Luxembourg / DHARPA project # Copyright (c) 2021, Markus Binsteiner @@ -7,7 +7,7 @@ # Mozilla Public License, version 2.0 (see LICENSE or https://www.mozilla.org/en-US/MPL/2.0/) -def test_render_python_script(api: KiaraAPI): +def test_render_python_script(api: BaseAPI): render_config = {"inputs": {"a": True, "b": False}}