From ef7ea1b32e3bddd43e12f5cb6dfa7c9fa379ec71 Mon Sep 17 00:00:00 2001 From: lainme Date: Fri, 17 Jan 2025 16:03:22 +0800 Subject: [PATCH 01/13] fix running in bash environment on windows --- src/aiida/cmdline/commands/cmd_computer.py | 4 +++- src/aiida/manage/configuration/config.py | 4 +++- src/aiida/manage/profile_access.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/aiida/cmdline/commands/cmd_computer.py b/src/aiida/cmdline/commands/cmd_computer.py index 42feb36d25..d0630bbde7 100644 --- a/src/aiida/cmdline/commands/cmd_computer.py +++ b/src/aiida/cmdline/commands/cmd_computer.py @@ -136,12 +136,14 @@ def _computer_create_temp_file(transport, scheduler, authinfo, computer): transport.makedirs(workdir, ignore_existing=True) - with tempfile.NamedTemporaryFile(mode='w+') as tempf: + with tempfile.NamedTemporaryFile(mode='w+', delete=False) as tempf: fname = os.path.split(tempf.name)[1] remote_file_path = os.path.join(workdir, fname) tempf.write(file_content) tempf.flush() + tempf.close() transport.putfile(tempf.name, remote_file_path) + os.remove(tempf.name) if not transport.path_exists(remote_file_path): return False, f'failed to create the file `{remote_file_path}` on the remote' diff --git a/src/aiida/manage/configuration/config.py b/src/aiida/manage/configuration/config.py index fff83f4330..2bf3cc5cdb 100644 --- a/src/aiida/manage/configuration/config.py +++ b/src/aiida/manage/configuration/config.py @@ -21,6 +21,7 @@ import json import os import uuid +import shutil from pathlib import Path from typing import Any, Dict, List, Optional, Tuple @@ -780,7 +781,8 @@ def _atomic_write(self, filepath=None): os.umask(umask) handle.flush() - os.rename(handle.name, self.filepath) + handle.close() + shutil.move(handle.name, self.filepath) def filepaths(self, profile: Profile): """Return the filepaths used by a profile. diff --git a/src/aiida/manage/profile_access.py b/src/aiida/manage/profile_access.py index c65364af03..be87be3e05 100644 --- a/src/aiida/manage/profile_access.py +++ b/src/aiida/manage/profile_access.py @@ -11,6 +11,7 @@ import contextlib import os import typing +import shutil from pathlib import Path import psutil @@ -76,7 +77,7 @@ def request_access(self) -> None: # it will read the incomplete command, won't be able to correctly compare it with its running # process, and will conclude the record is old and clean it up. filepath_tmp.write_text(str(self.process.cmdline())) - os.rename(filepath_tmp, filepath_pid) + shutil.move(filepath_tmp, filepath_pid) # Check again in case a lock was created in the time between the first check and creating the # access record file. From bd0926c8b7fe91724f76e5b62aff42d97c5f132a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 Jan 2025 08:06:04 +0000 Subject: [PATCH 02/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/aiida/manage/configuration/config.py | 2 +- src/aiida/manage/profile_access.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiida/manage/configuration/config.py b/src/aiida/manage/configuration/config.py index 2bf3cc5cdb..e38a89ae24 100644 --- a/src/aiida/manage/configuration/config.py +++ b/src/aiida/manage/configuration/config.py @@ -20,8 +20,8 @@ import io import json import os -import uuid import shutil +import uuid from pathlib import Path from typing import Any, Dict, List, Optional, Tuple diff --git a/src/aiida/manage/profile_access.py b/src/aiida/manage/profile_access.py index be87be3e05..888253abd5 100644 --- a/src/aiida/manage/profile_access.py +++ b/src/aiida/manage/profile_access.py @@ -10,8 +10,8 @@ import contextlib import os -import typing import shutil +import typing from pathlib import Path import psutil From 400289c8fe1731be8ad74e8ff13e257bdb93df68 Mon Sep 17 00:00:00 2001 From: lainme Date: Mon, 20 Jan 2025 13:55:42 +0800 Subject: [PATCH 03/13] fix running external code on windows --- pyproject.toml | 3 +- src/aiida/orm/nodes/data/code/abstract.py | 2 +- .../orm/nodes/data/code/containerized.py | 2 +- src/aiida/orm/nodes/data/code/installed.py | 6 ++-- src/aiida/orm/nodes/data/code/legacy.py | 4 +-- src/aiida/orm/nodes/data/code/portable.py | 8 ++--- src/aiida/orm/nodes/data/folder.py | 8 ++--- src/aiida/orm/nodes/data/singlefile.py | 2 +- src/aiida/orm/nodes/repository.py | 8 ++--- src/aiida/repository/repository.py | 34 +++++++++---------- .../sqlite_zip/migrations/legacy_to_main.py | 8 ++--- src/aiida/transports/plugins/local.py | 8 ++++- src/aiida/transports/transport.py | 12 ++++++- tests/orm/data/code/test_abstract.py | 4 +-- tests/repository/test_repository.py | 4 +-- 15 files changed, 65 insertions(+), 48 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4ee43b8c01..bfa4898554 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,8 @@ dependencies = [ 'tqdm~=4.45', 'typing-extensions~=4.0;python_version<"3.10"', 'upf_to_json~=0.9.2', - 'wrapt~=1.11' + 'wrapt~=1.11', + 'chardet' ] description = 'AiiDA is a workflow manager for computational science with a strong focus on provenance, performance and extensibility.' dynamic = ['version'] # read from aiida/__init__.py diff --git a/src/aiida/orm/nodes/data/code/abstract.py b/src/aiida/orm/nodes/data/code/abstract.py index b72a51e262..82a1e3713e 100644 --- a/src/aiida/orm/nodes/data/code/abstract.py +++ b/src/aiida/orm/nodes/data/code/abstract.py @@ -159,7 +159,7 @@ def can_run_on_computer(self, computer: Computer) -> bool: """ @abc.abstractmethod - def get_executable(self) -> pathlib.PurePosixPath: + def get_executable(self) -> pathlib.PurePath: """Return the executable that the submission script should execute to run the code. :return: The executable to be called in the submission script. diff --git a/src/aiida/orm/nodes/data/code/containerized.py b/src/aiida/orm/nodes/data/code/containerized.py index b2a8a8c439..85fd539cfa 100644 --- a/src/aiida/orm/nodes/data/code/containerized.py +++ b/src/aiida/orm/nodes/data/code/containerized.py @@ -62,7 +62,7 @@ def __init__(self, engine_command: str, image_name: str, **kwargs): self.image_name = image_name @property - def filepath_executable(self) -> pathlib.PurePosixPath: + def filepath_executable(self) -> pathlib.PurePath: """Return the filepath of the executable that this code represents. .. note:: This is overridden from the base class since the path does not have to be absolute. diff --git a/src/aiida/orm/nodes/data/code/installed.py b/src/aiida/orm/nodes/data/code/installed.py index 7546c2acb8..1db39ffa56 100644 --- a/src/aiida/orm/nodes/data/code/installed.py +++ b/src/aiida/orm/nodes/data/code/installed.py @@ -170,7 +170,7 @@ def can_run_on_computer(self, computer: Computer) -> bool: type_check(computer, Computer) return computer.pk == self.computer.pk - def get_executable(self) -> pathlib.PurePosixPath: + def get_executable(self) -> pathlib.PurePath: """Return the executable that the submission script should execute to run the code. :return: The executable to be called in the submission script. @@ -207,12 +207,12 @@ def full_label(self) -> str: return f'{self.label}@{self.computer.label}' @property - def filepath_executable(self) -> pathlib.PurePosixPath: + def filepath_executable(self) -> pathlib.PurePath: """Return the absolute filepath of the executable that this code represents. :return: The absolute filepath of the executable. """ - return pathlib.PurePosixPath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) @filepath_executable.setter def filepath_executable(self, value: str) -> None: diff --git a/src/aiida/orm/nodes/data/code/legacy.py b/src/aiida/orm/nodes/data/code/legacy.py index 5f62b1cf97..7c0b10faff 100644 --- a/src/aiida/orm/nodes/data/code/legacy.py +++ b/src/aiida/orm/nodes/data/code/legacy.py @@ -123,7 +123,7 @@ def can_run_on_computer(self, computer: Computer) -> bool: type_check(computer, orm.Computer) return computer.pk == self.get_remote_computer().pk - def get_executable(self) -> pathlib.PurePosixPath: + def get_executable(self) -> pathlib.PurePath: """Return the executable that the submission script should execute to run the code. :return: The executable to be called in the submission script. @@ -133,7 +133,7 @@ def get_executable(self) -> pathlib.PurePosixPath: else: exec_path = self.get_remote_exec_path() - return pathlib.PurePosixPath(exec_path) + return pathlib.PurePath(exec_path) def hide(self): """Hide the code (prevents from showing it in the verdi code list)""" diff --git a/src/aiida/orm/nodes/data/code/portable.py b/src/aiida/orm/nodes/data/code/portable.py index 9db96c24f8..b32a966a00 100644 --- a/src/aiida/orm/nodes/data/code/portable.py +++ b/src/aiida/orm/nodes/data/code/portable.py @@ -124,7 +124,7 @@ def can_run_on_computer(self, computer: Computer) -> bool: """ return True - def get_executable(self) -> pathlib.PurePosixPath: + def get_executable(self) -> pathlib.PurePath: """Return the executable that the submission script should execute to run the code. :return: The executable to be called in the submission script. @@ -161,12 +161,12 @@ def full_label(self) -> str: return self.label @property - def filepath_executable(self) -> pathlib.PurePosixPath: + def filepath_executable(self) -> pathlib.PurePath: """Return the relative filepath of the executable that this code represents. :return: The relative filepath of the executable. """ - return pathlib.PurePosixPath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) + return pathlib.PurePath(self.base.attributes.get(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE)) @filepath_executable.setter def filepath_executable(self, value: str) -> None: @@ -176,7 +176,7 @@ def filepath_executable(self, value: str) -> None: """ type_check(value, str) - if pathlib.PurePosixPath(value).is_absolute(): + if pathlib.PurePath(value).is_absolute(): raise ValueError('The `filepath_executable` should not be absolute.') self.base.attributes.set(self._KEY_ATTRIBUTE_FILEPATH_EXECUTABLE, value) diff --git a/src/aiida/orm/nodes/data/folder.py b/src/aiida/orm/nodes/data/folder.py index c0d385c961..bc59146f62 100644 --- a/src/aiida/orm/nodes/data/folder.py +++ b/src/aiida/orm/nodes/data/folder.py @@ -23,7 +23,7 @@ __all__ = ('FolderData',) -FilePath = t.Union[str, pathlib.PurePosixPath] +FilePath = t.Union[str, pathlib.PurePath] class FolderData(Data): @@ -177,7 +177,7 @@ def put_object_from_tree(self, filepath: str, path: str | None = None) -> None: """ return self.base.repository.put_object_from_tree(filepath, path) - def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePosixPath, list[str], list[str]]]: + def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePath, list[str], list[str]]]: """Walk over the directories and files contained within this repository. .. note:: the order of the dirname and filename lists that are returned is not necessarily sorted. This is in @@ -186,11 +186,11 @@ def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePos :param path: the relative path of the directory within the repository whose contents to walk. :return: tuples of root, dirnames and filenames just like ``os.walk``, with the exception that the root path is always relative with respect to the repository root, instead of an absolute path and it is an instance of - ``pathlib.PurePosixPath`` instead of a normal string + ``pathlib.PurePath`` instead of a normal string """ yield from self.base.repository.walk(path) - def glob(self) -> t.Iterable[pathlib.PurePosixPath]: + def glob(self) -> t.Iterable[pathlib.PurePath]: """Yield a recursive list of all paths (files and directories).""" yield from self.base.repository.glob() diff --git a/src/aiida/orm/nodes/data/singlefile.py b/src/aiida/orm/nodes/data/singlefile.py index d4cc4b7095..80e1d4b847 100644 --- a/src/aiida/orm/nodes/data/singlefile.py +++ b/src/aiida/orm/nodes/data/singlefile.py @@ -22,7 +22,7 @@ __all__ = ('SinglefileData',) -FilePath = t.Union[str, pathlib.PurePosixPath] +FilePath = t.Union[str, pathlib.PurePath] class SinglefileData(Data): diff --git a/src/aiida/orm/nodes/repository.py b/src/aiida/orm/nodes/repository.py index bc24fe1377..a19e7a7ace 100644 --- a/src/aiida/orm/nodes/repository.py +++ b/src/aiida/orm/nodes/repository.py @@ -20,7 +20,7 @@ __all__ = ('NodeRepository',) -FilePath = t.Union[str, pathlib.PurePosixPath] +FilePath = t.Union[str, pathlib.PurePath] class NodeRepository: @@ -317,7 +317,7 @@ def put_object_from_tree(self, filepath: str, path: str | None = None): self._repository.put_object_from_tree(filepath, path) self._update_repository_metadata() - def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePosixPath, list[str], list[str]]]: + def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePath, list[str], list[str]]]: """Walk over the directories and files contained within this repository. .. note:: the order of the dirname and filename lists that are returned is not necessarily sorted. This is in @@ -326,11 +326,11 @@ def walk(self, path: FilePath | None = None) -> t.Iterable[tuple[pathlib.PurePos :param path: the relative path of the directory within the repository whose contents to walk. :return: tuples of root, dirnames and filenames just like ``os.walk``, with the exception that the root path is always relative with respect to the repository root, instead of an absolute path and it is an instance of - ``pathlib.PurePosixPath`` instead of a normal string + ``pathlib.PurePath`` instead of a normal string """ yield from self._repository.walk(path) - def glob(self) -> t.Iterable[pathlib.PurePosixPath]: + def glob(self) -> t.Iterable[pathlib.PurePath]: """Yield a recursive list of all paths (files and directories).""" for dirpath, dirnames, filenames in self.walk(): for dirname in dirnames: diff --git a/src/aiida/repository/repository.py b/src/aiida/repository/repository.py index 992a96447d..97455595b0 100644 --- a/src/aiida/repository/repository.py +++ b/src/aiida/repository/repository.py @@ -12,7 +12,7 @@ __all__ = ('Repository',) -FilePath = Union[str, pathlib.PurePosixPath] +FilePath = Union[str, pathlib.PurePath] class Repository: @@ -127,23 +127,23 @@ def hash(self) -> str: return make_hash(objects) @staticmethod - def _pre_process_path(path: Optional[FilePath] = None) -> pathlib.PurePosixPath: - """Validate and convert the path to instance of ``pathlib.PurePosixPath``. + def _pre_process_path(path: Optional[FilePath] = None) -> pathlib.PurePath: + """Validate and convert the path to instance of ``pathlib.PurePath``. This should be called by every method of this class before doing anything, such that it can safely assume that - the path is a ``pathlib.PurePosixPath`` object, which makes path manipulation a lot easier. + the path is a ``pathlib.PurePath`` object, which makes path manipulation a lot easier. - :param path: the path as a ``pathlib.PurePosixPath`` object or `None`. - :raises TypeError: if the type of path was not a str nor a ``pathlib.PurePosixPath`` instance. + :param path: the path as a ``pathlib.PurePath`` object or `None`. + :raises TypeError: if the type of path was not a str nor a ``pathlib.PurePath`` instance. """ if path is None: - return pathlib.PurePosixPath() + return pathlib.PurePath() if isinstance(path, str): - path = pathlib.PurePosixPath(path) + path = pathlib.PurePath(path) - if not isinstance(path, pathlib.PurePosixPath): - raise TypeError('path is not of type `str` nor `pathlib.PurePosixPath`.') + if not isinstance(path, pathlib.PurePath): + raise TypeError('path is not of type `str` nor `pathlib.PurePath`.') if path.is_absolute(): raise TypeError(f'path `{path}` is not a relative path.') @@ -167,7 +167,7 @@ def set_backend(self, backend: AbstractRepositoryBackend) -> None: type_check(backend, AbstractRepositoryBackend) self._backend = backend - def _insert_file(self, path: pathlib.PurePosixPath, key: str) -> None: + def _insert_file(self, path: pathlib.PurePath, key: str) -> None: """Insert a new file object in the object mapping. .. note:: this assumes the path is a valid relative path, so should be checked by the caller. @@ -334,10 +334,10 @@ def put_object_from_tree(self, filepath: FilePath, path: Optional[FilePath] = No path = self._pre_process_path(path) if isinstance(filepath, str): - filepath = pathlib.PurePosixPath(filepath) + filepath = pathlib.PurePath(filepath) - if not isinstance(filepath, pathlib.PurePosixPath): - raise TypeError(f'filepath `{filepath}` is not of type `str` nor `pathlib.PurePosixPath`.') + if not isinstance(filepath, pathlib.PurePath): + raise TypeError(f'filepath `{filepath}` is not of type `str` nor `pathlib.PurePath`.') if not filepath.is_absolute(): raise TypeError(f'filepath `{filepath}` is not an absolute path.') @@ -347,7 +347,7 @@ def put_object_from_tree(self, filepath: FilePath, path: Optional[FilePath] = No self.create_directory(path) for root_str, dirnames, filenames in os.walk(filepath): - root = pathlib.PurePosixPath(root_str) + root = pathlib.PurePath(root_str) for dirname in dirnames: self.create_directory(path / root.relative_to(filepath) / dirname) @@ -457,7 +457,7 @@ def clone(self, source: 'Repository') -> None: with source.open(root / filename) as handle: self.put_object_from_filelike(handle, root / filename) - def walk(self, path: Optional[FilePath] = None) -> Iterable[Tuple[pathlib.PurePosixPath, List[str], List[str]]]: + def walk(self, path: Optional[FilePath] = None) -> Iterable[Tuple[pathlib.PurePath, List[str], List[str]]]: """Walk over the directories and files contained within this repository. .. note:: the order of the dirname and filename lists that are returned is not necessarily sorted. This is in @@ -466,7 +466,7 @@ def walk(self, path: Optional[FilePath] = None) -> Iterable[Tuple[pathlib.PurePo :param path: the relative path of the directory within the repository whose contents to walk. :return: tuples of root, dirnames and filenames just like ``os.walk``, with the exception that the root path is always relative with respect to the repository root, instead of an absolute path and it is an instance of - ``pathlib.PurePosixPath`` instead of a normal string + ``pathlib.PurePath`` instead of a normal string """ path = self._pre_process_path(path) diff --git a/src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py b/src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py index b6f40f7e98..1194104565 100644 --- a/src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py +++ b/src/aiida/storage/sqlite_zip/migrations/legacy_to_main.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from datetime import datetime from hashlib import sha256 -from pathlib import Path, PurePosixPath +from pathlib import Path, PurePath from typing import Any, Callable, ContextManager, Dict, Iterator, List, Optional, Tuple, Union from archive_path import ZipPath @@ -96,7 +96,7 @@ def _in_archive_context(_inpath): if len(parts) < 6 or parts[0] != 'nodes' or parts[4] not in ('raw_input', 'path'): continue uuid = ''.join(parts[1:4]) - posix_rel = PurePosixPath(*parts[5:]) + posix_rel = PurePath(*parts[5:]) hashkey = None if subpath.is_file(): with subpath.open('rb') as handle: @@ -270,7 +270,7 @@ def _create_repo_metadata(paths: List[Tuple[str, Optional[str]]]) -> Dict[str, A """ top_level = File() for _path, hashkey in paths: - path = PurePosixPath(_path) + path = PurePath(_path) if hashkey is None: _create_directory(top_level, path) else: @@ -279,7 +279,7 @@ def _create_repo_metadata(paths: List[Tuple[str, Optional[str]]]) -> Dict[str, A return top_level.serialize() -def _create_directory(top_level: File, path: PurePosixPath) -> File: +def _create_directory(top_level: File, path: PurePath) -> File: """Create a new directory with the given path. :param path: the relative path of the directory. diff --git a/src/aiida/transports/plugins/local.py b/src/aiida/transports/plugins/local.py index 8de49838e3..89cedfb0ea 100644 --- a/src/aiida/transports/plugins/local.py +++ b/src/aiida/transports/plugins/local.py @@ -13,6 +13,7 @@ import glob import io import os +import sys import shutil import subprocess @@ -734,9 +735,14 @@ def _exec_command_internal(self, command, workdir=None, **kwargs): else: cwd = self.getcwd() + if sys.platform == "win32": + shell = False + else: + shell = True + with subprocess.Popen( command, - shell=True, + shell=shell, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, diff --git a/src/aiida/transports/transport.py b/src/aiida/transports/transport.py index a6d755973e..8197cd9a33 100644 --- a/src/aiida/transports/transport.py +++ b/src/aiida/transports/transport.py @@ -13,6 +13,7 @@ import os import re import sys +import chardet from collections import OrderedDict from pathlib import Path @@ -429,7 +430,16 @@ def exec_command_wait(self, command, stdin=None, encoding='utf-8', workdir=None, command=command, stdin=stdin, workdir=workdir, **kwargs ) # Return the decoded strings - return (retval, stdout_bytes.decode(encoding), stderr_bytes.decode(encoding)) + if sys.platform == "win32": + outenc = chardet.detect(stdout_bytes)['encoding'] + errenc = chardet.detect(stderr_bytes)['encoding'] + if outenc == None: + outenc = 'utf-8' + if errenc == None: + errenc = 'utf-8' + return (retval, stdout_bytes.decode(outenc), stderr_bytes.decode(errenc)) + else: + return (retval, stdout_bytes.decode(encoding), stderr_bytes.decode(encoding)) @abc.abstractmethod def get(self, remotepath, localpath, *args, **kwargs): diff --git a/tests/orm/data/code/test_abstract.py b/tests/orm/data/code/test_abstract.py index 0985f90f40..6c16ee1cf3 100644 --- a/tests/orm/data/code/test_abstract.py +++ b/tests/orm/data/code/test_abstract.py @@ -22,9 +22,9 @@ def can_run_on_computer(self, computer) -> bool: """Return whether the code can run on a given computer.""" return True - def get_executable(self) -> pathlib.PurePosixPath: + def get_executable(self) -> pathlib.PurePath: """Return the executable that the submission script should execute to run the code.""" - return pathlib.PurePosixPath('/bin/executable') + return pathlib.PurePath('/bin/executable') @property def full_label(self) -> str: diff --git a/tests/repository/test_repository.py b/tests/repository/test_repository.py index f86b2456a8..ca10aa11a4 100644 --- a/tests/repository/test_repository.py +++ b/tests/repository/test_repository.py @@ -98,7 +98,7 @@ def test_initialise(repository_uninitialised): def test_pre_process_path(): """Test the ``Repository.pre_process_path`` classmethod.""" - with pytest.raises(TypeError, match=r'path is not of type `str` nor `pathlib.PurePosixPath`.'): + with pytest.raises(TypeError, match=r'path is not of type `str` nor `pathlib.PurePath`.'): Repository._pre_process_path(path=1) with pytest.raises(TypeError, match=r'path `.*` is not a relative path.'): @@ -377,7 +377,7 @@ def test_put_object_from_filelike(repository, generate_directory): def test_put_object_from_tree_raises(repository): """Test the ``Repository.put_object_from_tree`` method when it should raise.""" - with pytest.raises(TypeError, match=r'filepath `.*` is not of type `str` nor `pathlib.PurePosixPath`.'): + with pytest.raises(TypeError, match=r'filepath `.*` is not of type `str` nor `pathlib.PurePath`.'): repository.put_object_from_tree(None) with pytest.raises(TypeError, match=r'filepath `.*` is not an absolute path.'): From 5d862f247d4a6f2ba45b0c6356fdcecae4ec34e3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 06:59:19 +0000 Subject: [PATCH 04/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/aiida/transports/plugins/local.py | 4 ++-- src/aiida/transports/transport.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/aiida/transports/plugins/local.py b/src/aiida/transports/plugins/local.py index 89cedfb0ea..e26f69053a 100644 --- a/src/aiida/transports/plugins/local.py +++ b/src/aiida/transports/plugins/local.py @@ -13,9 +13,9 @@ import glob import io import os -import sys import shutil import subprocess +import sys from aiida.transports import cli as transport_cli from aiida.transports.transport import Transport, TransportInternalError @@ -735,7 +735,7 @@ def _exec_command_internal(self, command, workdir=None, **kwargs): else: cwd = self.getcwd() - if sys.platform == "win32": + if sys.platform == 'win32': shell = False else: shell = True diff --git a/src/aiida/transports/transport.py b/src/aiida/transports/transport.py index 8197cd9a33..adf4f7c957 100644 --- a/src/aiida/transports/transport.py +++ b/src/aiida/transports/transport.py @@ -13,10 +13,11 @@ import os import re import sys -import chardet from collections import OrderedDict from pathlib import Path +import chardet + from aiida.common.exceptions import InternalError from aiida.common.lang import classproperty from aiida.common.warnings import warn_deprecation @@ -430,7 +431,7 @@ def exec_command_wait(self, command, stdin=None, encoding='utf-8', workdir=None, command=command, stdin=stdin, workdir=workdir, **kwargs ) # Return the decoded strings - if sys.platform == "win32": + if sys.platform == 'win32': outenc = chardet.detect(stdout_bytes)['encoding'] errenc = chardet.detect(stderr_bytes)['encoding'] if outenc == None: From 5a287fcf4af1116d5c5dae95b9a13666cf10b3e3 Mon Sep 17 00:00:00 2001 From: lainme Date: Mon, 20 Jan 2025 16:48:32 +0800 Subject: [PATCH 05/13] Fix None detection --- src/aiida/transports/transport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aiida/transports/transport.py b/src/aiida/transports/transport.py index 8197cd9a33..3c58352ca6 100644 --- a/src/aiida/transports/transport.py +++ b/src/aiida/transports/transport.py @@ -433,9 +433,9 @@ def exec_command_wait(self, command, stdin=None, encoding='utf-8', workdir=None, if sys.platform == "win32": outenc = chardet.detect(stdout_bytes)['encoding'] errenc = chardet.detect(stderr_bytes)['encoding'] - if outenc == None: + if outenc is None: outenc = 'utf-8' - if errenc == None: + if errenc is None: errenc = 'utf-8' return (retval, stdout_bytes.decode(outenc), stderr_bytes.decode(errenc)) else: From e24e42d0c6461b70b3b784de19a7c58f2faa5262 Mon Sep 17 00:00:00 2001 From: lainme Date: Wed, 5 Feb 2025 13:18:10 +0800 Subject: [PATCH 06/13] fix daemon status command and chardet version --- pyproject.toml | 2 +- src/aiida/engine/daemon/client.py | 5 ++++- uv.lock | 13 ++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5b27bdac18..cf68717078 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,7 +49,7 @@ dependencies = [ 'typing-extensions~=4.0;python_version<"3.10"', 'upf_to_json~=0.9.2', 'wrapt~=1.11', - 'chardet' + 'chardet~=5.2.0;platform_system=="Windows"' ] description = 'AiiDA is a workflow manager for computational science with a strong focus on provenance, performance and extensibility.' dynamic = ['version'] # read from aiida/__init__.py diff --git a/src/aiida/engine/daemon/client.py b/src/aiida/engine/daemon/client.py index 4e47e7ed60..ffdd83d30c 100644 --- a/src/aiida/engine/daemon/client.py +++ b/src/aiida/engine/daemon/client.py @@ -84,7 +84,10 @@ class DaemonClient: """Client to interact with the daemon.""" _DAEMON_NAME = 'aiida-{name}' - _ENDPOINT_PROTOCOL = ControllerProtocol.IPC + if sys.platform == 'win32': + _ENDPOINT_PROTOCOL = ControllerProtocol.TCP + else: + _ENDPOINT_PROTOCOL = ControllerProtocol.IPC def __init__(self, profile: Profile): """Construct an instance for a given profile. diff --git a/uv.lock b/uv.lock index c4381b42de..93b34c4cc5 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,7 @@ dependencies = [ { name = "alembic" }, { name = "archive-path" }, { name = "asyncssh" }, + { name = "chardet", marker = "sys_platform == 'win32'" }, { name = "circus" }, { name = "click" }, { name = "click-spinner" }, @@ -171,6 +172,7 @@ requires-dist = [ { name = "ase", marker = "extra == 'atomic-tools'", specifier = "~=3.18" }, { name = "asyncssh", specifier = "~=2.19.0" }, { name = "bpython", marker = "extra == 'bpython'", specifier = "~=0.18.0" }, + { name = "chardet", marker = "sys_platform == 'win32'", specifier = "~=5.2.0" }, { name = "circus", specifier = "~=0.18.0" }, { name = "click", specifier = "~=8.1" }, { name = "click-spinner", specifier = "~=0.1.8" }, @@ -673,6 +675,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249 }, ] +[[package]] +name = "chardet" +version = "5.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/0d/f7b6ab21ec75897ed80c17d79b15951a719226b9fababf1e40ea74d69079/chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7", size = 2069618 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/6f/f5fbc992a329ee4e0f288c1fe0e2ad9485ed064cac731ed2fe47dcc38cbf/chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970", size = 199385 }, +] + [[package]] name = "charset-normalizer" version = "3.4.1" @@ -3099,7 +3110,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "python_full_version < '3.10' or sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450 } wheels = [ From e4ba7382e403621de91169768994259539fb8cee Mon Sep 17 00:00:00 2001 From: lainme Date: Wed, 5 Feb 2025 17:50:56 +0800 Subject: [PATCH 07/13] Fix postgres --- src/aiida/cmdline/commands/cmd_presto.py | 2 +- src/aiida/cmdline/commands/cmd_setup.py | 2 +- src/aiida/manage/configuration/profile.py | 5 +++-- src/aiida/manage/tests/pytest_fixtures.py | 2 +- src/aiida/storage/psql_dos/backend.py | 3 ++- src/aiida/tools/pytest_fixtures/storage.py | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/aiida/cmdline/commands/cmd_presto.py b/src/aiida/cmdline/commands/cmd_presto.py index eeb98fad75..a5fb24fda5 100644 --- a/src/aiida/cmdline/commands/cmd_presto.py +++ b/src/aiida/cmdline/commands/cmd_presto.py @@ -100,7 +100,7 @@ def detect_postgres_config( 'database_name': database_name, 'database_username': database_username, 'database_password': database_password, - 'repository_uri': f'file://{aiida_config_folder / "repository" / profile_name}', + 'repository_uri': pathlib.Path(f'{aiida_config_folder / "repository" / profile_name}').as_uri() } diff --git a/src/aiida/cmdline/commands/cmd_setup.py b/src/aiida/cmdline/commands/cmd_setup.py index ad86ee21db..5c900790d4 100644 --- a/src/aiida/cmdline/commands/cmd_setup.py +++ b/src/aiida/cmdline/commands/cmd_setup.py @@ -88,7 +88,7 @@ def setup( 'database_name': db_name, 'database_username': db_username, 'database_password': db_password, - 'repository_uri': f'file://{repository}', + 'repository_uri': Path(f'{repository}').as_uri(), }, ) profile.set_process_controller( diff --git a/src/aiida/manage/configuration/profile.py b/src/aiida/manage/configuration/profile.py index 93a3c5c911..2af280f4ac 100644 --- a/src/aiida/manage/configuration/profile.py +++ b/src/aiida/manage/configuration/profile.py @@ -211,6 +211,7 @@ def repository_path(self) -> pathlib.Path: :return: absolute filepath of the profile's file repository """ from urllib.parse import urlparse + from urllib.request import url2pathname from aiida.common.warnings import warn_deprecation @@ -224,10 +225,10 @@ def repository_path(self) -> pathlib.Path: if parts.scheme != 'file': raise exceptions.ConfigurationError('invalid repository protocol, only the local `file://` is supported') - if not os.path.isabs(parts.path): + if not os.path.isabs(url2pathname(parts.path)): raise exceptions.ConfigurationError('invalid repository URI: the path has to be absolute') - return pathlib.Path(os.path.expanduser(parts.path)) + return pathlib.Path(os.path.expanduser(url2pathname(parts.path))) @property def filepaths(self): diff --git a/src/aiida/manage/tests/pytest_fixtures.py b/src/aiida/manage/tests/pytest_fixtures.py index 8cc44dc608..7f5f3f2c61 100644 --- a/src/aiida/manage/tests/pytest_fixtures.py +++ b/src/aiida/manage/tests/pytest_fixtures.py @@ -225,7 +225,7 @@ def factory(custom_configuration: dict[str, t.Any] | None = None) -> dict[str, t 'storage': { 'backend': 'core.psql_dos', 'config': { - 'repository_uri': f'file://{tmp_path_factory.mktemp("repository")}', + 'repository_uri': Path(f'{tmp_path_factory.mktemp("repository")}').as_uri(), }, } } diff --git a/src/aiida/storage/psql_dos/backend.py b/src/aiida/storage/psql_dos/backend.py index b0d2dc813a..0acb746173 100644 --- a/src/aiida/storage/psql_dos/backend.py +++ b/src/aiida/storage/psql_dos/backend.py @@ -48,6 +48,7 @@ def get_filepath_container(profile: Profile) -> pathlib.Path: """Return the filepath of the disk-object store container.""" from urllib.parse import urlparse + from urllib.request import url2pathname try: parts = urlparse(profile.storage_config['repository_uri']) @@ -59,7 +60,7 @@ def get_filepath_container(profile: Profile) -> pathlib.Path: f'invalid profile {profile.name}: `storage.config.repository_uri` does not start with `file://`.' ) - filepath = pathlib.Path(parts.path) + filepath = pathlib.Path(url2pathname(parts.path)) if not filepath.is_absolute(): raise ConfigurationError(f'invalid profile {profile.name}: `storage.config.repository_uri` is not absolute') diff --git a/src/aiida/tools/pytest_fixtures/storage.py b/src/aiida/tools/pytest_fixtures/storage.py index dd47ad0f21..3652e221ef 100644 --- a/src/aiida/tools/pytest_fixtures/storage.py +++ b/src/aiida/tools/pytest_fixtures/storage.py @@ -98,7 +98,7 @@ def factory( database_username=database_username, database_password=database_password, ) - storage_config['repository_uri'] = f'file://{tmp_path_factory.mktemp("repository")}' + storage_config['repository_uri'] = Path(f'{tmp_path_factory.mktemp("repository")}').as_uri() return storage_config From b18cef8afee2d9f751a790208e6f537a44817d40 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 09:59:15 +0000 Subject: [PATCH 08/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/aiida/cmdline/commands/cmd_presto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aiida/cmdline/commands/cmd_presto.py b/src/aiida/cmdline/commands/cmd_presto.py index a5fb24fda5..7a468b8aeb 100644 --- a/src/aiida/cmdline/commands/cmd_presto.py +++ b/src/aiida/cmdline/commands/cmd_presto.py @@ -100,7 +100,7 @@ def detect_postgres_config( 'database_name': database_name, 'database_username': database_username, 'database_password': database_password, - 'repository_uri': pathlib.Path(f'{aiida_config_folder / "repository" / profile_name}').as_uri() + 'repository_uri': pathlib.Path(f'{aiida_config_folder / "repository" / profile_name}').as_uri(), } From cb43ffaa3a025d8eb8f2b09391ae5a386ce4148d Mon Sep 17 00:00:00 2001 From: lainme Date: Wed, 5 Feb 2025 18:03:42 +0800 Subject: [PATCH 09/13] fix pathlib --- src/aiida/cmdline/commands/cmd_setup.py | 3 ++- src/aiida/manage/tests/pytest_fixtures.py | 2 +- src/aiida/tools/pytest_fixtures/storage.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/aiida/cmdline/commands/cmd_setup.py b/src/aiida/cmdline/commands/cmd_setup.py index 5c900790d4..912d7ac51b 100644 --- a/src/aiida/cmdline/commands/cmd_setup.py +++ b/src/aiida/cmdline/commands/cmd_setup.py @@ -9,6 +9,7 @@ """The `verdi setup` and `verdi quicksetup` commands.""" import click +import pathlib from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.params import options @@ -88,7 +89,7 @@ def setup( 'database_name': db_name, 'database_username': db_username, 'database_password': db_password, - 'repository_uri': Path(f'{repository}').as_uri(), + 'repository_uri': pathlib.Path(f'{repository}').as_uri(), }, ) profile.set_process_controller( diff --git a/src/aiida/manage/tests/pytest_fixtures.py b/src/aiida/manage/tests/pytest_fixtures.py index 7f5f3f2c61..0482334864 100644 --- a/src/aiida/manage/tests/pytest_fixtures.py +++ b/src/aiida/manage/tests/pytest_fixtures.py @@ -225,7 +225,7 @@ def factory(custom_configuration: dict[str, t.Any] | None = None) -> dict[str, t 'storage': { 'backend': 'core.psql_dos', 'config': { - 'repository_uri': Path(f'{tmp_path_factory.mktemp("repository")}').as_uri(), + 'repository_uri': pathlib.Path(f'{tmp_path_factory.mktemp("repository")}').as_uri(), }, } } diff --git a/src/aiida/tools/pytest_fixtures/storage.py b/src/aiida/tools/pytest_fixtures/storage.py index 3652e221ef..537fad4141 100644 --- a/src/aiida/tools/pytest_fixtures/storage.py +++ b/src/aiida/tools/pytest_fixtures/storage.py @@ -98,7 +98,7 @@ def factory( database_username=database_username, database_password=database_password, ) - storage_config['repository_uri'] = Path(f'{tmp_path_factory.mktemp("repository")}').as_uri() + storage_config['repository_uri'] = pathlib.Path(f'{tmp_path_factory.mktemp("repository")}').as_uri() return storage_config From 440b983ba62d3454b2da267eecc96e3b48d681d5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Feb 2025 10:06:07 +0000 Subject: [PATCH 10/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/aiida/cmdline/commands/cmd_setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/aiida/cmdline/commands/cmd_setup.py b/src/aiida/cmdline/commands/cmd_setup.py index 912d7ac51b..6ba2ca8653 100644 --- a/src/aiida/cmdline/commands/cmd_setup.py +++ b/src/aiida/cmdline/commands/cmd_setup.py @@ -8,9 +8,10 @@ ########################################################################### """The `verdi setup` and `verdi quicksetup` commands.""" -import click import pathlib +import click + from aiida.cmdline.commands.cmd_verdi import verdi from aiida.cmdline.params import options from aiida.cmdline.params.options.commands import setup as options_setup From 80497617f9fd9fc77c49b126c016ea5dafc3d97e Mon Sep 17 00:00:00 2001 From: Alexander Goscinski Date: Mon, 10 Feb 2025 11:23:58 +0100 Subject: [PATCH 11/13] Move chardet import into the win32 if case --- src/aiida/transports/transport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/aiida/transports/transport.py b/src/aiida/transports/transport.py index 1afc842dee..89d582859e 100644 --- a/src/aiida/transports/transport.py +++ b/src/aiida/transports/transport.py @@ -503,6 +503,7 @@ def exec_command_wait( ) # Return the decoded strings if sys.platform == 'win32': + import chardet outenc = chardet.detect(stdout_bytes)['encoding'] errenc = chardet.detect(stderr_bytes)['encoding'] if outenc is None: From 1c6a2857ca1d92b86ed9706b3fc4932825fe8e29 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 10:25:30 +0000 Subject: [PATCH 12/13] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/aiida/transports/transport.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/aiida/transports/transport.py b/src/aiida/transports/transport.py index 89d582859e..2f19669a18 100644 --- a/src/aiida/transports/transport.py +++ b/src/aiida/transports/transport.py @@ -18,8 +18,6 @@ from pathlib import Path, PurePosixPath from typing import Optional, Union -import chardet - from aiida.common.exceptions import InternalError from aiida.common.lang import classproperty from aiida.common.warnings import warn_deprecation @@ -504,6 +502,7 @@ def exec_command_wait( # Return the decoded strings if sys.platform == 'win32': import chardet + outenc = chardet.detect(stdout_bytes)['encoding'] errenc = chardet.detect(stderr_bytes)['encoding'] if outenc is None: From ae25b13be3132de9df5682c5aeee597d728275c1 Mon Sep 17 00:00:00 2001 From: Alexander Goscinski Date: Sat, 15 Feb 2025 10:06:57 +0100 Subject: [PATCH 13/13] Add chardet to conda deps --- environment.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/environment.yml b/environment.yml index 3cbf1b4307..3560a817f5 100644 --- a/environment.yml +++ b/environment.yml @@ -37,3 +37,4 @@ dependencies: - typing-extensions~=4.0 - upf_to_json~=0.9.2 - wrapt~=1.11 +- chardet~=5.2.0 # [win]