refactor(BA-2136): Replace deprecated pkg_resources with importlib

changes/5609.fix.md

@@ -0,0 +1 @@
+Replace `pkg_resources` (deprecated since Python 3.12) with `importlib`

src/ai/backend/agent/agent.py

@@ -29,6 +29,7 @@
 from contextlib import contextmanager
 from dataclasses import dataclass
 from decimal import Decimal
+from importlib.resources import files
 from io import SEEK_END, BytesIO
 from pathlib import Path
 from types import TracebackType
@@ -46,7 +47,6 @@
 
 import aiotools
 import attrs
-import pkg_resources
 import zmq
 import zmq.asyncio
 from async_timeout import timeout
@@ -499,9 +499,10 @@ def get_krunner_info(self) -> tuple[str, str, str, str, str]:
             try:
                 krunner_pyver = (
                     Path(
-                        pkg_resources.resource_filename(
-                            f"ai.backend.krunner.{matched_distro_pkgname}",
-                            f"krunner-python.{matched_distro}.txt",
+                        str(
+                            files(f"ai.backend.krunner.{matched_distro_pkgname}").joinpath(
+                                f"krunner-python.{matched_distro}.txt"
+                            )
                         )
                     )
                     .read_text()
@@ -565,7 +566,7 @@ def _mount(
             krunner_volume,
             krunner_pyver,
         ) = self.get_krunner_info()
-        artifact_path = Path(pkg_resources.resource_filename("ai.backend.agent", "../runner"))
+        artifact_path = Path(str(files("ai.backend.agent").joinpath("../runner")))
 
         def find_artifacts(pattern: str) -> Mapping[str, str]:
             artifacts = {}

src/ai/backend/agent/docker/agent.py

@@ -16,6 +16,7 @@
 from decimal import Decimal
 from functools import partial
 from http import HTTPStatus
+from importlib.resources import files
 from io import StringIO
 from pathlib import Path
 from subprocess import CalledProcessError
@@ -38,7 +39,6 @@
 import aiofiles
 import aiohttp
 import aiotools
-import pkg_resources
 import zmq
 import zmq.asyncio
 from aiodocker.docker import Docker, DockerContainer
@@ -389,22 +389,18 @@ def _create_scratch_dirs():
             # directories when the agent is running as non-root.
             def _clone_dotfiles():
                 jupyter_custom_css_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", "jupyter-custom.css")
+                    str(files("ai.backend.runner").joinpath("jupyter-custom.css"))
                 )
-                logo_path = Path(pkg_resources.resource_filename("ai.backend.runner", "logo.svg"))
-                font_path = Path(pkg_resources.resource_filename("ai.backend.runner", "roboto.ttf"))
+                logo_path = Path(str(files("ai.backend.runner").joinpath("logo.svg")))
+                font_path = Path(str(files("ai.backend.runner").joinpath("roboto.ttf")))
                 font_italic_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", "roboto-italic.ttf")
-                )
-                bashrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".bashrc"))
-                bash_profile_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", ".bash_profile")
-                )
-                zshrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".zshrc"))
-                vimrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".vimrc"))
-                tmux_conf_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", ".tmux.conf")
+                    str(files("ai.backend.runner").joinpath("roboto-italic.ttf"))
                 )
+                bashrc_path = Path(str(files("ai.backend.runner").joinpath(".bashrc")))
+                bash_profile_path = Path(str(files("ai.backend.runner").joinpath(".bash_profile")))
+                zshrc_path = Path(str(files("ai.backend.runner").joinpath(".zshrc")))
+                vimrc_path = Path(str(files("ai.backend.runner").joinpath(".vimrc")))
+                tmux_conf_path = Path(str(files("ai.backend.runner").joinpath(".tmux.conf")))
                 jupyter_custom_dir = self.work_dir / ".jupyter" / "custom"
                 jupyter_custom_dir.mkdir(parents=True, exist_ok=True)
                 shutil.copy(jupyter_custom_css_path.resolve(), jupyter_custom_dir / "custom.css")
@@ -579,12 +575,7 @@ async def get_intrinsic_mounts(self) -> Sequence[Mount]:
         return mounts
 
     def resolve_krunner_filepath(self, filename) -> Path:
-        return Path(
-            pkg_resources.resource_filename(
-                "ai.backend.runner",
-                "../" + filename,
-            )
-        ).resolve()
+        return Path(str(files("ai.backend.runner").joinpath("../" + filename))).resolve()
 
     def get_runner_mount(
         self,

src/ai/backend/agent/docker/kernel.py

@@ -12,11 +12,11 @@
 import subprocess
 import textwrap
 from collections.abc import Mapping
+from importlib.resources import files
 from pathlib import Path, PurePosixPath
 from typing import Any, Dict, Final, FrozenSet, Optional, Tuple, cast, override
 
 import janus
-import pkg_resources
 from aiodocker.docker import Docker, DockerVolume
 from aiodocker.exceptions import DockerError
 from aiotools import TaskGroup
@@ -459,8 +459,10 @@ async def prepare_krunner_env_impl(distro: str, entrypoint_name: str) -> Tuple[s
     arch = get_arch_name()
     current_version = int(
         Path(
-            pkg_resources.resource_filename(
-                f"ai.backend.krunner.{entrypoint_name}", f"./krunner-version.{distro}.txt"
+            str(
+                files(f"ai.backend.krunner.{entrypoint_name}").joinpath(
+                    f"./krunner-version.{distro}.txt"
+                )
             )
         )
         .read_text()
@@ -477,8 +479,8 @@ async def prepare_krunner_env_impl(distro: str, entrypoint_name: str) -> Tuple[s
                 break
         else:
             log.info("preparing the Docker image for krunner extractor...")
-            extractor_archive = pkg_resources.resource_filename(
-                "ai.backend.runner", f"krunner-extractor.img.{arch}.tar.xz"
+            extractor_archive = str(
+                files("ai.backend.runner").joinpath(f"krunner-extractor.img.{arch}.tar.xz")
             )
             with lzma.open(extractor_archive, "rb") as reader:
                 image_tar = reader.read()
@@ -504,8 +506,10 @@ async def prepare_krunner_env_impl(distro: str, entrypoint_name: str) -> Tuple[s
                 do_create = True
         if do_create:
             archive_path = Path(
-                pkg_resources.resource_filename(
-                    f"ai.backend.krunner.{entrypoint_name}", f"krunner-env.{distro}.{arch}.tar.xz"
+                str(
+                    files(f"ai.backend.krunner.{entrypoint_name}").joinpath(
+                        f"krunner-env.{distro}.{arch}.tar.xz"
+                    )
                 )
             ).resolve()
             if not archive_path.exists():
@@ -517,7 +521,7 @@ async def prepare_krunner_env_impl(distro: str, entrypoint_name: str) -> Tuple[s
                     "Driver": "local",
                 })
                 extractor_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", "krunner-extractor.sh")
+                    str(files("ai.backend.runner").joinpath("krunner-extractor.sh"))
                 ).resolve()
                 proc = await asyncio.create_subprocess_exec(*[
                     "docker",
@@ -559,12 +563,7 @@ async def prepare_krunner_env(local_config: Mapping[str, Any]) -> Mapping[str, s
         plugin = entrypoint.load()
         await plugin.init({})  # currently does nothing
         provided_versions = (
-            Path(
-                pkg_resources.resource_filename(
-                    f"ai.backend.krunner.{entrypoint.name}",
-                    "versions.txt",
-                )
-            )
+            Path(str(files(f"ai.backend.krunner.{entrypoint.name}").joinpath("versions.txt")))
             .read_text()
             .splitlines()
         )
@@ -601,8 +600,8 @@ async def prepare_kernel_metadata_uri_handling(local_config: AgentUnifiedConfig)
     if local_config.agent.docker_mode == "linuxkit":
         # Docker Desktop mode
         arch = get_arch_name()
-        proxy_worker_binary = pkg_resources.resource_filename(
-            "ai.backend.agent.docker", f"linuxkit-metadata-proxy-worker.{arch}.bin"
+        proxy_worker_binary = str(
+            files("ai.backend.agent.docker").joinpath(f"linuxkit-metadata-proxy-worker.{arch}.bin")
         )
         shutil.copyfile(proxy_worker_binary, "/tmp/backend.ai/linuxkit-metadata-proxy")
         os.chmod("/tmp/backend.ai/linuxkit-metadata-proxy", 0o755)

src/ai/backend/agent/docker/utils.py

@@ -3,10 +3,10 @@
 import logging
 import subprocess
 from contextlib import closing
+from importlib.resources import files
 from pathlib import Path
 from typing import Any, Final, Mapping, Optional, Tuple
 
-import pkg_resources
 from aiodocker.docker import Docker
 from aiodocker.exceptions import DockerError
 
@@ -39,16 +39,18 @@ def __init__(
         self.container_config = container_config
         self.img_version = int(
             Path(
-                pkg_resources.resource_filename(
-                    "ai.backend.agent.docker",
-                    f"{default_container_name}.version.txt",
+                str(
+                    files("ai.backend.agent.docker").joinpath(
+                        f"{default_container_name}.version.txt"
+                    )
                 )
             ).read_text()
         )
         self.img_path = Path(
-            pkg_resources.resource_filename(
-                "ai.backend.agent.docker",
-                f"{default_container_name}.img.{arch}.tar.gz",
+            str(
+                files("ai.backend.agent.docker").joinpath(
+                    f"{default_container_name}.img.{arch}.tar.gz"
+                )
             )
         )
 

src/ai/backend/agent/kubernetes/agent.py

@@ -9,6 +9,7 @@
 import sys
 import uuid
 from decimal import Decimal
+from importlib.resources import files
 from io import StringIO
 from pathlib import Path
 from typing import (
@@ -27,7 +28,6 @@
 
 import aiotools
 import cattr
-import pkg_resources
 from kubernetes.client.models import V1Service, V1ServicePort
 from kubernetes_asyncio import client as kube_client
 from kubernetes_asyncio import config as kube_config
@@ -234,22 +234,18 @@ def _create_scratch_dirs():
             # directories when the agent is running as non-root.
             def _clone_dotfiles():
                 jupyter_custom_css_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", "jupyter-custom.css")
+                    str(files("ai.backend.runner").joinpath("jupyter-custom.css"))
                 )
-                logo_path = Path(pkg_resources.resource_filename("ai.backend.runner", "logo.svg"))
-                font_path = Path(pkg_resources.resource_filename("ai.backend.runner", "roboto.ttf"))
+                logo_path = Path(str(files("ai.backend.runner").joinpath("logo.svg")))
+                font_path = Path(str(files("ai.backend.runner").joinpath("roboto.ttf")))
                 font_italic_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", "roboto-italic.ttf")
-                )
-                bashrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".bashrc"))
-                bash_profile_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", ".bash_profile")
-                )
-                zshrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".zshrc"))
-                vimrc_path = Path(pkg_resources.resource_filename("ai.backend.runner", ".vimrc"))
-                tmux_conf_path = Path(
-                    pkg_resources.resource_filename("ai.backend.runner", ".tmux.conf")
+                    str(files("ai.backend.runner").joinpath("roboto-italic.ttf"))
                 )
+                bashrc_path = Path(str(files("ai.backend.runner").joinpath(".bashrc")))
+                bash_profile_path = Path(str(files("ai.backend.runner").joinpath(".bash_profile")))
+                zshrc_path = Path(str(files("ai.backend.runner").joinpath(".zshrc")))
+                vimrc_path = Path(str(files("ai.backend.runner").joinpath(".vimrc")))
+                tmux_conf_path = Path(str(files("ai.backend.runner").joinpath(".tmux.conf")))
                 jupyter_custom_dir = self.work_dir / ".jupyter" / "custom"
                 jupyter_custom_dir.mkdir(parents=True, exist_ok=True)
                 shutil.copy(jupyter_custom_css_path.resolve(), jupyter_custom_dir / "custom.css")

src/ai/backend/agent/kubernetes/kernel.py

@@ -4,10 +4,10 @@
 import os
 import shutil
 import textwrap
+from importlib.resources import files
 from pathlib import Path, PurePosixPath
 from typing import Any, Dict, FrozenSet, Mapping, Optional, Tuple, override
 
-import pkg_resources
 import zmq
 from aiodocker.docker import Docker
 from aiotools import TaskGroup
@@ -389,8 +389,10 @@ async def prepare_krunner_env_impl(
     arch = get_arch_name()
     current_version = int(
         Path(
-            pkg_resources.resource_filename(
-                f"ai.backend.krunner.{entrypoint_name}", f"./krunner-version.{distro}.txt"
+            str(
+                files(f"ai.backend.krunner.{entrypoint_name}").joinpath(
+                    f"./krunner-version.{distro}.txt"
+                )
             )
         )
         .read_text()
@@ -408,8 +410,8 @@ async def prepare_krunner_env_impl(
                 break
         else:
             log.info("preparing the Docker image for krunner extractor...")
-            extractor_archive = pkg_resources.resource_filename(
-                "ai.backend.runner", f"krunner-extractor.img.{arch}.tar.xz"
+            extractor_archive = str(
+                files("ai.backend.runner").joinpath(f"krunner-extractor.img.{arch}.tar.xz")
             )
             with lzma.open(extractor_archive, "rb") as reader:
                 proc = await asyncio.create_subprocess_exec(*["docker", "load"], stdin=reader)
@@ -422,12 +424,14 @@ async def prepare_krunner_env_impl(
             log.info("populating {} volume version {}", krunner_folder_name, current_version)
             target_path.mkdir(exist_ok=False)
             archive_path = Path(
-                pkg_resources.resource_filename(
-                    f"ai.backend.krunner.{entrypoint_name}", f"krunner-env.{distro}.{arch}.tar.xz"
+                str(
+                    files(f"ai.backend.krunner.{entrypoint_name}").joinpath(
+                        f"krunner-env.{distro}.{arch}.tar.xz"
+                    )
                 )
             ).resolve()
             extractor_path = Path(
-                pkg_resources.resource_filename("ai.backend.runner", "krunner-extractor.sh")
+                str(files("ai.backend.runner").joinpath("krunner-extractor.sh"))
             ).resolve()
 
             log.debug(
@@ -477,9 +481,9 @@ async def prepare_krunner_env_impl(
 
 
 async def copy_runner_files(scratch_path: Path) -> None:
-    artifact_path = Path(pkg_resources.resource_filename("ai.backend.agent", "../runner"))
-    kernel_path = Path(pkg_resources.resource_filename("ai.backend.agent", "../kernel"))
-    helpers_path = Path(pkg_resources.resource_filename("ai.backend.agent", "../helpers"))
+    artifact_path = Path(str(files("ai.backend.agent").joinpath("../runner")))
+    kernel_path = Path(str(files("ai.backend.agent").joinpath("../kernel")))
+    helpers_path = Path(str(files("ai.backend.agent").joinpath("../helpers")))
 
     destination_path = scratch_path
 
@@ -525,12 +529,7 @@ async def prepare_krunner_env(local_config: Mapping[str, Any]) -> Mapping[str, s
         plugin = entrypoint.load()
         await plugin.init({})  # currently does nothing
         provided_versions = (
-            Path(
-                pkg_resources.resource_filename(
-                    f"ai.backend.krunner.{entrypoint.name}",
-                    "versions.txt",
-                )
-            )
+            Path(str(files(f"ai.backend.krunner.{entrypoint.name}").joinpath("versions.txt")))
             .read_text()
             .splitlines()
         )

src/ai/backend/agent/kubernetes/utils.py

@@ -3,10 +3,10 @@
 import logging
 import subprocess
 from contextlib import closing
+from importlib.resources import files
 from pathlib import Path
 from typing import Any, Final, Mapping, Optional, Tuple
 
-import pkg_resources
 from aiodocker.docker import Docker
 from aiodocker.exceptions import DockerError
 
@@ -39,17 +39,15 @@ def __init__(
         self.container_config = container_config
         self.img_version = int(
             Path(
-                pkg_resources.resource_filename(
-                    "ai.backend.agent.docker",
-                    f"{default_container_name}.version.txt",
+                str(
+                    files("ai.backend.agent.docker").joinpath(
+                        f"{default_container_name}.version.txt"
+                    )
                 )
             ).read_text()
         )
         self.img_path = Path(
-            pkg_resources.resource_filename(
-                "ai.backend.agent.docker",
-                f"{default_container_name}.img.tar.gz",
-            )
+            str(files("ai.backend.agent.docker").joinpath(f"{default_container_name}.img.tar.gz"))
         )
 
     async def get_container_version_and_status(self) -> Tuple[int, bool]: