Merge pull request #1953 from ceph/newer-oses

Various updates to support newer OSes

bootstrap

@@ -187,7 +187,9 @@ if [ -f "$VENV/pyvenv.cfg" ]; then
     sed -i'' -e 's/\(include-system-site-packages\s*=\s*\)true/\1false/g' $VENV/pyvenv.cfg
 fi
 
-export LC_ALL=en_US.UTF-8
+# Attempt to force a UTF-8 locale without being specific to English
+export LANG=${LANG:-C.UTF-8}
+(echo $LANG | grep -qi utf-8) || export LC_ALL=$LANG.UTF-8
 
 if [ -z "$NO_CLOBBER" ] && \
   [ ! -e "./$VENV/bin/pip" -o ! -e "./$VENV/bin/$PYTHON" ] || \

containers/teuthology-dev/Dockerfile

@@ -1,5 +1,6 @@
-FROM ubuntu:focal
+FROM ubuntu:noble
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8
 RUN apt-get update && \
     apt-get install -y \
     git \
@@ -14,8 +15,10 @@ RUN apt-get update && \
     libvirt-dev \
     libffi-dev \
     libyaml-dev \
+    locales \
     lsb-release && \
-    apt-get clean all
+    apt-get clean all && \
+    locale-gen $LC_ALL
 WORKDIR /teuthology
 COPY requirements.txt requirements.yml ansible.cfg bootstrap /teuthology/
 RUN \

docs/docker-compose/testnode/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:focal
+FROM ubuntu:22.04
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt update && \
     apt -y install \

docs/docker-compose/testnode/testnode_start.sh

@@ -3,7 +3,11 @@ set -x
 echo "$SSH_PUBKEY" > /root/.ssh/authorized_keys
 echo "$SSH_PUBKEY" > /home/ubuntu/.ssh/authorized_keys
 chown ubuntu /home/ubuntu/.ssh/authorized_keys
-payload="{\"name\": \"$(hostname)\", \"machine_type\": \"testnode\", \"up\": true, \"locked\": false, \"os_type\": \"ubuntu\", \"os_version\": \"20.04\"}"
+. /etc/os-release
+if [ $ID = 'centos' ]; then
+    VERSION_ID=${VERSION_ID}.stream
+fi
+payload="{\"name\": \"$(hostname)\", \"machine_type\": \"testnode\", \"up\": true, \"locked\": false, \"os_type\": \"${ID}\", \"os_version\": \"${VERSION_ID}\"}"
 for i in $(seq 1 5); do
     echo "attempt $i"
     curl -v -f -d "$payload" http://paddles:8080/nodes/ && break

docs/docker-compose/teuthology/Dockerfile

@@ -1,6 +1,7 @@
-FROM ubuntu:latest
+FROM ubuntu:22.04
 ARG SSH_PRIVKEY_FILE=id_ed25519
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG=C.UTF-8
 RUN apt-get update && \
     apt-get install -y \
     git \
@@ -16,8 +17,10 @@ RUN apt-get update && \
     libvirt-dev \
     libffi-dev \
     libyaml-dev \
+    locales \
     lsb-release && \
-    apt-get clean all
+    apt-get clean all && \
+    locale-gen $LC_ALL
 WORKDIR /teuthology
 COPY requirements.txt requirements.yml ansible.cfg bootstrap /teuthology/
 RUN \

docs/docker-compose/teuthology/teuthology.sh

@@ -29,7 +29,7 @@ if [ -z "$TEUTHOLOGY_WAIT" ]; then
         -n 100 \
         --suite teuthology:no-ceph \
         --filter-out "libcephfs,kclient,stream,centos,rhel" \
-        -d ubuntu -D 20.04 \
+        -d ubuntu -D 22.04 \
         --suite-branch main \
         --subset 9000/100000 \
         -p 75 \

teuthology/orchestra/opsys.py

@@ -1,7 +1,11 @@
 import re
 
+from packaging.version import parse as parse_version, Version
+
+
 DISTRO_CODENAME_MAP = {
     "ubuntu": {
+        "24.04": "noble",
         "22.04": "jammy",
         "20.04": "focal",
         "18.04": "bionic",
@@ -31,6 +35,7 @@
         "6": "santiago",
     },
     "centos": {
+        "10": "stream",
         "9": "stream",
         "8": "core",
         "7": "core",
@@ -171,16 +176,18 @@ def from_os_release(cls, os_release_str):
             package_type = 'deb'
         """
         str_ = os_release_str.strip()
+        version = cls._get_value(str_, 'VERSION_ID')
         name = cls._get_value(str_, 'ID').lower()
         if name == 'sles':
             name = 'sle'
         elif name == 'opensuse-leap':
             name = 'opensuse'
         elif name == 'opensuse-tumbleweed':
             name = 'opensuse'
-        version = cls._get_value(str_, 'VERSION_ID')
+        elif name == 'centos':
+            if parse_version(version) >= Version("8.0"):
+                version = f"{version}.stream"
         obj = cls(name=name, version=version)
-
         return obj
 
 
@@ -244,7 +251,7 @@ def __repr__(self):
                     codename=repr(self.codename))
 
     def __eq__(self, other):
-        for slot in self.__slots__:
-            if not getattr(self, slot) == getattr(other, slot):
-                return False
-        return True
+        if self.name.lower() != other.name.lower():
+            return False
+        normalize = lambda s: s.lower().removesuffix(".stream")
+        return normalize(self.version) == normalize(other.version)

teuthology/orchestra/test/test_opsys.py

@@ -4,6 +4,23 @@
 
 
 class TestOS(object):
+    str_centos_9_os_release = dedent("""
+        NAME="CentOS Stream"
+        VERSION="9"
+        ID="centos"
+        ID_LIKE="rhel fedora"
+        VERSION_ID="9"
+        PLATFORM_ID="platform:el9"
+        PRETTY_NAME="CentOS Stream 9"
+        ANSI_COLOR="0;31"
+        LOGO="fedora-logo-icon"
+        CPE_NAME="cpe:/o:centos:centos:9"
+        HOME_URL="https://centos.org/"
+        BUG_REPORT_URL="https://issues.redhat.com/"
+        REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9"
+        REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream"
+    """)
+
     str_centos_7_os_release = dedent("""
         NAME="CentOS Linux"
         VERSION="7 (Core)"
@@ -235,6 +252,13 @@ class TestOS(object):
         HOME_URL="https://www.opensuse.org/"
     """)
 
+    def test_centos_9_os_release(self):
+        os = OS.from_os_release(self.str_centos_9_os_release)
+        assert os.name == 'centos'
+        assert os.version == '9.stream'
+        assert os.codename == 'stream'
+        assert os.package_type == 'rpm'
+
     def test_centos_7_os_release(self):
         os = OS.from_os_release(self.str_centos_7_os_release)
         assert os.name == 'centos'

teuthology/provision/fog.py

@@ -13,6 +13,7 @@
 from teuthology.config import config
 from teuthology.contextutil import safe_while
 from teuthology.exceptions import MaxWhileTries
+from teuthology.orchestra.opsys import OS
 from teuthology import misc
 
 log = logging.getLogger(__name__)
@@ -148,18 +149,28 @@ def get_image_data(self):
         represents it
         :returns: A dict describing the image
         """
-        name = '_'.join([
-            self.remote.machine_type, self.os_type.lower(), self.os_version])
-        resp = self.do_request(
-            '/image',
-            data=json.dumps(dict(name=name)),
-        )
-        obj = resp.json()
-        if not obj['count']:
+        def do_get(name):
+            resp = self.do_request(
+                '/image',
+                data=json.dumps(dict(name=name)),
+            )
+            obj = resp.json()
+            if obj['count']:
+                return obj['images'][0]
+
+        os_type = self.os_type.lower()
+        os_version = self.os_version
+        name = f"{self.remote.machine_type}_{os_type}_{os_version}"
+        if image := do_get(name):
+            return image
+        elif os_type == 'centos' and not os_version.endswith('.stream'):
+            image = do_get(f"{name}.stream")
+        if image:
+            return image
+        else:
             raise RuntimeError(
                 "Fog has no %s image. Available %s images: %s" %
                 (name, self.remote.machine_type, self.suggest_image_names()))
-        return obj['images'][0]
 
     def suggest_image_names(self):
         """
@@ -337,16 +348,11 @@ def _fix_hostname(self):
         )
 
     def _verify_installed_os(self):
-        # What we call "CentOS X.Stream", we will see as "CentOS X"
-        os_version = self.os_version.lower()
-        # When we drop support for python 3.8, str.removesuffix() is helpful
-        if os_version.endswith(".stream"):
-            os_version = os_version[:-len(".stream")]
-        if self.remote.os.name.lower() != self.os_type.lower() or \
-                self.remote.os.version.lower() != os_version:
+        wanted_os = OS(name=self.os_type, version=self.os_version)
+        if self.remote.os != wanted_os:
             raise RuntimeError(
-                f"Expected {self.remote.shortname}'s OS to be {self.os_type} {os_version} but "
-                f"found {self.remote.os.name} {self.remote.os.version}"
+                f"Expected {self.remote.shortname}'s OS to be {wanted_os} but "
+                f"found {self.remote.os}"
             )
 
     def destroy(self):