Add support for RAUC file parsing

Allow parsing of RAUC files using `PySquashFsImage`.  DB still needs migration.

goosebit/api/v1/software/routes.py

@@ -70,7 +70,7 @@ async def post_update(_: Request, file: UploadFile | None = File(None), url: str
         artifacts_dir = Path(config.artifacts_dir)
         file_path = artifacts_dir.joinpath(file.filename)
 
-        async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f:
+        async with aiofiles.tempfile.NamedTemporaryFile("w+b", delete_on_close=True) as f:
             await f.write(await file.read())
             absolute = await file_path.absolute()
             software = await create_software_update(absolute.as_uri(), Path(f.name))

goosebit/db/models.py

@@ -108,12 +108,28 @@ class Hardware(Model):
     revision = fields.CharField(max_length=255)
 
 
+class SoftwareImageFormat(IntEnum):
+    SWU = 0
+    RAUC = 1
+
+    def __str__(self):
+        return self.name.upper()
+
+    @classmethod
+    def from_str(cls, name):
+        try:
+            return cls[name.upper()]
+        except KeyError:
+            return cls.SWU
+
+
 class Software(Model):
     id = fields.IntField(primary_key=True)
     uri = fields.CharField(max_length=255)
     size = fields.BigIntField()
     hash = fields.CharField(max_length=255)
     version = fields.CharField(max_length=255)
+    image_format = fields.IntEnumField(SoftwareImageFormat, default=SoftwareImageFormat.SWU)
     compatibility = fields.ManyToManyField(
         "models.Hardware",
         related_name="softwares",

goosebit/ui/templates/software.html.jinja

@@ -53,7 +53,7 @@
                                         <div class="col">
                                             <input class="form-control"
                                                    type="file"
-                                                   accept=".swu"
+                                                   accept=".swu,.raucb"
                                                    id="file-upload"
                                                    name="file" />
                                         </div>

goosebit/updates/__init__.py

@@ -3,7 +3,7 @@
 from urllib.parse import unquote, urlparse
 from urllib.request import url2pathname
 
-from anyio import Path
+from anyio import Path, open_file
 from fastapi import HTTPException
 from fastapi.requests import Request
 from tortoise.expressions import Q
@@ -47,7 +47,16 @@ async def create_software_update(uri: str, temp_file: Path | None) -> Software:
         filename = Path(url2pathname(unquote(parsed_uri.path))).name
         path = Path(config.artifacts_dir).joinpath(update_info["hash"], filename)
         await path.parent.mkdir(parents=True, exist_ok=True)
-        await temp_file.rename(path)
+
+        # basically os.rename, but sync and async versions of that break across filesystems
+        async with await open_file(temp_file, "rb") as temp_file:
+            async with await open_file(path, "wb") as file:
+                while True:
+                    chunk = await temp_file.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    await file.write(chunk)
+
         absolute = await path.absolute()
         uri = absolute.as_uri()
 
@@ -57,6 +66,7 @@ async def create_software_update(uri: str, temp_file: Path | None) -> Software:
         version=str(update_info["version"]),
         size=update_info["size"],
         hash=update_info["hash"],
+        image_format=update_info["image_format"],
     )
 
     # create compatibility information

goosebit/updates/swdesc/__init__.py

@@ -0,0 +1,34 @@
+import logging
+
+import aiofiles
+import httpx
+from anyio import Path, open_file
+
+from ...db.models import SoftwareImageFormat
+from . import rauc, swu
+
+logger = logging.getLogger(__name__)
+
+
+async def parse_remote(url: str):
+    async with httpx.AsyncClient() as c:
+        file = await c.get(url)
+        async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f:
+            await f.write(file.content)
+            return await parse_file(Path(str(f.name)))
+
+
+async def parse_file(file: Path):
+    async with await open_file(file, "r+b") as f:
+        magic = await f.read(4)
+    if magic == swu.MAGIC:
+        image_format = SoftwareImageFormat.SWU
+        attributes = await swu.parse_file(file)
+    elif magic == rauc.MAGIC:
+        image_format = SoftwareImageFormat.RAUC
+        attributes = await rauc.parse_file(file)
+    else:
+        logger.warning(f"Unknown file format, magic={magic}")
+        raise ValueError(f"Unknown file format, magic={magic}")
+    attributes["image_format"] = image_format
+    return attributes

goosebit/updates/swdesc/func.py

@@ -0,0 +1,19 @@
+import hashlib
+
+from anyio import AsyncFile
+
+
+async def sha1_hash_file(fileobj: AsyncFile):
+    last = await fileobj.tell()
+    await fileobj.seek(0)
+    sha1_hash = hashlib.sha1()
+    buf = bytearray(2**18)
+    view = memoryview(buf)
+    while True:
+        size = await fileobj.readinto(buf)
+        if size == 0:
+            break
+        sha1_hash.update(view[:size])
+
+    await fileobj.seek(last)
+    return sha1_hash.hexdigest()

goosebit/updates/swdesc/rauc.py

@@ -0,0 +1,45 @@
+import configparser
+import logging
+
+import semver
+from anyio import Path, open_file
+from PySquashfsImage import SquashFsImage
+
+from .func import sha1_hash_file
+
+MAGIC = b"hsqs"
+
+logger = logging.getLogger(__name__)
+
+
+async def parse_file(file: Path):
+    async with await open_file(file, "r+b") as f:
+        image_data = await f.read()
+
+        image = SquashFsImage.from_bytes(image_data)
+        manifest = image.select("manifest.raucm")
+        manifest_str = manifest.read_bytes().decode("utf-8")
+        config = configparser.ConfigParser()
+        config.read_string(manifest_str)
+        swdesc_attrs = parse_descriptor(config)
+
+        stat = await file.stat()
+        swdesc_attrs["size"] = stat.st_size
+        swdesc_attrs["hash"] = await sha1_hash_file(f)
+    return swdesc_attrs
+
+
+def parse_descriptor(manifest: configparser.ConfigParser):
+    swdesc_attrs = {}
+    try:
+        swdesc_attrs["version"] = semver.Version.parse(manifest["system"].get("version"))
+        swdesc_attrs["compatibility"] = [{"hw_model": "default", "hw_revision": manifest["system"]["compatible"]}]
+    except KeyError:
+        try:
+            swdesc_attrs["version"] = semver.Version.parse(manifest["update"].get("version"))
+            swdesc_attrs["compatibility"] = [{"hw_model": "default", "hw_revision": manifest["update"]["compatible"]}]
+        except KeyError as e:
+            logger.warning(f"Parsing RAUC descriptor failed, error={e}")
+            raise ValueError("Parsing RAUC descriptor failed", e)
+
+    return swdesc_attrs

goosebit/updates/swdesc.py → goosebit/updates/swdesc/swu.py

@@ -1,15 +1,16 @@
-import hashlib
 import logging
 from typing import Any
 
-import aiofiles
-import httpx
 import libconf
 import semver
-from anyio import AsyncFile, Path, open_file
+from anyio import Path, open_file
+
+from .func import sha1_hash_file
 
 logger = logging.getLogger(__name__)
 
+MAGIC = b"0707"
+
 
 def _append_compatibility(boardname, value, compatibility):
     if "hardware-compatibility" in value:
@@ -34,7 +35,7 @@ def parse_descriptor(swdesc: libconf.AttrDict[Any, Any | None]):
 
         swdesc_attrs["compatibility"] = compatibility
     except KeyError as e:
-        logging.warning(f"Parsing swu descriptor failed, error={e}")
+        logger.warning(f"Parsing swu descriptor failed, error={e}")
         raise ValueError("Parsing swu descriptor failed", e)
 
     return swdesc_attrs
@@ -61,29 +62,6 @@ async def parse_file(file: Path):
         swdesc_attrs = parse_descriptor(swdesc)
         stat = await file.stat()
         swdesc_attrs["size"] = stat.st_size
-        swdesc_attrs["hash"] = await _sha1_hash_file(f)
-        return swdesc_attrs
+        swdesc_attrs["hash"] = await sha1_hash_file(f)
 
-
-async def parse_remote(url: str):
-    async with httpx.AsyncClient() as c:
-        file = await c.get(url)
-        async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f:
-            await f.write(file.content)
-            return await parse_file(Path(str(f.name)))
-
-
-async def _sha1_hash_file(fileobj: AsyncFile):
-    last = await fileobj.tell()
-    await fileobj.seek(0)
-    sha1_hash = hashlib.sha1()
-    buf = bytearray(2**18)
-    view = memoryview(buf)
-    while True:
-        size = await fileobj.readinto(buf)
-        if size == 0:
-            break
-        sha1_hash.update(view[:size])
-
-    await fileobj.seek(last)
-    return sha1_hash.hexdigest()
+        return swdesc_attrs

pyproject.toml

@@ -27,6 +27,7 @@ httpx = "^0.27.0"
 pydantic-settings = "^2.4.0"
 
 asyncpg = { version = "^0.29.0", optional = true }
+pysquashfsimage = "^0.9.0"
 
 [tool.poetry.extras]
 postgresql = ["asyncpg"]

tests/updates/test_swdesc.py → tests/updates/rauc/test_swdesc.py

@@ -2,7 +2,7 @@
 from anyio import Path
 from libconf import AttrDict
 
-from goosebit.updates.swdesc import parse_descriptor, parse_file
+from goosebit.updates.swdesc.swu import parse_descriptor, parse_file
 
 
 def test_parse_descriptor_simple():

tests/updates/swu/test_swdesc.py

@@ -0,0 +1,14 @@
+import pytest
+from anyio import Path
+
+from goosebit.updates.swdesc.rauc import parse_file
+
+
+@pytest.mark.asyncio
+async def test_parse_software_header():
+    resolved = await Path(__file__).resolve()
+    swdesc_attrs = await parse_file(resolved.parent / "software-header.raucb")
+    assert str(swdesc_attrs["version"]) == "8.8.1-11-g8c926e5+188370"
+    assert swdesc_attrs["compatibility"] == [
+        {"hw_model": "default", "hw_revision": "rauc-test-goosebit"},
+    ]