From 31ec99d0559a6a03c978c93d63b3f23f67b09365 Mon Sep 17 00:00:00 2001 From: Upstream Data Date: Tue, 10 Sep 2024 12:15:30 -0600 Subject: [PATCH 1/2] Add support for RAUC file parsing Allow parsing of RAUC files using `PySquashFsImage`. DB still needs migration. --- goosebit/api/v1/software/routes.py | 2 +- goosebit/db/models.py | 16 +++++++ goosebit/ui/templates/software.html.jinja | 2 +- goosebit/updates/__init__.py | 14 +++++- goosebit/updates/swdesc/__init__.py | 34 +++++++++++++ goosebit/updates/swdesc/func.py | 19 ++++++++ goosebit/updates/swdesc/rauc.py | 45 ++++++++++++++++++ goosebit/updates/{swdesc.py => swdesc/swu.py} | 38 ++++----------- pyproject.toml | 1 + tests/updates/rauc/__init__.py | 0 tests/updates/{ => rauc}/software-header.swu | Bin tests/updates/{ => rauc}/test_swdesc.py | 2 +- tests/updates/swu/__init__.py | 0 tests/updates/swu/software-header.raucb | Bin 0 -> 4096 bytes tests/updates/swu/test_swdesc.py | 14 ++++++ 15 files changed, 152 insertions(+), 35 deletions(-) create mode 100644 goosebit/updates/swdesc/__init__.py create mode 100644 goosebit/updates/swdesc/func.py create mode 100644 goosebit/updates/swdesc/rauc.py rename goosebit/updates/{swdesc.py => swdesc/swu.py} (67%) create mode 100644 tests/updates/rauc/__init__.py rename tests/updates/{ => rauc}/software-header.swu (100%) rename tests/updates/{ => rauc}/test_swdesc.py (98%) create mode 100644 tests/updates/swu/__init__.py create mode 100644 tests/updates/swu/software-header.raucb create mode 100644 tests/updates/swu/test_swdesc.py diff --git a/goosebit/api/v1/software/routes.py b/goosebit/api/v1/software/routes.py index 10dd3462..5fa9a36a 100644 --- a/goosebit/api/v1/software/routes.py +++ b/goosebit/api/v1/software/routes.py @@ -70,7 +70,7 @@ async def post_update(_: Request, file: UploadFile | None = File(None), url: str artifacts_dir = Path(config.artifacts_dir) file_path = artifacts_dir.joinpath(file.filename) - async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f: + async with aiofiles.tempfile.NamedTemporaryFile("w+b", delete_on_close=True) as f: await f.write(await file.read()) absolute = await file_path.absolute() software = await create_software_update(absolute.as_uri(), Path(f.name)) diff --git a/goosebit/db/models.py b/goosebit/db/models.py index 979f800a..8d9d363e 100644 --- a/goosebit/db/models.py +++ b/goosebit/db/models.py @@ -108,12 +108,28 @@ class Hardware(Model): revision = fields.CharField(max_length=255) +class SoftwareImageFormat(IntEnum): + SWU = 0 + RAUC = 1 + + def __str__(self): + return self.name.upper() + + @classmethod + def from_str(cls, name): + try: + return cls[name.upper()] + except KeyError: + return cls.SWU + + class Software(Model): id = fields.IntField(primary_key=True) uri = fields.CharField(max_length=255) size = fields.BigIntField() hash = fields.CharField(max_length=255) version = fields.CharField(max_length=255) + image_format = fields.IntEnumField(SoftwareImageFormat, default=SoftwareImageFormat.SWU) compatibility = fields.ManyToManyField( "models.Hardware", related_name="softwares", diff --git a/goosebit/ui/templates/software.html.jinja b/goosebit/ui/templates/software.html.jinja index e664527f..9b813a05 100644 --- a/goosebit/ui/templates/software.html.jinja +++ b/goosebit/ui/templates/software.html.jinja @@ -53,7 +53,7 @@
diff --git a/goosebit/updates/__init__.py b/goosebit/updates/__init__.py index 35c35dbf..ab2c6626 100644 --- a/goosebit/updates/__init__.py +++ b/goosebit/updates/__init__.py @@ -3,7 +3,7 @@ from urllib.parse import unquote, urlparse from urllib.request import url2pathname -from anyio import Path +from anyio import Path, open_file from fastapi import HTTPException from fastapi.requests import Request from tortoise.expressions import Q @@ -47,7 +47,16 @@ async def create_software_update(uri: str, temp_file: Path | None) -> Software: filename = Path(url2pathname(unquote(parsed_uri.path))).name path = Path(config.artifacts_dir).joinpath(update_info["hash"], filename) await path.parent.mkdir(parents=True, exist_ok=True) - await temp_file.rename(path) + + # basically os.rename, but sync and async versions of that break across filesystems + async with await open_file(temp_file, "rb") as temp_file: + async with await open_file(path, "wb") as file: + while True: + chunk = await temp_file.read(1024 * 1024) + if not chunk: + break + await file.write(chunk) + absolute = await path.absolute() uri = absolute.as_uri() @@ -57,6 +66,7 @@ async def create_software_update(uri: str, temp_file: Path | None) -> Software: version=str(update_info["version"]), size=update_info["size"], hash=update_info["hash"], + image_format=update_info["image_format"], ) # create compatibility information diff --git a/goosebit/updates/swdesc/__init__.py b/goosebit/updates/swdesc/__init__.py new file mode 100644 index 00000000..ce2b1af5 --- /dev/null +++ b/goosebit/updates/swdesc/__init__.py @@ -0,0 +1,34 @@ +import logging + +import aiofiles +import httpx +from anyio import Path, open_file + +from ...db.models import SoftwareImageFormat +from . import rauc, swu + +logger = logging.getLogger(__name__) + + +async def parse_remote(url: str): + async with httpx.AsyncClient() as c: + file = await c.get(url) + async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f: + await f.write(file.content) + return await parse_file(Path(str(f.name))) + + +async def parse_file(file: Path): + async with await open_file(file, "r+b") as f: + magic = await f.read(4) + if magic == swu.MAGIC: + image_format = SoftwareImageFormat.SWU + attributes = await swu.parse_file(file) + elif magic == rauc.MAGIC: + image_format = SoftwareImageFormat.RAUC + attributes = await rauc.parse_file(file) + else: + logger.warning(f"Unknown file format, magic={magic}") + raise ValueError(f"Unknown file format, magic={magic}") + attributes["image_format"] = image_format + return attributes diff --git a/goosebit/updates/swdesc/func.py b/goosebit/updates/swdesc/func.py new file mode 100644 index 00000000..779ea468 --- /dev/null +++ b/goosebit/updates/swdesc/func.py @@ -0,0 +1,19 @@ +import hashlib + +from anyio import AsyncFile + + +async def sha1_hash_file(fileobj: AsyncFile): + last = await fileobj.tell() + await fileobj.seek(0) + sha1_hash = hashlib.sha1() + buf = bytearray(2**18) + view = memoryview(buf) + while True: + size = await fileobj.readinto(buf) + if size == 0: + break + sha1_hash.update(view[:size]) + + await fileobj.seek(last) + return sha1_hash.hexdigest() diff --git a/goosebit/updates/swdesc/rauc.py b/goosebit/updates/swdesc/rauc.py new file mode 100644 index 00000000..de0eff19 --- /dev/null +++ b/goosebit/updates/swdesc/rauc.py @@ -0,0 +1,45 @@ +import configparser +import logging + +import semver +from anyio import Path, open_file +from PySquashfsImage import SquashFsImage + +from .func import sha1_hash_file + +MAGIC = b"hsqs" + +logger = logging.getLogger(__name__) + + +async def parse_file(file: Path): + async with await open_file(file, "r+b") as f: + image_data = await f.read() + + image = SquashFsImage.from_bytes(image_data) + manifest = image.select("manifest.raucm") + manifest_str = manifest.read_bytes().decode("utf-8") + config = configparser.ConfigParser() + config.read_string(manifest_str) + swdesc_attrs = parse_descriptor(config) + + stat = await file.stat() + swdesc_attrs["size"] = stat.st_size + swdesc_attrs["hash"] = await sha1_hash_file(f) + return swdesc_attrs + + +def parse_descriptor(manifest: configparser.ConfigParser): + swdesc_attrs = {} + try: + swdesc_attrs["version"] = semver.Version.parse(manifest["system"].get("version")) + swdesc_attrs["compatibility"] = [{"hw_model": "default", "hw_revision": manifest["system"]["compatible"]}] + except KeyError: + try: + swdesc_attrs["version"] = semver.Version.parse(manifest["update"].get("version")) + swdesc_attrs["compatibility"] = [{"hw_model": "default", "hw_revision": manifest["update"]["compatible"]}] + except KeyError as e: + logger.warning(f"Parsing RAUC descriptor failed, error={e}") + raise ValueError("Parsing RAUC descriptor failed", e) + + return swdesc_attrs diff --git a/goosebit/updates/swdesc.py b/goosebit/updates/swdesc/swu.py similarity index 67% rename from goosebit/updates/swdesc.py rename to goosebit/updates/swdesc/swu.py index c6e5685c..3caa728d 100644 --- a/goosebit/updates/swdesc.py +++ b/goosebit/updates/swdesc/swu.py @@ -1,15 +1,16 @@ -import hashlib import logging from typing import Any -import aiofiles -import httpx import libconf import semver -from anyio import AsyncFile, Path, open_file +from anyio import Path, open_file + +from .func import sha1_hash_file logger = logging.getLogger(__name__) +MAGIC = b"0707" + def _append_compatibility(boardname, value, compatibility): if "hardware-compatibility" in value: @@ -34,7 +35,7 @@ def parse_descriptor(swdesc: libconf.AttrDict[Any, Any | None]): swdesc_attrs["compatibility"] = compatibility except KeyError as e: - logging.warning(f"Parsing swu descriptor failed, error={e}") + logger.warning(f"Parsing swu descriptor failed, error={e}") raise ValueError("Parsing swu descriptor failed", e) return swdesc_attrs @@ -61,29 +62,6 @@ async def parse_file(file: Path): swdesc_attrs = parse_descriptor(swdesc) stat = await file.stat() swdesc_attrs["size"] = stat.st_size - swdesc_attrs["hash"] = await _sha1_hash_file(f) - return swdesc_attrs + swdesc_attrs["hash"] = await sha1_hash_file(f) - -async def parse_remote(url: str): - async with httpx.AsyncClient() as c: - file = await c.get(url) - async with aiofiles.tempfile.NamedTemporaryFile("w+b") as f: - await f.write(file.content) - return await parse_file(Path(str(f.name))) - - -async def _sha1_hash_file(fileobj: AsyncFile): - last = await fileobj.tell() - await fileobj.seek(0) - sha1_hash = hashlib.sha1() - buf = bytearray(2**18) - view = memoryview(buf) - while True: - size = await fileobj.readinto(buf) - if size == 0: - break - sha1_hash.update(view[:size]) - - await fileobj.seek(last) - return sha1_hash.hexdigest() + return swdesc_attrs diff --git a/pyproject.toml b/pyproject.toml index d6632b96..a2145ae4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ httpx = "^0.27.0" pydantic-settings = "^2.4.0" asyncpg = { version = "^0.29.0", optional = true } +pysquashfsimage = "^0.9.0" [tool.poetry.extras] postgresql = ["asyncpg"] diff --git a/tests/updates/rauc/__init__.py b/tests/updates/rauc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/updates/software-header.swu b/tests/updates/rauc/software-header.swu similarity index 100% rename from tests/updates/software-header.swu rename to tests/updates/rauc/software-header.swu diff --git a/tests/updates/test_swdesc.py b/tests/updates/rauc/test_swdesc.py similarity index 98% rename from tests/updates/test_swdesc.py rename to tests/updates/rauc/test_swdesc.py index 3649cdb5..013801c4 100644 --- a/tests/updates/test_swdesc.py +++ b/tests/updates/rauc/test_swdesc.py @@ -2,7 +2,7 @@ from anyio import Path from libconf import AttrDict -from goosebit.updates.swdesc import parse_descriptor, parse_file +from goosebit.updates.swdesc.swu import parse_descriptor, parse_file def test_parse_descriptor_simple(): diff --git a/tests/updates/swu/__init__.py b/tests/updates/swu/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/updates/swu/software-header.raucb b/tests/updates/swu/software-header.raucb new file mode 100644 index 0000000000000000000000000000000000000000..25d6188ba193be6c62d69c3d8a5c01f3367bce73 GIT binary patch literal 4096 zcmc~OE-YqZU|`s`?qM1O0}}%ykj=;-$Z!BivH;l%Knw!LAPFGQhSL9`UFDCh;*!+dSgz#!+=9fC%%q%D+oHtMWZja~;u78T{QTn7q|6eovecsD%=|oC z3q1=xLtR5d-E@m&OCz&XQ*A>F3uAKwE_H^ATggcghfXlCNME=0VK$!7z%F2Hw5Rp7 zvExz4Em{J`%!UlI+UyYuAcYAH3~m}{j~qBXVfKvaGp9wxEDE0zF>4MtgLXdaI)R1+ zsAD}qeg%ThP(B}2FRKPG1ET Date: Wed, 11 Sep 2024 08:16:18 -0600 Subject: [PATCH 2/2] Migrate DB schemas Adds migration for DB to add new values. --- .../models/1_20240911081506_add_image_format.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 goosebit/db/migrations/models/1_20240911081506_add_image_format.py diff --git a/goosebit/db/migrations/models/1_20240911081506_add_image_format.py b/goosebit/db/migrations/models/1_20240911081506_add_image_format.py new file mode 100644 index 00000000..731e341c --- /dev/null +++ b/goosebit/db/migrations/models/1_20240911081506_add_image_format.py @@ -0,0 +1,11 @@ +from tortoise import BaseDBAsyncClient + + +async def upgrade(db: BaseDBAsyncClient) -> str: + return """ + ALTER TABLE "software" ADD "image_format" SMALLINT NOT NULL DEFAULT 0 /* SWU: 0\nRAUC: 1 */;""" + + +async def downgrade(db: BaseDBAsyncClient) -> str: + return """ + ALTER TABLE "software" DROP COLUMN "image_format";"""