feat: Backwards compatibility (#17)

- Fix docker compose for launching tests via pycharm.
- Make a constant for `RUN_ENVIRONMENT`.
- Tiberian sun backwards compatibility
- Add random red alert map that I found on cncnet db.
- Red Alert backwards compatible upload
- Stop using `ZipStorage`. Revisit it at a later time.
- Add support for dta
- Dune 2000 wip
- Fix download endpoint for backwards compatible client
- Make sure tests end urls in `.zip` for backwards compatible uploads
- Fix tests for filtering on `is_backwards_compatible`
- Stop crashing when optional files are missing
- Fix tests from rebase
- Dune 2000 mis
- Tiberian dawn validator
- Add test for tiberian dawn
- Tidy up the config parser usage

Author: alexlambson

docker-compose.yml

@@ -43,3 +43,22 @@ services:
       - "80:80"
     depends_on:
       - django
+
+  debugger:
+    # Use this for launching via a debugger like PyCharm or VSCode.
+    # Just builds, and doesn't execute anything, your debugger will be in charge of executing.
+    build:
+      context: .
+      dockerfile: docker/Dockerfile
+      target: debugger
+    volumes:
+      - .:/cncnet-map-api
+      - ./data/tmp:/tmp/pytest-of-root  # For inspecting files during pytests
+    env_file:
+      - .env
+    environment:
+      POSTGRES_TEST_HOST: mapdb-postgres-dev  # Necessary to connect to docker db. Overrides the .env setting.
+#    ports:
+#      - "80:80"
+    depends_on:
+      - db

docker/Dockerfile

@@ -31,3 +31,6 @@ COPY . /cncnet-map-api
 
 ENTRYPOINT "/cncnet-map-api/web_entry_point.sh"
 
+FROM base AS debugger
+# Just build, but don't run anything. Your debugger will run pytest, manage.py, etc for you.
+RUN CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r ./requirements-dev.txt

example.env

@@ -45,3 +45,6 @@ TESTING_API_USERNAME=
 
 # Required for pytest. An account password to use for tests that need a JWT. DO NOT COMMIT
 TESTING_API_PASSWORD=
+
+# Options are in `kirovy.constants.RunEnvironment`
+RUN_ENVIRONMENT=

kirovy/constants/api_codes.py

@@ -19,3 +19,4 @@ class LegacyUploadApiCodes(enum.StrEnum):
     HASH_MISMATCH = "file-hash-does-not-match-zip-name"
     INVALID_FILE_TYPE = "invalid-file-type-in-zip"
     GAME_NOT_SUPPORTED = "game-not-supported"
+    MAP_FAILED_TO_PARSE = "map-failed-to-parse"

kirovy/migrations/0013_cncmap_is_mapdb1_compatible_alter_cncmapfile_file.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.20 on 2025-03-23 23:32
+
+from django.db import migrations, models
+import kirovy.models.file_base
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("kirovy", "0012_raw_zip_extension"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="cncmap",
+            name="is_mapdb1_compatible",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="cncmapfile",
+            name="file",
+            field=models.FileField(upload_to=kirovy.models.file_base._generate_upload_to),
+        ),
+    ]

kirovy/models/cnc_map.py

@@ -1,4 +1,5 @@
 import pathlib
+from uuid import UUID
 
 from django.conf import settings
 from django.db import models
@@ -117,6 +118,12 @@ class CncMap(cnc_user.CncNetUserOwnedModel):
     )
     """If set, then this map is a child of ``parent``. Used to track edits of other peoples' maps."""
 
+    is_mapdb1_compatible = models.BooleanField(default=False)
+    """If true, then this map was uploaded by a legacy CnCNet client and is backwards compatible with map db 1.0.
+
+    This should never be set for maps uploaded via the web UI.
+    """
+
     def next_version_number(self) -> int:
         """Generate the next version to use for a map file.
 
@@ -158,14 +165,26 @@ def set_ban(self, is_banned: bool, banned_by: cnc_user.CncUser) -> None:
         self.save(update_fields=["is_banned"])
 
 
-class CncMapFile(file_base.CncNetZippedFileBaseModel):
+class CncMapFileManager(models.Manager["CncMapFile"]):
+    def find_legacy_map_by_sha1(self, sha1: str, game_id: UUID) -> t.Union["CncMapFile", None]:
+        return (
+            super()
+            .get_queryset()
+            .filter(hash_sha1=sha1, cnc_game_id=game_id, cnc_map__is_mapdb1_compatible=True)
+            .first()
+        )
+
+
+class CncMapFile(file_base.CncNetFileBaseModel):
     """Represents the actual map file that a Command & Conquer game reads.
 
     .. warning::
 
         ``name`` is auto-generated for this file subclass.
     """
 
+    objects = CncMapFileManager()
+
     width = models.IntegerField()
     height = models.IntegerField()
     version = models.IntegerField(editable=False)

kirovy/models/cnc_user.py

@@ -19,7 +19,7 @@ class CncUserManager(models.Manager):
         constants.LegacyUploadUser.CNCNET_ID,
     }
 
-    def find_by_cncnet_id(self, cncnet_id: int) -> t.Tuple["CncUser"]:
+    def find_by_cncnet_id(self, cncnet_id: int) -> t.Union["CncUser", None]:
         return super().get_queryset().filter(cncnet_id=cncnet_id).first()
 
     def get_or_create_migration_user(self) -> "CncUser":

kirovy/services/cnc_gen_2_services.py

@@ -35,11 +35,39 @@ class CncGen2MapSections(enum.StrEnum):
     DIGEST = "Digest"
 
 
+class ParseErrorMsg(enum.StrEnum):
+    NO_BINARY = _("Binary files not allowed.")
+    CORRUPT_MAP = _("Could not parse map file.")
+    MISSING_INI = _("Missing necessary INI sections.")
+
+
 class MapConfigParser(configparser.ConfigParser):
     """Config parser with some helpers."""
 
     NAME_NOT_FOUND: str = "Map name not found in file"
 
+    @classmethod
+    def from_file(cls, file: File) -> "MapConfigParser":
+        parser = cls()
+        parser.read_django_file(file)
+        return parser
+
+    def read_django_file(self, file: File):
+        if file.closed:
+            file.open("r")
+        file.seek(0)
+        try:
+            # We can't use ConfigParser.read_file because parser expects the file to be read as a string,
+            # but django uploaded files are read as bytes. So we need to convert to string first.
+            # If `decode` is crashing in a test, make sure your test file is read in read-mode "rb".
+            self.read_string(file.read().decode())
+        except configparser.ParsingError as e:
+            raise exceptions.InvalidMapFile(
+                ParseErrorMsg.CORRUPT_MAP,
+                code=ParseErrorMsg.CORRUPT_MAP.name,
+                params={"e": e},
+            )
+
     def optionxform(self, optionstr: str) -> str:
         """Overwrite the base class to prevent lower-casing keys."""
         return optionstr
@@ -93,11 +121,6 @@ class CncGen2MapParser:
         CncGen2MapSections.DIGEST.value,
     }
 
-    class ErrorMsg(enum.StrEnum):
-        NO_BINARY = _("Binary files not allowed.")
-        CORRUPT_MAP = _("Could not parse map file.")
-        MISSING_INI = _("Missing necessary INI sections.")
-
     def __init__(self, uploaded_file: UploadedFile | File):
         self.validate_file_type(uploaded_file)
         self.file = uploaded_file
@@ -114,27 +137,13 @@ def _parse_file(self) -> None:
         :return:
             Nothing, but :attr:`~kirovy.services.MapParserService.parser` will be modified.
         """
-        if self.file.closed:
-            self.file.open("r")
-
-        try:
-            # We can't use read_file because parser expects the file to be read as a string,
-            # but django uploaded files are read as bytes. So we need to convert to string first.
-            # If `decode` is crashing in a test, make sure your test file is read in read-mode "rb".
-            self.ini.read_string(self.file.read().decode())
-        except configparser.ParsingError as e:
-            raise exceptions.InvalidMapFile(
-                self.ErrorMsg.CORRUPT_MAP,
-                code=self.ErrorMsg.CORRUPT_MAP.name,
-                params={"e": e},
-            )
-
+        self.ini.read_django_file(self.file)
         sections: t.Set[str] = set(self.ini.sections())
         missing_sections = self.required_sections - sections
         if missing_sections:
             raise exceptions.InvalidMapFile(
-                self.ErrorMsg.MISSING_INI,
-                code=self.ErrorMsg.MISSING_INI.name,
+                ParseErrorMsg.MISSING_INI,
+                code=ParseErrorMsg.MISSING_INI.name,
                 params={"missing": missing_sections},
             )
 
@@ -156,7 +165,7 @@ def validate_file_type(self, uploaded_file: File) -> None:
             Raised if file is binary.
         """
         if self.is_binary(uploaded_file):
-            raise exceptions.InvalidMimeType(self.ErrorMsg.NO_BINARY)
+            raise exceptions.InvalidMimeType(ParseErrorMsg.NO_BINARY)
 
     @classmethod
     def is_binary(cls, uploaded_file: File) -> bool:

kirovy/services/legacy_upload/__init__.py

@@ -1,14 +1,18 @@
 from kirovy.constants import GameSlugs
 from kirovy.constants.api_codes import LegacyUploadApiCodes
 from kirovy.exceptions import view_exceptions
-from kirovy.services.legacy_upload import westwood, dune_2000
+from kirovy.services.legacy_upload import westwood, dune_2000, tiberian_dawn
 from kirovy.services.legacy_upload.base import LegacyMapServiceBase
 from kirovy import typing as t
 
 
 _GAME_LEGACY_SERVICE_MAP: t.Dict[str, t.Type[LegacyMapServiceBase]] = {
     GameSlugs.yuris_revenge.value: westwood.YurisRevengeLegacyMapService,
     GameSlugs.dune_2000.value: dune_2000.Dune2000LegacyMapService,
+    GameSlugs.tiberian_sun.value: westwood.TiberianSunLegacyMapService,
+    GameSlugs.red_alert.value: westwood.RedAlertLegacyMapService,
+    GameSlugs.dawn_of_the_tiberium_age.value: westwood.DawnOfTheTiberiumAgeLegacyMapService,
+    GameSlugs.tiberian_dawn.value: tiberian_dawn.TiberianDawnLegacyMapService,
 }
 
 

kirovy/services/legacy_upload/base.py

@@ -1,5 +1,4 @@
 import dataclasses
-import functools
 import io
 import pathlib
 import zipfile
@@ -8,18 +7,18 @@
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import UploadedFile
 
-from kirovy import typing as t, constants
+from kirovy import typing as t, constants, exceptions
 from kirovy.constants.api_codes import LegacyUploadApiCodes
 from kirovy.exceptions import view_exceptions
-from kirovy.services.cnc_gen_2_services import CncGen2MapParser
+from kirovy.services.cnc_gen_2_services import MapConfigParser, CncGen2MapParser
 from kirovy.utils import file_utils
 from kirovy.utils.file_utils import ByteSized
 
 
 @dataclasses.dataclass
 class ExpectedFile:
     possible_extensions: t.Set[str]
-    file_validator: t.Callable[[str, ContentFile, zipfile.ZipInfo], bool]
+    file_validator: t.Callable[[str, ContentFile, zipfile.ZipInfo], None]
     required: bool = True
     """attr: If false, this file is not required to be present."""
 
@@ -53,15 +52,15 @@ def expected_files(self) -> t.List[ExpectedFile]:
         raise NotImplementedError("This Game's map validator hasn't implemented the expectd file structure.")
 
     def multi_file_validator(self):
-        file_list = self._file.infolist()
+        zip_file_list = self._file.infolist()
         min_files = len([x for x in self.expected_files if x.required])
         max_files = len(self.expected_files)
-        if min_files > len(file_list) > max_files:
+        if min_files > len(zip_file_list) > max_files:
             raise view_exceptions.KirovyValidationError(
                 "Incorrect file count", code=LegacyUploadApiCodes.BAD_ZIP_STRUCTURE
             )
 
-        for file_info in file_list:
+        for file_info in zip_file_list:
             expected_file = self._get_expected_file_for_extension(file_info)
             expected_file.file_validator(self._file.filename, ContentFile(self._file.read(file_info)), file_info)
 
@@ -101,8 +100,11 @@ def file_contents_merged(self) -> io.BytesIO:
         """
         output = io.BytesIO()
         for expected_file in self.expected_files:
-            file_info = self._find_file_info_by_extension(expected_file.possible_extensions)
-            output.write(self._file.read(file_info))
+            file_info = self._find_file_info_by_extension(
+                expected_file.possible_extensions, is_required=expected_file.required
+            )
+            if file_info:
+                output.write(self._file.read(file_info))
         output.seek(0)
         return output
 
@@ -111,9 +113,13 @@ def map_name(self) -> str:
         ini_file_info = self._find_file_info_by_extension(self.ini_extensions)
         fallback = f"legacy_client_upload_{self.map_sha1_from_filename}"
         ini_file = ContentFile(self._file.read(ini_file_info))
-        return CncGen2MapParser(ini_file).ini.get("Basic", "Name", fallback=fallback)
+        try:
+            return MapConfigParser.from_file(ini_file).get("Basic", "Name", fallback=fallback)
+        except exceptions.InvalidMapFile:
+            # Having a bad map name for a temporary upload is better than a 500 error.
+            return fallback
 
-    def _find_file_info_by_extension(self, extensions: t.Set[str]) -> zipfile.ZipInfo:
+    def _find_file_info_by_extension(self, extensions: t.Set[str], is_required: bool = True) -> zipfile.ZipInfo | None:
         """Find the zipinfo object for a file by a set of possible file extensions.
 
         This is meant to be used to find specific files in the zip.
@@ -132,11 +138,12 @@ def _find_file_info_by_extension(self, extensions: t.Set[str]) -> zipfile.ZipInf
         for file in self._file.infolist():
             if pathlib.Path(file.filename).suffix in extensions:
                 return file
-        raise view_exceptions.KirovyValidationError(
-            "No file matching the expected extensions was found",
-            LegacyUploadApiCodes.NO_VALID_MAP_FILE,
-            {"expected": extensions},
-        )
+        if is_required:
+            raise view_exceptions.KirovyValidationError(
+                "No file matching the expected extensions was found",
+                LegacyUploadApiCodes.NO_VALID_MAP_FILE,
+                {"expected": extensions},
+            )
 
     def _get_expected_file_for_extension(self, zip_info: zipfile.ZipInfo) -> ExpectedFile:
         """Get the ``expected_file`` class instance corresponding to the file in the zipfile.