Add server and other things

Author: mxsasha

irrd/conf/__init__.py

@@ -244,10 +244,7 @@ def _check_staging_config(self) -> List[str]:
         Validate the current staging configuration.
         Returns a list of any errors, or an empty list for a valid config.
         """
-        from irrd.utils.crypto import (
-            ed25519_private_key_from_str,
-            ed25519_public_key_from_str,
-        )
+        from irrd.utils.crypto import eckey_from_str
 
         errors = []
         config = self.user_config_staging
@@ -433,7 +430,7 @@ def _validate_subconfig(key, value):
 
             if details.get("nrtm4_client_initial_public_key"):
                 try:
-                    ed25519_public_key_from_str(details["nrtm4_client_initial_public_key"])
+                    eckey_from_str(details["nrtm4_client_initial_public_key"])
                 except ValueError as ve:
                     errors.append(
                         f"Invalid value for setting nrtm4_client_initial_public_key for source {name}: {ve}"
@@ -469,15 +466,15 @@ def _validate_subconfig(key, value):
 
             if details.get("nrtm4_server_private_key"):
                 try:
-                    ed25519_private_key_from_str(details["nrtm4_server_private_key"])
+                    eckey_from_str(details["nrtm4_server_private_key"], require_private=True)
                 except ValueError as ve:
                     errors.append(
                         f"Invalid value for setting nrtm4_server_private_key for source {name}: {ve}"
                     )
 
             if details.get("nrtm4_server_private_key_next"):
                 try:
-                    ed25519_private_key_from_str(details["nrtm4_server_private_key_next"])
+                    eckey_from_str(details["nrtm4_server_private_key_next"], require_private=True)
                 except ValueError as ve:
                     errors.append(
                         f"Invalid value for setting nrtm4_server_private_key_next for source {name}: {ve}"

irrd/conf/test_conf.py

@@ -6,6 +6,12 @@
 import pytest
 import yaml
 
+from ..mirroring.nrtm4.tests import (
+    MOCK_UNF_PRIVATE_KEY,
+    MOCK_UNF_PRIVATE_KEY_OTHER_STR,
+    MOCK_UNF_PRIVATE_KEY_STR,
+)
+from ..utils.crypto import eckey_public_key_as_str
 from . import (
     ConfigurationError,
     config_init,
@@ -119,10 +125,10 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                     "TESTDB-NRTM4": {
                         "keep_journal": True,
                         "nrtm4_client_notification_file_url": "https://testhost/",
-                        "nrtm4_client_initial_public_key": "kL7kSk56ASeaHl6Nj0eXC3XCHkCzktoPA3ceKz/cjOo=",
+                        "nrtm4_client_initial_public_key": eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY),
                         "nrtm4_server_base_url": "https://example.com",
-                        "nrtm4_server_private_key": "FalXchs8HIU22Efc3ipNcxVwYwB+Mp0x9TCM9BFtig0=",
-                        "nrtm4_server_private_key_next": "4YDgaXpRDIU8vJbFYeYgPQqEa4YAdHeRF1s6SLdXCsE=",
+                        "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR,
+                        "nrtm4_server_private_key_next": MOCK_UNF_PRIVATE_KEY_OTHER_STR,
                         "nrtm4_server_local_path": str(tmpdir),
                         "nrtm4_server_snapshot_frequency": 3600 * 2,
                     },
@@ -457,14 +463,8 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir):
         assert "Unknown setting key: log.unknown" in str(ce.value)
         assert "Unknown key(s) under source TESTDB: unknown" in str(ce.value)
 
-        assert (
-            "Invalid value for setting nrtm4_server_private_key for source TESTDB: Incorrect padding"
-            in str(ce.value)
-        )
-        assert (
-            "Invalid value for setting nrtm4_server_private_key_next for source TESTDB: Incorrect padding"
-            in str(ce.value)
-        )
+        assert "Invalid value for setting nrtm4_server_private_key for source TESTDB:" in str(ce.value)
+        assert "Invalid value for setting nrtm4_server_private_key_next for source TESTDB:" in str(ce.value)
         assert "Setting nrtm4_server_base_url for source TESTDB is not a valid https or file URL." in str(
             ce.value
         )

irrd/integration_tests/run.py

@@ -15,7 +15,7 @@
 import ujson
 import yaml
 from alembic import command, config
-from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
+from joserfc.rfc7518.ec_key import ECKey
 from python_graphql_client import GraphqlClient
 
 from irrd.conf import PASSWORD_HASH_DUMMY_VALUE, config_init
@@ -41,7 +41,7 @@
 )
 from irrd.utils.whois_client import whois_query, whois_query_irrd
 
-from ..utils.crypto import ed25519_private_key_as_str, ed25519_public_key_as_str
+from ..utils.crypto import eckey_private_key_as_str, eckey_public_key_as_str
 from .constants import (
     EMAIL_DISCARD_MSGS_COMMAND,
     EMAIL_END,
@@ -924,7 +924,7 @@ def _start_irrds(self):
         with open(self.config_path1, "w") as yaml_file:
             yaml.safe_dump(config1, yaml_file)
 
-        self.nrtm4_private_key = Ed25519PrivateKey.generate()
+        self.nrtm4_private_key = ECKey.generate_key()
         config2 = base_config.copy()
         config2["irrd"]["piddir"] = self.piddir2
         config2["irrd"]["database_url"] = self.database_url2
@@ -944,7 +944,7 @@ def _start_irrds(self):
             "nrtm_host": "127.0.0.1",
             "nrtm_port": str(self.port_whois1),
             "nrtm_access_list": "localhost",
-            "nrtm4_server_private_key": ed25519_private_key_as_str(self.nrtm4_private_key),
+            "nrtm4_server_private_key": eckey_private_key_as_str(self.nrtm4_private_key),
             "nrtm4_server_local_path": self.nrtm4_dir2,
             "nrtm4_server_base_url": f"file://{self.nrtm4_dir2}",
             "nrtm4_server_snapshot_frequency": 3600,
@@ -964,7 +964,7 @@ def _start_irrds(self):
         config3["irrd"]["sources"]["TEST"] = {
             "keep_journal": True,
             "nrtm4_client_notification_file_url": f"file://{self.nrtm4_dir2}update-notification-file.json",
-            "nrtm4_client_initial_public_key": ed25519_public_key_as_str(self.nrtm4_private_key.public_key()),
+            "nrtm4_client_initial_public_key": eckey_public_key_as_str(self.nrtm4_private_key),
         }
         with open(self.config_path3, "w") as yaml_file:
             yaml.safe_dump(config3, yaml_file)

irrd/mirroring/nrtm4/nrtm4_client.py

@@ -4,10 +4,7 @@
 from urllib.parse import urlparse
 
 import pydantic
-from joserfc import jws
-from joserfc.errors import BadSignatureError, JoseError
 from joserfc.rfc7515.model import CompactSignature
-from joserfc.rfc7518.ec_key import ECKey
 
 from irrd.conf import get_setting
 from irrd.mirroring.nrtm4.jsonseq import jsonseq_decode
@@ -30,6 +27,7 @@
     NRTM4ClientDatabaseStatus,
 )
 from irrd.storage.queries import DatabaseStatusQuery
+from irrd.utils.crypto import eckey_from_config, eckey_from_str, jws_deserialize
 from irrd.utils.misc import format_pydantic_errors
 
 logger = logging.getLogger(__name__)
@@ -155,24 +153,27 @@ def _deserialize_unf(self, unf_content: str) -> Tuple[bytes, str]:
             keys = [get_setting(f"sources.{self.source}.nrtm4_client_initial_public_key")]
 
         for key in keys:
-            if not key:
+            if not key:  # pragma: no cover
                 continue
+            pubkey = eckey_from_str(key)
             try:
-                pubkey = ECKey.import_key(key)
-            except JoseError as error:
-                logger.error(f"{self.source}: Invalid public key, ignoring: {key}", exc_info=error)
-                continue
-            try:
-                compact_signature = jws.deserialize_compact(unf_content_bytes, pubkey)
+                compact_signature = jws_deserialize(unf_content_bytes, pubkey)
                 return compact_signature.payload, key
-            except BadSignatureError:
+            except ValueError:
                 continue
 
         if self.last_status.current_key:
+            compact_signature = None
+
             try:
-                compact_signature = jws.deserialize_compact(unf_content_bytes, ECKey.import_key(config_key))
-            except JoseError:
-                compact_signature = None
+                ec_key = eckey_from_config(f"sources.{self.source}.nrtm4_client_initial_public_key")
+                if ec_key:
+                    compact_signature = jws_deserialize(
+                        unf_content_bytes,
+                        ec_key,
+                    )
+            except ValueError:  # pragma: no cover
+                pass
             if compact_signature:
                 # While technically just a "signature not valid case", it is a rather
                 # confusing situation for the user, so gets a special message.

irrd/mirroring/nrtm4/nrtm4_server.py

@@ -1,7 +1,5 @@
-import base64
 import datetime
 import gzip
-import hashlib
 import logging
 import os
 import secrets
@@ -24,7 +22,7 @@
     RPSLDatabaseJournalStatisticsQuery,
     RPSLDatabaseQuery,
 )
-from irrd.utils.crypto import ed25519_private_key_from_config, ed25519_public_key_as_str
+from irrd.utils.crypto import eckey_from_config, eckey_public_key_as_str, jws_serialize
 from irrd.utils.text import remove_auth_hashes
 
 from ...utils.process_support import get_lockfile
@@ -222,13 +220,11 @@ def _write_unf(self) -> None:
         This is based on settings and self.status.
         """
         assert self.status
-        next_signing_private_key = ed25519_private_key_from_config(
+        next_signing_private_key = eckey_from_config(
             f"sources.{self.source}.nrtm4_server_private_key_next", permit_empty=True
         )
         next_signing_public_key = (
-            ed25519_public_key_as_str(next_signing_private_key.public_key())
-            if next_signing_private_key
-            else None
+            eckey_public_key_as_str(next_signing_private_key) if next_signing_private_key else None
         )
         unf = NRTM4UpdateNotificationFile(
             nrtm_version=4,
@@ -251,14 +247,11 @@ def _write_unf(self) -> None:
             ],
         )
         unf_content = unf.model_dump_json(exclude_none=True, include=unf.model_fields_set).encode("ascii")
-        private_key = ed25519_private_key_from_config(f"sources.{self.source}.nrtm4_server_private_key")
+        private_key = eckey_from_config(f"sources.{self.source}.nrtm4_server_private_key")
         assert private_key
-        signature = private_key.sign(unf_content)
-        unf_hash = hashlib.sha256(unf_content).hexdigest()
-        with open(self.path / f"update-notification-file-signature-{unf_hash}.sig", "wb") as sig_file:
-            sig_file.write(base64.b64encode(signature))
-        with open(self.path / "update-notification-file.json", "wb") as unf_file:
-            unf_file.write(unf_content)
+        unf_serialized = jws_serialize(unf_content, private_key)
+        with open(self.path / "update-notification-file.json", "w") as unf_file:
+            unf_file.write(unf_serialized)
         self.status.last_update_notification_file_update = unf.timestamp
 
     def _expire_deltas(self) -> None:

irrd/mirroring/nrtm4/nrtm4_types.py

@@ -4,7 +4,6 @@
 from uuid import UUID
 
 import pydantic
-from joserfc.errors import JoseError
 from joserfc.rfc7518.ec_key import ECKey
 from pytz import UTC
 from typing_extensions import Self
@@ -145,7 +144,7 @@ def validate_next_signing_key(cls, next_signing_key: Optional[str]):
         if next_signing_key:
             try:
                 ECKey.import_key(next_signing_key)
-            except JoseError as ve:
+            except ValueError as ve:
                 raise ValueError(
                     f"Update Notification File has invalid next_signing_key {next_signing_key}: {ve}"
                 )

irrd/mirroring/nrtm4/tests/__init__.py

@@ -1,17 +1,22 @@
-from joserfc.rfc7518.ec_key import ECKey
+from irrd.utils.crypto import eckey_from_str, eckey_public_key_as_str
 
-MOCK_UNF_PRIVATE_KEY = ECKey.import_key(
+MOCK_UNF_PRIVATE_KEY_STR = (
     "-----BEGIN PRIVATE"
     " KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgGQrdALKHTVC4sVav\nmKUjXaPB22CWZP3t5XSkLqKHMO2hRANCAAQ9U/aaZwLV4koey4Jvu9cRaxiXna9k\naQ3YwrPzZlwd5MQSZ59kfT2+LAbQmXbZg0NGzptqHoOK0YD3YVBjv4kc\n-----END"
     " PRIVATE KEY-----\n"
 )
 
-MOCK_UNF_PUBLIC_KEY = MOCK_UNF_PRIVATE_KEY.as_pem(private=False).decode("ascii")
+MOCK_UNF_PRIVATE_KEY = eckey_from_str(MOCK_UNF_PRIVATE_KEY_STR)
 
-MOCK_UNF_PRIVATE_KEY_OTHER = ECKey.import_key(
+MOCK_UNF_PUBLIC_KEY = eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY)
+
+MOCK_UNF_PRIVATE_KEY_OTHER_STR = (
     "-----BEGIN PRIVATE"
     " KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgmHtbXrQ0uEcrzeZK\niaK8UnpD5c/YAmqdUHqoHLz997ShRANCAAQ18hSL1o3ynp1kLsfXgZBtlWSYwKvc\nLT2qRj7QeJPxHA6X3XMk7eD6xbdeyNFnLXiKwNPFMPcwRLC6oLN81Fvb\n-----END"
     " PRIVATE KEY-----\n"
 )
 
-MOCK_UNF_PUBLIC_KEY_OTHER = MOCK_UNF_PRIVATE_KEY_OTHER.as_pem(private=False).decode("ascii")
+
+MOCK_UNF_PRIVATE_KEY_OTHER = eckey_from_str(MOCK_UNF_PRIVATE_KEY_OTHER_STR)
+
+MOCK_UNF_PUBLIC_KEY_OTHER = eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY_OTHER)

irrd/mirroring/nrtm4/tests/test_nrtm4_server.py

@@ -1,7 +1,5 @@
-import base64
 import dataclasses
 import gzip
-import hashlib
 import json
 import os
 import time
@@ -11,7 +9,7 @@
 from irrd.conf import NRTM4_SERVER_DELTA_EXPIRY_TIME, PASSWORD_HASH_DUMMY_VALUE
 from irrd.mirroring.nrtm4.jsonseq import jsonseq_decode
 from irrd.mirroring.nrtm4.nrtm4_server import NRTM4Server, NRTM4ServerWriter
-from irrd.mirroring.nrtm4.tests import MOCK_UNF_PRIVATE_KEY, MOCK_UNF_PUBLIC_KEY
+from irrd.mirroring.nrtm4.tests import MOCK_UNF_PRIVATE_KEY, MOCK_UNF_PRIVATE_KEY_STR
 from irrd.mirroring.retrieval import check_file_hash_sha256
 from irrd.storage.models import DatabaseOperation, NRTM4ServerDatabaseStatus
 from irrd.storage.queries import (
@@ -20,7 +18,7 @@
     RPSLDatabaseJournalStatisticsQuery,
     RPSLDatabaseQuery,
 )
-from irrd.utils.crypto import ed25519_private_key_as_str, ed25519_public_key_from_str
+from irrd.utils.crypto import jws_deserialize
 from irrd.utils.rpsl_samples import SAMPLE_MNTNER
 from irrd.utils.test_utils import MockDatabaseHandler
 from irrd.utils.text import remove_auth_hashes
@@ -64,7 +62,7 @@ def test_nrtm4_server(self, tmpdir, config_override):
                 "piddir": pid_path,
                 "sources": {
                     "TEST": {
-                        "nrtm4_server_private_key": ed25519_private_key_as_str(MOCK_UNF_PRIVATE_KEY),
+                        "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR,
                         "nrtm4_server_local_path": str(nrtm_path),
                         "nrtm4_server_base_url": BASE_URL,
                         # "nrtm4_server_snapshot_frequency": 0,
@@ -253,16 +251,11 @@ def test_nrtm4_server(self, tmpdir, config_override):
     def _load_unf(self, nrtm_path):
         with open(nrtm_path / "update-notification-file.json", "rb") as f:
             unf_content = f.read()
-        unf = json.loads(unf_content)
+        unf_payload = jws_deserialize(unf_content, MOCK_UNF_PRIVATE_KEY)
+        unf = json.loads(unf_payload.payload)
         assert unf["nrtm_version"] == 4
         assert unf["source"] == "TEST"
         assert unf["type"] == "notification"
-
-        unf_hash = hashlib.sha256(unf_content).hexdigest()
-        with open(nrtm_path / f"update-notification-file-signature-{unf_hash}.sig", "r") as sig_file:
-            sig_content = base64.b64decode(sig_file.read())
-        public_key = ed25519_public_key_from_str(MOCK_UNF_PUBLIC_KEY)
-        public_key.verify(sig_content, unf_content)
         return unf
 
     def _status_to_dict(self, status: NRTM4ServerDatabaseStatus, force_reload=False):