Skip to content

Commit

Permalink
Fix #745 - Add RPKI test results to overall score
Browse files Browse the repository at this point in the history
  • Loading branch information
@mxsasha
mxsasha committed Mar 7, 2024
1 parent 44bd852 commit d0163da
Show file tree
Hide file tree
Showing 9 changed files with 138 additions and 31 deletions.
17 changes: 15 additions & 2 deletions checks/categories.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,7 @@ class RpkiExists(Subtest):
_label = None
_test_label = "exists"
_test_name = None
_score_field = None

def __init__(self):
super().__init__(
Expand All @@ -268,7 +269,9 @@ def __init__(self):
explanation=f"detail {self._label} rpki {self._test_label} exp",
tech_string=f"detail {self._label} rpki {self._test_label} tech table",
init_tech_type="table_multi_col",
worst_status=STATUS_INFO,
worst_status=STATUS_FAIL,
full_score=scoring.RPKI_EXISTS_GOOD,
model_score_field=self._score_field,
)

def was_tested(self):
Expand Down Expand Up @@ -299,6 +302,7 @@ def result_validator_error(self):
class WebRpkiExists(RpkiExists):
_label = "web"
_test_name = "web_rpki_exists"
_score_field = "web_exists_score"

def was_tested(self):
self.worst_status = scoring.WEB_RPKI_EXISTENCE_WORST_STATUS
Expand All @@ -307,6 +311,7 @@ def was_tested(self):
class MailRpkiExists(RpkiExists):
_label = "mail"
_test_name = "mail_rpki_exists"
_score_field = "mail_exists_score"

def was_tested(self):
self.worst_status = scoring.MAIL_RPKI_EXISTENCE_WORST_STATUS
Expand All @@ -316,6 +321,7 @@ class RpkiValid(Subtest):
_label = None
_test_label = "valid"
_test_name = None
_score_field = None

def __init__(self):
super().__init__(
Expand All @@ -324,7 +330,8 @@ def __init__(self):
explanation=f"detail {self._label} rpki {self._test_label} exp",
tech_string=f"detail {self._label} rpki {self._test_label} tech table",
init_tech_type="table",
worst_status=STATUS_INFO,
full_score=scoring.RPKI_VALID_GOOD,
model_score_field=self._score_field,
)

def was_tested(self):
Expand Down Expand Up @@ -365,6 +372,7 @@ def result_validator_error(self):
class WebRpkiValid(RpkiValid):
_label = "web"
_test_name = "web_rpki_valid"
_score_field = "web_valid_score"

def was_tested(self):
self.worst_status = scoring.WEB_RPKI_VALIDITY_WORST_STATUS
Expand All @@ -373,6 +381,7 @@ def was_tested(self):
class MailRpkiValid(RpkiValid):
_label = "mail"
_test_name = "mail_rpki_valid"
_score_field = "mail_valid_score"

def was_tested(self):
self.worst_status = scoring.MAIL_RPKI_VALIDITY_WORST_STATUS
Expand All @@ -382,6 +391,7 @@ class NsRpkiExists(RpkiExists):
_label = "web-mail"
_test_label = "ns-exists"
_test_name = "ns_rpki_exists"
_score_field = "ns_exists_score"

def was_tested(self):
self.worst_status = scoring.RPKI_NS_EXISTENCE_WORST_STATUS
Expand All @@ -391,6 +401,7 @@ class NsRpkiValid(RpkiValid):
_label = "web-mail"
_test_label = "ns-valid"
_test_name = "ns_rpki_valid"
_score_field = "ns_valid_score"

def was_tested(self):
self.worst_status = scoring.RPKI_NS_VALIDITY_WORST_STATUS
Expand All @@ -400,6 +411,7 @@ class MailMxNsRpkiExists(RpkiExists):
_label = "mail"
_test_label = "mx-ns-exists"
_test_name = "mail_mx_ns_rpki_exists"
_score_field = "mx_ns_exists_score"

def was_tested(self):
self.worst_status = scoring.MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS
Expand All @@ -409,6 +421,7 @@ class MailMxNsRpkiValid(RpkiValid):
_label = "mail"
_test_label = "mx-ns-valid"
_test_name = "mail_mx_ns_rpki_valid"
_score_field = "mx_ns_valid_score"

def was_tested(self):
self.worst_status = scoring.MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS
Expand Down
78 changes: 78 additions & 0 deletions checks/migrations/0015_add_rpki_scoring.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# Generated by Django 3.2.19 on 2023-06-15 14:09

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
("checks", "0015_auto_20240212_1616"),
]

operations = [
migrations.RemoveField(
model_name="mailtestrpki",
name="mail_score",
),
migrations.RemoveField(
model_name="mailtestrpki",
name="ns_score",
),
migrations.RemoveField(
model_name="webtestrpki",
name="ns_score",
),
migrations.RemoveField(
model_name="webtestrpki",
name="web_score",
),
migrations.AddField(
model_name="mailtestrpki",
name="mail_exists_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="mailtestrpki",
name="mail_valid_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="mailtestrpki",
name="mx_ns_exists_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="mailtestrpki",
name="mx_ns_valid_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="mailtestrpki",
name="ns_exists_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="mailtestrpki",
name="ns_valid_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="webtestrpki",
name="ns_exists_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="webtestrpki",
name="ns_valid_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="webtestrpki",
name="web_exists_score",
field=models.IntegerField(null=True),
),
migrations.AddField(
model_name="webtestrpki",
name="web_valid_score",
field=models.IntegerField(null=True),
),
]
14 changes: 10 additions & 4 deletions checks/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,8 +783,10 @@ class WebTestRpki(BaseTestModel):
timestamp = models.DateTimeField(auto_now_add=True)
domain = models.CharField(max_length=255, default="")
report = ListField(default="")
web_score = models.IntegerField(null=True)
ns_score = models.IntegerField(null=True)
web_exists_score = models.IntegerField(null=True)
web_valid_score = models.IntegerField(null=True)
ns_exists_score = models.IntegerField(null=True)
ns_valid_score = models.IntegerField(null=True)
score = models.IntegerField(null=True)
max_score = models.IntegerField(null=True)

Expand All @@ -799,8 +801,12 @@ class MailTestRpki(BaseTestModel):
timestamp = models.DateTimeField(auto_now_add=True)
domain = models.CharField(max_length=255, default="")
report = ListField(default="")
mail_score = models.IntegerField(null=True)
ns_score = models.IntegerField(null=True)
mail_exists_score = models.IntegerField(null=True)
mail_valid_score = models.IntegerField(null=True)
ns_exists_score = models.IntegerField(null=True)
ns_valid_score = models.IntegerField(null=True)
mx_ns_exists_score = models.IntegerField(null=True)
mx_ns_valid_score = models.IntegerField(null=True)
score = models.IntegerField(null=True)
max_score = models.IntegerField(null=True)

Expand Down
21 changes: 13 additions & 8 deletions checks/scoring.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,13 @@
IPV6_NS_CONN_FAIL = NO_POINTS
IPV6_NS_CONN_WORST_STATUS = STATUS_FAIL

RPKI_NS_EXISTENCE_WORST_STATUS = STATUS_NOTICE
RPKI_NS_VALIDITY_WORST_STATUS = STATUS_NOTICE
RPKI_NS_EXISTENCE_WORST_STATUS = STATUS_FAIL
RPKI_NS_VALIDITY_WORST_STATUS = STATUS_FAIL

RPKI_EXISTS_GOOD = HALF_WEIGHT_POINTS
RPKI_EXISTS_FAIL = NO_POINTS
RPKI_VALID_GOOD = FULL_WEIGHT_POINTS
RPKI_VALID_FAIL = NO_POINTS


# --- WEBTEST
Expand All @@ -84,8 +89,8 @@
WEB_DNSSEC_ERROR = NO_POINTS
WEB_DNSSEC_WORST_STATUS = STATUS_FAIL

WEB_RPKI_EXISTENCE_WORST_STATUS = STATUS_NOTICE
WEB_RPKI_VALIDITY_WORST_STATUS = STATUS_NOTICE
WEB_RPKI_EXISTENCE_WORST_STATUS = STATUS_FAIL
WEB_RPKI_VALIDITY_WORST_STATUS = STATUS_FAIL

# The HTTPS_EXISTS test is not scored. The scoring of the category relies on
# the subsequent tests.
Expand Down Expand Up @@ -240,10 +245,10 @@
MAIL_AUTH_SPF_POLICY_FAIL = NO_POINTS
MAIL_AUTH_SPF_POLICY_WORST_STATUS = STATUS_FAIL

MAIL_RPKI_EXISTENCE_WORST_STATUS = STATUS_NOTICE
MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS = STATUS_NOTICE
MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS = STATUS_NOTICE
MAIL_RPKI_VALIDITY_WORST_STATUS = STATUS_NOTICE
MAIL_RPKI_EXISTENCE_WORST_STATUS = STATUS_FAIL
MAIL_RPKI_MX_NS_EXISTENCE_WORST_STATUS = STATUS_FAIL
MAIL_RPKI_MX_NS_VALIDITY_WORST_STATUS = STATUS_FAIL
MAIL_RPKI_VALIDITY_WORST_STATUS = STATUS_FAIL

MAIL_TLS_STARTTLS_EXISTS_GOOD = FULL_WEIGHT_POINTS
MAIL_TLS_STARTTLS_EXISTS_WORST_STATUS = STATUS_FAIL
Expand Down
31 changes: 18 additions & 13 deletions checks/tasks/rpki.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
TeamCymruIPtoASN,
RelyingPartyUnvailableError,
)
from .. import categories
from .. import categories, scoring
from ..models import (
MailTestRpki,
WebTestRpki,
Expand Down Expand Up @@ -249,7 +249,7 @@ def batch_mail_mx_ns_rpki(self, mx_ips_pairs, url, *args, **kwargs):
return do_mx_ns_rpki(mx_ips_pairs, url, self, *args, **kwargs)


def generate_roa_existence_report(subtestname, category, hostset) -> None:
def generate_roa_existence_report(subtestname, category, hostset) -> int:
"""Generate a test report for the existence of ROAs."""

def gen_tech_data(host, ip, validity) -> List[List[str]]:
Expand Down Expand Up @@ -296,11 +296,13 @@ def gen_tech_data(host, ip, validity) -> List[List[str]]:
category.subtests[subtestname].result_no_addresses()
elif missing_count > 0:
category.subtests[subtestname].result_bad(tech_data)
return scoring.RPKI_EXISTS_FAIL
else:
category.subtests[subtestname].result_good(tech_data)
return scoring.RPKI_EXISTS_GOOD


def generate_validity_report(subtestname, category, hostset) -> None:
def generate_validity_report(subtestname, category, hostset) -> int:
"""Generate a test report based on Route Origin Validation.
This compares routing data from BGP with published ROAs.
Expand Down Expand Up @@ -368,12 +370,15 @@ def gen_tech_data(host, asn, prefix, validity, errors) -> List[str]:
category.subtests[subtestname].result_no_addresses()
elif invalid_count > 0:
category.subtests[subtestname].result_invalid(tech_data)
return scoring.RPKI_VALID_FAIL
elif not_valid_count > 0:
category.subtests[subtestname].result_bad(tech_data)
return scoring.RPKI_VALID_FAIL
elif not_routed_count == count: # no BGP data for all IPs
category.subtests[subtestname].result_not_routed(tech_data)
else:
category.subtests[subtestname].result_good(tech_data)
return scoring.RPKI_VALID_GOOD


def build_summary_report(parent, parent_name, category) -> None:
Expand All @@ -382,22 +387,22 @@ def build_summary_report(parent, parent_name, category) -> None:
webset = parent.webhosts.all().order_by("host")
nsset = parent.nshosts.all().order_by("host")

generate_roa_existence_report("web_rpki_exists", category, webset)
generate_validity_report("web_rpki_valid", category, webset)
generate_roa_existence_report("ns_rpki_exists", category, nsset)
generate_validity_report("ns_rpki_valid", category, nsset)
parent.web_exists_score = generate_roa_existence_report("web_rpki_exists", category, webset)
parent.web_valid_score = generate_validity_report("web_rpki_valid", category, webset)
parent.ns_exists_score = generate_roa_existence_report("ns_rpki_exists", category, nsset)
parent.ns_valid_score = generate_validity_report("ns_rpki_valid", category, nsset)

elif parent_name == "mailtestrpki":
mxset = parent.mxhosts.all().order_by("host")
nsset = parent.nshosts.all().order_by("host")
mxnsset = parent.mxnshosts.all().order_by("host")

generate_roa_existence_report("mail_rpki_exists", category, mxset)
generate_validity_report("mail_rpki_valid", category, mxset)
generate_roa_existence_report("ns_rpki_exists", category, nsset)
generate_validity_report("ns_rpki_valid", category, nsset)
generate_roa_existence_report("mail_mx_ns_rpki_exists", category, mxnsset)
generate_validity_report("mail_mx_ns_rpki_valid", category, mxnsset)
parent.mail_exists_score = generate_roa_existence_report("mail_rpki_exists", category, mxset)
parent.mail_valid_score = generate_validity_report("mail_rpki_valid", category, mxset)
parent.ns_exists_score = generate_roa_existence_report("ns_rpki_exists", category, nsset)
parent.ns_valid_score = generate_validity_report("ns_rpki_valid", category, nsset)
parent.mx_ns_exists_score = generate_roa_existence_report("mail_mx_ns_rpki_exists", category, mxnsset)
parent.mx_ns_valid_score = generate_validity_report("mail_mx_ns_rpki_valid", category, mxnsset)

parent.report = category.gen_report()
parent.save()
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/develop/test_website.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
INVALID_DOMAIN = "invalid-domain.example.com"

TEST_DOMAIN_EXPECTED_SCORE = 100
TEST_DOMAIN_EXPECTED_SCORE_NO_IPV6 = 48
TEST_DOMAIN_EXPECTED_SCORE_NO_IPV6 = 61

LONG_TIMEOUT = 1000 * 3 * 60

Expand Down
2 changes: 1 addition & 1 deletion integration_tests/integration/test_batch.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
TEST_DOMAIN = "target.test"
TEST_DOMAIN_EXPECTED_SCORE = 100
# TODO: improve test environment to allow 100% score result
TEST_DOMAIN_EXPECTED_SCORE = 48
TEST_DOMAIN_EXPECTED_SCORE = 49


def wait_for_request_status(url, expected_status, timeout=10, interval=1, auth=None):
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/integration/test_email.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
ALL_EMAIL_PROBES = {"ipv6", "dnssec", "tls", "auth", "rpki"}
TEST_EMAIL_EXPECTED_SCORE = 100
# TODO: improve test environment to allow 100% score result
TEST_EMAIL_EXPECTED_SCORE = 17
TEST_EMAIL_EXPECTED_SCORE = 20


def test_your_email_score(page, app_url, test_email):
Expand Down
2 changes: 1 addition & 1 deletion integration_tests/integration/test_website.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
ALL_PROBES = {"ipv6", "dnssec", "tls", "appsecpriv", "rpki"}
TEST_DOMAIN_EXPECTED_SCORE = 100
# TODO: improve test environment to allow 100% score result
TEST_DOMAIN_EXPECTED_SCORE = 48
TEST_DOMAIN_EXPECTED_SCORE = 49


def test_reject_invalid_domain(page, app_url):
Expand Down

1 comment on commit d0163da

@mdavids
Copy link

@mdavids mdavids commented on d0163da Mar 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Commit nr 1000! 🎉

Please sign in to comment.