From 74df7452b4ee69770afb93d62a1dd9363e1cc3b3 Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Fri, 7 Feb 2025 17:32:25 -0600 Subject: [PATCH 1/3] Randomize the millisecond time on the timestamp to ensure that ConfigPlans are unique This commit will look for duplicate ConfigPlans and resolve the date to datetime issue before enforcing the Unique Constraint. --- changes/781.fixed | 1 + .../0029_alter_configplan_unique_together.py | 28 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 changes/781.fixed diff --git a/changes/781.fixed b/changes/781.fixed new file mode 100644 index 00000000..95cbf985 --- /dev/null +++ b/changes/781.fixed @@ -0,0 +1 @@ +Fixed UniqueViolation error when applying migration 0029 with multiple config plans sharing same device, date and plan_type. diff --git a/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py b/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py index 1efba606..5615c178 100644 --- a/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py +++ b/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py @@ -1,7 +1,26 @@ # Generated by Django 3.2.20 on 2023-09-16 17:31 -from django.db import migrations +from datetime import timedelta +import secrets +from django.db import migrations, models + + +def ensure_config_plan_created_timestamps_are_unique(apps, schema_editor): + ConfigPlan = apps.get_model('nautobot_golden_config', 'ConfigPlan') + natural_key_fields = ['plan_type', 'device', 'created'] + + # We append some random milliseconds of time to avoid duplicate timestamps for Config Plan objects created field + + duplicate_records = ConfigPlan.objects.values(*natural_key_fields).order_by().annotate( + count=models.Count('pk'), + ).filter(count__gt=1) + + for duplicate_record in duplicate_records: + duplicate_record.pop('count') + for record in ConfigPlan.objects.filter(**duplicate_record): + record.created += timedelta(milliseconds=secrets.randbelow(1000)) + record.save() class Migration(migrations.Migration): dependencies = [ @@ -10,6 +29,13 @@ class Migration(migrations.Migration): ] operations = [ + # I know that data migrations are not recommended in the same migration as schema changes, + # but since there are already released migrations with users that had successful migrations + # doing this is the only way to ensure that we don't break their data, but are also able to fix the issue. + migrations.RunPython( + code=ensure_config_plan_created_timestamps_are_unique, + reverse_code=migrations.RunPython.noop, + ), migrations.AlterUniqueTogether( name="configplan", unique_together={("plan_type", "device", "created")}, From c9bce7002aaec4ab2263621b451f0510b7a0131d Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Mon, 10 Feb 2025 10:47:33 -0600 Subject: [PATCH 2/3] Ruff --- .../0029_alter_configplan_unique_together.py | 20 ++++++++++++------- nautobot_golden_config/utilities/helper.py | 6 ++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py b/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py index 5615c178..b4ba6672 100644 --- a/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py +++ b/nautobot_golden_config/migrations/0029_alter_configplan_unique_together.py @@ -1,27 +1,33 @@ # Generated by Django 3.2.20 on 2023-09-16 17:31 -from datetime import timedelta import secrets +from datetime import timedelta from django.db import migrations, models def ensure_config_plan_created_timestamps_are_unique(apps, schema_editor): - ConfigPlan = apps.get_model('nautobot_golden_config', 'ConfigPlan') - natural_key_fields = ['plan_type', 'device', 'created'] + ConfigPlan = apps.get_model("nautobot_golden_config", "ConfigPlan") + natural_key_fields = ["plan_type", "device", "created"] # We append some random milliseconds of time to avoid duplicate timestamps for Config Plan objects created field - duplicate_records = ConfigPlan.objects.values(*natural_key_fields).order_by().annotate( - count=models.Count('pk'), - ).filter(count__gt=1) + duplicate_records = ( + ConfigPlan.objects.values(*natural_key_fields) + .order_by() + .annotate( + count=models.Count("pk"), + ) + .filter(count__gt=1) + ) for duplicate_record in duplicate_records: - duplicate_record.pop('count') + duplicate_record.pop("count") for record in ConfigPlan.objects.filter(**duplicate_record): record.created += timedelta(milliseconds=secrets.randbelow(1000)) record.save() + class Migration(migrations.Migration): dependencies = [ ("dcim", "0049_remove_slugs_and_change_device_primary_ip_fields"), diff --git a/nautobot_golden_config/utilities/helper.py b/nautobot_golden_config/utilities/helper.py index 5457af21..a0264aa5 100644 --- a/nautobot_golden_config/utilities/helper.py +++ b/nautobot_golden_config/utilities/helper.py @@ -16,8 +16,8 @@ from nautobot.core.utils.data import render_jinja2 from nautobot.dcim.filters import DeviceFilterSet from nautobot.dcim.models import Device -from nautobot.extras.models import Job from nautobot.extras.choices import DynamicGroupTypeChoices +from nautobot.extras.models import Job from nornir_nautobot.exceptions import NornirNautobotException from nautobot_golden_config import config as app_config @@ -71,7 +71,9 @@ def get_job_filter(data=None): raw_qs = Q() # If scope is set to {} do not loop as all devices are in scope. - if not models.GoldenConfigSetting.objects.filter(dynamic_group__filter__iexact="{}", dynamic_group__group_type=DynamicGroupTypeChoices.TYPE_DYNAMIC_FILTER).exists(): + if not models.GoldenConfigSetting.objects.filter( + dynamic_group__filter__iexact="{}", dynamic_group__group_type=DynamicGroupTypeChoices.TYPE_DYNAMIC_FILTER + ).exists(): for obj in models.GoldenConfigSetting.objects.all(): raw_qs = raw_qs | obj.dynamic_group.generate_query() From 5307b945b9be86fde126082b8f613da750373c2d Mon Sep 17 00:00:00 2001 From: Stephen Kiely Date: Tue, 11 Feb 2025 10:34:13 -0600 Subject: [PATCH 3/3] Bump minimum supported Nautobot version to 2.4.2 in testing Per the compatibility matrix this is the lowest version supported now. --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a87ed1ab..c318caf4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: fail-fast: true matrix: python-version: ["3.11"] - nautobot-version: ["2.0.0"] + nautobot-version: ["2.4.2"] env: INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}" INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" @@ -146,7 +146,7 @@ jobs: include: - python-version: "3.11" db-backend: "postgresql" - nautobot-version: "2.0.0" + nautobot-version: "2.4.2" - python-version: "3.12" db-backend: "mysql" nautobot-version: "stable"