refactor: isoler les settings de test (#864)

## Description

🎸 Isoler les `settings` de test, pour nettoyer les valeurs par défaut
des `settings` de base et de dev.
🎸 Utiliser ces `settings` de test pour rendre les tests executables dès
le tirage du dépôt.
🎸 Utiliser ces `settings` dans la `CI` pour harmoniser l'execution des
tests en local et dans les actions github

## Type de changement

🚧 technique

### Points d'attention

🦺 refactor de l'appel aux settings liés aux envois de mails
🦺 simplification de la gestion des routes `brevo`
🦺 harmonisation des `settings` utilisés pour envoyer le lien magique par
mail ou dans les `message` django
🦺 prise en compte des noms de bucket issus des `settings` dans les
fichiers de migration

📦 prerequis à la PR #857

.env.template

@@ -1,42 +1,5 @@
-PYTHONPATH=.
-
-# for Django
-DJANGO_SETTINGS_MODULE=config.settings.dev
-POSTGRESQL_ADDON_HOST=localhost
-POSTGRESQL_ADDON_DB=communaute
-POSTGRESQL_ADDON_USER=communaute
-POSTGRESQL_ADDON_PASSWORD=password
-
-# GITHUB_ACCESS_TOKEN is used to update changelog from last release.
-GITHUB_ACCESS_TOKEN=__key_to_be_set__
-GITHUB_REPO=betagouv/itou-communaute-django
-
-# SENDINBLUE API KEY
-SIB_API_KEY=__key_to_be_set__
-
-# for Sentry
-#SENTRY_DSN=__url_to_be_set__
-
-# for Pro Connect
-OPENID_CONNECT_BASE_URL=http://127.0.0.1:8080
-OPENID_CONNECT_CLIENT_ID=local_openid_connect
-OPENID_CONNECT_CLIENT_SECRET=password
-
-# parking page
-PARKING_PAGE=True
-
-# Path to the itou-backup project repository.
-PATH_TO_BACKUPS=~/path/to/backups
-
-# bucket for test purpose only
-CELLAR_ADDON_KEY_ID=minioadmin
-CELLAR_ADDON_KEY_SECRET=minioadmin
-CELLAR_ADDON_HOST=localhost:9000
-CELLAR_ADDON_PROTOCOL=http
-
-# itou-backups
-export RCLONE_S3_ACCESS_KEY_ID=ACCESS_KEY_ID
-export RCLONE_S3_SECRET_ACCESS_KEY=SECRET_ACCESS_KEY
-export RCLONE_CRYPT_PASSWORD=CRYPT-PASSWORD
-export RCLONE_CRYPT_PASSWORD2=CRYPT-PASSWORD2
-export RCLONE_REMOTE_NAME=communaute
+# for psql
+PGDATABASE=communaute
+PGHOST=localhost
+PGUSER=communaute
+PGPASSWORD=password

.github/workflows/ci.yml

@@ -8,19 +8,12 @@ jobs:
     runs-on: ubuntu-latest
     env:
       PYTHONPATH: .
-      DJANGO_SETTINGS_MODULE: config.settings.base
-      DJANGO_SECRET_KEY: ministryofsillywalks
+      DJANGO_SETTINGS_MODULE: config.settings.test
       CPUCOUNT: 1
       PGPASSWORD: password
       PGHOST: localhost
       PGUSER: postgres
-      POSTGRESQL_ADDON_DB: communaute
-      POSTGRESQL_ADDON_USER: postgres
-      POSTGRESQL_ADDON_PASSWORD: password
-      CELLAR_ADDON_KEY_ID: minioadmin
-      CELLAR_ADDON_KEY_SECRET: minioadmin
-      CELLAR_ADDON_PROTOCOL: http
-      CELLAR_ADDON_HOST: localhost:9000
+
     services:
       minio:
         image: bitnami/minio
@@ -54,7 +47,7 @@ jobs:
       - name: 💂 Install Python
         uses: actions/setup-python@v4
         with:
-          python-version: '3.11'
+          python-version: '3.12'
           cache: pip
           cache-dependency-path: requirements/dev.txt
       - name: 📥 Install dependencies

.github/workflows/review-app-creation.yml

@@ -18,7 +18,7 @@ env:
   CONFIGURATION_ADDON: ${{ secrets.CLEVER_REVIEW_APPS_CONFIGURATION_ADDON }}
   S3_ADDON: ${{ secrets.CLEVER_REVIEW_APPS_S3_ADDON }}
   BRANCH: ${{ github.head_ref }}
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.12"
 
 jobs:
   create:

config/settings/base.py

@@ -268,14 +268,12 @@
 # S3 uploads
 # ------------------------------------------------------------------------------
 
-AWS_S3_ACCESS_KEY_ID = os.getenv("CELLAR_ADDON_KEY_ID", "123")
-AWS_S3_SECRET_ACCESS_KEY = os.getenv("CELLAR_ADDON_KEY_SECRET", "secret")
-AWS_S3_ENDPOINT_URL = (
-    f"{os.getenv('CELLAR_ADDON_PROTOCOL', 'https')}://{os.getenv('CELLAR_ADDON_HOST', 'set-var-env.com')}"
-)
-AWS_STORAGE_BUCKET_NAME = os.getenv("S3_STORAGE_BUCKET_NAME", "private-bucket")
-AWS_STORAGE_BUCKET_NAME_PUBLIC = os.getenv("S3_STORAGE_BUCKET_NAME_PUBLIC", "public-bucket")
-AWS_S3_STORAGE_BUCKET_REGION = os.getenv("S3_STORAGE_BUCKET_REGION", "eu-west-3")
+AWS_S3_ACCESS_KEY_ID = os.getenv("CELLAR_ADDON_KEY_ID")
+AWS_S3_SECRET_ACCESS_KEY = os.getenv("CELLAR_ADDON_KEY_SECRET")
+AWS_S3_ENDPOINT_URL = f"{os.getenv('CELLAR_ADDON_PROTOCOL')}://{os.getenv('CELLAR_ADDON_HOST')}"
+AWS_STORAGE_BUCKET_NAME = os.getenv("S3_STORAGE_BUCKET_NAME")
+AWS_STORAGE_BUCKET_NAME_PUBLIC = os.getenv("S3_STORAGE_BUCKET_NAME_PUBLIC")
+AWS_S3_STORAGE_BUCKET_REGION = os.getenv("S3_STORAGE_BUCKET_REGION")
 
 # MEDIA CONFIGURATION
 # ------------------------------------------------------------------------------
@@ -359,11 +357,11 @@
 
 # SENDINBLUE
 # ---------------------------------------
-SIB_URL = os.getenv("SIB_URL", "http://test.com")
-SIB_SMTP_URL = os.path.join(SIB_URL, "smtp/email")
-SIB_CONTACTS_URL = os.path.join(SIB_URL, "contacts/import")
+SIB_URL = os.getenv("SIB_URL")
+SIB_SMTP_ROUTE = "smtp/email"
+SIB_CONTACTS_ROUTE = "contacts/import"
 
-SIB_API_KEY = os.getenv("SIB_API_KEY", "set-sib-api-key")
+SIB_API_KEY = os.getenv("SIB_API_KEY")
 DEFAULT_FROM_EMAIL = os.getenv("DEFAULT_FROM_EMAIL", "[email protected]")
 
 SIB_MAGIC_LINK_TEMPLATE = 31

config/settings/dev.py

@@ -3,14 +3,11 @@
 
 from lacommunaute.utils.enums import Environment
 
-from .base import *  # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F403 F401
+from .test import *  # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F403 F401
 
 
 # Django settings
 # ---------------
-SECRET_KEY = "foobar"
-
-
 DEBUG = True
 ENVIRONMENT = Environment.DEV
 

config/settings/test.py

@@ -0,0 +1,39 @@
+import os
+
+from lacommunaute.utils.enums import Environment
+
+from .base import *  # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F403 F401
+
+
+# Django settings
+# ---------------
+SECRET_KEY = "v3ry_s3cr3t_k3y"
+
+ENVIRONMENT = Environment.TEST
+
+# Database
+# ------------------------------------------------------------------------------
+DATABASES["default"]["HOST"] = os.getenv("PGHOST", "localhost")  # noqa: F405
+DATABASES["default"]["PORT"] = os.getenv("PGPORT", "5432")  # noqa: F405
+DATABASES["default"]["NAME"] = os.getenv("PGDATABASE", "communaute")  # noqa: F405
+DATABASES["default"]["USER"] = os.getenv("PGUSER", "postgres")  # noqa: F405
+DATABASES["default"]["PASSWORD"] = os.getenv("PGPASSWORD", "password")  # noqa: F405
+
+# S3 uploads
+# ------------------------------------------------------------------------------
+
+AWS_S3_ACCESS_KEY_ID = os.getenv("CELLAR_ADDON_KEY_ID", "minioadmin")
+AWS_S3_SECRET_ACCESS_KEY = os.getenv("CELLAR_ADDON_KEY_SECRET", "minioadmin")
+AWS_S3_ENDPOINT_URL = (
+    f"{os.getenv('CELLAR_ADDON_PROTOCOL', 'http')}://{os.getenv('CELLAR_ADDON_HOST', 'localhost:9000')}"
+)
+AWS_STORAGE_BUCKET_NAME = "private-bucket"
+AWS_STORAGE_BUCKET_NAME_PUBLIC = "public-bucket"
+AWS_S3_STORAGE_BUCKET_REGION = "eu-west-3"
+
+MEDIA_URL = f"{AWS_S3_ENDPOINT_URL}/"
+
+# SENDINBLUE
+# ---------------------------------------
+SIB_URL = "http://test.com"
+SIB_API_KEY = "dummy-sib-api-key"

docker/django/entrypoint.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e
 
-while ! pg_isready -h $POSTGRESQL_ADDON_HOST -p 5432; do
+while ! pg_isready -h $PGHOST -p 5432; do
     >&2 echo "Postgres is unavailable - sleeping"
     sleep 1
 done

lacommunaute/notification/emails.py

@@ -1,14 +1,19 @@
 import logging
+from urllib.parse import urljoin
 
 import httpx
 from django.conf import settings
 
 from lacommunaute.notification.enums import EmailSentTrackKind
 from lacommunaute.notification.models import EmailSentTrack
+from lacommunaute.utils.enums import Environment
 
 
 logger = logging.getLogger(__name__)
 
+SIB_SMTP_URL = urljoin(settings.SIB_URL, settings.SIB_SMTP_ROUTE)
+SIB_CONTACTS_URL = urljoin(settings.SIB_URL, settings.SIB_CONTACTS_ROUTE)
+
 
 def send_email(to, params, template_id, kind, bcc=None):
     headers = {"api-key": settings.SIB_API_KEY, "Content-Type": "application/json", "Accept": "application/json"}
@@ -21,11 +26,11 @@ def send_email(to, params, template_id, kind, bcc=None):
     if bcc:
         payload["bcc"] = bcc
 
-    if settings.DEBUG:
-        # We don't want to send emails in debug mode, payload is saved in the database
+    if settings.ENVIRONMENT == Environment.DEV:
+        # We don't want to send emails in DEV mode, payload is saved in the database
         response = httpx.Response(200, json={"message": "OK"})
     else:
-        response = httpx.post(settings.SIB_SMTP_URL, headers=headers, json=payload)
+        response = httpx.post(SIB_SMTP_URL, headers=headers, json=payload)
 
     EmailSentTrack.objects.create(
         status_code=response.status_code,
@@ -47,7 +52,7 @@ def bulk_send_user_to_list(users, list_id):
         "emptyContactsAttributes": True,
     }
     headers = {"accept": "application/json", "content-type": "application/json", "api-key": settings.SIB_API_KEY}
-    response = httpx.post(settings.SIB_CONTACTS_URL, headers=headers, json=payload)
+    response = httpx.post(SIB_CONTACTS_URL, headers=headers, json=payload)
 
     EmailSentTrack.objects.create(
         status_code=response.status_code, response=response.text, datas=payload, kind=EmailSentTrackKind.ONBOARDING

lacommunaute/notification/tasks.py

@@ -3,7 +3,6 @@
 from django.urls import reverse
 from django.utils import timezone
 
-from config.settings.base import NEW_MESSAGES_EMAIL_MAX_PREVIEW, SIB_NEW_MESSAGES_TEMPLATE
 from lacommunaute.forum_conversation.models import Topic
 from lacommunaute.forum_member.shortcuts import get_forum_member_display_name
 from lacommunaute.notification.emails import bulk_send_user_to_list, send_email
@@ -31,13 +30,13 @@ def get_grouped_notifications():
 
         params = {
             "email_thumbnail": (f"Vous avez {message_count_text} à découvrir sur la communauté de l'inclusion"),
-            "messages": get_serialized_messages(recipient_notifications[:NEW_MESSAGES_EMAIL_MAX_PREVIEW]),
+            "messages": get_serialized_messages(recipient_notifications[: settings.NEW_MESSAGES_EMAIL_MAX_PREVIEW]),
         }
         send_email(
             to=[{"email": recipient}],
             params=params,
             kind=EmailSentTrackKind.BULK_NOTIFS,
-            template_id=SIB_NEW_MESSAGES_TEMPLATE,
+            template_id=settings.SIB_NEW_MESSAGES_TEMPLATE,
         )
 
     notifications.update(sent_at=timezone.now())

lacommunaute/notification/tests/tests_emails.py

@@ -2,11 +2,11 @@
 
 import httpx
 import respx
+from django.conf import settings
 from django.test import TestCase
 from faker import Faker
 
-from config.settings.base import DEFAULT_FROM_EMAIL, SIB_CONTACTS_URL, SIB_SMTP_URL
-from lacommunaute.notification.emails import bulk_send_user_to_list, send_email
+from lacommunaute.notification.emails import SIB_CONTACTS_URL, SIB_SMTP_URL, bulk_send_user_to_list, send_email
 from lacommunaute.notification.models import EmailSentTrack
 from lacommunaute.users.factories import UserFactory
 
@@ -23,7 +23,7 @@ def setUpTestData(cls):
         cls.template_id = faker.random_int()
         cls.kind = "first_reply"
         cls.payload = {
-            "sender": {"name": "La Communauté", "email": DEFAULT_FROM_EMAIL},
+            "sender": {"name": "La Communauté", "email": settings.DEFAULT_FROM_EMAIL},
             "to": cls.to,
             "params": cls.params,
             "templateId": cls.template_id,

lacommunaute/notification/tests/tests_tasks.py

@@ -8,17 +8,11 @@
 from django.urls import reverse
 from faker import Faker
 
-from config.settings.base import (
-    DEFAULT_FROM_EMAIL,
-    SIB_CONTACTS_URL,
-    SIB_ONBOARDING_LIST,
-    SIB_SMTP_URL,
-    SIB_UNANSWERED_QUESTION_TEMPLATE,
-)
 from lacommunaute.forum_conversation.factories import (
     TopicFactory,
 )
 from lacommunaute.forum_member.shortcuts import get_forum_member_display_name
+from lacommunaute.notification.emails import SIB_CONTACTS_URL, SIB_SMTP_URL
 from lacommunaute.notification.enums import NotificationDelay
 from lacommunaute.notification.factories import NotificationFactory
 from lacommunaute.notification.models import EmailSentTrack
@@ -128,7 +122,7 @@ def test_add_user_to_list_when_register(self):
             ],
             "emailBlacklist": False,
             "smsBlacklist": False,
-            "listIds": [SIB_ONBOARDING_LIST],
+            "listIds": [settings.SIB_ONBOARDING_LIST],
             "updateExistingContacts": True,
             "emptyContactsAttributes": True,
         }
@@ -163,10 +157,10 @@ def payload_for_staff_user_to_notify_on_unanswered_topics_fixture():
     )
     params = {"count": 1, "link": "".join(url)}
     payload = {
-        "sender": {"name": "La Communauté", "email": DEFAULT_FROM_EMAIL},
+        "sender": {"name": "La Communauté", "email": settings.DEFAULT_FROM_EMAIL},
         "to": to,
         "params": params,
-        "templateId": SIB_UNANSWERED_QUESTION_TEMPLATE,
+        "templateId": settings.SIB_UNANSWERED_QUESTION_TEMPLATE,
     }
     yield payload
 

lacommunaute/partner/migrations/0001_initial.py

@@ -2,6 +2,7 @@
 
 import machina.models.fields
 import storages.backends.s3
+from django.conf import settings
 from django.db import migrations, models
 
 import lacommunaute.utils.validators
@@ -35,7 +36,9 @@ class Migration(migrations.Migration):
                     "logo",
                     models.ImageField(
                         help_text="1200x600 recommandé",
-                        storage=storages.backends.s3.S3Storage(bucket_name="private-bucket", file_overwrite=False),
+                        storage=storages.backends.s3.S3Storage(
+                            bucket_name=settings.AWS_STORAGE_BUCKET_NAME, file_overwrite=False
+                        ),
                         upload_to="logos/",
                         validators=[lacommunaute.utils.validators.validate_image_size],
                     ),

lacommunaute/users/tests/__snapshots__/tests_views.ambr

@@ -613,9 +613,9 @@
           </main>
   '''
 # ---
-# name: TestSendMagicLink.test_send_magic_link[DEV-1][send_magic_link_payload]
+# name: TestSendMagicLink.test_send_magic[PROD-True-0][send_magic_link_payload]
   '{"sender": {"name": "La Communaut\\u00e9", "email": "[email protected]"}, "to": [{"email": "[email protected]"}], "params": {"display_name": "Samuel J.", "login_link": "LOGIN_LINK"}, "templateId": 31}'
 # ---
-# name: TestSendMagicLink.test_send_magic_link[PROD-0][send_magic_link_payload]
+# name: TestSendMagicLink.test_send_magic[TEST-True-0][send_magic_link_payload]
   '{"sender": {"name": "La Communaut\\u00e9", "email": "[email protected]"}, "to": [{"email": "[email protected]"}], "params": {"display_name": "Samuel J.", "login_link": "LOGIN_LINK"}, "templateId": 31}'
 # ---