Add providers.json from API and use it for specific greenwebcheck URLs

.gitignore

@@ -21,6 +21,7 @@
 
 content/provider/
 data/results/
+data/filtered_providers.json
 data/implementing_clients.json
 data/providers.json
 downloads/

Dockerfile

@@ -20,8 +20,11 @@ ARG BASEURL=https://providers.xmpp.net/
 # Build and copy in place
 WORKDIR /var/tmp/src/xmpp-providers
 COPY . /var/tmp/src/xmpp-providers
+
 ADD https://invent.kde.org/melvo/xmpp-providers/-/jobs/artifacts/master/download/?job=filtered-provider-lists /var/tmp/src/xmpp-providers/downloads-docker/providers_data.zip
 ADD https://invent.kde.org/melvo/xmpp-providers/-/jobs/artifacts/master/download/?job=badges /var/tmp/src/xmpp-providers/downloads-docker/badges_data.zip
+ADD https://data.xmpp.net/providers/v1/providers.json /var/tmp/src/xmpp-providers/downloads-docker/providers.json
+
 RUN cd /var/tmp/src/xmpp-providers && make publish BASEURL=$BASEURL
 
 FROM nginx

themes/xmpp-providers/layouts/index.html

@@ -53,7 +53,7 @@ <h3 id="category-a" class="pt-4">
             </h3>
             <p>Providers in this category have the best properties, allow you to register via an app and are free of charge.</p>
             <div class="accordion pb-5" id="providers-accordion-a">
-                {{ partial "provider-list-items.html" (where .Site.Data.providers ".category" "==" "A") }}
+                {{ partial "provider-list-items.html" (where .Site.Data.filtered_providers ".category" "==" "A") }}
             </div>
             <h3 id="category-b">
                 Category&nbsp;
@@ -62,7 +62,7 @@ <h3 id="category-b">
             </h3>
             <p>Providers in this category have the best properties but may only allow you to register via a web page or are <i>not</i> free of charge.</p>
             <div class="accordion pb-5" id="providers-accordion-b">
-                {{ partial "provider-list-items.html" (where .Site.Data.providers ".category" "==" "B") }}
+                {{ partial "provider-list-items.html" (where .Site.Data.filtered_providers ".category" "==" "B") }}
             </div>
             <h3 id="category-c">
                 Category&nbsp;
@@ -71,7 +71,7 @@ <h3 id="category-c">
             </h3>
             <p>Providers in this category have average properties and should only be used if there is a certain reason for it.</p>
             <div class="accordion pb-5" id="providers-accordion-c">
-                {{ partial "provider-list-items.html" (where .Site.Data.providers ".category" "==" "C") }}
+                {{ partial "provider-list-items.html" (where .Site.Data.filtered_providers ".category" "==" "C") }}
             </div>
             <h3 id="category-d">
                 Category&nbsp;
@@ -80,7 +80,7 @@ <h3 id="category-d">
             </h3>
             <p>Providers in this category cannot or should not be used for registration.</p>
             <div class="accordion pb-5" id="providers-accordion-d">
-                {{ partial "provider-list-items.html" (where .Site.Data.providers ".category" "==" "D") }}
+                {{ partial "provider-list-items.html" (where .Site.Data.filtered_providers ".category" "==" "D") }}
             </div>
         </div>
     </div>

themes/xmpp-providers/layouts/shortcodes/provider-details.html

@@ -1,6 +1,9 @@
 {{ $provider_name := .Get "provider" }}
-{{ $providers_data := .Site.Data.providers }}
-{{ $provider := where $providers_data ".jid" "==" $provider_name }}
+{{ $filtered_providers_data := .Site.Data.filtered_providers }}
+{{ $provider := where $filtered_providers_data ".jid" "==" $provider_name }}
+
+{{ $providers_data := index .Site.Data.providers $provider_name }}
+
 {{ $permalink := .Page.Permalink }}
 
 {{ $category_criteria := "" }}
@@ -144,7 +147,8 @@ <h2>Service</h2>
                     </td>
                     <td>
                         {{ if .ratingGreenWebCheck }}
-                            <a href="/faq/#green-hosting" class="abbr">Green hosting</a>
+                            {{ $url := replace $providers_data.ratingGreenWebCheck.source "https://admin.thegreenwebfoundation.org/greencheck/" "" }}
+                            <a href="https://www.thegreenwebfoundation.org/green-web-check/?url={{ $url }}" target="_blank">Green hosting</a>
                         {{ else }}
                             No <a href="/faq/#green-hosting" class="abbr">green hosting</a>
                         {{ end }}

themes/xmpp-providers/layouts/shortcodes/provider-table.html

@@ -77,7 +77,7 @@
             </tr>
         </thead>
         <tbody>
-            {{ range sort .Site.Data.providers "category" }}
+            {{ range sort .Site.Data.filtered_providers "category" }}
                 <tr>
                     <td>
                         <a href="/provider/{{ .jid }}/">{{ .jid }}</a>

tools/prepare.py

@@ -26,7 +26,7 @@
 PROVIDERS_JSON_PATH = DATA_PATH / "results"
 PROVIDERS_PAGES_PATH = Path("content/provider")
 
-PROVIDERS_DATA_URL = (
+FILTERED_PROVIDERS_DATA_URL = (
     "https://invent.kde.org/melvo/xmpp-providers/"
     "-/jobs/artifacts/master/download/?job=filtered-provider-lists"
 )
@@ -35,10 +35,11 @@
     "-/jobs/artifacts/master/download/?job=badges"
 )
 CLIENTS_DATA_URL = (
-    "https://invent.kde.org/melvo/xmpp-providers/" "-/raw/master/clients.json"
+    "https://invent.kde.org/melvo/xmpp-providers/-/raw/master/clients.json"
 )
+PROVIDERS_FILE_URL = "https://data.xmpp.net/providers/v1/providers.json"
 XSF_SOFTWARE_LIST_URL = (
-    "https://raw.githubusercontent.com/xsf/" "xmpp.org/master/data/software.json"
+    "https://raw.githubusercontent.com/xsf/xmpp.org/master/data/software.json"
 )
 
 DOAP_NS = "http://usefulinc.com/ns/doap#"
@@ -56,9 +57,8 @@ def initialize_directory(path: Path) -> None:
     """
     if path.exists() and path.is_dir():
         shutil.rmtree(path)
-        os.mkdir(path)
-    else:
-        os.mkdir(path)
+
+    os.mkdir(path)
 
 
 def prepare_provider_data_files() -> None:
@@ -79,8 +79,9 @@ def prepare_provider_data_files() -> None:
     initialize_directory(STATIC_PATH)
     initialize_directory(DATA_PATH)
 
-    get_providers_data()
+    get_filtered_providers_data()
     get_badges()
+    get_providers_file()
 
     shutil.copytree(DOWNLOAD_PATH / "logo", STATIC_PATH / "logo")
     shutil.copytree(DOWNLOAD_PATH / "images", STATIC_PATH / "images")
@@ -118,7 +119,7 @@ def download_file(url: str, path: Path) -> bool:
     return True
 
 
-def get_providers_data() -> None:
+def get_filtered_providers_data() -> None:
     """
     Download, extract, and move providers data.
     Use downloaded file from docker build, if it exists.
@@ -129,24 +130,24 @@ def get_providers_data() -> None:
     if providers_docker_path.exists():
         shutil.copyfile(providers_docker_path, DOWNLOAD_PATH / "providers_data.zip")
     else:
-        success = download_file(PROVIDERS_DATA_URL, Path("providers_data.zip"))
+        success = download_file(FILTERED_PROVIDERS_DATA_URL, Path("providers_data.zip"))
         if not success:
-            sys.exit(f"Error while trying to download from {PROVIDERS_DATA_URL}")
+            sys.exit(f"Error while trying to download from {FILTERED_PROVIDERS_DATA_URL}")
 
     with zipfile.ZipFile(DOWNLOAD_PATH / "providers_data.zip", "r") as zip_file:
         zip_file.extractall(DOWNLOAD_PATH / "providers_data")
 
     shutil.copyfile(
         DOWNLOAD_PATH / "providers_data" / "providers-D.json",
-        DATA_PATH / "providers.json",
+        DATA_PATH / "filtered_providers.json",
     )
     shutil.copytree(DOWNLOAD_PATH / "providers_data" / "results", DATA_PATH / "results")
 
 
 def url_escape_support_addresses() -> None:
     """Escaoe support addresses in order to display them correctly
     in browsers."""
-    with open(DATA_PATH / "providers.json", "rb") as json_file:
+    with open(DATA_PATH / "filtered_providers.json", "rb") as json_file:
         providers = json.load(json_file)
 
     support_categories = ["emailSupport", "chatSupport", "groupChatSupport"]
@@ -156,7 +157,7 @@ def url_escape_support_addresses() -> None:
                 for index, address in enumerate(addresses):
                     addresses[index] = quote(address)
 
-    with open(DATA_PATH / "providers.json", "w", encoding="utf-8") as esc_providers:
+    with open(DATA_PATH / "filtered_providers.json", "w", encoding="utf-8") as esc_providers:
         json.dump(providers, esc_providers, indent=4)
 
 
@@ -183,6 +184,26 @@ def get_badges() -> None:
     )
 
 
+def get_providers_file() -> None:
+    """
+    Download providers.json from https://data.xmpp.net/ API endpoint.
+    Use downloaded file from docker build, if it exists.
+    Docker builds need to download providers.json directly in order to
+    correctly apply caching.
+    """
+    providers_docker_path = Path(DOCKER_DOWNLOAD_PATH / "providers.json")
+    if providers_docker_path.exists():
+        shutil.copyfile(providers_docker_path, DOWNLOAD_PATH / "providers.json")
+    else:
+        success = download_file(PROVIDERS_FILE_URL, Path("providers.json"))
+        if not success:
+            sys.exit(f"Error while trying to download from {PROVIDERS_FILE_URL}")
+    shutil.copyfile(
+        DOWNLOAD_PATH / "providers.json",
+        DATA_PATH / "providers.json",
+    )
+
+
 def create_provider_pages() -> None:
     """
     Create a .md page per provider