feat: remove matomo dump backup (#228)

SocialGouv · Aug 23, 2024 · 6c1c452 · 6c1c452
1 parent 59d5a43
commit 6c1c452
Show file tree

Hide file tree

Showing 12 changed files with 4 additions and 220 deletions.
diff --git a/.github/workflows/ingest_day.yml b/.github/workflows/ingest_day.yml
diff --git a/.github/workflows/schedule.yaml b/.github/workflows/schedule.yaml
@@ -15,18 +15,8 @@ jobs:
         with:
           node-version: 20.3.1
           cache: "yarn"
-      - name: Backup
-        uses: docker://mcr.microsoft.com/azure-cli
-        with:
-          args: "bash ./scripts/dump_matomo_yesterday.sh"
-        env:
-          AZ_STORAGE_TOKEN: ${{ secrets.AZ_STORAGE_TOKEN }}
-      - name: Download
-        uses: docker://mcr.microsoft.com/azure-cli
-        with:
-          args: "bash ./scripts/download_dump.sh"
-        env:
-          AZ_STORAGE_TOKEN: ${{ secrets.AZ_STORAGE_TOKEN }}
+      - name: Retrieve data
+        run: "./scripts/dump_matomo_yesterday.sh"
       - name: Ingest
         run: |
           yarn

diff --git a/.github/workflows/schedule_day.yml b/.github/workflows/schedule_day.yml
@@ -17,18 +17,8 @@ jobs:
         with:
           node-version: 20.3.1
           cache: "yarn"
-      - name: Backup
-        uses: docker://mcr.microsoft.com/azure-cli
-        with:
-          args: bash ./scripts/dump_matomo_precise_date.sh ${{ github.event.inputs.date }}
-        env:
-          AZ_STORAGE_TOKEN: ${{ secrets.AZ_STORAGE_TOKEN }}
-      - name: Download
-        uses: docker://mcr.microsoft.com/azure-cli
-        with:
-          args: bash ./scripts/download_dump_precise_date.sh ${{ github.event.inputs.date }}
-        env:
-          AZ_STORAGE_TOKEN: ${{ secrets.AZ_STORAGE_TOKEN }}
+      - name: Retrieve data
+        run: "./scripts/dump_matomo_yesterday.sh"
       - name: Ingest
         run: |
           yarn

diff --git a/README.md b/README.md
@@ -13,7 +13,6 @@ As this project lies between different services, it is useful to understand the
 
 ```
 MATOMO_URL # URL of the Matomo server where raw logs can be found
-AZ_STORAGE_TOKEN # Azure token to push dump to Azure blob
 ELASTICSEARCH_URL # URL of the Elastic instance where the logs are stored eventually
 ELASTIC_TOKEN # Token to use the Elastic API, read-only token is enough for the query lib
 CDTN_API_URL # URL of the CDTN API required to generate a cache of the search engine results (without the api and final /)
@@ -31,14 +30,9 @@ However you may need to create manually all elastic indices which can be achieve
 
 ## Log storage
 
-### Backup
-
-We use Azure blob to store daily dumps of the Matomo content. Downloading the data from Matomo and pushing it to Azure is done through a bash script `dump_matomo_yesterday.sh` executed from the Azure Docker image.
-
 ### `ingest`
 
 The `ingest` task takes a Matomo dump file in `data/`, convert it, and push the actions to Elastic.
-The `download_dump.sh` script allows you to get a dump file from Azure.
 
 ```console
 ELASTICSEARCH_URL=xxxx API_KEY=yyyy yarn monolog ingest data/

diff --git a/scripts/_back/analyse_week.sh b/scripts/_back/analyse_week.sh
diff --git a/scripts/_back/download_many.sh b/scripts/_back/download_many.sh
diff --git a/scripts/_back/dump_ingest.sh b/scripts/_back/dump_ingest.sh
diff --git a/scripts/_back/dump_matomo_many.sh b/scripts/_back/dump_matomo_many.sh
diff --git a/scripts/download_dump.sh b/scripts/download_dump.sh
diff --git a/scripts/download_dump_precise_date.sh b/scripts/download_dump_precise_date.sh
diff --git a/scripts/dump_matomo_precise_date.sh b/scripts/dump_matomo_precise_date.sh
@@ -48,14 +48,3 @@ file=$name.json
 jq -c -s '[.[][]]' $prefix*.json >$file
 
 rm $prefix*.json
-
-echo "Push file to Azure"
-az storage blob upload \
-  --account-key "$AZ_STORAGE_TOKEN" \
-  --account-name=cdtndata \
-  --container logs \
-  --file $file \
-  --name $name \
-  --overwrite
-
-rm $file
diff --git a/scripts/dump_matomo_yesterday.sh b/scripts/dump_matomo_yesterday.sh
@@ -50,13 +50,3 @@ file=$name.json
 jq -c -s '[.[][]]' $prefix*.json > $file
 
 rm $prefix*.json
-
-echo "Push file to Azure"
-az storage blob upload \
-  --account-key "$AZ_STORAGE_TOKEN" \
-  --account-name=cdtndata \
-  --container logs \
-  --file $file \
-  --name $name
-
-rm $file