feat: add developer docs (#328)

Adds a hidden statcan.github.io/dev wiki, which does not hold any original content, but instead scrapes all statcan repos (and applies a blacklist to remove ones that don't seem interesting) and wgets the README, CHANGELOG, and DEVELOPMENT files.

Author: blairdrummond

.github/workflows/publish.yml

@@ -13,12 +13,15 @@ jobs:
           python-version: 3.8
       - name: Install dependencies
         run: |
+          apt-get install --yes jq curl wget
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
       - name: Build
         run: |
           mkdocs build -c -f mkdocs-en.yml
           mkdocs build -c -f mkdocs-fr.yml
+          bash build-dev-docs.sh
+          mkdocs build -c -f mkdocs-dev.yml
           cp docs/index.html site/
       - name: Publish
         uses: peaceiris/actions-gh-pages@v3

.gitignore

@@ -4,3 +4,5 @@ node_modules
 package-lock.json
 site
 yarn-error.log
+docs/dev/*
+!docs/dev/index.md

.prettierignore

@@ -1,3 +1,5 @@
 README.md
 CHANGELOG.md
-.venv/
+docs/dev/
+build-dev-docs.sh
+.venv/

Makefile

@@ -58,6 +58,10 @@ serve-en:
 serve-fr:
 	. .venv/bin/activate && mkdocs serve -f mkdocs-fr.yml
 
+serve-dev:
+	bash build-dev-docs.sh
+	. .venv/bin/activate && mkdocs serve -f mkdocs-dev.yml
+
 test: check-format check-spelling
 
 .PHONY: \

build-dev-docs.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+# NOTE/WARNING: This will break once we go over 100 repos.
+# Then we'll need to introduce pagination.
+
+
+ORG=StatCan
+DOCS_DIR=docs/dev/
+
+camelcase () {
+    echo $@ | tr '-' ' ' | sed 's/[^ ]\+/\L\u&/g'
+}
+
+
+# Repo blacklist
+ignore () {
+    cat <<EOF | grep -q "$1"
+block-sequence
+cae-eac
+ccei
+census_age65_vs_population_growth
+census_birthplace
+census_income
+census_occupations
+dataviz-components
+dns
+EpiSim
+ESM-Mobile-App
+experimental-react-native-app
+experimental_data_api
+express-api-server
+jsonapi-pagination
+jstree
+katacoda-courses
+MetaTagGenerator
+orbital-viz
+pg-evolve
+site-ccei
+time-series-library
+transportation
+website
+wellBeingCheck
+WellbeingCheckChangeRequest
+WellbeingCheckUAT
+wellbeing_react_native
+EOF
+}
+
+
+let NUM_REPOS=$(curl --silent 'https://api.github.com/users/statcan' | jq -r .public_repos)
+
+echo "Total number of repos: $NUM_REPOS"
+echo "Fetching all READMEs. This might take a minute."
+{
+    let i=1
+    while [ $NUM_REPOS -gt 0 ]; do
+        let "NUM_REPOS-=50"
+        echo curl --silent "https://api.github.com/users/statcan/repos?per_page=50&page=$i" >&2
+        curl --silent "https://api.github.com/users/statcan/repos?per_page=50&page=$i"
+        ((i++))
+    done
+} | jq -cr '.[] | @text "\(.name) \(.url)"' |
+    while IFS=" " read name url; do
+        # Keep a blacklist of repos
+        ignore $name && continue
+
+        NAME=$(camelcase $name)
+        mkdir -p "$DOCS_DIR/$NAME/"
+        for f in CHANGELOG.md README.md DEVELOPMENT.md; do
+            # If the file exists, then write it to file and
+            # prepend the filename.
+            wget --quiet https://raw.githubusercontent.com/$ORG/$name/master/$f -O "$DOCS_DIR/$NAME/$f.tmp"
+            if [ -s "$DOCS_DIR/$NAME/$f.tmp" ]; then
+                cat <<EOF > "$DOCS_DIR/$NAME/$f"
+Link: [$name]($url)
+
+$(cat "$DOCS_DIR/$NAME/$f.tmp")
+EOF
+            fi
+            rm "$DOCS_DIR/$NAME/$f.tmp"
+        done
+done

docs/dev/index.md

@@ -0,0 +1,5 @@
+# AAW Dev docs
+
+Everything here is simply scraped from the respective repos.
+
+This content is not "curated".

mkdocs-dev.yml

@@ -0,0 +1,39 @@
+site_name: AAW Dev Docs
+site_url: https://statcan.github.io/daaas/en/
+site_description: Developer Docs
+copyright: Copyright © 2020 Statistics Canada
+
+docs_dir: docs/dev
+site_dir: site/dev
+
+repo_name: statcan/daaas
+repo_url: https://github.com/statcan/daaas
+edit_uri: edit/master/docs/dev
+
+theme:
+  name: material
+  language: en
+  #features:
+  #  - tabs
+  palette:
+    primary: indigo
+    accent: teal
+  show_sidebar: false
+  #logo: images/statcan.png
+
+markdown_extensions:
+  - toc:
+      baselevel: 2
+      permalink: true
+  - attr_list
+  - admonition
+  - codehilite
+  - mdx_truly_sane_lists
+  - pymdownx.details
+
+extra:
+  social:
+    - icon: fontawesome/brands/github-alt
+      link: https://github.com/statcan/daaas
+    - icon: fontawesome/brands/slack
+      link: https://statcan-aaw.slack.com