docs: Add support to readthedocs.io

Add support to mkdocs and readthedocs.io in order to build EVE's
documentation through mkdocs and allow to publish it at readthedocs.io.
Document sources (.md, .txt, etc) are located through the whole repository.
Unfortunately, mkdocs doesn't allow to keep the configuration file along
with document sources, so this commit adds a python script which
reassembles the repository's tree only with documentation sources and
related files, such as figures (.png, .svg). Additionally, it changes all
relative links that point to some other file in the repository, such as .go
and .sh files, with the full URL that will point to Github EVE's
repository.

The documentation can be also built locally, through the following
commands:

python3 docs/mkdocs/prepare_mkdocs.py \
	dist/docs-src \
	"https://github.com/lf-edge/eve/blob/master"
cp docs/mkdocs/mkdocs.yml .
mkdocs build

Documentation files will be built at dist/docs-src and the links for source
files throughout the documentation will point to master branch.

To visualize the documentation through a local web server, use:

mkdocs serve

And access it through the URL:

http://127.0.0.1:8000/projects/eve/

Signed-off-by: Renê de Souza Pinto <[email protected]>

Author: rene-zededa

.gitignore

@@ -6,3 +6,4 @@ images/*.yml
 .go/
 tags
 tmp/
+mkdocs.yml

.readthedocs.yaml

@@ -0,0 +1,16 @@
+---
+version: 2
+build:
+  os: "ubuntu-20.04"
+  tools:
+    python: "3.10"
+  jobs:
+    post_checkout:
+      - python3 docs/mkdocs/prepare_mkdocs.py dist/docs-src "https://github.com/lf-edge/eve/blob/master"
+      - cp docs/mkdocs/mkdocs.yml ./
+mkdocs:
+  configuration: mkdocs.yml
+  fail_on_warning: false
+python:
+  install:
+    - requirements: docs/mkdocs/requirements.txt

docs/mkdocs/mkdocs.yml

@@ -0,0 +1,219 @@
+---
+site_name: EVE-OS
+site_url: https://www.lfedge.org/projects/eve/
+repo_name: GitHub
+repo_url: https://github.com/lf-edge/eve
+docs_dir: dist/docs-src/
+site_dir: dist/doc/
+plugins:
+  - search:
+      indexing: 'full'
+theme:
+  name: readthedocs
+  highlightjs: true
+  hljs_languages:
+    - yaml
+    - rust
+extra:
+  version: 2.0
+nav:
+  - Introduction: 'docs/README.md'
+  - 'Building EVE': 'docs/BUILD.md'
+  - 'EVE Configuration': 'docs/CONFIG.md'
+  - 'BIOS and Firmware management with EVE': 'docs/BIOS-FIRMWARE.md'
+  - 'Deploying EVE-OS': 'docs/DEPLOYMENT.md'
+  - 'Booting EVE': 'docs/BOOTING.md'
+  - 'Device Connectivity': 'docs/DEVICE-CONNECTIVITY.md'
+  - Communications: 'docs/COMMS.md'
+  - Registration: 'docs/REGISTRATION.md'
+  - 'Update base image': 'docs/BASEIMAGE-UPDATE.md'
+  - Build-tools: 'build-tools/ver-update/README.md'
+  - 'Direct support of OCI containers': 'docs/CONTAINERS.md'
+  - 'Edge Containers': 'docs/ECOS.md'
+  - 'Edge-View Containers API': 'docs/EDGEVIEW-CONTAINER-API.md'
+  - Hypervisors: 'docs/HYPERVISORS.md'
+  - IPC: 'docs/IPC.md'
+  - 'Keys and Certificates': 'docs/KEYS-AND-CERTS.md'
+  - 'LED indication': 'docs/LED-INDICATION.md'
+  - Wireless: 'docs/WIRELESS.md'
+  - Tasks: 'docs/TASKS.md'
+  - Upstreaming: 'docs/UPSTREAMING.md'
+  - 'Bringing EVE up on a new CPU architecture': 'docs/PORTING.md'
+  - 'Bringing EVE up on new Hardware': 'docs/HARDWARE-BRINGUP.md'
+  - 'Hardware model description and testing': 'docs/HARDWARE-MODEL.md'
+  - Cleanup: 'docs/CLEANUP.md'
+  - 'Configuration properties': 'docs/CONFIG-PROPERTIES.md'
+  - CI-CD: 'docs/CI-CD.md'
+  - FAQ: 'docs/FAQ.md'
+  - Networking:
+      - 'Network ACLS': 'docs/NETWORK-ACLS.md'
+      - 'Network Models': 'docs/NETWORK-MODELS.md'
+  - 'ECO Initialization':
+      - Initialization: 'docs/ECO-INIT.md'
+      - 'ECO Metadata': 'docs/ECO-METADATA.md'
+  - Logging:
+      - Introduction: 'docs/LOGGING.md'
+      - 'Log levels': 'docs/LOG-LEVELS.md'
+  - Security:
+      - Information: 'docs/SECURITY.md'
+      - 'Object level encryption': 'docs/OBJECT-LEVEL-ENCRYPTION.md'
+      - 'Fault injection': 'docs/FAULT-INJECTION.md'
+  - Storage:
+      - 'RAID configuration': 'docs/STORAGE-RAID-CONFIGURATION.md'
+      - ZFS: 'docs/ZFS.md'
+  - Boards:
+      - 'NVIDIA Jetson': 'boards/nvidia/jetson/README.md'
+  - Debugging:
+      - Information: 'pkg/debug/abuild/README.md'
+      - Debugging: 'docs/DEBUGGING.md'
+  - API:
+      - Information: 'api/README.md'
+      - V1: 'api/APIv1.md'
+      - V2: 'api/APIv2.md'
+      - 'Object signing': 'api/OBJECT-SIGNING.md'
+      - Profile: 'api/PROFILE.md'
+      - 'GO Info': 'api/go/README.md'
+  - Tests:
+      - app: 'tests/eden/app/README.md'
+      - eclient: 'tests/eden/eclient/README.md'
+      - escript: 'tests/eden/escript/README.md'
+      - flir: 'tests/eden/flir/README.md'
+      - 'Hardware reboot': 'tests/eden/hardware_reboot/README.md'
+      - 'I/O performance': 'tests/eden/io_performance/README.md'
+      - lim: 'tests/eden/lim/README.md'
+      - Network: 'tests/eden/network/README.md'
+      - Phoronix: 'tests/eden/phoronix/README.md'
+      - Reboot: 'tests/eden/reboot/README.md'
+      - Registry: 'tests/eden/registry/README.md'
+      - Volume: 'tests/eden/volume/README.md'
+  - Libs:
+      - depgraph: 'libs/depgraph/README.md'
+      - zedUpload: 'libs/zedUpload/README.md'
+      - reconciler: 'libs/reconciler/README.md'
+  - EdgeView:
+      - Information: 'pkg/edgeview/README.md'
+  - Newlog:
+      - ipinfo: 'pkg/newlog/vendor/github.com/eriknordmark/ipinfo/README.md'
+      - 'go.uuid': 'pkg/newlog/vendor/github.com/satori/go.uuid/README.md'
+      - netdns: 'pkg/newlog/vendor/github.com/vishvananda/netns/README.md'
+      - unix: 'pkg/newlog/vendor/golang.org/x/sys/unix/README.md'
+      - fsnotify: 'pkg/newlog/vendor/github.com/fsnotify/fsnotify/README.md'
+      - logrus: 'pkg/newlog/vendor/github.com/sirupsen/logrus/README.md'
+  - Pillar:
+      - Introduction: 'pkg/pillar/README.md'
+      - Documentation: 'pkg/pillar/docs/README.md'
+      - containerd: 'pkg/pillar/containerd/README.md'
+      - domainmgr: 'pkg/pillar/docs/domainmgr.md'
+      - NIM: 'pkg/pillar/docs/nim.md'
+      - 'Radio silence': 'pkg/pillar/docs/radio-silence.md'
+      - tpmmgr: 'pkg/pillar/docs/tpmmgr.md'
+      - vaultmgr: 'pkg/pillar/docs/vaultmgr.md'
+      - volumemgr: 'pkg/pillar/docs/volumemgr.md'
+      - zedmanager: 'pkg/pillar/docs/zedmanager.md'
+      - zedrouter: 'pkg/pillar/docs/zedrouter.md'
+      - zedUpload: 'pkg/pillar/vendor/github.com/lf-edge/eve/libs/zedUpload/README.md'
+      - go-ieproxy: 'pkg/pillar/vendor/github.com/mattn/go-ieproxy/README.md'
+      - dns: 'pkg/pillar/vendor/github.com/miekg/dns/README.md'
+      - go-homedir: 'pkg/pillar/vendor/github.com/mitchellh/go-homedir/README.md'
+      - locker: 'pkg/pillar/vendor/github.com/moby/locker/README.md'
+      - term: 'pkg/pillar/vendor/github.com/moby/term/README.md'
+      - aec: 'pkg/pillar/vendor/github.com/morikuni/aec/README.md'
+      - go-connections: 'pkg/pillar/vendor/github.com/docker/go-connections/sockets/README.md'
+      - Azure: 'pkg/pillar/vendor/github.com/Azure/go-ansiterm/README.md'
+      - Focinfi: 'pkg/pillar/vendor/github.com/Focinfi/go-dns-resolver/README.md'
+      - go-winio: 'pkg/pillar/vendor/github.com/Microsoft/go-winio/README.md'
+      - hcsshim: 'pkg/pillar/vendor/github.com/Microsoft/hcsshim/README.md'
+      - wmi: 'pkg/pillar/vendor/github.com/StackExchange/wmi/README.md'
+      - godmi: 'pkg/pillar/vendor/github.com/VictorLowther/godmi/README.md'
+      - smart.go: 'pkg/pillar/vendor/github.com/anatol/smart.go/README.md'
+      - backoff: 'pkg/pillar/vendor/github.com/cenkalti/backoff/README.md'
+      - eve-tpm2-tools: 'pkg/pillar/vendor/github.com/cshari-zededa/eve-tpm2-tools/eventlog/README.md'
+      - tpm2: 'pkg/pillar/vendor/github.com/google/go-tpm/tpm2/README.md'
+      - gax-go: 'pkg/pillar/vendor/github.com/googleapis/gax-go/v2/apierror/internal/proto/README.md'
+      - mux: 'pkg/pillar/vendor/github.com/gorilla/mux/README.md'
+      - websocket: 'pkg/pillar/vendor/github.com/gorilla/websocket/README.md'
+      - zerconf: 'pkg/pillar/vendor/github.com/grandcat/zeroconf/README.md'
+      - gopac: 'pkg/pillar/vendor/github.com/jackwakefield/gopac/README.md'
+      - pcidb: 'pkg/pillar/vendor/github.com/jaypipes/pcidb/README.md'
+      - go-jmespath: 'pkg/pillar/vendor/github.com/jmespath/go-jmespath/README.md'
+      - go-pcap: 'pkg/pillar/vendor/github.com/packetcap/go-pcap/README.md'
+      - errors: 'pkg/pillar/vendor/github.com/pkg/errors/README.md'
+      - go.uuid: 'pkg/pillar/vendor/github.com/satori/go.uuid/README.md'
+      - w32: 'pkg/pillar/vendor/github.com/shirou/w32/README.md'
+      - go-fastping: 'pkg/pillar/vendor/github.com/tatsushid/go-fastping/README.md'
+      - netns: 'pkg/pillar/vendor/github.com/vishvananda/netns/README.md'
+      - plist: 'pkg/pillar/vendor/howett.net/plist/README.md'
+      - prometheus: 'pkg/pillar/vendor/github.com/prometheus/client_golang/prometheus/README.md'
+      - unix: 'pkg/pillar/vendor/golang.org/x/sys/unix/README.md'
+      - ipinfo: 'pkg/pillar/vendor/github.com/eriknordmark/ipinfo/README.md'
+      - cloud.google.com: 'pkg/pillar/vendor/cloud.google.com/go/README.md'
+      - 'cloud.google.com/Security': 'pkg/pillar/vendor/cloud.google.com/go/SECURITY.md'
+      - 'cloud.google.com/Testing': 'pkg/pillar/vendor/cloud.google.com/go/testing.md'
+      - 'cloud.google.com/internal': 'pkg/pillar/vendor/cloud.google.com/go/internal/README.md'
+      - 'cloud.google.com/IAM': 'pkg/pillar/vendor/cloud.google.com/go/iam/README.md'
+      - 'cloud.google.com/Storage': 'pkg/pillar/vendor/cloud.google.com/go/storage/README.md'
+      - go-libzfs: 'pkg/pillar/vendor/github.com/bicomsystems/go-libzfs/README.md'
+      - xxhash: 'pkg/pillar/vendor/github.com/cespare/xxhash/v2/README.md'
+      - containerd: 'pkg/pillar/vendor/github.com/containerd/containerd/README.md'
+      - 'containerd/sysx': 'pkg/pillar/vendor/github.com/containerd/continuity/sysx/README.md'
+      - 'containerd/fifo': 'pkg/pillar/vendor/github.com/containerd/fifo/readme.md'
+      - 'containerd/ttrpc': 'pkg/pillar/vendor/github.com/containerd/ttrpc/README.md'
+      - 'containerd/typeurl': 'pkg/pillar/vendor/github.com/containerd/typeurl/README.md'
+      - go-libvirt: 'pkg/pillar/vendor/github.com/digitalocean/go-libvirt/README.md'
+      - go-qemu: 'pkg/pillar/vendor/github.com/digitalocean/go-qemu/qmp/README.md'
+      - 'docker/client/': 'pkg/pillar/vendor/github.com/docker/docker/client/README.md'
+      - 'docker/socket/': 'pkg/pillar/vendor/github.com/docker/docker/pkg/stringid/README.md'
+      - distribution: 'pkg/pillar/vendor/github.com/docker/distribution/README.md'
+      - API: 'pkg/pillar/vendor/github.com/docker/docker/api/README.md'
+      - 'API versions': 'pkg/pillar/vendor/github.com/docker/docker/api/types/versions/README.md'
+      - go-events: 'pkg/pillar/vendor/github.com/docker/go-events/README.md'
+      - go-metrics: 'pkg/pillar/vendor/github.com/docker/go-metrics/README.md'
+      - go-units: 'pkg/pillar/vendor/github.com/docker/go-units/README.md'
+      - fsnotify: 'pkg/pillar/vendor/github.com/fsnotify/fsnotify/README.md'
+      - yaml: 'pkg/pillar/vendor/github.com/ghodss/yaml/README.md'
+      - go-ole: 'pkg/pillar/vendor/github.com/go-ole/go-ole/README.md'
+      - jwt: 'pkg/pillar/vendor/github.com/golang-jwt/jwt/README.md'
+      - 'go-containerregistry/authn': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/authn/README.md'
+      - 'go-containerregistry/name': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/name/README.md'
+      - 'go-containerregistry/partial': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/v1/partial/README.md'
+      - 'go-containerregistry/remote': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/v1/remote/README.md'
+      - 'go-containerregistry/transport': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/v1/remote/transport/README.md'
+      - 'go-containerregistry/stream': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/v1/stream/README.md'
+      - 'go-containerregistry/tarball': 'pkg/pillar/vendor/github.com/google/go-containerregistry/pkg/v1/tarball/README.md'
+      - gopacket: 'pkg/pillar/vendor/github.com/google/gopacket/README.md'
+      - uuid: 'pkg/pillar/vendor/github.com/google/uuid/README.md'
+      - ghw: 'pkg/pillar/vendor/github.com/jaypipes/ghw/README.md'
+      - 'compress': 'pkg/pillar/vendor/github.com/klauspost/compress/README.md'
+      - 'compress/fse': 'pkg/pillar/vendor/github.com/klauspost/compress/fse/README.md'
+      - 'compress/huff0': 'pkg/pillar/vendor/github.com/klauspost/compress/huff0/README.md'
+      - 'compress/zstd': 'pkg/pillar/vendor/github.com/klauspost/compress/zstd/README.md'
+      - 'compres/xxhash': 'pkg/pillar/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md'
+      - gomega: 'pkg/pillar/vendor/github.com/onsi/gomega/README.md'
+      - opencontainers: 'pkg/pillar/vendor/github.com/opencontainers/go-digest/README.md'
+      - 'opencontainers/pwalk': 'pkg/pillar/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md'
+      - 'opencontainers/pwalkdir': 'pkg/pillar/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md'
+      - sftp: 'pkg/pillar/vendor/github.com/pkg/sftp/README.md'
+      - procfs: 'pkg/pillar/vendor/github.com/prometheus/procfs/README.md'
+      - otto: 'pkg/pillar/vendor/github.com/robertkrimen/otto/README.markdown'
+      - 'otto/Design': 'pkg/pillar/vendor/github.com/robertkrimen/otto/DESIGN.markdown'
+      - 'otto/ast': 'pkg/pillar/vendor/github.com/robertkrimen/otto/ast/README.markdown'
+      - 'otto/file': 'pkg/pillar/vendor/github.com/robertkrimen/otto/file/README.markdown'
+      - 'otto/parser': 'pkg/pillar/vendor/github.com/robertkrimen/otto/parser/README.markdown'
+      - 'otto/registry': 'pkg/pillar/vendor/github.com/robertkrimen/otto/registry/README.markdown'
+      - 'otto/token': 'pkg/pillar/vendor/github.com/robertkrimen/otto/token/README.markdown'
+      - logrus: 'pkg/pillar/vendor/github.com/sirupsen/logrus/README.md'
+      - netlink: 'pkg/pillar/vendor/github.com/vishvananda/netlink/README.md'
+      - go.opencensus.io: 'pkg/pillar/vendor/go.opencensus.io/README.md'
+      - oauth2: 'pkg/pillar/vendor/golang.org/x/oauth2/README.md'
+      - appengine: 'pkg/pillar/vendor/google.golang.org/appengine/README.md'
+      - grpc: 'pkg/pillar/vendor/google.golang.org/grpc/README.md'
+      - sourcemap: 'pkg/pillar/vendor/gopkg.in/sourcemap.v1/README.md'
+      - yaml V2: 'pkg/pillar/vendor/gopkg.in/yaml.v2/README.md'
+      - yaml V3: 'pkg/pillar/vendor/gopkg.in/yaml.v3/README.md'
+  - Hypervisors:
+      - Xen: 'pkg/xen/README.md'
+  - Miscellaneous:
+      - RNGD: 'pkg/rngd/cmd/rngd/vendor/golang.org/x/sys/unix/README.md'
+      - U-boot: 'pkg/u-boot/README.md'
+      - UEFI: 'pkg/uefi/rpi/README.md'
+      - WLAN: 'pkg/wlan/FIXME.md'

docs/mkdocs/prepare_mkdocs.py

@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""This script prepares documentation files for mkdocs."""
+
+import os
+import sys
+import shutil
+import re
+
+# Output directory for document sources
+if len(sys.argv) <= 2:
+    print("Use: python", sys.argv[0], "<output-directory> <repository-url>")
+    sys.exit(1)
+else:
+    outdir  = sys.argv[1]
+    repourl = sys.argv[2]
+
+# Regex
+reLinks  = re.compile(r'\[([^\]]+)\]\(([^)]+)\)')
+reURL    = re.compile(r'[A-Za-z0-9]+://[A-Za-z0-9%-_]+(/[A-Za-z0-9%-_])*(#|\\?)[A-Za-z0-9%-_&=]*')
+reMAIL   = re.compile(r'mailto:.*')
+reANCHOR = re.compile(r'^#.*')
+
+# All extensions for document sources
+docsources = [".md", ".markdown", ".txt", ".png", ".svg", ".gif", ".jpg", ".jpeg"]
+# Extensions for markdown files
+mkdfiles   = [".md", ".markdown"]
+
+# Create output directory
+os.makedirs(outdir, exist_ok=True)
+
+# Search for all document sources in repository
+pwd = os.getcwd()
+for root, sub, files in os.walk(pwd):
+    for f in files:
+        fstr  = os.path.splitext(f)
+        fext  = fstr[len(fstr) - 1]
+        try:
+            # Only get files with extensions in the list
+            ext   = docsources.index(fext)
+            fpath = os.path.join(root, f)
+            rpath = os.path.relpath(fpath, start=pwd)
+
+            # Get directory path from file
+            dpath = os.path.dirname(rpath)
+
+            # Don't look on output directory
+            if os.path.commonpath([dpath, outdir]) == outdir:
+                break
+
+            # Create corresponding directory
+            npath = os.path.join(outdir, dpath)
+            if os.path.isdir(npath) is False:
+                try:
+                    os.makedirs(npath, exist_ok=True)
+                except OSError as error:
+                    print(error)
+
+            # Destination file
+            destfile = os.path.join(npath, f)
+
+            # Process document source file
+            try:
+                sidx = mkdfiles.index(docsources[ext])
+                with open(fpath, encoding=sys.getdefaultencoding()) as fp:
+                    with open(destfile, "w", encoding=sys.getdefaultencoding()) as fout:
+                        ftxt = fp.read()
+
+                        # Search for all markdown links in the file
+                        mdlinks = reLinks.findall(ftxt)
+
+                        for l in mdlinks:
+                            # If the link is not an URL, check if it points to a
+                            # non-document source file. If so, convert it to
+                            # an URL for source code repository
+                            link = l[1].strip()
+                            if reURL.match(link) or reANCHOR.match(link) or reMAIL.match(link):
+                                continue
+                            # Not an URL, remove anchor link (if exist)
+                            flink = re.sub(r'#.*','',link)
+
+                            try:
+                                # Get file extension of the link
+                                lstr = os.path.splitext(flink)
+                                lext = lstr[len(lstr) - 1]
+                                didx = docsources.index(lext)
+                                # It's a document source, we don't need to process
+                                continue
+                            except ValueError:
+                                # Build the full URL for the link
+                                lrpath = os.path.relpath(fpath, start=pwd)
+                                ldpath = os.path.dirname(lrpath)
+                                lurl   = repourl + "/" + ldpath + "/" + flink
+                                # Substitute link with the full URL in the text
+                                ftxt = ftxt.replace("(" + flink, "(" + lurl)
+
+                        fout.write(ftxt)
+                        fout.close()
+                        fp.close()
+            except ValueError:
+                # Not a source file, just copy the file
+                shutil.copy(fpath, destfile)
+        finally:
+            continue

docs/mkdocs/requirements.txt

@@ -0,0 +1,2 @@
+jinja2<3.1.0
+mkdocs==1.3.1