Add a link checker

Tim Bannister · celestehorgan · commit 2eb6ab1ad349 · 2020-05-26T16:06:46.000-07:00
Signed-off-by: Celeste Horgan &lt;celeste@cncf.io&gt;
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,4 @@ resources/
 # Netlify Functions build output
 package-lock.json
 functions/
-node_modules/
+node_modules/
diff --git a/.htmltest.yml b/.htmltest.yml
@@ -0,0 +1,16 @@
+DirectoryPath: public/docs
+IgnoreDirectoryMissingTrailingSlash: true
+CheckExternal: false
+IgnoreAltMissing: true
+CheckImages: false
+CheckScripts: false
+CheckMeta: false
+CheckMetaRefresh: false
+CheckLinks: false
+EnforceHTML5: false
+EnforceHTTPS: false
+IgnoreDirectoryMissingTrailingSlash: false
+IgnoreInternalEmptyHash: true
+IgnoreEmptyHref: true
+IgnoreDirs:
+  - "reference/generated/kubernetes-api"
diff --git a/Makefile b/Makefile
@@ -50,3 +50,11 @@ docker-serve:
 test-examples:
 	scripts/test_examples.sh install
 	scripts/test_examples.sh run
+
+.PHONY: link-checker-setup
+link-checker-image-pull:
+	docker pull wjdp/htmltest
+
+docker-internal-linkcheck: link-checker-image-pull
+	$(DOCKER_RUN) $(DOCKER_IMAGE) hugo --config config.toml,linkcheck-config.toml --buildFuture
+	$(DOCKER) run --mount type=bind,source=$(CURDIR),target=/test --rm wjdp/htmltest htmltest
diff --git a/config.toml b/config.toml
@@ -313,4 +313,4 @@ contentDir = "content/uk"
 [languages.uk.params]
 time_format_blog = "02.01.2006"
 # A list of language codes to look for untranslated content, ordered from left to right.
-language_alternatives = ["en"]
+language_alternatives = ["en"]
diff --git a/content/en/docs/concepts/architecture/cloud-controller.md b/content/en/docs/concepts/architecture/cloud-controller.md
@@ -212,4 +212,4 @@ The cloud controller manager uses Go interfaces to allow implementations from an
 The implementation of the shared controllers highlighted in this document (Node, Route, and Service), and some scaffolding along with the shared cloudprovider interface, is part of the Kubernetes core. Implementations specific to cloud providers are outside the core of Kubernetes and implement the `CloudProvider` interface.
 
 For more information about developing plugins, see [Developing Cloud Controller Manager](/docs/tasks/administer-cluster/developing-cloud-controller-manager/).
-{{% /capture %}}
+{{% /capture %}}
diff --git a/content/en/docs/contribute/new-content/overview.md b/content/en/docs/contribute/new-content/overview.md
@@ -54,5 +54,8 @@ was wrong, you (and only you, the submitter) can change it.
 
 Limit pull requests to one language per PR. If you need to make an identical change to the same code sample in multiple languages, open a separate PR for each language.
 
+## Tools for contributors
+
+The [doc contributors tools](https://github.com/kubernetes/website/tree/master/content/en/docs/doc-contributor-tools) directory in the `kubernetes/website` repository contains tools to help your contribution journey go more smoothly.
 
 {{% /capture %}}
diff --git a/content/en/docs/doc-contributor-tools/linkchecker/README.md b/content/en/docs/doc-contributor-tools/linkchecker/README.md
@@ -0,0 +1,76 @@
+# Internal link checking tool
+
+You can use [htmltest](https://github.com/wjdp/htmltest) to check for broken links in [`/content/en/`](https://git.k8s.io/website/content/en/). This is useful when refactoring sections of content, moving pages around, or renaming files or page headers.
+
+## How the tool works
+
+`htmltest` scans links in the generated HTML files of the kubernetes website repository. It runs using a `make` command which does the following:
+
+- Builds the site and generates output HTML in the `/public` directory of your local `kubernetes/website` repository
+- Pulls the `wdjp/htmltest` Docker image
+- Mounts your local `kubernetes/website` repository to the Docker image
+- Scans the files generated in the `/public` directory and provides command line output when it encounters broken internal links
+
+## What it does and doesn't check
+
+The link checker scans generated HTML files, not raw Markdown. The htmltest tool depends on a configuration file, [`.htmltest.yml`](https://git.k8s.io/website/.htmltest.yml), to determine which content to examine.
+
+The link checker scans the following:
+
+- All content generated from Markdown in [`/content/en/docs`](https://git.k8s.io/website/content/en/docs/) directory, excluding:
+  - Generated API references, for example https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/
+- All internal links, excluding:
+  - Empty hashes (`<a href="#">` or `[title](#)`) and empty hrefs (`<a href="">` or `[title]()`)
+  - Internal links to images and other media files
+
+The link checker does not scan the following:
+
+- Links included in the top and side nav bars, footer links, or links in a page's `<head>` section, such as links to CSS stylesheets, scripts, and meta information
+- Top level pages and their children, for example: `/training`, `/community`, `/case-studies/adidas`
+- Blog posts
+- API Reference documentation, for example: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/
+- Localizations
+
+## Prerequisites and installation
+
+You must install
+* [Docker](https://docs.docker.com/get-docker/)
+* [make](https://www.gnu.org/software/make/)
+
+## Running the link checker
+
+To run the link checker:
+
+1. Navigate to the root directory of your local `kubernetes/website` repository.
+
+2. Run the following command:
+
+  ```
+  make docker-internal-linkcheck
+  ```
+
+## Understanding the output
+
+If the link checker finds broken links, the output is similar to the following:
+
+```
+tasks/access-kubernetes-api/custom-resources/index.html
+  hash does not exist --- tasks/access-kubernetes-api/custom-resources/index.html --> #preserving-unknown-fields
+  hash does not exist --- tasks/access-kubernetes-api/custom-resources/index.html --> #preserving-unknown-fields
+```
+
+This is one set of broken links. The log adds an output for each page with broken links.
+
+In this output, the file with broken links is `tasks/access-kubernetes-api/custom-resources.md`.
+
+The tool gives a reason: `hash does not exist`. In most cases, you can ignore this.
+
+The target URL is `#preserving-unknown-fields`.
+
+One way to fix this is to:
+
+1. Navigate to the Markdown file with a broken link.
+2. Using a text editor, do a full-text search (usually Ctrl+F or Command+F) for the broken link's URL, `#preserving-unknown-fields`.
+3. Fix the link. For a broken page hash (or _anchor_) link, check whether the topic was renamed or removed.
+
+Run htmltest to verify that broken links are fixed.
diff --git a/content/en/docs/home/_index.md b/content/en/docs/home/_index.md
@@ -15,7 +15,7 @@ menu:
     title: "Documentation"
     weight: 20
     post: >
-      <p>Learn how to use Kubernetes with conceptual, tutorial, and reference documentation. You can even <a href="/editdocs/" data-auto-burger-exclude>help contribute to the docs</a>!</p>
+      <p>Learn how to use Kubernetes with conceptual, tutorial, and reference documentation. You can even <a href="/editdocs/" data-auto-burger-exclude data-proofer-ignore>help contribute to the docs</a>!</p>
 description: >
   Kubernetes is an open source container orchestration engine for automating deployment, scaling, and management of containerized applications. The open source project is hosted by the Cloud Native Computing Foundation.
 overview: >
@@ -38,7 +38,7 @@ cards:
   button_path: "/docs/setup"
 - name: tasks
   title: "Learn how to use Kubernetes"
-  description: "Look up common tasks and how to perform them using a short sequence of steps."  
+  description: "Look up common tasks and how to perform them using a short sequence of steps."
   button: "View Tasks"
   button_path: "/docs/tasks"
 - name: training
@@ -62,4 +62,4 @@ cards:
 - name: about
   title: About the documentation
   description: This website contains documentation for the current and previous 4 versions of Kubernetes.
----
+---
diff --git a/layouts/partials/docs/content-page.html b/layouts/partials/docs/content-page.html
@@ -1,7 +1,7 @@
 {{- $filepath := .page.File.Path }}
 {{- $editLink := printf "https://github.com/kubernetes/website/edit/master/content/%s/%s" .page.Language.Lang $filepath }}
 <p>
-  <a href="{{ $editLink }}" id="editPageButton" target="_blank">
+  <a href="{{ $editLink }}" id="editPageButton" target="_blank" data-proofer-ignore>
     Edit This Page
   </a>
 </p>
@@ -15,4 +15,4 @@ <h1>{{ .page.Title }}</h1>
 {{ .page.TableOfContents }}
 {{ end }}
 {{ .page.Content }}
-{{ end }}
+{{ end }}
diff --git a/layouts/partials/docs/top-menu.html b/layouts/partials/docs/top-menu.html
@@ -12,10 +12,10 @@ <h5>{{ .Params.abstract }}</h5>
 	<ul>
 		{{ range $menuSections }}
 		{{ $yah := $p.IsDescendant . }}
-		<li><a href="{{ .RelPermalink }}"{{ if $yah }} class="YAH"{{ end }}>{{ .LinkTitle | upper }}</a></li>
+		<li><a href="{{ .RelPermalink }}"{{ if $yah }} class="YAH"{{ end }} data-proofer-ignore>{{ .LinkTitle | upper }}</a></li>
 		{{ end }}
 	</ul>
 	<form id="searchBox" action="/docs/search/" role="search">
 		<input type="text" id="search" name="q" placeholder="{{ T "ui_search_placeholder" }}" aria-label="Search">
 	</form>
-</div>
+</div>
diff --git a/layouts/partials/footer.html b/layouts/partials/footer.html
@@ -1,10 +1,10 @@
 <footer>
     <div class="light-text main-section">
         <nav>
-            {{ with site.GetPage "page" "docs/tutorials/stateless-application/hello-minikube" }}<a href="{{ .RelPermalink }}">{{ .LinkTitle }}</a>{{ end }}
+            {{ with site.GetPage "page" "docs/tutorials/stateless-application/hello-minikube" }}<a href="{{ .RelPermalink }}" data-proofer-ignore>{{ .LinkTitle }}</a>{{ end }}
             {{ $sections := slice "docs/home" "blog" "training" "partners" "community" "case-studies" }}
             {{ range $sections }}
-            {{ with site.GetPage "section" . }}<a href="{{ .RelPermalink }}">{{ .LinkTitle }}</a>{{ end }}
+            {{ with site.GetPage "section" . }}<a href="{{ .RelPermalink }}" data-proofer-ignore>{{ .LinkTitle }}</a>{{ end }}
             {{ end }}
         </nav>
         <div class="social" role="region" aria-label="Social hyperlinks">
@@ -34,4 +34,4 @@
             ICP license: 京ICP备17074266号-3
         </div>
     </div>
-</footer>
+</footer>
diff --git a/layouts/partials/header.html b/layouts/partials/header.html
@@ -1,40 +1,40 @@
 <div id="cellophane" onclick="kub.toggleMenu()"></div>
 
 <header>
-    <a href="{{ site.Home.RelPermalink }}" class="logo" title="{{ site.Home.Title }} - {{ site.Params.title }}" aria-label="Kubernetes website"></a>
+    <a href="{{ site.Home.RelPermalink }}" class="logo" title="{{ site.Home.Title }} - {{ site.Params.title }}" aria-label="Kubernetes website" data-proofer-ignore></a>
 
     <div class="nav-buttons" data-auto-burger="primary">
         <ul class="global-nav">
             {{ $sections := slice "docs" "blog" "training" "partners" "community" "case-studies" }}
             {{ range $sections }}
-            {{ with site.GetPage "section" . }}<li><a href="{{ .RelPermalink }}"{{ if eq .Section $.Section }} class="active"{{ end}}>{{ .LinkTitle }}</a></li>{{ end }}
+            {{ with site.GetPage "section" . }}<li><a href="{{ .RelPermalink }}"{{ if eq .Section $.Section }} class="active"{{ end}} data-proofer-ignore>{{ .LinkTitle }}</a></li>{{ end }}
             {{ end }}
             {{/* Link directly to documentation etc., if possible. */}}
             {{ $langPage := cond (gt (len .Translations) 0) . site.Home }}
              <li>
-                <a href="#">
+                <a href="#" data-proofer-ignore>
                     {{ $langPage.Language.LanguageName }} <span class="ui-icon ui-icon-carat-1-s"></span>
                 </a>
                 <ul>
                 {{ range $langPage.Translations }}
-                    <li><a href="{{ .RelPermalink }}">{{ .Language.LanguageName }}</a></li>
+                    <li><a href="{{ .RelPermalink }}" data-proofer-ignore>{{ .Language.LanguageName }}</a></li>
                 {{ end }}
                 </ul>
             </li>
 
             <li>
-                <a href="#">
+                <a href="#" data-proofer-ignore>
                     {{ site.Params.version }} <span class="ui-icon ui-icon-carat-1-s"></span>
                 </a>
                 <ul>
                 {{ range site.Params.versions }}
-                    <li><a href="{{ .url }}{{ $.RelPermalink }}">{{ .version }}</a></li>
+                    <li><a href="{{ .url }}{{ $.RelPermalink }}" data-proofer-ignore>{{ .version }}</a></li>
                 {{ end }}
                 </ul>
             </li>
         </ul>
         {{ with site.GetPage "section" "docs/tutorials/kubernetes-basics" }}
-        <a href="{{ .RelPermalink }}" class="button" id="tryKubernetes" data-auto-burger-exclude>{{ .LinkTitle }}</a>
+        <a href="{{ .RelPermalink }}" class="button" id="tryKubernetes" data-auto-burger-exclude data-proofer-ignore>{{ .LinkTitle }}</a>
         {{ end }}
 
         <button id="hamburger" onclick="kub.toggleMenu()" data-auto-burger-exclude><div></div></button>
@@ -44,7 +44,7 @@
         <div class="main-section" data-auto-burger="primary">
          {{ range site.Menus.main }}
            <div class="nav-box">
-            <h3><a href="{{ .URL }}">{{ .Title }}</a></h3>
+            <h3><a href="{{ .URL }}" data-proofer-ignore>{{ .Title }}</a></h3>
            {{ .Post }}
             </div>
          {{ end }}
@@ -70,4 +70,4 @@ <h5 class="github-invite">{{ T "main_community_explore" }}</h5>
             <div class="clear" style="clear: both"></div>
         </div>
     </nav>
-</header>
+</header>
diff --git a/layouts/partials/tree.html b/layouts/partials/tree.html
@@ -1,9 +1,9 @@
-<a class="item" data-title="{{ .LinkTitle }}" href="{{ .RelPermalink }}"></a>
+<a class="item" data-title="{{ .LinkTitle }}" href="{{ .RelPermalink }}" data-proofer-ignore></a>
 {{ template "section-tree-nav" (dict "ctx" . "section" .) }}
 {{ define "section-tree-nav" }}
 	{{ $pages :=  (union .section.Pages .section.Sections) }}
 	{{ with site.Params.language_alternatives }}
-		{{ range . }}	
+		{{ range . }}
 			{{ with (where $.section.Translations ".Lang" . ) }}
 			{{ $p := index . 0 }}
 			{{ $pages =  $pages | lang.Merge (union $p.Pages $p.Sections) }}
@@ -25,13 +25,13 @@
 		{{ if ge (len .section.Content) 10 }}
 		{{/* The section page has content, so link to it. */}}
 		{{ $isForeignLanguage := (ne .section.Lang $.ctx.Lang)}}
-		<a class="item" {{ if $isForeignLanguage }}target="_blank" {{ end }}data-title="{{ .section.LinkTitle }}{{ if $isForeignLanguage }} <small>({{ .section.Lang | upper }})</small>{{ end }}" href="{{ .section.RelPermalink }}"></a>
+		<a class="item" {{ if $isForeignLanguage }}target="_blank" {{ end }}data-title="{{ .section.LinkTitle }}{{ if $isForeignLanguage }} <small>({{ .section.Lang | upper }})</small>{{ end }}" href="{{ .section.RelPermalink }}" data-proofer-ignore></a>
 		{{ end }}
 		{{ template "section-tree-nav" . }}
 	</div>
 </div>
 {{ end }}
 {{ define "section-tree-nav-page" }}
 {{ $isForeignLanguage := (ne .page.Lang $.ctx.Lang)}}
-<a class="item" {{ if $isForeignLanguage }}target="_blank" {{ end }}data-title="{{ .page.LinkTitle }}{{ if $isForeignLanguage }} <small>({{ .page.Lang | upper }})</small>{{ end }}" href="{{ .page.RelPermalink }}"></a>
-{{ end }}
+<a class="item" {{ if $isForeignLanguage }}target="_blank" {{ end }}data-title="{{ .page.LinkTitle }}{{ if $isForeignLanguage }} <small>({{ .page.Lang | upper }})</small>{{ end }}" href="{{ .page.RelPermalink }}" data-proofer-ignore></a>
+{{ end }}
diff --git a/linkcheck-config.toml b/linkcheck-config.toml
@@ -0,0 +1 @@
+canonifyURLs = true
