Merge pull request #183 from 18F/develop

10x final sync before P3 close-out

.github/workflows/xspecRunner.yml → .github/workflows/run-tests.yml

@@ -17,43 +17,39 @@ jobs:
       # Check-out the repository under $GITHUB_WORKSPACE
       - name: Checkout repository
         uses: actions/checkout@v2
-      - run: |
-          git submodule update --init --recursive
 
-      # Install Saxon HE to /tmp
-      - name: Install Saxon HE
-        run: |
-          echo "Installing Saxon"
-          mkdir -p /tmp/saxon
-          echo "Dowloading Saxon"
-          export SAXON_CP=/tmp/saxon/Saxon-HE-10.2.jar
-          wget -O "${SAXON_CP}" https://repo1.maven.org/maven2/net/sf/saxon/Saxon-HE/10.2/Saxon-HE-10.2.jar
-          echo "saxon_cp is ${SAXON_CP}"
+      - name: Read node version from `.nvmrc` file
+        id: nvmrc
+        uses: browniebroke/read-nvmrc-action@v1
+
+      - name: Install required node.js version
+        uses: actions/setup-node@v1
+        with:
+          node-version: "${{ steps.nvmrc.outputs.node_version }}"
 
-      # Run XSpec after the dependencies are completed
-      - name: Run XSpec
+      # Initialize the workspace with submodules and dependencies.
+      - name: Initialize workspace
+        run: make init
+
+      - name: Run test suite
         run: |
-          echo "Running XSpec"
-          cd $GITHUB_WORKSPACE/src/validations
-          export SAXON_CP=/tmp/saxon/Saxon-HE-10.2.jar
-          export TEST_DIR=$(pwd)/report/test
-          $GITHUB_WORKSPACE/vendor/xspec/bin/xspec.sh -s -j test/test_all.xspec
-      
+          make test-web test-validations
+
       # Sets the test report path for visibility
       - name: Publish XSpec Test Results
         uses: mikepenz/action-junit-report@v1
         with:
-          report_paths: '**/report/test/*junit.xml'
+          report_paths: "**/report/test/*junit.xml"
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
-      # publish the test summary as comment on the PR
+      # Publish the test summary as comment on the PR
       - name: Publish XSpec Test Results Summary
         uses: EnricoMi/[email protected]
         if: always()
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           check_name: XSpec Test Results
-          files: '**/report/test/*junit.xml'
+          files: "**/report/test/*junit.xml"
           report_individual_runs: true
           deduplicate_classes_by_file_name: false
 
@@ -64,4 +60,4 @@ jobs:
           path: |
             ./src/validations/report/schematron/**/*.*
             ./src/validations/report/test/**/*.*
-          if-no-files-found: error
+          if-no-files-found: error

Makefile

@@ -1,67 +1,30 @@
-REQUIRED_NODE_VERSION = $(shell cat .nvmrc)
-INSTALLED_NODE_VERSION = $(shell node --version)
+export BASE_DIR=$(shell pwd)
+
+# Most of the real work of the build is in sub-project Makefiles.
+include src/examples/module.mk
+include src/validations/module.mk
+include src/web/module.mk
 
 .PHONY: help
 
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+	@grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 all: clean test build  ## Complete clean build with tests
 
-init: init-repo init-web  ## Initialize project dependencies
-
-node:
-ifneq ($(REQUIRED_NODE_VERSION),$(INSTALLED_NODE_VERSION))
-	$(error node.js version $(REQUIRED_NODE_VERSION) required)
-endif
+init: init-repo init-validations init-web  ## Initialize project dependencies
 
 init-repo:
 	git submodule update --init --recursive
 
-init-web: node
-	cd src/web && \
-		npm install
-
 clean: clean-dist clean-validations clean-web  ## Clean all
 
 clean-dist:  ## Clean non-RCS-tracked dist files
 	@echo "Cleaning dist..."
 	git clean -xfd dist
 
-clean-validations:  ## Clean validations artifact
-	@echo "Cleaning validations..."
-	cd src/validations \
-		rm -rf report target
-
-clean-web:  ## Clean web artifacts
-	@echo "Cleaning web..."
-	cd src/web && \
-		npm run clean
-
 test: test-validations test-web test-examples ## Test all
 
-test-validations:  ## Test validations
-	@echo "Running validations tests..."
-	cd src/validations && \
-		../../vendor/xspec/bin/xspec.sh -s -j test/test_all.xspec
-
-test-web:  ## Test web codebase
-	@echo "Running web tests..."
-	cd src/web && \
-		npm run test
-
-test-examples: test-example-java test-example-python  ## Test example code projects
-
-test-example-java:  ## Test example Java project
-	@echo "Verifying Java example..."
-	cd src/examples/java && \
-		docker-compose run example mvn test
-
-test-example-python:  ## Test example Python project
-	@echo "Verifying Python example..."
-	cd src/examples/python && \
-		docker-compose run example pytest
-
 build: build-validations build-web dist  ## Build all artifacts and copy into dist directory
 	# Symlink for Federalist
 	ln -sf ./src/web/build _site
@@ -70,13 +33,3 @@ build: build-validations build-web dist  ## Build all artifacts and copy into di
 	mkdir -p dist/validations
 	cp src/validations/target/ssp.xsl dist/validations
 	cp -r src/validations/rules/ssp.sch dist/validations
-
-build-validations:  ## Build Schematron validations
-	@echo "Building Schematron validations..."
-	cd src/validations && \
-		./bin/validate_with_schematron.sh
-
-build-web: node ## Build web bundle
-	@echo "Building web bundle..."
-	cd src/web && \
-	  npm run build

src/examples/module.mk

@@ -0,0 +1,11 @@
+test-examples: test-example-java test-example-python  ## Test example code projects
+
+test-example-java:  ## Test example Java project
+	@echo "Verifying Java example..."
+	cd src/examples/java && \
+		docker-compose run example mvn test
+
+test-example-python:  ## Test example Python project
+	@echo "Verifying Python example..."
+	cd src/examples/python && \
+		docker-compose run example pytest

src/validations/CONTRIBUTING.md

@@ -6,7 +6,7 @@ All directory references are local to the `fedramp-automation/src/validations`
 
 * `bin` has the validation script.
 * `docs/adr` has a list of [Architectural Decision Records](https://adr.github.io) in which the product team documented technical decisions for the project.
-* `report/test` for the XSpec outputs.
+* `report/test` for XSpec and SCH test outputs
 * `report/schematron` for the final validations in Schematron SVRL reporting format.
 * `rules` has the Schematron files for the SSP.
 * `sch` for the XSpec and Schematron files.
@@ -70,12 +70,22 @@ docker-compose run \
 
 ## Unit Test
 
-**Prerequesite**: *To ensure that you have all required dependencies (see .gitmodules), run the following command:*
+Unit tests consist of:
+
+- src/validations/test/*.xspec
+- src/validations/styleguides/sch.sch
+
+A make target is provided to run all unit tests:
 
 ```sh
-git submodule update --init --recursive
+# If you haven't done so previously: initialize your workspace.
+make init
+# Run xspec and Schematron tests
+make test-validations
 ```
 
+Alternately, you may follow the instructions below to run the tests manually.
+
 ### Running Unit Test
 
 Run the unit tests from with `./fedramp-automation/src/validations` directory. The `SAXON_CP` and `TEST_DIR` environment variables must be set to the **Saxon HE** and test report directories, respectively. *Note that you may choose to run your preferred version of the Saxon HE JAR file.*

src/validations/bin/assert-svrl.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+import xml.etree.ElementTree as ET
+
+
+def assert_svrl(svrl_string):
+    root = ET.fromstring(svrl_string)
+    failed_asserts = root.findall('{http://purl.oclc.org/dsdl/svrl}failed-assert')
+    for failed_assert in failed_asserts:
+        text = failed_assert.find('{http://purl.oclc.org/dsdl/svrl}text').text
+        diagnostic = failed_assert.find('{http://purl.oclc.org/dsdl/svrl}diagnostic-reference').text
+        location = failed_assert.get('location')
+        print(f"""* failed-assert at {location}
+    {text}
+    {diagnostic}
+        """)
+    print(f'Found {len(failed_asserts)} failed assertions')
+    return len(failed_asserts)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(prog='assert_svrl')
+    parser.add_argument('file_name')
+    arguments = parser.parse_args(sys.argv[1:])
+    with open(arguments.file_name, 'r') as f:
+        svrl_string = f.read()
+        assertion_count = assert_svrl(svrl_string)
+        sys.exit(assertion_count)

src/validations/bin/compile-sch.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+SOURCE=$1
+DESTINATION=$2
+
+echo "Preprocessing stage 1/3 of ${SOURCE}..."
+STAGE_1=$(mktemp)
+java -cp "${SAXON_CP}" net.sf.saxon.Transform \
+    -o:"${STAGE_1}" \
+    -s:"${SOURCE}" \
+    "${BASE_DIR}/vendor/schematron/trunk/schematron/code/iso_dsdl_include.xsl" \
+    ${SAXON_OPTS}
+
+echo "Preprocessing stage 2/3 of ${SOURCE}..."
+STAGE_2=$(mktemp)
+java -cp "${SAXON_CP}" net.sf.saxon.Transform \
+    -o:"${STAGE_2}" \
+    -s:"${STAGE_1}" \
+    "${BASE_DIR}/vendor/schematron/trunk/schematron/code/iso_abstract_expand.xsl" \
+    ${SAXON_OPTS}
+
+echo "Preprocessing stage 3/3 of ${SOURCE}..."
+java -cp "${SAXON_CP}" net.sf.saxon.Transform \
+    -o:"${DESTINATION}" \
+    -s:"${STAGE_2}" \
+    "${BASE_DIR}/vendor/schematron/trunk/schematron/code/iso_svrl_for_xslt2.xsl" \
+    ${SAXON_OPTS}

src/validations/bin/evaluate-compiled-schematron.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+SOURCE_XSL=$1
+SOURCE_XML=$2
+SVRL_DESTINATION=$3
+
+java -cp "${SAXON_CP}" net.sf.saxon.Transform \
+    -o:"${SVRL_DESTINATION}" \
+    -s:"${SOURCE_XML}" \
+    "${SOURCE_XSL}" \
+    ${SAXON_OPTS}
+if "${BASE_DIR}/src/validations/bin/assert-svrl.py" "${SVRL_DESTINATION}" ; then
+  echo "Schematron evaluation succeeded"
+  exit 0
+else
+  echo "Schematron evaluation failed"
+  exit 1
+fi

src/validations/module.mk

@@ -0,0 +1,41 @@
+SAXON_VERSION := 10.5
+SAXON_JAR := Saxon-HE-$(SAXON_VERSION).jar
+SAXON_LOCATION := saxon/Saxon-HE/$(SAXON_VERSION)/$(SAXON_JAR)
+SAXON_URL := https://repo1.maven.org/maven2/net/sf/$(SAXON_LOCATION)
+export SAXON_OPTS = allow-foreign=true diagnose=true
+export SAXON_CP = $(BASE_DIR)/vendor/$(SAXON_JAR)
+
+VALIDATIONS_DIR := $(BASE_DIR)/src/validations
+
+COMPILE_SCH := $(VALIDATIONS_DIR)/bin/compile-sch.sh
+EVAL_SCHEMATRON := $(VALIDATIONS_DIR)/bin/evaluate-compiled-schematron.sh
+EVAL_XSPEC := TEST_DIR=$(VALIDATIONS_DIR)/report/test $(BASE_DIR)/vendor/xspec/bin/xspec.sh -s -j
+
+init-validations: $(SAXON_CP)  ## Initialize validations dependencies
+
+$(SAXON_CP):  ## Download Saxon-HE to the vendor directory
+	curl -H "Accept: application/zip" -o "$(SAXON_CP)" "$(SAXON_URL)" &> /dev/null
+
+clean-validations:  ## Clean validations artifact
+	@echo "Cleaning validations..."
+	rm -rf $(VALIDATIONS_DIR)/target
+	git clean -xfd $(VALIDATIONS_DIR)/report
+
+test-validations: $(SAXON_CP) test-xspec test-sch  ## Test validations
+
+test-xspec: $(VALIDATIONS_DIR)/test/test_all.xspec
+	$(EVAL_XSPEC) $^
+
+$(VALIDATIONS_DIR)/target/%.sch.xsl: $(VALIDATIONS_DIR)/styleguides/%.sch
+	$(COMPILE_SCH) $^ $@
+
+$(VALIDATIONS_DIR)/report/test/%.svrl.xml: $(VALIDATIONS_DIR)/target/%.sch.xsl $(VALIDATIONS_DIR)/rules/ssp.sch
+	$(EVAL_SCHEMATRON) $^ $@
+
+test-sch: $(VALIDATIONS_DIR)/report/test/sch.svrl.xml $(VALIDATIONS_DIR)/report/test/xspec.svrl.xml
+
+$(VALIDATIONS_DIR)/target/ssp.xsl: $(VALIDATIONS_DIR)/rules/ssp.sch
+	@echo "Building Schematron validations..."
+	$(COMPILE_SCH) $^ $@
+
+build-validations: $(SAXON_CP) $(VALIDATIONS_DIR)/target/ssp.xsl ## Build Schematron validations

src/validations/rules/Writing_FedRAMP_Validations.md

@@ -45,7 +45,7 @@ Schematron constraints have thus far been for system security plan documents. Th
 
 A companion document [`fedramp_values.xml`](../../dist/content/resources/xml/fedramp_values.xml) defines sets of allowed values for various FedRAMP OSCAL XML elements and their attributes.
 
-An additional Schematron document [`sch.sch`](../sch/sch.sch) defines a style guide for FedRAMP Schematron to ensure developers continue to follow best practices. These augmentations provide
+An additional Schematron document [`sch.sch`](../styleguides/sch.sch) defines a style guide for FedRAMP Schematron to ensure developers continue to follow best practices. These augmentations provide
 
 - References to associated unit tests ([ssp.xspec](../test/ssp.xspec))
 - References to FedRAMP documentation related to the constraint
@@ -58,7 +58,7 @@ Use a validating (XML) editor with the `sch.sch` employed in order to enforce th
 
 The following examples use XML namespace prefixes declared at the root element of the Schematron document.
 
-```
+```xml
 <sch:schema
     queryBinding="xslt2"
     xmlns:array="http://www.w3.org/2005/xpath-functions/array"
@@ -151,7 +151,7 @@ and the following code block
 </sch:rule>
 ```
 
-The `<sch:rule>` element sets the locus within the document to be used as a relative reference for the subordinate assertions (`<sch:assert>` elements). In this case, the locus is `oscal:system-characteristics`: an XPath statement identifying the context element using a (namespace-)qualified element name. Statement subordinate to `<rule>` can refer to the con text implicitly or explicitly using the XPath function `current()`.
+The `<sch:rule>` element sets the locus within the document to be used as a relative reference for the subordinate assertions (`<sch:assert>` elements). In this case, the locus is `oscal:system-characteristics`: an XPath statement identifying the context element using a (namespace-)qualified element name. Statement subordinate to `<rule>` can refer to the context implicitly or explicitly using the XPath function `current()`.
 
 The first assertion `id="has-system-id"`has a test (an XPath statement in the `test` attribute stating `oscal:system-id[@identifier-type eq 'https://fedramp.gov']` which asserts "the `oscal:system-characteristics` element has a child element `oscal:system-id` which must have an `@identifier-type eq 'https://fedramp.gov'` with a specific value. The natural language prose equivalent "A FedRAMP SSP must have a FedRAMP system identifier." resides within the `<sch:assert>` element.
 
@@ -175,7 +175,7 @@ The following assertion (`id="has-fedramp-authorization-type"`) tests that a chi
 
 ## Example 2 - intra-document references
 
-This exmaple illustrates intra-document references relative to the context.
+This example illustrates intra-document references relative to the context.
 
 Consider the following document fragment
 ```xml
@@ -262,14 +262,14 @@ The second (`id="responsible-party-has-party-uuid"`) assertion is just an existe
 
 The third (`id="responsible-party-has-definition"`) assertion is an existential test that every `<party-uuid>` reference (there may be more than one subordinate to `<responsible-party>`) references a `<party>` anywhere within the ndocument. 
 
-The fourth () assertion ensures that specific role types identify a responsible **person** (i.e., a `<party>` defined as a person).
+The fourth (`id="responsible-party-is-person"`) assertion ensures that specific role types identify a responsible **person** (i.e., a `<party>` defined as a person).
 
 ## Example 3 - remote resource reference
 
 This example uses remote HTTPS resource availability to ensure a Cryptographic Module Validation Program (CMVP) citation is actually found on the related NIST web site.
 
 Consider the following document fragment
-```
+```xml
 <component uuid="95beec7e-6f82-4aaa-8211-969cd7c1f1ab" type="validation">
     <title>[SAMPLE]Module Name</title>
     <description>
@@ -284,7 +284,7 @@ Consider the following document fragment
 </component>
 ```
 and the following code block
-```
+```xml
 <sch:rule
     context="oscal:link[@rel eq 'validation-details']"
     doc:guide-reference="Guide to OSCAL-based FedRAMP System Security Plans Appendix A">
@@ -319,7 +319,7 @@ The second (`id="has-accessible-CMVP-validation-details"`) assertion uses the XP
 
 The third (`id="has-consonant-CMVP-validation-details"`) assertion uses the preceding-sibling axis to ensure that the certificate number in the `<link>` href matches the value found in the preceding sibling `<prop name="validation-reference">`. (The OSCAL SSP XML Schema constrains the prop to precede the link element.)
 
-## Example 5 - Use of JSON in XPath
+## Example 4 - Use of JSON in XPath
 
 This example uses a network-resident JSON document to validate the FedRAMP package ID of the system security plan.