feature(api): /scan endpoint for CI integration (#65)

.gitignore

@@ -5,4 +5,5 @@ api-examples.http
 mdn-observatory-webext/dist
 .DS_Store
 compare_output.txt
-load-script.js
+load-script.js
+.vscode/

README.md

@@ -1,65 +1,150 @@
-# Welcome to Mozilla's MDN Observatory
+# Welcome to Mozilla's HTTP Observatory
 
-MDN HTTP Observatory is a library and service that checks web sites for security-relevant headers.
-
-MDN HTTP Observatory is hosted by [MDN Web Docs](https://github.com/mdn).
+[HTTP Observatory](https://developer.mozilla.org/en-US/observatory/) is a service that checks web sites for security-relevant headers. It is hosted by [MDN Web Docs](https://github.com/mdn).
 
 ## Getting Started
 
-To get up and running, follow these steps:
+If you just want to scan a host, please head over to <https://developer.mozilla.org/en-US/observatory/>. If you want to
+run the code locally or on your premises, continue reading.
 
-<!--
+### Installation
 
-TODO:
+Install dependencies by running this from the root of the repository:
 
-Include enough details to get started using the project here and link to other docs with more detail as needed.
-This should look like:
+```sh
+npm i
+```
 
-- quick installation/build instructions
-- a few simple examples of use
+### Running a local scan
 
-More detailed build instructions (e.g., prerequisites and testing hints) should be in the CONTRIBUTING.md file.
--->
+To run a scan on a host, a command line script `scan` is available. It returns the a JSON of the form described below. For example, to scan `mdn.dev`:
 
-## Contributing
+```sh
+./scan mdn.dev
 
-Our project welcomes contributions from any member of our community.
-To get started contributing, please see our [Contributor Guide](CONTRIBUTING.md).
+{
+  "scan": {
+    "algorithmVersion": 4,
+    "grade": "A+",
+    "error": null,
+    "score": 105,
+    "statusCode": 200,
+    "testsFailed": 0,
+    "testsPassed": 10,
+    "testsQuantity": 10,
+    "responseHeaders": {
+      ...
+    }
+  },
+  "tests": {
+    "cross-origin-resource-sharing": {
+      "expectation": "cross-origin-resource-sharing-not-implemented",
+      "pass": true,
+      "result": "cross-origin-resource-sharing-not-implemented",
+      "scoreModifier": 0,
+      "data": null
+    },
+    ...
+  }
+}
 
-By participating in and contributing to our projects and discussions, you acknowledge that you have read and agree to our [Code of Conduct](CODE_OF_CONDUCT.md).
+```
 
-<!-- ## Resources
+### Running a local API server
 
-For more information about MDN HTTP Observatory, see the following resources: -->
+This needs a [postgres](https://www.postgresql.org/) database for the API to use as a persistence layer. All scans and results initiated via the API are stored in the database.
 
-<!-- [TODO: Add links to other helpful information (roadmap, docs, website, etc.)] -->
+#### Configuration
 
-## Communications
+Default configuration is read from a default `config/config.json` file. See [this file](src/config.js) for a list of possible configuration options.
 
-If you have any questions, please reach out to us on [Mozilla Developer Network](https://developer.mozilla.org).
+Create a configuration file by copying the [`config/config-example.json`](conf/config-example.json) to `config/config.json`.
+Put in your database credentials into `config/config.json`:
 
-<!--
+```json
+{
+  "database": {
+    "database": "observatory",
+    "user": "postgres"
+  }
+}
 
-TODO:
+```
 
-Details (with links) to meetings, mailing lists, Slack, and any other communication channels]
+To initialize the database with the proper tables, use this command to migrate. This is a one-time action, but future code changes
+might need further database changes, so run this migration every time the code is updated from the repository.
 
-- User Mailing List:
-- Developer Mailing List:
-- Slack Channel:
-- Public Meeting Schedule and Links:
-- Social Media:
+```sh
+npm run migrate
+```
 
--->
+Finally, start the server by running
 
-## License
+```sh
+npm start
+```
 
-This project is licensed under the [Mozilla Public License 2.0](LICENSE).
+The server is listening on your local interface on port `8080`. You can check the root path by opening <http://localhost:8080/> in your browser or `curl` the URL. The server should respond with `Welcome to the MDN Observatory!`.
+
+## JSON API
+
+**Note:** We provide these endpoints on our public deployment of HTTP Observatory at <https://observatory-api.mdn.mozilla.net/>
+
+### POST `/api/v2/scan`
+
+For integration in CI pipelines or similar applications, a JSON API endpoint is provided. The request rate is limited to one scan per host per `api.cooldown` (default: One minute) seconds. If exceeded, a cached result will be returned.
+
+#### Query parameters
+
+* `host` hostname (required)
+
+#### Examples
+
+* `POST /api/v2/scan?host=mdn.dev`
+* `POST /api/v2/scan?host=google.com`
 
-<!--
+#### Result
 
-We generally use the Mozilla Public License 2.0 and CCSA 2.5 licenses for our projects, see: https://github.com/mdn/content/blob/main/LICENSE.md.
+On success, a JSON object is returned, structured like this example response:
 
-This template is on based on the [CNCF project template](https://github.com/cncf/project-template) distributed under an [Apache license 2.0](https://github.com/cncf/project-template/blob/main/LICENSE).
+```json
+{
+  "id": 77666718,
+  "details_url": "https://developer.mozilla.org/en-US/observatory/analyze?host=mdn.dev",
+  "algorithm_version": 4,
+  "scanned_at": "2024-08-12T08:20:18.926Z",
+  "error": null,
+  "grade": "A+",
+  "score": 105,
+  "status_code": 200,
+  "tests_failed": 0,
+  "tests_passed": 10,
+  "tests_quantity": 10
+}
+```
 
--->
+**Note:** For a full set of details about the host, use the provided link in the `details_url` field.
+
+If an error occurred, an object like this is returned:
+
+```json
+{
+  "error": "invalid-hostname-lookup",
+  "message": "some.invalid.hostname.dev cannot be resolved"
+}
+```
+
+## Contributing
+
+Our project welcomes contributions from any member of our community.
+To get started contributing, please see our [Contributor Guide](CONTRIBUTING.md).
+
+By participating in and contributing to our projects and discussions, you acknowledge that you have read and agree to our [Code of Conduct](CODE_OF_CONDUCT.md).
+
+## Communications
+
+If you have any questions, please reach out to us on [Mozilla Developer Network](https://developer.mozilla.org).
+
+## License
+
+This project is licensed under the [Mozilla Public License 2.0](LICENSE).

conf/config-example.json

@@ -0,0 +1,6 @@
+{
+  "database": {
+    "database": "observatory",
+    "user": "postgres"
+  }
+}

package.json

@@ -12,9 +12,7 @@
     "dev": "nodemon src/api/index.js",
     "test": "CONFIG_FILE=conf/config-test.json mocha",
     "test:nodb": "CONFIG_FILE=conf/config-test.json SKIP_DB_TESTS=1 mocha",
-    "test:compare": "CONFIG_FILE=conf/config-test.json COMPARE_RESULT_TESTS=1 mocha test/compare-results.test.js",
     "tsc": "tsc -p jsconfig.json",
-    "scan": "node src/cli/index.js",
     "updateHsts": "node src/retrieve-hsts.js",
     "refreshMaterializedViews": "node src/maintenance/index.js",
     "migrate": "node -e 'import(\"./src/database/migrate.js\").then( m => m.migrateDatabase() )'"

scan

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+if [ -z "$1" ]; then
+  echo "Usage: $0 <hostname>"
+  exit 1
+fi
+
+node src/cli/index.js scan $1

src/api/global-error-handler.js

@@ -18,15 +18,13 @@ const errorInfo = {
 export default async function globalErrorHandler(error, request, reply) {
   if (error instanceof AppError) {
     return reply.status(error.statusCode).send({
-      statusCode: error.statusCode,
       error: error.name,
       message: error.message,
     });
   }
   return reply
     .status(error.statusCode ?? STATUS_CODES.internalServerError)
     .send({
-      statusCode: error.statusCode ?? STATUS_CODES.internalServerError,
       error: "error-unknown",
       message: error.message,
     });

src/api/server.js

@@ -14,7 +14,6 @@ import pool from "@fastify/postgres";
 import { poolOptions } from "../database/repository.js";
 import { CONFIG } from "../config.js";
 
-
 if (CONFIG.sentry.dsn) {
   initSentry({
     dsn: CONFIG.sentry.dsn,

src/api/v2/analyze/index.js

@@ -1,17 +1,9 @@
 import { CONFIG } from "../../../config.js";
-import {
-  ensureSite,
-  insertScan,
-  insertTestResults,
-  ScanState,
-  selectScanLatestScanByHost,
-  updateScanState,
-} from "../../../database/repository.js";
-import { scan } from "../../../scanner/index.js";
-import { ScanFailedError } from "../../errors.js";
+import { selectScanLatestScanByHost } from "../../../database/repository.js";
 import { SCHEMAS } from "../schemas.js";
 import {
   checkHostname,
+  executeScan,
   historyForSite,
   hydrateTests,
   testsForScan,
@@ -67,38 +59,6 @@ export default async function (fastify) {
   );
 }
 
-/**
- *
- * @param {Pool} pool
- * @param {string} hostname
- * @returns {Promise<import("../../../database/repository.js").ScanRow>}
- */
-async function executeScan(pool, hostname) {
-  const siteId = await ensureSite(pool, hostname);
-  let scanRow = await insertScan(pool, siteId);
-  const scanId = scanRow.id;
-  let scanResult;
-  try {
-    scanResult = await scan(hostname);
-  } catch (e) {
-    if (e instanceof Error) {
-      await updateScanState(pool, scanId, ScanState.FAILED, e.message);
-      throw new ScanFailedError(e);
-    } else {
-      const unknownError = new Error("Unknown error occurred");
-      await updateScanState(
-        pool,
-        scanId,
-        ScanState.FAILED,
-        unknownError.message
-      );
-      throw new ScanFailedError(unknownError);
-    }
-  }
-  scanRow = await insertTestResults(pool, siteId, scanId, scanResult);
-  return scanRow;
-}
-
 /**
  *
  * @param {import("fastify").FastifyInstance} fastify

src/api/v2/recommendations/index.js

@@ -1,4 +1,3 @@
-import { CookiesOutput } from "../../../analyzer/tests/cookies.js";
 import { ALL_RESULTS, ALL_TESTS } from "../../../constants.js";
 import { SCORE_TABLE, TEST_TOPIC_LINKS } from "../../../grader/charts.js";
 import { SCHEMAS } from "../schemas.js";

src/api/v2/scan/index.js

@@ -1,7 +1,11 @@
-import { selectScanById } from "../../../database/repository.js";
-import { NotFoundError } from "../../errors.js";
+import { CONFIG } from "../../../config.js";
+import { selectScanLatestScanByHost } from "../../../database/repository.js";
 import { SCHEMAS } from "../schemas.js";
-import { hydrateTests, testsForScan } from "./../utils.js";
+import { checkHostname, executeScan } from "../utils.js";
+
+/**
+ * @typedef {import("pg").Pool} Pool
+ */
 
 /**
  * Register the API - default export
@@ -10,23 +14,39 @@ import { hydrateTests, testsForScan } from "./../utils.js";
  */
 export default async function (fastify) {
   const pool = fastify.pg.pool;
-
-  fastify.get("/scan", { schema: SCHEMAS.scan }, async (request, reply) => {
+  fastify.post("/scan", { schema: SCHEMAS.scan }, async (request, reply) => {
     const query = /** @type {import("../../v2/schemas.js").ScanQuery} */ (
       request.query
     );
-    const scanId = query.scan;
-    const scanRow = await selectScanById(pool, scanId);
-
-    if (!scanRow) {
-      throw new NotFoundError();
-    }
-    const siteId = scanRow.site_id;
-    const tests = hydrateTests(await testsForScan(pool, scanId));
-    scanRow.scanned_at = scanRow.start_time;
-    return {
-      scan: scanRow,
-      tests,
-    };
+    let hostname = query.host.trim().toLowerCase();
+    hostname = await checkHostname(hostname);
+    return await scanOrReturnRecent(
+      fastify,
+      pool,
+      hostname,
+      CONFIG.api.cooldown
+    );
   });
 }
+
+/**
+ *
+ * @param {import("fastify").FastifyInstance} fastify
+ * @param {Pool} pool
+ * @param {string} hostname
+ * @param {number} age
+ * @returns {Promise<any>}
+ */
+async function scanOrReturnRecent(fastify, pool, hostname, age) {
+  let scanRow = await selectScanLatestScanByHost(pool, hostname, age);
+  if (!scanRow) {
+    // do a rescan
+    fastify.log.info("Rescanning because no recent scan could be found");
+    scanRow = await executeScan(pool, hostname);
+  } else {
+    fastify.log.info("Returning a recent scan result");
+  }
+  scanRow.scanned_at = scanRow.start_time;
+  const siteLink = `https://developer.mozilla.org/en-US/observatory/analyze?host=${encodeURIComponent(hostname)}`;
+  return { details_url: siteLink, ...scanRow };
+}