index.js

/**
 * MIT License
 *
 * Copyright (c) 2018 Simo Ahava
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
const {URL} = require(`url`);
const fs = require(`fs`);
const {promisify} = require(`util`);
const crypto = require('crypto');

const puppeteer = require(`puppeteer`);
const lighthouse = require(`lighthouse`);
const uuidv1 = require(`uuid/v1`);
const {Validator} = require(`jsonschema`);


const {BigQuery} = require(`@google-cloud/bigquery`);
const {Storage} = require(`@google-cloud/storage`);

const bqSchema = require(`./bigquery-schema.json`);
const config = require(`./config.json`);
const configSchema = require(`./config.schema.json`);

// Make filesystem write work with async/await
const writeFile = promisify(fs.writeFile);
const readFile = promisify(fs.readFile);

// Initialize new GC clients
const bigquery = new BigQuery({
  projectId: config.projectId
});
const storage = new Storage({
  projectId: config.projectId
});

const validator = new Validator;

const log = console.log;

/**
 * Function that runs lighthouse in a headless browser instance.
 *
 * @param {string} id ID of the source for logging purposes.
 * @param {string} url URL to audit.
 * @returns {Promise<object>} The object containing the lighthouse report.
 */
async function launchBrowserWithLighthouse(id, url) {

  log(`${id}: Starting browser for ${url}`);

  const browser = await puppeteer.launch({args: ['--no-sandbox']});

  log(`${id}: Browser started for ${url}`);

  config.lighthouseFlags = config.lighthouseFlags || {};

  config.lighthouseFlags.port = (new URL(browser.wsEndpoint())).port;

  log(`${id}: Starting lighthouse for ${url}`);

  const lhr = await lighthouse(url, config.lighthouseFlags);

  log(`${id}: Lighthouse done for ${url}`);

  await browser.close();

  log(`${id}: Browser closed for ${url}`);

  return lhr;
}

/**
 * Parse the Lighthouse report into an object format corresponding to the BigQuery schema.
 *
 * @param {object} obj The lhr object.
 * @param {string} id ID of the source.
 * @returns {object} The parsed lhr object corresponding to the BigQuery schema.
 */
function createJSON(obj, id) {
  return {
    fetch_time: obj.fetchTime,
    site_url: obj.finalUrl,
    site_id: id,
    user_agent: obj.userAgent,
    emulated_as: obj.configSettings.emulatedFormFactor,
    accessibility: [{
      total_score: obj.categories.accessibility.score,
      bypass_repetitive_content: obj.audits.bypass.score === 1,
      color_contrast: obj.audits['color-contrast'].score === 1,
      document_title_found: obj.audits['document-title'].score === 1,
      no_duplicate_id_attribute: obj.audits['duplicate-id'].score === 1,
      html_has_lang_attribute: obj.audits['html-has-lang'].score === 1,
      html_lang_is_valid: obj.audits['html-lang-valid'].score === 1,
      images_have_alt_attribute: obj.audits['image-alt'].score === 1,
      form_elements_have_labels: obj.audits.label.score === 1,
      links_have_names: obj.audits['link-name'].score === 1,
      lists_are_well_formed: obj.audits.list.score === 1,
      list_items_within_proper_parents: obj.audits['listitem'].score === 1,
      meta_viewport_allows_zoom: obj.audits['meta-viewport'].score === 1
    }],
    best_practices: [{
      total_score: obj.categories['best-practices'].score,
      avoid_application_cache: obj.audits['appcache-manifest'].score === 1,
      uses_https: obj.audits['is-on-https'].score === 1,
      uses_http2: obj.audits['uses-http2'].score === 1,
      uses_passive_event_listeners: obj.audits['uses-passive-event-listeners'].score === 1,
      no_document_write: obj.audits['no-document-write'].score === 1,
      external_anchors_use_rel_noopener: obj.audits['external-anchors-use-rel-noopener'].score === 1,
      no_geolocation_on_start: obj.audits['geolocation-on-start'].score === 1,
      doctype_defined: obj.audits.doctype.score === 1,
      no_vulnerable_libraries: obj.audits['no-vulnerable-libraries'].score === 1,
      notification_asked_on_start: obj.audits['notification-on-start'].score === 1,
      avoid_deprecated_apis: obj.audits.deprecations.score === 1,
      allow_paste_to_password_field: obj.audits['password-inputs-can-be-pasted-into'].score === 1,
      errors_in_console: obj.audits['errors-in-console'].score === 1,
      images_have_correct_aspect_ratio: obj.audits['image-aspect-ratio'].score === 1
    }],
    performance: [{
      total_score: obj.categories.performance.score,
      first_contentful_paint: [{
        raw_value: obj.audits['first-contentful-paint'].rawValue,
        score: obj.audits['first-contentful-paint'].score
      }],
      first_meaningful_paint: [{
        raw_value: obj.audits['first-meaningful-paint'].rawValue,
        score: obj.audits['first-meaningful-paint'].score
      }],
      speed_index: [{
        raw_value: obj.audits['speed-index'].rawValue,
        score: obj.audits['speed-index'].score
      }],
      page_interactive: [{
        raw_value: obj.audits.interactive.rawValue,
        score: obj.audits.interactive.score
      }],
      first_cpu_idle: [{
        raw_value: obj.audits['first-cpu-idle'].rawValue,
        score: obj.audits['first-cpu-idle'].score
      }]
    }],
    pwa: [{
      total_score: obj.categories.pwa.score,
      load_fast_enough: obj.audits['load-fast-enough-for-pwa'].score === 1,
      works_offline: obj.audits['works-offline'].score === 1,
      installable_manifest: obj.audits['installable-manifest'].score === 1,
      uses_https: obj.audits['is-on-https'].score === 1,
      redirects_http_to_https: obj.audits['redirects-http'].score === 1,
      has_meta_viewport: obj.audits.viewport.score === 1,
      uses_service_worker: obj.audits['service-worker'].score === 1,
      works_without_javascript: obj.audits['without-javascript'].score === 1,
      splash_screen_found: obj.audits['splash-screen'].score === 1,
      themed_address_bar: obj.audits['themed-omnibox'].score === 1
    }],
    seo: [{
      total_score: obj.categories.seo.score,
      has_meta_viewport: obj.audits.viewport.score === 1,
      document_title_found: obj.audits['document-title'].score === 1,
      meta_description: obj.audits['meta-description'].score === 1,
      http_status_code: obj.audits['http-status-code'].score === 1,
      descriptive_link_text: obj.audits['link-text'].score === 1,
      is_crawlable: obj.audits['is-crawlable'].score === 1,
      robots_txt_valid: obj.audits['robots-txt'].score === 1,
      hreflang_valid: obj.audits.hreflang.score === 1,
      font_size_ok: obj.audits['font-size'].score === 1,
      plugins_ok: obj.audits.plugins.score === 1
    }]
  }
}

/**
 * Converts input object to newline-delimited JSON
 *
 * @param {object} data Object to convert.
 * @returns {string} The stringified object.
 */
function toNdjson(data) {
  data = Array.isArray(data) ? data : [data];
  let outNdjson = '';
  data.forEach(item => {
    outNdjson += JSON.stringify(item) + '\n';
  });
  return outNdjson;
}

/**
 * Write the lhr log object and reports to GCS. Only write reports if lighthouseFlags.output is defined in config.json.
 *
 * @param {object} obj The lighthouse audit object.
 * @param {string} id ID of the source.
 * @returns {Promise<void>} Resolved promise when all write operations are complete.
 */
async function writeLogAndReportsToStorage(obj, id) {
  const bucket = storage.bucket(config.gcs.bucketName);
  config.lighthouseFlags.output = config.lighthouseFlags.output || [];
  await Promise.all(config.lighthouseFlags.output.map(async (fileType, idx) => {
    let filePath = `${id}/report_${obj.lhr.fetchTime}`;
    let mimetype;
    switch (fileType) {
      case 'csv':
        mimetype = 'text/csv';
        filePath += '.csv';
        break;
      case 'json':
        mimetype = 'application/json';
        filePath += '.json';
        break;
      default:
        filePath += '.html';
        mimetype = 'text/html';
    }
    const file = bucket.file(filePath);
    log(`${id}: Writing ${fileType} report to bucket ${config.gcs.bucketName}`);
    return await file.save(obj.report[idx], {
      metadata: {contentType: mimetype}
    });
  }));
  const file = bucket.file(`${id}/log_${obj.lhr.fetchTime}.json`);
  log(`${id}: Writing log to bucket ${config.gcs.bucketName}`);
  return await file.save(JSON.stringify(obj.lhr, null, " "), {
    metadata: {contentType: 'application/json'}
  });
}

/**
 * Check events in GCS states.json to see if an event with given ID has been pushed to Pub/Sub less than
 * minTimeBetweenTriggers (in config.json) ago.
 *
 * @param {string} id ID of the source (and the Pub/Sub message).
 * @param {number} timeNow Timestamp when this method was invoked.
 * @returns {Promise<object>} Object describing active state and time delta between invocation and when the state entry was created, if necessary.
 */
async function checkEventState(id, timeNow) {
  let eventStates = {};
  try {
    // Try to load existing state file from storage
    const destination = `/tmp/state_${id}.json`;
    await storage
      .bucket(config.gcs.bucketName)
      .file(`${id}/state.json`)
      .download({destination: destination});
    eventStates = JSON.parse(await readFile(destination));
  } catch(e) {}

  // Check if event corresponding to id has been triggered less than the timeout ago
  const delta = id in eventStates && (timeNow - eventStates[id].created);
  if (delta && delta < config.minTimeBetweenTriggers) {
    return {active: true, delta: Math.round(delta/1000)}
  }

  // Otherwise write the state of the event with current timestamp and save to bucket
  eventStates[id] = {created: timeNow};
  await storage.bucket(config.gcs.bucketName).file(`${id}/state.json`).save(JSON.stringify(eventStates, null, " "), {
    metadata: {contentType: 'application/json'}
  });
  return {active: false}
}

/**
 * The Cloud Function. Triggers on HTTP Request
 *
 * @returns {Promise<*>} Promise when BigQuery load starts.
 * @param url
 */
async function launchLighthouse (url) {
  try {
    const uuid = uuidv1();
    const metadata = {
      sourceFormat: 'NEWLINE_DELIMITED_JSON',
      schema: {fields: bqSchema},
      jobId: uuid
    };

    const id = crypto.createHash('md5').update(url).digest('hex');

    log(`${id}: Received message to start with URL ${url}`);

    const timeNow = new Date().getTime();
    const eventState = await checkEventState(id, timeNow);
    if (eventState.active) {
      return log(`${id}: Found active event (${Math.round(eventState.delta)}s < ${Math.round(config.minTimeBetweenTriggers/1000)}s), aborting...`);
    }

    const res = await launchBrowserWithLighthouse(id, url);

    await writeLogAndReportsToStorage(res, id);
    const json = createJSON(res.lhr, id);

    json.job_id = uuid;

    await writeFile(`/tmp/${uuid}.json`, toNdjson(json));

    log(`${id}: BigQuery job with ID ${uuid} starting for ${url}`);

    return bigquery
      .dataset(config.datasetId)
      .table('reports')
      .load(`/tmp/${uuid}.json`, metadata);

  } catch(e) {
    console.error(e);
  }
}

/**
 * Initialization function - only run when Cloud Function is deployed and/or a new instance is started. Validates the configuration file against its schema.
 */
function init() {
  // Validate config schema
  const result = validator.validate(config, configSchema);
  if (result.errors.length) {
    throw new Error(`Error(s) in configuration file: ${JSON.stringify(result.errors, null, " ")}`);
  } else {
    log(`Configuration validated successfully`);
  }
}

if (process.env.NODE_ENV !== 'test') {
  init();
} else {
  // For testing
  module.exports = {
    _init: init,
    _writeLogAndReportsToStorage: writeLogAndReportsToStorage,
    _toNdJson: toNdjson,
    _createJSON: createJSON,
    _launchBrowserWithLighthouse: launchBrowserWithLighthouse,
    _checkEventState: checkEventState
  }
}

module.exports.launchLighthouse = async (req,res) => {
  if (req.body.apiKey === config.apiKey) {
    await launchLighthouse(req.body.url);
    res.send('Done');
  } else {
    res.send('Wrong API Key');
  }
};