cli: add --audit and --auditList options

bin/citgm-all.js

@@ -39,6 +39,14 @@ const yargs = commonArgs(require('yargs'))
  type: 'array',
  description: 'Define which tags from the lookup to skip'
  })
+ .option('audit', {
+ type: 'array',
+ description: 'Run only module tests that are skipped or flaky'
+ })
+ .option('auditList', {
+ type: 'array',
+ description: 'List module tests that are skipped or flaky'
+ })
  .example('citgm-all -t /path/to/output.tap',
  'Write test results as tap to file.')
  .example('citgm-all -l /path/to/lookup.json', 'Test a custom set of modules.')
@@ -72,7 +80,11 @@ const options = {
  tmpDir: app.tmpDir,
  customTest: app.customTest,
  includeTags: app.includeTags || [],
- excludeTags: app.excludeTags || []
+ excludeTags: app.excludeTags || [],
+ audit: app.audit !== undefined,
+ auditReasons: app.audit || [],
+ auditList: app.auditList !== undefined,
+ auditListReasons: app.auditList || []
 };
 
 if (options.includeTags.length){
@@ -114,8 +126,39 @@ if (!citgm.windows) {
 
 const modules = [];
 
+function matchedWithReason(match, reasons) {
+ if (!match.matched)
+ return false;
+ if (reasons.length === 0)
+ return true;
+ return reasons.indexOf(match.reason) !== -1;
+}
+
 function runCitgm (mod, name, next) {
- if (isMatch(mod.skip)) {
+ let skip = isMatch(mod.skip);
+ let flaky = isMatch(mod.flaky);
+ if (options.audit || options.auditList) {
+ const reasons = options.audit
+ ? options.auditReasons
+ : options.auditListReasons;
+ const run = options.audit;
+ let result;
+ let what;
+ if (matchedWithReason(skip, reasons)) {
+ result = skip;
+ what = 'skipped';
+ } else if (matchedWithReason(flaky, reasons)) {
+ result = flaky;
+ what = 'flaky';
+ } else {
+ return next();
+ }
+ log.info('audit', name + ' is ' + what + ' because it matches \'' +
+ result.reason + '\'');
+ mod.skip = false;
+ mod.flaky = false;
+ if (!run) return next();
+ } else if (skip.matched) {
  modules.push({
  name,
  skipped: true

lib/lookup.js

@@ -115,9 +115,9 @@ function resolve(context, next) {
  context.module.tags = rep.tags;
  }
  context.module.flaky = context.options.failFlaky ?
- false : isMatch(rep.flaky);
+ false : isMatch(rep.flaky).matched;
  context.module.expectFail = context.options.expectFail ?
- false : isMatch(rep.expectFail);
+ false : isMatch(rep.expectFail).matched;
  } else {
  context.emit('data', 'info', 'lookup-notfound', detail.name);
  if (meta.gitHead) {

lib/match-conditions.js

@@ -1,5 +1,4 @@
 'use strict';
-const _ = require('lodash');
 const execSync = require('child_process').execSync;
 const fs = require('fs');
 const semver = require('semver');
@@ -37,38 +36,75 @@ const endian = process.config.variables.node_byteorder || '';
 
 function isStringMatch(conditions) {
  if (semver.validRange(conditions) && semver.satisfies(semVersion, conditions))
- return true;
- const checks = [distro, release, version, [platform, arch].join('-'),
- endian, fips];
- return _.some(checks, function (check) {
- return check.search(conditions) !== -1;
- });
+ return {
+ matched: true,
+ reason: 'version'
+ };
+ const checks = {
+ 'distro': distro,
+ 'release': release,
+ 'version': version,
+ 'platform': [platform, arch].join('-'),
+ 'endian': endian,
+ 'fips': fips
+ };
+ for (let key in checks) {
+ if (checks[key].search(conditions) !== -1) {
+ return {
+ matched: true,
+ reason: key
+ };
+ }
+ }
+ return { matched: false };
 }
 
 function isArrayMatch(arr) {
- return _.some(arr, function(item) {
- if (typeof item === 'string') return isStringMatch(item);
- if (typeof item === 'object') return isObjectMatch(item);
- return false;
- });
+ for (let idx in arr) {
+ const item = arr[idx];
+ let result;
+ if (typeof item === 'string') result = isStringMatch(item);
+ if (typeof item === 'object') result = isObjectMatch(item);
+ if (result && result.matched) {
+ return result;
+ }
+ }
+ return { matched: false };
 }
 
 function isObjectMatch(obj) {
- return _.some(obj, function(value, key) {
- if (!isStringMatch(key)) return false;
- if (typeof value === 'boolean') return value;
- if (typeof value === 'string') return isStringMatch(value);
- if (value instanceof Array) return isArrayMatch(value);
- return false;
- });
+ for (let key in obj) {
+ if (!isStringMatch(key).matched) {
+ continue;
+ }
+ let value = obj[key];
+ if (typeof value === 'boolean') {
+ return {
+ result: value,
+ reason: 'explicit'
+ };
+ }
+ let result;
+ if (typeof value === 'string') result = isStringMatch(value);
+ if (value instanceof Array) result = isArrayMatch(value);
+ if (result && result.matched) {
+ return result;
+ }
+ }
+ return { matched: false };
 }
 
 function isMatch(conditions) {
- if (typeof conditions === 'boolean') return conditions;
+ if (typeof conditions === 'boolean' && conditions === true) {
+ return {
+ matched: true,
+ reason: 'explicit'
+ };
+ }
  if (typeof conditions === 'string') return isStringMatch(conditions);
  if (conditions instanceof Array) return isArrayMatch(conditions);
  if (typeof conditions === 'object') return isObjectMatch(conditions);
- return false;
+ return { matched: false };
 }
 
 module.exports = isMatch;

test/test-match-conditions.js

@@ -45,31 +45,37 @@ function revertShim() {
 }
 
 function testVersions(t, testFunction) {
- t.ok(testFunction(process.version),
+ t.ok(testFunction(process.version).matched,
  'the current version is what it is matched against');
+ t.equal(testFunction(process.version).reason, 'version',
+ 'the version match reason is "version"');
  shim();
- t.ok(testFunction('v5'), 'the module is matched on the current platform');
- t.ok(testFunction('> 5.0.0'),
+ t.ok(testFunction('v5').matched,
  'the module is matched on the current platform');
- t.ok(testFunction('macos'), 'the distro is correct');
- t.notok(testFunction('v2'),
+ t.ok(testFunction('> 5.0.0').matched,
+ 'the module is matched on the current platform');
+ t.ok(testFunction('macos').matched, 'the distro is correct');
+ t.notok(testFunction('v2').matched,
  'the module is not matched on the current platform');
- t.notok(testFunction('<=v2.0.0'),
+ t.notok(testFunction('<=v2.0.0').matched,
  'the module is not matched on the current platform');
  revertShim();
 }
 
 function testPlatforms(t, testFunction) {
- t.ok(testFunction(process.platform),
+ t.ok(testFunction(process.platform).matched,
  'the current platform is what it is matched against');
+ t.equal(testFunction(process.platform).reason, 'platform',
+ 'the platform matched reason is "platform"');
+
  shim();
- t.ok(testFunction('darwin'), 'darwin is matched');
- t.ok(testFunction('x64'), 'x64 is matched');
- t.ok(testFunction('darwin-x64'), 'darwin-x64 is matched');
- t.notok(testFunction('darwin-x86'), 'darwin-x86 is not matched');
- t.notok(testFunction('hurd-x86'), 'hurd-x86 is not matched');
- t.notok(testFunction('hurd-x64'), 'hurd-x64 is not matched');
- t.notok(testFunction('hurd'), 'hurd is not matched');
+ t.ok(testFunction('darwin').matched, 'darwin is matched');
+ t.ok(testFunction('x64').matched, 'x64 is matched');
+ t.ok(testFunction('darwin-x64').matched, 'darwin-x64 is matched');
+ t.notok(testFunction('darwin-x86').matched, 'darwin-x86 is not matched');
+ t.notok(testFunction('hurd-x86').matched, 'hurd-x86 is not matched');
+ t.notok(testFunction('hurd-x64').matched, 'hurd-x64 is not matched');
+ t.notok(testFunction('hurd').matched, 'hurd is not matched');
  revertShim();
 }
 
@@ -80,46 +86,46 @@ function testArrays(t, testFunction) {
  match,
  notMatch,
  invalid
- ]), 'matched array of object');
+ ]).matched, 'matched array of object');
  t.notok(testFunction([
  notMatch,
  notMatch,
  notMatch,
  invalid
- ]), 'not matched array of objects');
+ ]).matched, 'not matched array of objects');
 
  t.ok(testFunction([
  'hurd',
  'x86',
  'v4',
  'darwin'
- ]), 'matchy array of string');
+ ]).matched, 'matchy array of string');
 
  t.notok(testFunction([
  'hurd',
  'x86',
  'v4'
- ]), 'not matchy array of string');
+ ]).matched, 'not matchy array of string');
 
  t.notok(testFunction([
  true,
  false,
  123
- ]), 'not matched invalid input');
+ ]).matched, 'not matched invalid input');
 
  revertShim();
 }
 
 function testObjects(t, testFunction) {
  shim();
- t.ok(testFunction(match), 'it should be matched');
- t.notok(testFunction(notMatch), 'it should not be matched');
- t.notok(testFunction(invalid),
+ t.ok(testFunction(match).matched, 'it should be matched');
+ t.notok(testFunction(notMatch).matched, 'it should not be matched');
+ t.notok(testFunction(invalid).matched,
  'invalid input should not give a false positive');
  t.notok(testFunction({
  a: 123,
  v5: false
- }), 'another invalid input that should not give a false positive');
+ }).matched, 'another invalid input that should not give a false positive');
  revertShim();
 }
 
@@ -147,8 +153,9 @@ test('isMatch', function (t) {
  testPlatforms(t, isMatch);
  testArrays(t, isMatch);
  testObjects(t, isMatch);
- t.ok(isMatch(true), 'true is matched');
- t.notok(isMatch(false), 'false is not matched');
- t.notok(isMatch(123), 'invalid input is not matched');
+ t.ok(isMatch(true).matched, 'true is matched');
+ t.equal(isMatch(true).reason, 'explicit', 'true match reason is "explicit"');
+ t.notok(isMatch(false).matched, 'false is not matched');
+ t.notok(isMatch(123).matched, 'invalid input is not matched');
  t.end();
 });