esmDependencyCheck.js

/* eslint-disable no-prototype-builtins */
/* eslint-disable no-console */
const { exec } = require('child_process');
const fs = require('fs');

const { dependencies, devDependencies } = JSON.parse(
  fs.readFileSync('./package.json'),
);

const allDependencies = { ...dependencies, ...devDependencies };
const dependencyTable = [];

const dateNow = new Date().getTime();
const datediff = (first, second) =>
  Math.round((second - first) / (1000 * 60 * 60 * 24));
const gitHubToken = process.env.GITHUB_TOKEN;

let countedRepos = 0;

const getRemoteGitFile = async (gitDepUrl, args) => {
  let url = `https://api.github.com/repos/${gitDepUrl}/contents/package.json`;
  if (url.includes('/tree/')) {
    url = url
      .replace(/\/tree\/[^/]+\//, '/contents/')
      .replace('//contents/', '/')
      .replace('/contents/package.json', '/package.json');
  }

  return fetch(url, {
    method: 'GET',
    headers: {
      Authorization: `token ${gitHubToken}`,
      Accept: 'application/vnd.github.v3.raw',
    },
  })
    .then(res => {
      if (res.status === 200) {
        return res.json().then(data => {
          if (!data.type) {
            return { type: 'commonjs', ...args };
          }
          if (data.type) {
            return { type: data.type, ...args };
          }
          return false;
        });
      }
      return { type: 'error', ...args };
    })
    .catch(e => {
      countedRepos += 1;
      console.error(e);
      return { type: 'error', ...args };
    });
};

const dealWithNonNumericCharacters = (versionString, timeJson) => {
  const patchVersion = versionString.match(/patch/);
  if (patchVersion) {
    const possibleVersionStrings = versionString.match(
      /@npm:([\d.]+)|@([\d.]+)/,
    );
    return possibleVersionStrings
      ? possibleVersionStrings[1] || possibleVersionStrings[2]
      : `Version number not processed in current form ${versionString}`;
  }

  const plainVersion = versionString.match(/^[\d.]+$/g);
  if (plainVersion) {
    return versionString; // if it's just numbers and dots
  }
  const lowestVersionMatches = versionString.match(/[\d.]+$/g);
  if (!lowestVersionMatches) {
    return 'Unknown'; // if it contains a string that doesn't end in numbers and dots we give up
  }
  const splitOurVersionArray = lowestVersionMatches[0].split('.');

  let versionMatcherString = '';
  if (versionString.indexOf('^') !== -1) {
    [versionMatcherString] = splitOurVersionArray; // caret means we are going to get anything belonging to major
  }
  if (versionString.indexOf('~') !== -1) {
    splitOurVersionArray.pop(); // tilde means we get all patches of the minor
    versionMatcherString = splitOurVersionArray.join('\\.');
  }
  versionMatcherString += '\\.';

  let versionToReturn = '';
  // loop through response from npm (which is handily in series order) and match our version with regex
  Object.keys(timeJson).forEach(version => {
    const ourRegex = new RegExp(`^${versionMatcherString}`, 'gi');
    if (version.match(ourRegex)) {
      versionToReturn = version;
    }
  });
  return versionToReturn;
};

const simplifyDate = modifiedDate => {
  const year = modifiedDate.getFullYear().toString();
  const month = (modifiedDate.getMonth() + 1).toString().padStart(2, '0');
  const day = modifiedDate.getDate().toString().padStart(2, '0');
  return `${year}-${month}-${day}`;
};

const getRepoFromNpmData = npmData => {
  if (
    npmData.hasOwnProperty('repository') &&
    npmData.repository.hasOwnProperty('url')
  ) {
    return npmData.repository.url;
  }
  if (npmData.hasOwnProperty('repository')) {
    return npmData.repository;
  }
  return npmData.url || npmData.homepage;
};

const writeCsvFile = data => {
  let csvContents =
    'Dependency,Type,Most Recent Version,Most Recent Version Date,Our Version,Our Version Date,Our Version Freshness in Days';

  data
    .sort((a, b) => {
      if (a.name < b.name) {
        return -1;
      }
      if (a.name > b.name) {
        return 1;
      }
      return 0;
    })
    .forEach(
      ({
        name,
        type,
        mostRecentVersion,
        mostRecentVersionDate,
        ourVersion,
        ourVersionDate,
        ourFreshnessInDays,
      }) => {
        csvContents += `\n${name},${type},${mostRecentVersion},${mostRecentVersionDate},${ourVersion},${ourVersionDate},${ourFreshnessInDays}`;
      },
    );

  fs.writeFileSync('./esmDependencyTable.csv', csvContents);
};

const repoLength = Object.keys(allDependencies).length;

if (gitHubToken) {
  console.log('Please wait. Gathering npm data...');
  Object.keys(allDependencies).forEach((dep, i) => {
    let gitRepo;
    console.log(`checking npm details for ${dep}`);
    exec(`npm view ${dep} --json`, (err, stdout) => {
      if (err) {
        console.error(err);
      }
      if (stdout) {
        const depRepository = JSON.parse(stdout);
        const modifiedDate = new Date(depRepository.time.modified);
        const simplifiedModifiedDate = simplifyDate(modifiedDate);
        const ourVersion = dealWithNonNumericCharacters(
          allDependencies[dep],
          depRepository.time,
          dep,
        );

        const dateOfOurVersion = new Date(depRepository.time[ourVersion]);
        const simplifiedDateOfOurVersion = simplifyDate(dateOfOurVersion);

        const ourFreshnessInDays = datediff(
          dateOfOurVersion.getTime(),
          dateNow,
        );
        setTimeout(() => {
          let latestVersion = '';
          if (Array.isArray(depRepository.versions)) {
            latestVersion = depRepository.versions.at(-1);
          } else {
            latestVersion = depRepository.versions;
          }
          gitRepo = getRepoFromNpmData(depRepository);
          if (gitRepo) {
            gitRepo = gitRepo.replace(
              /^(?:git)?(?:\+)?(https|ssh)?:\/\/(?:git@)?github.com\/|\.git(?:#?.*)$/g,
              '',
            );
            const gitRepoArgs = {
              latestVersion,
              ourVersion,
              ourFreshnessInDays,
              dateOfOurVersion,
              modifiedDate,
            };
            try {
              getRemoteGitFile(gitRepo, gitRepoArgs)
                .then(data => {
                  countedRepos += 1;
                  dependencyTable.push({
                    name: dep,
                    type: data.type,
                    mostRecentVersion: data.latestVersion,
                    mostRecentVersionDate: simplifiedModifiedDate,
                    ourVersion: data.ourVersion,
                    ourVersionDate: simplifiedDateOfOurVersion,
                    ourFreshnessInDays: data.ourFreshnessInDays,
                  });
                  console.log(
                    `checking repo ${countedRepos} out of ${repoLength}: ${gitRepo}`,
                  );
                  if (countedRepos >= Object.keys(allDependencies).length) {
                    console.log('FINISHED');
                    writeCsvFile(dependencyTable);
                  }
                })
                .catch(e => {
                  console.error(e);
                  countedRepos += 1;
                });
            } catch (e) {
              console.error(e);
            }
          } else {
            countedRepos += 1;
            console.error(
              `dep ${dep} has no public repo so we're reading from local`,
            );
            const repository = JSON.parse(
              fs.readFileSync(`./node_modules/${dep}/package.json`),
            );
            const depType = repository.type ? repository.type : 'commonjs';
            dependencyTable.push({
              name: dep,
              type: depType,
              mostRecentVersion: latestVersion,
              mostRecentVersionDate: simplifiedModifiedDate,
              ourVersion,
              ourVersionDate: simplifiedDateOfOurVersion,
              ourFreshnessInDays,
            });
          }
        }, i * 50);
      } else {
        countedRepos += 1;
        console.log('no stdout', dep, stdout);
      }
    });
  });
} else {
  console.error(
    'No github token supplied. Please see ./scripts/README.md for details',
  );
}