feat: initial commit

Author: remy

.eslintrc.js

@@ -0,0 +1,8 @@
+module.exports = {
+  plugins: ['node'],
+  extends: ['eslint:recommended', 'plugin:node/recommended'],
+  rules: {
+    'node/no-deprecated-api': 0,
+    'no-console': 0,
+  },
+};

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+generated/*
+.vscode

.nvmrc

@@ -0,0 +1 @@
+8

data/bad-hosts

@@ -0,0 +1,2 @@
+http://pipes.yahoo.com
+http://andreaarbogast.org

data/known-hosts

@@ -0,0 +1,10 @@
+https://jkphl.is https://jkphl.is/webmention.php
+http://aaronparecki.com https://webmention.io/aaronpk/webmention
+https://indiewebcamp.com https://webmention.io/indiewebcamp/webmention
+http://indiewebcamp.com https://webmention.io/indiewebcamp/webmention
+http://webmention.io https://webmention.io/pingback/webmention
+https://quill.p3k.io https://webmention.io/aaronpk/webmention
+https://ben.thatmustbe.me https://ben.thatmustbe.me/api/webmention
+https://ownyourgram.com http://ownyourgram.com/webmention
+http://xtof.withknown.com https://xtof.withknown.com/webmention/
+https://adactio.com https://adactio.com/webmention

data/unknown-hosts

@@ -0,0 +1,16 @@
+https://border-none.net
+https://gist.github.com
+http://microformats.org
+https://wordpress.org
+https://www.brid.gy
+https://ifttt.com
+https://www.flickr.com
+http://instagram.com
+https://indieauth.com
+http://www.wired.com
+http://davidmead.withknown.com
+https://twitter.com
+http://ind.ie
+https://ind.ie
+http://nourel.net
+http://bit.ly

generated/.keep

@@ -0,0 +1,18 @@
+const cheerio = require('cheerio');
+
+function main(html) {
+  const base = cheerio.load(html.toString());
+
+  // FIXME: this doesn't handle multiple entries
+  let $ = base('.h-entry');
+
+  if ($.length === 0) {
+    $ = base;
+  } else {
+    $ = $.find.bind($);
+  }
+
+  return $;
+}
+
+module.exports = main;

lib/dom.js

@@ -0,0 +1,26 @@
+const readFileSync = require('fs').readFileSync;
+const writeFileSync = require('fs').writeFileSync;
+
+function parse(type) {
+  const data = readFileSync(__dirname + `/../data/${type}-hosts`, 'utf8');
+  return data
+    .split('\n')
+    .filter(Boolean)
+    .map(_ => {
+      const [source, endpoint = null] = _.split(' ').filter(Boolean);
+      return { source, endpoint };
+    });
+}
+
+const hosts = []
+  .concat(parse('known'), parse('unknown'), parse('bad'))
+  .reduce(
+    (acc, curr) => Object.assign(acc, { [curr.source]: curr.endpoint }),
+    {}
+  );
+
+writeFileSync(
+  __dirname + '/../generated/hosts.json',
+  JSON.stringify(hosts),
+  'utf8'
+);

lib/generate-data.js

@@ -0,0 +1,31 @@
+const html = require('../lib/html');
+const dom = require('../lib/dom');
+const links = require('../lib/links');
+const getEndpoints = require('./get-wm-endpoints');
+const parse = require('url').parse;
+
+async function main(url) {
+  const host = parse(url).hostname;
+
+  const ignoreOwn = url => {
+    if (url.includes(`/${host}/`) || url.endsWith(`/${host}`)) {
+      return false;
+    }
+
+    return true;
+  };
+
+  const urls = links(dom(await html(url))).filter(ignoreOwn);
+  const endpoints = await getEndpoints(urls);
+
+  // this is a bit confusing…maybe refactor?
+  return endpoints.map(({ url: target, endpoint }) => {
+    return {
+      endpoint,
+      source: url,
+      target,
+    };
+  });
+}
+
+module.exports = main;