Re-implements as modular server using Express

.github/workflows/test-and-lint.yml

@@ -1,26 +1,27 @@
 name: test-and-lint
 on:
   push:
-    branches: [ master ]
+    branches: [ master, modular ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, modular ]
 jobs:
   build:
     name: node.js
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         # Support LTS versions based on https://nodejs.org/en/about/releases/
         node-version: ['18', '20', '21']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
       - name: Install dependencies
         run: npm ci
       - name: Run linter
         run: npm run lint
       - name: Run tests
-        run: npm test
+        run: npm test -- --timeout 10000

.gitignore

@@ -10,3 +10,13 @@ dev-log
 dev-storage
 .vscode
 .idea
+
+# misc
+.DS_Store
+.env.local
+.env.development.local
+.env.test.local
+.env.production.local
+
+# Data stores
+myminio

bin/dev-conf.json

@@ -1,17 +1,27 @@
 {
+  "host_identity": "example.com",
   "basePath": "",
   "allow_signup": true,
   "storage_path": "./dev-storage",
   "lock_timeout_ms": 30000,
   "lock_stale_after_ms": 60000,
   "cache_views": true,
   "http": {
-    "host": "127.0.0.1",
+    "host": "0.0.0.0",
     "port": 8000
   },
+  "https": {
+    "enable": true,
+    "portXXX": 4443,
+    "key": "../../example.com+5-key.pem",
+    "cert": "../../example.com+5.pem"
+  },
   "logging": {
     "log_dir": "./dev-log",
-    "stdout": ["debug"],
+    "stdout": ["info"],
     "log_files": ["notice"]
+  },
+  "s3": {
+    "user_name_suffix": null
   }
 }

bin/www

@@ -0,0 +1,221 @@
+#!/usr/bin/env node
+
+const http = require('http');
+const fs = require("fs");
+const path = require("path");
+const {ArgumentParser} = require("argparse");
+const appFactory = require('../lib/appFactory');
+const {configureLogger, getLogger} = require("../lib/logger");
+const S3Handler = require("../lib/routes/S3_store_handler");
+const process = require("process");
+const https = require("https");
+
+const SSL_CIPHERS = 'ECDHE-RSA-AES256-SHA384:AES256-SHA256:RC4-SHA:RC4:HIGH:!MD5:!aNULL:!EDH:!AESGCM';
+const SSL_OPTIONS = require('crypto').constants.SSL_OP_CIPHER_SERVER_PREFERENCE;
+
+/** Command-line arguments and configuration loading */
+
+const args = parseArgs();
+let conf = {};
+
+if (args.exampleConf) {
+  console.info(fs.readFileSync(path.join(__dirname, '/conf.example.json'), 'utf8'));
+  return -1;
+}
+
+try {
+  conf = JSON.parse(fs.readFileSync(args.conf, 'utf8'));
+} catch (err) {
+  console.error(`Unable to load config file “${args.conf}”:`, err);
+  return -2;
+}
+
+/** Configures application */
+
+configureLogger(conf.logging);
+
+let basePath = conf.basePath || '';
+if (basePath && !basePath.startsWith('/')) { basePath = '/' + basePath; }
+
+let jwtSecret = process.env.JWT_SECRET || process.env.S3_SECRET_KEY;
+if (!jwtSecret) {
+  process.env.SECRET = jwtSecret = String(Math.round(Math.random() * Number.MAX_SAFE_INTEGER))
+  getLogger().warning(`neither JWT_SECRET nor S3_SECRET_KEY were set in the environment. Setting it to “${jwtSecret}”`)
+}
+
+const hostIdentity = conf.host_identity?.trim();
+if (!hostIdentity) {
+  getLogger().emerg(`host_identity MUST be set in the configuration file`);
+  process.exit(1);
+}
+const userNameSuffix = conf.user_name_suffix ?? '-' + hostIdentity;
+
+if (conf.http?.port) {
+  start( Object.assign({}, conf.http, process.env.PORT && {port: process.env.PORT}));
+}
+
+if (conf.https?.port) {
+  start(conf.https);
+}
+
+
+function start(network) {
+  // If the environment variables aren't set, s3handler uses a shared public account on play.min.io,
+  // to which anyone in the world can read and write!
+  // It is not entirely compatible with S3Handler.
+  const s3handler = new S3Handler({endPoint: process.env.S3_ENDPOINT,
+    accessKey: process.env.S3_ACCESS_KEY, secretKey: process.env.S3_SECRET_KEY, region: process.env.S3_REGION || 'us-east-1',
+    userNameSuffix});
+
+  const app = appFactory({hostIdentity, jwtSecret, account: s3handler, store: s3handler, basePath});
+
+  const port = normalizePort( network?.port || '8000');
+  app.set('port', port);
+
+  app.set('forceSSL', Boolean(conf.https?.force));
+  if (network?.port && conf.https?.port) {
+    app.set('httpsPort', parseInt(conf.https?.port));   // only for redirecting to HTTPS
+  }
+
+  app.locals.title = "Modular Armadietto";
+  // Before rendering, `locals.host` should be set to `getHost(req)`
+  app.locals.host = (network?.host || '0.0.0.0') + (port ? ':' + port : '');
+  app.locals.signup = conf.allow_signup;
+
+  /** Creates HTTP server. */
+
+  let server;
+  if (network.key && network.cert) {
+    const key = fs.readFileSync(network.key);
+    const cert = fs.readFileSync(network.cert);
+    const ca = network.ca ? fs.readFileSync(network.ca) : null;
+    const sslOptions = {
+      key,
+      cert,
+      ciphers: SSL_CIPHERS,
+      secureOptions: SSL_OPTIONS,
+      ca
+    };
+
+    server = https.createServer(sslOptions, app);
+  } else {
+    server = http.createServer(app);
+  }
+
+  /** Listens on provided port, on network interfaces specified by 'host'. */
+
+  server.listen(port);
+  server.on('error', onError);
+  server.on('clientError', clientError);
+  server.on('listening', onListening);
+
+  /** Event listener for HTTP server "error" event. */
+  function onError(error) {
+    if (error.syscall !== 'listen') {
+      throw error;
+    }
+
+    const bind = typeof port === 'string'
+      ? 'Pipe ' + port
+      : 'Port ' + port;
+
+    // handle specific listen errors with friendly messages
+    switch (error.code) {
+      case 'EACCES':
+        getLogger().crit(bind + ' requires elevated privileges');
+        process.exit(1);
+        break;
+      case 'EADDRINUSE':
+        getLogger().crit(bind + ' is already in use');
+        process.exit(1);
+        break;
+      default:
+        throw error;
+    }
+  }
+
+  /** Event listener for HTTP server "listening" event. */
+  function onListening() {
+    getLogger().notice(`Accepting remoteStorage connections: http${network.key ? 's' : ''}://${app.locals.host}${basePath}/`);
+  }
+
+  /** Adds listeners for shutdown and serious problems */
+
+// These are the happy paths for shutdown.
+  process.on('SIGINT', stop.bind(this, 'SIGINT'));
+  process.on('SIGTERM', stop.bind(this, 'SIGTERM'));
+
+  function stop(signal) {
+    getLogger().debug(`${signal} signal received: closing HTTP server`);
+    server.close(() => {
+      getLogger().notice(`No longer accepting remoteStorage connections: http${network.key ? 's' : ''}://${app.locals.host}${basePath}/`);
+    });
+  }
+
+// Without these listeners, these events would not be logged, only sent to stdout or stderr.
+  process.on('uncaughtExceptionMonitor', (err, origin) => {
+    getLogger().crit(`${origin} ${err}`);
+  });
+
+  process.on('warning', (warning) => {
+    getLogger().warning(`${warning.name} ${warning.message} ${warning.stack}`);
+  });
+
+  process.on('multipleResolves', (type, promise, reason) => {
+    getLogger().debug(`multipleResolves ${type} “${reason}”`);
+  });
+}
+
+/** parses command-line arguments */
+function parseArgs () {
+  const version = require(path.join(__dirname, '/../package.json')).version;
+  const parser = new ArgumentParser({
+    add_help: true,
+    description: 'NodeJS remoteStorage server / ' + version
+  });
+
+  parser.add_argument('-c', '--conf', {
+    help: 'Path to configuration'
+  });
+
+  parser.add_argument('-e', '--exampleConf', {
+    help: 'Print configuration example',
+    action: 'store_true'
+  });
+
+  return parser.parse_args();
+}
+
+/** Normalizes a port into a number, string, or false. */
+function normalizePort(val) {
+  const port = parseInt(val, 10);
+
+  if (isNaN(port)) {
+    // named pipe
+    return val;
+  }
+
+  if (port >= 0) {
+    // port number
+    return port;
+  }
+
+  return false;
+}
+
+function clientError (err, socket) {
+  let status, message;
+  if (err.code === 'HPE_HEADER_OVERFLOW') {
+    status = 431;
+    message = 'Request Header Fields Too Large';
+  } else {
+    status = 400;
+    message = 'Bad Request';
+  }
+  getLogger().warning(`${socket.address().address} n/a n/a ${status} ${message} ${err.toString()}`);
+
+  if (err.code !== 'ECONNRESET' && socket.writable) {
+    socket.end(`HTTP/1.1 ${status} ${message}\r\n\r\n`);
+  }
+  socket.destroy(err);
+}

lib/appFactory.js

@@ -0,0 +1,85 @@
+const express = require('express');
+const path = require('path');
+const { loggingMiddleware } = require('./logger');
+const indexRouter = require('./routes/index');
+const signupRouter = require('./routes/signup');
+const webFingerRouter = require('./routes/webfinger');
+const oAuthRouter = require('./routes/oauth');
+const storageCommon = require('./routes/storage_common');
+const errorPage = require('./util/errorPage');
+const helmet = require('helmet');
+const shorten = require('./util/shorten');
+
+module.exports = function ({ hostIdentity, jwtSecret, account, store, basePath = '' }) {
+  if (basePath && !basePath.startsWith('/')) { basePath = '/' + basePath; }
+
+  const app = express();
+  app.locals.basePath = basePath;
+
+  // view engine setup
+  app.engine('.html', require('ejs').__express);
+  app.engine('.xml', require('ejs').__express);
+  app.set('view engine', 'html');
+  app.set('views', path.join(__dirname, 'views'));
+
+  express.static.mime.define({ 'text/javascript': ['js'] });
+
+  app.set('account', account);
+
+  app.use(loggingMiddleware);
+  app.use(helmet({
+    contentSecurityPolicy: {
+      directives: {
+        sandbox: ['allow-scripts', 'allow-forms', 'allow-popups', 'allow-same-origin'],
+        defaultSrc: ['\'self\''],
+        scriptSrc: ['\'self\''],
+        scriptSrcAttr: ['\'none\''],
+        styleSrc: ['\'self\''],
+        imgSrc: ['\'self\''],
+        fontSrc: ['\'self\''],
+        objectSrc: ['\'none\''],
+        childSrc: ['\'none\''],
+        connectSrc: ['\'none\''],
+        baseUri: ['\'self\''],
+        frameAncestors: ['\'none\''],
+        formAction: (process.env.NODE_ENV === 'production' ? ['https:'] : ['https:', 'http:']), // allows redirect to any RS app
+        upgradeInsecureRequests: []
+      }
+    }
+  }));
+  app.use(express.urlencoded({ extended: false }));
+  app.use(`${basePath}/assets`, express.static(path.join(__dirname, 'assets')));
+
+  app.use(`${basePath}/`, indexRouter);
+
+  app.use(`${basePath}/signup`, signupRouter);
+
+  app.use([`${basePath}/.well-known`, `${basePath}/webfinger`], webFingerRouter);
+
+  app.use(`${basePath}/oauth`, oAuthRouter(hostIdentity, jwtSecret));
+  app.use(`${basePath}/storage`, storageCommon(hostIdentity, jwtSecret));
+  app.use(`${basePath}/storage`, store);
+
+  // catches paths not handled and returns Not Found
+  app.use(basePath, function (req, res, next) {
+    const name = req.path.slice(1);
+    errorPage(req, res, 404, { title: 'Not Found', message: `“${name}” doesn't exist` });
+  });
+
+  // redirect for paths outside the app
+  app.use(function (req, res, next) {
+    res.status(308).set('Location', basePath).end();
+  });
+
+  // error handler
+  app.use(function (err, req, res, _next) {
+    const message = err?.message || err?.errors?.find(e => e.message).message || err?.cause?.message || 'indescribable error';
+    errorPage(req, res, err.status || 500, {
+      title: shorten(message, 30),
+      message,
+      error: req.app.get('env') === 'development' ? err : {}
+    });
+  });
+
+  return app;
+};