Merge httpclient and rest.js from Monkshu.

flows/pipeline_RESTtoSOAPtoRESTUsingURL.json

@@ -0,0 +1,42 @@
+{
+ "flow": {
+ "name":"Expose SOAP API as REST",
+ "description": "Exposes NOAA Weather SOAP Web Service as a REST API. Uses URLs. Input LAT and LON with west as negative and south as negative, and using only two decimal places.",
+ "disabled": false,
+ "autoGC": false
+ },
+ "listener": {
+ "type":"rest_listener", 
+ "isMessageGenerator": true,
+ "host": "127.0.0.1",
+ "port":9091,
+ "url":"/weather",
+ "allow_origin": "*",
+ "timeout": 120000
+ },
+ "route0":{
+ "type": "httpclient",
+ "dependencies":["listener"],
+ "url":"https://graphical.weather.gov/xml/SOAP_server/ndfdXMLclient.php?whichClient=NDFDgen&lat={{message.content.lat}}&lon={{message.content.lon}}&listLatLon=&lat1=&lon1=&lat2=&lon2=&resolutionSub=&listLat1=&listLon1=&listLat2=&listLon2=&resolutionList=&endPoint1Lat=&endPoint1Lon=&endPoint2Lat=&endPoint2Lon=&listEndPoint1Lat=&listEndPoint1Lon=&listEndPoint2Lat=&listEndPoint2Lon=&zipCodeList=&listZipCodeList=&centerPointLat=&centerPointLon=&distanceLat=&distanceLon=&resolutionSquare=&listCenterPointLat=&listCenterPointLon=&listDistanceLat=&listDistanceLon=&listResolutionSquare=&citiesLevel=&listCitiesLevel=&sector=&gmlListLatLon=&featureType=&requestedTime=&startTime=&endTime=&compType=&propertyName=&product=glance&begin=2004-01-01T00%3A00%3A00&end=2022-09-16T00%3A00%3A00&Unit=e&temp=temp&Submit=Submit",
+ "timeout": 180000,
+ "headers":["USER-AGENT: JSON_ESB", "ACCEPT: text/html,application/xhtml+xml,application/xml"]
+ },
+ "route1": {
+ "type":"xmlparser",
+ "dependencies":["route0"]
+ },
+ "route.error": {
+ "type":"js",
+ "dependencies":[["listener.error"],["route0.error"],["route1.error"]],
+ "js":"message.content={}; message.content.result=false;"
+ },
+ "output": {
+ "type":"rest_responder",
+ "dependencies":[["route1"],["route.error"]]
+ },
+ "garbagecollector": {
+ "type": "simple",
+ "dependencies":[["output"],["route.error"],["output.error"]],
+ "isMessageConsumer": true
+ }
+}

flows/pipeline_emailFileNotification.json

@@ -1,7 +1,7 @@
 {
  "flow":{
  "name":"Email file notification",
- "disabled":false
+ "disabled":true
  },
  "listener": {
  "type":"file", 

install.sh.bat

@@ -0,0 +1,7 @@
+echo "Use sh ./install.sh.bat to run on Linux"
+
+npm install line-by-line
+npm install papaparse
+npm install fast-xml-parser
+npm install cron
+npm install nodemailer

lib/httpclient.js

@@ -1,170 +1,155 @@
-/* 
- * (C) 2018 TekMonks. All rights reserved.
+/** 
+ * (C) 2020 TekMonks. All rights reserved.
+ * 
+ * httpClient.js - perform HTTP client calls. Accepts callback. 
+ * Returns promise if callback is not passed.
  * 
  * callback format -> callback(error, data)
+ * promise resolves to -> { data, status, resHeaders, error }
+ * promise rejects -> error
+ * 
+ * Returned data is a Buffer object. It should be converted based on
+ * returned MIME headers.
  */
-
-const http = require("http");
-const https = require("https");
-
-const querystring = require("querystring");
-
-function post(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) req = (typeof (req) == "object" ? querystring.stringify(req):req);
- headers["Content-Length"] = Buffer.byteLength(req, "utf8");
-
- let optionspost = {
- host : host,
- port : port,
- path : path,
- method : "POST",
- headers : headers,
- timeout: timeout
- };
-
- doCall(req, optionspost, false, callback);
-}
-
-function postHttps(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) req = (typeof (req) == "object" ? querystring.stringify(req):req);
- headers["Content-Length"] = Buffer.byteLength(req, "utf8");
-
- let optionspost = {
- host : host,
- port : port,
- path : path,
- method : 'POST',
- headers : headers,
- timeout: timeout
- };
-
- doCall(req, optionspost, true, callback);
-}
-
-function put(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) req = (typeof (req) == "object" ? querystring.stringify(req):req);
- headers["Content-Length"] = Buffer.byteLength(req, "utf8");
-
- let optionsput = {
- host : host,
- port : port,
- path : path,
- method : 'PUT',
- headers : headers,
- timeout: timeout
- };
-
- doCall(req, optionsput, false, callback);
-}
-
-function putHttps(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) req = (typeof (req) == "object" ? querystring.stringify(req):req);
- headers["Content-Length"] = Buffer.byteLength(req, "utf8");
-
- let optionsput = {
- host : host,
- port : port,
- path : path,
- method : 'PUT',
- headers : headers,
- timeout: timeout
- };
-
- doCall(req, optionsput, true, callback);
-}
-
-function get(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) path += "?" + (typeof (req) == "object" ? querystring.stringify(req):req);
-
- let optionsget = {
- host : host,
- port : port,
- path : path,
- method : 'GET',
- headers : headers,
- timeout: timeout
- };
-
- doCall(null, optionsget, false, callback);
-}
-
-function getHttps(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) path += "?" + (typeof (req) == "object" ? querystring.stringify(req):req);
-
- let optionsget = {
- host : host,
- port : port,
- path : path,
- method : 'GET',
- headers : headers,
- timeout: timeout
- };
-
- doCall(null, optionsget, true, callback);
-}
-
-function deleteHttp(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) path += "?" + (typeof (req) == "object" ? querystring.stringify(req):req);
-
- let optionsdelete = {
- host : host,
- port : port,
- path : path,
- method : 'DELETE',
- headers : headers,
- timeout: timeout
- };
-
- doCall(null, optionsdelete, false, callback);
-}
-
-function deleteHttps(host, port, path, headers, req, timeout = 120000, callback) {
- if (req) path += "?" + (typeof (req) == "object" ? querystring.stringify(req):req);
-
- let optionsdelete = {
- host : host,
- port : port,
- path : path,
- method : 'GET',
- headers : headers,
- timeout: timeout
- };
-
- doCall(null, optionsdelete, true, callback);
-}
-
-function doCall(reqStr, options, secure, callback) {
- let caller = secure ? https : http;
- let response = "";
- let req = caller.request(options, res => {
- res.on("data", d => response += d);
-
- res.on("end", function() {
- let status = this.statusCode;
- callback(null, {response, status});
- });
- });
-
- if (reqStr) req.write(reqStr);
- req.end();
- req.on("error", (e) => {callback(e, null)})
-}
-
-if (require.main === module) {
- let args = process.argv.slice(2);
-
- if (args.length == 0) console.log("Usage: httpclient <host> <port> <path> <data>");
- else post(args[0], args[1], args[2], args[3], (e, data) => { 
- if (!e) console.log(JSON.stringify(data)); else console.log(e); 
- });
-}
-
-exports.get = get;
-exports.post = post;
-exports.put = put;
-exports.delete = deleteHttp;
-
-exports.getHttps = getHttps;
-exports.postHttps = postHttps;
-exports.putHttps = putHttps;
-exports.deleteHttps = deleteHttps;
+ if (!global.CONSTANTS) global.CONSTANTS = require(__dirname + "/constants.js"); // to support direct execution
+
+ const PROC_MEMORY = {};
+ const http = require("http");
+ const zlib = require("zlib");
+ const https = require("https");
+ const fspromises = require("fs").promises;
+ const querystring = require("querystring");
+ const utils = require(CONSTANTS.LIBDIR + "/utils.js");
+ const crypt = require(CONSTANTS.LIBDIR + "/crypt.js");
+
+ const DEFAULT_HTTP_TIMEOUT = 120000;
+
+ function post(host, port, path, headers, req, timeout, callback) {
+ headers = headers||{}; const body = req; _addHeaders(headers, body);
+ const optionspost = { host, port, path, method: 'POST', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(body, optionspost, false); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function postHttps(host, port, path, headers, req, timeout, sslObj, callback) {
+ headers = headers||{}; const body = req; _addHeaders(headers, body);
+ const optionspost = { host, port, path, method: 'POST', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(body, optionspost, true, sslObj); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function put(host, port, path, headers, req, timeout, callback) {
+ headers = headers||{}; const body = req; _addHeaders(headers, body);
+ const optionsput = { host, port, path, method: 'PUT', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(body, optionsput, false); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function putHttps(host, port, path, headers, req, timeout, sslObj, callback) {
+ headers = headers||{}; const body = req; _addHeaders(headers, body);
+ const optionsput = { host, port, path, method: 'PUT', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(body, optionsput, true, sslObj); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function get(host, port, path, headers, req, timeout, callback) {
+ headers = headers||{}; _addHeaders(headers, null);
+ if (req && typeof req == "object") req = querystring.stringify(req); if (req && req.trim() !== "") path += `?${req}`; 
+ const optionsget = { host, port, path, method: 'GET', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(null, optionsget, false); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function getHttps(host, port, path, headers = {}, req, timeout, sslObj, callback) {
+ headers = headers||{}; _addHeaders(headers, null);
+ if (req && typeof req == "object") req = querystring.stringify(req); if (req && req.trim() !== "") path += `?${req}`; 
+ const optionsget = { host, port, path, method: 'GET', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(null, optionsget, true, sslObj); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function deleteHttp(host, port, path, headers, req, timeout, callback) {
+ headers = headers||{}; _addHeaders(headers, null);
+ if (req && typeof req == "object") req = querystring.stringify(req); if (req && req.trim() !== "") path += `?${req}`; 
+ const optionsdelete = { host, port, path, method: 'DELETE', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(null, optionsdelete, false); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function deleteHttps(host, port, path, headers = {}, req, timeout, sslObj, callback) {
+ headers = headers||{}; _addHeaders(headers, null);
+ if (req && typeof req == "object") req = querystring.stringify(req); if (req && req.trim() !== "") path += `?${req}`; 
+ const optionsdelete = { host, port, path, method: 'DELETE', headers, timeout: timeout||DEFAULT_HTTP_TIMEOUT };
+ const result = _doCall(null, optionsdelete, true, sslObj); if (!callback) return result;
+ result.then(({ data, status, resHeaders, error }) => callback(error, data, status, resHeaders)).catch((error) => callback(error, null));
+ }
+
+ function _addHeaders(headers, body) {
+ if (body) headers["Content-Type"] = headers["Content-Type"] || "application/x-www-form-urlencoded";
+ if (body) headers["Content-Length"] = Buffer.byteLength(body, "utf8");
+ headers["Accept"] = headers["Accept"] || "*/*";
+ }
+
+ function _doCall(reqStr, options, secure, sslObj) {
+ return new Promise(async (resolve, reject) => {
+ const caller = secure ? https : http;
+ let resp, ignoreEvents = false, resPiped;
+ if (sslObj & typeof sslObj == "object") try{await _addSecureOptions(options, sslObj)} catch (err) {reject(err); return;};
+ const req = caller.request(options, res => {
+ const encoding = utils.getObjectKeyValueCaseInsensitive(res.headers, "Content-Encoding") || "identity";
+ if (encoding.toLowerCase() == "gzip") { resPiped = zlib.createGunzip(); res.pipe(resPiped); } else resPiped = res;
+
+ resPiped.on("data", chunk => { if (!ignoreEvents) resp = resp ? Buffer.concat([resp,chunk]) : chunk });
+
+ const sendError = error => { reject(error); ignoreEvents = true; };
+ res.on("error", error => sendError(error)); resPiped.on("error", error => sendError(error));
+
+ resPiped.on("end", () => {
+ if (ignoreEvents) return;
+ const status = res.statusCode, resHeaders = { ...res.headers };
+ const statusOK = Math.trunc(status / 200) == 1 && status % 200 < 100;
+
+ if (!statusOK) resolve({ error: `Bad status: ${status}`, data: resp, status, resHeaders });
+ else resolve({ error: null, data: resp, status, resHeaders });
+ });
+ });
+
+ if (reqStr) req.write(reqStr);
+ req.end();
+ req.on("error", error => reject(error));
+ });
+ }
+
+ async function _addSecureOptions(options, sslObj) {
+ const _cacheReadFile = async filepath => {
+ if (!PROC_MEMORY[filepath]) PROC_MEMORY[filepath] = await fspromises.readFile(filepath);
+ return PROC_MEMORY[filepath];
+ }
+
+ if (sslObj.pfxPath && sslObj.encryptedPassphrase) {
+ options.pfx = await _cacheReadFile(sslObj.pfxPath);
+ options.passphrase = crypt.decrypt(sslObj.encryptedPassphrase, sslObj.encryptionKey);
+ } else if (sslObj.certPath && sslObj.encryptedKeyPath) {
+ options.cert = await _cacheReadFile(ssl.certPath);
+ options.key = crypt.decrypt(await _cacheReadFile(ssl.encryptedKeyPath), sslObj.encryptionKey);
+ }
+ }
+
+ if (require.main === module) main();
+
+ function main() {
+ const args = process.argv.slice(2); if (args[0]) args[0] = args[0].toLowerCase();
+ if (args.length == 0) console.error("Usage: httpClient.js <method> <url> <body> <headers> [ssl-options]");
+ else {
+ const url = new URL(args[1]); if (url.protocol == "https:") args[0] = args[0]+"Https"; if (args[0]=="delete") args[0] = "deleteHttp"; 
+ const port = url.port && url.port != "" ? url.port : (url.protocol=="https:"?443:80), out = process.stdout.write.bind(process.stdout),
+ headers = args[3] && args[3] != "" ? JSON.parse(args[3]):{}, sslOptions = args[4] ? JSON.parse(args[4]):null,
+ totalPath = url.pathname + (url.search?url.search:"");
+ eval(args[0]).call( null, url.hostname, port, totalPath, headers, args[2], sslOptions, (err, data) => {
+ const funcToWriteTo = err?console.error:out, dataToWrite = err ? err : (process.stdout.isTTY?data.toString("utf8"):data);
+ funcToWriteTo(dataToWrite);
+ });
+ }
+ }
+
+ module.exports = { get, post, put, delete: deleteHttp, getHttps, postHttps, putHttps, deleteHttps, main };