feat(infra): enhance server infrastructure with Redis connection pool and observability stack

package.json

@@ -20,6 +20,8 @@
     "packages/devreact",
     "packages/devexpo",
     "packages/devreactnative",
+    "packages/analytics-server",
+    "packages/analytics-client",
     "packages/devnext",
     "packages/deve2e",
     "packages/playground-next"

packages/analytics-client/README.md

@@ -0,0 +1 @@
+# analytics-client

packages/analytics-client/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "analytics-client",
+  "packageManager": "[email protected]"
+}

packages/analytics-server/.eslintignore

@@ -0,0 +1,4 @@
+dist/
+node_modules/
+*.js
+*.d.ts 

packages/analytics-server/.eslintrc.js

@@ -0,0 +1,19 @@
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint'],
+  extends: [
+    'eslint:recommended',
+    'plugin:@typescript-eslint/recommended',
+    'prettier',
+  ],
+  env: {
+    node: true,
+    es6: true,
+  },
+  rules: {
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'warn',
+    '@typescript-eslint/no-unused-vars': ['error', { argsIgnorePattern: '^_' }],
+  },
+}; 

packages/analytics-server/.gitignore

@@ -0,0 +1,29 @@
+# Dependencies
+node_modules/
+yarn.lock
+package-lock.json
+
+# Build output
+dist/
+
+# Environment variables
+.env
+.env.local
+.env.*.local
+
+# Logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db 

packages/analytics-server/.prettierignore

@@ -0,0 +1,7 @@
+dist/
+node_modules/
+*.js
+*.d.ts
+package.json
+package-lock.json
+yarn.lock 

packages/analytics-server/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "semi": true,
+  "trailingComma": "es5",
+  "singleQuote": true,
+  "printWidth": 100,
+  "tabWidth": 2
+} 

packages/analytics-server/Dockerfile

@@ -0,0 +1,33 @@
+# Build stage
+FROM node:18-alpine AS builder
+
+# Install build dependencies and build the project
+WORKDIR /app
+COPY package.json ./
+RUN yarn install
+COPY . .
+RUN yarn build
+
+# Runtime stage
+FROM node:18-alpine
+
+# Install runtime dependencies
+WORKDIR /app
+COPY --from=builder /app/package.json ./
+RUN yarn install --production
+
+# Copy built project and .env file from the build stage
+COPY --from=builder /app/dist ./dist
+# Do not copy .env file, it should be mounted separately
+# COPY .env ./
+
+# Expose the server port
+EXPOSE 2002
+
+# Start the server
+CMD ["node", "dist/src/index.js"]
+#  CMD ["sh", "-c", "DEBUG= node dist/index.js"]
+
+# Start the server with DEBUG mode enabled
+# CMD ["sh", "-c", "DEBUG=socket.io-redis-streams-adapter node dist/index.js"]
+#  CMD ["sh", "-c", "DEBUG=socket.io-redis node dist/index.js"]

packages/analytics-server/README.md

@@ -0,0 +1 @@
+# analytics-server

packages/analytics-server/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@metamask/analytics-server",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Analytics server for MetaMask SDK",
+  "main": "dist/src/index.js",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/src/index.js",
+    "dev": "ts-node src/index.ts",
+    "lint": "eslint . --ext .ts",
+    "lint:fix": "eslint . --ext .ts --fix",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "typecheck": "tsc --noEmit",
+    "allow-scripts": "allow-scripts"
+  },
+  "dependencies": {
+    "analytics-node": "^6.2.0",
+    "body-parser": "^1.20.2",
+    "cors": "^2.8.5",
+    "dotenv": "^16.3.1",
+    "express": "^4.18.2",
+    "express-rate-limit": "^7.1.5",
+    "helmet": "^5.1.1",
+    "ioredis": "^5.6.0",
+    "winston": "^3.11.0"
+  },
+  "devDependencies": {
+    "@lavamoat/allow-scripts": "^2.3.1",
+    "@types/analytics-node": "^3.1.13",
+    "@types/body-parser": "^1.19.4",
+    "@types/cors": "^2.8.15",
+    "@types/express": "^4.17.20",
+    "@types/node": "^20.4.1",
+    "@typescript-eslint/eslint-plugin": "^4.20.0",
+    "@typescript-eslint/parser": "^4.20.0",
+    "eslint": "^7.30.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-prettier": "^3.4.0",
+    "prettier": "^2.8.8",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.3.2"
+  }
+}

packages/analytics-server/src/index.ts

@@ -0,0 +1,171 @@
+/* eslint-disable import/first */
+import dotenv from 'dotenv';
+
+// Dotenv must be loaded before importing local files
+dotenv.config();
+
+import crypto from 'crypto';
+import Analytics from 'analytics-node';
+import bodyParser from 'body-parser';
+import cors from 'cors';
+import express from 'express';
+import { rateLimit } from 'express-rate-limit';
+import helmet from 'helmet';
+import { createLogger } from './logger';
+
+const logger = createLogger(process.env.NODE_ENV === 'development');
+
+const app = express();
+
+app.use(bodyParser.urlencoded({ extended: true }));
+app.use(bodyParser.json());
+app.use(cors());
+app.options('*', cors());
+app.use(helmet());
+app.disable('x-powered-by');
+
+// Rate limiting configuration
+const limiter = rateLimit({
+    windowMs: 60 * 1000, // 1 minute
+    max: 100000, // limit each IP to 100,000 requests per windowMs
+    legacyHeaders: false,
+});
+
+app.use(limiter);
+
+const analytics = new Analytics(
+    process.env.NODE_ENV === 'development'
+        ? process.env.SEGMENT_API_KEY_DEBUG || ''
+        : process.env.SEGMENT_API_KEY_PRODUCTION || '',
+    {
+        flushInterval: process.env.NODE_ENV === 'development' ? 1000 : 10000,
+        errorHandler: (err: Error) => {
+            logger.error(`ERROR> Analytics-node flush failed: ${err}`);
+        },
+    },
+);
+
+app.get('/', (req, res) => {
+    if (process.env.NODE_ENV === 'development') {
+        logger.info(`health check from`, {
+            'x-forwarded-for': req.headers['x-forwarded-for'],
+            'cf-connecting-ip': req.headers['cf-connecting-ip'],
+        });
+    }
+    res.json({ success: true });
+});
+
+app.post('/evt', async (req, res) => {
+    try {
+        const { body } = req;
+
+        if (!body.event) {
+            logger.error(`Event is required`);
+            return res.status(400).json({ error: 'event is required' });
+        }
+
+        if (!body.event.startsWith('sdk_')) {
+            logger.error(`Wrong event name: ${body.event}`);
+            return res.status(400).json({ error: 'wrong event name' });
+        }
+
+        const toCheckEvents = ['sdk_rpc_request_done', 'sdk_rpc_request'];
+        const allowedMethods = [
+            "eth_sendTransaction",
+            "wallet_switchEthereumChain",
+            "personal_sign",
+            "eth_signTypedData_v4",
+            "wallet_requestPermissions",
+            "metamask_connectSign"
+        ];
+
+        if (toCheckEvents.includes(body.event) &&
+            (!body.method || !allowedMethods.includes(body.method))) {
+            return res.json({ success: true });
+        }
+
+        let channelId: string = body.id || 'sdk';
+        let isExtensionEvent = body.from === 'extension';
+
+        if (typeof channelId !== 'string') {
+            logger.error(`Received event with invalid channelId: ${channelId}`, body);
+            return res.status(400).json({ status: 'error' });
+        }
+
+        let isAnonUser = false;
+
+        if (channelId === 'sdk') {
+            isAnonUser = true;
+            isExtensionEvent = true;
+        }
+
+        logger.debug(
+            `Received event /evt channelId=${channelId} isExtensionEvent=${isExtensionEvent}`,
+            body,
+        );
+
+        const userIdHash = isAnonUser
+            ? crypto.createHash('sha1').update(channelId).digest('hex')
+            : crypto.createHash('sha1').update(channelId).digest('hex');
+
+        const event = {
+            userId: userIdHash,
+            event: body.event,
+            properties: {
+                userId: userIdHash,
+                ...body.properties,
+            },
+        };
+
+        if (!event.properties.dappId) {
+            const newDappId =
+                event.properties.url && event.properties.url !== 'N/A'
+                    ? event.properties.url
+                    : event.properties.title || 'N/A';
+            event.properties.dappId = newDappId;
+            logger.debug(
+                `event: ${event.event} - dappId missing - replacing with '${newDappId}'`,
+                event,
+            );
+        }
+
+        const propertiesToExclude: string[] = ['icon', 'originationInfo', 'id'];
+
+        for (const property in body) {
+            if (
+                Object.prototype.hasOwnProperty.call(body, property) &&
+                body[property] &&
+                !propertiesToExclude.includes(property)
+            ) {
+                event.properties[property] = body[property];
+            }
+        }
+
+        if (process.env.EVENTS_DEBUG_LOGS === 'true') {
+            logger.debug('Event object:', event);
+        }
+
+        analytics.track(event, function (err: Error) {
+            if (process.env.EVENTS_DEBUG_LOGS === 'true') {
+                logger.info('Segment batch', JSON.stringify({ event }, null, 2));
+            } else {
+                logger.info('Segment batch', { event });
+            }
+
+            if (err) {
+                logger.error('Segment error:', err);
+            }
+        });
+
+        return res.json({ success: true });
+    } catch (error) {
+        return res.json({ error });
+    }
+});
+
+const port = process.env.PORT || 3001;
+app.listen(port, () => {
+    logger.info(`Analytics server listening on port ${port}`);
+});
+
+export { app }; 

packages/analytics-server/src/logger.ts

@@ -0,0 +1,19 @@
+import winston from 'winston';
+
+export const createLogger = (isDevelopment: boolean) => {
+  return winston.createLogger({
+    level: isDevelopment ? 'debug' : 'info',
+    format: winston.format.combine(
+      winston.format.timestamp(),
+      winston.format.json(),
+    ),
+    transports: [
+      new winston.transports.Console({
+        format: winston.format.combine(
+          winston.format.colorize(),
+          winston.format.simple(),
+        ),
+      }),
+    ],
+  });
+}; 

packages/analytics-server/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "es2018",
+    "module": "commonjs",
+    "lib": ["es2018"],
+    "declaration": true,
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+} 

packages/sdk-communication-layer/src/Analytics.ts

@@ -99,9 +99,9 @@ async function sendBufferedEvents(parameters: AnalyticsProps) {
 // Modified SendAnalytics to add events to buffer instead of sending directly
 export const SendAnalytics = async (
   parameters: AnalyticsProps,
-  socketServerUrl: string,
+  analyticsServerUrl: string,
 ) => {
-  targetUrl = socketServerUrl;
+  targetUrl = analyticsServerUrl;
 
   // Safely add the analytics event to the buffer
   addToBuffer(parameters);