Skip to content

Commit f6ad40c

Browse files
ShakirFarhanShakirFarhan
committed
feat: analytics
1 parent efc1e7d commit f6ad40c

File tree

10 files changed

+159
-30
lines changed

10 files changed

+159
-30
lines changed

‎api-server/src/services/kafka.service.ts

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
1-
import { Kafka, Offsets } from 'kafkajs';
1+
import { Kafka } from 'kafkajs';
22
import { config } from '../config/production';
33
import fs from 'fs';
44
import path from 'path';
55
import ProjectService from './project.service';
6-
import { prismaClient } from '../client';
76
import eventEmitter from '../utils/eventEmitter';
87
export const kafkaClient = new Kafka({
98
clientId: `API-SERVER`,
@@ -17,11 +16,10 @@ export const kafkaClient = new Kafka({
1716
ca: [fs.readFileSync(path.resolve(__dirname, '../../ca.pem'), 'utf-8')],
1817
},
1918
});
20-
2119
export async function logsConsumer() {
2220
const consumer = kafkaClient.consumer({ groupId: 'deployment-logs' });
2321
await consumer.connect();
24-
await consumer.subscribe({ topic: 'builder-logs' });
22+
await consumer.subscribe({ topics: ['builder-logs', 'proxy-analytics'] });
2523
await consumer.run({
2624
autoCommit: false,
2725
eachBatch: async function ({
@@ -34,25 +32,31 @@ export async function logsConsumer() {
3432
const messages = batch.messages;
3533
for (const message of messages) {
3634
if (!message.value) return;
37-
38-
const { DEPLOYMENT_ID, log, status } = JSON.parse(
39-
message.value.toString()
40-
) as {
41-
DEPLOYMENT_ID: string;
42-
log: string;
43-
status?: any;
44-
};
45-
4635
try {
47-
if (status) {
48-
await ProjectService.updateDeployment(DEPLOYMENT_ID, { status });
36+
if (batch.topic === 'builder-logs') {
37+
const { DEPLOYMENT_ID, log, status } = JSON.parse(
38+
message.value.toString()
39+
) as {
40+
DEPLOYMENT_ID: string;
41+
log: string;
42+
status?: any;
43+
};
44+
45+
if (status) {
46+
await ProjectService.updateDeployment(DEPLOYMENT_ID, { status });
47+
}
48+
// Emitting a new Event for SSE
49+
eventEmitter.emit(
50+
'log',
51+
JSON.stringify({ deploymentId: DEPLOYMENT_ID, log })
52+
);
53+
await ProjectService.createLog(DEPLOYMENT_ID, log);
54+
} else if (batch.topic === 'proxy-analytics') {
55+
/*
56+
Handling Analytics
57+
Either Storing in Current DB(Postgres) or Influx
58+
*/
4959
}
50-
// Emitting a new Event for SSE
51-
eventEmitter.emit(
52-
'log',
53-
JSON.stringify({ deploymentId: DEPLOYMENT_ID, log })
54-
);
55-
await ProjectService.createLog(DEPLOYMENT_ID, log);
5660
resolveOffset(message.offset);
5761
await commitOffsetsIfNecessary({
5862
topics: [

‎builder/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ COPY . .
1515
RUN chmod +x main.sh
1616
RUN chmod +x script.js
1717

18-
ENTRYPOINT [ "/home/app/main.sh" ]
18+
ENTRYPOINT [ "/home/app/main.sh" ]

‎builder/script.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ async function publishLog(log, status) {
2020
],
2121
});
2222
}
23-
console.log(process.env);
2423

2524
async function init() {
2625
await producer.connect();

‎proxy-server/package-lock.json

Lines changed: 10 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎proxy-server/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"dependencies": {
2121
"dotenv": "^16.4.5",
2222
"express": "^4.19.2",
23-
"http-proxy": "^1.18.1"
23+
"http-proxy": "^1.18.1",
24+
"kafkajs": "^2.2.4"
2425
}
2526
}

‎proxy-server/src/config/index.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
export const config = {
2+
KAFKA: {
3+
BROKER: process.env.APP_KAFKA_BROKER as unknown as string,
4+
USERNAME: process.env.APP_KAFKA_USERNAME as string,
5+
PASSWORD: process.env.APP_KAFKA_PASSWORD as string,
6+
},
7+
};

‎proxy-server/src/index.ts

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,90 @@
11
import express from 'express';
22
import dotenv from 'dotenv';
3-
dotenv.config();
43
import httpProxy from 'http-proxy';
4+
import { isAssetRequest } from './utils/validations';
5+
import { publishAnalytic } from './services/kafka';
6+
7+
dotenv.config();
8+
59
const app = express();
610
const PORT = 8000;
711
const proxy = httpProxy.createProxy();
12+
13+
const fallbackHtml = `
14+
<!DOCTYPE html>
15+
<html lang="en">
16+
<head>
17+
<meta charset="UTF-8">
18+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
19+
<title>Site Not Found</title>
20+
<style>
21+
body { font-family: Arial, sans-serif; text-align: center; padding: 50px; }
22+
h1 { color: #444; }
23+
</style>
24+
</head>
25+
<body>
26+
<h1>Site Not Found</h1>
27+
<p>The requested site could not be found. Please check the URL and try again.</p>
28+
</body>
29+
</html>
30+
`;
31+
832
app.use((req, res) => {
9-
const host = req.hostname;
10-
const subDomain = host.split('.')[0];
33+
const host = req.headers.host;
34+
const subDomain = host ? host.split('.')[0] : null;
35+
1136
if (!subDomain) {
1237
return res.status(400).send('Invalid subdomain');
1338
}
39+
40+
const startTime = process.hrtime();
41+
42+
res.on('finish', async () => {
43+
console.log(req.path);
44+
if (isAssetRequest(req.path)) return;
45+
const [seconds, nanoseconds] = process.hrtime(startTime);
46+
// Convert the duration to milliseconds
47+
const duration = seconds * 1000 + nanoseconds / 1e6;
48+
const analyticsData = {
49+
id: Date.now().toString(),
50+
timestamp: new Date().toISOString(),
51+
subdomain: subDomain,
52+
path: req.url,
53+
method: req.method,
54+
statusCode: res.statusCode,
55+
userAgent: req.get('User-Agent'),
56+
referer: req.get('Referer'),
57+
ip: req.ip,
58+
country: req.get('CF-IPCountry'),
59+
duration: duration.toFixed(2),
60+
contentLength: res.get('Content-Length'),
61+
};
62+
await publishAnalytic(analyticsData);
63+
});
64+
1465
return proxy.web(req, res, {
15-
target: `${process.env.S3_BASE_URL}/${subDomain}`, // forwards the request
16-
changeOrigin: true, // modifies the host header
66+
target: `${process.env.S3_BASE_URL}/${subDomain}`,
67+
changeOrigin: true,
1768
});
1869
});
1970
proxy.on('proxyReq', (proxy, req, res) => {
2071
if (req.url === '/') {
2172
proxy.path += 'index.html';
2273
}
2374
});
75+
proxy.on('proxyRes', (proxyRes, req, res) => {
76+
if (proxyRes.statusCode === 403 || proxyRes.statusCode === 404) {
77+
// For 403 or 404 errors, serve the fallback HTML
78+
proxyRes.destroy(); // End the original response
79+
80+
res.writeHead(200, {
81+
'Content-Type': 'text/html',
82+
'Content-Length': Buffer.byteLength(fallbackHtml),
83+
});
84+
res.end(fallbackHtml);
85+
}
86+
});
87+
2488
app.listen(PORT, () => {
2589
console.log(`Server running on port ${PORT}`);
2690
});

‎proxy-server/src/middlewares/analytics.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { Request, Response, NextFunction } from 'express';
2+
export const analytics = async (
3+
req: Request,
4+
res: Response,
5+
next: NextFunction
6+
) => {};

‎proxy-server/src/services/kafka.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
import { Kafka } from 'kafkajs';
2+
import fs from 'fs';
3+
import path from 'path';
4+
import { config } from '../config';
5+
const kafkaClient = new Kafka({
6+
clientId: `proxy-server`,
7+
brokers: [config.KAFKA.BROKER],
8+
sasl: {
9+
username: config.KAFKA.USERNAME,
10+
password: config.KAFKA.PASSWORD,
11+
mechanism: 'plain',
12+
},
13+
ssl: {
14+
ca: [fs.readFileSync(path.resolve(__dirname, '../../ca.pem'), 'utf-8')],
15+
},
16+
});
17+
const producer = kafkaClient.producer();
18+
export async function publishAnalytic(data: any) {
19+
await producer.send({
20+
topic: 'proxy-analytics',
21+
messages: [{ key: 'analytic', value: JSON.stringify({ data }) }],
22+
});
23+
}

‎proxy-server/src/utils/validations.ts

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
export const isAssetRequest = (path: string) => {
2+
const assetExtensions = [
3+
'.js',
4+
'.css',
5+
'.png',
6+
'.jpg',
7+
'.jpeg',
8+
'.gif',
9+
'.svg',
10+
'.ico',
11+
'.woff',
12+
'.woff2',
13+
'.json',
14+
];
15+
return assetExtensions.some((ext) => path.endsWith(ext));
16+
};

0 commit comments

Comments
 (0)