-
Notifications
You must be signed in to change notification settings - Fork 93
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Discussion: cortex benchmark cli #610
Comments
My current implementation, gonna refactor const si = require('systeminformation');
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
const fs = require('fs');
const yaml = require('js-yaml');
const { Command } = require('commander');
const program = new Command();
program.option('--config <path>', 'Path to the configuration file');
program.parse(process.argv);
const options = program.opts();
if (!options.config) {
console.error('Configuration file path is required.');
process.exit(1);
}
let config;
try {
config = yaml.load(fs.readFileSync(options.config, 'utf8'));
console.log('Configuration loaded successfully:', config);
} catch (error) {
console.error('Failed to read or parse the configuration file:', error);
process.exit(1);
}
async function getSystemResources() {
const data = {
cpu: await si.currentLoad(),
mem: await si.mem(),
gpu: (await si.graphics()).controllers
};
console.log('Current system resources:', data);
return data;
}
async function getResourceMetadata() {
const hardware = {};
const components = ['cpu', 'memory', 'gpu', 'disk', 'osInfo', 'memLayout', 'mem', 'chassis', 'baseboard'];
for (let component of components) {
if (config.hardware.includes(component)) {
switch(component) {
case 'memory':
hardware.memory = await si.mem();
break;
case 'memLayout':
hardware.memLayout = await si.memLayout();
break;
case 'gpu':
hardware.gpu = await si.graphics();
delete hardware.gpu.displays; // Assuming displays are not needed in hardware logs
break;
case 'disk':
hardware.disk = await si.diskLayout();
break;
case 'osInfo':
hardware.osInfo = await si.osInfo();
console.log('Operating System Info:', hardware.osInfo);
break;
case 'chassis':
hardware.chassis = await si.chassis();
break;
case 'baseboard':
hardware.baseboard = await si.baseboard();
break;
default:
hardware[component] = await si[component]();
break;
}
console.log(`Hardware component ${component} data:`, hardware[component]);
}
}
return hardware;
}
function generateTokenLengths(min, max, samples) {
const step = (max - min) / (samples - 1);
const lengths = Array.from({length: samples}, (_, index) => Math.round(min + step * index));
console.log('Generated token lengths:', lengths);
return lengths;
}
async function getResourceChange(startData, endData) {
const change = {
cpu: (endData.cpu.currentload - startData.cpu.currentload) / startData.cpu.currentload * 100,
mem: (endData.mem.used - startData.mem.used) / startData.mem.total * 100,
gpu: endData.gpu.map((gpu, index) => ({
gpuUsageChange: gpu.utilizationGpu - startData.gpu[index].utilizationGpu
}))
};
console.log('Resource change calculated:', change);
return change;
}
async function benchmarkUser() {
const results = [];
const tokenLengths = generateTokenLengths(config.prompts.min, config.prompts.max, config.prompts.samples);
for (const length of tokenLengths) {
const startResources = await getSystemResources();
const start = Date.now();
const response = await fetch(config.api.url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${config.api.api_key}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({...config.api.parameters, max_tokens: length})
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const latency = Date.now() - start;
const endResources = await getSystemResources();
const resourceChange = await getResourceChange(startResources, endResources);
results.push({
tokens: length,
latency,
resourceChange
});
console.log(`Result for token length ${length}:`, results[results.length - 1]);
}
return results;
}
async function runBenchmarks() {
const hardware = await getResourceMetadata();
const userPromises = Array.from({ length: config.concurrency }, benchmarkUser);
const allResults = await Promise.all(userPromises);
const output = {
hardware,
results: allResults
};
fs.writeFileSync('output.json', JSON.stringify(output, null, 2));
console.log('Benchmark results saved to output.json');
console.log('Detailed Results:', JSON.stringify(allResults, null, 2));
}
runBenchmarks(); api:
url: http://127.0.0.1:1337/v1/chat/completions
api_key: <api_key>
parameters:
messages:
- content: You are a helpful assistant.
role: system
- content: Hello!
role: user
model: tinyllama-1.1b
stream: true
max_tokens: 2048
stop:
- hello
frequency_penalty: 0
presence_penalty: 0
temperature: 0.7
top_p: 0.95
prompts:
min: 1024
max: 2048
samples: 10
output: json
hardware:
- cpu
- gpu
- psu
- chassis
- ram
concurrency: 1 |
|
|
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Motivation
Discussion
Resources
The text was updated successfully, but these errors were encountered: