hiro-v
commented
1 month ago My current implementation, gonna refactor
const si = require('systeminformation');
const fetch = (...args) => import('node-fetch').then(({default: fetch}) => fetch(...args));
const fs = require('fs');
const yaml = require('js-yaml');
const { Command } = require('commander');
const program = new Command();
program.option('--config <path>', 'Path to the configuration file');
program.parse(process.argv);
const options = program.opts();
if (!options.config) {
console.error('Configuration file path is required.');
process.exit(1);
}
let config;
try {
config = yaml.load(fs.readFileSync(options.config, 'utf8'));
console.log('Configuration loaded successfully:', config);
} catch (error) {
console.error('Failed to read or parse the configuration file:', error);
process.exit(1);
}
async function getSystemResources() {
const data = {
cpu: await si.currentLoad(),
mem: await si.mem(),
gpu: (await si.graphics()).controllers
};
console.log('Current system resources:', data);
return data;
}
async function getResourceMetadata() {
const hardware = {};
const components = ['cpu', 'memory', 'gpu', 'disk', 'osInfo', 'memLayout', 'mem', 'chassis', 'baseboard'];
for (let component of components) {
if (config.hardware.includes(component)) {
switch(component) {
case 'memory':
hardware.memory = await si.mem();
break;
case 'memLayout':
hardware.memLayout = await si.memLayout();
break;
case 'gpu':
hardware.gpu = await si.graphics();
delete hardware.gpu.displays; // Assuming displays are not needed in hardware logs
break;
case 'disk':
hardware.disk = await si.diskLayout();
break;
case 'osInfo':
hardware.osInfo = await si.osInfo();
console.log('Operating System Info:', hardware.osInfo);
break;
case 'chassis':
hardware.chassis = await si.chassis();
break;
case 'baseboard':
hardware.baseboard = await si.baseboard();
break;
default:
hardware[component] = await si[component]();
break;
}
console.log(`Hardware component ${component} data:`, hardware[component]);
}
}
return hardware;
}
function generateTokenLengths(min, max, samples) {
const step = (max - min) / (samples - 1);
const lengths = Array.from({length: samples}, (_, index) => Math.round(min + step * index));
console.log('Generated token lengths:', lengths);
return lengths;
}
async function getResourceChange(startData, endData) {
const change = {
cpu: (endData.cpu.currentload - startData.cpu.currentload) / startData.cpu.currentload * 100,
mem: (endData.mem.used - startData.mem.used) / startData.mem.total * 100,
gpu: endData.gpu.map((gpu, index) => ({
gpuUsageChange: gpu.utilizationGpu - startData.gpu[index].utilizationGpu
}))
};
console.log('Resource change calculated:', change);
return change;
}
async function benchmarkUser() {
const results = [];
const tokenLengths = generateTokenLengths(config.prompts.min, config.prompts.max, config.prompts.samples);
for (const length of tokenLengths) {
const startResources = await getSystemResources();
const start = Date.now();
const response = await fetch(config.api.url, {
method: 'POST',
headers: {
'Authorization': `Bearer ${config.api.api_key}`,
'Content-Type': 'application/json'
},
body: JSON.stringify({...config.api.parameters, max_tokens: length})
});
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
const latency = Date.now() - start;
const endResources = await getSystemResources();
const resourceChange = await getResourceChange(startResources, endResources);
results.push({
tokens: length,
latency,
resourceChange
});
console.log(`Result for token length ${length}:`, results[results.length - 1]);
}
return results;
}
async function runBenchmarks() {
const hardware = await getResourceMetadata();
const userPromises = Array.from({ length: config.concurrency }, benchmarkUser);
const allResults = await Promise.all(userPromises);
const output = {
hardware,
results: allResults
};
fs.writeFileSync('output.json', JSON.stringify(output, null, 2));
console.log('Benchmark results saved to output.json');
console.log('Detailed Results:', JSON.stringify(allResults, null, 2));
}
runBenchmarks();api:
url: http://127.0.0.1:1337/v1/chat/completions
api_key: <api_key>
parameters:
messages:
- content: You are a helpful assistant.
role: system
- content: Hello!
role: user
model: tinyllama-1.1b
stream: true
max_tokens: 2048
stop:
- hello
frequency_penalty: 0
presence_penalty: 0
temperature: 0.7
top_p: 0.95
prompts:
min: 1024
max: 2048
samples: 10
output: json
hardware:
- cpu
- gpu
- psu
- chassis
- ram
concurrency: 1
Motivation
Discussion
2. Initialize a compatible engine
cortex init
3. Download a GGUF model from Hugging Face
Models save to $(npm list -g)/node_modules
cortex models pull janhq/TinyLlama-1.1B-Chat-v1.0-GGUF
4. Load the model
cortex models start janhq/TinyLlama-1.1B-Chat-v1.0-GGUF
Resources