chore: 增加scripts目录
This commit is contained in:
506
scripts/ollama/demo-parallel.js
Normal file
506
scripts/ollama/demo-parallel.js
Normal file
@@ -0,0 +1,506 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* 演示并行处理效果的脚本
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
async function demoParallelProcessing() {
|
||||
console.log('🚀 Parallel Processing Demo');
|
||||
console.log('============================');
|
||||
console.log('');
|
||||
|
||||
const rootDir = path.dirname(__dirname); // 回到项目根目录
|
||||
const distDir = path.join(rootDir, 'dist');
|
||||
|
||||
console.log('📁 Checking dist directory...');
|
||||
|
||||
if (!fs.existsSync(distDir)) {
|
||||
console.log('❌ Dist directory not found');
|
||||
console.log('💡 Building demo files instead...');
|
||||
|
||||
// 创建演示文件
|
||||
await createDemoFiles();
|
||||
return await demoWithCreatedFiles();
|
||||
}
|
||||
|
||||
console.log(`✅ Found dist directory: ${distDir}`);
|
||||
|
||||
// 统计dist目录中的JS文件
|
||||
const jsFiles = await countJsFiles(distDir);
|
||||
console.log(`📝 Found ${jsFiles} JavaScript files`);
|
||||
|
||||
if (jsFiles > 3) {
|
||||
console.log('🎯 Perfect for parallel processing!');
|
||||
console.log('');
|
||||
console.log('💡 Commands to try:');
|
||||
console.log(' node scripts/optimize-dist.js # Automatic parallel processing');
|
||||
console.log(' node scripts/optimized-processor.js ./dist # Manual parallel processing');
|
||||
console.log(' node scripts/performance-test.js # Performance comparison');
|
||||
console.log('');
|
||||
console.log('⚡ Parallel processing features:');
|
||||
console.log(' • ✅ Worker threads for concurrent file processing');
|
||||
console.log(' • ✅ Up to 4 parallel workers');
|
||||
console.log(' • ✅ Intelligent caching to avoid reprocessing');
|
||||
console.log(' • ✅ Fallback to single-thread when needed');
|
||||
} else {
|
||||
console.log('📝 Fewer than 4 files, single-thread may be more efficient');
|
||||
}
|
||||
}
|
||||
|
||||
async function createDemoFiles() {
|
||||
const demoDir = path.join(__dirname, 'demo-files');
|
||||
|
||||
// 清理旧的演示文件
|
||||
if (fs.existsSync(demoDir)) {
|
||||
fs.rmSync(demoDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
fs.mkdirSync(demoDir, { recursive: true });
|
||||
|
||||
console.log('📝 Creating demo files for parallel processing test...');
|
||||
|
||||
// 创建多个不同大小的JS文件
|
||||
const demoFiles = [
|
||||
{
|
||||
name: 'utils.js',
|
||||
content: `
|
||||
// Utility functions
|
||||
function formatNumber(num) {
|
||||
console.log("Formatting number:", num);
|
||||
return num.toLocaleString();
|
||||
}
|
||||
|
||||
function formatDate(date) {
|
||||
console.info("Formatting date:", date);
|
||||
return date.toISOString().split('T')[0];
|
||||
}
|
||||
|
||||
function validateEmail(email) {
|
||||
console.debug("Validating email:", email);
|
||||
const regex = /^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$/;
|
||||
const result = regex.test(email);
|
||||
console.log("Email validation result:", result);
|
||||
return result;
|
||||
}
|
||||
|
||||
console.warn("Utils module loaded");
|
||||
`
|
||||
},
|
||||
{
|
||||
name: 'api-client.js',
|
||||
content: `
|
||||
// API client
|
||||
class ApiClient {
|
||||
constructor(baseUrl) {
|
||||
console.log("Creating API client for:", baseUrl);
|
||||
this.baseUrl = baseUrl;
|
||||
this.cache = new Map();
|
||||
}
|
||||
|
||||
async request(endpoint, options = {}) {
|
||||
console.info("Making request to:", endpoint);
|
||||
console.debug("Request options:", options);
|
||||
|
||||
const url = this.baseUrl + endpoint;
|
||||
console.log("Full URL:", url);
|
||||
|
||||
try {
|
||||
const response = await fetch(url, options);
|
||||
console.debug("Response status:", response.status);
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("Request failed:", response.statusText);
|
||||
throw new Error(response.statusText);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.info("Request successful, got data");
|
||||
return data;
|
||||
} catch (error) {
|
||||
console.error("Request error:", error.message);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
get(endpoint) {
|
||||
console.log("GET request to:", endpoint);
|
||||
return this.request(endpoint, { method: 'GET' });
|
||||
}
|
||||
|
||||
post(endpoint, data) {
|
||||
console.log("POST request to:", endpoint);
|
||||
console.debug("POST data:", data);
|
||||
return this.request(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(data)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("API client module loaded");
|
||||
`
|
||||
},
|
||||
{
|
||||
name: 'data-processor.js',
|
||||
content: `
|
||||
// Data processing utilities
|
||||
class DataProcessor {
|
||||
constructor() {
|
||||
console.log("Initializing DataProcessor");
|
||||
this.processedCount = 0;
|
||||
}
|
||||
|
||||
processArray(data, processor) {
|
||||
console.log("Processing array with", data.length, "items");
|
||||
console.time("array-processing");
|
||||
|
||||
const results = data.map((item, index) => {
|
||||
console.debug("Processing item", index, ":", item);
|
||||
const result = processor(item);
|
||||
this.processedCount++;
|
||||
return result;
|
||||
});
|
||||
|
||||
console.timeEnd("array-processing");
|
||||
console.info("Array processing complete");
|
||||
console.log("Total processed so far:", this.processedCount);
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
filterAndMap(data, filterFn, mapFn) {
|
||||
console.log("Filter and map operation on", data.length, "items");
|
||||
console.time("filter-map");
|
||||
|
||||
const filtered = data.filter(item => {
|
||||
const result = filterFn(item);
|
||||
console.debug("Filter result:", result);
|
||||
return result;
|
||||
});
|
||||
|
||||
console.log("Filtered to", filtered.length, "items");
|
||||
|
||||
const mapped = filtered.map(item => {
|
||||
console.debug("Mapping item:", item);
|
||||
const result = mapFn(item);
|
||||
this.processedCount++;
|
||||
return result;
|
||||
});
|
||||
|
||||
console.timeEnd("filter-map");
|
||||
console.info("Filter and map complete");
|
||||
|
||||
return mapped;
|
||||
}
|
||||
|
||||
async processAsync(data, asyncProcessor) {
|
||||
console.log("Async processing of", data.length, "items");
|
||||
console.time("async-processing");
|
||||
|
||||
const promises = data.map(async (item, index) => {
|
||||
console.debug("Async processing item", index);
|
||||
try {
|
||||
const result = await asyncProcessor(item);
|
||||
this.processedCount++;
|
||||
console.debug("Async item processed:", index);
|
||||
return result;
|
||||
} catch (error) {
|
||||
console.error("Error processing item", index, ":", error.message);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
const results = await Promise.all(promises);
|
||||
console.timeEnd("async-processing");
|
||||
console.info("Async processing complete");
|
||||
|
||||
return results.filter(r => r !== null);
|
||||
}
|
||||
|
||||
getStats() {
|
||||
console.info("Getting processor stats");
|
||||
return {
|
||||
processedCount: this.processedCount,
|
||||
timestamp: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Data processor module loaded");
|
||||
`
|
||||
},
|
||||
{
|
||||
name: 'event-emitter.js',
|
||||
content: `
|
||||
// Custom event emitter
|
||||
class EventEmitter {
|
||||
constructor() {
|
||||
console.log("Creating EventEmitter");
|
||||
this.events = new Map();
|
||||
this.maxListeners = 10;
|
||||
}
|
||||
|
||||
on(event, listener) {
|
||||
console.log("Adding listener for event:", event);
|
||||
if (!this.events.has(event)) {
|
||||
this.events.set(event, []);
|
||||
}
|
||||
|
||||
const listeners = this.events.get(event);
|
||||
listeners.push(listener);
|
||||
|
||||
console.debug("Total listeners for", event, ":", listeners.length);
|
||||
|
||||
if (listeners.length > this.maxListeners) {
|
||||
console.warn("Possible memory leak detected for event:", event);
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
emit(event, ...args) {
|
||||
console.info("Emitting event:", event);
|
||||
console.debug("Event args:", args);
|
||||
|
||||
const listeners = this.events.get(event);
|
||||
if (!listeners || listeners.length === 0) {
|
||||
console.debug("No listeners for event:", event);
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log("Notifying", listeners.length, "listeners");
|
||||
|
||||
listeners.forEach(listener => {
|
||||
try {
|
||||
listener(...args);
|
||||
} catch (error) {
|
||||
console.error("Error in event listener:", error.message);
|
||||
}
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
off(event, listener) {
|
||||
console.log("Removing listener for event:", event);
|
||||
const listeners = this.events.get(event);
|
||||
|
||||
if (listeners) {
|
||||
const index = listeners.indexOf(listener);
|
||||
if (index > -1) {
|
||||
listeners.splice(index, 1);
|
||||
console.debug("Listener removed, remaining:", listeners.length);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
console.debug("Listener not found for event:", event);
|
||||
return false;
|
||||
}
|
||||
|
||||
once(event, listener) {
|
||||
console.log("Adding once listener for event:", event);
|
||||
|
||||
const onceWrapper = (...args) => {
|
||||
console.debug("Once wrapper called for event:", event);
|
||||
this.off(event, onceWrapper);
|
||||
listener(...args);
|
||||
};
|
||||
|
||||
return this.on(event, onceWrapper);
|
||||
}
|
||||
|
||||
removeAllListeners(event) {
|
||||
if (event) {
|
||||
console.log("Removing all listeners for event:", event);
|
||||
this.events.delete(event);
|
||||
} else {
|
||||
console.log("Removing all listeners for all events");
|
||||
this.events.clear();
|
||||
}
|
||||
}
|
||||
|
||||
listenerCount(event) {
|
||||
const listeners = this.events.get(event);
|
||||
const count = listeners ? listeners.length : 0;
|
||||
console.debug("Listener count for", event, ":", count);
|
||||
return count;
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Event emitter module loaded");
|
||||
`
|
||||
},
|
||||
{
|
||||
name: 'logger.js',
|
||||
content: `
|
||||
// Logger utility
|
||||
class Logger {
|
||||
constructor(level = 'info') {
|
||||
console.log("Creating logger with level:", level);
|
||||
this.level = level;
|
||||
this.levels = {
|
||||
debug: 0,
|
||||
info: 1,
|
||||
warn: 2,
|
||||
error: 3
|
||||
};
|
||||
}
|
||||
|
||||
debug(message, ...args) {
|
||||
console.debug("DEBUG:", message, ...args);
|
||||
this.log('debug', message, args);
|
||||
}
|
||||
|
||||
info(message, ...args) {
|
||||
console.info("INFO:", message, ...args);
|
||||
this.log('info', message, args);
|
||||
}
|
||||
|
||||
warn(message, ...args) {
|
||||
console.warn("WARN:", message, ...args);
|
||||
this.log('warn', message, args);
|
||||
}
|
||||
|
||||
error(message, ...args) {
|
||||
console.error("ERROR:", message, ...args);
|
||||
this.log('error', message, args);
|
||||
}
|
||||
|
||||
log(level, message, args) {
|
||||
const shouldLog = this.levels[level] >= this.levels[this.level];
|
||||
|
||||
if (shouldLog) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const logMessage = [timestamp, level.toUpperCase(), message, ...args].join(' ');
|
||||
|
||||
console.log("📝", logMessage);
|
||||
|
||||
// 这里可以添加文件日志等
|
||||
this.writeToStorage(level, logMessage);
|
||||
}
|
||||
}
|
||||
|
||||
writeToStorage(level, message) {
|
||||
// 模拟写入存储
|
||||
console.debug("Writing to storage:", level, message);
|
||||
}
|
||||
|
||||
setLevel(level) {
|
||||
console.log("Setting log level to:", level);
|
||||
this.level = level;
|
||||
}
|
||||
|
||||
createChild(prefix) {
|
||||
console.log("Creating child logger with prefix:", prefix);
|
||||
return new Logger(this.level);
|
||||
}
|
||||
}
|
||||
|
||||
console.warn("Logger module loaded");
|
||||
`
|
||||
}
|
||||
];
|
||||
|
||||
// 写入演示文件
|
||||
demoFiles.forEach(file => {
|
||||
const filePath = path.join(demoDir, file.name);
|
||||
fs.writeFileSync(filePath, file.content);
|
||||
console.log(`✅ Created: ${file.name}`);
|
||||
});
|
||||
|
||||
console.log(`📁 Demo files created in: ${demoDir}`);
|
||||
return demoDir;
|
||||
}
|
||||
|
||||
async function countJsFiles(dir) {
|
||||
let count = 0;
|
||||
|
||||
function traverse(currentDir) {
|
||||
const items = fs.readdirSync(currentDir);
|
||||
|
||||
for (const item of items) {
|
||||
const itemPath = path.join(currentDir, item);
|
||||
const stats = fs.statSync(itemPath);
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
traverse(itemPath);
|
||||
} else if (item.endsWith('.js')) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
traverse(dir);
|
||||
return count;
|
||||
}
|
||||
|
||||
async function demoWithCreatedFiles() {
|
||||
const demoDir = path.join(__dirname, 'demo-files');
|
||||
|
||||
console.log('');
|
||||
console.log('🚀 Testing parallel processing with demo files...');
|
||||
console.log('');
|
||||
|
||||
try {
|
||||
// 使用优化处理器处理演示文件
|
||||
const { processDirectory } = require('./optimized-processor.js');
|
||||
|
||||
const result = await processDirectory(demoDir, {
|
||||
useOllama: false,
|
||||
parallel: true,
|
||||
dryRun: false
|
||||
});
|
||||
|
||||
console.log('');
|
||||
console.log('📊 Processing Results:');
|
||||
console.log(` Total files: ${result.totalFiles}`);
|
||||
console.log(` Successful: ${result.successful}`);
|
||||
console.log(` Failed: ${result.failed}`);
|
||||
console.log(` Processing time: ${result.processingTime}ms`);
|
||||
console.log(` Size reduction: ${result.totalReduction} characters`);
|
||||
console.log('');
|
||||
|
||||
if (result.successful > 0 && result.processingTime > 0) {
|
||||
const avgTime = result.processingTime / result.successful;
|
||||
console.log(` Average time per file: ${avgTime.toFixed(1)}ms`);
|
||||
|
||||
if (result.totalFiles >= 4) {
|
||||
console.log('✅ Parallel processing was used (≥4 files)');
|
||||
} else {
|
||||
console.log('⚠️ Single-thread processing was used (<4 files)');
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Demo processing failed:', error.message);
|
||||
} finally {
|
||||
// 清理演示文件
|
||||
console.log('');
|
||||
console.log('🧹 Cleaning up demo files...');
|
||||
if (fs.existsSync(demoDir)) {
|
||||
fs.rmSync(demoDir, { recursive: true, force: true });
|
||||
}
|
||||
console.log('✅ Demo completed!');
|
||||
}
|
||||
}
|
||||
|
||||
// 主程序
|
||||
async function main() {
|
||||
try {
|
||||
await demoParallelProcessing();
|
||||
} catch (error) {
|
||||
console.error('❌ Demo failed:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = { demoParallelProcessing, createDemoFiles };
|
||||
Reference in New Issue
Block a user