506 lines
15 KiB
JavaScript
506 lines
15 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
/**
|
|
* 演示并行处理效果的脚本
|
|
*/
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
|
|
async function demoParallelProcessing() {
|
|
console.log('🚀 Parallel Processing Demo');
|
|
console.log('============================');
|
|
console.log('');
|
|
|
|
const rootDir = path.dirname(__dirname); // 回到项目根目录
|
|
const distDir = path.join(rootDir, 'dist');
|
|
|
|
console.log('📁 Checking dist directory...');
|
|
|
|
if (!fs.existsSync(distDir)) {
|
|
console.log('❌ Dist directory not found');
|
|
console.log('💡 Building demo files instead...');
|
|
|
|
// 创建演示文件
|
|
await createDemoFiles();
|
|
return await demoWithCreatedFiles();
|
|
}
|
|
|
|
console.log(`✅ Found dist directory: ${distDir}`);
|
|
|
|
// 统计dist目录中的JS文件
|
|
const jsFiles = await countJsFiles(distDir);
|
|
console.log(`📝 Found ${jsFiles} JavaScript files`);
|
|
|
|
if (jsFiles > 3) {
|
|
console.log('🎯 Perfect for parallel processing!');
|
|
console.log('');
|
|
console.log('💡 Commands to try:');
|
|
console.log(' node scripts/optimize-dist.js # Automatic parallel processing');
|
|
console.log(' node scripts/optimized-processor.js ./dist # Manual parallel processing');
|
|
console.log(' node scripts/performance-test.js # Performance comparison');
|
|
console.log('');
|
|
console.log('⚡ Parallel processing features:');
|
|
console.log(' • ✅ Worker threads for concurrent file processing');
|
|
console.log(' • ✅ Up to 4 parallel workers');
|
|
console.log(' • ✅ Intelligent caching to avoid reprocessing');
|
|
console.log(' • ✅ Fallback to single-thread when needed');
|
|
} else {
|
|
console.log('📝 Fewer than 4 files, single-thread may be more efficient');
|
|
}
|
|
}
|
|
|
|
async function createDemoFiles() {
|
|
const demoDir = path.join(__dirname, 'demo-files');
|
|
|
|
// 清理旧的演示文件
|
|
if (fs.existsSync(demoDir)) {
|
|
fs.rmSync(demoDir, { recursive: true, force: true });
|
|
}
|
|
|
|
fs.mkdirSync(demoDir, { recursive: true });
|
|
|
|
console.log('📝 Creating demo files for parallel processing test...');
|
|
|
|
// 创建多个不同大小的JS文件
|
|
const demoFiles = [
|
|
{
|
|
name: 'utils.js',
|
|
content: `
|
|
// Utility functions
|
|
function formatNumber(num) {
|
|
console.log("Formatting number:", num);
|
|
return num.toLocaleString();
|
|
}
|
|
|
|
function formatDate(date) {
|
|
console.info("Formatting date:", date);
|
|
return date.toISOString().split('T')[0];
|
|
}
|
|
|
|
function validateEmail(email) {
|
|
console.debug("Validating email:", email);
|
|
const regex = /^[^\\s@]+@[^\\s@]+\\.[^\\s@]+$/;
|
|
const result = regex.test(email);
|
|
console.log("Email validation result:", result);
|
|
return result;
|
|
}
|
|
|
|
console.warn("Utils module loaded");
|
|
`
|
|
},
|
|
{
|
|
name: 'api-client.js',
|
|
content: `
|
|
// API client
|
|
class ApiClient {
|
|
constructor(baseUrl) {
|
|
console.log("Creating API client for:", baseUrl);
|
|
this.baseUrl = baseUrl;
|
|
this.cache = new Map();
|
|
}
|
|
|
|
async request(endpoint, options = {}) {
|
|
console.info("Making request to:", endpoint);
|
|
console.debug("Request options:", options);
|
|
|
|
const url = this.baseUrl + endpoint;
|
|
console.log("Full URL:", url);
|
|
|
|
try {
|
|
const response = await fetch(url, options);
|
|
console.debug("Response status:", response.status);
|
|
|
|
if (!response.ok) {
|
|
console.error("Request failed:", response.statusText);
|
|
throw new Error(response.statusText);
|
|
}
|
|
|
|
const data = await response.json();
|
|
console.info("Request successful, got data");
|
|
return data;
|
|
} catch (error) {
|
|
console.error("Request error:", error.message);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
get(endpoint) {
|
|
console.log("GET request to:", endpoint);
|
|
return this.request(endpoint, { method: 'GET' });
|
|
}
|
|
|
|
post(endpoint, data) {
|
|
console.log("POST request to:", endpoint);
|
|
console.debug("POST data:", data);
|
|
return this.request(endpoint, {
|
|
method: 'POST',
|
|
headers: { 'Content-Type': 'application/json' },
|
|
body: JSON.stringify(data)
|
|
});
|
|
}
|
|
}
|
|
|
|
console.warn("API client module loaded");
|
|
`
|
|
},
|
|
{
|
|
name: 'data-processor.js',
|
|
content: `
|
|
// Data processing utilities
|
|
class DataProcessor {
|
|
constructor() {
|
|
console.log("Initializing DataProcessor");
|
|
this.processedCount = 0;
|
|
}
|
|
|
|
processArray(data, processor) {
|
|
console.log("Processing array with", data.length, "items");
|
|
console.time("array-processing");
|
|
|
|
const results = data.map((item, index) => {
|
|
console.debug("Processing item", index, ":", item);
|
|
const result = processor(item);
|
|
this.processedCount++;
|
|
return result;
|
|
});
|
|
|
|
console.timeEnd("array-processing");
|
|
console.info("Array processing complete");
|
|
console.log("Total processed so far:", this.processedCount);
|
|
|
|
return results;
|
|
}
|
|
|
|
filterAndMap(data, filterFn, mapFn) {
|
|
console.log("Filter and map operation on", data.length, "items");
|
|
console.time("filter-map");
|
|
|
|
const filtered = data.filter(item => {
|
|
const result = filterFn(item);
|
|
console.debug("Filter result:", result);
|
|
return result;
|
|
});
|
|
|
|
console.log("Filtered to", filtered.length, "items");
|
|
|
|
const mapped = filtered.map(item => {
|
|
console.debug("Mapping item:", item);
|
|
const result = mapFn(item);
|
|
this.processedCount++;
|
|
return result;
|
|
});
|
|
|
|
console.timeEnd("filter-map");
|
|
console.info("Filter and map complete");
|
|
|
|
return mapped;
|
|
}
|
|
|
|
async processAsync(data, asyncProcessor) {
|
|
console.log("Async processing of", data.length, "items");
|
|
console.time("async-processing");
|
|
|
|
const promises = data.map(async (item, index) => {
|
|
console.debug("Async processing item", index);
|
|
try {
|
|
const result = await asyncProcessor(item);
|
|
this.processedCount++;
|
|
console.debug("Async item processed:", index);
|
|
return result;
|
|
} catch (error) {
|
|
console.error("Error processing item", index, ":", error.message);
|
|
return null;
|
|
}
|
|
});
|
|
|
|
const results = await Promise.all(promises);
|
|
console.timeEnd("async-processing");
|
|
console.info("Async processing complete");
|
|
|
|
return results.filter(r => r !== null);
|
|
}
|
|
|
|
getStats() {
|
|
console.info("Getting processor stats");
|
|
return {
|
|
processedCount: this.processedCount,
|
|
timestamp: new Date().toISOString()
|
|
};
|
|
}
|
|
}
|
|
|
|
console.warn("Data processor module loaded");
|
|
`
|
|
},
|
|
{
|
|
name: 'event-emitter.js',
|
|
content: `
|
|
// Custom event emitter
|
|
class EventEmitter {
|
|
constructor() {
|
|
console.log("Creating EventEmitter");
|
|
this.events = new Map();
|
|
this.maxListeners = 10;
|
|
}
|
|
|
|
on(event, listener) {
|
|
console.log("Adding listener for event:", event);
|
|
if (!this.events.has(event)) {
|
|
this.events.set(event, []);
|
|
}
|
|
|
|
const listeners = this.events.get(event);
|
|
listeners.push(listener);
|
|
|
|
console.debug("Total listeners for", event, ":", listeners.length);
|
|
|
|
if (listeners.length > this.maxListeners) {
|
|
console.warn("Possible memory leak detected for event:", event);
|
|
}
|
|
|
|
return this;
|
|
}
|
|
|
|
emit(event, ...args) {
|
|
console.info("Emitting event:", event);
|
|
console.debug("Event args:", args);
|
|
|
|
const listeners = this.events.get(event);
|
|
if (!listeners || listeners.length === 0) {
|
|
console.debug("No listeners for event:", event);
|
|
return false;
|
|
}
|
|
|
|
console.log("Notifying", listeners.length, "listeners");
|
|
|
|
listeners.forEach(listener => {
|
|
try {
|
|
listener(...args);
|
|
} catch (error) {
|
|
console.error("Error in event listener:", error.message);
|
|
}
|
|
});
|
|
|
|
return true;
|
|
}
|
|
|
|
off(event, listener) {
|
|
console.log("Removing listener for event:", event);
|
|
const listeners = this.events.get(event);
|
|
|
|
if (listeners) {
|
|
const index = listeners.indexOf(listener);
|
|
if (index > -1) {
|
|
listeners.splice(index, 1);
|
|
console.debug("Listener removed, remaining:", listeners.length);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
console.debug("Listener not found for event:", event);
|
|
return false;
|
|
}
|
|
|
|
once(event, listener) {
|
|
console.log("Adding once listener for event:", event);
|
|
|
|
const onceWrapper = (...args) => {
|
|
console.debug("Once wrapper called for event:", event);
|
|
this.off(event, onceWrapper);
|
|
listener(...args);
|
|
};
|
|
|
|
return this.on(event, onceWrapper);
|
|
}
|
|
|
|
removeAllListeners(event) {
|
|
if (event) {
|
|
console.log("Removing all listeners for event:", event);
|
|
this.events.delete(event);
|
|
} else {
|
|
console.log("Removing all listeners for all events");
|
|
this.events.clear();
|
|
}
|
|
}
|
|
|
|
listenerCount(event) {
|
|
const listeners = this.events.get(event);
|
|
const count = listeners ? listeners.length : 0;
|
|
console.debug("Listener count for", event, ":", count);
|
|
return count;
|
|
}
|
|
}
|
|
|
|
console.warn("Event emitter module loaded");
|
|
`
|
|
},
|
|
{
|
|
name: 'logger.js',
|
|
content: `
|
|
// Logger utility
|
|
class Logger {
|
|
constructor(level = 'info') {
|
|
console.log("Creating logger with level:", level);
|
|
this.level = level;
|
|
this.levels = {
|
|
debug: 0,
|
|
info: 1,
|
|
warn: 2,
|
|
error: 3
|
|
};
|
|
}
|
|
|
|
debug(message, ...args) {
|
|
console.debug("DEBUG:", message, ...args);
|
|
this.log('debug', message, args);
|
|
}
|
|
|
|
info(message, ...args) {
|
|
console.info("INFO:", message, ...args);
|
|
this.log('info', message, args);
|
|
}
|
|
|
|
warn(message, ...args) {
|
|
console.warn("WARN:", message, ...args);
|
|
this.log('warn', message, args);
|
|
}
|
|
|
|
error(message, ...args) {
|
|
console.error("ERROR:", message, ...args);
|
|
this.log('error', message, args);
|
|
}
|
|
|
|
log(level, message, args) {
|
|
const shouldLog = this.levels[level] >= this.levels[this.level];
|
|
|
|
if (shouldLog) {
|
|
const timestamp = new Date().toISOString();
|
|
const logMessage = [timestamp, level.toUpperCase(), message, ...args].join(' ');
|
|
|
|
console.log("📝", logMessage);
|
|
|
|
// 这里可以添加文件日志等
|
|
this.writeToStorage(level, logMessage);
|
|
}
|
|
}
|
|
|
|
writeToStorage(level, message) {
|
|
// 模拟写入存储
|
|
console.debug("Writing to storage:", level, message);
|
|
}
|
|
|
|
setLevel(level) {
|
|
console.log("Setting log level to:", level);
|
|
this.level = level;
|
|
}
|
|
|
|
createChild(prefix) {
|
|
console.log("Creating child logger with prefix:", prefix);
|
|
return new Logger(this.level);
|
|
}
|
|
}
|
|
|
|
console.warn("Logger module loaded");
|
|
`
|
|
}
|
|
];
|
|
|
|
// 写入演示文件
|
|
demoFiles.forEach(file => {
|
|
const filePath = path.join(demoDir, file.name);
|
|
fs.writeFileSync(filePath, file.content);
|
|
console.log(`✅ Created: ${file.name}`);
|
|
});
|
|
|
|
console.log(`📁 Demo files created in: ${demoDir}`);
|
|
return demoDir;
|
|
}
|
|
|
|
async function countJsFiles(dir) {
|
|
let count = 0;
|
|
|
|
function traverse(currentDir) {
|
|
const items = fs.readdirSync(currentDir);
|
|
|
|
for (const item of items) {
|
|
const itemPath = path.join(currentDir, item);
|
|
const stats = fs.statSync(itemPath);
|
|
|
|
if (stats.isDirectory()) {
|
|
traverse(itemPath);
|
|
} else if (item.endsWith('.js')) {
|
|
count++;
|
|
}
|
|
}
|
|
}
|
|
|
|
traverse(dir);
|
|
return count;
|
|
}
|
|
|
|
async function demoWithCreatedFiles() {
|
|
const demoDir = path.join(__dirname, 'demo-files');
|
|
|
|
console.log('');
|
|
console.log('🚀 Testing parallel processing with demo files...');
|
|
console.log('');
|
|
|
|
try {
|
|
// 使用优化处理器处理演示文件
|
|
const { processDirectory } = require('./optimized-processor.js');
|
|
|
|
const result = await processDirectory(demoDir, {
|
|
useOllama: false,
|
|
parallel: true,
|
|
dryRun: false
|
|
});
|
|
|
|
console.log('');
|
|
console.log('📊 Processing Results:');
|
|
console.log(` Total files: ${result.totalFiles}`);
|
|
console.log(` Successful: ${result.successful}`);
|
|
console.log(` Failed: ${result.failed}`);
|
|
console.log(` Processing time: ${result.processingTime}ms`);
|
|
console.log(` Size reduction: ${result.totalReduction} characters`);
|
|
console.log('');
|
|
|
|
if (result.successful > 0 && result.processingTime > 0) {
|
|
const avgTime = result.processingTime / result.successful;
|
|
console.log(` Average time per file: ${avgTime.toFixed(1)}ms`);
|
|
|
|
if (result.totalFiles >= 4) {
|
|
console.log('✅ Parallel processing was used (≥4 files)');
|
|
} else {
|
|
console.log('⚠️ Single-thread processing was used (<4 files)');
|
|
}
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error('❌ Demo processing failed:', error.message);
|
|
} finally {
|
|
// 清理演示文件
|
|
console.log('');
|
|
console.log('🧹 Cleaning up demo files...');
|
|
if (fs.existsSync(demoDir)) {
|
|
fs.rmSync(demoDir, { recursive: true, force: true });
|
|
}
|
|
console.log('✅ Demo completed!');
|
|
}
|
|
}
|
|
|
|
// 主程序
|
|
async function main() {
|
|
try {
|
|
await demoParallelProcessing();
|
|
} catch (error) {
|
|
console.error('❌ Demo failed:', error.message);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
|
|
if (require.main === module) {
|
|
main().catch(console.error);
|
|
}
|
|
|
|
module.exports = { demoParallelProcessing, createDemoFiles }; |