chore: 增加scripts目录
This commit is contained in:
818
scripts/ollama/optimized-processor-fixed.js
Normal file
818
scripts/ollama/optimized-processor-fixed.js
Normal file
@@ -0,0 +1,818 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');
|
||||
|
||||
// Ollama配置
|
||||
const OLLAMA_CONFIG = {
|
||||
baseUrl: 'http://localhost:11434',
|
||||
model: 'deepseek-coder:6.7b',
|
||||
timeout: 30000
|
||||
};
|
||||
|
||||
// 缓存目录
|
||||
const CACHE_DIR = path.join(__dirname, '.cache');
|
||||
const METADATA_FILE = path.join(CACHE_DIR, 'metadata.json');
|
||||
|
||||
// 并发控制
|
||||
const MAX_CONCURRENT_WORKERS = 4;
|
||||
const WORKER_SCRIPT = path.join(__dirname, 'worker.js');
|
||||
|
||||
/**
|
||||
* Worker线程代码（内联）
|
||||
*/
|
||||
const workerCode = `
|
||||
const { parentPort, workerData } = require('worker_threads');
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { inputPath, outputPath, useOllama, OLLAMA_CONFIG } = workerData;
|
||||
|
||||
// 本地备用方法
|
||||
function removeConsoleLocally(jsCode) {
|
||||
return jsCode
|
||||
.replace(/console\.(log|error|warn|info|debug)\s*\([^)]*\);?\s*/g, '')
|
||||
.replace(/\/\*[\s\S]*?\*\//g, '')
|
||||
.replace(/\/\/.*$/gm, '');
|
||||
}
|
||||
|
||||
// 简单的fetch polyfill
|
||||
async function simpleFetch(url, options) {
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
const urlObj = new URL(url);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const lib = urlObj.protocol === 'https:' ? https : http;
|
||||
const req = lib.request(url, options, (res) => {
|
||||
let data = '';
|
||||
res.on('data', chunk => data += chunk);
|
||||
res.on('end', () => {
|
||||
resolve({
|
||||
ok: res.statusCode >= 200 && res.statusCode < 300,
|
||||
json: async () => JSON.parse(data),
|
||||
text: async () => data,
|
||||
status: res.statusCode,
|
||||
statusText: res.statusMessage
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', reject);
|
||||
if (options.body) {
|
||||
req.write(options.body);
|
||||
}
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function removeConsoleWithOllama(jsCode) {
|
||||
const prompt = \`Please remove all console statements from this JavaScript code while preserving the code structure and functionality. Only remove console.log, console.error, console.warn, console.info, console.debug, console.assert, console.trace, console.table, console.group, console.groupEnd, console.time, console.timeEnd calls. Return only the cleaned code without any explanations.
|
||||
|
||||
JavaScript code:
|
||||
\`\`\`javascript
|
||||
\${jsCode}
|
||||
\`\`\`
|
||||
|
||||
Cleaned code:\`;
|
||||
|
||||
try {
|
||||
const response = await simpleFetch(\`\${OLLAMA_CONFIG.baseUrl}/api/generate\`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: OLLAMA_CONFIG.model,
|
||||
prompt: prompt,
|
||||
stream: false,
|
||||
options: {
|
||||
temperature: 0.1,
|
||||
top_p: 0.9,
|
||||
max_tokens: 8000
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(\`Ollama API error: \${response.status} \${response.statusText}\`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.response) {
|
||||
throw new Error('No response from Ollama');
|
||||
}
|
||||
|
||||
let cleanedCode = data.response.trim();
|
||||
|
||||
if (cleanedCode.startsWith('\`\`\`')) {
|
||||
const lines = cleanedCode.split('\\n');
|
||||
if (lines[0].includes('javascript')) {
|
||||
lines.shift();
|
||||
} else {
|
||||
lines.shift();
|
||||
}
|
||||
if (lines[lines.length - 1] === '\`\`\`') {
|
||||
lines.pop();
|
||||
}
|
||||
cleanedCode = lines.join('\\n');
|
||||
}
|
||||
|
||||
return cleanedCode;
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 检查明显的JavaScript语法错误
|
||||
function hasSyntaxErrors(code) {
|
||||
const lines = code.split('\\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
|
||||
// 检查常见的语法错误模式
|
||||
if (line.includes('p[s][0]') && !line.includes('p[s][0];')) {
|
||||
console.log(\`Syntax error detected at line \${i + 1}: missing semicolon after p[s][0]\`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查未闭合的括号（简单检查）
|
||||
const openBrackets = (line.match(/\\[/g) || []).length;
|
||||
const closeBrackets = (line.match(/\\]/g) || []).length;
|
||||
if (Math.abs(openBrackets - closeBrackets) > 2) {
|
||||
console.log(\`Bracket mismatch detected at line \${i + 1}\`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查其他常见错误模式
|
||||
if (line.includes('void 0)') && line.includes('if(')) {
|
||||
console.log(\`Conditional syntax error detected at line \${i + 1}\`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function processFile() {
|
||||
try {
|
||||
const originalCode = fs.readFileSync(inputPath, 'utf8');
|
||||
|
||||
// 检查是否有明显的语法错误
|
||||
if (hasSyntaxErrors(originalCode)) {
|
||||
return {
|
||||
success: false,
|
||||
error: 'JavaScript syntax error detected - skipping file to avoid Worker crashes',
|
||||
syntaxError: true
|
||||
};
|
||||
}
|
||||
|
||||
if (useOllama) {
|
||||
// 尝试使用Ollama
|
||||
try {
|
||||
const cleanedCode = await removeConsoleWithOllama(originalCode);
|
||||
return { success: true, cleanedCode, useOllama: true };
|
||||
} catch (ollamaError) {
|
||||
// Ollama失败，使用本地方法
|
||||
console.log('Ollama failed, using local method:', ollamaError.message);
|
||||
const cleanedCode = removeConsoleLocally(originalCode);
|
||||
return { success: true, cleanedCode, useOllama: false, fallback: true };
|
||||
}
|
||||
} else {
|
||||
// 直接使用本地方法
|
||||
const cleanedCode = removeConsoleLocally(originalCode);
|
||||
return { success: true, cleanedCode, useOllama: false };
|
||||
}
|
||||
} catch (error) {
|
||||
return { success: false, error: error.message };
|
||||
}
|
||||
}
|
||||
|
||||
// Worker消息处理
|
||||
processFile().then(result => {
|
||||
parentPort.postMessage(result);
|
||||
}).catch(error => {
|
||||
parentPort.postMessage({ success: false, error: error.message });
|
||||
});
|
||||
`;
|
||||
|
||||
/**
|
||||
* 初始化缓存目录
|
||||
*/
|
||||
function initCache() {
|
||||
if (!fs.existsSync(CACHE_DIR)) {
|
||||
fs.mkdirSync(CACHE_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
if (!fs.existsSync(METADATA_FILE)) {
|
||||
fs.writeFileSync(METADATA_FILE, JSON.stringify({}));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 计算文件哈希值
|
||||
*/
|
||||
function calculateFileHash(filePath) {
|
||||
const fileContent = fs.readFileSync(filePath, 'utf8');
|
||||
return crypto.createHash('md5').update(fileContent).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件元数据
|
||||
*/
|
||||
function getMetadata() {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(METADATA_FILE, 'utf8'));
|
||||
} catch (error) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存文件元数据
|
||||
*/
|
||||
function saveMetadata(metadata) {
|
||||
fs.writeFileSync(METADATA_FILE, JSON.stringify(metadata, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查文件是否需要处理（增量处理）
|
||||
*/
|
||||
function needsProcessing(inputPath, outputPath, metadata) {
|
||||
// 如果输出文件不存在，需要处理
|
||||
if (!fs.existsSync(outputPath)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 获取当前文件哈希
|
||||
const currentHash = calculateFileHash(inputPath);
|
||||
const fileInfo = metadata[inputPath];
|
||||
|
||||
// 如果没有记录，需要处理
|
||||
if (!fileInfo) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 如果哈希值不同，需要处理
|
||||
if (fileInfo.hash !== currentHash) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 从缓存获取处理结果
|
||||
*/
|
||||
function getCachedResult(fileHash) {
|
||||
const cacheFile = path.join(CACHE_DIR, `${fileHash}.cache`);
|
||||
if (fs.existsSync(cacheFile)) {
|
||||
try {
|
||||
return fs.readFileSync(cacheFile, 'utf8');
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 保存处理结果到缓存
|
||||
*/
|
||||
function saveCachedResult(fileHash, result) {
|
||||
const cacheFile = path.join(CACHE_DIR, `${fileHash}.cache`);
|
||||
try {
|
||||
fs.writeFileSync(cacheFile, result, 'utf8');
|
||||
} catch (error) {
|
||||
console.warn('Failed to save cache:', error.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 本地备用方法
|
||||
*/
|
||||
function removeConsoleLocally(jsCode) {
|
||||
return jsCode
|
||||
.replace(/console\.(log|error|warn|info|debug|assert|trace|table|group|groupEnd|time|timeEnd)\s*\([^;]*?\);?\s*/g, '')
|
||||
.replace(/\/\*[\s\S]*?\*\//g, '')
|
||||
.replace(/\/\/.*$/gm, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查Ollama服务
|
||||
*/
|
||||
async function checkOllamaHealth() {
|
||||
try {
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
|
||||
return new Promise((resolve) => {
|
||||
const urlObj = new URL(`${OLLAMA_CONFIG.baseUrl}/api/tags`);
|
||||
const lib = urlObj.protocol === 'https:' ? https : http;
|
||||
|
||||
const req = lib.request(urlObj, { timeout: 5000 }, (res) => {
|
||||
resolve(res.statusCode >= 200 && res.statusCode < 300);
|
||||
});
|
||||
|
||||
req.on('error', () => resolve(false));
|
||||
req.on('timeout', () => {
|
||||
req.destroy();
|
||||
resolve(false);
|
||||
});
|
||||
|
||||
req.end();
|
||||
});
|
||||
} catch (error) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取目录中的所有JS文件
|
||||
*/
|
||||
function collectJsFiles(dir, baseDir = dir) {
|
||||
const files = [];
|
||||
|
||||
function traverse(currentDir) {
|
||||
const items = fs.readdirSync(currentDir);
|
||||
|
||||
for (const item of items) {
|
||||
const itemPath = path.join(currentDir, item);
|
||||
const stats = fs.statSync(itemPath);
|
||||
|
||||
if (stats.isDirectory()) {
|
||||
traverse(itemPath);
|
||||
} else if (path.extname(item) === '.js') {
|
||||
const relativePath = path.relative(baseDir, itemPath);
|
||||
files.push({
|
||||
inputPath: itemPath,
|
||||
relativePath: relativePath,
|
||||
outputPath: path.join(baseDir + '-optimized', relativePath)
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
traverse(dir);
|
||||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* 单线程处理方法（备用）
|
||||
*/
|
||||
async function processFilesSingleThread(files, useOllama, dryRun = false, metadata = {}) {
|
||||
const results = [];
|
||||
|
||||
for (const file of files) {
|
||||
if (!needsProcessing(file.inputPath, file.outputPath, metadata)) {
|
||||
console.log(\`⏭️ Skipping unchanged file: \${file.relativePath}\`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(\`🔧 Processing: \${file.relativePath}\`);
|
||||
|
||||
try {
|
||||
const originalCode = fs.readFileSync(file.inputPath, 'utf8');
|
||||
const fileHash = calculateFileHash(file.inputPath);
|
||||
|
||||
// 检查缓存
|
||||
let cleanedCode;
|
||||
if (useOllama) {
|
||||
const cached = getCachedResult(fileHash);
|
||||
if (cached) {
|
||||
console.log(\`📋 Using cached result for: \${file.relativePath}\`);
|
||||
cleanedCode = cached;
|
||||
} else {
|
||||
console.log(\`🤖 Processing with Ollama: \${file.relativePath}\`);
|
||||
cleanedCode = await removeConsoleWithOllama(originalCode);
|
||||
saveCachedResult(fileHash, cleanedCode);
|
||||
}
|
||||
} else {
|
||||
console.log(\`🔧 Processing locally: \${file.relativePath}\`);
|
||||
cleanedCode = removeConsoleLocally(originalCode);
|
||||
}
|
||||
|
||||
if (!dryRun) {
|
||||
const outputDir = path.dirname(file.outputPath);
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(file.outputPath, cleanedCode, 'utf8');
|
||||
|
||||
// 更新元数据
|
||||
metadata[file.inputPath] = {
|
||||
hash: fileHash,
|
||||
processedAt: new Date().toISOString(),
|
||||
size: {
|
||||
original: originalCode.length,
|
||||
processed: cleanedCode.length
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
results.push({
|
||||
file: file.relativePath,
|
||||
success: true,
|
||||
sizeReduction: originalCode.length - cleanedCode.length
|
||||
});
|
||||
|
||||
console.log(\`✅ Completed: \${file.relativePath}\`);
|
||||
|
||||
} catch (error) {
|
||||
console.error(\`❌ Error processing \${file.relativePath}:\`, error.message);
|
||||
|
||||
if (!dryRun) {
|
||||
// 复制原文件作为备用
|
||||
const outputDir = path.dirname(file.outputPath);
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
fs.copyFileSync(file.inputPath, file.outputPath);
|
||||
}
|
||||
|
||||
results.push({
|
||||
file: file.relativePath,
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* 并行处理文件（多线程）
|
||||
*/
|
||||
async function processFilesParallel(files, useOllama, dryRun = false, metadata = {}) {
|
||||
console.log(\`🚀 Parallel processing \${files.length} files with \${MAX_CONCURRENT_WORKERS} workers...\`);
|
||||
|
||||
// 准备需要处理的文件
|
||||
const filesToProcess = files.filter(file =>
|
||||
needsProcessing(file.inputPath, file.outputPath, metadata)
|
||||
);
|
||||
|
||||
console.log(\`📋 Files to process: \${filesToProcess.length}, Skipping: \${files.length - filesToProcess.length}\`);
|
||||
|
||||
if (filesToProcess.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
// 创建Worker池
|
||||
const workers = [];
|
||||
const results = [];
|
||||
let processedCount = 0;
|
||||
|
||||
// 创建worker脚本文件
|
||||
if (!fs.existsSync(WORKER_SCRIPT)) {
|
||||
fs.writeFileSync(WORKER_SCRIPT, workerCode);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const processNext = () => {
|
||||
if (filesToProcess.length === 0 && workers.length === 0) {
|
||||
console.log(\`🎉 All \${processedCount} files processed!\`);
|
||||
resolve(results);
|
||||
return;
|
||||
}
|
||||
|
||||
if (filesToProcess.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (workers.length >= MAX_CONCURRENT_WORKERS) {
|
||||
return;
|
||||
}
|
||||
|
||||
const file = filesToProcess.shift();
|
||||
console.log(\`🔧 Starting: \${file.relativePath} (\${processedCount + 1}/\${filesToProcess.length + processedCount + 1})\`);
|
||||
|
||||
const worker = new Worker(WORKER_SCRIPT, {
|
||||
workerData: {
|
||||
inputPath: file.inputPath,
|
||||
outputPath: file.outputPath,
|
||||
useOllama: useOllama,
|
||||
OLLAMA_CONFIG: OLLAMA_CONFIG
|
||||
}
|
||||
});
|
||||
|
||||
workers.push(worker);
|
||||
|
||||
worker.on('message', (result) => {
|
||||
if (result.success) {
|
||||
if (!dryRun) {
|
||||
try {
|
||||
const outputDir = path.dirname(file.outputPath);
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(file.outputPath, result.cleanedCode, 'utf8');
|
||||
|
||||
// 更新元数据
|
||||
const originalCode = fs.readFileSync(file.inputPath, 'utf8');
|
||||
const fileHash = calculateFileHash(file.inputPath);
|
||||
metadata[file.inputPath] = {
|
||||
hash: fileHash,
|
||||
processedAt: new Date().toISOString(),
|
||||
size: {
|
||||
original: originalCode.length,
|
||||
processed: result.cleanedCode.length
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(\`❌ Failed to save \${file.relativePath}:\`, error.message);
|
||||
}
|
||||
}
|
||||
|
||||
results.push({
|
||||
file: file.relativePath,
|
||||
success: true,
|
||||
sizeReduction: result.sizeReduction || 0
|
||||
});
|
||||
} else {
|
||||
if (result.syntaxError) {
|
||||
console.log(\`⚠️ Skipped \${file.relativePath} - \${result.error}\`);
|
||||
} else {
|
||||
console.error(\`❌ Failed: \${file.relativePath} - \${result.error}\`);
|
||||
}
|
||||
results.push({
|
||||
file: file.relativePath,
|
||||
success: false,
|
||||
error: result.error,
|
||||
syntaxError: result.syntaxError || false
|
||||
});
|
||||
}
|
||||
|
||||
processedCount++;
|
||||
const index = workers.indexOf(worker);
|
||||
if (index > -1) {
|
||||
workers.splice(index, 1);
|
||||
}
|
||||
|
||||
processNext();
|
||||
});
|
||||
|
||||
worker.on('error', (error) => {
|
||||
console.error(\`❌ Worker error for \${file.relativePath}:\`, error.message);
|
||||
results.push({
|
||||
file: file.relativePath,
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
|
||||
processedCount++;
|
||||
const index = workers.indexOf(worker);
|
||||
if (index > -1) {
|
||||
workers.splice(index, 1);
|
||||
}
|
||||
|
||||
processNext();
|
||||
});
|
||||
|
||||
worker.on('exit', () => {
|
||||
const index = workers.indexOf(worker);
|
||||
if (index > -1) {
|
||||
workers.splice(index, 1);
|
||||
}
|
||||
processNext();
|
||||
});
|
||||
};
|
||||
|
||||
// 启动处理
|
||||
for (let i = 0; i < Math.min(MAX_CONCURRENT_WORKERS, filesToProcess.length); i++) {
|
||||
processNext();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 检查明显的JavaScript语法错误
|
||||
*/
|
||||
function hasSyntaxErrors(code) {
|
||||
const lines = code.split('\\n');
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i].trim();
|
||||
|
||||
// 检查常见的语法错误模式
|
||||
if (line.includes('p[s][0]') && !line.includes('p[s][0];')) {
|
||||
console.log(\`Syntax error detected at line \${i + 1}: missing semicolon after p[s][0]\`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查未闭合的括号（简单检查）
|
||||
const openBrackets = (line.match(/\\[/g) || []).length;
|
||||
const closeBrackets = (line.match(/\\]/g) || []).length;
|
||||
if (Math.abs(openBrackets - closeBrackets) > 2) {
|
||||
console.log(\`Bracket mismatch detected at line \${i + 1}\`);
|
||||
return true;
|
||||
}
|
||||
|
||||
// 检查其他常见错误模式
|
||||
if (line.includes('void 0)') && line.includes('if(')) {
|
||||
console.log(\`Conditional syntax error detected at line \${i + 1}\`);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* 主处理函数
|
||||
*/
|
||||
async function processDirectory(inputDir, options = {}) {
|
||||
const {
|
||||
useOllama = true,
|
||||
parallel = true,
|
||||
dryRun = false,
|
||||
clearCache = false
|
||||
} = options;
|
||||
|
||||
// 初始化
|
||||
initCache();
|
||||
|
||||
if (clearCache) {
|
||||
console.log('🗑️ Clearing cache...');
|
||||
fs.rmSync(CACHE_DIR, { recursive: true, force: true });
|
||||
initCache();
|
||||
}
|
||||
|
||||
const outputDir = inputDir + '-optimized';
|
||||
const metadata = getMetadata();
|
||||
|
||||
// 收集文件
|
||||
console.log('📂 Scanning files...');
|
||||
const files = collectJsFiles(inputDir);
|
||||
const jsFiles = files.filter(f => f.relativePath.endsWith('.js'));
|
||||
console.log(\`📝 Found \${jsFiles.length} JavaScript files\`);
|
||||
|
||||
// 检查Ollama
|
||||
let isOllamaAvailable = false;
|
||||
if (useOllama) {
|
||||
console.log('🔍 Checking Ollama service...');
|
||||
isOllamaAvailable = await checkOllamaHealth();
|
||||
console.log(\` Status: \${isOllamaAvailable ? '✅ Available' : '❌ Not available'}\`);
|
||||
}
|
||||
|
||||
// 处理文件
|
||||
const startTime = Date.now();
|
||||
let results;
|
||||
|
||||
if (parallel && isMainThread && jsFiles.length > 3) {
|
||||
results = await processFilesParallel(jsFiles, isOllamaAvailable, dryRun, metadata);
|
||||
} else {
|
||||
results = await processFilesSingleThread(jsFiles, isOllamaAvailable, dryRun, metadata);
|
||||
}
|
||||
|
||||
// 保存元数据
|
||||
if (!dryRun) {
|
||||
saveMetadata(metadata);
|
||||
}
|
||||
|
||||
// 统计结果
|
||||
const successful = results.filter(r => r.success);
|
||||
const failed = results.filter(r => !r.success);
|
||||
const syntaxErrors = results.filter(r => r.syntaxError);
|
||||
const otherErrors = results.filter(r => !r.success && !r.syntaxError);
|
||||
const totalReduction = successful.reduce((sum, r) => sum + (r.sizeReduction || 0), 0);
|
||||
const processingTime = (Date.now() - startTime) / 1000;
|
||||
|
||||
console.log('');
|
||||
console.log('📊 Processing Summary:');
|
||||
console.log(\`⏱️ Processing time: \${processingTime.toFixed(2)}s\`);
|
||||
console.log(\`✅ Successful: \${successful.length}\`);
|
||||
console.log(\`❌ Failed: \${failed.length}\`);
|
||||
if (syntaxErrors.length > 0) {
|
||||
console.log(\`⚠️ Syntax errors: \${syntaxErrors.length}\`);
|
||||
}
|
||||
if (otherErrors.length > 0) {
|
||||
console.log(\`💥 Other errors: \${otherErrors.length}\`);
|
||||
}
|
||||
console.log(\`💾 Size reduction: \${formatBytes(totalReduction)}\`);
|
||||
console.log(\`📁 Output directory: \${outputDir}\`);
|
||||
|
||||
if (failed.length > 0) {
|
||||
console.log('');
|
||||
if (syntaxErrors.length > 0) {
|
||||
console.log('⚠️ Files with syntax errors (skipped):');
|
||||
syntaxErrors.forEach(f => console.log(\` \${f.file}: \${f.error}\`));
|
||||
}
|
||||
if (otherErrors.length > 0) {
|
||||
console.log('❌ Other failed files:');
|
||||
otherErrors.forEach(f => console.log(\` \${f.file}: \${f.error}\`));
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
totalFiles: jsFiles.length,
|
||||
successful: successful.length,
|
||||
failed: failed.length,
|
||||
totalReduction,
|
||||
processingTime,
|
||||
outputDir
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化字节数
|
||||
*/
|
||||
function formatBytes(bytes) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB', 'GB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* 显示帮助信息
|
||||
*/
|
||||
function showHelp() {
|
||||
console.log(\`
|
||||
Usage: node optimized-processor.js [options] <input-directory>
|
||||
|
||||
Arguments:
|
||||
input-directory Directory containing JavaScript files to process
|
||||
|
||||
Options:
|
||||
--local, -l Use local fallback method instead of Ollama
|
||||
--single, -s Use single-threaded processing (no parallel)
|
||||
--dry-run, -d Preview mode - don't save files
|
||||
--clear-cache, -c Clear existing cache before processing
|
||||
--help, -h Show this help message
|
||||
|
||||
Features:
|
||||
🎯 Parallel processing with worker threads
|
||||
📋 Intelligent caching to avoid reprocessing
|
||||
⏭️ Incremental processing (only changed files)
|
||||
📊 Detailed performance statistics
|
||||
\`);
|
||||
}
|
||||
|
||||
// 主程序
|
||||
async function main() {
|
||||
if (!isMainThread) {
|
||||
return;
|
||||
}
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
let inputDir = '';
|
||||
const options = {};
|
||||
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
const arg = args[i];
|
||||
|
||||
if (arg === '--help' || arg === '-h') {
|
||||
showHelp();
|
||||
return;
|
||||
} else if (arg === '--local' || arg === '-l') {
|
||||
options.useOllama = false;
|
||||
} else if (arg === '--single' || arg === '-s') {
|
||||
options.parallel = false;
|
||||
} else if (arg === '--dry-run' || arg === '-d') {
|
||||
options.dryRun = true;
|
||||
} else if (arg === '--clear-cache' || arg === '-c') {
|
||||
options.clearCache = true;
|
||||
} else if (!inputDir) {
|
||||
inputDir = arg;
|
||||
}
|
||||
}
|
||||
|
||||
if (!inputDir) {
|
||||
console.error('❌ Input directory is required');
|
||||
showHelp();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(inputDir)) {
|
||||
console.error(\`❌ Input directory "\${inputDir}" does not exist\`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('🚀 Optimized JavaScript Console Remover');
|
||||
console.log('');
|
||||
|
||||
try {
|
||||
await processDirectory(inputDir, options);
|
||||
} catch (error) {
|
||||
console.error('❌ Processing failed:', error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module && isMainThread) {
|
||||
main().catch(console.error);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
processDirectory,
|
||||
checkOllamaHealth,
|
||||
collectJsFiles,
|
||||
hasSyntaxErrors
|
||||
};
|
||||
Reference in New Issue
Block a user