diff --git a/common/js/voice-recorder.js b/common/js/voice-recorder.js index 66a4629..629de65 100644 --- a/common/js/voice-recorder.js +++ b/common/js/voice-recorder.js @@ -5,11 +5,20 @@ * 统一 H5 / 微信小程序 / App-Plus / 鸿蒙元服务 四端的录音接口, * 把麦克风采集 + 振幅事件 + 自动停止 + 资源回收 收敛到一处。 * + * PR-FRONTEND-VOICE-WAV (2026-09-25) + * H5 分支从 MediaRecorder (产出 webm/opus) 改为 AudioContext + ScriptProcessor + * 采原始 PCM, 停止时手动封装成标准 WAV (PCM 16-bit LE mono, 16kHz). + * 原因: Chrome MediaRecorder 只支持 webm/ogg 容器, 不支持 wav/mp3 录制; + * 大多数 STT / 后端 Agent 框架无法解析 webm/opus. + * 产 wav 后所有 STT 服务 (Whisper / 达摩院 / 腾讯 ASR / ...) 都吃. + * * 平台分支: - * - H5 : navigator.mediaDevices.getUserMedia + MediaRecorder, - * 走 AudioContext + AnalyserNode 采样振幅 + * - H5 : navigator.mediaDevices.getUserMedia → AudioContext + * → ScriptProcessor 采 PCM Float32 → 停止时封装 WAV. + * 振幅采样走 AnalyserNode (与采 PCM 并行, 不冲突). * - MP / APP / 鸿蒙: uni.getRecorderManager()(uni-app 鸿蒙 runtime 已映射) - * 走 onFrameRecorded 推 progress 事件 + * 走 onFrameRecorded 推 progress 事件. + * uni-app format 参数可指定 wav/mp3/aac. * * 统一事件: * progress → { level: 0..1, duration: ms } 持续采样, 用于驱动波形动画 @@ -18,28 +27,25 @@ * 统一结果 (stop() 解析): * { * filePath?: string, // MP/APP/鸿蒙: tempFilePath; H5: blob URL - * blob?: Blob, // H5: 原始 Blob (供 FileReader / uploadMedia 二次处理) - * fileName?: string, // H5: 自动生成的 .webm/.ogg/.mp4 文件名 + * blob?: Blob, // H5: WAV Blob (供 FileReader / uploadMedia 二次处理) + * fileName?: string, // H5: voice_.wav; MP 按 uni 约定 * duration: number, // ms - * mime: string, // 录音产物 MIME - * size: number, // bytes (H5 = blob.size; MP 在 onStop 给出) + * mime: string, // 录音产物 MIME: H5='audio/wav', MP/APP/鸿蒙按 format + * size: number, // bytes * cancelled: false * } * * 取消时 resolve(null);调用方拿到 null 时不要 push 到 pendingFiles。 */ -// H5 录音 MIME 偏好顺序 (从高到低) -const H5_MIME_PREFERENCES = [ - 'audio/webm;codecs=opus', - 'audio/webm', - 'audio/ogg;codecs=opus', - 'audio/ogg', - 'audio/mp4' -] - const DEFAULT_MAX_DURATION_MS = 60 * 1000 +// H5 录音参数: 16kHz mono PCM → 16-bit LE WAV +// 16kHz/16-bit 是大多数 STT 服务 (Whisper / 达摩院 / 腾讯 ASR 等) 最常见的输入规格. +const H5_SAMPLE_RATE = 16000 +const H5_NUM_CHANNELS = 1 +const H5_SCRIPT_PROCESSOR_BUFFER_SIZE = 4096 + /** * 构造带 code 字段的 Error * - code: 稳定的英文枚举, 供调用方按 code 走 $lang() 翻译 @@ -74,33 +80,88 @@ function isMpOrApp() { // #endif } -/** - * 选取浏览器实际支持的 MediaRecorder MIME - * 兜底用空字符串 → MediaRecorder 用浏览器默认编码 - */ -function pickH5Mime() { - if (typeof MediaRecorder === 'undefined') return '' - for (const m of H5_MIME_PREFERENCES) { - try { - if (MediaRecorder.isTypeSupported(m)) return m - } catch (e) { /* noop */ } - } - return '' -} - /** * 由 mime 反推文件扩展名 */ function mimeToExt(mime) { - if (!mime) return 'webm' + if (!mime) return 'wav' + if (mime.indexOf('wav') !== -1) return 'wav' if (mime.indexOf('webm') !== -1) return 'webm' if (mime.indexOf('ogg') !== -1) return 'ogg' if (mime.indexOf('mp4') !== -1) return 'm4a' if (mime.indexOf('mpeg') !== -1) return 'mp3' - if (mime.indexOf('wav') !== -1) return 'wav' + if (mime.indexOf('aac') !== -1) return 'aac' return 'audio' } +/** + * 合并多个 Float32Array (PCM 帧缓冲) 为单个 Float32Array + */ +function mergeFloat32Arrays(arrays) { + let totalLength = 0 + for (let i = 0; i < arrays.length; i++) totalLength += arrays[i].length + const out = new Float32Array(totalLength) + let offset = 0 + for (let i = 0; i < arrays.length; i++) { + out.set(arrays[i], offset) + offset += arrays[i].length + } + return out +} + +/** + * 将 Float32 PCM (-1..1) 编码为标准 WAV (PCM 16-bit LE, RIFF/WAVE 头). + * 所有 STT 服务都吃这个格式; 浏览器 H5 录音不依赖 MediaRecorder 也能产出. + * + * 参考 RIFF/WAVE 规范: + * https://docs.fileformat.com/audio/wav/ + * http://soundfile.sapp.org/doc/WaveFormat/ + */ +function encodeWavPcm16LE(samples, sampleRate, numChannels) { + const bytesPerSample = 2 // 16-bit + const dataSize = samples.length * numChannels * bytesPerSample + const buffer = new ArrayBuffer(44 + dataSize) + const view = new DataView(buffer) + + // RIFF chunk descriptor (12 bytes) + writeAscii(view, 0, 'RIFF') + view.setUint32(4, 36 + dataSize, true) // ChunkSize = 文件总字节 - 8 + writeAscii(view, 8, 'WAVE') + + // fmt subchunk (24 bytes) + writeAscii(view, 12, 'fmt ') + view.setUint32(16, 16, true) // Subchunk1Size for PCM = 16 + view.setUint16(20, 1, true) // AudioFormat: 1 = PCM + view.setUint16(22, numChannels, true) + view.setUint32(24, sampleRate, true) + view.setUint32(28, sampleRate * numChannels * bytesPerSample, true) // ByteRate + view.setUint16(32, numChannels * bytesPerSample, true) // BlockAlign + view.setUint16(34, 16, true) // BitsPerSample + + // data subchunk (8 + N bytes) + writeAscii(view, 36, 'data') + view.setUint32(40, dataSize, true) + + // PCM 样本: Float32 (-1..1) → Int16 LE (clamp 到 [-1, 1] 防溢出) + let offset = 44 + for (let i = 0; i < samples.length; i++) { + let s = samples[i] + if (s > 1) s = 1 + else if (s < -1) s = -1 + const int16 = s < 0 ? Math.round(s * 0x8000) : Math.round(s * 0x7FFF) + view.setInt16(offset, int16, true) + offset += 2 + } + + return buffer +} + +function writeAscii(view, offset, str) { + for (let i = 0; i < str.length; i++) { + view.setUint8(offset + i, str.charCodeAt(i)) + } +} + /** * 创建录音器实例。 * @@ -198,6 +259,18 @@ class VoiceRecorder { } // ===================== H5 实现 ===================== + // PR-FRONTEND-VOICE-WAV (2026-09-25): 改为 AudioContext + ScriptProcessor 采 PCM, + // 停止时手动封装标准 WAV. 不再依赖 MediaRecorder, 解决 Chrome 只能产 webm/opus + // 导致后端 STT / Agent framework 无法解析的问题. + // + // 关键设计: + // 1. PCM 采样率固定 16kHz mono, 浏览器会自动 resample 麦克风输入 + // (实际由 AudioContext sampleRate 参数保证) + // 2. ScriptProcessor 必须 connect 到 audioCtx.destination 才会触发 + // onaudioprocess; 通过 GainNode(gain=0) 静音防止麦克风外放回声 + // 3. 振幅采样走 AnalyserNode (与 PCM 采样并行, 不冲突) + // 4. onaudioprocess 每帧必须拷贝 channelData (new Float32Array(input)), + // 否则下一帧会覆盖导致 PCM 数据丢失 _startH5() { return new Promise((resolve, reject) => { @@ -205,117 +278,91 @@ class VoiceRecorder { reject(_err('H5_NO_MIC_API', 'mic API not available (HTTPS or localhost required)')) return } - const mime = pickH5Mime() - const chunks = [] - let mediaRecorder = null + const Ctx = window.AudioContext || window.webkitAudioContext + if (!Ctx) { + reject(_err('H5_NO_AUDIO_CTX', 'AudioContext not available')) + return + } + + const buffers = [] let stream = null let audioCtx = null + let source = null + let processor = null let analyser = null + let silentGain = null let rafId = null - navigator.mediaDevices.getUserMedia({ audio: true }) + // 清理 H5 资源 (失败 / 异常路径统一调用) + const cleanup = () => { + try { if (rafId) cancelAnimationFrame(rafId) } catch (e) { /* noop */ } + try { if (processor) processor.disconnect() } catch (e) { /* noop */ } + try { if (analyser) analyser.disconnect() } catch (e) { /* noop */ } + try { if (source) source.disconnect() } catch (e) { /* noop */ } + try { if (silentGain) silentGain.disconnect() } catch (e) { /* noop */ } + try { if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {}) } catch (e) { /* noop */ } + try { if (stream) stream.getTracks().forEach(t => t.stop()) } catch (e) { /* noop */ } + } + + navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: H5_NUM_CHANNELS, + // 注意: 部分浏览器会忽略 sampleRate 约束, 实际采样率以 AudioContext 为准 + sampleRate: H5_SAMPLE_RATE, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true + } + }) .then((s) => { stream = s - try { - mediaRecorder = mime - ? new MediaRecorder(stream, { mimeType: mime }) - : new MediaRecorder(stream) - } catch (e) { - try { stream.getTracks().forEach(t => t.stop()) } catch (_) {} - reject(e) - return - } + audioCtx = new Ctx({ sampleRate: H5_SAMPLE_RATE }) - mediaRecorder.ondataavailable = (ev) => { - if (ev.data && ev.data.size > 0) chunks.push(ev.data) + // 麦克风 → ScriptProcessor → 静音 GainNode → destination + // (ScriptProcessor 必须连到 destination 才会触发 onaudioprocess; + // 用 GainNode(gain=0) 静音, 防止麦克风声音外放产生回声) + source = audioCtx.createMediaStreamSource(stream) + const BufferSize = H5_SCRIPT_PROCESSOR_BUFFER_SIZE + processor = audioCtx.createScriptProcessor(BufferSize, H5_NUM_CHANNELS, H5_NUM_CHANNELS) + processor.onaudioprocess = (e) => { + const channelData = e.inputBuffer.getChannelData(0) + // 必须拷贝: inputBuffer 数据会被下一帧覆盖, 不拷贝会导致 PCM 丢失 + buffers.push(new Float32Array(channelData)) } + silentGain = audioCtx.createGain() + silentGain.gain.value = 0 + source.connect(processor) + processor.connect(silentGain) + silentGain.connect(audioCtx.destination) - mediaRecorder.onerror = (ev) => { - const err = (ev && ev.error) || new Error('MediaRecorder error') - this._emit('error', err) - } - - mediaRecorder.onstop = () => { - // 收到 onstop 才算真正的 stop 完成,产物在 _stopH5 的 promise 里组装 + // AnalyserNode 并行采振幅 (驱动波形动画) + analyser = audioCtx.createAnalyser() + analyser.fftSize = 256 + source.connect(analyser) + const freqBuf = new Uint8Array(analyser.frequencyBinCount) + const tick = () => { + if (!this._isRecording) return try { - if (rafId) cancelAnimationFrame(rafId) - if (analyser) analyser.disconnect() - if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {}) - if (stream) stream.getTracks().forEach(t => t.stop()) + analyser.getByteFrequencyData(freqBuf) + let sum = 0 + for (let i = 0; i < freqBuf.length; i++) sum += freqBuf[i] + const level = Math.min(1, sum / (freqBuf.length * 255)) + this._emit('progress', { level, duration: Date.now() - this._startTs }) } catch (e) { /* noop */ } - - this._h5 = null - this._isRecording = false - this._clearAutoStopTimer() - - const blob = new Blob(chunks, { type: mediaRecorder.mimeType || mime || 'audio/webm' }) - const finalMime = blob.type || (mime || 'audio/webm') - const fileName = `voice_${Date.now()}.${mimeToExt(finalMime)}` - // 把产物挂回实例,供 _stopH5 的 promise 读取 - this._h5PendingResult = { - blob, - mime: finalMime, - size: blob.size, - duration: Date.now() - this._startTs, - fileName - } - if (this._h5Resolve) { - const r = this._h5Resolve - this._h5Resolve = null - r(this._h5PendingResult) - } + rafId = requestAnimationFrame(tick) } + tick() - // AudioContext + AnalyserNode 采样振幅 (驱动波形) - try { - const Ctx = window.AudioContext || window.webkitAudioContext - if (Ctx) { - audioCtx = new Ctx() - const source = audioCtx.createMediaStreamSource(stream) - analyser = audioCtx.createAnalyser() - analyser.fftSize = 256 - source.connect(analyser) - const data = new Uint8Array(analyser.frequencyBinCount) - const tick = () => { - if (!this._isRecording) return - analyser.getByteFrequencyData(data) - let sum = 0 - for (let i = 0; i < data.length; i++) sum += data[i] - const level = Math.min(1, sum / (data.length * 255)) - this._emit('progress', { level, duration: Date.now() - this._startTs }) - rafId = requestAnimationFrame(tick) - } - tick() - } - } catch (e) { - // AudioContext 创建失败不影响录音本身 - console.warn('[voice-recorder] AudioContext 初始化失败:', e) - } - - this._h5 = { mediaRecorder, stream, audioCtx, analyser, rafId, mime } + this._h5 = { stream, audioCtx, source, processor, analyser, silentGain, rafId, buffers, sampleRate: audioCtx.sampleRate, cleanup } this._isRecording = true this._startTs = Date.now() this._setAutoStopTimer() - try { - mediaRecorder.start(250) // 每 250ms 触发一次 ondataavailable - } catch (e) { - // 启动失败, 主动清理资源并回滚状态 - this._isRecording = false - this._clearAutoStopTimer() - try { - if (rafId) cancelAnimationFrame(rafId) - if (analyser) analyser.disconnect() - if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {}) - stream.getTracks().forEach(t => t.stop()) - } catch (e2) { /* noop */ } - this._h5 = null - reject(e) - return - } resolve() }) .catch((err) => { - reject(err) + // 麦克风权限拒绝 / 设备不可用 + cleanup() + reject(_err('H5_MIC_PERMISSION', (err && err.message) || 'mic permission denied or device unavailable')) }) }) } @@ -323,51 +370,60 @@ class VoiceRecorder { _stopH5() { return new Promise((resolve) => { const h5 = this._h5 - if (!h5 || !h5.mediaRecorder) { + if (!h5) { resolve(null); return } + + // 1) 拆 audio graph + 停流 (避免 onprocess 继续累积 buffer) + try { h5.cleanup() } catch (e) { /* noop */ } + + // 2) 合并 PCM 帧缓冲 + const sampleRate = h5.sampleRate || H5_SAMPLE_RATE + const merged = mergeFloat32Arrays(h5.buffers || []) + + // 3) 编码标准 WAV (PCM 16-bit LE mono) + const wavBuffer = encodeWavPcm16LE(merged, sampleRate, H5_NUM_CHANNELS) + const blob = new Blob([wavBuffer], { type: 'audio/wav' }) + + this._h5 = null + this._isRecording = false + this._clearAutoStopTimer() + + if (merged.length === 0) { + // 没有采到任何 PCM (例如用户没说话或立即 stop) resolve(null) return } - // onstop 回调里会 set _h5PendingResult 并调用 _h5Resolve - this._h5Resolve = resolve - try { h5.mediaRecorder.stop() } catch (e) { resolve(null) } - }).then((result) => { - if (!result) return null + + const fileName = `voice_${Date.now()}.wav` const blobUrl = (typeof URL !== 'undefined' && URL.createObjectURL) - ? URL.createObjectURL(result.blob) + ? URL.createObjectURL(blob) : '' - return { + + resolve({ filePath: blobUrl, - blob: result.blob, - fileName: result.fileName, - duration: result.duration, - mime: result.mime, - size: result.size, + blob, + fileName, + duration: Date.now() - this._startTs, + mime: 'audio/wav', + size: blob.size, cancelled: false - } + }) }) } _cancelH5() { return new Promise((resolve) => { const h5 = this._h5 - if (!h5 || !h5.mediaRecorder) { + if (!h5) { this._isRecording = false this._clearAutoStopTimer() resolve() return } - this._h5Resolve = (result) => { - // 取消时丢掉 blob URL, 但产物本身已不再使用 - if (result && result.blob) { - // Blob 没有显式 release 接口, 浏览器 GC 即可 - } - resolve() - } - try { h5.mediaRecorder.stop() } catch (e) { - this._isRecording = false - this._clearAutoStopTimer() - resolve() - } + try { h5.cleanup() } catch (e) { /* noop */ } + this._h5 = null + this._isRecording = false + this._clearAutoStopTimer() + resolve() }) } diff --git a/lang/en-us/ai-chat/index.js b/lang/en-us/ai-chat/index.js index b9f06b9..3623d8a 100644 --- a/lang/en-us/ai-chat/index.js +++ b/lang/en-us/ai-chat/index.js @@ -22,8 +22,19 @@ export const lang = { // ====== ai-chat-message ====== aiNickname: 'AI Assistant', defaultNickname: 'Me', + // PR-FRONTEND-VOICE-INPUT (2026-09-24): query text for attachment-only messages + // audioMessage / videoMessage: short label for the user-bubble filename fallback (still used by bubble template) + // *Instruction* keys: explicit directive sent to the AI as the query. + // Just a label gets ignored — the AI needs an explicit "please listen/look" + // to actually invoke STT/vision/file parsing on the attachment payload. audioMessage: 'Voice Message', videoMessage: 'Video Message', + audioInstruction: 'I sent a voice message. Please listen to it and reply.', + audioInstructionWithDuration: 'I sent a voice message ({seconds}s). Please listen to it and reply.', + imageInstruction: 'I sent an image. Please look at it and reply.', + videoInstruction: 'I sent a video. Please watch it and reply.', + fileInstruction: 'I sent a file. Please review it and reply.', + attachmentInstruction: 'I sent attachments. Please review them and reply.', inputPlaceholder: 'Please enter your question...', expandEditorTitle: 'Edit Content', loadMoreHistory: 'Load more history', diff --git a/lang/zh-cn/ai-chat/index.js b/lang/zh-cn/ai-chat/index.js index e8a0cdc..d4dee67 100644 --- a/lang/zh-cn/ai-chat/index.js +++ b/lang/zh-cn/ai-chat/index.js @@ -22,8 +22,18 @@ export const lang = { // ====== ai-chat-message ====== aiNickname: '智能助手', defaultNickname: '我', + // PR-FRONTEND-VOICE-INPUT (2026-09-24): 纯附件时发给后端的 query 文案 + // audioMessage / videoMessage: 用户气泡里文件名的兜底 (短标签, 仍在 bubble 模板用) + // *Instruction* 系列: 纯附件时发给 AI 的 query, 显式要求 AI "去看/听" 附件内容 + // AI 必须看到指令才会主动调 STT / 视觉 / 文件解析, 光有 label 容易被忽略 audioMessage: '语音消息', videoMessage: '视频消息', + audioInstruction: '我发送了一段语音，请听一下并回复我', + audioInstructionWithDuration: '我发送了一段语音（{seconds} 秒），请听一下并回复我', + imageInstruction: '我发送了一张图片，请查看图片内容并回复我', + videoInstruction: '我发送了一段视频，请查看视频内容并回复我', + fileInstruction: '我发送了一个文件，请查看文件内容并回复我', + attachmentInstruction: '我发送了附件，请查看附件内容并回复我', inputPlaceholder: '请输入您的问题...', expandEditorTitle: '编辑内容', loadMoreHistory: '加载更多历史消息', diff --git a/pages_tool/ai-chat/ai-chat-message.vue b/pages_tool/ai-chat/ai-chat-message.vue index 9f8a504..bb0e8eb 100644 --- a/pages_tool/ai-chat/ai-chat-message.vue +++ b/pages_tool/ai-chat/ai-chat-message.vue @@ -1519,6 +1519,7 @@ async loadChatHistoryIfExist() { // backend MediaGuard 旁路 (≤5MB 时直接送 data+encoding, 不依赖 URL 公网可达)。 // 这与图片的 enrichedPendingAttachments 行为对齐, 后端 WbAgentOsClient::buildMediaBlocks // 已经按 docs §3.1.5 处理 audio + data 内联。 + // 顺带把录音时长 (ms) 一起塞给后端, 方便 query 文案生成 ("语音消息 (12 秒)") const fileAttachments = await Promise.all( pendingFilesList .filter(p => p.uploaded) @@ -1530,7 +1531,9 @@ async loadChatHistoryIfExist() { url: p.uploaded.url, mime: p.uploaded.mime, name: p.uploaded.name, - size: p.uploaded.size + size: p.uploaded.size, + // PR-FRONTEND-VOICE-INPUT: 录音时长 (ms) 仅对 audio 有意义; 文件/视频缺省 0 + duration: p.duration || 0 } // audio 走 inline-base64 路径 (≤5MB) → 避免 LAN URL 被 MediaGuard SSRF 闸门拦截 if (base.type === 'audio' && p.tempPath) { @@ -1577,12 +1580,24 @@ async loadChatHistoryIfExist() { // 实际发送给后端的内容 = 图片(含 base64) + 文件/视频/音频 const readyAttachments = [...fileAttachments, ...enrichedPendingAttachments] + // PR-FRONTEND-VOICE-INPUT (2026-09-25): 不再给 attachment 挂自定义 instruction 字段. + // 实测反馈: 后端 Agent framework 不识别 attachment.instruction 字段, + // 会把它当成"事件标识"误读, 反而干扰 STT/视觉链路触发. + // 改为纯 queryText 携带指令 (见下方 _buildAttachmentQuery + merge 逻辑). + // 将来后端若支持 instruction 字段, 再单独 PR 加回. + // 记录用户发出消息的时间戳, AI 回复完成时算 elapsed this._aiRequestStart = Date.now() - const queryText = hasContent - ? this.inputText.trim() - : (readyAttachments.length ? '[附件]' : '') + // PR-FRONTEND-VOICE-INPUT (2026-09-25): content 只发用户文本, 不再拼接指令 hint + // 历史: 之前给 query 拼 "请听一下 / 请查看图片..." 指令, 实测 AI 会把这条 hint + // 当成"事件标识"误读 (raw i18n key 漏出 / 后端透传逻辑不一致), + // 还不如信任后端 Agent framework 自己按 type 分发处理附件 (语音→STT, 图片→视觉). + // 现在: text + 语音 时 content 只有 text, 音频附件独立传给后端, AI 自行判断. + // 纯附件场景 (无用户文本) 保留 _buildAttachmentQuery 的回退逻辑作为类型标识兜底, + // 后续如果后端 Agent 完全不识别某类附件, 再单独 PR 加回 hint. + const userText = hasContent ? this.inputText.trim() : '' + const queryText = userText || this._buildAttachmentQuery(readyAttachments) // 带图片时消息类型为 image, 图片 + 配文一起显示; 纯文件/文字时仍为 text const firstPendingImage = pendingImagesList.length ? pendingImagesList[0] : null // 用户气泡内展示用的附件: 图片(供 messageImages 渲染多图) + 文件/视频/音频 @@ -1623,7 +1638,9 @@ async loadChatHistoryIfExist() { id: ++this.messageId, role: 'user', type: 'text', - content: queryText, + // PR-FRONTEND-VOICE-INPUT (2026-09-24): 用户气泡只显示用户自己输入的文字 + // 给 AI 的指令 hint 在 queryText 里, 这里不要污染用户气泡 + content: userText, // 纯文件(无图)时把文件挂到 attachments, 用户消息气泡下方展示文件卡片 attachments: displayAttachments, timestamp: Date.now() @@ -1658,7 +1675,9 @@ async loadChatHistoryIfExist() { // 直接发送消息（跳过健康检查，避免健康检查接口格式不匹配导致误判） // 如果服务不可用，实际发送时会抛出异常，由 catch 块处理 // 纯图片无文字时, 给 AI 的 query 用占位文案, 避免空 query - const enhancedMessage = this.enhanceEmojiMessage(userMessage.content || (readyAttachments.length ? '[图片]' : '')) + // PR-FRONTEND-VOICE-INPUT (2026-09-25): content 只发用户文本 (有文本时), + // 纯附件时 fallback 到 _buildAttachmentQuery 的类型标识, 不再 append 指令 hint. + const enhancedMessage = this.enhanceEmojiMessage(queryText) if (this.enableStreaming) { await this.sendStreamMessage(enhancedMessage, readyAttachments) } else { @@ -3355,6 +3374,46 @@ _handleStreamError(msg) { this._discardVoiceRecorder() }, + // PR-FRONTEND-VOICE-INPUT (2026-09-24): 按附件类型生成给后端 AI 的 query 文案 + // 纯图片: "我发送了一张图片，请查看图片内容并回复我" + // 纯语音: "我发送了一段语音，请听一下并回复我" / "…（{n} 秒）…" + // 纯视频: "我发送了一段视频，请查看视频内容并回复我" + // 纯文件: "我发送了一个文件，请查看文件内容并回复我" + // 多类型混合: "我发送了附件，请查看附件内容并回复我" + // 空: '' + // 必须是带动作的指令 (请听 / 请查看), 不是单纯的类型标签. + // AI 只看到 "语音消息" 不会主动调 STT, 必须有 "请听一下" 才会触发. + // + // 返回合并后的单一字符串, 作为 query text 的 hint 兜底 (老 Agent 不识 + // attachment.instruction 字段时仍能据此判断如何处理附件). + _buildAttachmentQuery(attachments) { + if (!attachments || !attachments.length) return '' + // 取首个非空 type; 多类型时降级到 attachmentInstruction + const seen = new Set() + let firstAudio = null + for (const a of attachments) { + const kind = a.type || '' + if (!seen.has(kind)) seen.add(kind) + if (kind === 'audio' && !firstAudio) firstAudio = a + } + if (seen.size === 1) { + const onlyKind = attachments[0].type + if (onlyKind === 'image') return this.$lang('ai-chat.index.imageInstruction') + if (onlyKind === 'video') return this.$lang('ai-chat.index.videoInstruction') + if (onlyKind === 'file') return this.$lang('ai-chat.index.fileInstruction') + if (onlyKind === 'audio') { + // 音频带时长 (毫秒 → 秒, 至少 1s 起步避免 0s 看起来像出 bug) + const dur = (firstAudio && Number(firstAudio.duration)) || 0 + if (dur >= 1000) { + const seconds = Math.max(1, Math.round(dur / 1000)) + return this.$lang('ai-chat.index.audioInstructionWithDuration', { seconds }) + } + return this.$lang('ai-chat.index.audioInstruction') + } + } + return this.$lang('ai-chat.index.attachmentInstruction') + }, + // ============== PR-FRONTEND-VOICE-INPUT: 录音产物处理 ============== /** @@ -3395,9 +3454,11 @@ _handleStreamError(msg) { // 录完自动退出语音模式, 让发送按钮显出来 (用户可加文字或直接发) if (this.voiceMode) this.exitVoiceMode() try { - // H5 优先传 blob (uploadMedia 内部会处理 blob URL → fetch 上传), - // 非 H5 传 filePath (uni.uploadFile) - const inputArg = result.blob || result.filePath + // uploadMedia 要求 inputArg 是 URL/path 字符串: + // H5 → blob URL (URL.createObjectURL), 内部 fetch → Blob → File + // MP / APP / 鸿蒙 → tempFilePath (uni.uploadFile 直传) + // 不能传 raw Blob — uploadMedia 内部 pathToFile 会调 filePath.split(), Blob 没该方法 + const inputArg = result.filePath const uploaded = await uploadMedia(inputArg, 'audio', { name: pending.fileName, mime: result.mime, @@ -3409,6 +3470,10 @@ _handleStreamError(msg) { this.$set(pending, 'imageUrl', uploaded.url) this.$set(pending, 'url', uploaded.url) } + // PR-FRONTEND-VOICE-INPUT: 不主动 revoke blob URL — sendMessage 时 + // 还要 fileToBase64(tempPath) 再 fetch 一次 (走 base64 内联避开 MediaGuard), + // revoke 后 fetch 失效, 退回 URL-only 路径, LAN 环境会被后端 SSRF 闸门丢。 + // blob URL 生命周期跟随 pendingFiles, 用户「删除附件 / 发送后」自然随 JS GC 释放。 } catch (e) { this.$set(pending, 'uploading', false) this.$set(pending, 'uploadError', e.message || this.$lang('ai-chat.index.uploadFailed'))