feat(voice-input): 改进音频消息处理，支持音频时长和指令文本，优化后端交互

2026-09-28 09:33:57 +08:00
parent fff10a2e19
commit 306f2f7ff7
4 changed files with 301 additions and 159 deletions
--- a/common/js/voice-recorder.js
+++ b/common/js/voice-recorder.js
@@ -5,11 +5,20 @@
 *   统一 H5 / 微信小程序 / App-Plus / 鸿蒙元服务 四端的录音接口,
 *   把麦克风采集 + 振幅事件 + 自动停止 + 资源回收 收敛到一处。
 *
+ * PR-FRONTEND-VOICE-WAV (2026-09-25)
+ *   H5 分支从 MediaRecorder (产出 webm/opus) 改为 AudioContext + ScriptProcessor
+ *   采原始 PCM, 停止时手动封装成标准 WAV (PCM 16-bit LE mono, 16kHz).
+ *   原因: Chrome MediaRecorder 只支持 webm/ogg 容器, 不支持 wav/mp3 录制;
+ *         大多数 STT / 后端 Agent 框架无法解析 webm/opus.
+ *         产 wav 后所有 STT 服务 (Whisper / 达摩院 / 腾讯 ASR / ...) 都吃.
+ *
 * 平台分支:
- *   - H5           : navigator.mediaDevices.getUserMedia + MediaRecorder,
- *                    走 AudioContext + AnalyserNode 采样振幅
+ *   - H5           : navigator.mediaDevices.getUserMedia → AudioContext
+ *                    → ScriptProcessor 采 PCM Float32 → 停止时封装 WAV.
+ *                    振幅采样走 AnalyserNode (与采 PCM 并行, 不冲突).
 *   - MP / APP / 鸿蒙: uni.getRecorderManager()(uni-app 鸿蒙 runtime 已映射)
- *                    走 onFrameRecorded 推 progress 事件
+ *                    走 onFrameRecorded 推 progress 事件.
+ *                    uni-app format 参数可指定 wav/mp3/aac.
 *
 * 统一事件:
 *   progress → { level: 0..1, duration: ms }    持续采样, 用于驱动波形动画
@@ -18,28 +27,25 @@
 * 统一结果 (stop() 解析):
 *   {
 *     filePath?: string,    // MP/APP/鸿蒙: tempFilePath; H5: blob URL
- *     blob?: Blob,          // H5: 原始 Blob (供 FileReader / uploadMedia 二次处理)
- *     fileName?: string,    // H5: 自动生成的 .webm/.ogg/.mp4 文件名
+ *     blob?: Blob,          // H5: WAV Blob (供 FileReader / uploadMedia 二次处理)
+ *     fileName?: string,    // H5: voice_<ts>.wav; MP 按 uni 约定
 *     duration: number,     // ms
- *     mime: string,         // 录音产物 MIME
- *     size: number,         // bytes (H5 = blob.size; MP 在 onStop 给出)
+ *     mime: string,         // 录音产物 MIME: H5='audio/wav', MP/APP/鸿蒙按 format
+ *     size: number,         // bytes
 *     cancelled: false
 *   }
 *
 * 取消时 resolve(null);调用方拿到 null 时不要 push 到 pendingFiles。
 */

-// H5 录音 MIME 偏好顺序 (从高到低)
-const H5_MIME_PREFERENCES = [
-  'audio/webm;codecs=opus',
-  'audio/webm',
-  'audio/ogg;codecs=opus',
-  'audio/ogg',
-  'audio/mp4'
-]
-
 const DEFAULT_MAX_DURATION_MS = 60 * 1000

+// H5 录音参数: 16kHz mono PCM → 16-bit LE WAV
+// 16kHz/16-bit 是大多数 STT 服务 (Whisper / 达摩院 / 腾讯 ASR 等) 最常见的输入规格.
+const H5_SAMPLE_RATE = 16000
+const H5_NUM_CHANNELS = 1
+const H5_SCRIPT_PROCESSOR_BUFFER_SIZE = 4096
+
 /**
 * 构造带 code 字段的 Error
 *   - code: 稳定的英文枚举, 供调用方按 code 走 $lang() 翻译
@@ -74,33 +80,88 @@ function isMpOrApp() {
  // #endif
 }

-/**
- * 选取浏览器实际支持的 MediaRecorder MIME
- *   兜底用空字符串 → MediaRecorder 用浏览器默认编码
- */
-function pickH5Mime() {
-  if (typeof MediaRecorder === 'undefined') return ''
-  for (const m of H5_MIME_PREFERENCES) {
-    try {
-      if (MediaRecorder.isTypeSupported(m)) return m
-    } catch (e) { /* noop */ }
-  }
-  return ''
-}
-
 /**
 * 由 mime 反推文件扩展名
 */
 function mimeToExt(mime) {
-  if (!mime) return 'webm'
+  if (!mime) return 'wav'
+  if (mime.indexOf('wav') !== -1) return 'wav'
  if (mime.indexOf('webm') !== -1) return 'webm'
  if (mime.indexOf('ogg') !== -1) return 'ogg'
  if (mime.indexOf('mp4') !== -1) return 'm4a'
  if (mime.indexOf('mpeg') !== -1) return 'mp3'
-  if (mime.indexOf('wav') !== -1) return 'wav'
+  if (mime.indexOf('aac') !== -1) return 'aac'
  return 'audio'
 }

+/**
+ * 合并多个 Float32Array (PCM 帧缓冲) 为单个 Float32Array
+ */
+function mergeFloat32Arrays(arrays) {
+  let totalLength = 0
+  for (let i = 0; i < arrays.length; i++) totalLength += arrays[i].length
+  const out = new Float32Array(totalLength)
+  let offset = 0
+  for (let i = 0; i < arrays.length; i++) {
+    out.set(arrays[i], offset)
+    offset += arrays[i].length
+  }
+  return out
+}
+
+/**
+ * 将 Float32 PCM (-1..1) 编码为标准 WAV (PCM 16-bit LE, RIFF/WAVE 头).
+ *   所有 STT 服务都吃这个格式; 浏览器 H5 录音不依赖 MediaRecorder 也能产出.
+ *
+ * 参考 RIFF/WAVE 规范:
+ *   https://docs.fileformat.com/audio/wav/
+ *   http://soundfile.sapp.org/doc/WaveFormat/
+ */
+function encodeWavPcm16LE(samples, sampleRate, numChannels) {
+  const bytesPerSample = 2 // 16-bit
+  const dataSize = samples.length * numChannels * bytesPerSample
+  const buffer = new ArrayBuffer(44 + dataSize)
+  const view = new DataView(buffer)
+
+  // RIFF chunk descriptor (12 bytes)
+  writeAscii(view, 0, 'RIFF')
+  view.setUint32(4, 36 + dataSize, true) // ChunkSize = 文件总字节 - 8
+  writeAscii(view, 8, 'WAVE')
+
+  // fmt subchunk (24 bytes)
+  writeAscii(view, 12, 'fmt ')
+  view.setUint32(16, 16, true) // Subchunk1Size for PCM = 16
+  view.setUint16(20, 1, true) // AudioFormat: 1 = PCM
+  view.setUint16(22, numChannels, true)
+  view.setUint32(24, sampleRate, true)
+  view.setUint32(28, sampleRate * numChannels * bytesPerSample, true) // ByteRate
+  view.setUint16(32, numChannels * bytesPerSample, true) // BlockAlign
+  view.setUint16(34, 16, true) // BitsPerSample
+
+  // data subchunk (8 + N bytes)
+  writeAscii(view, 36, 'data')
+  view.setUint32(40, dataSize, true)
+
+  // PCM 样本: Float32 (-1..1) → Int16 LE (clamp 到 [-1, 1] 防溢出)
+  let offset = 44
+  for (let i = 0; i < samples.length; i++) {
+    let s = samples[i]
+    if (s > 1) s = 1
+    else if (s < -1) s = -1
+    const int16 = s < 0 ? Math.round(s * 0x8000) : Math.round(s * 0x7FFF)
+    view.setInt16(offset, int16, true)
+    offset += 2
+  }
+
+  return buffer
+}
+
+function writeAscii(view, offset, str) {
+  for (let i = 0; i < str.length; i++) {
+    view.setUint8(offset + i, str.charCodeAt(i))
+  }
+}
+
 /**
 * 创建录音器实例。
 *
@@ -198,6 +259,18 @@ class VoiceRecorder {
  }

  // ===================== H5 实现 =====================
+  // PR-FRONTEND-VOICE-WAV (2026-09-25): 改为 AudioContext + ScriptProcessor 采 PCM,
+  //   停止时手动封装标准 WAV. 不再依赖 MediaRecorder, 解决 Chrome 只能产 webm/opus
+  //   导致后端 STT / Agent framework 无法解析的问题.
+  //
+  // 关键设计:
+  //   1. PCM 采样率固定 16kHz mono, 浏览器会自动 resample 麦克风输入
+  //      (实际由 AudioContext sampleRate 参数保证)
+  //   2. ScriptProcessor 必须 connect 到 audioCtx.destination 才会触发
+  //      onaudioprocess; 通过 GainNode(gain=0) 静音防止麦克风外放回声
+  //   3. 振幅采样走 AnalyserNode (与 PCM 采样并行, 不冲突)
+  //   4. onaudioprocess 每帧必须拷贝 channelData (new Float32Array(input)),
+  //      否则下一帧会覆盖导致 PCM 数据丢失

  _startH5() {
    return new Promise((resolve, reject) => {
@@ -205,117 +278,91 @@ class VoiceRecorder {
        reject(_err('H5_NO_MIC_API', 'mic API not available (HTTPS or localhost required)'))
        return
      }
-      const mime = pickH5Mime()
-      const chunks = []
-      let mediaRecorder = null
+      const Ctx = window.AudioContext || window.webkitAudioContext
+      if (!Ctx) {
+        reject(_err('H5_NO_AUDIO_CTX', 'AudioContext not available'))
+        return
+      }
+
+      const buffers = []
      let stream = null
      let audioCtx = null
+      let source = null
+      let processor = null
      let analyser = null
+      let silentGain = null
      let rafId = null

-      navigator.mediaDevices.getUserMedia({ audio: true })
+      // 清理 H5 资源 (失败 / 异常路径统一调用)
+      const cleanup = () => {
+        try { if (rafId) cancelAnimationFrame(rafId) } catch (e) { /* noop */ }
+        try { if (processor) processor.disconnect() } catch (e) { /* noop */ }
+        try { if (analyser) analyser.disconnect() } catch (e) { /* noop */ }
+        try { if (source) source.disconnect() } catch (e) { /* noop */ }
+        try { if (silentGain) silentGain.disconnect() } catch (e) { /* noop */ }
+        try { if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {}) } catch (e) { /* noop */ }
+        try { if (stream) stream.getTracks().forEach(t => t.stop()) } catch (e) { /* noop */ }
+      }
+
+      navigator.mediaDevices.getUserMedia({
+        audio: {
+          channelCount: H5_NUM_CHANNELS,
+          // 注意: 部分浏览器会忽略 sampleRate 约束, 实际采样率以 AudioContext 为准
+          sampleRate: H5_SAMPLE_RATE,
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true
+        }
+      })
        .then((s) => {
          stream = s
-          try {
-            mediaRecorder = mime
-              ? new MediaRecorder(stream, { mimeType: mime })
-              : new MediaRecorder(stream)
-          } catch (e) {
-            try { stream.getTracks().forEach(t => t.stop()) } catch (_) {}
-            reject(e)
-            return
-          }
+          audioCtx = new Ctx({ sampleRate: H5_SAMPLE_RATE })

-          mediaRecorder.ondataavailable = (ev) => {
-            if (ev.data && ev.data.size > 0) chunks.push(ev.data)
+          // 麦克风 → ScriptProcessor → 静音 GainNode → destination
+          // (ScriptProcessor 必须连到 destination 才会触发 onaudioprocess;
+          //  用 GainNode(gain=0) 静音, 防止麦克风声音外放产生回声)
+          source = audioCtx.createMediaStreamSource(stream)
+          const BufferSize = H5_SCRIPT_PROCESSOR_BUFFER_SIZE
+          processor = audioCtx.createScriptProcessor(BufferSize, H5_NUM_CHANNELS, H5_NUM_CHANNELS)
+          processor.onaudioprocess = (e) => {
+            const channelData = e.inputBuffer.getChannelData(0)
+            // 必须拷贝: inputBuffer 数据会被下一帧覆盖, 不拷贝会导致 PCM 丢失
+            buffers.push(new Float32Array(channelData))
          }
+          silentGain = audioCtx.createGain()
+          silentGain.gain.value = 0
+          source.connect(processor)
+          processor.connect(silentGain)
+          silentGain.connect(audioCtx.destination)

-          mediaRecorder.onerror = (ev) => {
-            const err = (ev && ev.error) || new Error('MediaRecorder error')
-            this._emit('error', err)
-          }
-
-          mediaRecorder.onstop = () => {
-            // 收到 onstop 才算真正的 stop 完成,产物在 _stopH5 的 promise 里组装
+          // AnalyserNode 并行采振幅 (驱动波形动画)
+          analyser = audioCtx.createAnalyser()
+          analyser.fftSize = 256
+          source.connect(analyser)
+          const freqBuf = new Uint8Array(analyser.frequencyBinCount)
+          const tick = () => {
+            if (!this._isRecording) return
            try {
-              if (rafId) cancelAnimationFrame(rafId)
-              if (analyser) analyser.disconnect()
-              if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {})
-              if (stream) stream.getTracks().forEach(t => t.stop())
+              analyser.getByteFrequencyData(freqBuf)
+              let sum = 0
+              for (let i = 0; i < freqBuf.length; i++) sum += freqBuf[i]
+              const level = Math.min(1, sum / (freqBuf.length * 255))
+              this._emit('progress', { level, duration: Date.now() - this._startTs })
            } catch (e) { /* noop */ }
-
-            this._h5 = null
-            this._isRecording = false
-            this._clearAutoStopTimer()
-
-            const blob = new Blob(chunks, { type: mediaRecorder.mimeType || mime || 'audio/webm' })
-            const finalMime = blob.type || (mime || 'audio/webm')
-            const fileName = `voice_${Date.now()}.${mimeToExt(finalMime)}`
-            // 把产物挂回实例,供 _stopH5 的 promise 读取
-            this._h5PendingResult = {
-              blob,
-              mime: finalMime,
-              size: blob.size,
-              duration: Date.now() - this._startTs,
-              fileName
-            }
-            if (this._h5Resolve) {
-              const r = this._h5Resolve
-              this._h5Resolve = null
-              r(this._h5PendingResult)
-            }
+            rafId = requestAnimationFrame(tick)
          }
+          tick()

-          // AudioContext + AnalyserNode 采样振幅 (驱动波形)
-          try {
-            const Ctx = window.AudioContext || window.webkitAudioContext
-            if (Ctx) {
-              audioCtx = new Ctx()
-              const source = audioCtx.createMediaStreamSource(stream)
-              analyser = audioCtx.createAnalyser()
-              analyser.fftSize = 256
-              source.connect(analyser)
-              const data = new Uint8Array(analyser.frequencyBinCount)
-              const tick = () => {
-                if (!this._isRecording) return
-                analyser.getByteFrequencyData(data)
-                let sum = 0
-                for (let i = 0; i < data.length; i++) sum += data[i]
-                const level = Math.min(1, sum / (data.length * 255))
-                this._emit('progress', { level, duration: Date.now() - this._startTs })
-                rafId = requestAnimationFrame(tick)
-              }
-              tick()
-            }
-          } catch (e) {
-            // AudioContext 创建失败不影响录音本身
-            console.warn('[voice-recorder] AudioContext 初始化失败:', e)
-          }
-
-          this._h5 = { mediaRecorder, stream, audioCtx, analyser, rafId, mime }
+          this._h5 = { stream, audioCtx, source, processor, analyser, silentGain, rafId, buffers, sampleRate: audioCtx.sampleRate, cleanup }
          this._isRecording = true
          this._startTs = Date.now()
          this._setAutoStopTimer()
-          try {
-            mediaRecorder.start(250) // 每 250ms 触发一次 ondataavailable
-          } catch (e) {
-            // 启动失败, 主动清理资源并回滚状态
-            this._isRecording = false
-            this._clearAutoStopTimer()
-            try {
-              if (rafId) cancelAnimationFrame(rafId)
-              if (analyser) analyser.disconnect()
-              if (audioCtx && audioCtx.state !== 'closed') audioCtx.close().catch(() => {})
-              stream.getTracks().forEach(t => t.stop())
-            } catch (e2) { /* noop */ }
-            this._h5 = null
-            reject(e)
-            return
-          }
          resolve()
        })
        .catch((err) => {
-          reject(err)
+          // 麦克风权限拒绝 / 设备不可用
+          cleanup()
+          reject(_err('H5_MIC_PERMISSION', (err && err.message) || 'mic permission denied or device unavailable'))
        })
    })
  }
@@ -323,51 +370,60 @@ class VoiceRecorder {
  _stopH5() {
    return new Promise((resolve) => {
      const h5 = this._h5
-      if (!h5 || !h5.mediaRecorder) {
+      if (!h5) { resolve(null); return }
+
+      // 1) 拆 audio graph + 停流 (避免 onprocess 继续累积 buffer)
+      try { h5.cleanup() } catch (e) { /* noop */ }
+
+      // 2) 合并 PCM 帧缓冲
+      const sampleRate = h5.sampleRate || H5_SAMPLE_RATE
+      const merged = mergeFloat32Arrays(h5.buffers || [])
+
+      // 3) 编码标准 WAV (PCM 16-bit LE mono)
+      const wavBuffer = encodeWavPcm16LE(merged, sampleRate, H5_NUM_CHANNELS)
+      const blob = new Blob([wavBuffer], { type: 'audio/wav' })
+
+      this._h5 = null
+      this._isRecording = false
+      this._clearAutoStopTimer()
+
+      if (merged.length === 0) {
+        // 没有采到任何 PCM (例如用户没说话或立即 stop)
        resolve(null)
        return
      }
-      // onstop 回调里会 set _h5PendingResult 并调用 _h5Resolve
-      this._h5Resolve = resolve
-      try { h5.mediaRecorder.stop() } catch (e) { resolve(null) }
-    }).then((result) => {
-      if (!result) return null
+
+      const fileName = `voice_${Date.now()}.wav`
      const blobUrl = (typeof URL !== 'undefined' && URL.createObjectURL)
-        ? URL.createObjectURL(result.blob)
+        ? URL.createObjectURL(blob)
        : ''
-      return {
+
+      resolve({
        filePath: blobUrl,
-        blob: result.blob,
-        fileName: result.fileName,
-        duration: result.duration,
-        mime: result.mime,
-        size: result.size,
+        blob,
+        fileName,
+        duration: Date.now() - this._startTs,
+        mime: 'audio/wav',
+        size: blob.size,
        cancelled: false
-      }
+      })
    })
  }

  _cancelH5() {
    return new Promise((resolve) => {
      const h5 = this._h5
-      if (!h5 || !h5.mediaRecorder) {
+      if (!h5) {
        this._isRecording = false
        this._clearAutoStopTimer()
        resolve()
        return
      }
-      this._h5Resolve = (result) => {
-        // 取消时丢掉 blob URL, 但产物本身已不再使用
-        if (result && result.blob) {
-          // Blob 没有显式 release 接口, 浏览器 GC 即可
-        }
-        resolve()
-      }
-      try { h5.mediaRecorder.stop() } catch (e) {
-        this._isRecording = false
-        this._clearAutoStopTimer()
-        resolve()
-      }
+      try { h5.cleanup() } catch (e) { /* noop */ }
+      this._h5 = null
+      this._isRecording = false
+      this._clearAutoStopTimer()
+      resolve()
    })
  }