diff --git a/pkg/audio/tts/dashscope_tts.go b/pkg/audio/tts/dashscope_tts.go new file mode 100644 index 0000000000..304804d3af --- /dev/null +++ b/pkg/audio/tts/dashscope_tts.go @@ -0,0 +1,180 @@ +package tts + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/sipeed/picoclaw/pkg/providers/common" +) + +const dashScopeTTSEndpoint = "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation" + +type DashScopeTTSProvider struct { + apiKey string + model string + voice string + workspace string + httpClient *http.Client +} + +type dashScopeTTSAudioStream struct { + io.ReadCloser + fileExt string + contentType string +} + +func (s *dashScopeTTSAudioStream) AudioFileMeta() (string, string) { + return s.fileExt, s.contentType +} + +type dashScopeTTSRequest struct { + Model string `json:"model"` + Input dashScopeTTSInput `json:"input"` + Params dashScopeTTSParams `json:"parameters,omitempty"` +} + +type dashScopeTTSInput struct { + Text string `json:"text"` +} + +type dashScopeTTSParams struct { + Voice string `json:"voice,omitempty"` + Format string `json:"format,omitempty"` +} + +type dashScopeTTSResponse struct { + Output struct { + Audio struct { + Data string `json:"data"` + URL string `json:"url"` + ExpiresAt int64 `json:"expires_at"` + } `json:"audio"` + } `json:"output"` + RequestID string `json:"request_id"` +} + +func NewDashScopeTTSProvider(apiKey, workspace, model, voice string) *DashScopeTTSProvider { + client := common.NewHTTPClient("") + client.Timeout = 60 * time.Second + + return &DashScopeTTSProvider{ + apiKey: apiKey, + model: model, + voice: voice, + workspace: workspace, + httpClient: client, + } +} + +func (t *DashScopeTTSProvider) Name() string { + return "dashscope-tts" +} + +func (t *DashScopeTTSProvider) Synthesize(ctx context.Context, text string) (io.ReadCloser, error) { + // Step 1: Call DashScope TTS API + reqBody := dashScopeTTSRequest{ + Model: t.model, + Input: dashScopeTTSInput{Text: text}, + } + if t.voice != "" { + reqBody.Params = dashScopeTTSParams{ + Voice: t.voice, + Format: "wav", + } + } + + jsonData, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("dashscope tts: marshal failed: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, "POST", dashScopeTTSEndpoint, bytes.NewReader(jsonData)) + if err != nil { + return nil, fmt.Errorf("dashscope tts: create request failed: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+t.apiKey) + if t.workspace != "" { + req.Header.Set("X-DashScope-WorkSpace", t.workspace) + } + + resp, err := t.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("dashscope tts: request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("dashscope tts: API error %d: %s", resp.StatusCode, string(body)) + } + + var ttsResp dashScopeTTSResponse + if err := json.NewDecoder(resp.Body).Decode(&ttsResp); err != nil { + return nil, fmt.Errorf("dashscope tts: parse response failed: %w", err) + } + + audioURL := strings.TrimSpace(ttsResp.Output.Audio.URL) + if audioURL == "" { + return nil, fmt.Errorf("dashscope tts: no audio URL in response") + } + + // Step 2: Download audio from OSS + dlReq, err := http.NewRequestWithContext(ctx, "GET", audioURL, nil) + if err != nil { + return nil, fmt.Errorf("dashscope tts: create download request failed: %w", err) + } + + dlResp, err := t.httpClient.Do(dlReq) + if err != nil { + return nil, fmt.Errorf("dashscope tts: download audio failed: %w", err) + } + + if dlResp.StatusCode != http.StatusOK { + defer dlResp.Body.Close() + body, _ := io.ReadAll(dlResp.Body) + return nil, fmt.Errorf("dashscope tts: download audio HTTP %d: %s", dlResp.StatusCode, string(body)) + } + + audioBytes, err := io.ReadAll(dlResp.Body) + dlResp.Body.Close() + if err != nil { + return nil, fmt.Errorf("dashscope tts: read audio failed: %w", err) + } + + if len(audioBytes) == 0 { + return nil, fmt.Errorf("dashscope tts: empty audio response") + } + + return &dashScopeTTSAudioStream{ + ReadCloser: io.NopCloser(bytes.NewReader(audioBytes)), + fileExt: ".wav", + contentType: "audio/wav", + }, nil +} + +// extractWorkspaceFromBaseURL extracts the workspace ID from a Bailian maas URL. +// e.g. "https://llm-hubsv5i2n0jjbjb6.cn-beijing.maas.aliyuncs.com/..." → "llm-hubsv5i2n0jjbjb6" +func extractWorkspaceFromBaseURL(apiBase string) string { + if apiBase == "" { + return "" + } + u, err := url.Parse(apiBase) + if err != nil { + return "" + } + host := u.Hostname() + // Match maas host pattern: ..maas.aliyuncs.com + parts := strings.Split(host, ".") + if len(parts) >= 4 && strings.HasSuffix(host, ".maas.aliyuncs.com") { + return parts[0] + } + return "" +} diff --git a/pkg/audio/tts/tts.go b/pkg/audio/tts/tts.go index fc8a36ea23..470e224fd7 100644 --- a/pkg/audio/tts/tts.go +++ b/pkg/audio/tts/tts.go @@ -36,6 +36,18 @@ func providerFromModelConfig(mc *config.ModelConfig) TTSProvider { switch protocol { case "mimo": return NewMimoTTSProvider(mc.APIKey(), providers.ResolveAPIBase(mc), modelID, mc.Proxy) + case "dashscope": + voice := "" + if mc.ExtraBody != nil { + if v, ok := mc.ExtraBody["voice"].(string); ok { + voice = strings.TrimSpace(v) + } + } + workspace := extractWorkspaceFromBaseURL(mc.APIBase) + if workspace == "" { + workspace = extractWorkspaceFromBaseURL(providers.ResolveAPIBase(mc)) + } + return NewDashScopeTTSProvider(mc.APIKey(), workspace, modelID, voice) default: return NewOpenAITTSProviderWithOptions( mc.APIKey(), diff --git a/pkg/channels/weixin/media.go b/pkg/channels/weixin/media.go index cf1b456126..178bcb5af9 100644 --- a/pkg/channels/weixin/media.go +++ b/pkg/channels/weixin/media.go @@ -446,6 +446,67 @@ func selectInboundMediaItem(msg WeixinMessage) *MessageItem { return nil } +// tryTranscodeAudioToSilk converts an audio file (OGG/MP3/WAV/etc) to SILK format +// suitable for WeChat voice messages. Uses ffmpeg to decode to PCM, then silk_encoder +// with -tencent flag to produce WeChat-compatible SILK. +func tryTranscodeAudioToSilk(ctx context.Context, audioPath string) ([]byte, error) { + transcodeCtx, cancel := context.WithTimeout(ctx, weixinVoiceTranscodeTimeout) + defer cancel() + + tmpDir := media.TempDir() + pcmPath := filepath.Join(tmpDir, "weixin-voice-"+uuid.New().String()+".pcm") + silkPath := filepath.Join(tmpDir, "weixin-voice-"+uuid.New().String()+".silk") + defer os.Remove(pcmPath) + defer os.Remove(silkPath) + + // Step 1: Decode audio to 24kHz 16-bit mono PCM via ffmpeg + ffmpegBin, err := exec.LookPath("ffmpeg") + if err != nil { + return nil, fmt.Errorf("ffmpeg not found: %w", err) + } + ffCmd := exec.CommandContext(transcodeCtx, ffmpegBin, + "-y", "-i", audioPath, + "-f", "s16le", "-acodec", "pcm_s16le", + "-ar", "24000", "-ac", "1", + pcmPath, + ) + ffOut, ffErr := ffCmd.CombinedOutput() + if ffErr != nil { + return nil, fmt.Errorf("ffmpeg decode failed: %w (output: %s)", ffErr, strings.TrimSpace(string(ffOut))) + } + + // Step 2: Encode PCM to Tencent SILK + encoderBin := "/vol1/picoclaw/bin/silk_encoder" + if _, err := os.Stat(encoderBin); err != nil { + // Fallback: search PATH + if found, lookErr := exec.LookPath("silk_encoder"); lookErr == nil { + encoderBin = found + } else { + return nil, fmt.Errorf("silk_encoder not found at %s", encoderBin) + } + } + silkCmd := exec.CommandContext(transcodeCtx, encoderBin, + pcmPath, silkPath, + "-tencent", + "-Fs_API", "24000", + "-rate", "25000", + "-quiet", + ) + silkOut, silkErr := silkCmd.CombinedOutput() + if silkErr != nil { + return nil, fmt.Errorf("silk encode failed: %w (output: %s)", silkErr, strings.TrimSpace(string(silkOut))) + } + + silk, err := os.ReadFile(silkPath) + if err != nil { + return nil, fmt.Errorf("failed to read SILK output: %w", err) + } + if len(silk) == 0 { + return nil, fmt.Errorf("silk_encoder produced empty output") + } + return silk, nil +} + func tryTranscodeSilkToWAV(ctx context.Context, silk []byte) ([]byte, error) { decoders := []struct { name string @@ -610,6 +671,8 @@ func outboundMediaKind(partType, filename, contentType string) int { return UploadMediaTypeImage case "video": return UploadMediaTypeVideo + case "voice", "audio": + return UploadMediaTypeVoice } ct := strings.ToLower(contentType) @@ -618,7 +681,16 @@ func outboundMediaKind(partType, filename, contentType string) int { return UploadMediaTypeImage case strings.HasPrefix(ct, "video/"): return UploadMediaTypeVideo + // WeChat iLink bots cannot send native voice bubbles to users. + // Audio is always sent as a file attachment instead. + case strings.HasPrefix(ct, "audio/"): + return UploadMediaTypeFile default: + ext := strings.ToLower(filepath.Ext(filename)) + switch ext { + case ".ogg", ".mp3", ".wav", ".opus", ".aac", ".flac", ".m4a": + return UploadMediaTypeFile + } return UploadMediaTypeFile } } @@ -993,6 +1065,17 @@ func (c *WeixinChannel) sendUploadedMedia( }, }) + case UploadMediaTypeVoice: + return c.sendMessageItem(ctx, toUserID, contextToken, MessageItem{ + Type: MessageItemTypeVoice, + VoiceItem: &VoiceItem{ + Media: mediaRef, + EncodeType: 1, // SILK + BitsPerSample: 16, + SampleRate: 24000, + }, + }) + default: return c.sendMessageItem(ctx, toUserID, contextToken, MessageItem{ Type: MessageItemTypeFile, @@ -1133,6 +1216,7 @@ func (c *WeixinChannel) SendMedia(ctx context.Context, msg bus.OutboundMediaMess } kind := outboundMediaKind(part.Type, filename, contentType) + uploaded, uploadErr := c.uploadLocalFile(ctx, localPath, filename, msg.ChatID, kind) if uploadErr != nil { err = uploadErr