Skip to content

feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口 #793

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bigsongeth wants to merge 1 commit into
base: beta
Choose a base branch
from
Open

feat(asr): 百炼 ASR 支持 fun-asr-flash 并统一三协议为单一入口 #793

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
405 changes: 405 additions & 0 deletions openless-all/app/src-tauri/src/asr/dashscope_multimodal.rs
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions openless-all/app/src-tauri/src/asr/mimo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ fn pcm_duration_ms(pcm: &[u8]) -> u64 {
super::pcm::pcm_duration_ms(pcm)
}

fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
/// 按时长把 PCM 切成多段（base64 进 JSON 的批量 ASR 都受单请求体积/时长限制）。
/// `dashscope_multimodal` 复用同一套切分逻辑，故 `pub(crate)`。
pub(crate) fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
if max_chunk_duration_ms == 0 {
return vec![pcm];
}
Expand All @@ -195,7 +197,9 @@ fn split_pcm_by_duration(pcm: &[u8], max_chunk_duration_ms: u64) -> Vec<&[u8]> {
pcm.chunks(bytes_per_chunk).collect()
}

fn join_transcript_chunks(chunks: &[String]) -> String {
/// 把分段识别文本按 CJK/标点规则拼回一句（段间按需补空格）。
/// `dashscope_multimodal` 复用同一套拼接逻辑，故 `pub(crate)`。
pub(crate) fn join_transcript_chunks(chunks: &[String]) -> String {
let mut joined = String::new();
for chunk in chunks.iter().map(|chunk| chunk.trim()) {
if chunk.is_empty() {
Expand Down
2 changes: 2 additions & 0 deletions openless-all/app/src-tauri/src/asr/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
//! `volcengine.rs`.

pub mod bailian;
pub mod dashscope_multimodal;
mod frame;
pub mod local;
pub mod mimo;
Expand All @@ -16,6 +17,7 @@ pub mod wav;
pub mod whisper;

pub use bailian::{BailianCredentials, BailianRealtimeASR};
pub use dashscope_multimodal::DashScopeMultimodalASR;
pub use mimo::MimoBatchASR;
pub use qwen_realtime::{Qwen3RealtimeASR, Qwen3RealtimeCredentials};
pub use volcengine::{VolcengineCredentials, VolcengineStreamingASR};
Expand Down
29 changes: 16 additions & 13 deletions openless-all/app/src-tauri/src/commands/credentials.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@ fn volcengine_configured(snap: &CredentialsSnapshot) -> bool {
}

pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
if provider == "volcengine" {
return volcengine_configured(snap);
}
// 本地 / 无凭据引擎不属于云端分类枚举（ActiveAsrProviderKind），由平台 cfg 门
// 在此单独判定；移动端上这些引擎不可用直接判未配置。
if cfg!(mobile)
&& (provider == crate::asr::local::PROVIDER_ID
|| provider == crate::asr::local::sherpa::PROVIDER_ID
Expand All @@ -46,17 +45,21 @@ pub(crate) fn asr_configured_for_provider(provider: &str, snap: &CredentialsSnap
// 本地 ASR 不依赖云端凭据。
return true;
}
if provider == crate::asr::bailian::PROVIDER_ID
|| provider == crate::asr::qwen_realtime::PROVIDER_ID
{
return configured(&snap.asr_api_key);
}
if provider == crate::asr::mimo::PROVIDER_ID {
return configured(&snap.asr_api_key)
&& configured(&snap.asr_endpoint)
&& configured(&snap.asr_model);
// 云端 provider：所需字段由 ActiveAsrProviderKind 统一判定（穷尽 match，新增
// kind 编译器强制补齐）。volcengine 亦经此路（VolcAppKey）。
use crate::coordinator::{active_asr_provider_kind, AsrConfiguredFields};
match active_asr_provider_kind(provider).configured_fields() {
AsrConfiguredFields::ApiKeyOnly => configured(&snap.asr_api_key),
AsrConfiguredFields::ApiKeyEndpointModel => {
configured(&snap.asr_api_key)
&& configured(&snap.asr_endpoint)
&& configured(&snap.asr_model)
}
AsrConfiguredFields::EndpointModelOnly => {
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
}
AsrConfiguredFields::VolcAppKey => volcengine_configured(snap),
}
configured(&snap.asr_endpoint) && configured(&snap.asr_model)
}

pub(crate) fn llm_configured_for_provider(provider: &str, snap: &CredentialsSnapshot) -> bool {
Expand Down
122 changes: 112 additions & 10 deletions openless-all/app/src-tauri/src/commands/providers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,22 @@ pub async fn validate_provider_credentials(kind: String) -> Result<ProviderCheck
#[tauri::command]
pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult, String> {
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::bailian::PROVIDER_ID {
// Bailian 实时 ASR 没有模型列表 HTTP 接口，列表只能是静态的。但直接返回
// 硬编码列表会让「拉取模型」在 Key / endpoint 全错时也显示成功，给用户
// "已连通"的错觉（其他厂商这颗按钮是真实带 Key GET /models）。先跑一次
// 与「验证」相同的 WebSocket 连通性检查，语义对齐后再返回静态列表。
validate_bailian_asr_provider().await?;
// 统一「阿里云百炼」入口:三条协议(实时 fun-asr-realtime / 实时 qwen3 /
// 录音文件 fun-asr-flash)收成一个 provider。百炼各网关都没有模型列表 HTTP
// 接口,列表是静态的;但先跑一次与「验证」相同的、按当前所选模型对应协议的
// 连通性检查(validate_asr_provider 已按模型路由),避免 Key/endpoint 全错时
// 也显示成功。随后返回三个可选模型供下拉。
validate_asr_provider().await?;
// 静态清单只是常用快捷项;协议按模型名自动路由,用户也可在模型框直接手填
// 任意 DashScope ASR 模型(如 paraformer-realtime-v2 等),不受此列表限制。
return Ok(ProviderModelsResult {
models: vec![crate::asr::bailian::DEFAULT_MODEL.to_string()],
models: vec![
crate::asr::bailian::DEFAULT_MODEL.to_string(),
crate::asr::qwen_realtime::DEFAULT_MODEL.to_string(),
"qwen3-asr-flash-realtime-2026-02-10".to_string(),
"qwen3-asr-flash-realtime-2025-10-27".to_string(),
crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string(),
],
});
}
if kind == "asr" && CredentialsVault::get_active_asr() == crate::asr::qwen_realtime::PROVIDER_ID
Expand All @@ -55,6 +64,14 @@ pub async fn list_provider_models(kind: String) -> Result<ProviderModelsResult,
models: vec![crate::asr::mimo::DEFAULT_MODEL.to_string()],
});
}
if kind == "asr"
&& CredentialsVault::get_active_asr() == crate::asr::dashscope_multimodal::PROVIDER_ID
{
// multimodal-generation 无模型列表 HTTP 接口；与 mimo 同，返回静态别名。
return Ok(ProviderModelsResult {
models: vec![crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string()],
});
}
if kind == "llm" && CredentialsVault::get_active_llm() == CODEX_OAUTH_PROVIDER_ID {
return Ok(ProviderModelsResult {
models: vec![
Expand Down Expand Up @@ -203,6 +220,19 @@ async fn validate_asr_provider() -> Result<(), String> {
}

if active_asr == crate::asr::bailian::PROVIDER_ID {
// 统一百炼:按所选模型验证对应协议（endpoint 由前端按模型同步，各 validator
// 读到的都是该协议的正确地址）。
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.ok()
.flatten()
.unwrap_or_default();
let effective = crate::coordinator::resolve_effective_asr_provider(&active_asr, &model);
if effective == crate::asr::qwen_realtime::PROVIDER_ID {
return validate_qwen3_realtime_asr_provider().await;
}
if effective == crate::asr::dashscope_multimodal::PROVIDER_ID {
return validate_dashscope_multimodal_asr_provider().await;
}
return validate_bailian_asr_provider().await;
}
if active_asr == crate::asr::qwen_realtime::PROVIDER_ID {
Expand All @@ -211,6 +241,9 @@ async fn validate_asr_provider() -> Result<(), String> {
if active_asr == crate::asr::mimo::PROVIDER_ID {
return validate_mimo_asr_provider().await;
}
if active_asr == crate::asr::dashscope_multimodal::PROVIDER_ID {
return validate_dashscope_multimodal_asr_provider().await;
}

let config = read_openai_provider_config("asr")?;
let model = CredentialsVault::get(CredentialAccount::AsrModel)
Expand All @@ -237,6 +270,69 @@ async fn validate_mimo_asr_provider() -> Result<(), String> {
.map_err(|e| e.to_string())
}

/// fun-asr-flash 官方公开示例音频，用于连通性校验。该模型对纯静音会返回
/// 400（"no speech" 类错误），无法像 Whisper/Mimo 那样发静音探活；改用这段
/// 阿里官方文档在案的示例 wav（由 DashScope 侧拉取），key/endpoint/model 有效
/// 即返回 200。
const DASHSCOPE_ASR_VALIDATE_SAMPLE_URL: &str =
"https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav";

async fn validate_dashscope_multimodal_asr_provider() -> Result<(), String> {
// 统一「阿里云百炼」下三协议共用一个 endpoint 字段(存的是经典实时的 wss 地址),
// 对 multimodal(https)不适用；直接用协议默认地址 + 共用 key,绕过按存储 endpoint
// 走(且会拒 wss)的 read_openai_provider_config。别名 id 仍走标准读取。
let (api_key, base_url) = if crate::coordinator::unified_bailian_is_active() {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.ok_or_else(|| "API Key 为空".to_string())?;
(
api_key,
crate::asr::dashscope_multimodal::DEFAULT_ENDPOINT.to_string(),
)
} else {
let config = read_openai_provider_config("asr")?;
(config.api_key, config.base_url)
};
let model = CredentialsVault::get(CredentialAccount::AsrModel)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::dashscope_multimodal::DEFAULT_MODEL.to_string());
let url = crate::asr::dashscope_multimodal::generation_url(&base_url)
.map_err(|_| "endpointInvalid".to_string())?;
let body = serde_json::json!({
"model": model,
"input": { "messages": [{ "role": "user", "content": [{
"type": "input_audio",
"input_audio": { "data": DASHSCOPE_ASR_VALIDATE_SAMPLE_URL },
}]}]},
"parameters": { "format": "wav", "sample_rate": "16000" },
});
let client = http_client_builder(&url, 20)
.build()
.map_err(|_| "providerClientInitFailed".to_string())?;
let response = client
.post(&url)
.header("Authorization", format!("Bearer {}", api_key))
.header("Content-Type", "application/json")
.header("X-DashScope-SSE", "disable")
.json(&body)
.send()
.await
.map_err(|e| {
if e.is_timeout() {
"providerRequestTimeout".to_string()
} else {
"providerNetworkError".to_string()
}
})?;
let status = response.status();
if !status.is_success() {
return Err(format!("providerHttpStatus:{}", status.as_u16()));
}
Ok(())
}

async fn validate_bailian_asr_provider() -> Result<(), String> {
let api_key = CredentialsVault::get(CredentialAccount::AsrApiKey)
.map_err(|e| e.to_string())?
Expand Down Expand Up @@ -293,10 +389,16 @@ async fn validate_qwen3_realtime_asr_provider() -> Result<(), String> {
if api_key.trim().is_empty() {
return Err("API Key 为空".to_string());
}
let endpoint = CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string());
// 统一百炼下共用 endpoint 存的是经典实时地址，对 qwen3 realtime 网关不对；用协议
// 默认地址。别名 id 仍读自己存的 endpoint。
let endpoint = if crate::coordinator::unified_bailian_is_active() {
crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string()
} else {
CredentialsVault::get(CredentialAccount::AsrEndpoint)
.map_err(|e| e.to_string())?
.filter(|s| !s.trim().is_empty())
.unwrap_or_else(|| crate::asr::qwen_realtime::DEFAULT_ENDPOINT.to_string())
};
if !crate::asr::qwen_realtime::endpoint_scheme_is_secure_websocket(&endpoint) {
return Err("qwen3EndpointSchemeInvalid".to_string());
}
Expand Down
Loading
Loading