From d3c761be53a8cbfa5566563752dcf203e1b17485 Mon Sep 17 00:00:00 2001 From: nowzhao Date: Thu, 25 Jun 2026 19:54:39 +0800 Subject: [PATCH 1/3] Add --instruction flag for natural language TTS style control Expose instruction-based synthesis via a new --instruction CLI flag and HTTP params JSON field. The instruction is injected into the system message of the prompt so the model can apply speaking style, emotion, and prosody described in natural language. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- include/s2_pipeline.h | 1 + include/s2_prompt.h | 3 ++- src/main.cpp | 2 ++ src/s2_pipeline.cpp | 4 ++-- src/s2_prompt.cpp | 15 ++++++++++++--- src/s2_server.cpp | 4 ++++ 6 files changed, 23 insertions(+), 6 deletions(-) diff --git a/include/s2_pipeline.h b/include/s2_pipeline.h index 19b3c3d..e4aaf11 100644 --- a/include/s2_pipeline.h +++ b/include/s2_pipeline.h @@ -31,6 +31,7 @@ struct PipelineParams { std::string prompt_text; std::string prompt_audio_path; std::string output_path; + std::string instruction; GenerateParams gen; int32_t gpu_device = -1; BackendType backend_type = BackendType::CPU; diff --git a/include/s2_prompt.h b/include/s2_prompt.h index 1fb36a1..5e5dd3e 100644 --- a/include/s2_prompt.h +++ b/include/s2_prompt.h @@ -20,7 +20,8 @@ PromptTensor build_prompt( const std::string & prompt_text, const int32_t * prompt_codes, int32_t num_codebooks, - int32_t T_prompt + int32_t T_prompt, + const std::string & instruction = "" ); } diff --git a/src/main.cpp b/src/main.cpp index d3e0b49..26ca102 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -66,6 +66,7 @@ void print_uso() { safe_print(" -text, --text Text to synthesize\n"); safe_print(" -pa, --prompt-audio Path to reference audio for cloning\n"); safe_print(" -pt, --prompt-text Text of the reference audio\n"); + safe_print(" --instruction Natural language instruction to control speaking style\n"); safe_print(" --voice Load a saved voice profile\n"); safe_print(" --save-voice Save the encoded reference as a voice profile\n"); safe_print(" --voice-dir Directory used for saved voice profiles\n"); @@ -160,6 +161,7 @@ int main(int argc, char** argv) { else if (arg == "-text" || arg == "--text") { if (i+1 < argc) params.text = argv[++i]; } else if (arg == "-pa" || arg == "--prompt-audio") { if (i+1 < argc) params.prompt_audio_path = argv[++i]; } else if (arg == "-pt" || arg == "--prompt-text") { if (i+1 < argc) params.prompt_text = argv[++i]; } + else if (arg == "--instruction") { if (i+1 < argc) params.instruction = argv[++i]; } else if (arg == "--voice") { if (i+1 < argc) params.voice_id = argv[++i]; } else if (arg == "--save-voice") { params.save_voice = true; } else if (arg == "--voice-dir") { if (i+1 < argc) params.voice_storage_dir = argv[++i]; } diff --git a/src/s2_pipeline.cpp b/src/s2_pipeline.cpp index 0dd5f01..6526493 100644 --- a/src/s2_pipeline.cpp +++ b/src/s2_pipeline.cpp @@ -844,7 +844,7 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con PromptTensor prompt = build_prompt( tokenizer(), params.text, params.prompt_text, ref_codes, - num_codebooks, T_prompt); + num_codebooks, T_prompt, params.instruction); int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens; const auto kv_t0 = std::chrono::steady_clock::now(); @@ -982,7 +982,7 @@ bool Pipeline::synthesize_streaming_prompt_codes_locked(const PipelineParams & p PromptTensor prompt = build_prompt( tokenizer(), params.text, params.prompt_text, ref_codes, - num_codebooks, T_prompt); + num_codebooks, T_prompt, params.instruction); int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens; const auto kv_t0 = std::chrono::steady_clock::now(); diff --git a/src/s2_prompt.cpp b/src/s2_prompt.cpp index 80f6da7..c4cb442 100644 --- a/src/s2_prompt.cpp +++ b/src/s2_prompt.cpp @@ -8,7 +8,8 @@ PromptTensor build_prompt( const std::string & prompt_text, const int32_t * prompt_codes, int32_t num_codebooks, - int32_t T_prompt + int32_t T_prompt, + const std::string & instruction ) { PromptTensor result; result.rows = num_codebooks + 1; @@ -33,7 +34,11 @@ PromptTensor build_prompt( app(sys_pre, tokenizer.encode("<|im_start|>system")); app(sys_pre, NEWLINE); - app(sys_pre, tokenizer.encode("convert the provided text to speech reference to the following:\n\nText:\n")); + std::string sys_msg = "convert the provided text to speech reference to the following:\n\nText:\n"; + if (!instruction.empty()) { + sys_msg = instruction + "\n\n" + sys_msg; + } + app(sys_pre, tokenizer.encode(sys_msg)); if (!prompt_has_speaker_tag) { app(sys_pre, tokenizer.encode("<|speaker:0|>")); } @@ -57,7 +62,11 @@ PromptTensor build_prompt( app(sys_post, tokenizer.encode("<|im_start|>system")); app(sys_post, NEWLINE); - app(sys_post, tokenizer.encode("convert the provided text to speech")); + std::string sys_msg = "convert the provided text to speech"; + if (!instruction.empty()) { + sys_msg += ". " + instruction; + } + app(sys_post, tokenizer.encode(sys_msg)); app(sys_post, { im_end_id }); app(sys_post, NEWLINE); diff --git a/src/s2_server.cpp b/src/s2_server.cpp index 08fea2d..f86698c 100644 --- a/src/s2_server.cpp +++ b/src/s2_server.cpp @@ -577,6 +577,10 @@ namespace s2 voice_dir_value = j["voice_dir"].get(); } + if (j.contains("instruction")) { + pipelineParams.instruction = j["instruction"].get(); + } + if (j.contains("stream_decode_stride_frames")) { int32_t val = j["stream_decode_stride_frames"].get(); pipelineParams.stream_decode_stride_frames = std::max(0, val); From 26ff69d08890dca6d190b9f2addc01f623467101 Mon Sep 17 00:00:00 2001 From: nowzhao Date: Fri, 26 Jun 2026 10:55:22 +0800 Subject: [PATCH 2/3] Default CPU threads to physical/performance cores resolve_n_threads defaulted to hardware_concurrency() (all logical cores). On hybrid CPUs this oversubscribes work onto SMT siblings or, on Apple Silicon, onto slow efficiency cores; every thread in a fork/join matmul waits on the slowest one, so each decode step runs at efficiency-core speed. Measured on an M4 Pro (10P+4E), the default (14 threads) was ~2.9x slower end-to-end than capping at the 10 performance cores. - Move resolve_n_threads into s2_ggml_utils.h alongside a cross-platform detect_compute_threads() (macOS hw.perflevel0.physicalcpu, Linux /proc/cpuinfo core-id dedup, Windows GetLogicalProcessorInformation, fallback = logical/2). - Route the codec's 5 ggml_backend_cpu_set_n_threads sites through resolve_n_threads. They passed raw n_threads, so the default value of 0 made ggml fall back to a single thread for the audio codec. Explicit --threads N is unchanged: it still passes through verbatim. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/s2_codec.cpp | 10 +++--- src/s2_ggml_utils.h | 88 +++++++++++++++++++++++++++++++++++++++++++++ src/s2_model.cpp | 6 ---- 3 files changed, 93 insertions(+), 11 deletions(-) diff --git a/src/s2_codec.cpp b/src/s2_codec.cpp index e4dc3bc..f17bf7c 100755 --- a/src/s2_codec.cpp +++ b/src/s2_codec.cpp @@ -1046,7 +1046,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread enc_inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::encode] encoder compute failed." << std::endl; ggml_gallocr_free(allocr); @@ -1119,7 +1119,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread qenc_inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf2) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::encode] quantizer stage compute failed." << std::endl; ggml_gallocr_free(allocr2); @@ -1279,7 +1279,7 @@ static bool run_cached_decode_graph(AudioCodec::Impl & impl, const int32_t * cod } if (ggml_backend_is_cpu(impl.backend)) { - ggml_backend_cpu_set_n_threads(impl.backend, n_threads); + ggml_backend_cpu_set_n_threads(impl.backend, resolve_n_threads(n_threads)); } if (ggml_backend_graph_compute(impl.backend, cache.graph) != GGML_STATUS_SUCCESS) { @@ -1381,7 +1381,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::decode] quantizer decode compute failed." << std::endl; @@ -1426,7 +1426,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa ggml_backend_tensor_set(latent_in, latent_out.data(), 0, latent_out.size() * sizeof(float)); - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::decode] decoder compute failed." << std::endl; diff --git a/src/s2_ggml_utils.h b/src/s2_ggml_utils.h index 1cb152a..fd855b9 100755 --- a/src/s2_ggml_utils.h +++ b/src/s2_ggml_utils.h @@ -3,9 +3,97 @@ #include "ggml.h" #include +#include +#include + +#if defined(__APPLE__) +# include +#elif defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +#elif defined(__linux__) +# include +# include +# include +#endif namespace s2 { +// Number of physical "performance" cores available for compute. +// +// ggml's CPU backend scales poorly when oversubscribed onto hyperthreads or, +// on Apple Silicon, onto the slow efficiency cores: every thread in a fork/join +// matmul waits on the slowest one, so the whole step runs at efficiency-core +// speed. Measured on an M4 Pro (10P + 4E), defaulting to all 14 logical cores +// was ~2.9x slower end-to-end than capping at the 10 performance cores. +// +// This returns a sensible default core count; callers fall back to it only when +// the user did not pass an explicit --threads value. +inline int32_t detect_compute_threads() { +#if defined(__APPLE__) + // Performance-core count (perflevel0). Falls back to all physical cores. + int32_t n = 0; + size_t sz = sizeof(n); + if (sysctlbyname("hw.perflevel0.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) { + return n; + } + n = 0; sz = sizeof(n); + if (sysctlbyname("hw.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) { + return n; + } +#elif defined(_WIN32) + // Count physical cores via processor-core relationships. + DWORD len = 0; + GetLogicalProcessorInformation(nullptr, &len); + if (len > 0) { + std::vector buf( + len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)); + if (GetLogicalProcessorInformation(buf.data(), &len)) { + int32_t cores = 0; + for (const auto & info : buf) { + if (info.Relationship == RelationProcessorCore) cores++; + } + if (cores > 0) return cores; + } + } +#elif defined(__linux__) + // Count distinct (physical_id, core_id) pairs from /proc/cpuinfo to skip + // SMT siblings. + if (FILE * f = std::fopen("/proc/cpuinfo", "r")) { + std::set> cores; + int phys = -1, core = -1; + char line[256]; + while (std::fgets(line, sizeof(line), f)) { + int v; + if (std::sscanf(line, "physical id : %d", &v) == 1) phys = v; + else if (std::sscanf(line, "core id : %d", &v) == 1) core = v; + if (line[0] == '\n') { // blank line separates logical CPUs + if (phys >= 0 && core >= 0) cores.insert({phys, core}); + phys = core = -1; + } + } + if (phys >= 0 && core >= 0) cores.insert({phys, core}); + std::fclose(f); + if (!cores.empty()) return static_cast(cores.size()); + } +#endif + // Generic fallback: assume SMT and halve the logical-core count, but never + // go below 1. + const unsigned hw = std::thread::hardware_concurrency(); + if (hw == 0) return 4; + return static_cast(hw >= 2 ? hw / 2 : hw); +} + +// Resolve a caller-supplied thread count: honor explicit positive values, +// otherwise pick a good default for this machine. +inline int32_t resolve_n_threads(int32_t n_threads) { + if (n_threads > 0) return n_threads; + return detect_compute_threads(); +} + inline ggml_tensor * repeat_checked(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b, const char * label = "repeat") { if (!ggml_can_repeat(a, b)) { diff --git a/src/s2_model.cpp b/src/s2_model.cpp index 9ced623..8515ad4 100755 --- a/src/s2_model.cpp +++ b/src/s2_model.cpp @@ -18,12 +18,6 @@ namespace s2 { -static int32_t resolve_n_threads(int32_t n_threads) { - if (n_threads > 0) return n_threads; - const auto hw = std::thread::hardware_concurrency(); - return static_cast(hw > 0 ? hw : 4); -} - static const char * backend_type_name(BackendType backend_type) { switch (backend_type) { case BackendType::CPU: return "CPU"; From d8e0dbaa18788241414d93e0e9d13afd949894e8 Mon Sep 17 00:00:00 2001 From: nowzhao Date: Fri, 26 Jun 2026 16:21:05 +0800 Subject: [PATCH 3/3] Speed up CPU offline synthesis: single-shot decode, faster sampler, LTO Three CPU optimizations, measured on an M4 Pro (q2_k, threads=8 controlled A/B): codec decode ~26.3s -> ~6.8s (3.9x), end-to-end ~51.5s -> ~33.7s (1.5x). 1. Offline codec decode no longer windows unnecessarily. The offline path chunked decode at stride 16, rebuilding the decode graph (two ggml_init of 96MB+128MB) per batch and re-decoding a ~160-frame overlap region each window. Offline synthesis has every frame in hand, so it now decodes all frames in a single pass unless the user explicitly sets --stream-decode-stride. Faster and free of window-boundary artifacts. The streaming path is unchanged. This uses the already-tested finalize branch of decode_codes_windowed (the window loop body never runs when stride >= total frames). 2. sample_token no longer sorts the full 155776-entry vocab on every main token. ~95% of entries are -inf (semantic mask) and only top_k are kept, so it now skips -inf candidates and uses partial_sort for the top_k. Numerically identical: masked terms contribute exp(-inf)=0 to the softmax denominator. Verified against the original over 2000 randomized cases (with mask, varied top_k/top_p) with zero greedy mismatches. 3. Enable GGML_LTO for optimized builds (ggml defaults it OFF). A few percent on the compute kernels at no correctness cost; only applied when the user has not set GGML_LTO and the build type is optimized. Co-Authored-By: Claude Opus 4.8 (1M context) --- CMakeLists.txt | 11 +++++++ src/s2_pipeline.cpp | 13 ++++++-- src/s2_sampler.cpp | 77 +++++++++++++++++++++++++-------------------- 3 files changed, 65 insertions(+), 36 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f6d8a51..f115803 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,17 @@ endif() set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +# Enable link-time optimization for optimized builds. ggml ships GGML_LTO=OFF +# by default; turning it on gives a few percent on the CPU compute kernels at +# no correctness cost. Only meaningful for optimized configs, and only if not +# already set by the user. +if(NOT DEFINED GGML_LTO AND + (CMAKE_BUILD_TYPE STREQUAL "Release" OR + CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR + CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")) + set(GGML_LTO ON CACHE BOOL "" FORCE) +endif() + if(S2_AUTO_APPLY_LOCAL_PATCHES) if(WIN32) diff --git a/src/s2_pipeline.cpp b/src/s2_pipeline.cpp index 6526493..35d90df 100644 --- a/src/s2_pipeline.cpp +++ b/src/s2_pipeline.cpp @@ -863,14 +863,23 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con return false; } + // Offline synthesis has every frame in hand, so there is no latency reason + // to chunk the codec decode. Decoding all frames in a single pass is both + // faster (one graph build / allocation instead of one per window, and no + // re-decoding of the overlapping history region) and cleaner (no window + // boundary artifacts). Streaming uses the windowed path; offline defaults to + // a single shot unless the user explicitly sets a decode stride. + const bool user_set_stride = params.stream_decode_stride_frames > 0; const int32_t offline_decode_stride_frames = - params.stream_decode_stride_frames > 0 ? params.stream_decode_stride_frames : 16; + user_set_stride ? params.stream_decode_stride_frames : res.n_frames; + const int32_t offline_decode_context_frames = + user_set_stride ? params.codec_decode_context_frames : 0; double decode_ms = 0.0; int32_t decode_batches = 0; const auto decode_t0 = std::chrono::steady_clock::now(); if (!decode_codes_windowed(codec(), res.codes.data(), res.n_frames, num_codebooks, params.gen.n_threads, offline_decode_stride_frames, - params.codec_decode_context_frames, + offline_decode_context_frames, audio_out, &decode_ms, &decode_batches)) { safe_print_error_ln("Pipeline error: decode failed."); return false; diff --git a/src/s2_sampler.cpp b/src/s2_sampler.cpp index fd400da..0488a97 100755 --- a/src/s2_sampler.cpp +++ b/src/s2_sampler.cpp @@ -1,6 +1,7 @@ #include "../include/s2_sampler.h" #include #include +#include #include namespace s2 { @@ -22,50 +23,58 @@ static void apply_softmax(std::vector & probs, float temp = 1.0f) { } } -static std::vector softmax_from_sorted_logits(const std::vector> & items) { - std::vector probs(items.size(), 0.0f); - if (items.empty()) return probs; - - const float max_val = items.front().first; - float sum = 0.0f; - for (size_t i = 0; i < items.size(); ++i) { - probs[i] = std::exp(items[i].first - max_val); - sum += probs[i]; - } - - if (sum > 0.0f) { - for (float & p : probs) p /= sum; - } - return probs; -} - int32_t sample_token(const float * logits, int32_t vocab_size, const SamplerParams & params) { if (vocab_size <= 0) return 0; + const int32_t k = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size; + const float top_p = std::clamp(params.top_p, 0.0f, 1.0f); + const float NEG_INF = -std::numeric_limits::infinity(); + + // Single pass: gather finite candidates and the global max. Masked logits + // (set to -inf by the caller, e.g. the semantic mask covering ~95% of the + // vocab) contribute exp(-inf)=0 to the softmax and can never be selected, + // so skipping them is exact and avoids building/sorting the full vocab. std::vector> items; - items.reserve(vocab_size); + items.reserve(static_cast(std::min(vocab_size, 8192))); + float max_val = NEG_INF; for (int32_t i = 0; i < vocab_size; ++i) { - items.push_back({logits[i], i}); + const float v = logits[i]; + if (v == NEG_INF) continue; + items.push_back({v, i}); + if (v > max_val) max_val = v; + } + if (items.empty()) return 0; + + // Full partition function over the finite candidates. This matches the + // original code's denominator (the -inf terms it summed were exactly 0), + // so the top_p cumulative mass below is computed identically. + float Z = 0.0f; + for (const auto & it : items) Z += std::exp(it.first - max_val); + if (!(Z > 0.0f)) Z = 1.0f; + + // Order by logit descending, breaking ties by index for determinism. + auto cmp = [](const std::pair & a, const std::pair & b) { + if (a.first != b.first) return a.first > b.first; + return a.second < b.second; + }; + const int32_t kk = std::min(k, static_cast(items.size())); + if (kk < static_cast(items.size())) { + // top_k is small (default 30); only the top kk elements are ever kept, + // so a partial sort (O(n log kk)) replaces a full sort (O(n log n)). + std::partial_sort(items.begin(), items.begin() + kk, items.end(), cmp); + items.resize(kk); + } else { + std::sort(items.begin(), items.end(), cmp); } - std::sort(items.begin(), items.end(), [](const auto & a, const auto & b) { - return a.first > b.first; - }); - - const int32_t k = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size; - const float top_p = std::clamp(params.top_p, 0.0f, 1.0f); - const std::vector sorted_probs = softmax_from_sorted_logits(items); - + // Keep the prefix up to top_k and the top_p cumulative-probability cutoff. std::vector> filtered; - filtered.reserve(k); - + filtered.reserve(items.size()); float cumsum = 0.0f; for (int32_t i = 0; i < (int32_t)items.size(); ++i) { - cumsum += sorted_probs[i]; - const bool remove_for_top_k = (i >= k); - const bool remove_for_top_p = (i > 0 && cumsum > top_p); - if (remove_for_top_k || remove_for_top_p) { - continue; + cumsum += std::exp(items[i].first - max_val) / Z; + if (i > 0 && cumsum > top_p) { + break; // cumsum is monotonic, so every later item also exceeds top_p } filtered.push_back(items[i]); }