diff --git a/CMakeLists.txt b/CMakeLists.txt index f6d8a51..f115803 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,17 @@ endif() set(GGML_BUILD_TESTS OFF CACHE BOOL "" FORCE) set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +# Enable link-time optimization for optimized builds. ggml ships GGML_LTO=OFF +# by default; turning it on gives a few percent on the CPU compute kernels at +# no correctness cost. Only meaningful for optimized configs, and only if not +# already set by the user. +if(NOT DEFINED GGML_LTO AND + (CMAKE_BUILD_TYPE STREQUAL "Release" OR + CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR + CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")) + set(GGML_LTO ON CACHE BOOL "" FORCE) +endif() + if(S2_AUTO_APPLY_LOCAL_PATCHES) if(WIN32) diff --git a/include/s2_pipeline.h b/include/s2_pipeline.h index 19b3c3d..e4aaf11 100644 --- a/include/s2_pipeline.h +++ b/include/s2_pipeline.h @@ -31,6 +31,7 @@ struct PipelineParams { std::string prompt_text; std::string prompt_audio_path; std::string output_path; + std::string instruction; GenerateParams gen; int32_t gpu_device = -1; BackendType backend_type = BackendType::CPU; diff --git a/include/s2_prompt.h b/include/s2_prompt.h index 1fb36a1..5e5dd3e 100644 --- a/include/s2_prompt.h +++ b/include/s2_prompt.h @@ -20,7 +20,8 @@ PromptTensor build_prompt( const std::string & prompt_text, const int32_t * prompt_codes, int32_t num_codebooks, - int32_t T_prompt + int32_t T_prompt, + const std::string & instruction = "" ); } diff --git a/src/main.cpp b/src/main.cpp index d3e0b49..26ca102 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -66,6 +66,7 @@ void print_uso() { safe_print(" -text, --text Text to synthesize\n"); safe_print(" -pa, --prompt-audio Path to reference audio for cloning\n"); safe_print(" -pt, --prompt-text Text of the reference audio\n"); + safe_print(" --instruction Natural language instruction to control speaking style\n"); safe_print(" --voice Load a saved voice profile\n"); safe_print(" --save-voice Save the encoded reference as a voice profile\n"); safe_print(" --voice-dir Directory used for saved voice profiles\n"); @@ -160,6 +161,7 @@ int main(int argc, char** argv) { else if (arg == "-text" || arg == "--text") { if (i+1 < argc) params.text = argv[++i]; } else if (arg == "-pa" || arg == "--prompt-audio") { if (i+1 < argc) params.prompt_audio_path = argv[++i]; } else if (arg == "-pt" || arg == "--prompt-text") { if (i+1 < argc) params.prompt_text = argv[++i]; } + else if (arg == "--instruction") { if (i+1 < argc) params.instruction = argv[++i]; } else if (arg == "--voice") { if (i+1 < argc) params.voice_id = argv[++i]; } else if (arg == "--save-voice") { params.save_voice = true; } else if (arg == "--voice-dir") { if (i+1 < argc) params.voice_storage_dir = argv[++i]; } diff --git a/src/s2_codec.cpp b/src/s2_codec.cpp index e4dc3bc..f17bf7c 100755 --- a/src/s2_codec.cpp +++ b/src/s2_codec.cpp @@ -1046,7 +1046,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread enc_inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::encode] encoder compute failed." << std::endl; ggml_gallocr_free(allocr); @@ -1119,7 +1119,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread qenc_inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf2) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::encode] quantizer stage compute failed." << std::endl; ggml_gallocr_free(allocr2); @@ -1279,7 +1279,7 @@ static bool run_cached_decode_graph(AudioCodec::Impl & impl, const int32_t * cod } if (ggml_backend_is_cpu(impl.backend)) { - ggml_backend_cpu_set_n_threads(impl.backend, n_threads); + ggml_backend_cpu_set_n_threads(impl.backend, resolve_n_threads(n_threads)); } if (ggml_backend_graph_compute(impl.backend, cache.graph) != GGML_STATUS_SUCCESS) { @@ -1381,7 +1381,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa inp.mask_values.size() * sizeof(float)); } - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::decode] quantizer decode compute failed." << std::endl; @@ -1426,7 +1426,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa ggml_backend_tensor_set(latent_in, latent_out.data(), 0, latent_out.size() * sizeof(float)); - if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads); + if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads)); if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) { std::cerr << "[Codec::decode] decoder compute failed." << std::endl; diff --git a/src/s2_ggml_utils.h b/src/s2_ggml_utils.h index 1cb152a..fd855b9 100755 --- a/src/s2_ggml_utils.h +++ b/src/s2_ggml_utils.h @@ -3,9 +3,97 @@ #include "ggml.h" #include +#include +#include + +#if defined(__APPLE__) +# include +#elif defined(_WIN32) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif +# include +# include +#elif defined(__linux__) +# include +# include +# include +#endif namespace s2 { +// Number of physical "performance" cores available for compute. +// +// ggml's CPU backend scales poorly when oversubscribed onto hyperthreads or, +// on Apple Silicon, onto the slow efficiency cores: every thread in a fork/join +// matmul waits on the slowest one, so the whole step runs at efficiency-core +// speed. Measured on an M4 Pro (10P + 4E), defaulting to all 14 logical cores +// was ~2.9x slower end-to-end than capping at the 10 performance cores. +// +// This returns a sensible default core count; callers fall back to it only when +// the user did not pass an explicit --threads value. +inline int32_t detect_compute_threads() { +#if defined(__APPLE__) + // Performance-core count (perflevel0). Falls back to all physical cores. + int32_t n = 0; + size_t sz = sizeof(n); + if (sysctlbyname("hw.perflevel0.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) { + return n; + } + n = 0; sz = sizeof(n); + if (sysctlbyname("hw.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) { + return n; + } +#elif defined(_WIN32) + // Count physical cores via processor-core relationships. + DWORD len = 0; + GetLogicalProcessorInformation(nullptr, &len); + if (len > 0) { + std::vector buf( + len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION)); + if (GetLogicalProcessorInformation(buf.data(), &len)) { + int32_t cores = 0; + for (const auto & info : buf) { + if (info.Relationship == RelationProcessorCore) cores++; + } + if (cores > 0) return cores; + } + } +#elif defined(__linux__) + // Count distinct (physical_id, core_id) pairs from /proc/cpuinfo to skip + // SMT siblings. + if (FILE * f = std::fopen("/proc/cpuinfo", "r")) { + std::set> cores; + int phys = -1, core = -1; + char line[256]; + while (std::fgets(line, sizeof(line), f)) { + int v; + if (std::sscanf(line, "physical id : %d", &v) == 1) phys = v; + else if (std::sscanf(line, "core id : %d", &v) == 1) core = v; + if (line[0] == '\n') { // blank line separates logical CPUs + if (phys >= 0 && core >= 0) cores.insert({phys, core}); + phys = core = -1; + } + } + if (phys >= 0 && core >= 0) cores.insert({phys, core}); + std::fclose(f); + if (!cores.empty()) return static_cast(cores.size()); + } +#endif + // Generic fallback: assume SMT and halve the logical-core count, but never + // go below 1. + const unsigned hw = std::thread::hardware_concurrency(); + if (hw == 0) return 4; + return static_cast(hw >= 2 ? hw / 2 : hw); +} + +// Resolve a caller-supplied thread count: honor explicit positive values, +// otherwise pick a good default for this machine. +inline int32_t resolve_n_threads(int32_t n_threads) { + if (n_threads > 0) return n_threads; + return detect_compute_threads(); +} + inline ggml_tensor * repeat_checked(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b, const char * label = "repeat") { if (!ggml_can_repeat(a, b)) { diff --git a/src/s2_model.cpp b/src/s2_model.cpp index 9ced623..8515ad4 100755 --- a/src/s2_model.cpp +++ b/src/s2_model.cpp @@ -18,12 +18,6 @@ namespace s2 { -static int32_t resolve_n_threads(int32_t n_threads) { - if (n_threads > 0) return n_threads; - const auto hw = std::thread::hardware_concurrency(); - return static_cast(hw > 0 ? hw : 4); -} - static const char * backend_type_name(BackendType backend_type) { switch (backend_type) { case BackendType::CPU: return "CPU"; diff --git a/src/s2_pipeline.cpp b/src/s2_pipeline.cpp index 0dd5f01..35d90df 100644 --- a/src/s2_pipeline.cpp +++ b/src/s2_pipeline.cpp @@ -844,7 +844,7 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con PromptTensor prompt = build_prompt( tokenizer(), params.text, params.prompt_text, ref_codes, - num_codebooks, T_prompt); + num_codebooks, T_prompt, params.instruction); int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens; const auto kv_t0 = std::chrono::steady_clock::now(); @@ -863,14 +863,23 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con return false; } + // Offline synthesis has every frame in hand, so there is no latency reason + // to chunk the codec decode. Decoding all frames in a single pass is both + // faster (one graph build / allocation instead of one per window, and no + // re-decoding of the overlapping history region) and cleaner (no window + // boundary artifacts). Streaming uses the windowed path; offline defaults to + // a single shot unless the user explicitly sets a decode stride. + const bool user_set_stride = params.stream_decode_stride_frames > 0; const int32_t offline_decode_stride_frames = - params.stream_decode_stride_frames > 0 ? params.stream_decode_stride_frames : 16; + user_set_stride ? params.stream_decode_stride_frames : res.n_frames; + const int32_t offline_decode_context_frames = + user_set_stride ? params.codec_decode_context_frames : 0; double decode_ms = 0.0; int32_t decode_batches = 0; const auto decode_t0 = std::chrono::steady_clock::now(); if (!decode_codes_windowed(codec(), res.codes.data(), res.n_frames, num_codebooks, params.gen.n_threads, offline_decode_stride_frames, - params.codec_decode_context_frames, + offline_decode_context_frames, audio_out, &decode_ms, &decode_batches)) { safe_print_error_ln("Pipeline error: decode failed."); return false; @@ -982,7 +991,7 @@ bool Pipeline::synthesize_streaming_prompt_codes_locked(const PipelineParams & p PromptTensor prompt = build_prompt( tokenizer(), params.text, params.prompt_text, ref_codes, - num_codebooks, T_prompt); + num_codebooks, T_prompt, params.instruction); int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens; const auto kv_t0 = std::chrono::steady_clock::now(); diff --git a/src/s2_prompt.cpp b/src/s2_prompt.cpp index 80f6da7..c4cb442 100644 --- a/src/s2_prompt.cpp +++ b/src/s2_prompt.cpp @@ -8,7 +8,8 @@ PromptTensor build_prompt( const std::string & prompt_text, const int32_t * prompt_codes, int32_t num_codebooks, - int32_t T_prompt + int32_t T_prompt, + const std::string & instruction ) { PromptTensor result; result.rows = num_codebooks + 1; @@ -33,7 +34,11 @@ PromptTensor build_prompt( app(sys_pre, tokenizer.encode("<|im_start|>system")); app(sys_pre, NEWLINE); - app(sys_pre, tokenizer.encode("convert the provided text to speech reference to the following:\n\nText:\n")); + std::string sys_msg = "convert the provided text to speech reference to the following:\n\nText:\n"; + if (!instruction.empty()) { + sys_msg = instruction + "\n\n" + sys_msg; + } + app(sys_pre, tokenizer.encode(sys_msg)); if (!prompt_has_speaker_tag) { app(sys_pre, tokenizer.encode("<|speaker:0|>")); } @@ -57,7 +62,11 @@ PromptTensor build_prompt( app(sys_post, tokenizer.encode("<|im_start|>system")); app(sys_post, NEWLINE); - app(sys_post, tokenizer.encode("convert the provided text to speech")); + std::string sys_msg = "convert the provided text to speech"; + if (!instruction.empty()) { + sys_msg += ". " + instruction; + } + app(sys_post, tokenizer.encode(sys_msg)); app(sys_post, { im_end_id }); app(sys_post, NEWLINE); diff --git a/src/s2_sampler.cpp b/src/s2_sampler.cpp index fd400da..0488a97 100755 --- a/src/s2_sampler.cpp +++ b/src/s2_sampler.cpp @@ -1,6 +1,7 @@ #include "../include/s2_sampler.h" #include #include +#include #include namespace s2 { @@ -22,50 +23,58 @@ static void apply_softmax(std::vector & probs, float temp = 1.0f) { } } -static std::vector softmax_from_sorted_logits(const std::vector> & items) { - std::vector probs(items.size(), 0.0f); - if (items.empty()) return probs; - - const float max_val = items.front().first; - float sum = 0.0f; - for (size_t i = 0; i < items.size(); ++i) { - probs[i] = std::exp(items[i].first - max_val); - sum += probs[i]; - } - - if (sum > 0.0f) { - for (float & p : probs) p /= sum; - } - return probs; -} - int32_t sample_token(const float * logits, int32_t vocab_size, const SamplerParams & params) { if (vocab_size <= 0) return 0; + const int32_t k = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size; + const float top_p = std::clamp(params.top_p, 0.0f, 1.0f); + const float NEG_INF = -std::numeric_limits::infinity(); + + // Single pass: gather finite candidates and the global max. Masked logits + // (set to -inf by the caller, e.g. the semantic mask covering ~95% of the + // vocab) contribute exp(-inf)=0 to the softmax and can never be selected, + // so skipping them is exact and avoids building/sorting the full vocab. std::vector> items; - items.reserve(vocab_size); + items.reserve(static_cast(std::min(vocab_size, 8192))); + float max_val = NEG_INF; for (int32_t i = 0; i < vocab_size; ++i) { - items.push_back({logits[i], i}); + const float v = logits[i]; + if (v == NEG_INF) continue; + items.push_back({v, i}); + if (v > max_val) max_val = v; + } + if (items.empty()) return 0; + + // Full partition function over the finite candidates. This matches the + // original code's denominator (the -inf terms it summed were exactly 0), + // so the top_p cumulative mass below is computed identically. + float Z = 0.0f; + for (const auto & it : items) Z += std::exp(it.first - max_val); + if (!(Z > 0.0f)) Z = 1.0f; + + // Order by logit descending, breaking ties by index for determinism. + auto cmp = [](const std::pair & a, const std::pair & b) { + if (a.first != b.first) return a.first > b.first; + return a.second < b.second; + }; + const int32_t kk = std::min(k, static_cast(items.size())); + if (kk < static_cast(items.size())) { + // top_k is small (default 30); only the top kk elements are ever kept, + // so a partial sort (O(n log kk)) replaces a full sort (O(n log n)). + std::partial_sort(items.begin(), items.begin() + kk, items.end(), cmp); + items.resize(kk); + } else { + std::sort(items.begin(), items.end(), cmp); } - std::sort(items.begin(), items.end(), [](const auto & a, const auto & b) { - return a.first > b.first; - }); - - const int32_t k = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size; - const float top_p = std::clamp(params.top_p, 0.0f, 1.0f); - const std::vector sorted_probs = softmax_from_sorted_logits(items); - + // Keep the prefix up to top_k and the top_p cumulative-probability cutoff. std::vector> filtered; - filtered.reserve(k); - + filtered.reserve(items.size()); float cumsum = 0.0f; for (int32_t i = 0; i < (int32_t)items.size(); ++i) { - cumsum += sorted_probs[i]; - const bool remove_for_top_k = (i >= k); - const bool remove_for_top_p = (i > 0 && cumsum > top_p); - if (remove_for_top_k || remove_for_top_p) { - continue; + cumsum += std::exp(items[i].first - max_val) / Z; + if (i > 0 && cumsum > top_p) { + break; // cumsum is monotonic, so every later item also exceeds top_p } filtered.push_back(items[i]); } diff --git a/src/s2_server.cpp b/src/s2_server.cpp index 08fea2d..f86698c 100644 --- a/src/s2_server.cpp +++ b/src/s2_server.cpp @@ -577,6 +577,10 @@ namespace s2 voice_dir_value = j["voice_dir"].get(); } + if (j.contains("instruction")) { + pipelineParams.instruction = j["instruction"].get(); + } + if (j.contains("stream_decode_stride_frames")) { int32_t val = j["stream_decode_stride_frames"].get(); pipelineParams.stream_decode_stride_frames = std::max(0, val);