Perf/cpu thread default

CMakeLists.txt

@@ -29,6 +29,17 @@ endif()
 set(GGML_BUILD_TESTS  OFF CACHE BOOL "" FORCE)
 set(GGML_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
 
+# Enable link-time optimization for optimized builds. ggml ships GGML_LTO=OFF
+# by default; turning it on gives a few percent on the CPU compute kernels at
+# no correctness cost. Only meaningful for optimized configs, and only if not
+# already set by the user.
+if(NOT DEFINED GGML_LTO AND
+   (CMAKE_BUILD_TYPE STREQUAL "Release" OR
+    CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR
+    CMAKE_BUILD_TYPE STREQUAL "MinSizeRel"))
+    set(GGML_LTO ON CACHE BOOL "" FORCE)
+endif()
+
 if(S2_AUTO_APPLY_LOCAL_PATCHES)
 
 	if(WIN32)

include/s2_pipeline.h

@@ -31,6 +31,7 @@ struct PipelineParams {
     std::string prompt_text;
     std::string prompt_audio_path;
     std::string output_path;
+    std::string instruction;
     GenerateParams gen;
     int32_t gpu_device = -1;
     BackendType backend_type = BackendType::CPU;

include/s2_prompt.h

@@ -20,7 +20,8 @@ PromptTensor build_prompt(
     const std::string & prompt_text,
     const int32_t * prompt_codes,
     int32_t num_codebooks,
-    int32_t T_prompt
+    int32_t T_prompt,
+    const std::string & instruction = ""
 );
 
 }

src/main.cpp

@@ -66,6 +66,7 @@ void print_uso() {
     safe_print("  -text, --text      <text>   Text to synthesize\n");
     safe_print("  -pa, --prompt-audio <path>  Path to reference audio for cloning\n");
     safe_print("  -pt, --prompt-text <text>   Text of the reference audio\n");
+    safe_print("  --instruction      <text>   Natural language instruction to control speaking style\n");
     safe_print("  --voice            <id>     Load a saved voice profile\n");
     safe_print("  --save-voice                Save the encoded reference as a voice profile\n");
     safe_print("  --voice-dir        <path>   Directory used for saved voice profiles\n");
@@ -160,6 +161,7 @@ int main(int argc, char** argv) {
         else if (arg == "-text" || arg == "--text")       { if (i+1 < argc) params.text               = argv[++i]; }
         else if (arg == "-pa" || arg == "--prompt-audio") { if (i+1 < argc) params.prompt_audio_path = argv[++i]; }
         else if (arg == "-pt" || arg == "--prompt-text")  { if (i+1 < argc) params.prompt_text        = argv[++i]; }
+        else if (arg == "--instruction")                  { if (i+1 < argc) params.instruction         = argv[++i]; }
         else if (arg == "--voice")                        { if (i+1 < argc) params.voice_id           = argv[++i]; }
         else if (arg == "--save-voice")                   { params.save_voice = true; }
         else if (arg == "--voice-dir")                    { if (i+1 < argc) params.voice_storage_dir  = argv[++i]; }

src/s2_codec.cpp

@@ -1046,7 +1046,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread
                                 enc_inp.mask_values.size() * sizeof(float));
     }
 
-    if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads);
+    if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads));
     if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) {
         std::cerr << "[Codec::encode] encoder compute failed." << std::endl;
         ggml_gallocr_free(allocr);
@@ -1119,7 +1119,7 @@ bool AudioCodec::encode(const float * audio, int32_t n_samples, int32_t n_thread
                                     qenc_inp.mask_values.size() * sizeof(float));
         }
 
-        if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads);
+        if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads));
         if (ggml_backend_graph_compute(impl_->backend, gf2) != GGML_STATUS_SUCCESS) {
             std::cerr << "[Codec::encode] quantizer stage compute failed." << std::endl;
             ggml_gallocr_free(allocr2);
@@ -1279,7 +1279,7 @@ static bool run_cached_decode_graph(AudioCodec::Impl & impl, const int32_t * cod
     }
 
     if (ggml_backend_is_cpu(impl.backend)) {
-        ggml_backend_cpu_set_n_threads(impl.backend, n_threads);
+        ggml_backend_cpu_set_n_threads(impl.backend, resolve_n_threads(n_threads));
     }
 
     if (ggml_backend_graph_compute(impl.backend, cache.graph) != GGML_STATUS_SUCCESS) {
@@ -1381,7 +1381,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa
                 inp.mask_values.size() * sizeof(float));
         }
 
-        if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads);
+        if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads));
 
         if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) {
             std::cerr << "[Codec::decode] quantizer decode compute failed." << std::endl;
@@ -1426,7 +1426,7 @@ bool AudioCodec::decode(const int32_t * codes, int32_t n_frames, int32_t n_threa
 
     ggml_backend_tensor_set(latent_in, latent_out.data(), 0, latent_out.size() * sizeof(float));
 
-    if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, n_threads);
+    if (ggml_backend_is_cpu(impl_->backend)) ggml_backend_cpu_set_n_threads(impl_->backend, resolve_n_threads(n_threads));
 
     if (ggml_backend_graph_compute(impl_->backend, gf) != GGML_STATUS_SUCCESS) {
             std::cerr << "[Codec::decode] decoder compute failed." << std::endl;

src/s2_ggml_utils.h

@@ -3,9 +3,97 @@
 #include "ggml.h"
 
 #include <cstdio>
+#include <cstdint>
+#include <thread>
+
+#if defined(__APPLE__)
+#  include <sys/sysctl.h>
+#elif defined(_WIN32)
+#  ifndef WIN32_LEAN_AND_MEAN
+#    define WIN32_LEAN_AND_MEAN
+#  endif
+#  include <windows.h>
+#  include <vector>
+#elif defined(__linux__)
+#  include <cstdio>
+#  include <set>
+#  include <utility>
+#endif
 
 namespace s2 {
 
+// Number of physical "performance" cores available for compute.
+//
+// ggml's CPU backend scales poorly when oversubscribed onto hyperthreads or,
+// on Apple Silicon, onto the slow efficiency cores: every thread in a fork/join
+// matmul waits on the slowest one, so the whole step runs at efficiency-core
+// speed. Measured on an M4 Pro (10P + 4E), defaulting to all 14 logical cores
+// was ~2.9x slower end-to-end than capping at the 10 performance cores.
+//
+// This returns a sensible default core count; callers fall back to it only when
+// the user did not pass an explicit --threads value.
+inline int32_t detect_compute_threads() {
+#if defined(__APPLE__)
+    // Performance-core count (perflevel0). Falls back to all physical cores.
+    int32_t n = 0;
+    size_t  sz = sizeof(n);
+    if (sysctlbyname("hw.perflevel0.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) {
+        return n;
+    }
+    n = 0; sz = sizeof(n);
+    if (sysctlbyname("hw.physicalcpu", &n, &sz, nullptr, 0) == 0 && n > 0) {
+        return n;
+    }
+#elif defined(_WIN32)
+    // Count physical cores via processor-core relationships.
+    DWORD len = 0;
+    GetLogicalProcessorInformation(nullptr, &len);
+    if (len > 0) {
+        std::vector<SYSTEM_LOGICAL_PROCESSOR_INFORMATION> buf(
+            len / sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION));
+        if (GetLogicalProcessorInformation(buf.data(), &len)) {
+            int32_t cores = 0;
+            for (const auto & info : buf) {
+                if (info.Relationship == RelationProcessorCore) cores++;
+            }
+            if (cores > 0) return cores;
+        }
+    }
+#elif defined(__linux__)
+    // Count distinct (physical_id, core_id) pairs from /proc/cpuinfo to skip
+    // SMT siblings.
+    if (FILE * f = std::fopen("/proc/cpuinfo", "r")) {
+        std::set<std::pair<int, int>> cores;
+        int phys = -1, core = -1;
+        char line[256];
+        while (std::fgets(line, sizeof(line), f)) {
+            int v;
+            if (std::sscanf(line, "physical id : %d", &v) == 1) phys = v;
+            else if (std::sscanf(line, "core id : %d", &v) == 1) core = v;
+            if (line[0] == '\n') { // blank line separates logical CPUs
+                if (phys >= 0 && core >= 0) cores.insert({phys, core});
+                phys = core = -1;
+            }
+        }
+        if (phys >= 0 && core >= 0) cores.insert({phys, core});
+        std::fclose(f);
+        if (!cores.empty()) return static_cast<int32_t>(cores.size());
+    }
+#endif
+    // Generic fallback: assume SMT and halve the logical-core count, but never
+    // go below 1.
+    const unsigned hw = std::thread::hardware_concurrency();
+    if (hw == 0) return 4;
+    return static_cast<int32_t>(hw >= 2 ? hw / 2 : hw);
+}
+
+// Resolve a caller-supplied thread count: honor explicit positive values,
+// otherwise pick a good default for this machine.
+inline int32_t resolve_n_threads(int32_t n_threads) {
+    if (n_threads > 0) return n_threads;
+    return detect_compute_threads();
+}
+
 inline ggml_tensor * repeat_checked(ggml_context * ctx, ggml_tensor * a, ggml_tensor * b,
                                     const char * label = "repeat") {
     if (!ggml_can_repeat(a, b)) {

src/s2_model.cpp

@@ -18,12 +18,6 @@
 
 namespace s2 {
 
-static int32_t resolve_n_threads(int32_t n_threads) {
-    if (n_threads > 0) return n_threads;
-    const auto hw = std::thread::hardware_concurrency();
-    return static_cast<int32_t>(hw > 0 ? hw : 4);
-}
-
 static const char * backend_type_name(BackendType backend_type) {
     switch (backend_type) {
         case BackendType::CPU:    return "CPU";

src/s2_pipeline.cpp

@@ -844,7 +844,7 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con
     PromptTensor prompt = build_prompt(
         tokenizer(), params.text, params.prompt_text,
         ref_codes,
-        num_codebooks, T_prompt);
+        num_codebooks, T_prompt, params.instruction);
 
     int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens;
     const auto kv_t0 = std::chrono::steady_clock::now();
@@ -863,14 +863,23 @@ bool Pipeline::synthesize_prompt_codes_locked(const PipelineParams & params, con
         return false;
     }
 
+    // Offline synthesis has every frame in hand, so there is no latency reason
+    // to chunk the codec decode. Decoding all frames in a single pass is both
+    // faster (one graph build / allocation instead of one per window, and no
+    // re-decoding of the overlapping history region) and cleaner (no window
+    // boundary artifacts). Streaming uses the windowed path; offline defaults to
+    // a single shot unless the user explicitly sets a decode stride.
+    const bool user_set_stride = params.stream_decode_stride_frames > 0;
     const int32_t offline_decode_stride_frames =
-        params.stream_decode_stride_frames > 0 ? params.stream_decode_stride_frames : 16;
+        user_set_stride ? params.stream_decode_stride_frames : res.n_frames;
+    const int32_t offline_decode_context_frames =
+        user_set_stride ? params.codec_decode_context_frames : 0;
     double decode_ms = 0.0;
     int32_t decode_batches = 0;
     const auto decode_t0 = std::chrono::steady_clock::now();
     if (!decode_codes_windowed(codec(), res.codes.data(), res.n_frames, num_codebooks,
                                params.gen.n_threads, offline_decode_stride_frames,
-                               params.codec_decode_context_frames,
+                               offline_decode_context_frames,
                                audio_out, &decode_ms, &decode_batches)) {
         safe_print_error_ln("Pipeline error: decode failed.");
         return false;
@@ -982,7 +991,7 @@ bool Pipeline::synthesize_streaming_prompt_codes_locked(const PipelineParams & p
     PromptTensor prompt = build_prompt(
         tokenizer(), params.text, params.prompt_text,
         ref_codes,
-        num_codebooks, T_prompt);
+        num_codebooks, T_prompt, params.instruction);
 
     int32_t max_seq_len = prompt.cols + params.gen.max_new_tokens;
     const auto kv_t0 = std::chrono::steady_clock::now();

src/s2_prompt.cpp

@@ -8,7 +8,8 @@ PromptTensor build_prompt(
     const std::string & prompt_text,
     const int32_t * prompt_codes,
     int32_t num_codebooks,
-    int32_t T_prompt
+    int32_t T_prompt,
+    const std::string & instruction
 ) {
     PromptTensor result;
     result.rows = num_codebooks + 1;
@@ -33,7 +34,11 @@ PromptTensor build_prompt(
 
         app(sys_pre, tokenizer.encode("<|im_start|>system"));
         app(sys_pre, NEWLINE);
-        app(sys_pre, tokenizer.encode("convert the provided text to speech reference to the following:\n\nText:\n"));
+        std::string sys_msg = "convert the provided text to speech reference to the following:\n\nText:\n";
+        if (!instruction.empty()) {
+            sys_msg = instruction + "\n\n" + sys_msg;
+        }
+        app(sys_pre, tokenizer.encode(sys_msg));
         if (!prompt_has_speaker_tag) {
             app(sys_pre, tokenizer.encode("<|speaker:0|>"));
         }
@@ -57,7 +62,11 @@ PromptTensor build_prompt(
 
         app(sys_post, tokenizer.encode("<|im_start|>system"));
         app(sys_post, NEWLINE);
-        app(sys_post, tokenizer.encode("convert the provided text to speech"));
+        std::string sys_msg = "convert the provided text to speech";
+        if (!instruction.empty()) {
+            sys_msg += ". " + instruction;
+        }
+        app(sys_post, tokenizer.encode(sys_msg));
         app(sys_post, { im_end_id });
         app(sys_post, NEWLINE);
 

src/s2_sampler.cpp

@@ -1,6 +1,7 @@
 #include "../include/s2_sampler.h"
 #include <cmath>
 #include <algorithm>
+#include <limits>
 #include <random>
 
 namespace s2 {
@@ -22,50 +23,58 @@ static void apply_softmax(std::vector<float> & probs, float temp = 1.0f) {
     }
 }
 
-static std::vector<float> softmax_from_sorted_logits(const std::vector<std::pair<float, int32_t>> & items) {
-    std::vector<float> probs(items.size(), 0.0f);
-    if (items.empty()) return probs;
-
-    const float max_val = items.front().first;
-    float sum = 0.0f;
-    for (size_t i = 0; i < items.size(); ++i) {
-        probs[i] = std::exp(items[i].first - max_val);
-        sum += probs[i];
-    }
-
-    if (sum > 0.0f) {
-        for (float & p : probs) p /= sum;
-    }
-    return probs;
-}
-
 int32_t sample_token(const float * logits, int32_t vocab_size, const SamplerParams & params) {
     if (vocab_size <= 0) return 0;
 
+    const int32_t k     = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size;
+    const float   top_p = std::clamp(params.top_p, 0.0f, 1.0f);
+    const float   NEG_INF = -std::numeric_limits<float>::infinity();
+
+    // Single pass: gather finite candidates and the global max. Masked logits
+    // (set to -inf by the caller, e.g. the semantic mask covering ~95% of the
+    // vocab) contribute exp(-inf)=0 to the softmax and can never be selected,
+    // so skipping them is exact and avoids building/sorting the full vocab.
     std::vector<std::pair<float, int32_t>> items;
-    items.reserve(vocab_size);
+    items.reserve(static_cast<size_t>(std::min(vocab_size, 8192)));
+    float max_val = NEG_INF;
     for (int32_t i = 0; i < vocab_size; ++i) {
-        items.push_back({logits[i], i});
+        const float v = logits[i];
+        if (v == NEG_INF) continue;
+        items.push_back({v, i});
+        if (v > max_val) max_val = v;
+    }
+    if (items.empty()) return 0;
+
+    // Full partition function over the finite candidates. This matches the
+    // original code's denominator (the -inf terms it summed were exactly 0),
+    // so the top_p cumulative mass below is computed identically.
+    float Z = 0.0f;
+    for (const auto & it : items) Z += std::exp(it.first - max_val);
+    if (!(Z > 0.0f)) Z = 1.0f;
+
+    // Order by logit descending, breaking ties by index for determinism.
+    auto cmp = [](const std::pair<float, int32_t> & a, const std::pair<float, int32_t> & b) {
+        if (a.first != b.first) return a.first > b.first;
+        return a.second < b.second;
+    };
+    const int32_t kk = std::min<int32_t>(k, static_cast<int32_t>(items.size()));
+    if (kk < static_cast<int32_t>(items.size())) {
+        // top_k is small (default 30); only the top kk elements are ever kept,
+        // so a partial sort (O(n log kk)) replaces a full sort (O(n log n)).
+        std::partial_sort(items.begin(), items.begin() + kk, items.end(), cmp);
+        items.resize(kk);
+    } else {
+        std::sort(items.begin(), items.end(), cmp);
     }
 
-    std::sort(items.begin(), items.end(), [](const auto & a, const auto & b) {
-        return a.first > b.first;
-    });
-
-    const int32_t k = params.top_k > 0 ? std::min(params.top_k, vocab_size) : vocab_size;
-    const float top_p = std::clamp(params.top_p, 0.0f, 1.0f);
-    const std::vector<float> sorted_probs = softmax_from_sorted_logits(items);
-
+    // Keep the prefix up to top_k and the top_p cumulative-probability cutoff.
     std::vector<std::pair<float, int32_t>> filtered;
-    filtered.reserve(k);
-
+    filtered.reserve(items.size());
     float cumsum = 0.0f;
     for (int32_t i = 0; i < (int32_t)items.size(); ++i) {
-        cumsum += sorted_probs[i];
-        const bool remove_for_top_k = (i >= k);
-        const bool remove_for_top_p = (i > 0 && cumsum > top_p);
-        if (remove_for_top_k || remove_for_top_p) {
-            continue;
+        cumsum += std::exp(items[i].first - max_val) / Z;
+        if (i > 0 && cumsum > top_p) {
+            break; // cumsum is monotonic, so every later item also exceeds top_p
         }
         filtered.push_back(items[i]);
     }

src/s2_server.cpp

@@ -577,6 +577,10 @@ namespace s2
                             voice_dir_value = j["voice_dir"].get<std::string>();
                         }
 
+                        if (j.contains("instruction")) {
+                            pipelineParams.instruction = j["instruction"].get<std::string>();
+                        }
+
                         if (j.contains("stream_decode_stride_frames")) {
                             int32_t val = j["stream_decode_stride_frames"].get<int32_t>();
                             pipelineParams.stream_decode_stride_frames = std::max(0, val);