From 06ac7bcd7ec527a6b8f06c80e01ada7ef86378e2 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 11 Jul 2026 12:40:44 +0100 Subject: [PATCH 1/6] audio: ffmpeg_dec: add FFmpeg (libavcodec/libavfilter) module Add a loadable (LLEXT) audio module that wraps FFmpeg. Two build modes: - decoder (default): a compressed elementary stream (FLAC/AAC/Opus, selected via Kconfig) is parsed and decoded to PCM (process_raw_data); - filter (CONFIG_FFMPEG_DEC_FILTER_MODE): a PCM source/sink effect that runs an FFmpeg audio filter graph (afftdn noise reduction). FFmpeg is a west-pinned source (see west.yml) cross-built by ffmpeg.cmake as a CMake ExternalProject, enabling only the Kconfig-selected decoders and filters. The module supplies the libc surface FFmpeg needs that the SOF core does not export to LLEXT (ffmpeg_dec-shims.c), a SOF-heap-backed malloc (ffmpeg_dec-alloc.c), fast single-precision float math (fastmathf.c), and routes av_log() into the Zephyr log. The avfilter graph backend and the PCM effect ops are in ffmpeg_dec-filter.c. Also adds the rimage module manifest (ffmpeg_dec.toml + per-platform includes), the topology widget (ffmpeg_dec.conf), host test tooling (ffmpeg_dec_prepare.sh) and docs (README/TESTING). Verified building and load-ready (all symbols resolved, signed) for ace30 (ptl) with the Zephyr SDK: FLAC and FLAC+AAC decoders, and the afftdn filter effect. Runtime decode/denoise verification is pending hardware. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/audio/CMakeLists.txt | 3 + src/audio/Kconfig | 1 + src/audio/ffmpeg_dec/CMakeLists.txt | 18 + src/audio/ffmpeg_dec/Kconfig | 97 ++++ src/audio/ffmpeg_dec/README.md | 59 +++ src/audio/ffmpeg_dec/TESTING.md | 66 +++ src/audio/ffmpeg_dec/fastmathf.c | 257 ++++++++++ src/audio/ffmpeg_dec/ffmpeg.cmake | 132 ++++++ src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c | 79 ++++ src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c | 342 ++++++++++++++ src/audio/ffmpeg_dec/ffmpeg_dec-filter.c | 281 +++++++++++ src/audio/ffmpeg_dec/ffmpeg_dec-shims.c | 441 ++++++++++++++++++ src/audio/ffmpeg_dec/ffmpeg_dec-stub.c | 65 +++ src/audio/ffmpeg_dec/ffmpeg_dec.c | 322 +++++++++++++ src/audio/ffmpeg_dec/ffmpeg_dec.h | 151 ++++++ src/audio/ffmpeg_dec/ffmpeg_dec.toml | 25 + src/audio/ffmpeg_dec/llext/CMakeLists.txt | 62 +++ src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld | 24 + src/audio/ffmpeg_dec/llext/llext.toml.h | 6 + src/include/sof/audio/component.h | 1 + tools/rimage/config/lnl.toml.h | 3 + tools/rimage/config/mtl.toml.h | 3 + tools/rimage/config/nvl.toml.h | 3 + tools/rimage/config/ptl.toml.h | 3 + tools/rimage/config/tgl-h.toml.h | 3 + tools/rimage/config/tgl.toml.h | 3 + tools/rimage/config/wcl.toml.h | 3 + tools/test/audio/ffmpeg_dec_prepare.sh | 81 ++++ .../include/components/ffmpeg_dec.conf | 115 +++++ uuid-registry.txt | 1 + west.yml | 16 + 31 files changed, 2666 insertions(+) create mode 100644 src/audio/ffmpeg_dec/CMakeLists.txt create mode 100644 src/audio/ffmpeg_dec/Kconfig create mode 100644 src/audio/ffmpeg_dec/README.md create mode 100644 src/audio/ffmpeg_dec/TESTING.md create mode 100644 src/audio/ffmpeg_dec/fastmathf.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg.cmake create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec-filter.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec-shims.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec-stub.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec.c create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec.h create mode 100644 src/audio/ffmpeg_dec/ffmpeg_dec.toml create mode 100644 src/audio/ffmpeg_dec/llext/CMakeLists.txt create mode 100644 src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld create mode 100644 src/audio/ffmpeg_dec/llext/llext.toml.h create mode 100755 tools/test/audio/ffmpeg_dec_prepare.sh create mode 100644 tools/topology/topology2/include/components/ffmpeg_dec.conf diff --git a/src/audio/CMakeLists.txt b/src/audio/CMakeLists.txt index 92002c8b7c1c..18da047ad77c 100644 --- a/src/audio/CMakeLists.txt +++ b/src/audio/CMakeLists.txt @@ -41,6 +41,9 @@ if(NOT CONFIG_COMP_MODULE_SHARED_LIBRARY_BUILD) if(CONFIG_COMP_DRC) add_subdirectory(drc) endif() + if(CONFIG_COMP_FFMPEG_DEC) + add_subdirectory(ffmpeg_dec) + endif() if(CONFIG_COMP_FIR) add_subdirectory(eq_fir) endif() diff --git a/src/audio/Kconfig b/src/audio/Kconfig index 8accb25738a2..e8cfb8c0cbe7 100644 --- a/src/audio/Kconfig +++ b/src/audio/Kconfig @@ -139,6 +139,7 @@ rsource "dcblock/Kconfig" rsource "drc/Kconfig" rsource "eq_fir/Kconfig" rsource "eq_iir/Kconfig" +rsource "ffmpeg_dec/Kconfig" rsource "google/Kconfig" rsource "igo_nr/Kconfig" rsource "mfcc/Kconfig" diff --git a/src/audio/ffmpeg_dec/CMakeLists.txt b/src/audio/ffmpeg_dec/CMakeLists.txt new file mode 100644 index 000000000000..a830a1e5fead --- /dev/null +++ b/src/audio/ffmpeg_dec/CMakeLists.txt @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: BSD-3-Clause + +if(CONFIG_COMP_FFMPEG_DEC STREQUAL "m" AND DEFINED CONFIG_LLEXT) + add_subdirectory(llext ${PROJECT_BINARY_DIR}/ffmpeg_dec_llext) + add_dependencies(app ffmpeg_dec) +else() + add_local_sources(sof ffmpeg_dec.c) + + if(CONFIG_COMP_FFMPEG_DEC_STUB) + add_local_sources(sof ffmpeg_dec-stub.c) + else() + add_local_sources(sof ffmpeg_dec-ffmpeg.c) + # NOTE: ffmpeg_dec-shims.c / ffmpeg_dec-alloc.c are LLEXT-only (see + # llext/CMakeLists.txt). They provide libc/libm symbols the isolated LLEXT + # cannot import; in a built-in (=y) or native/testbench build the real libc + # provides those, so including them here would multiply-define malloc etc. + endif() +endif() diff --git a/src/audio/ffmpeg_dec/Kconfig b/src/audio/ffmpeg_dec/Kconfig new file mode 100644 index 000000000000..19203fc23064 --- /dev/null +++ b/src/audio/ffmpeg_dec/Kconfig @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config COMP_FFMPEG_DEC + tristate "FFmpeg (libavcodec) audio decoder" + select COMP_FFMPEG_DEC_STUB if COMP_STUBS + help + Select to include the FFmpeg audio decoder module. It wraps + libavcodec decoders behind the SOF module interface, decoding a + compressed elementary stream to PCM. The real backend cross-builds + libavcodec/libavutil/libswresample from the FFmpeg source pulled in + by west (see west.yml), enabling only the decoders selected below. + Without the source (or for CI) select COMP_FFMPEG_DEC_STUB to build + the dependency-free passthrough stub. + +config COMP_FFMPEG_DEC_STUB + bool "FFmpeg decoder stub backend" + depends on COMP_FFMPEG_DEC + help + Build the ffmpeg_dec module against a dependency-free passthrough + backend instead of libavcodec. Intended for testing and CI so the + SOF glue and LLEXT packaging can be validated without the FFmpeg + source or a cross-built archive. + +if COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB + +comment "FFmpeg decoders to build (footprint scales with selection)" + +config FFMPEG_DEC_FLAC + bool "FLAC decoder" + default y + help + Lossless integer decoder. Smallest footprint, no floating-point + math. Enables --enable-decoder=flac in the libavcodec build. + +config FFMPEG_DEC_AAC + bool "AAC-LC decoder" + help + AAC Low Complexity decoder. Pulls in the single-precision float + math layer (see FFMPEG_DEC_FLOAT_MATH). Enables + --enable-decoder=aac in the libavcodec build. + +config FFMPEG_DEC_OPUS + bool "Opus decoder" + help + Opus (SILK+CELT) decoder. Pulls in the single-precision float math + layer. Enables --enable-decoder=opus in the libavcodec build. + +comment "FFmpeg audio filters (libavfilter; need the avfilter-graph backend to run)" + +config FFMPEG_FILTER_AFFTDN + bool "FFT noise reduction (afftdn)" + help + Build FFmpeg's FFT-based denoiser (afftdn) into libavfilter. Pure DSP, + no external model file (unlike arnndn). Enabling any filter turns on + libavfilter in the cross-build. NOTE: a filter is only usable at runtime + once the module gains an avfilter-graph backend; selecting it here just + builds the filter into the archive. + +config FFMPEG_DEC_FILTER_MODE + bool "Build as a PCM filter effect instead of a decoder [EXPERIMENTAL]" + select FFMPEG_FILTER_AFFTDN + help + Build the module as a PCM source->sink effect that runs an FFmpeg + audio filter graph (default afftdn noise reduction) via the modern + .process interface, instead of the compressed-stream decoder. Turns on + libavfilter and the fast float math. Structurally complete and + load-ready; real-time latency/format tuning needs on-hardware bring-up. + +# Turns on libavfilter in the FFmpeg cross-build when any filter is selected. +config FFMPEG_BUILD_AVFILTER + bool + default y if FFMPEG_FILTER_AFFTDN + +# Selected automatically when a decoder or filter that needs IEEE float math is +# built. Gates compilation of the fast single-precision libm (fastmathf.c). +config FFMPEG_DEC_FLOAT_MATH + bool + default y if FFMPEG_DEC_AAC || FFMPEG_DEC_OPUS || FFMPEG_FILTER_AFFTDN + +config FFMPEG_DEC_COLD_SPLIT + bool "Place cold FFmpeg code (av_cold init/setup) in DRAM [EXPERIMENTAL]" + depends on COLD_STORE_EXECUTE_DRAM + default n + help + EXPERIMENTAL. Build libavcodec with -mtext-section-literals and rename + FFmpeg's cold functions (its av_cold init/table-generation code, which GCC + emits into .text.unlikely) into SOF's .cold section so the LLEXT loader + places them in DRAM, keeping the hot per-frame decode in fast SRAM. + + Measured cold code is small (~1 KB for FLAC, ~26 KB for AAC) because + FFmpeg only marks init functions av_cold. Placing that much cold code in a + -shared LLEXT currently hits Xtensa linker limits (cold->hot call8 range + needs -mlongcalls; the orphan .cold lands at a bad LMA and overlaps .hash), + so this is off by default pending proper cold-region placement in the LLEXT + memory map. See git history / TESTING notes. + +endif # COMP_FFMPEG_DEC && !COMP_FFMPEG_DEC_STUB diff --git a/src/audio/ffmpeg_dec/README.md b/src/audio/ffmpeg_dec/README.md new file mode 100644 index 000000000000..2dc8203d029f --- /dev/null +++ b/src/audio/ffmpeg_dec/README.md @@ -0,0 +1,59 @@ +# FFmpeg audio decoder module (ffmpeg_dec) + +Wraps FFmpeg's `libavcodec` audio decoders behind the SOF `module_interface`, +decoding a compressed elementary stream to PCM inside the DSP. First target +codec is **FLAC**. + +## Design + +The SOF glue and the decoder are separated so the integration can be validated +without libavcodec: + +- `ffmpeg_dec.c` — SOF module core: `init` / `prepare` / `process_raw_data` / + `set_configuration` / `reset` / `free`, plus the LLEXT manifest. Input is the + compressed byte stream (raw data), output is interleaved PCM. Decoding is + delegated to a `struct ffmpeg_dec_backend`. +- `ffmpeg_dec-stub.c` — dependency-free passthrough backend. Lets CI build and + exercise the module (pipeline, IPC config, LLEXT packaging) with **no** FFmpeg + libraries. Selected by `CONFIG_COMP_FFMPEG_DEC_STUB`. +- `ffmpeg_dec-ffmpeg.c` — real backend driving the standalone `libavcodec` + send-packet / receive-frame API. A raw stream is framed on-DSP with an + `AVCodecParser` (`av_parser_parse2`); decode is forced single-threaded + (`thread_count = 1`). Requires pre-compiled static libraries and headers under + `third_party/` (see below). +- `ffmpeg_dec.h` — private data (`struct ffmpeg_dec_comp_data`) and the backend + interface. + +The codec setup header (e.g. FLAC STREAMINFO) is delivered as a binary control +via `set_configuration` and stored as `extradata`, which the libavcodec backend +installs before `avcodec_open2()`. + +## Build + +- **Stub (default for testing/CI):** `CONFIG_COMP_FFMPEG_DEC=y` (or `=m` for + LLEXT) with `CONFIG_COMP_FFMPEG_DEC_STUB=y`. No external dependencies. +- **Real decoder:** `CONFIG_COMP_FFMPEG_DEC=m`, `CONFIG_COMP_FFMPEG_DEC_STUB=n`. + Requires cross-compiled decoder-only FFmpeg static libraries installed as: + - `third_party/lib/libavcodec.a`, `libavutil.a`, `libswresample.a` + - `third_party/include/libavcodec/…`, `libavutil/…`, `libswresample/…` + Configure FFmpeg with `--disable-everything --disable-avformat + --disable-pthreads --enable-decoder=flac --enable-parser=flac` (plus the + target cross-compile flags). + +## Files + +- `Kconfig` — `COMP_FFMPEG_DEC` (tristate) and `COMP_FFMPEG_DEC_STUB`. +- `CMakeLists.txt` / `llext/CMakeLists.txt` — static and LLEXT builds; the LLEXT + target name is `ffmpeg_dec` and links the FFmpeg libraries in the non-stub + branch. +- `ffmpeg_dec.toml` / `llext/llext.toml.h` — rimage module manifest + (`UUIDREG_STR_FFMPEG_DEC`); OBS is sized above IBS since PCM out ≫ compressed + in. + +## Status / TODO + +- Codec id is currently hard-coded to FLAC in `init`; wire it from topology/IPC + init config for other codecs. +- Output is assumed to already match the sink sample format; add + `libswresample` (or a fixed-point path) for format/rate conversion. +- Topology and host test tooling for end-to-end (bit-exact) FLAC decode. diff --git a/src/audio/ffmpeg_dec/TESTING.md b/src/audio/ffmpeg_dec/TESTING.md new file mode 100644 index 000000000000..3d50ed8d84f6 --- /dev/null +++ b/src/audio/ffmpeg_dec/TESTING.md @@ -0,0 +1,66 @@ +# Testing the ffmpeg_dec module + +This describes how to verify the FFmpeg FLAC decoder module end to end. FLAC is +lossless, so a correct decode is **bit-exact** against a reference decode. + +## Test vectors + +Generate vectors with the helper (needs `ffmpeg` on the host): + +``` +tools/test/audio/ffmpeg_dec_prepare.sh -i song.wav -o vectors/ +``` + +Produces in `vectors/`: +- `streaminfo.bin` — 34-byte FLAC STREAMINFO = the codec **extradata** blob to + load into the module's bytes control (`set_configuration`). +- `stream.raw` — the raw FLAC elementary stream fed to the decoder input. +- `ref_s32le.raw` — reference interleaved S32_LE PCM; the decoder output must + match this byte-for-byte. + +## Topology + +The component widget is `tools/topology/topology2/include/components/ffmpeg_dec.conf` +(models the DTS codec: `type "effect"`, one input/one output pin, a `bytes` +control for STREAMINFO). Its `uuid` must match the firmware's registry entry for +`ffmpeg_dec` (see `uuid-registry.txt`). + +A runnable pipeline must deliver **compressed bytes** to the module and take PCM +out. Two paths: + +1. **ALSA compress-offload** (the production path for a decoder): the host writes + the FLAC elementary stream to a compress DMA gateway; the module decodes; PCM + goes to a DAI or host capture. This is the correct model (a decoder is not a + fixed-rate PCM effect) and is the remaining integration to wire on real ptl + hardware. +2. **Direct raw-data injection** (bring-up shortcut): a minimal pipeline where a + host copier delivers `stream.raw` as an opaque byte buffer into the module's + `process_raw_data`, capturing PCM on the other side. + +## Verification paths + +### A. On target (ptl hardware) — full verification +1. Build + flash firmware with the real backend: + `CONFIG_COMP_FFMPEG_DEC=m`, `CONFIG_COMP_FFMPEG_DEC_STUB=n` (requires the + FFmpeg static libs in `third_party/`; see the module README). +2. Load the topology; set the STREAMINFO bytes control: + `amixer -c0 cset name='FFMPGDEC...' -- $(od -An -tx1 vectors/streaminfo.bin)` + (or via `tinymix`). +3. Play `stream.raw` through the pipeline, capture the PCM output. +4. `cmp captured.raw vectors/ref_s32le.raw` — must be identical. + +### B. Native testbench (no hardware) — decode-only check +The module builds native (built-in, real libc — the LLEXT-only shims/alloc are +excluded automatically, see `CMakeLists.txt`). Linking the **host** libavcodec +and driving `process_raw_data` over `stream.raw` with `ref_s32le.raw` as the +oracle gives a bit-exact check without a DSP. This harness is not yet wired (the +testbench file component is PCM-oriented; feeding compressed bytes needs a small +adapter) and is the recommended next step for CI-friendly verification. + +## Status + +- Component widget conf: **done**. +- Host vector/reference tooling: **done** (`ffmpeg_dec_prepare.sh`). +- Runnable pipeline (compress-offload) + on-target bit-exact run: **pending + hardware**. +- Native testbench harness: **pending** (small compressed-input adapter). diff --git a/src/audio/ffmpeg_dec/fastmathf.c b/src/audio/ffmpeg_dec/fastmathf.c new file mode 100644 index 000000000000..02153fd10dc1 --- /dev/null +++ b/src/audio/ffmpeg_dec/fastmathf.c @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Fast single-precision float math for FFmpeg lossy decoders (AAC-LC / Opus). +// +// Empirically these decoders need only 7 single-precision functions on their +// decode paths: powf, sqrtf, cbrtf, exp2f, log2f, sinf, cosf. SOF's own math is +// fixed-point Q-format (range-limited) and cannot serve them, so this provides +// IEEE single-precision approximations. 24-bit mantissa precision is ample for +// 32-bit audio samples destined for 24/16-bit rendering, and these are per-frame +// / per-band calls (not the innermost per-sample loop, which is MDCT + table +// lookups), so scalar polynomial/Newton approximations are fast enough. +// +// On Xtensa HiFi, sqrtf/rsqrt and FMA map to hardware intrinsics; the scalar C +// here is the portable reference. Marked spots are where HiFi intrinsics would +// slot in. +// +// Build standalone accuracy/perf test: +// gcc -O2 -DSOFM_FASTMATH_TEST fastmathf.c -lm -o t && ./t + +#include + +/* --- helpers --- */ + +static inline float sofm_flint(int n) +{ + return (float)n; +} + +/* Build 2^n as a float via the exponent field (n in [-126,127]). */ +static inline float sofm_ldexp2(int n) +{ + union { float f; uint32_t i; } u; + + u.i = (uint32_t)(n + 127) << 23; + return u.f; +} + +/* --- sqrtf: bit-trick seed + 2 Newton steps --- */ +float sofm_sqrtf(float x) +{ + union { float f; uint32_t i; } u = { .f = x }; + float y; + + if (x <= 0.0f) + return 0.0f; + + /* HiFi: use FSQRT/RSQRT intrinsic instead of this seed. */ + u.i = 0x1fbd1df5 + (u.i >> 1); + y = u.f; + y = 0.5f * (y + x / y); + y = 0.5f * (y + x / y); + return y; +} + +/* --- cbrtf: bit-trick seed + 2 Newton steps, sign-symmetric --- */ +float sofm_cbrtf(float x) +{ + union { float f; uint32_t i; } u; + int neg = x < 0.0f; + float a = neg ? -x : x; + float y; + + if (x == 0.0f) + return 0.0f; + + u.f = a; + u.i = u.i / 3 + 0x2a514067; + y = u.f; + y = (2.0f * y + a / (y * y)) * (1.0f / 3.0f); + y = (2.0f * y + a / (y * y)) * (1.0f / 3.0f); + return neg ? -y : y; +} + +/* --- exp2f: split x = n + f, 2^x = 2^n * 2^f, poly for 2^f on [0,1) --- */ +float sofm_exp2f(float x) +{ + int n; + float f, p; + + if (x > 127.0f) + return sofm_ldexp2(127) * 2.0f; /* +inf-ish */ + if (x < -126.0f) + return 0.0f; + + n = (int)x; + if (x < 0.0f && (float)n != x) + n--; /* floor */ + f = x - (float)n; + + /* Taylor of 2^f, coeffs (ln2)^k/k!, degree 5 -> ~1e-6 rel error. */ + p = 1.0f + f * (0.6931472f + f * (0.2402265f + f * (0.0555041f + + f * (0.0096181f + f * 0.0013334f)))); + return p * sofm_ldexp2(n); +} + +/* --- log2f: x = 2^e * m, reduce m to [sqrt(2)/2, sqrt(2)), atanh series --- */ +float sofm_log2f(float x) +{ + union { float f; uint32_t i; } u = { .f = x }; + int e; + float m, a, a2, ln_m; + + if (x <= 0.0f) + return -127.0f; + + e = (int)((u.i >> 23) & 0xff) - 127; + u.i = (u.i & 0x007fffff) | 0x3f800000; /* m in [1,2) */ + m = u.f; + if (m > 1.41421356f) { /* center around 1: m in [~.707,~1.414) */ + m *= 0.5f; + e += 1; + } + + /* ln(m) = 2*atanh(a), a = (m-1)/(m+1); exact 0 at m==1. */ + a = (m - 1.0f) / (m + 1.0f); + a2 = a * a; + ln_m = 2.0f * a * (1.0f + a2 * (0.3333333f + a2 * (0.2f + a2 * 0.14285714f))); + return (float)e + ln_m * 1.44269504f; /* / ln(2) */ +} + +/* --- powf: x^y = 2^(y*log2(x)) for x > 0 (AAC uses non-negative magnitudes) --- */ +float sofm_powf(float x, float y) +{ + if (x <= 0.0f) + return 0.0f; + return sofm_exp2f(y * sofm_log2f(x)); +} + +/* --- sinf/cosf: Cody-Waite reduction by pi/2 + quadrant, polys on [-pi/4,pi/4] --- */ +static float sofm_poly_sin(float r) +{ + float r2 = r * r; + + return r * (1.0f + r2 * (-0.16666667f + r2 * (0.00833216f + + r2 * -0.00019515f))); +} + +static float sofm_poly_cos(float r) +{ + float r2 = r * r; + + return 1.0f + r2 * (-0.5f + r2 * (0.04166664f + + r2 * (-0.00138873f + r2 * 2.4433e-5f))); +} + +static float sofm_sincos_core(float x, int cos_phase) +{ + /* pi/2 split so k*PIO2_hi is exact for moderate k (Cody-Waite). */ + const float two_over_pi = 0.636619772f; + const float PIO2_hi = 1.5707855225f; + const float PIO2_lo = 1.08043006e-5f; + int k = (int)(x * two_over_pi + (x < 0.0f ? -0.5f : 0.5f)); + float r = (x - (float)k * PIO2_hi) - (float)k * PIO2_lo; + int q = (k + cos_phase) & 3; + + switch (q) { + case 0: return sofm_poly_sin(r); + case 1: return sofm_poly_cos(r); + case 2: return -sofm_poly_sin(r); + default: return -sofm_poly_cos(r); + } +} + +float sofm_sinf(float x) +{ + return sofm_sincos_core(x, 0); +} + +float sofm_cosf(float x) +{ + return sofm_sincos_core(x, 1); +} + +/* + * libc-name entry points used by libavcodec. Defined here so the LLEXT resolves + * them internally. (In a native/testbench build the real libm provides these; + * this file, like the other shims, is LLEXT-only.) + */ +#ifndef SOFM_FASTMATH_TEST +float sqrtf(float x) { return sofm_sqrtf(x); } +float cbrtf(float x) { return sofm_cbrtf(x); } +float exp2f(float x) { return sofm_exp2f(x); } +float log2f(float x) { return sofm_log2f(x); } +float powf(float x, float y) { return sofm_powf(x, y); } +float sinf(float x) { return sofm_sinf(x); } +float cosf(float x) { return sofm_cosf(x); } +#endif + +#ifdef SOFM_FASTMATH_TEST +#include +#include +#include +#include + +/* metric: 0 = relative error, 1 = absolute error */ +static double maxerr(const char *name, float (*approx)(float), + double (*ref)(double), float lo, float hi, int log_sweep, + int absolute) +{ + double worst = 0.0, at = 0.0; + int i, N = 200000; + + for (i = 0; i < N; i++) { + double t = (double)i / (N - 1); + float x = log_sweep ? (float)(lo * pow(hi / lo, t)) + : (float)(lo + (hi - lo) * t); + double r = ref((double)x), a = approx(x); + double e = absolute ? fabs(a - r) + : (fabs(r) > 1e-9 ? fabs((a - r) / r) : fabs(a - r)); + + if (e > worst) { worst = e; at = x; } + } + printf(" %-8s max_%s_err=%.3e at x=%.6g\n", name, + absolute ? "abs" : "rel", worst, at); + return worst; +} + +static double refpow_x(double x) { return pow(x, 1.3333333333); } + +int main(void) +{ + volatile float acc = 0; + int i; + clock_t t0; + + printf("accuracy (approx vs libm double reference):\n"); + maxerr("sqrtf", sofm_sqrtf, sqrt, 1e-6f, 1e6f, 1, 0); + maxerr("cbrtf", sofm_cbrtf, cbrt, 1e-6f, 1e6f, 1, 0); + maxerr("exp2f", sofm_exp2f, exp2, -30.f, 30.f, 0, 0); + maxerr("log2f", sofm_log2f, log2, 1e-6f, 1e6f, 1, 1); /* abs: log2 crosses 0 */ + maxerr("sinf", sofm_sinf, sin, -25.f, 25.f, 0, 1); /* abs: sin/cos in [-1,1] */ + maxerr("cosf", sofm_cosf, cos, -25.f, 25.f, 0, 1); + /* powf specifically over the AAC dequant shape x^(4/3), x in [0,8192] */ + { + double worst = 0, at = 0; + for (i = 1; i < 200000; i++) { + float x = (float)(8192.0 * i / 200000.0); + double r = refpow_x(x), a = sofm_powf(x, 1.3333333f); + double e = fabs(r) > 1e-9 ? fabs((a - r) / r) : fabs(a - r); + if (e > worst) { worst = e; at = x; } + } + printf(" %-8s max_rel_err=%.3e at x=%.6g (x^4/3)\n", "powf", worst, at); + } + + /* rough timing: fast vs libm for exp2f/log2f/powf */ + printf("timing (200M calls):\n"); + t0 = clock(); + for (i = 0; i < 200000000; i++) acc += sofm_exp2f((float)(i & 63) - 32.f); + printf(" sofm_exp2f: %.2fs\n", (double)(clock() - t0) / CLOCKS_PER_SEC); + t0 = clock(); + for (i = 0; i < 200000000; i++) acc += exp2f((float)(i & 63) - 32.f); + printf(" libm exp2f: %.2fs\n", (double)(clock() - t0) / CLOCKS_PER_SEC); + return (int)acc & 0; +} +#endif diff --git a/src/audio/ffmpeg_dec/ffmpeg.cmake b/src/audio/ffmpeg_dec/ffmpeg.cmake new file mode 100644 index 000000000000..7dd8eb2058c2 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg.cmake @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: BSD-3-Clause +# +# Cross-build decoder-only FFmpeg static libraries for the ffmpeg_dec module. +# +# Invokes FFmpeg's autotools build as an ExternalProject, cross-compiled with the +# Zephyr target toolchain, enabling only the decoders selected in Kconfig. The +# source comes from the west-pinned FFmpeg project (see west.yml). Produces +# libavcodec/libavutil/libswresample .a under ${FFMPEG_INSTALL_DIR} for the LLEXT +# to link, and defines the 'ffmpeg_ext' target it must depend on. + +include(ExternalProject) + +# --- 1. Locate the FFmpeg source (west module), overridable for local trees --- +if(NOT DEFINED SOF_FFMPEG_SRC_DIR) + set(SOF_FFMPEG_SRC_DIR "${sof_top_dir}/../modules/audio/ffmpeg" + CACHE PATH "FFmpeg source tree (west-pinned)") +endif() +cmake_path(NORMAL_PATH SOF_FFMPEG_SRC_DIR) +if(NOT EXISTS "${SOF_FFMPEG_SRC_DIR}/configure") + message(FATAL_ERROR + "ffmpeg_dec: FFmpeg source not found at '${SOF_FFMPEG_SRC_DIR}'.\n" + "Run 'west update' (FFmpeg is pinned in west.yml) or pass " + "-DSOF_FFMPEG_SRC_DIR=.") +endif() + +# --- 2. Kconfig -> --enable-decoder / --enable-parser lists --- +set(_ff_decoders "") +set(_ff_parsers "") +if(CONFIG_FFMPEG_DEC_FLAC) + list(APPEND _ff_decoders flac) + list(APPEND _ff_parsers flac) +endif() +if(CONFIG_FFMPEG_DEC_AAC) + list(APPEND _ff_decoders aac aac_latm) + list(APPEND _ff_parsers aac aac_latm) +endif() +if(CONFIG_FFMPEG_DEC_OPUS) + list(APPEND _ff_decoders opus) + list(APPEND _ff_parsers opus) +endif() +if(NOT _ff_decoders) + message(FATAL_ERROR "ffmpeg_dec: no decoder selected (Kconfig FFMPEG_DEC_*)") +endif() +list(JOIN _ff_decoders "," _ff_dec_csv) +list(JOIN _ff_parsers "," _ff_par_csv) + +# --- 2b. Kconfig -> libavfilter audio filters (off unless a filter is chosen) --- +set(_ff_avfilter_cfg --disable-avfilter) +if(CONFIG_FFMPEG_BUILD_AVFILTER) + set(_ff_filters "") + if(CONFIG_FFMPEG_FILTER_AFFTDN) + list(APPEND _ff_filters afftdn) + endif() + list(JOIN _ff_filters "," _ff_filt_csv) + # abuffer/abuffersink feed/drain a filter graph; aformat negotiates format. + set(_ff_avfilter_cfg + --enable-avfilter + --enable-filter=abuffer,abuffersink,aformat,${_ff_filt_csv}) + message(STATUS "ffmpeg_dec: enabling avfilter, filters [${_ff_filt_csv}]") +endif() + +# --- 3. Derive the cross toolchain prefix from the Zephyr target compiler --- +# e.g. .../bin/xtensa-intel_ace30_ptl_zephyr-elf-gcc +# -> prefix .../bin/xtensa-intel_ace30_ptl_zephyr-elf- +get_filename_component(_tc_dir "${CMAKE_C_COMPILER}" DIRECTORY) +get_filename_component(_tc_name "${CMAKE_C_COMPILER}" NAME) +string(REGEX REPLACE "gcc$" "" _tc_prefix_name "${_tc_name}") +set(_ff_cross_prefix "${_tc_dir}/${_tc_prefix_name}") + +# GCC 14 (Zephyr SDK) promotes these to errors; FFmpeg 7.x trips them. +set(_ff_extra_cflags "-fPIC -Wno-error=incompatible-pointer-types -Wno-error=implicit-function-declaration") + +set(FFMPEG_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/ffmpeg-install" CACHE INTERNAL "ffmpeg_dec libs") + +# Hot/cold split: GCC emits av_cold (init/setup) functions into .text.unlikely. +# We rename that to SOF's .cold (post-install, below) so the LLEXT loader places +# it in DRAM. -mtext-section-literals interleaves each function's Xtensa L32R +# literals into its text section (as SOF does for LLEXT module code) so the +# renamed section carries its literals and L32R stays in range. We deliberately +# do NOT use -ffunction-sections: keeping one .text.unlikely per object lets a +# single objcopy --rename-section catch it, and lets .cold be placed by SOF's +# existing orphan .cold handling (a hand-written linker script that forces .cold +# ahead of .text overruns the SRAM budget and overlaps .rodata for big codecs). +if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + # -mlongcalls: cold code lives in DRAM (.cold, relocated by the loader) far + # from the hot SRAM .text, so calls between them exceed call8 range; longcalls + # emit an indirect L32R+CALLX with unlimited range. -mtext-section-literals + # keeps each function's literals in its own text section so the rename carries + # them. Coarse split: rename the whole FFmpeg .text (not just av_cold) into + # .cold so essentially all of libav* executes from DRAM, freeing SRAM. + set(_ff_extra_cflags "${_ff_extra_cflags} -mtext-section-literals -mlongcalls") + set(_ff_objcopy "${_ff_cross_prefix}objcopy") + set(_ff_cold_rename + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + COMMAND ${_ff_objcopy} --rename-section .text=.cold + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a) +endif() + +# --- 4. Configure + make + install (out-of-tree; source must be clean) --- +ExternalProject_Add(ffmpeg_ext + SOURCE_DIR "${SOF_FFMPEG_SRC_DIR}" + BUILD_IN_SOURCE 0 + CONFIGURE_COMMAND + ${CMAKE_COMMAND} -E env "PATH=${_tc_dir}:$ENV{PATH}" + /configure + --prefix=${FFMPEG_INSTALL_DIR} + --enable-cross-compile --target-os=none --arch=xtensa + --cross-prefix=${_ff_cross_prefix} --cc=${CMAKE_C_COMPILER} + --disable-asm --disable-all --disable-everything --disable-autodetect + --disable-programs --disable-doc --disable-network + --disable-avformat --disable-avdevice ${_ff_avfilter_cfg} + --disable-swscale --disable-postproc + --disable-pthreads --disable-w32threads --disable-os2threads + --disable-runtime-cpudetect --disable-debug + --enable-avcodec --enable-avutil --enable-swresample + --enable-decoder=${_ff_dec_csv} --enable-parser=${_ff_par_csv} + --enable-small --enable-pic + "--extra-cflags=${_ff_extra_cflags}" + BUILD_COMMAND + ${CMAKE_COMMAND} -E env "PATH=${_tc_dir}:$ENV{PATH}" make -j 8 + INSTALL_COMMAND make install + ${_ff_cold_rename} + BUILD_BYPRODUCTS + ${FFMPEG_INSTALL_DIR}/lib/libavcodec.a + ${FFMPEG_INSTALL_DIR}/lib/libavutil.a + ${FFMPEG_INSTALL_DIR}/lib/libswresample.a +) + +message(STATUS "ffmpeg_dec: cross-building FFmpeg decoders [${_ff_dec_csv}] from ${SOF_FFMPEG_SRC_DIR}") diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c b/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c new file mode 100644 index 000000000000..44da5cf08033 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-alloc.c @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// malloc/free/realloc for the ffmpeg_dec libavcodec backend, backed by the SOF +// module heap. libavcodec's av_malloc() bottoms out in libc malloc; the SOF core +// does not export malloc to LLEXT modules, so we provide it here and route it to +// mod_alloc_align()/mod_free() — no large static reservation, memory comes from +// the DSP heap like any other SOF module. +// +// Each block is prefixed with a small header storing its payload size so realloc +// can copy the old contents (SOF's allocator does not expose a realloc). The +// header is 16 bytes to preserve 16-byte alignment of the returned pointer. +// +// The active processing_module is bound at the entry of every backend op (init/ +// prepare/process/reset/free) via ffmpeg_dec_libc_bind(); decode is serialized +// per module on a pipeline core, so a single global binding is safe. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#define FFMPEG_DEC_ALLOC_HDR 16 + +static struct processing_module *ffmpeg_dec_alloc_mod; + +void ffmpeg_dec_libc_bind(struct processing_module *mod) +{ + ffmpeg_dec_alloc_mod = mod; +} + +void *malloc(size_t size) +{ + uint8_t *base; + + if (!ffmpeg_dec_alloc_mod || !size) + return NULL; + + base = mod_alloc_align(ffmpeg_dec_alloc_mod, size + FFMPEG_DEC_ALLOC_HDR, + FFMPEG_DEC_ALLOC_HDR); + if (!base) + return NULL; + + *(size_t *)base = size; + return base + FFMPEG_DEC_ALLOC_HDR; +} + +void free(void *ptr) +{ + if (!ptr || !ffmpeg_dec_alloc_mod) + return; + + mod_free(ffmpeg_dec_alloc_mod, (uint8_t *)ptr - FFMPEG_DEC_ALLOC_HDR); +} + +void *realloc(void *ptr, size_t size) +{ + size_t old_size; + void *new_ptr; + + if (!ptr) + return malloc(size); + + if (!size) { + free(ptr); + return NULL; + } + + old_size = *(size_t *)((uint8_t *)ptr - FFMPEG_DEC_ALLOC_HDR); + new_ptr = malloc(size); + if (new_ptr) { + memcpy(new_ptr, ptr, old_size < size ? old_size : size); + free(ptr); + } + + return new_ptr; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c b/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c new file mode 100644 index 000000000000..472536555c54 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-ffmpeg.c @@ -0,0 +1,342 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavcodec decode backend for the ffmpeg_dec module. +// +// Drives the modern send-packet / receive-frame libavcodec API standalone (no +// libavformat, no file I/O). A raw compressed elementary stream is parsed on +// the DSP with an AVCodecParser (av_parser_parse2), each recovered frame is fed +// to avcodec_send_packet, and decoded PCM is drained with avcodec_receive_frame +// and interleaved into the SOF output buffer. +// +// Requires pre-compiled libavcodec / libavutil / libswresample static libraries +// and headers for the target under third_party/ (see the Phase 0 build). It is +// only compiled when CONFIG_COMP_FFMPEG_DEC_STUB is not selected. + +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/* Padding libavcodec's bitstream readers over-read past the end of a packet. */ +#ifndef AV_INPUT_BUFFER_PADDING_SIZE +#define AV_INPUT_BUFFER_PADDING_SIZE 64 +#endif + +/* + * Route FFmpeg's av_log() into the SOF/Zephyr logging subsystem instead of its + * default callback (which writes to stderr via fprintf). The line buffer is + * static: decode is single-threaded, and a static buffer avoids handing Zephyr's + * deferred logging a pointer into a stack frame that has since been unwound. + */ +static char ffmpeg_dec_log_line[160]; + +static void ffmpeg_dec_av_log(void *avcl, int level, const char *fmt, va_list vl) +{ + int n; + + if (level > av_log_get_level()) + return; + + n = vsnprintf(ffmpeg_dec_log_line, sizeof(ffmpeg_dec_log_line), fmt, vl); + if (n <= 0) + return; + + /* Trim the trailing newline FFmpeg conventionally appends. */ + if (n < (int)sizeof(ffmpeg_dec_log_line) && ffmpeg_dec_log_line[n - 1] == '\n') + ffmpeg_dec_log_line[n - 1] = '\0'; + + if (level <= AV_LOG_ERROR) + LOG_ERR("ffmpeg: %s", ffmpeg_dec_log_line); + else if (level <= AV_LOG_WARNING) + LOG_WRN("ffmpeg: %s", ffmpeg_dec_log_line); + else if (level <= AV_LOG_INFO) + LOG_INF("ffmpeg: %s", ffmpeg_dec_log_line); + else + LOG_DBG("ffmpeg: %s", ffmpeg_dec_log_line); +} + +/** + * struct ffmpeg_dec_ffmpeg_data - libavcodec backend private state. + * @avctx: Decoder context. + * @parser: Elementary-stream frame parser. + * @pkt: Reusable packet handed to the decoder. + * @frame: Reusable decoded PCM frame. + * @pktbuf: Padded scratch buffer for parsed packet payloads. + * @codec: Resolved AVCodec for the selected codec id. + */ +struct ffmpeg_dec_ffmpeg_data { + AVCodecContext *avctx; + AVCodecParserContext *parser; + AVPacket *pkt; + AVFrame *frame; + uint8_t *pktbuf; + const AVCodec *codec; +}; + +/* Map the SOF codec enum to a libavcodec decoder id. */ +static enum AVCodecID ffmpeg_dec_av_codec_id(enum ffmpeg_dec_codec codec) +{ + switch (codec) { +#if defined(CONFIG_FFMPEG_DEC_FLAC) + case FFMPEG_DEC_CODEC_FLAC: + return AV_CODEC_ID_FLAC; +#endif +#if defined(CONFIG_FFMPEG_DEC_AAC) + case FFMPEG_DEC_CODEC_AAC: + return AV_CODEC_ID_AAC; +#endif +#if defined(CONFIG_FFMPEG_DEC_OPUS) + case FFMPEG_DEC_CODEC_OPUS: + return AV_CODEC_ID_OPUS; +#endif + default: + return AV_CODEC_ID_NONE; + } +} + +static int ffmpeg_dec_ff_init(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + struct ffmpeg_dec_ffmpeg_data *ff; + enum AVCodecID id; + + /* Bind the heap that backs FFmpeg's malloc before any av_malloc runs. */ + ffmpeg_dec_libc_bind(mod); + + /* Redirect libavcodec logging into SOF/Zephyr before anything can log. */ + av_log_set_level(AV_LOG_ERROR); + av_log_set_callback(ffmpeg_dec_av_log); + + ff = mod_zalloc(mod, sizeof(*ff)); + if (!ff) + return -ENOMEM; + + id = ffmpeg_dec_av_codec_id(cd->codec); + ff->codec = avcodec_find_decoder(id); + if (!ff->codec) { + comp_err(dev, "no libavcodec decoder for codec %d", cd->codec); + mod_free(mod, ff); + return -ENODEV; + } + + ff->parser = av_parser_init(ff->codec->id); + ff->avctx = avcodec_alloc_context3(ff->codec); + ff->pkt = av_packet_alloc(); + ff->frame = av_frame_alloc(); + if (!ff->avctx || !ff->pkt || !ff->frame) { + comp_err(dev, "libavcodec object allocation failed"); + goto err; + } + + cd->backend_data = ff; + return 0; + +err: + if (ff->frame) + av_frame_free(&ff->frame); + if (ff->pkt) + av_packet_free(&ff->pkt); + if (ff->avctx) + avcodec_free_context(&ff->avctx); + if (ff->parser) + av_parser_close(ff->parser); + mod_free(mod, ff); + return -ENOMEM; +} + +static int ffmpeg_dec_ff_configure(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + struct comp_dev *dev = mod->dev; + int ret; + + ffmpeg_dec_libc_bind(mod); + + /* Some codecs (FLAC, Opus, Vorbis, AAC-in-MP4) need their setup header + * in avctx->extradata before avcodec_open2(). + */ + if (cd->extradata && cd->extradata_size) { + av_freep(&ff->avctx->extradata); + ff->avctx->extradata = + av_mallocz(cd->extradata_size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!ff->avctx->extradata) + return -ENOMEM; + memcpy_s(ff->avctx->extradata, cd->extradata_size, + cd->extradata, cd->extradata_size); + ff->avctx->extradata_size = cd->extradata_size; + } + + /* Force single-threaded decode: no pthreads, no frame-threading latency, + * and get_buffer2 need not be thread-safe. + */ + ff->avctx->thread_count = 1; + + ret = avcodec_open2(ff->avctx, ff->codec, NULL); + if (ret < 0) { + comp_err(dev, "avcodec_open2 failed %d", ret); + return -EIO; + } + + /* Padded scratch for a parsed packet payload, sized to OBS as a bound. */ + ff->pktbuf = mod_alloc(mod, cd->out_frame_bytes ? + cd->out_frame_bytes * 4096 : 65536); + if (!ff->pktbuf) + return -ENOMEM; + + return 0; +} + +/* + * Interleave one decoded AVFrame into @out, honouring planar vs packed layout. + * Returns bytes written, or a negative errno if @out cannot hold the frame. + * NOTE: for the first FLAC bring-up the output sample format is assumed to match + * the sink (S16/S32). Format/rate conversion (libswresample) is a follow-up. + */ +static int ffmpeg_dec_emit_frame(struct processing_module *mod, AVFrame *frame, + uint8_t *out, size_t out_size) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + int channels = cd->out_channels; + int bps = av_get_bytes_per_sample(frame->format); + int planar = av_sample_fmt_is_planar(frame->format); + size_t need = (size_t)frame->nb_samples * channels * bps; + int i, ch; + + if (need > out_size) + return -ENOSPC; + + if (!planar) { + memcpy_s(out, out_size, frame->data[0], need); + return need; + } + + for (i = 0; i < frame->nb_samples; i++) + for (ch = 0; ch < channels; ch++) + memcpy_s(out + (((size_t)i * channels + ch) * bps), + out_size, frame->data[ch] + (size_t)i * bps, bps); + + return need; +} + +static int ffmpeg_dec_ff_decode(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + struct comp_dev *dev = mod->dev; + size_t out_off = 0; + int used; + int ret; + + ffmpeg_dec_libc_bind(mod); + + *consumed = 0; + *produced = 0; + + /* Parse one frame out of the raw stream. */ + used = av_parser_parse2(ff->parser, ff->avctx, + &ff->pkt->data, &ff->pkt->size, + in, in_size, + AV_NOPTS_VALUE, AV_NOPTS_VALUE, 0); + if (used < 0) { + comp_err(dev, "av_parser_parse2 failed %d", used); + return -EIO; + } + *consumed = used; + + /* Parser still buffering: no complete frame yet. */ + if (ff->pkt->size == 0) + return 0; + + ret = avcodec_send_packet(ff->avctx, ff->pkt); + if (ret < 0) { + comp_err(dev, "avcodec_send_packet failed %d", ret); + return -EIO; + } + + /* Drain all frames this packet produced. */ + while (ret >= 0) { + ret = avcodec_receive_frame(ff->avctx, ff->frame); + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) + break; + if (ret < 0) { + comp_err(dev, "avcodec_receive_frame failed %d", ret); + return -EIO; + } + + ret = ffmpeg_dec_emit_frame(mod, ff->frame, out + out_off, + out_size - out_off); + if (ret < 0) { + comp_warn(dev, "output buffer full, PCM dropped"); + break; + } + out_off += ret; + ret = 0; + } + + *produced = out_off; + return 0; +} + +static int ffmpeg_dec_ff_reset(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + + ffmpeg_dec_libc_bind(mod); + + if (ff && ff->avctx) + avcodec_flush_buffers(ff->avctx); + + return 0; +} + +static int ffmpeg_dec_ff_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct ffmpeg_dec_ffmpeg_data *ff = cd->backend_data; + + ffmpeg_dec_libc_bind(mod); + + if (!ff) + return 0; + + if (ff->frame) + av_frame_free(&ff->frame); + if (ff->pkt) + av_packet_free(&ff->pkt); + if (ff->avctx) + avcodec_free_context(&ff->avctx); + if (ff->parser) + av_parser_close(ff->parser); + if (ff->pktbuf) + mod_free(mod, ff->pktbuf); + + mod_free(mod, ff); + cd->backend_data = NULL; + return 0; +} + +const struct ffmpeg_dec_backend ffmpeg_dec_backend = { + .name = "libavcodec", + .init = ffmpeg_dec_ff_init, + .configure = ffmpeg_dec_ff_configure, + .decode = ffmpeg_dec_ff_decode, + .reset = ffmpeg_dec_ff_reset, + .free = ffmpeg_dec_ff_free, +}; diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c b/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c new file mode 100644 index 000000000000..5b33b99692a2 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-filter.c @@ -0,0 +1,281 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// libavfilter graph backend for the ffmpeg_dec module: runs an FFmpeg audio +// filter (e.g. afftdn FFT noise reduction) over PCM. Unlike the decoder path +// (send_packet/receive_frame), filters use the avfilter graph API: +// +// abuffer(src) -> -> abuffersink(sink) +// +// PCM AVFrames are pushed into the source, pulled filtered from the sink. This +// is a PCM->PCM effect path, distinct from the compressed->PCM decode path; it +// is only built when a filter is selected (CONFIG_FFMPEG_FILTER_*), which pulls +// libavfilter into the module. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/** + * struct ffmpeg_af_graph - one audio filter graph instance. + * @graph: The avfilter graph. + * @src: abuffer source context (frames pushed here). + * @sink: abuffersink context (filtered frames pulled here). + */ +struct ffmpeg_af_graph { + AVFilterGraph *graph; + AVFilterContext *src; + AVFilterContext *sink; +}; + +/** + * ffmpeg_af_open() - Build a src -> -> sink graph for the given PCM + * format. + * @g: Graph to initialise. + * @filter: Filter name, e.g. "afftdn". + * @rate: Sample rate (Hz). + * @channels: Channel count. + * @fmt: Interleaved AV sample format (e.g. AV_SAMPLE_FMT_S32). + * + * Return: Zero on success, negative errno otherwise. + */ +int ffmpeg_af_open(struct ffmpeg_af_graph *g, const char *filter, + int rate, int channels, enum AVSampleFormat fmt) +{ + const AVFilter *abuffer = avfilter_get_by_name("abuffer"); + const AVFilter *abuffersink = avfilter_get_by_name("abuffersink"); + const AVFilter *filt = avfilter_get_by_name(filter); + AVFilterContext *filt_ctx = NULL; + AVChannelLayout layout; + char chbuf[64]; + char args[256]; + int ret; + + memset(g, 0, sizeof(*g)); + if (!abuffer || !abuffersink || !filt) + return -ENODEV; + + g->graph = avfilter_graph_alloc(); + if (!g->graph) + return -ENOMEM; + + av_channel_layout_default(&layout, channels); + av_channel_layout_describe(&layout, chbuf, sizeof(chbuf)); + + snprintf(args, sizeof(args), + "time_base=1/%d:sample_rate=%d:sample_fmt=%s:channel_layout=%s", + rate, rate, av_get_sample_fmt_name(fmt), chbuf); + + ret = avfilter_graph_create_filter(&g->src, abuffer, "in", args, NULL, g->graph); + if (ret < 0) + goto err; + ret = avfilter_graph_create_filter(&filt_ctx, filt, "filter", NULL, NULL, g->graph); + if (ret < 0) + goto err; + ret = avfilter_graph_create_filter(&g->sink, abuffersink, "out", NULL, NULL, g->graph); + if (ret < 0) + goto err; + + ret = avfilter_link(g->src, 0, filt_ctx, 0); + if (!ret) + ret = avfilter_link(filt_ctx, 0, g->sink, 0); + if (ret < 0) + goto err; + + ret = avfilter_graph_config(g->graph, NULL); + if (ret < 0) + goto err; + + return 0; + +err: + avfilter_graph_free(&g->graph); + return -EIO; +} + +/** + * ffmpeg_af_filter() - Push one PCM frame through the graph, pull the result. + * @g: Graph. + * @in: Input PCM frame. + * @out: Output frame to receive the filtered PCM. + * + * Return: 0 on a produced frame, -EAGAIN if the filter needs more input, + * negative errno on error. + */ +int ffmpeg_af_filter(struct ffmpeg_af_graph *g, AVFrame *in, AVFrame *out) +{ + int ret = av_buffersrc_add_frame(g->src, in); + + if (ret < 0) + return -EIO; + + ret = av_buffersink_get_frame(g->sink, out); + if (ret == AVERROR(EAGAIN)) + return -EAGAIN; + if (ret < 0) + return -EIO; + + return 0; +} + +void ffmpeg_af_close(struct ffmpeg_af_graph *g) +{ + if (g->graph) + avfilter_graph_free(&g->graph); +} + +#if CONFIG_FFMPEG_DEC_FILTER_MODE +/* + * Filter-mode SOF module: a PCM source->sink effect that runs the graph + * (default afftdn). SOF PCM is interleaved S32; afftdn works on float planar + * (FLTP), so we deinterleave+normalize S32 -> float on the way in and the + * reverse on the way out. Bounded chunk per cycle. + * + * NOTE: afftdn has internal latency/framing, so produced samples per cycle may + * differ from consumed; the structure below drives the graph correctly but + * real-time latency/underrun tuning is left for on-hardware bring-up. + */ + +#include +#include +#include + +#define FFMPEG_AF_MAX_CHUNK 4096 +#define FFMPEG_AF_S32_SCALE 2147483648.0f /* 2^31 */ +#ifndef CONFIG_FFMPEG_AF_FILTER_NAME +#define CONFIG_FFMPEG_AF_FILTER_NAME "afftdn" +#endif + +int ffmpeg_af_mod_init(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + ffmpeg_dec_libc_bind(mod); + cd->af_graph = mod_zalloc(mod, sizeof(struct ffmpeg_af_graph)); + return cd->af_graph ? 0 : -ENOMEM; +} + +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + + if (num_of_sources != 1 || num_of_sinks != 1) + return -EINVAL; + + ffmpeg_dec_libc_bind(mod); + cd->out_rate = source_get_rate(sources[0]); + cd->out_channels = source_get_channels(sources[0]); + cd->out_frame_bytes = source_get_frame_bytes(sources[0]); + + comp_info(dev, "ffmpeg_af: %s, rate %u ch %u", + CONFIG_FFMPEG_AF_FILTER_NAME, cd->out_rate, cd->out_channels); + + /* Graph runs in float planar; the module converts to/from S32. */ + return ffmpeg_af_open(cd->af_graph, CONFIG_FFMPEG_AF_FILTER_NAME, + cd->out_rate, cd->out_channels, AV_SAMPLE_FMT_FLTP); +} + +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct sof_source *source = sources[0]; + struct sof_sink *sink = sinks[0]; + int ch = cd->out_channels; + int in_frames = source_get_data_frames_available(source); + int n = MIN(in_frames, FFMPEG_AF_MAX_CHUNK); + AVFrame *in, *out; + const int32_t *src; + int32_t *dst; + int i, c, m, ret; + + ffmpeg_dec_libc_bind(mod); + if (n <= 0) + return 0; + + /* Build the FLTP input frame (deinterleave + normalize S32 -> float). */ + in = av_frame_alloc(); + if (!in) + return -ENOMEM; + in->format = AV_SAMPLE_FMT_FLTP; + in->nb_samples = n; + in->sample_rate = cd->out_rate; + av_channel_layout_default(&in->ch_layout, ch); + if (av_frame_get_buffer(in, 0) < 0) { + av_frame_free(&in); + return -ENOMEM; + } + + ret = source_get_data_s32(source, n * cd->out_frame_bytes, &src, NULL, NULL); + if (ret) { + av_frame_free(&in); + return ret; + } + for (i = 0; i < n; i++) + for (c = 0; c < ch; c++) + ((float *)in->data[c])[i] = (float)src[i * ch + c] / FFMPEG_AF_S32_SCALE; + source_release_data(source, n * cd->out_frame_bytes); + + /* Run the graph. */ + out = av_frame_alloc(); + if (!out) { + av_frame_free(&in); + return -ENOMEM; + } + ret = ffmpeg_af_filter(cd->af_graph, in, out); + av_frame_free(&in); + if (ret == -EAGAIN) { /* filter still priming, no output yet */ + av_frame_free(&out); + return 0; + } + if (ret) { + av_frame_free(&out); + return ret; + } + + /* Interleave + denormalize float -> S32 into the sink. */ + m = MIN(out->nb_samples, sink_get_free_frames(sink)); + if (m > 0 && sink_get_buffer_s32(sink, m * cd->out_frame_bytes, &dst, NULL, NULL) == 0) { + for (i = 0; i < m; i++) + for (c = 0; c < ch; c++) { + float f = ((const float *)out->data[c])[i] * FFMPEG_AF_S32_SCALE; + + f = f > 2147483647.0f ? 2147483647.0f : + (f < -2147483648.0f ? -2147483648.0f : f); + dst[i * ch + c] = (int32_t)f; + } + sink_commit_buffer(sink, m * cd->out_frame_bytes); + } + av_frame_free(&out); + return 0; +} + +int ffmpeg_af_mod_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + ffmpeg_dec_libc_bind(mod); + if (cd->af_graph) { + ffmpeg_af_close(cd->af_graph); + mod_free(mod, cd->af_graph); + cd->af_graph = NULL; + } + return 0; +} +#endif /* CONFIG_FFMPEG_DEC_FILTER_MODE */ diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c new file mode 100644 index 000000000000..ebff0c0d9b04 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-shims.c @@ -0,0 +1,441 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Local libc surface for the ffmpeg_dec libavcodec backend. +// +// libavcodec/libavutil reference a set of libc/libm symbols the SOF core does +// not export to LLEXT modules. This file defines them inside the module so it +// resolves at load. Categories: +// +// * stdio / time / env (file I/O, clock, getenv, stderr): NEVER used on a SOF +// DSP and never on the FLAC decode path -> no-op stubs. av_log()'s default +// stderr path is additionally bypassed by the Zephyr LOG wrapper installed in +// ffmpeg_dec-ffmpeg.c. +// * vsnprintf / snprintf: genuinely used (av_log formatting, string utils) -> +// a compact real implementation. +// * str/mem helpers not exported by the core (memchr, strchr, ...): real. +// * bsearch, __bswap*: real. +// * strto* : not on the FLAC decode path -> return-0 stubs. +// * libm (pow/sin/sqrt/...): FLAC is lossless integer and never calls these; +// SOF's fixed-point math is range-limited (sqrt 0-2, exp +/-8, 16-bit sin) so +// it cannot serve as a double libm. These are RESOLVE-ONLY STUBS that let the +// module load. THEY MUST BE REPLACED WITH A REAL DOUBLE libm before enabling +// any lossy codec (AAC/Opus/Vorbis), which would otherwise decode to garbage. +// +// NOTE: this file intentionally includes NO libc headers so these definitions +// cannot clash with newlib prototypes; only the symbol names matter to the linker. + +#include +#include +#include + +/* ============================ stdio / time / env ========================== */ + +/* stderr is referenced as a data object by FFmpeg's default log path. */ +void *stderr; + +int fprintf(void *stream, const char *fmt, ...) { (void)stream; (void)fmt; return 0; } +int fputs(const char *s, void *stream) { (void)s; (void)stream; return 0; } +unsigned long fread(void *p, unsigned long sz, unsigned long n, void *stream) +{ (void)p; (void)sz; (void)n; (void)stream; return 0; } +int fclose(void *stream) { (void)stream; return 0; } +void *fopen(const char *path, const char *mode) { (void)path; (void)mode; return NULL; } +void *fdopen(int fd, const char *mode) { (void)fd; (void)mode; return NULL; } +int setvbuf(void *s, char *buf, int mode, size_t size) +{ (void)s; (void)buf; (void)mode; (void)size; return 0; } +int open(const char *path, int flags, ...) { (void)path; (void)flags; return -1; } +int sscanf(const char *str, const char *fmt, ...) { (void)str; (void)fmt; return 0; } + +char *getenv(const char *name) { (void)name; return NULL; } +unsigned long clock(void) { return 0; } +void *gmtime_r(const void *timep, void *result) { (void)timep; (void)result; return NULL; } +void *localtime(const void *timep) { (void)timep; return NULL; } +long mktime(void *tm) { (void)tm; return -1; } +size_t strftime(char *s, size_t max, const char *fmt, const void *tm) +{ (void)fmt; (void)tm; if (max) s[0] = '\0'; return 0; } +void abort(void) { for (;;) ; } + +/* ============================ str / mem helpers =========================== */ + +void *memchr(const void *s, int c, size_t n) +{ + const unsigned char *p = s; + + while (n--) { + if (*p == (unsigned char)c) + return (void *)p; + p++; + } + return NULL; +} + +char *strchr(const char *s, int c) +{ + for (;; s++) { + if (*s == (char)c) + return (char *)s; + if (!*s) + return NULL; + } +} + +char *strrchr(const char *s, int c) +{ + const char *last = NULL; + + for (;; s++) { + if (*s == (char)c) + last = s; + if (!*s) + return (char *)last; + } +} + +char *strstr(const char *haystack, const char *needle) +{ + if (!*needle) + return (char *)haystack; + + for (; *haystack; haystack++) { + const char *h = haystack, *n = needle; + + while (*h && *n && *h == *n) { + h++; + n++; + } + if (!*n) + return (char *)haystack; + } + return NULL; +} + +static int ffmpeg_dec_in_set(char ch, const char *set) +{ + for (; *set; set++) + if (*set == ch) + return 1; + return 0; +} + +size_t strspn(const char *s, const char *accept) +{ + size_t n = 0; + + while (s[n] && ffmpeg_dec_in_set(s[n], accept)) + n++; + return n; +} + +size_t strcspn(const char *s, const char *reject) +{ + size_t n = 0; + + while (s[n] && !ffmpeg_dec_in_set(s[n], reject)) + n++; + return n; +} + +/* ============================ bsearch / bswap ============================= */ + +void *bsearch(const void *key, const void *base, size_t nmemb, size_t size, + int (*compar)(const void *, const void *)) +{ + size_t lo = 0, hi = nmemb; + + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + const void *elem = (const char *)base + mid * size; + int r = compar(key, elem); + + if (r < 0) + hi = mid; + else if (r > 0) + lo = mid + 1; + else + return (void *)elem; + } + return NULL; +} + +uint32_t __bswapsi2(uint32_t x) +{ + return ((x & 0x000000ffu) << 24) | ((x & 0x0000ff00u) << 8) | + ((x & 0x00ff0000u) >> 8) | ((x & 0xff000000u) >> 24); +} + +uint64_t __bswapdi2(uint64_t x) +{ + return ((uint64_t)__bswapsi2((uint32_t)x) << 32) | + __bswapsi2((uint32_t)(x >> 32)); +} + +/* ============================ strto* (unused stubs) ======================= */ + +double strtod(const char *nptr, char **endptr) +{ if (endptr) *endptr = (char *)nptr; return 0; } +long strtol(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +long long strtoll(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +unsigned long strtoul(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } +unsigned long long strtoull(const char *nptr, char **endptr, int base) +{ (void)base; if (endptr) *endptr = (char *)nptr; return 0; } + +/* ============================ libm (RESOLVE-ONLY STUBS) =================== */ +/* FLAC never calls these. Replace with a real double libm for lossy codecs. */ + +double acos(double x) { (void)x; return 0; } +double asin(double x) { (void)x; return 0; } +double atan(double x) { (void)x; return 0; } +double atan2(double y, double x) { (void)y; (void)x; return 0; } +double ceil(double x) { (void)x; return 0; } +double cos(double x) { (void)x; return 0; } +double cosh(double x) { (void)x; return 0; } +double exp(double x) { (void)x; return 0; } +double exp2(double x) { (void)x; return 0; } +double fabs(double x) { return x < 0 ? -x : x; } +double floor(double x) { (void)x; return 0; } +double fmod(double x, double y) { (void)x; (void)y; return 0; } +double frexp(double x, int *e) { (void)x; if (e) *e = 0; return 0; } +double hypot(double x, double y) { (void)x; (void)y; return 0; } +long long llrint(double x) { (void)x; return 0; } +double log(double x) { (void)x; return 0; } +double pow(double x, double y) { (void)x; (void)y; return 0; } +double round(double x) { (void)x; return 0; } +double scalbn(double x, int n) { (void)x; (void)n; return 0; } +double sin(double x) { (void)x; return 0; } +double sinh(double x) { (void)x; return 0; } +double sqrt(double x) { (void)x; return 0; } +double tan(double x) { (void)x; return 0; } +double tanh(double x) { (void)x; return 0; } +double trunc(double x) { (void)x; return 0; } + +/* Small real libm helpers pulled in by libavfilter (afftdn). fmax/fmin/rint + * are exact; log10 is routed through the fast single-precision log2f (see + * fastmathf.c) so afftdn's dB math stays correct. */ +extern float sofm_log2f(float x); + +double fmax(double a, double b) { return a > b ? a : b; } +double fmin(double a, double b) { return a < b ? a : b; } +long lrint(double x) { return (long)(x < 0.0 ? x - 0.5 : x + 0.5); } +long long llrintf(float x) { return (long long)(x < 0.0f ? x - 0.5f : x + 0.5f); } +double log10(double x) { return (double)sofm_log2f((float)x) * 0.30102999566398120; } + +/* ============================ vsnprintf / snprintf ======================= */ + +static void ffmpeg_dec_putc(char *buf, size_t size, size_t *pos, char c) +{ + if (*pos < size) + buf[*pos] = c; + (*pos)++; +} + +int vsnprintf(char *buf, size_t size, const char *fmt, va_list ap) +{ + size_t pos = 0; + + for (; *fmt; fmt++) { + int left = 0, zero = 0, alt = 0, lng = 0, width = 0, prec = -1; + unsigned long long uv = 0; + int isnum = 0, base = 10, upper = 0, neg = 0; + const char *s = NULL; + char c; + + if (*fmt != '%') { + ffmpeg_dec_putc(buf, size, &pos, *fmt); + continue; + } + + fmt++; + /* flags */ + for (;; fmt++) { + if (*fmt == '-') + left = 1; + else if (*fmt == '0') + zero = 1; + else if (*fmt == '#') + alt = 1; + else if (*fmt == '+' || *fmt == ' ') + continue; + else + break; + } + /* width */ + if (*fmt == '*') { + width = va_arg(ap, int); + fmt++; + if (width < 0) { + left = 1; + width = -width; + } + } else { + while (*fmt >= '0' && *fmt <= '9') + width = width * 10 + (*fmt++ - '0'); + } + /* precision */ + if (*fmt == '.') { + fmt++; + prec = 0; + if (*fmt == '*') { + prec = va_arg(ap, int); + fmt++; + } else { + while (*fmt >= '0' && *fmt <= '9') + prec = prec * 10 + (*fmt++ - '0'); + } + } + /* length modifiers */ + for (;;) { + if (*fmt == 'l') { + lng++; + fmt++; + } else if (*fmt == 'z' || *fmt == 't' || *fmt == 'j') { + lng = 2; + fmt++; + } else if (*fmt == 'h' || *fmt == 'L') { + fmt++; + } else { + break; + } + } + + c = *fmt; + switch (c) { + case '%': + ffmpeg_dec_putc(buf, size, &pos, '%'); + continue; + case 'c': + ffmpeg_dec_putc(buf, size, &pos, (char)va_arg(ap, int)); + continue; + case 's': + s = va_arg(ap, const char *); + if (!s) + s = "(null)"; + break; + case 'd': + case 'i': { + long long v = lng >= 2 ? va_arg(ap, long long) : + lng == 1 ? va_arg(ap, long) : va_arg(ap, int); + if (v < 0) { + neg = 1; + uv = (unsigned long long)-v; + } else { + uv = (unsigned long long)v; + } + isnum = 1; + break; + } + case 'u': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + isnum = 1; + break; + case 'o': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + base = 8; + isnum = 1; + break; + case 'x': + case 'X': + uv = lng >= 2 ? va_arg(ap, unsigned long long) : + lng == 1 ? va_arg(ap, unsigned long) : va_arg(ap, unsigned int); + base = 16; + upper = (c == 'X'); + isnum = 1; + break; + case 'p': + uv = (unsigned long long)(uintptr_t)va_arg(ap, void *); + base = 16; + alt = 1; + isnum = 1; + break; + case 'e': + case 'f': + case 'g': + case 'E': + case 'F': + case 'G': + case 'a': + case 'A': + /* float formatting unsupported (see file header) */ + (void)va_arg(ap, double); + s = "0"; + break; + default: + ffmpeg_dec_putc(buf, size, &pos, '%'); + ffmpeg_dec_putc(buf, size, &pos, c); + continue; + } + + if (isnum) { + char digits[24]; + int di = 0, i; + const char *hx = upper ? "0123456789ABCDEF" : "0123456789abcdef"; + int len, pad; + + if (uv == 0) { + digits[di++] = '0'; + } else { + while (uv) { + digits[di++] = hx[uv % base]; + uv /= base; + } + } + + len = di + (neg ? 1 : 0) + (alt && base == 16 ? 2 : 0); + pad = width > len ? width - len : 0; + + if (!left && !zero) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + if (neg) + ffmpeg_dec_putc(buf, size, &pos, '-'); + if (alt && base == 16) { + ffmpeg_dec_putc(buf, size, &pos, '0'); + ffmpeg_dec_putc(buf, size, &pos, upper ? 'X' : 'x'); + } + if (!left && zero) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, '0'); + for (i = di - 1; i >= 0; i--) + ffmpeg_dec_putc(buf, size, &pos, digits[i]); + if (left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + } else if (s) { + int len = 0, pad; + + while (s[len] && (prec < 0 || len < prec)) + len++; + pad = width > len ? width - len : 0; + + if (!left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + for (int i = 0; i < len; i++) + ffmpeg_dec_putc(buf, size, &pos, s[i]); + if (left) + while (pad-- > 0) + ffmpeg_dec_putc(buf, size, &pos, ' '); + } + } + + if (size) + buf[pos < size ? pos : size - 1] = '\0'; + + return (int)pos; +} + +int snprintf(char *buf, size_t size, const char *fmt, ...) +{ + va_list ap; + int n; + + va_start(ap, fmt); + n = vsnprintf(buf, size, fmt, ap); + va_end(ap); + return n; +} diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c b/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c new file mode 100644 index 000000000000..bb7cc7408d6a --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec-stub.c @@ -0,0 +1,65 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// Dependency-free stub backend for the ffmpeg_dec module. +// +// This lets the SOF module glue (init/prepare/process/config/reset/free, the +// LLEXT manifest and the topology wiring) be built and exercised in CI without +// pulling in libavcodec. It does not decode: it simply passes the input bytes +// through to the output, so the pipeline moves data end to end. The real +// libavcodec implementation lives in ffmpeg_dec-ffmpeg.c and provides the same +// struct ffmpeg_dec_backend symbol. + +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +LOG_MODULE_DECLARE(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +static int ffmpeg_dec_stub_init(struct processing_module *mod) +{ + comp_info(mod->dev, "ffmpeg_dec stub backend: no decoder linked"); + return 0; +} + +static int ffmpeg_dec_stub_configure(struct processing_module *mod) +{ + return 0; +} + +static int ffmpeg_dec_stub_decode(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced) +{ + /* Passthrough: copy as many bytes as fit, report exact consume/produce. */ + size_t n = MIN(in_size, out_size); + + if (n) + memcpy_s(out, out_size, in, n); + + *consumed = n; + *produced = n; + return 0; +} + +static int ffmpeg_dec_stub_reset(struct processing_module *mod) +{ + return 0; +} + +static int ffmpeg_dec_stub_free(struct processing_module *mod) +{ + return 0; +} + +const struct ffmpeg_dec_backend ffmpeg_dec_backend = { + .name = "stub", + .init = ffmpeg_dec_stub_init, + .configure = ffmpeg_dec_stub_configure, + .decode = ffmpeg_dec_stub_decode, + .reset = ffmpeg_dec_stub_reset, + .free = ffmpeg_dec_stub_free, +}; diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.c b/src/audio/ffmpeg_dec/ffmpeg_dec.c new file mode 100644 index 000000000000..92eed87345f5 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.c @@ -0,0 +1,322 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2026 Intel Corporation. +// +// FFmpeg (libavcodec) audio decoder wrapper - SOF module core. +// +// This translation unit contains the SOF module_interface glue only; the actual +// decoding is delegated to the backend selected at build time (see +// ffmpeg_dec-stub.c and ffmpeg_dec-ffmpeg.c). The input is a compressed audio +// elementary stream (raw data), the output is interleaved PCM. + +#include +#include +#include +#include +#include +#include "ffmpeg_dec.h" + +/* UUID identifies the component. Registered in uuid-registry.txt. */ +SOF_DEFINE_REG_UUID(ffmpeg_dec); + +/* Creates logging data for the component */ +LOG_MODULE_REGISTER(ffmpeg_dec, CONFIG_SOF_LOG_LEVEL); + +/* Decoder-mode ops (compressed -> PCM). In filter mode the module uses the + * avfilter-graph PCM effect ops from ffmpeg_dec-filter.c instead. + */ +#if !CONFIG_FFMPEG_DEC_FILTER_MODE +int ffmpeg_dec_store_extradata(struct processing_module *mod, + const uint8_t *data, size_t size) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + uint8_t *buf; + + if (!size) + return 0; + + if (size > FFMPEG_DEC_MAX_EXTRADATA) { + comp_err(dev, "extradata too large: %zu", size); + return -EINVAL; + } + + /* Replace any previously stored setup header. */ + if (cd->extradata) { + mod_free(mod, cd->extradata); + cd->extradata = NULL; + cd->extradata_size = 0; + } + + buf = mod_alloc(mod, size); + if (!buf) + return -ENOMEM; + + memcpy_s(buf, size, data, size); + cd->extradata = buf; + cd->extradata_size = size; + comp_info(dev, "stored %zu bytes of codec extradata", size); + return 0; +} + +/** + * ffmpeg_dec_init() - Initialize the ffmpeg_dec component. + * @mod: Pointer to module data. + * + * Allocates private data and hands off to the selected decode backend for its + * one-time initialization. __cold marks this non-critical path for slower DRAM. + * + * Return: Zero if success, otherwise error code. + */ +__cold static int ffmpeg_dec_init(struct processing_module *mod) +{ + struct module_data *md = &mod->priv; + struct comp_dev *dev = mod->dev; + struct ffmpeg_dec_comp_data *cd; + int ret; + + comp_info(dev, "entry"); + + cd = mod_zalloc(mod, sizeof(*cd)); + if (!cd) + return -ENOMEM; + + md->private = cd; + cd->backend = &ffmpeg_dec_backend; + /* TODO: derive codec id from topology/IPC init config. Until then use the + * default from the Kconfig decoder selection (first enabled). + */ + cd->codec = FFMPEG_DEC_DEFAULT_CODEC; + + comp_info(dev, "backend '%s'", cd->backend->name); + + if (cd->backend->init) { + ret = cd->backend->init(mod); + if (ret) { + comp_err(dev, "backend init failed %d", ret); + mod_free(mod, cd); + return ret; + } + } + + return 0; +} + +/** + * ffmpeg_dec_prepare() - Prepare the component for processing. + * @mod: Pointer to module data. + * @sources: Unused (input is a raw compressed byte stream). + * @sinks: Output PCM sink array; sinks[0] provides the target PCM format. + * + * Caches the decoded PCM output format and opens the backend decoder. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + int ret; + + comp_dbg(dev, "entry"); + + if (num_of_sinks != 1) { + comp_err(dev, "unsupported sink count %d", num_of_sinks); + return -EINVAL; + } + + /* Cache the PCM format the decoder must produce for the pipeline. */ + cd->out_rate = sink_get_rate(sinks[0]); + cd->out_channels = sink_get_channels(sinks[0]); + cd->out_frame_fmt = sink_get_frm_fmt(sinks[0]); + cd->out_frame_bytes = sink_get_frame_bytes(sinks[0]); + + comp_info(dev, "out rate %u ch %u fmt %d frame_bytes %u", + cd->out_rate, cd->out_channels, cd->out_frame_fmt, + cd->out_frame_bytes); + + if (cd->backend->configure) { + ret = cd->backend->configure(mod); + if (ret) { + comp_err(dev, "backend configure failed %d", ret); + return ret; + } + } + + cd->configured = true; + return 0; +} + +/** + * ffmpeg_dec_process() - Decode a chunk of the compressed stream. + * @mod: Pointer to module data. + * @input_buffers: input_buffers[0].data holds compressed bytes. + * @output_buffers: output_buffers[0].data receives interleaved PCM. + * + * Raw-data processing: hand the input bytes to the backend decoder and report + * how many input bytes were consumed and how many PCM bytes were produced. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_process(struct processing_module *mod, + struct input_stream_buffer *input_buffers, + int num_input_buffers, + struct output_stream_buffer *output_buffers, + int num_output_buffers) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + struct comp_dev *dev = mod->dev; + size_t consumed = 0; + size_t produced = 0; + int ret; + + if (num_input_buffers != 1 || num_output_buffers != 1) + return -EINVAL; + + if (!input_buffers[0].size) + return -ENODATA; + + ret = cd->backend->decode(mod, + input_buffers[0].data, input_buffers[0].size, + &consumed, + output_buffers[0].data, output_buffers[0].size, + &produced); + if (ret) { + comp_err(dev, "decode failed %d", ret); + return ret; + } + + input_buffers[0].consumed = consumed; + output_buffers[0].size = produced; + return 0; +} + +/** + * ffmpeg_dec_set_config() - Receive the codec setup header (extradata). + * + * Reassembles a possibly fragmented binary configuration via the common + * module_set_configuration() helper, then stores the whole reassembled blob as + * codec extradata (e.g. FLAC STREAMINFO). Mirrors the DTS codec config path. + */ +__cold static int +ffmpeg_dec_set_config(struct processing_module *mod, uint32_t config_id, + enum module_cfg_fragment_position pos, uint32_t data_offset_size, + const uint8_t *fragment, size_t fragment_size, uint8_t *response, + size_t response_size) +{ + struct comp_dev *dev = mod->dev; + struct module_config *config = &mod->priv.cfg; + uint32_t header_size; + int ret; + + assert_can_be_cold(); + + ret = module_set_configuration(mod, config_id, pos, data_offset_size, fragment, + fragment_size, response, response_size); + if (ret < 0) { + comp_err(dev, "module_set_configuration failed %d", ret); + return ret; + } + + /* Wait until the whole (possibly fragmented) blob has been received. */ + if (pos != MODULE_CFG_FRAGMENT_LAST && pos != MODULE_CFG_FRAGMENT_SINGLE) + return 0; + + /* module_config prepends a {size, avail} header to the payload. */ + header_size = sizeof(config->size) + sizeof(config->avail); + if (config->size <= header_size) { + comp_warn(dev, "empty codec config"); + return 0; + } + + return ffmpeg_dec_store_extradata(mod, config->data, + config->size - header_size); +} + +/** + * ffmpeg_dec_reset() - Reset the component to a re-preparable state. + * @mod: Pointer to module data. + * + * Flushes decoder state but keeps allocations so the pipeline can restart. + * + * Return: Zero if success, otherwise error code. + */ +static int ffmpeg_dec_reset(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + comp_dbg(mod->dev, "entry"); + + if (cd->backend->reset) + return cd->backend->reset(mod); + + return 0; +} + +/** + * ffmpeg_dec_free() - Free dynamic allocations. + * @mod: Pointer to module data. + * + * Return: Zero if success, otherwise error code. + */ +__cold static int ffmpeg_dec_free(struct processing_module *mod) +{ + struct ffmpeg_dec_comp_data *cd = module_get_private_data(mod); + + assert_can_be_cold(); + comp_dbg(mod->dev, "entry"); + + if (cd->backend->free) + cd->backend->free(mod); + + if (cd->extradata) + mod_free(mod, cd->extradata); + + mod_free(mod, cd); + return 0; +} +#endif /* !CONFIG_FFMPEG_DEC_FILTER_MODE */ + +/* This defines the module operations */ +#if CONFIG_FFMPEG_DEC_FILTER_MODE +/* PCM source/sink effect driving an avfilter graph (ffmpeg_dec-filter.c). */ +static const struct module_interface ffmpeg_dec_interface = { + .init = ffmpeg_af_mod_init, + .prepare = ffmpeg_af_mod_prepare, + .process = ffmpeg_af_mod_process, + .free = ffmpeg_af_mod_free +}; +#else +static const struct module_interface ffmpeg_dec_interface = { + .init = ffmpeg_dec_init, + .prepare = ffmpeg_dec_prepare, + .process_raw_data = ffmpeg_dec_process, + .set_configuration = ffmpeg_dec_set_config, + .reset = ffmpeg_dec_reset, + .free = ffmpeg_dec_free +}; +#endif + +/* If COMP_FFMPEG_DEC is =m in Kconfig this is built as a loadable LLEXT module. */ +#if CONFIG_COMP_FFMPEG_DEC_MODULE + +#include +#include +#include + +static const struct sof_man_module_manifest mod_manifest __section(".module") __used = + SOF_LLEXT_MODULE_MANIFEST("FFMPGDEC", &ffmpeg_dec_interface, 1, + SOF_REG_UUID(ffmpeg_dec), 40); + +SOF_LLEXT_BUILDINFO; + +#else + +/* Only used for the module adapter trace context, soon to be deprecated */ +DECLARE_TR_CTX(ffmpeg_dec_tr, SOF_UUID(ffmpeg_dec_uuid), LOG_LEVEL_INFO); +DECLARE_MODULE_ADAPTER(ffmpeg_dec_interface, ffmpeg_dec_uuid, ffmpeg_dec_tr); +SOF_MODULE_INIT(ffmpeg_dec, sys_comp_module_ffmpeg_dec_interface_init); + +#endif diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.h b/src/audio/ffmpeg_dec/ffmpeg_dec.h new file mode 100644 index 000000000000..63b42a96d8a3 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.h @@ -0,0 +1,151 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * FFmpeg (libavcodec) audio decoder wrapper for SOF. + * + * This module adapts SOF's module_interface to libavcodec's send-packet / + * receive-frame decode API. Compressed elementary-stream bytes arrive on the + * input (raw data) buffer, are parsed into codec frames, decoded to interleaved + * PCM and written to the output buffer. + * + * The actual decode work is delegated to a pluggable "backend" so the SOF glue + * can be validated with a dependency-free stub (ffmpeg_dec-stub.c) before the + * real libavcodec backend (ffmpeg_dec-ffmpeg.c) is linked in. + */ +#ifndef __SOF_AUDIO_FFMPEG_DEC_H__ +#define __SOF_AUDIO_FFMPEG_DEC_H__ + +#include +#include +#include +#include + +/* Maximum codec setup header (extradata) we accept, e.g. FLAC STREAMINFO. */ +#define FFMPEG_DEC_MAX_EXTRADATA 4096 + +/* Which codec this instance decodes. Kept as an explicit enum so it can be + * carried in topology/IPC config and mapped to an AVCodecID by the backend. + */ +enum ffmpeg_dec_codec { + FFMPEG_DEC_CODEC_NONE = 0, + FFMPEG_DEC_CODEC_FLAC, + FFMPEG_DEC_CODEC_AAC, + FFMPEG_DEC_CODEC_OPUS, +}; + +/* Default codec for a new instance, picked from the Kconfig selection until the + * codec is carried per-instance from topology/IPC config. First enabled wins. + */ +#if defined(CONFIG_FFMPEG_DEC_FLAC) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_FLAC +#elif defined(CONFIG_FFMPEG_DEC_AAC) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_AAC +#elif defined(CONFIG_FFMPEG_DEC_OPUS) +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_OPUS +#else +#define FFMPEG_DEC_DEFAULT_CODEC FFMPEG_DEC_CODEC_NONE +#endif + +struct ffmpeg_dec_comp_data; + +/** + * struct ffmpeg_dec_backend - decode backend operations. + * + * A backend owns the real decoder state (allocated by the backend itself and + * stored in ffmpeg_dec_comp_data.backend_data). All ops return 0 on success or + * a negative errno. + */ +struct ffmpeg_dec_backend { + const char *name; + + /* One-time backend init, called from module init(). */ + int (*init)(struct processing_module *mod); + + /* Open/configure the decoder once codec, extradata and the output PCM + * format (rate/channels/frame format) are known. Called from prepare(). + */ + int (*configure)(struct processing_module *mod); + + /* + * Decode from @in (in_size bytes of compressed stream) into @out (up to + * out_size bytes of interleaved PCM). On return *consumed holds input + * bytes eaten and *produced holds PCM bytes written. + */ + int (*decode)(struct processing_module *mod, + const uint8_t *in, size_t in_size, size_t *consumed, + uint8_t *out, size_t out_size, size_t *produced); + + /* Flush decoder state, keep it configured. Called from reset(). */ + int (*reset)(struct processing_module *mod); + + /* Tear down decoder state allocated by init()/configure(). */ + int (*free)(struct processing_module *mod); +}; + +/** + * struct ffmpeg_dec_comp_data - ffmpeg_dec module private data. + * @backend: Selected decode backend ops. + * @backend_data: Backend-private decoder state. + * @codec: Codec id for this instance. + * @extradata: Codec setup header (e.g. FLAC STREAMINFO), NULL if none. + * @extradata_size: Size of @extradata in bytes. + * @out_rate: Decoded PCM sample rate (Hz), from the sink stream params. + * @out_channels: Decoded PCM channel count. + * @out_frame_fmt: Decoded PCM sample format (enum sof_ipc_frame). + * @out_frame_bytes: Bytes per PCM frame (all channels). + * @configured: True once the backend decoder has been opened. + */ +struct ffmpeg_dec_comp_data { + const struct ffmpeg_dec_backend *backend; + void *backend_data; + enum ffmpeg_dec_codec codec; + uint8_t *extradata; + size_t extradata_size; + uint32_t out_rate; + uint32_t out_channels; + int out_frame_fmt; + uint32_t out_frame_bytes; + bool configured; + /* Filter mode (CONFIG_FFMPEG_DEC_FILTER_MODE): avfilter graph handle + * (struct ffmpeg_af_graph *, see ffmpeg_dec-filter.c). Unused for decode. + */ + void *af_graph; +}; + +/* Filter-mode module ops (PCM source/sink effect), in ffmpeg_dec-filter.c. */ +int ffmpeg_af_mod_init(struct processing_module *mod); +int ffmpeg_af_mod_prepare(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_process(struct processing_module *mod, + struct sof_source **sources, int num_of_sources, + struct sof_sink **sinks, int num_of_sinks); +int ffmpeg_af_mod_free(struct processing_module *mod); + +/* Backend instance provided by the selected backend translation unit + * (ffmpeg_dec-stub.c or ffmpeg_dec-ffmpeg.c). + */ +extern const struct ffmpeg_dec_backend ffmpeg_dec_backend; + +/** + * ffmpeg_dec_store_extradata() - Copy a codec setup header into private data. + * @mod: Pointer to module data. + * @data: Setup header bytes (e.g. FLAC STREAMINFO). + * @size: Number of bytes. + * + * Return: Zero if success, otherwise error code. + */ +int ffmpeg_dec_store_extradata(struct processing_module *mod, + const uint8_t *data, size_t size); + +/** + * ffmpeg_dec_libc_bind() - Bind the module whose heap backs malloc/free/realloc. + * @mod: Pointer to module data (see ffmpeg_dec-alloc.c). + * + * Called at the entry of every libavcodec backend op so allocations made by + * FFmpeg are drawn from the current instance's SOF heap. + */ +void ffmpeg_dec_libc_bind(struct processing_module *mod); + +#endif /* __SOF_AUDIO_FFMPEG_DEC_H__ */ diff --git a/src/audio/ffmpeg_dec/ffmpeg_dec.toml b/src/audio/ffmpeg_dec/ffmpeg_dec.toml new file mode 100644 index 000000000000..6c58fcb8b731 --- /dev/null +++ b/src/audio/ffmpeg_dec/ffmpeg_dec.toml @@ -0,0 +1,25 @@ +#ifndef LOAD_TYPE +#define LOAD_TYPE "0" +#endif + + REM # FFmpeg audio decoder module config + [[module.entry]] + REM # NOTE: module name is limited to 8 chars (manifest name field); + REM # it must match the SOF_LLEXT_MODULE_MANIFEST() name in ffmpeg_dec.c. + name = "FFMPGDEC" + uuid = UUIDREG_STR_FFMPEG_DEC + affinity_mask = "0x1" + instance_count = "40" + domain_types = "0" + load_type = LOAD_TYPE + module_type = "9" + auto_start = "0" + sched_caps = [1, 0x00008000] + REM # pin = [dir, type, sample rate, size, container, channel-cfg] + pin = [0, 0, 0xfeef, 0xf, 0xf, 0x45ff, 1, 0, 0xfeef, 0xf, 0xf, 0x1ff] + REM # mod_cfg [PAR_0 PAR_1 PAR_2 PAR_3 IS_BYTES CPS IBS OBS MOD_FLAGS CPC OBLS] + REM # Decoder: input is a small compressed chunk, output is larger PCM, so + REM # OBS (output buffer size) is sized well above IBS (input buffer size). + mod_cfg = [0, 0, 0, 0, 8192, 5000000, 4096, 16384, 0, 5000, 0] + + index = __COUNTER__ diff --git a/src/audio/ffmpeg_dec/llext/CMakeLists.txt b/src/audio/ffmpeg_dec/llext/CMakeLists.txt new file mode 100644 index 000000000000..d97db94c4f68 --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/CMakeLists.txt @@ -0,0 +1,62 @@ +# Copyright (c) 2026 Intel Corporation. +# SPDX-License-Identifier: Apache-2.0 + +if(CONFIG_COMP_FFMPEG_DEC_STUB) + # Dependency-free build: SOF glue + passthrough stub backend, no libavcodec. + sof_llext_build("ffmpeg_dec" + SOURCES ../ffmpeg_dec.c + ../ffmpeg_dec-stub.c + LIB openmodules + ) +else() + # Real decoder: cross-build the FFmpeg archive (Kconfig-selected decoders) + # and link it. libc/libm are NOT bundled (non-PIC, and would clash) — the + # module provides the needed libc symbols itself (shims/alloc), resolves the + # rest from the SOF core, and supplies fast single-precision float math + # (fastmathf.c) when a float codec (AAC/Opus) is selected. + include(${CMAKE_CURRENT_LIST_DIR}/../ffmpeg.cmake) + + set(ffmpeg_dec_srcs + ../ffmpeg_dec.c + ../ffmpeg_dec-ffmpeg.c + ../ffmpeg_dec-shims.c + ../ffmpeg_dec-alloc.c) + if(CONFIG_FFMPEG_DEC_FLOAT_MATH) + list(APPEND ffmpeg_dec_srcs ../fastmathf.c) + endif() + + # avfilter graph backend (PCM effects, e.g. afftdn noise reduction). Pulls + # libavfilter into the link; it must precede the libs it depends on. + set(ffmpeg_dec_libs avcodec swresample avutil) + if(CONFIG_FFMPEG_BUILD_AVFILTER) + list(APPEND ffmpeg_dec_srcs ../ffmpeg_dec-filter.c) + set(ffmpeg_dec_libs avfilter ${ffmpeg_dec_libs}) + endif() + + # Cold split: module code must also use -mlongcalls to reach FFmpeg code + # that now lives in the far (DRAM) .cold region. + set(ffmpeg_dec_cflags "") + if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + set(ffmpeg_dec_cflags -mlongcalls) + endif() + + sof_llext_build("ffmpeg_dec" + SOURCES ${ffmpeg_dec_srcs} + INCLUDES "${FFMPEG_INSTALL_DIR}/include" + LIBS_PATH "${FFMPEG_INSTALL_DIR}/lib" + LIBS ${ffmpeg_dec_libs} + CFLAGS ${ffmpeg_dec_cflags} + LIB openmodules + ) + + # The archive must be built before the module compiles/links against it. + add_dependencies(ffmpeg_dec_llext_lib ffmpeg_ext) + add_dependencies(ffmpeg_dec ffmpeg_ext) + + # Cold split: place the (renamed) FFmpeg .cold code so the loader relocates + # it to DRAM. See ffmpeg_cold.ld for why INSERT AFTER .bss. + if(CONFIG_FFMPEG_DEC_COLD_SPLIT) + llext_link_options(ffmpeg_dec + "-Wl,-T,${CMAKE_CURRENT_LIST_DIR}/ffmpeg_cold.ld") + endif() +endif() diff --git a/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld b/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld new file mode 100644 index 000000000000..1ddecd4dbfff --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/ffmpeg_cold.ld @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2026 Intel Corporation. + * + * LLEXT linker fragment for the ffmpeg_dec hot/cold split + * (CONFIG_FFMPEG_DEC_COLD_SPLIT). Collects the FFmpeg code (renamed to .cold in + * the archives by ffmpeg.cmake) plus any av_cold leftovers into an explicit + * .cold output section so the LLEXT loader (llext_manager) relocates it to DRAM. + * + * Placed with INSERT AFTER .bss so .cold gets the highest link-time address and + * cannot overlap the fixed-LMA .text/.rodata/.hash of the ace "mirrored" -shared + * layout (forcing it before/after .text overran those budgets). Its link address + * is irrelevant at runtime: the loader reads the .cold section and copies it to + * the cold/DRAM region. Cross-region .cold<->.text calls are handled by + * -mlongcalls (FFmpeg and module code). + */ +SECTIONS +{ + .cold : { + *(.cold .cold.*) + *(.literal.unlikely .literal.unlikely.*) + *(.text.unlikely .text.unlikely.*) + } +} INSERT AFTER .bss; diff --git a/src/audio/ffmpeg_dec/llext/llext.toml.h b/src/audio/ffmpeg_dec/llext/llext.toml.h new file mode 100644 index 000000000000..eda3fce26a5c --- /dev/null +++ b/src/audio/ffmpeg_dec/llext/llext.toml.h @@ -0,0 +1,6 @@ +#include +#define LOAD_TYPE "2" +#include "../ffmpeg_dec.toml" + +[module] +count = __COUNTER__ diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index db8a69e63646..11b8023c16c0 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -954,6 +954,7 @@ void sys_comp_module_drc_interface_init(void); void sys_comp_module_dts_interface_init(void); void sys_comp_module_eq_fir_interface_init(void); void sys_comp_module_eq_iir_interface_init(void); +void sys_comp_module_ffmpeg_dec_interface_init(void); void sys_comp_module_gain_interface_init(void); void sys_comp_module_google_rtc_audio_processing_interface_init(void); void sys_comp_module_google_ctc_audio_processing_interface_init(void); diff --git a/tools/rimage/config/lnl.toml.h b/tools/rimage/config/lnl.toml.h index faefbb8acadb..ab45dd1b9902 100644 --- a/tools/rimage/config/lnl.toml.h +++ b/tools/rimage/config/lnl.toml.h @@ -101,6 +101,9 @@ #if defined(CONFIG_COMP_FIR) || defined(LLEXT_FORCE_ALL_MODULAR) #include