diff --git a/CMakeLists.txt b/CMakeLists.txt index a9b83bd..4e8bfd3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,7 @@ find_package(TBB REQUIRED) # ================================================ add_subdirectory(arras) add_subdirectory(moonray) +add_subdirectory(profiling) add_subdirectory(rats) add_subdirectory(scripts) diff --git a/profiling/CMakeLists.txt b/profiling/CMakeLists.txt new file mode 100644 index 0000000..7e024ef --- /dev/null +++ b/profiling/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2026 DreamWorks Animation LLC +# SPDX-License-Identifier: Apache-2.0 + +list(APPEND CMAKE_MESSAGE_CONTEXT ProfilingTest) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) + +include(ProfileTest) +add_profile_tests() diff --git a/profiling/README.md b/profiling/README.md new file mode 100644 index 0000000..4c33537 --- /dev/null +++ b/profiling/README.md @@ -0,0 +1,264 @@ +# Profile render suite + +The profiling suite renders a collection of scenes from a scenes repo such as: +[dwanim/example-scenes](https://github.com/dwanim/example-scenes) repository +using MoonRay. It tracks rendering performance across builds +and, when canonical images have been generated, also validates visual +correctness via an image-diff step — similar to [RATS](../rats/README.md). + +The suite is built on [CTest](https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html) +and is driven by the `ProfileTest.cmake` module found in `cmake/`. At +configure time CMake scans `PROFILE_SCENES_DIR` for scene files and +registers up to three CTests per scene (`update`, `render`, `diff`). If +`PROFILE_SCENES_DIR` is not set, no profile tests are registered. + +## Test stages + +Each scene produces three CTest stages (mirroring RATS): + +| Stage | Label | Purpose | +|---|---|---| +| `update` | `profiling;update` | Renders once and saves the result as the canonical reference image in `PROFILE_CANONICAL_DIR`. | +| `render` | `profiling;render` | Renders the scene for profiling; saves a timestamped image+log and a fixed `.exr` used by the diff stage. | +| `diff` | `profiling;diff` | Runs `idiff` to compare the latest render result against the canonical. Requires `PROFILE_CANONICAL_DIR` at runtime. | + +> **Note:** `update` and `diff` tests are only registered when `idiff` is +> found at CMake configure time. The `render` test always runs regardless. + +## Quick start + +```bash +# 1. Clone the scene repo next to (or anywhere near) your source tree. +git clone git@github.com:dwanim/example-scenes.git /path/to/example-scenes + +# 2. Export the scene location so CMake picks it up automatically. +export PROFILE_SCENES_DIR=/path/to/example-scenes + +# Optional: override where images and logs are written (defaults to build tree). +export PROFILE_OUTPUT_DIR=/path/to/output + +# Optional: version tag embedded in output filenames (e.g. 1). +export PROFILE_VERSION=1 + +# Optional: directory where canonical images are stored (required for update/diff). +export PROFILE_CANONICAL_DIR=/path/to/canonicals + +# 3. Build as normal — no extra cmake flags needed. +rez-env buildtools -c "rez-build -i --variants 0" + +# 4. Run the profile tests from the build directory. +cd build/ +source ../../installs/openmoonray/scripts/setup.sh +ctest -L profiling -j $(nproc) +``` + +## Setting PROFILE_CANONICAL_DIR + +`PROFILE_CANONICAL_DIR` is the directory where canonical (reference) images +are stored. It is a **runtime** env var — checked at test time, not at +configure time — so there is no corresponding CMake cache variable. + +```bash +export PROFILE_CANONICAL_DIR=/path/to/canonicals +``` + +Canonical images are organized as: +``` +///.exr +# e.g. /path/to/canonicals/bitterli/bedroom/scalar/scene.exr +``` + +To generate canonicals for the first time, run the `update` stage: +```bash +ctest -L profiling -L update +``` + +Subsequently, the `diff` stage compares every new render against these +canonicals: +```bash +ctest -L profiling -L diff +``` + +## Setting PROFILE_SCENES_DIR + +`PROFILE_SCENES_DIR` can be supplied in two equivalent ways: + +| Method | Example | +|---|---| +| Shell environment variable (recommended) | `export PROFILE_SCENES_DIR=/path/to/example-scenes` | +| CMake cache variable | `rez-build -i -- -DPROFILE_SCENES_DIR=/path/to/example-scenes` | + +The environment variable is read at cmake-configure time by +`profiling/cmake/ProfileTest.cmake` and stored in the CMake cache, so +it only needs to be set when running `rez-build` (not when running `ctest`). + +## Setting PROFILE_OUTPUT_DIR + +`PROFILE_OUTPUT_DIR` controls where rendered images and log files are written. +When not set, output lands in `/profiling/profile/`. + +| Method | Example | +|---|---| +| Shell environment variable | `export PROFILE_OUTPUT_DIR=/path/to/output` | +| CMake cache variable | `rez-build -i -- -DPROFILE_OUTPUT_DIR=/path/to/output` | + +Like `PROFILE_SCENES_DIR`, this only needs to be set at `rez-build` time. + +## Setting PROFILE_VERSION + +`PROFILE_VERSION` is an optional version tag embedded in output filenames. +When set to `1`, output files look like `2026-06-03_1_sca_scene.txt`. +When empty (the default), the version field is omitted: `2026-06-03_sca_scene.txt`. + +Use `PROFILE_VERSION` to distinguish multiple runs performed on the same date +(e.g. `PROFILE_VERSION=1`, `PROFILE_VERSION=2`, …) so that earlier results are +not overwritten. + +| Method | Example | +|---|---| +| Shell environment variable | `export PROFILE_VERSION=1` | +| CMake cache variable | `rez-build -i -- -DPROFILE_VERSION=1` | + +## Supported scene types + +| Extension | Renderer | +|---|---| +| `.rdla` | `moonray` — three tests per scene: `sca`, `vec`, and `xpu` exec modes | + +All scene files found recursively under `PROFILE_SCENES_DIR` are included. + +## Test labels + +Every profile CTest is tagged with the following labels: + +| Label | Meaning | +|---|---| +| `profiling` | Selects all profile tests: `ctest -L profiling` | +| `update` | Selects only the update (canonical generation) stage | +| `render` | Selects only the render (profiling + result image) stage | +| `diff` | Selects only the diff (image comparison) stage | +| `moonray` | Selects by renderer: `ctest -L profiling -L moonray` | +| `sca`, `vec`, `xpu` | Selects by moonray exec mode: `ctest -L profiling -L sca` | + +## Test names + +Tests follow the same naming convention as RaTS: + +``` +--[.] +``` + +where `` is `update`, `render`, or `diff`; `` is `sca`, `vec`, +`xpu` (moonray); and `` is +the scene file's parent directory relative to `PROFILE_SCENES_DIR`. +`diff` tests also include the image filename suffix. For example: + +``` +bitterli/bedroom/scene.rdla -> + update-sca-bitterli/bedroom + render-sca-bitterli/bedroom + diff-sca-bitterli/bedroom-scene.exr + update-vec-bitterli/bedroom + render-vec-bitterli/bedroom + diff-vec-bitterli/bedroom-scene.exr + ... + +bitterli/coffee_maker/scene.usdc -> + update-hd-bitterli/coffee_maker + render-hd-bitterli/coffee_maker + diff-hd-bitterli/coffee_maker-scene.exr +``` + +The `profiling` label distinguishes these from RaTS tests when filtering +with `ctest -L`. + +Use `ctest -N -L profiling` to list all registered profile tests without +running them. + +## Output files + +Each scene gets its own output directory matching its test name path under +`PROFILE_OUTPUT_DIR` (or the build tree when unset). Files within that +directory are named with the date, optional version tag, mode, and scene stem +so that successive runs accumulate and can be compared over time: + +``` +/ + bitterli/ + bedroom/ + 2026-06-03_1_sca_scene.exr + 2026-06-03_1_sca_scene.txt + 2026-06-03_1_vec_scene.exr + 2026-06-03_1_vec_scene.txt + 2026-06-03_1_xpu_scene.exr + 2026-06-03_1_xpu_scene.txt + coffee_maker/ + 2026-06-03_1_sca_scene.exr + ... +``` + +| File | Contents | +|---|---| +| `[_version]__.exr` | Rendered image | +| `[_version]__.txt` | Combined stdout/stderr from the renderer | + +## Running a subset of tests + +```bash +# Run only the render stage (profiling, no diff) +ctest -L profiling -L render + +# Generate canonicals (first time or after a known-good build) +ctest -L profiling -L update + +# Run only the diff stage to check image quality +ctest -L profiling -L diff + +# Run only moonray profile tests +ctest -L profiling -L moonray + +# Run a single scene by name (supports regex) +ctest -R render-sca-bitterli/bedroom + +# Dry-run: list matching tests without executing them +ctest -N -L profiling +``` + +## GitHub Actions + +The `CI/Profile - Rocky9/Cobalt` workflow lives in the +[dwanim/openmoonray_dwa](https://github.com/dwanim/openmoonray_dwa) repository +at `.github/workflows/run_CI_Profile.yml`. It automates the full cycle on a +self-hosted runner: + +1. Clones `openmoonray_dwa` (with submodules, including this repo). +2. Clones the configured scenes repo (default `dwanim/example-scenes`) to + `example_scenes/`. +3. Validates that at least one scene file was found. +4. Calls `openmoonray_dwa/.github/workflows/scripts/build_profile.sh` which + exports `PROFILE_SCENES_DIR` and runs `rez-build`, then writes + `PROFILE_TEST_PACKAGE`, `PROFILE_VARIANT`, and `PROFILE_BUILD_DIR` to + `$GITHUB_ENV` for subsequent steps. +5. Calls `openmoonray_dwa/.github/workflows/scripts/run_profile.sh` which runs + `ctest` directly (bypassing `rez-test`) so that the label filter + `-L profiling -L ` can be applied at runtime. + +The workflow is triggered manually via **workflow_dispatch** with the +following inputs: + +| Input | Default | Description | +|---|---|---| +| `output_dir` | _(build tree)_ | Directory on the runner where images and logs are saved | +| `version_tag` | `1` | Version tag embedded in output filenames (e.g. `1`) | +| `scenes_repo` | `dwanim/example-scenes` | Example-scenes repository in `owner/repo` format | +| `scenes_ref` | `main` | Branch, tag, or SHA of the example-scenes repo to clone | +| `run_render` | `true` | Run the render tests (`ctest` label: `render`) | +| `run_update` | `false` | Run the update/canonical-generation tests (`ctest` label: `update`) | +| `run_diff` | `false` | Run the diff/image-comparison tests (`ctest` label: `diff`) | +| `canonical_dir` | _(empty)_ | Directory containing (or to write) canonical reference images; required when `run_update` or `run_diff` is true | +| `machine_type` | `cobalt` | Runner label / EF resource class | +| `borrow_from_EF` | `true` | Whether to borrow a render host from EF | +| `variant` | `0` | `package.py` variant index to build and profile | + +> **Note:** At least one of `run_render`, `run_update`, or `run_diff` must +> be enabled; the workflow fails fast if all three are false. diff --git a/profiling/cmake/ProfileTest.cmake b/profiling/cmake/ProfileTest.cmake new file mode 100644 index 0000000..e250c3c --- /dev/null +++ b/profiling/cmake/ProfileTest.cmake @@ -0,0 +1,328 @@ +# Copyright 2026 DreamWorks Animation LLC +# SPDX-License-Identifier: Apache-2.0 + +# ---------------------------------------------------------------------------- +# PROFILE_SCENES_DIR +# +# Path to a directory of profiling scenes (e.g. a local clone of +# dwanim/example-scenes). When not set, no profile tests are registered. +# May also be supplied via the PROFILE_SCENES_DIR environment variable at +# cmake-configure time; the cmake cache variable takes precedence. +# ---------------------------------------------------------------------------- +set(PROFILE_SCENES_DIR "" CACHE PATH + "Path to a directory of profiling scenes (e.g. a clone of \ +dwanim/example-scenes). When empty, no profile tests are added.") + +if(NOT PROFILE_SCENES_DIR AND DEFINED ENV{PROFILE_SCENES_DIR}) + set(PROFILE_SCENES_DIR "$ENV{PROFILE_SCENES_DIR}" CACHE PATH + "Path to a directory of profiling scenes." FORCE) +endif() + +# ---------------------------------------------------------------------------- +# PROFILE_OUTPUT_DIR +# +# Directory where rendered images and logs are written. Defaults to +# /profiling/profile when empty. +# May also be supplied via the PROFILE_OUTPUT_DIR environment variable at +# cmake-configure time; the cmake cache variable takes precedence. +# ---------------------------------------------------------------------------- +set(PROFILE_OUTPUT_DIR "" CACHE PATH + "Directory for profile render output (images and logs). \ +Defaults to /profiling/profile when empty.") + +if(NOT PROFILE_OUTPUT_DIR AND DEFINED ENV{PROFILE_OUTPUT_DIR}) + set(PROFILE_OUTPUT_DIR "$ENV{PROFILE_OUTPUT_DIR}" CACHE PATH + "Directory for profile render output." FORCE) +endif() + +# ---------------------------------------------------------------------------- +# PROFILE_VERSION +# +# Optional version tag embedded in output filenames. May also be supplied via +# the PROFILE_VERSION environment variable. See profiling/README.md for details. +# ---------------------------------------------------------------------------- +set(PROFILE_VERSION "" CACHE STRING + "Version tag embedded in output filenames (e.g. 1). Omitted when empty.") + +if(DEFINED ENV{PROFILE_VERSION}) + set(PROFILE_VERSION "$ENV{PROFILE_VERSION}" CACHE STRING + "Version tag embedded in output filenames." FORCE) +endif() + +# ---------------------------------------------------------------------------- +# add_profile_tests() +# +# Scans PROFILE_SCENES_DIR recursively for scene files and registers the +# profiling CTests (labeled 'profiling'). This function is a no-op when +# PROFILE_SCENES_DIR is unset or empty. See profiling/README.md for the +# supported scene types, output layout, test labels, and naming conventions. +# ---------------------------------------------------------------------------- +function(add_profile_tests) + if(NOT PROFILE_SCENES_DIR) + message(STATUS "PROFILE_SCENES_DIR is not set; skipping profile tests.") + return() + endif() + + if(NOT IS_DIRECTORY "${PROFILE_SCENES_DIR}") + message(WARNING "PROFILE_SCENES_DIR='${PROFILE_SCENES_DIR}' \ +is not a valid directory; skipping profile tests.") + return() + endif() + + # Python and idiff are only required when profiling tests are actually being + # registered (PROFILE_SCENES_DIR is set). Placing the calls here avoids + # making them hard configure-time dependencies for every build. + find_package(Python REQUIRED COMPONENTS Interpreter) + find_program(IDIFF idiff QUIET) + + cmake_path(NATIVE_PATH PROFILE_SCENES_DIR NORMALIZE scenes_dir) + + if(PROFILE_OUTPUT_DIR) + set(output_dir "${PROFILE_OUTPUT_DIR}") + else() + set(output_dir "${CMAKE_CURRENT_BINARY_DIR}/profile") + endif() + file(MAKE_DIRECTORY "${output_dir}") + + # profile_render.py is the profiling-specific wrapper that captures the + # renderer's output to a log file alongside the rendered image. + set(_render_script "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/profile_render.py") + set(_update_script "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/update_profile_canonical.py") + + set(_total 0) + + # Compute a CTest name following the same convention as RaTS: + # render-- + # e.g. bitterli/bedroom/scene.rdla -> render-sca-bitterli/bedroom + # The 'profiling' label (not the name) is what distinguishes these from + # RaTS tests when filtering with ctest -L. + macro(_profile_test_name mode rel_path out_var) + cmake_path(GET rel_path PARENT_PATH _ptn_parent) + set(${out_var} "render-${mode}-${_ptn_parent}") + endmacro() + + # Guard against two scenes that map to the same sanitized test name. + macro(_profile_check_unique tname) + if(TEST "${tname}") + message(FATAL_ERROR "Duplicate test name '${tname}'. \ +Two scenes in PROFILE_SCENES_DIR produce the same sanitized test name.") + endif() + endmacro() + + # --- moonray (.rdla) scenes ------------------------------------------- + file(GLOB_RECURSE _rdla_scenes CONFIGURE_DEPENDS "${scenes_dir}/*.rdla") + foreach(scene_path ${_rdla_scenes}) + cmake_path(RELATIVE_PATH scene_path + BASE_DIRECTORY "${scenes_dir}" + OUTPUT_VARIABLE rel_path) + + cmake_path(GET scene_path PARENT_PATH scene_dir) + + # Build the list of -in args: the .rdla file first, then any .rdlb + # files in the same directory (sorted for a deterministic order). + # Moonray scenes commonly split across multiple files, e.g. + # scene.rdla + scene.rdlb + set(_in_args -in "${scene_path}") + file(GLOB _rdlb_files CONFIGURE_DEPENDS "${scene_dir}/*.rdlb") + list(SORT _rdlb_files) + foreach(_rdlb ${_rdlb_files}) + list(APPEND _in_args -in "${_rdlb}") + endforeach() + + # Output subdir = the scene's parent directory relative to scenes_dir. + # The scene stem is passed separately and incorporated into the filename + # so that multiple scenes in the same directory remain distinct. + cmake_path(GET rel_path PARENT_PATH _scene_parent_subpath) + cmake_path(GET rel_path STEM _scene_stem) + if(_scene_parent_subpath) + set(_test_output_dir "${output_dir}/${_scene_parent_subpath}") + else() + set(_test_output_dir "${output_dir}") + endif() + + # Register one test per execution mode. + # Abbreviations match the RaTS convention: sca / vec / xpu. + foreach(_exec_mode IN ITEMS scalar vector xpu) + if(_exec_mode STREQUAL "scalar") + set(_abbrev "sca") + elseif(_exec_mode STREQUAL "vector") + set(_abbrev "vec") + else() + set(_abbrev "${_exec_mode}") # xpu + endif() + _profile_test_name("${_abbrev}" "${rel_path}" tname) + _profile_check_unique("${tname}") + + # Per-mode render dir: holds the fixed-name result image that the + # diff stage compares against the canonical. + set(_render_dir + "${CMAKE_CURRENT_BINARY_DIR}/render/${_scene_parent_subpath}/${_exec_mode}") + file(MAKE_DIRECTORY "${_render_dir}") + set(_result_image "${_render_dir}/${_scene_stem}.exr") + set(_result_image_args --result-image "${_result_image}") + + add_test(NAME "${tname}" + COMMAND ${Python_EXECUTABLE} "${_render_script}" + --output-dir "${_test_output_dir}" + --scene-name "${_scene_stem}" + --mode "${_abbrev}" + --version-tag "${PROFILE_VERSION}" + ${_result_image_args} + moonray + ${_in_args} + -exec_mode ${_exec_mode} + -info + WORKING_DIRECTORY "${scene_dir}" + ) + set_tests_properties("${tname}" PROPERTIES + LABELS "profiling;render;moonray;${_abbrev}" + TIMEOUT 3600 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + + # The update (canonical generation) and diff (image comparison) + # stages both require idiff, so register them together. + if(IDIFF) + # update test: render once and save as the canonical image. + set(_update_tname "update-${_abbrev}-${_scene_parent_subpath}") + _profile_check_unique("${_update_tname}") + add_test(NAME "${_update_tname}" + COMMAND ${Python_EXECUTABLE} "${_update_script}" + --exec-mode "${_exec_mode}" + --test-rel-path "${_scene_parent_subpath}" + --image-name "${_scene_stem}.exr" + moonray + ${_in_args} + -exec_mode ${_exec_mode} + -info + WORKING_DIRECTORY "${scene_dir}" + ) + set_tests_properties("${_update_tname}" PROPERTIES + LABELS "profiling;update;moonray;${_abbrev}" + TIMEOUT 3600 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + + # diff test: compare result against canonical. + set(_diff_tname "diff-${_abbrev}-${_scene_parent_subpath}-${_scene_stem}.exr") + _profile_check_unique("${_diff_tname}") + add_test(NAME "${_diff_tname}" + WORKING_DIRECTORY "${_render_dir}" + COMMAND ${CMAKE_COMMAND} + "-DEXEC_MODE=${_exec_mode}" + "-DIDIFF_TOOL=${IDIFF}" + "-DIMAGE_FILENAME=${_scene_stem}.exr" + "-DTEST_REL_PATH=${_scene_parent_subpath}" + -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/diff_profile.cmake" + ) + set_tests_properties("${_diff_tname}" PROPERTIES + LABELS "profiling;diff;moonray;${_abbrev}" + DEPENDS "${tname}" + TIMEOUT 300 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + endif() + + math(EXPR _total "${_total} + 1") + endforeach() + endforeach() + + # --- hd_render (.usd / .usdc / .usda) scenes -------------------------- + file(GLOB_RECURSE _usd_scenes CONFIGURE_DEPENDS + "${scenes_dir}/*.usd" + "${scenes_dir}/*.usdc" + "${scenes_dir}/*.usda" + ) + foreach(scene_path ${_usd_scenes}) + cmake_path(RELATIVE_PATH scene_path + BASE_DIRECTORY "${scenes_dir}" + OUTPUT_VARIABLE rel_path) + _profile_test_name("hd" "${rel_path}" tname) + _profile_check_unique("${tname}") + + cmake_path(GET scene_path PARENT_PATH scene_dir) + + cmake_path(GET rel_path PARENT_PATH _scene_parent_subpath) + cmake_path(GET rel_path STEM _scene_stem) + if(_scene_parent_subpath) + set(_test_output_dir "${output_dir}/${_scene_parent_subpath}") + else() + set(_test_output_dir "${output_dir}") + endif() + + # Per-mode render dir: holds the fixed-name result image that the + # diff stage compares against the canonical. + set(_render_dir + "${CMAKE_CURRENT_BINARY_DIR}/render/${_scene_parent_subpath}/default") + file(MAKE_DIRECTORY "${_render_dir}") + set(_result_image "${_render_dir}/${_scene_stem}.exr") + set(_result_image_args --result-image "${_result_image}") + + add_test(NAME "${tname}" + COMMAND ${Python_EXECUTABLE} "${_render_script}" + --output-dir "${_test_output_dir}" + --scene-name "${_scene_stem}" + --mode "hd" + --version-tag "${PROFILE_VERSION}" + ${_result_image_args} + hd_render + -in "${scene_path}" + WORKING_DIRECTORY "${scene_dir}" + ) + set_tests_properties("${tname}" PROPERTIES + LABELS "profiling;render;hd_render" + TIMEOUT 3600 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + + # The update (canonical generation) and diff (image comparison) + # stages both require idiff, so register them together. + if(IDIFF) + # update test: render once and save as the canonical image. + set(_update_tname "update-hd-${_scene_parent_subpath}") + _profile_check_unique("${_update_tname}") + add_test(NAME "${_update_tname}" + COMMAND ${Python_EXECUTABLE} "${_update_script}" + --exec-mode "default" + --test-rel-path "${_scene_parent_subpath}" + --image-name "${_scene_stem}.exr" + hd_render + -in "${scene_path}" + WORKING_DIRECTORY "${scene_dir}" + ) + set_tests_properties("${_update_tname}" PROPERTIES + LABELS "profiling;update;hd_render" + TIMEOUT 3600 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + + # diff test: compare result against canonical. + set(_diff_tname "diff-hd-${_scene_parent_subpath}-${_scene_stem}.exr") + _profile_check_unique("${_diff_tname}") + add_test(NAME "${_diff_tname}" + WORKING_DIRECTORY "${_render_dir}" + COMMAND ${CMAKE_COMMAND} + "-DEXEC_MODE=default" + "-DIDIFF_TOOL=${IDIFF}" + "-DIMAGE_FILENAME=${_scene_stem}.exr" + "-DTEST_REL_PATH=${_scene_parent_subpath}" + -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/diff_profile.cmake" + ) + set_tests_properties("${_diff_tname}" PROPERTIES + LABELS "profiling;diff;hd_render" + DEPENDS "${tname}" + TIMEOUT 300 + ENVIRONMENT "PYTHONPATH=$ENV{PYTHONPATH}:$ENV{OIIO_PYTHON}" + ) + endif() + + math(EXPR _total "${_total} + 1") + endforeach() + + if(_total EQUAL 0) + message(FATAL_ERROR "No scene files (.rdla, .usd, .usdc, .usda) \ +found in '${PROFILE_SCENES_DIR}'. Ensure the scenes repository was cloned correctly \ +and PROFILE_SCENES_DIR points to the right directory.") + endif() + message(STATUS "Registered ${_total} profile test(s) from: ${PROFILE_SCENES_DIR}") +endfunction() diff --git a/profiling/cmake/diff_profile.cmake b/profiling/cmake/diff_profile.cmake new file mode 100644 index 0000000..53a517c --- /dev/null +++ b/profiling/cmake/diff_profile.cmake @@ -0,0 +1,71 @@ +# Copyright 2026 DreamWorks Animation LLC +# SPDX-License-Identifier: Apache-2.0 + +# ===================================================================================== +# This script is executed during profiling CTest 'diff' stages via: +# +# cmake -DEXEC_MODE=... -DIDIFF_TOOL=... -DIMAGE_FILENAME=... \ +# -DTEST_REL_PATH=... -P diff_profile.cmake +# +# It compares a rendered result image against the stored canonical using idiff. +# The WORKING_DIRECTORY of the test must be the directory that contains the +# result image (IMAGE_FILENAME is relative to WORKING_DIRECTORY). +# ------------------------------------------------------------------------------------- + +# Validate required definitions. +foreach(required_def + EXEC_MODE # scalar | vector | xpu | default + IDIFF_TOOL # full path to the OpenImageIO 'idiff' command + IMAGE_FILENAME # image filename relative to WORKING_DIRECTORY, e.g. scene.exr + TEST_REL_PATH) # scene parent path, e.g. bitterli/bedroom + if(NOT DEFINED ${required_def}) + message(FATAL_ERROR "[ProfileDiff] ${required_def} is undefined") + endif() +endforeach() + +# Require PROFILE_CANONICAL_DIR at runtime (not a CMake cache var). +if(NOT DEFINED ENV{PROFILE_CANONICAL_DIR}) + message(FATAL_ERROR + "[ProfileDiff] PROFILE_CANONICAL_DIR is not set.\n" + "Run 'ctest -L profiling -L update' first to generate canonicals, then\n" + "set PROFILE_CANONICAL_DIR to the directory containing them.") +endif() + +set(canonicals_root "$ENV{PROFILE_CANONICAL_DIR}") +cmake_path(NORMAL_PATH canonicals_root) + +if(NOT EXISTS "${canonicals_root}") + message(FATAL_ERROR + "[ProfileDiff] PROFILE_CANONICAL_DIR '${canonicals_root}' does not exist.") +endif() + +set(canonical_image + "${canonicals_root}/${TEST_REL_PATH}/${EXEC_MODE}/${IMAGE_FILENAME}") + +if(NOT EXISTS "${canonical_image}") + message(FATAL_ERROR + "[ProfileDiff] Canonical not found: ${canonical_image}\n" + "Run 'ctest -L profiling -L update' to generate it.") +endif() + +if(NOT EXISTS "${IMAGE_FILENAME}") + message(FATAL_ERROR + "[ProfileDiff] Result image not found: ${IMAGE_FILENAME}\n" + "Run 'ctest -L profiling -L render' first.") +endif() + +message(STATUS "[ProfileDiff] Comparing:") +message(STATUS " canonical: ${canonical_image}") +message(STATUS " result: ${IMAGE_FILENAME} (relative to test working dir)") + +execute_process( + COMMAND "${IDIFF_TOOL}" -a -v -abs "${canonical_image}" "${IMAGE_FILENAME}" + RESULT_VARIABLE idiff_result +) + +if(NOT idiff_result EQUAL 0) + message(FATAL_ERROR + "[ProfileDiff] Images differ:\n" + " canonical: ${canonical_image}\n" + " result: ${IMAGE_FILENAME}") +endif() diff --git a/profiling/cmake/profile_render.py b/profiling/cmake/profile_render.py new file mode 100644 index 0000000..129ec23 --- /dev/null +++ b/profiling/cmake/profile_render.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 + +# Copyright 2026 DreamWorks Animation LLC +# SPDX-License-Identifier: Apache-2.0 + +""" +Render wrapper for the profiling suite. + +Runs the renderer and simultaneously streams its combined stdout/stderr to the +terminal (so CTest can display it) and to a dated log file. Output +files are placed in a per-test subdirectory with names like: + + 2026-06-03_1_sca_scene.txt (log) + 2026-06-03_1_sca_scene.exr (rendered image) + +Multiple runs accumulate in the same directory, enabling performance tracking +over time. + +Usage: + profile_render.py --output-dir --mode [--version-tag ] + [renderer-args ...] + +Inherits the RATS_MOONRAY_THREADS behaviour from render.py: when the +environment variable is set to a positive integer, '-threads N' is injected +into the moonray command line. +""" + +import argparse +import os +import shutil +import subprocess +import sys +from datetime import datetime + + +def main(): + parser = argparse.ArgumentParser( + description='Profiling render wrapper: runs the renderer and saves dated output.' + ) + parser.add_argument( + '--output-dir', + required=True, + metavar='DIR', + help='Per-test output directory. Images and logs are written here.' + ) + parser.add_argument( + '--scene-name', + required=True, + metavar='NAME', + help='Scene stem name included in the output filename (e.g. veach-mis).' + ) + parser.add_argument( + '--mode', + required=True, + metavar='MODE', + help='Exec-mode abbreviation embedded in the filename (e.g. sca, vec, xpu, hd).' + ) + parser.add_argument( + '--version-tag', + default='', + metavar='VER', + help='Version string embedded in the filename (e.g. 1). Omitted when empty.' + ) + parser.add_argument( + '--result-image', + default='', + metavar='PATH', + help='When set, also copy the rendered image to this fixed path after a ' + 'successful render. Used by the diff test stage.' + ) + parser.add_argument( + 'renderer', + help='Renderer executable (moonray, hd_render, etc.)' + ) + parser.add_argument( + 'args', + nargs=argparse.REMAINDER, + help='Arguments forwarded verbatim to the renderer.' + ) + + parsed = parser.parse_args() + + # Build the base name: YYYY-MM-DD[_version]_mode_scenename. + date_str = datetime.now().strftime('%Y-%m-%d') + if parsed.version_tag: + base_name = f"{date_str}_{parsed.version_tag}_{parsed.mode}_{parsed.scene_name}" + else: + base_name = f"{date_str}_{parsed.mode}_{parsed.scene_name}" + + os.makedirs(parsed.output_dir, exist_ok=True) + log_path = os.path.join(parsed.output_dir, base_name + '.txt') + img_path = os.path.join(parsed.output_dir, base_name + '.exr') + + cmd = [parsed.renderer] + + # Inject -threads for moonray when RATS_MOONRAY_THREADS is set. + if parsed.renderer == 'moonray': + moonray_threads = os.getenv('RATS_MOONRAY_THREADS', '') + if moonray_threads: + try: + num_threads = int(moonray_threads) + if num_threads > 0: + cmd.extend(['-threads', str(num_threads)]) + print(f'[ProfileTest] Using {num_threads} threads for moonray', flush=True) + except ValueError: + print( + f'[ProfileTest WARNING] RATS_MOONRAY_THREADS={moonray_threads!r} ' + 'is not a valid integer, ignoring.', + file=sys.stderr + ) + + cmd.extend(parsed.args) + # profile_render.py owns -out so the filename can be determined at run time. + cmd.extend(['-out', img_path]) + + with open(log_path, 'w') as log: + log.write(f"Command: {' '.join(cmd)}\n\n") + log.flush() + + try: + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + ) + + for line in process.stdout: + sys.stdout.write(line) + sys.stdout.flush() + log.write(line) + log.flush() + + process.wait() + returncode = process.returncode + + if returncode == 0 and parsed.result_image: + os.makedirs( + os.path.dirname(os.path.abspath(parsed.result_image)), + exist_ok=True + ) + shutil.copy2(img_path, parsed.result_image) + msg = ( + f'[ProfileRender] Result image: {parsed.result_image}\n' + ) + sys.stdout.write(msg) + log.write(msg) + + return returncode + + except Exception as e: + msg = f'Error executing renderer: {e}\n' + sys.stderr.write(msg) + log.write(msg) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/profiling/cmake/update_profile_canonical.py b/profiling/cmake/update_profile_canonical.py new file mode 100644 index 0000000..19fffe6 --- /dev/null +++ b/profiling/cmake/update_profile_canonical.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +# Copyright 2026 DreamWorks Animation LLC +# SPDX-License-Identifier: Apache-2.0 + +""" +Render a profiling scene once and save the result as the canonical image. + +Called by the CTest 'update' stage registered in ProfileTest.cmake: + + update_profile_canonical.py --exec-mode --test-rel-path + --image-name + [renderer-args ...] + +The canonical image is written to: + $PROFILE_CANONICAL_DIR/// + +PROFILE_CANONICAL_DIR must be set in the environment before running. +Inherits the RATS_MOONRAY_THREADS behaviour: when set to a positive integer, +'-threads N' is injected into the moonray command line. +""" + +import argparse +import os +import shutil +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser( + description='Generate a canonical image for a profiling scene.' + ) + parser.add_argument( + '--exec-mode', + required=True, + metavar='MODE', + help='Execution mode used as the canonical subdirectory name ' + '(e.g. scalar, vector, xpu, default).' + ) + parser.add_argument( + '--test-rel-path', + required=True, + metavar='PATH', + help='Scene parent path relative to PROFILE_SCENES_DIR ' + '(e.g. bitterli/bedroom).' + ) + parser.add_argument( + '--image-name', + required=True, + metavar='FILE', + help='Output image filename (e.g. scene.exr).' + ) + parser.add_argument( + 'renderer_cmd', + nargs=argparse.REMAINDER, + help='Renderer executable and all its arguments. ' + '-out is appended automatically.' + ) + + parsed = parser.parse_args() + + canonical_dir_root = os.getenv('PROFILE_CANONICAL_DIR') + if not canonical_dir_root: + print('[ProfileUpdate] ERROR: PROFILE_CANONICAL_DIR is not set.', + file=sys.stderr) + print('[ProfileUpdate] Set this environment variable to the directory', + file=sys.stderr) + print('[ProfileUpdate] where canonical images should be stored.', + file=sys.stderr) + sys.exit(1) + + canonical_dir = os.path.join( + canonical_dir_root, parsed.test_rel_path, parsed.exec_mode + ) + os.makedirs(canonical_dir, exist_ok=True) + canonical_path = os.path.join(canonical_dir, parsed.image_name) + + # Render to a temp path first so a partial render never corrupts the canonical. + temp_path = canonical_path + '.updating' + + renderer_args = parsed.renderer_cmd + if not renderer_args: + print('[ProfileUpdate] ERROR: no renderer command was provided.', + file=sys.stderr) + sys.exit(1) + + renderer = renderer_args[0] + cmd = list(renderer_args) + + # Inject -threads for moonray when RATS_MOONRAY_THREADS is set. + if renderer == 'moonray': + threads = os.getenv('RATS_MOONRAY_THREADS', '') + if threads: + try: + n = int(threads) + if n > 0: + cmd = [renderer, '-threads', str(n)] + cmd[1:] + print(f'[ProfileUpdate] Using {n} moonray threads.', + flush=True) + except ValueError: + print( + f'[ProfileUpdate] WARNING: RATS_MOONRAY_THREADS={threads!r} ' + 'is not a valid integer, ignoring.', + file=sys.stderr + ) + + cmd.extend(['-out', temp_path]) + + print( + f'[ProfileUpdate] Rendering canonical: ' + f'{parsed.test_rel_path}/{parsed.exec_mode}/{parsed.image_name}', + flush=True + ) + print(f'[ProfileUpdate] Command: {" ".join(cmd)}', flush=True) + + result = subprocess.run(cmd) + + if result.returncode != 0: + if os.path.exists(temp_path): + os.unlink(temp_path) + print( + f'[ProfileUpdate] Render failed (exit {result.returncode}).', + file=sys.stderr + ) + sys.exit(result.returncode) + + if not os.path.exists(temp_path): + print( + f'[ProfileUpdate] ERROR: renderer did not produce {temp_path}.', + file=sys.stderr + ) + sys.exit(1) + + shutil.move(temp_path, canonical_path) + print(f'[ProfileUpdate] Canonical saved: {canonical_path}', flush=True) + + +main()