From 6801b121ae89527de81fe8e142403b273032f872 Mon Sep 17 00:00:00 2001 From: mwlasiuk Date: Fri, 14 Aug 2026 22:23:18 +0200 Subject: [PATCH 1/2] Add python utiliti scripts --- .github/workflows/check_code_formating.yml | 2 +- _Update.bat | 3 - scripts/README.md | 6 + .../check_clang_format.py | 24 +- .../run_clang_format.py | 3 +- scripts/submodules_sync_update.py | 43 ++ scripts/update_submodules.py | 374 ++++++++++++++++++ 7 files changed, 446 insertions(+), 9 deletions(-) delete mode 100644 _Update.bat create mode 100644 scripts/README.md rename check_clang_format.py => scripts/check_clang_format.py (81%) rename run_clang_format.py => scripts/run_clang_format.py (90%) mode change 100755 => 100644 create mode 100644 scripts/submodules_sync_update.py create mode 100644 scripts/update_submodules.py diff --git a/.github/workflows/check_code_formating.yml b/.github/workflows/check_code_formating.yml index e398db98..d054d2bd 100644 --- a/.github/workflows/check_code_formating.yml +++ b/.github/workflows/check_code_formating.yml @@ -38,4 +38,4 @@ jobs: clang-format --version - name: Run clang-format check - run: python3 check_clang_format.py \ No newline at end of file + run: python3 scripts/check_clang_format.py \ No newline at end of file diff --git a/_Update.bat b/_Update.bat deleted file mode 100644 index 02b9e337..00000000 --- a/_Update.bat +++ /dev/null @@ -1,3 +0,0 @@ -git pull --recurse-submodules -git submodule update --init --recursive -pause \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..922a033e --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,6 @@ +# Repository management utilities + +- check_clang_format.py - script for CI to check whether clang-format was ran before commiting +- run_clang_format.py - runs clang-format on all project files +- submodules_sync_update.py - performs ```git submodule sync && git submodule update --init --recursive``` in case repository was cloned without ```--recursive``` option +- update_submodules.py - updates all submodules to pinned branch and performs code fetch (for submodules without explicit branch pin it will report no pinned branch and do nothing) \ No newline at end of file diff --git a/check_clang_format.py b/scripts/check_clang_format.py similarity index 81% rename from check_clang_format.py rename to scripts/check_clang_format.py index f8b99a43..0165ce81 100644 --- a/check_clang_format.py +++ b/scripts/check_clang_format.py @@ -4,7 +4,8 @@ import subprocess import sys -PROJECT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) +PROJECT_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "..")) FILE_LOCATIONS = [ os.path.join(PROJECT_DIRECTORY, "core"), @@ -24,11 +25,13 @@ def get_files(input_paths: list[str], extensions: list[str]): files = [] + for input_path in input_paths: for dirpath, _, filenames in os.walk(input_path): for filename in filenames: if filename.endswith(tuple(extensions)): files.append(os.path.normpath(os.path.join(dirpath, filename))) + return files @@ -45,17 +48,28 @@ def main(): formatted_files = set(get_files(FILE_LOCATIONS, FILE_EXTENSIONS)) formatted_files = { - os.path.relpath(path, PROJECT_DIRECTORY) - for path in formatted_files + os.path.relpath(path, PROJECT_DIRECTORY) for path in formatted_files } - result = run([sys.executable, "run_clang_format.py"]) + clang_format_script = os.path.join( + SCRIPT_DIR, + "run_clang_format.py", + ) + + result = run( + [ + sys.executable, + clang_format_script, + ] + ) + if result.returncode != 0: print("ERROR: run_clang_format.py failed") print(result.stderr) sys.exit(result.returncode) status = run(["git", "status", "--porcelain"]) + if status.returncode != 0: print("ERROR: git status failed") print(status.stderr) @@ -71,8 +85,10 @@ def main(): if offending_files: print("ERROR: clang-format produced changes in the following files:") + for f in offending_files: print(f" {f}") + sys.exit(1) print("clang-format check passed") diff --git a/run_clang_format.py b/scripts/run_clang_format.py old mode 100755 new mode 100644 similarity index 90% rename from run_clang_format.py rename to scripts/run_clang_format.py index 35dc605a..3f78d1a5 --- a/run_clang_format.py +++ b/scripts/run_clang_format.py @@ -3,7 +3,8 @@ import os import subprocess -PROJECT_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__file__))) +SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__)) +PROJECT_DIRECTORY = os.path.abspath(os.path.join(SCRIPT_DIR, "..")) FILE_LOCATIONS = [os.path.join(PROJECT_DIRECTORY, 'core'), os.path.join(PROJECT_DIRECTORY, 'core_hd_mapping'), diff --git a/scripts/submodules_sync_update.py b/scripts/submodules_sync_update.py new file mode 100644 index 00000000..ff174900 --- /dev/null +++ b/scripts/submodules_sync_update.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) + + +def run_git(args): + result = subprocess.run( + ["git"] + args, + cwd=REPO_ROOT, + ) + + if result.returncode != 0: + print("ERROR: git " + " ".join(args)) + return False + + return True + + +def main(): + if not os.path.isdir(os.path.join(REPO_ROOT, ".git")): + print("ERROR: repository root not found") + sys.exit(1) + + print("Running: git submodule sync") + if not run_git(["submodule", "sync"]): + sys.exit(1) + + print() + print("Running: git submodule update --init --recursive") + if not run_git(["submodule", "update", "--init", "--recursive"]): + sys.exit(1) + + print() + print("Submodules synchronized and initialized successfully.") + + +if __name__ == "__main__": + main() diff --git a/scripts/update_submodules.py b/scripts/update_submodules.py new file mode 100644 index 00000000..2450f1c9 --- /dev/null +++ b/scripts/update_submodules.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys + + +# ============================================================================ +# Configuration +# ============================================================================ + +SUBMODULES_DIR = "3rdparty" + + +# ============================================================================ +# Paths +# ============================================================================ + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +REPO_ROOT = os.path.dirname(SCRIPT_DIR) +SUBMODULES_ROOT = os.path.join(REPO_ROOT, SUBMODULES_DIR) + + +# ============================================================================ +# Git helpers +# ============================================================================ + +def run_git(args, cwd, quiet=False): + result = subprocess.run( + ["git"] + args, + cwd=cwd, + text=True, + capture_output=True, + ) + + if result.returncode != 0: + if not quiet: + print("ERROR: git " + " ".join(args)) + + if result.stderr: + print(result.stderr.strip()) + + return None + + return result.stdout.strip() + + +# ============================================================================ +# Submodules +# ============================================================================ + +def get_submodules(): + output = run_git( + [ + "config", + "--file", + ".gitmodules", + "--get-regexp", + r"^submodule\..*\.path$", + ], + REPO_ROOT, + ) + + if output is None: + return [] + + submodules = [] + + for line in output.splitlines(): + key, path = line.split(None, 1) + + # Extract submodule name from: + # submodule.NAME.path + name = key[len("submodule."):-len(".path")] + + path = os.path.normpath(path) + + # Only include submodules located inside SUBMODULES_DIR. + if ( + path == SUBMODULES_DIR + or path.startswith(SUBMODULES_DIR + os.sep) + ): + submodules.append((name, path)) + + return submodules + + +def get_branch(submodule_name): + return run_git( + [ + "config", + "--file", + ".gitmodules", + "--get", + "submodule." + submodule_name + ".branch", + ], + REPO_ROOT, + quiet=True, + ) + + +# ============================================================================ +# Update +# ============================================================================ + +def update_submodule(submodule_name, submodule_path): + submodule_dir = os.path.join(REPO_ROOT, submodule_path) + + print() + print("=" * 70) + print("Updating:", submodule_path) + print("=" * 70) + + branch = get_branch(submodule_name) + + if branch is None: + print("ERROR: no branch configured in .gitmodules") + print("Submodule:", submodule_name) + print("Skipping:", submodule_path) + + return False, None, False, "No branch configured" + + print("Pinned branch:", branch) + + # ------------------------------------------------------------------------ + # Initialize missing submodule + # ------------------------------------------------------------------------ + + if not os.path.isdir(submodule_dir): + result = subprocess.run( + [ + "git", + "submodule", + "update", + "--init", + "--", + submodule_path, + ], + cwd=REPO_ROOT, + ) + + if result.returncode != 0: + print("ERROR: failed to initialize submodule") + + return False, branch, False, "Initialization failed" + + # ------------------------------------------------------------------------ + # Fetch + # ------------------------------------------------------------------------ + + result = subprocess.run( + ["git", "fetch", "origin"], + cwd=submodule_dir, + ) + + if result.returncode != 0: + print("ERROR: fetch failed") + + return False, branch, False, "Fetch failed" + + # ------------------------------------------------------------------------ + # Get current branch + # ------------------------------------------------------------------------ + + current_branch = run_git( + ["symbolic-ref", "--short", "HEAD"], + submodule_dir, + quiet=True, + ) + + # ------------------------------------------------------------------------ + # Switch to pinned branch if necessary + # ------------------------------------------------------------------------ + + if current_branch != branch: + if current_branch is None: + print("Switching: HEAD ->", branch) + else: + print("Switching:", current_branch, "->", branch) + + result = subprocess.run( + ["git", "checkout", branch], + cwd=submodule_dir, + ) + + if result.returncode != 0: + # Local branch does not exist. + result = subprocess.run( + [ + "git", + "checkout", + "-b", + branch, + "--track", + "origin/" + branch, + ], + cwd=submodule_dir, + ) + + if result.returncode != 0: + print("ERROR: checkout failed") + + return False, branch, False, "Checkout failed" + + # ------------------------------------------------------------------------ + # Get local and remote commits BEFORE pull + # ------------------------------------------------------------------------ + + before_commit = run_git( + ["rev-parse", "HEAD"], + submodule_dir, + quiet=True, + ) + + remote_commit = run_git( + ["rev-parse", "origin/" + branch], + submodule_dir, + quiet=True, + ) + + if before_commit is None: + print("ERROR: could not determine local commit") + + return False, branch, False, "Could not determine local commit" + + if remote_commit is None: + print("ERROR: could not determine remote commit") + + return False, branch, False, "Could not determine remote commit" + + # ------------------------------------------------------------------------ + # Pull + # ------------------------------------------------------------------------ + + result = subprocess.run( + [ + "git", + "pull", + "--ff-only", + "origin", + branch, + ], + cwd=submodule_dir, + ) + + if result.returncode != 0: + print("ERROR: pull failed") + + return False, branch, False, "Pull failed" + + # ------------------------------------------------------------------------ + # Get final commit + # ------------------------------------------------------------------------ + + after_commit = run_git( + ["rev-parse", "HEAD"], + submodule_dir, + quiet=True, + ) + + if after_commit is None: + print("ERROR: could not determine final commit") + + return False, branch, False, "Could not determine final commit" + + # ------------------------------------------------------------------------ + # Determine whether anything was actually pulled + # ------------------------------------------------------------------------ + + updated = before_commit != after_commit + + if updated: + print("Updated successfully:", submodule_path) + + return True, branch, True, "Changes pulled" + + print("Already up to date:", submodule_path) + + return True, branch, False, "Already up to date" + + +# ============================================================================ +# Main +# ============================================================================ + +def main(): + if not os.path.isdir(os.path.join(REPO_ROOT, ".git")): + print("ERROR: repository root not found") + sys.exit(1) + + if not os.path.isdir(SUBMODULES_ROOT): + print( + "ERROR: submodule directory does not exist:", + SUBMODULES_ROOT, + ) + sys.exit(1) + + submodules = get_submodules() + + if not submodules: + print( + "No submodules found under:", + SUBMODULES_DIR, + ) + return + + print("Found submodules:") + + for submodule_name, submodule_path in submodules: + print(" ", submodule_path) + + success = True + results = [] + + for submodule_name, submodule_path in submodules: + result, branch, updated, reason = update_submodule( + submodule_name, + submodule_path, + ) + + if not result: + success = False + + results.append( + ( + submodule_path, + branch if branch is not None else "-", + updated, + reason, + ) + ) + + # ------------------------------------------------------------------------ + # Summary + # ------------------------------------------------------------------------ + + print() + print("=" * 105) + print("SUBMODULE UPDATE SUMMARY") + print("=" * 105) + + print( + "{:<40} {:<20} {:<10} {}".format( + "Submodule", + "Branch", + "Updated", + "Reason", + ) + ) + + print("-" * 105) + + for submodule_path, branch, updated, reason in results: + print( + "{:<40} {:<20} {:<10} {}".format( + submodule_path, + branch, + "YES" if updated else "NO", + reason, + ) + ) + + print("=" * 105) + + if success: + print("All submodules updated successfully.") + sys.exit(0) + + print("Some submodules failed to update.") + sys.exit(1) + + +if __name__ == "__main__": + main() \ No newline at end of file From d8fa0eb1e83851709650f45e7604ece2653fa298 Mon Sep 17 00:00:00 2001 From: mwlasiuk Date: Sat, 15 Aug 2026 14:13:20 +0200 Subject: [PATCH 2/2] Update naming and readme --- .github/workflows/check_code_formating.yml | 2 +- scripts/README.md | 27 ++++++++++++++++--- ...ng_format.py => run_check_clang_format.py} | 0 ...ules.py => run_pull_submodules_changes.py} | 0 ...pdate.py => run_submodules_sync_update.py} | 0 5 files changed, 24 insertions(+), 5 deletions(-) rename scripts/{check_clang_format.py => run_check_clang_format.py} (100%) rename scripts/{update_submodules.py => run_pull_submodules_changes.py} (100%) rename scripts/{submodules_sync_update.py => run_submodules_sync_update.py} (100%) diff --git a/.github/workflows/check_code_formating.yml b/.github/workflows/check_code_formating.yml index d054d2bd..14b2d66e 100644 --- a/.github/workflows/check_code_formating.yml +++ b/.github/workflows/check_code_formating.yml @@ -38,4 +38,4 @@ jobs: clang-format --version - name: Run clang-format check - run: python3 scripts/check_clang_format.py \ No newline at end of file + run: python3 scripts/run_check_clang_format.py \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index 922a033e..93a7ba7e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1,6 +1,25 @@ # Repository management utilities -- check_clang_format.py - script for CI to check whether clang-format was ran before commiting -- run_clang_format.py - runs clang-format on all project files -- submodules_sync_update.py - performs ```git submodule sync && git submodule update --init --recursive``` in case repository was cloned without ```--recursive``` option -- update_submodules.py - updates all submodules to pinned branch and performs code fetch (for submodules without explicit branch pin it will report no pinned branch and do nothing) \ No newline at end of file +* `run_clang_format.py` — runs `clang-format` on all C/C++ source and header files located in: + + * `core` + * `core_hd_mapping` + * `apps` + * `pybind` + * `shared` + +* `run_check_clang_format.py` — checks whether running `clang-format` would modify any tracked project files. The script runs `run_clang_format.py` and then checks the Git working tree for changes. If formatting would produce changes, the check fails and lists the affected files. + +* `run_submodules_sync_update.py` — initializes and updates Git submodules recursively using: + `git submodule sync` and `git submodule update --init --recursive`. + +* `run_pull_submodules_changes.py` — updates submodules located under `3rdparty`. For each submodule, the script: + + * reads the branch configured in `.gitmodules`; + * initializes the submodule if it is missing; + * fetches from `origin`; + * switches to the configured branch; + * performs a fast-forward-only pull; + * reports whether the submodule was actually updated. + + Submodules without a branch configured in `.gitmodules` are reported and skipped. diff --git a/scripts/check_clang_format.py b/scripts/run_check_clang_format.py similarity index 100% rename from scripts/check_clang_format.py rename to scripts/run_check_clang_format.py diff --git a/scripts/update_submodules.py b/scripts/run_pull_submodules_changes.py similarity index 100% rename from scripts/update_submodules.py rename to scripts/run_pull_submodules_changes.py diff --git a/scripts/submodules_sync_update.py b/scripts/run_submodules_sync_update.py similarity index 100% rename from scripts/submodules_sync_update.py rename to scripts/run_submodules_sync_update.py