diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ffff67872..47a19d437 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -223,7 +223,7 @@ packages: - ~netcdf hdf5: require: - - '@1.14.5' + - '@2.1.0' - +hl - +fortran - +mpi @@ -301,9 +301,13 @@ packages: nemsiogfs: require: - '@2.5.4' + # Note: per spack, the first is "preferred", so if you + # need 1.9.0.1, you need to explicitly request it in your spec. nccmp: require: - - '@1.9.0.1' + - any_of: + - '@1.11.0.0' + - '@1.9.0.1' ncio: require: - '@1.1.2' diff --git a/configs/sites/tier1/discover/README.md b/configs/sites/tier1/discover/README.md new file mode 100644 index 000000000..f823e3279 --- /dev/null +++ b/configs/sites/tier1/discover/README.md @@ -0,0 +1,432 @@ +Discover (Tier 1 / JCSDA) + +--- + +## Table of Contents + +- [Overview](#overview) +- [Clone spack-stack](#clone-spack-stack) +- [Setup spack-stack](#setup-spack-stack) +- [Using batch\_install.sh script](#using-batch_installsh-script) + - [Help and dry-run](#help-and-dry-run) + - [Roles and modes](#roles-and-modes) + - [The -H flag](#the--h-flag) + - [The -e flag](#the--e-flag) + - [The -s flag](#the--s-flag) + - [The -a flag](#the--a-flag) + - [The -p, -q, and --constraint flags](#the--p--q-and---constraint-flags) + - [Typical workflow](#typical-workflow) + - [Environment names](#environment-names) + - [Loading the stack](#loading-the-stack) +- [Adding or Updating a Single Package](#adding-or-updating-a-single-package) + - [Scenario A: Add an additional version](#scenario-a-add-an-additional-version) + - [Scenario B: Replace an existing version](#scenario-b-replace-an-existing-version) + - [Pushing to the build cache after manual changes](#pushing-to-the-build-cache-after-manual-changes) +- [Updating a Package Build Recipe](#updating-a-package-build-recipe) +- [Additional Options Reference](#additional-options-reference) +- [Debugging Package Builds](#debugging-package-builds) + +--- + +## Overview + +This configuration provides the Spack Stack setup for the Tier 1 JCSDA installation on the NASA Discover supercomputer. It is maintained under the JCSDA `/discover/swdev/` space. + +The site builds the `unified-dev` environment template with the following compilers: + +- `gcc@=14.2.0` +- `oneapi@=2024.2.0` +- `oneapi@=2025.3.0` + +Module files are managed with `lmod`. Shared caches are located under `/discover/swdev/jcsda/spack-stack/`. + +> **Note:** This site (`discover`) is distinct from the GMAO Tier 2 site (`discover-gmao`). The two sites use different cache locations, compiler sets, and environment templates. If you are doing GMAO-specific work, see `configs/sites/tier2/discover-gmao/README.md`. + +--- + +## Clone spack-stack + +Use the appropriate branch or tag: + +### Fixed Tag - JCSDA + +```bash +git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git -b x.y.z spack-stack-x.y.z +``` + +### Development - JCSDA + +```bash +git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git -b develop spack-stack-dev +``` + +### Development - SI Team + +```bash +git clone --recurse-submodules https://github.com/GMAO-SI-Team/spack-stack.git -b develop spack-stack-siteam +``` + +--- + +## Setup spack-stack + +Activate the base spack-stack environment from the root of the repository: + +```bash +cd spack-stack-dev +. ./setup.sh +``` + +> **Important:** You must run `. ./setup.sh` before running interactive Spack commands. The `batch_install.sh` script sources `setup.sh` internally, so this step is not required before calling it. + +--- + +## Using batch_install.sh script + +The `batch_install.sh` script automates environment creation, cache population, and the Spack installation pipeline. It is the recommended way to build or install the full stack on Discover. + +### Help and dry-run + +```bash +./util/gmao/batch_install.sh -h +``` + +Before running a full build, use the `-n` (dry-run) flag to preview exactly what the script would execute without making any changes: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s -n +``` + +--- + +### Roles and modes + +The script requires two key flags on every invocation: + +- **`-r ROLE`** — either `dev` or `ops` +- **`-m MODE`** — either `build` or `install` + +| Role + Mode | What it does | +|---|---| +| `-r dev -m build` | For SI Team developers with write access to the shared mirrors. Builds packages from source and writes results to the **default shared build cache**. | +| `-r dev -m install` | Installs packages from the shared build cache (no compilation) and generates module files. | +| `-r ops -m build` | For operational partners **without** write access to the shared mirrors. Builds packages and writes to a **custom build cache** specified with `-c BUILDCACHE_DIR`. Allows partners to maintain their own binary cache independently. | +| `-r ops -m install` | Installs packages from the shared build cache and generates module files. | + +The `-m install` step is **required** to generate usable `lmod` module files. A `-m build` run alone does not produce modules. + +--- + +### The -H flag + +Always pass `-H discover` explicitly on Discover: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s +``` + +Without `-H`, the script autodetects the hostname by stripping digits and the domain suffix from `$HOSTNAME`. On a node like `discover23`, this produces `discover` correctly — but if you are logged into a node whose name does not strip cleanly (or if you are on a `discover-gmao`-type node), explicit `-H` avoids any ambiguity. + +--- + +### The -e flag + +The `-e` flag tells the script to continue even if the environment directories already exist. Without it, the script exits with an error if any target environment directory is present. + +- On the **very first run**, you do not need `-e` (the directories do not exist yet). +- On **every subsequent run** (rebuilds, retries, installs), you must pass `-e`. + +--- + +### The -s flag + +The `-s` flag submits the `spack install` step to the SLURM scheduler via `salloc` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: + +- Allocates a single node with 120 tasks +- Targets Milan nodes (`--constraint=mil`) by default +- Walltime: 8 hours +- Output is redirected to `spack...log` in the current directory + +If the default account (`s1873`) is active, the job defaults to `--partition=preops --qos=benchmark`. Use `-p`, `-q`, and `--constraint` to override these. See `-a`, `-p`, `-q`, and `--constraint` below. + +--- + +### The -a flag + +Override the SLURM account (default: `s1873`): + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s -a +``` + +When the account is `s1873`, the job defaults to `--partition=preops --qos=benchmark`. For any other account, no partition or QOS flags are set by default. Use `-p` and `-q` to override. + +--- + +### The -p, -q, and --constraint flags + +Override the SLURM partition, QOS, and node constraint independently. These options only apply to NCCS hosts (`discover`, `discover-gmao`). + +```bash +# Use a different partition +./util/gmao/batch_install.sh -r dev -m build -H discover -s -p compute + +# Use a different QOS +./util/gmao/batch_install.sh -r dev -m build -H discover -s -q high + +# Override the node constraint (default: mil for Milan nodes) +./util/gmao/batch_install.sh -r dev -m build -H discover -s --constraint=cas + +# Override all three +./util/gmao/batch_install.sh -r dev -m build -H discover -s -p compute -q normal --constraint=sky +``` + +`-p` and `-q` also accept long-option forms (`--partition=`, `--qos=`). `--constraint` has no short form because `-c` and `-C` are already in use. + +--- + +### Typical workflow + +The normal progression from a fresh clone to a fully installed, module-ready stack is three steps: + +#### Step 1 — First run: build and populate caches + +On the very first run, use `-u` to populate the bootstrap, source, and cargo mirrors. This requires `-r dev -m build`. + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s -u +``` + +- `-u`: Builds and populates the bootstrap mirror, source cache, and cargo mirror under `/discover/swdev/jcsda/spack-stack/`. Only needed once (or when the caches need to be refreshed). +- `-s`: Submits the install step to SLURM. +- No `-e` needed since the environment directories do not exist yet. + +#### Step 2 — Rebuild or retry runs + +After the first run, the environment directories exist and the caches are populated. Drop `-u` and add `-e`: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover -s -e +``` + +Use this if a build failed and you need to retry, or if you are iterating on the build configuration. The `-e` flag is required because the environment directories were created in Step 1. + +#### Step 3 — Install from cache and generate modules + +Once the build cache is ready, run in install mode to install from the cache and generate `lmod` module files: + +```bash +./util/gmao/batch_install.sh -r dev -m install -H discover -s -e +``` + +This step is **required** to produce usable modules. The `-m build` steps above do not generate module files. + +--- + +### Environment names + +The script creates one environment per compiler under `envs/` (relative to the repository root, unless overridden with `-d`): + +| Mode | Environment names created | +|---|---| +| `build` | `ue-gcc-14.2.0-build`, `ue-oneapi-2024.2.0-build`, `ue-oneapi-2025.3.0-build` | +| `install` | `ue-gcc-14.2.0`, `ue-oneapi-2024.2.0`, `ue-oneapi-2025.3.0` | + +The `ue-` prefix comes from the `unified-dev` template used by this site. + +--- + +### Loading the stack + +After the `install` step completes, point your shell to the generated module files: + +```bash +module use -a /path/to/spack-stack/envs/ue-gcc-14.2.0/install/modulefiles/Core +module load stack-gcc stack-openmpi +``` + +Replace `ue-gcc-14.2.0` with the appropriate environment name for your compiler of choice. + +--- + +## Adding or Updating a Single Package + +If the full stack is already installed and you need to add or change a single package — for example, someone needs `nco@5.3.8` but `nco@5.3.9` was built — it is easier to work inside the environment directly rather than re-running the full `batch_install.sh` pipeline. + +First, activate spack-stack (required for interactive Spack commands) and activate the relevant environment: + +```bash +. ./setup.sh +cd envs/ue-gcc-14.2.0 +spack env activate -p . +``` + +### Scenario A: Add an additional version + +If you want both `nco@5.3.9` and `nco@5.3.8` to coexist as separate modules: + +1. Edit `spack.yaml` in the environment directory and add the additional spec: + + ```yaml + specs: + - nco@5.3.9 + - nco@5.3.8 # add this line + ``` + +2. Re-concretize the environment: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +3. Install only the new package: + + ```bash + spack install nco@5.3.8 2>&1 | tee log.install.nco + ``` + +4. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +5. Deactivate when done: + + ```bash + spack env deactivate + ``` + +### Scenario B: Replace an existing version + +If you want to replace `nco@5.3.9` with `nco@5.3.8` (removing the old version): + +1. Edit `spack.yaml` to change the spec: + + ```yaml + specs: + - nco@5.3.8 # was nco@5.3.9 + ``` + +2. Re-concretize: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +3. Install the new version: + + ```bash + spack install nco@5.3.8 2>&1 | tee log.install.nco + ``` + +4. Uninstall the old version: + + ```bash + spack uninstall nco@5.3.9 + ``` + +5. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +6. Deactivate when done: + + ```bash + spack env deactivate + ``` + +### Pushing to the build cache after manual changes + +If you want the manually installed package to be available in the shared build cache for others to install from, push it after installation: + +```bash +spack buildcache push -u local-binary +spack buildcache update-index local-binary +``` + +> **Warning:** This writes to the default shared build cache at `/discover/swdev/jcsda/spack-stack/build-cache`. Only do this if you have write access and intend to publish the change. + +--- + +## Updating a Package Build Recipe + +If a package needs a new version or checksum update (e.g., a new upstream release of `nco` or `cdo`), you can update the recipe and install the new version by hand without running the full pipeline. + +1. Update the Spack package recipe (e.g., in `var/spack/repos/` or the upstream Spack packages repo) to add the new version and checksum. + +2. Activate spack-stack (required for interactive Spack commands): + + ```bash + . ./setup.sh + ``` + +3. Activate the relevant environment: + + ```bash + cd envs/ue-gcc-14.2.0 + spack env activate -p . + ``` + +4. If the version spec in `spack.yaml` needs to change, edit it now. + +5. Re-concretize: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +6. Install the updated package (Spack will only build what has changed): + + ```bash + spack install @ 2>&1 | tee log.install. + ``` + +7. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +8. Deactivate: + + ```bash + spack env deactivate + ``` + +--- + +## Additional Options Reference + +The following flags are less commonly used. The descriptions below reflect our best current understanding; confirm with the SI Team if you need to use them. + +| Flag | Description | +|---|---| +| `-c BUILDCACHE_DIR` | Use a custom build cache directory. Required with `-r ops -m build` for operational partners who do not have write access to the shared mirrors. | +| `-d ENV_DIRS` | Override the default environment directory (default: `envs/` in the repo root). | +| `-C COMPILERS` | Comma-separated list of compilers to build (e.g., `gcc@=14.2.0,oneapi@=2025.3.0`). Overrides the site defaults. | +| `-p` / `--partition` | Override the SLURM partition. Defaults to `preops` when account is `s1873`, otherwise unset. NCCS hosts only. | +| `-q` / `--qos` | Override the SLURM QOS. Defaults to `benchmark` when account is `s1873`, otherwise unset. NCCS hosts only. | +| `--constraint` | Override the SLURM node constraint. Defaults to `mil` (Milan nodes). No short form. NCCS hosts only. | +| `-t` | Run tests for specific third-party dependencies after installation. The list of packages to test is hardcoded in `batch_install.sh`. | +| `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. Only needed on first run or when refreshing caches. | + +--- + +## Debugging Package Builds + +If a specific package fails to build, drop into Spack's build environment to debug it manually: + +```bash +spack clean +spack stage +cd $(spack location -s ) +spack build-env -- bash --norc --noprofile +``` + +This drops you into a clean bash shell with the exact environment variables, compiler wrappers, and dependencies loaded that Spack uses during the build. diff --git a/configs/sites/tier2/discover-gmao/README.md b/configs/sites/tier2/discover-gmao/README.md new file mode 100644 index 000000000..ed8b466be --- /dev/null +++ b/configs/sites/tier2/discover-gmao/README.md @@ -0,0 +1,430 @@ +Discover GMAO + +--- + +## Table of Contents + +- [Overview](#overview) +- [Clone spack-stack](#clone-spack-stack) +- [Setup spack-stack](#setup-spack-stack) +- [Using batch\_install.sh script](#using-batch_installsh-script) + - [Help and dry-run](#help-and-dry-run) + - [Roles and modes](#roles-and-modes) + - [The -H flag](#the--h-flag) + - [The -e flag](#the--e-flag) + - [The -s flag](#the--s-flag) + - [The -a flag](#the--a-flag) + - [The -p, -q, and --constraint flags](#the--p--q-and---constraint-flags) + - [Typical workflow](#typical-workflow) + - [Environment names](#environment-names) + - [Loading the stack](#loading-the-stack) +- [Adding or Updating a Single Package](#adding-or-updating-a-single-package) + - [Scenario A: Add an additional version](#scenario-a-add-an-additional-version) + - [Scenario B: Replace an existing version](#scenario-b-replace-an-existing-version) + - [Pushing to the build cache after manual changes](#pushing-to-the-build-cache-after-manual-changes) +- [Updating a Package Build Recipe](#updating-a-package-build-recipe) +- [Additional Options Reference](#additional-options-reference) +- [Debugging Package Builds](#debugging-package-builds) + +--- + +## Overview + +This configuration provides the Spack Stack setup for GMAO users on the NASA Discover supercomputer. It is a Tier 2 site (`discover-gmao`) maintained by the GMAO SI Team. + +The site builds the `geos-dev` environment template with the following compilers: + +- `gcc@=15.2.0` +- `oneapi@=2024.2.0` +- `oneapi@=2025.3.0` + +Module files are managed with `lmod`. Shared caches are located under `/discover/nobackup/projects/gmao/SIteam/spack-stack/`. + +--- + +## Clone spack-stack + +Use the appropriate branch or tag: + +### Fixed Tag - JCSDA + +```bash +git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git -b x.y.z spack-stack-x.y.z +``` + +### Development - JCSDA + +```bash +git clone --recurse-submodules https://github.com/JCSDA/spack-stack.git -b develop spack-stack-dev +``` + +### Development - SI Team + +```bash +git clone --recurse-submodules https://github.com/GMAO-SI-Team/spack-stack.git -b develop spack-stack-siteam +``` + +--- + +## Setup spack-stack + +Activate the base spack-stack environment from the root of the repository: + +```bash +cd spack-stack-siteam +. ./setup.sh +``` + +> **Important:** You must run `. ./setup.sh` before running interactive Spack commands. The `batch_install.sh` script sources `setup.sh` internally, so this step is not required before calling it. + +--- + +## Using batch_install.sh script + +The `batch_install.sh` script automates environment creation, cache population, and the Spack installation pipeline. It is the recommended way to build or install the full stack on Discover. + +### Help and dry-run + +```bash +./util/gmao/batch_install.sh -h +``` + +Before running a full build, use the `-n` (dry-run) flag to preview exactly what the script would execute without making any changes: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -n +``` + +--- + +### Roles and modes + +The script requires two key flags on every invocation: + +- **`-r ROLE`** — either `dev` or `ops` +- **`-m MODE`** — either `build` or `install` + +| Role + Mode | What it does | +|---|---| +| `-r dev -m build` | For SI Team developers with write access to the shared mirrors. Builds packages from source and writes results to the **default shared build cache**. | +| `-r dev -m install` | Installs packages from the shared build cache (no compilation) and generates module files. | +| `-r ops -m build` | For operational partners **without** write access to the shared mirrors. Builds packages and writes to a **custom build cache** specified with `-c BUILDCACHE_DIR`. Allows partners to maintain their own binary cache independently. | +| `-r ops -m install` | Installs packages from the shared build cache and generates module files. | + +The `-m install` step is **required** to generate usable `lmod` module files. A `-m build` run alone does not produce modules. + +--- + +### The -H flag + +Always pass `-H discover-gmao` explicitly on Discover: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s +``` + +Without `-H`, the script autodetects the hostname by stripping digits and the domain suffix from `$HOSTNAME`. On a node like `discover23`, this produces `discover` (without the `-gmao` suffix), which is a different site configuration. Providing `-H discover-gmao` forces the correct site. + +--- + +### The -e flag + +The `-e` flag tells the script to continue even if the environment directories already exist. Without it, the script exits with an error if any target environment directory is present. + +- On the **very first run**, you do not need `-e` (the directories do not exist yet). +- On **every subsequent run** (rebuilds, retries, installs), you must pass `-e`. + +--- + +### The -s flag + +The `-s` flag submits the `spack install` step to the SLURM scheduler via `salloc` rather than running it directly in your current shell. This is strongly recommended for full stack builds on Discover: + +- Allocates a single node with 120 tasks +- Targets Milan nodes (`--constraint=mil`) +- Walltime: 8 hours +- Output is redirected to `spack...log` in the current directory + +If the default account (`s1873`) is active, the job defaults to `--partition=preops --qos=benchmark`. Use `-p`, `-q`, and `--constraint` to override these. See `-a`, `-p`, `-q`, and `--constraint` below. + +--- + +### The -a flag + +Override the SLURM account (default: `s1873`): + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -a +``` + +When the account is `s1873`, the job defaults to `--partition=preops --qos=benchmark`. For any other account, no partition or QOS flags are set by default. Use `-p` and `-q` to override. + +--- + +### The -p, -q, and --constraint flags + +Override the SLURM partition, QOS, and node constraint independently: + +```bash +# Use a different partition +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -p compute + +# Use a different QOS +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -q high + +# Override the node constraint (default: mil for Milan nodes) +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s --constraint=cas + +# Override all three +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -p compute -q normal --constraint=sky +``` + +`-p` and `-q` also accept long-option forms (`--partition=`, `--qos=`). `--constraint` has no short form because `-c` and `-C` are already in use. + +These flags override the site defaults regardless of the account. Useful when working under a non-`s1873` account that still has access to specific partitions, QOS levels, or node types. + +--- + +### Typical workflow + +The normal progression from a fresh clone to a fully installed, module-ready stack is three steps: + +#### Step 1 — First run: build and populate caches + +On the very first run, use `-u` to populate the bootstrap, source, and cargo mirrors. This requires `-r dev -m build`. + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -u +``` + +- `-u`: Builds and populates the bootstrap mirror, source cache, and cargo mirror under `/discover/nobackup/projects/gmao/SIteam/spack-stack/`. Only needed once (or when the caches need to be refreshed). +- `-s`: Submits the install step to SLURM. +- No `-e` needed since the environment directories do not exist yet. + +#### Step 2 — Rebuild or retry runs + +After the first run, the environment directories exist and the caches are populated. Drop `-u` and add `-e`: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -e +``` + +Use this if a build failed and you need to retry, or if you are iterating on the build configuration. The `-e` flag is required because the environment directories were created in Step 1. + +#### Step 3 — Install from cache and generate modules + +Once the build cache is ready, run in install mode to install from the cache and generate `lmod` module files: + +```bash +./util/gmao/batch_install.sh -r dev -m install -H discover-gmao -s -e +``` + +This step is **required** to produce usable modules. The `-m build` steps above do not generate module files. + +--- + +### Environment names + +The script creates one environment per compiler under `envs/` (relative to the repository root, unless overridden with `-d`): + +| Mode | Environment names created | +|---|---| +| `build` | `ge-gcc-15.2.0-build`, `ge-oneapi-2024.2.0-build`, `ge-oneapi-2025.3.0-build` | +| `install` | `ge-gcc-15.2.0`, `ge-oneapi-2024.2.0`, `ge-oneapi-2025.3.0` | + +--- + +### Loading the stack + +After the `install` step completes, point your shell to the generated module files: + +```bash +module use -a /path/to/spack-stack-siteam/envs/ge-gcc-15.2.0/install/modulefiles/Core +module load stack-gcc stack-openmpi geos-gcm-env +``` + +Replace `ge-gcc-15.2.0` with the appropriate environment name for your compiler of choice. + +--- + +## Adding or Updating a Single Package + +If the full stack is already installed and you need to add or change a single package — for example, someone needs `nco@5.3.8` but `nco@5.3.9` was built — it is easier to work inside the environment directly rather than re-running the full `batch_install.sh` pipeline. + +First, activate spack-stack (required for interactive Spack commands) and activate the relevant environment: + +```bash +. ./setup.sh +cd envs/ge-gcc-15.2.0 +spack env activate -p . +``` + +### Scenario A: Add an additional version + +If you want both `nco@5.3.9` and `nco@5.3.8` to coexist as separate modules: + +1. Edit `spack.yaml` in the environment directory and add the additional spec: + + ```yaml + specs: + - nco@5.3.9 + - nco@5.3.8 # add this line + ``` + +2. Re-concretize the environment: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +3. Install only the new package: + + ```bash + spack install nco@5.3.8 2>&1 | tee log.install.nco + ``` + +4. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +5. Deactivate when done: + + ```bash + spack env deactivate + ``` + +### Scenario B: Replace an existing version + +If you want to replace `nco@5.3.9` with `nco@5.3.8` (removing the old version): + +1. Edit `spack.yaml` to change the spec: + + ```yaml + specs: + - nco@5.3.8 # was nco@5.3.9 + ``` + +2. Re-concretize: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +3. Install the new version: + + ```bash + spack install nco@5.3.8 2>&1 | tee log.install.nco + ``` + +4. Uninstall the old version: + + ```bash + spack uninstall nco@5.3.9 + ``` + +5. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +6. Deactivate when done: + + ```bash + spack env deactivate + ``` + +### Pushing to the build cache after manual changes + +If you want the manually installed package to be available in the shared build cache for others to install from, push it after installation: + +```bash +spack buildcache push -u local-binary +spack buildcache update-index local-binary +``` + +> **Warning:** This writes to the default shared build cache at `/discover/nobackup/projects/gmao/SIteam/spack-stack/build-cache`. Only do this if you have write access and intend to publish the change. + +--- + +## Updating a Package Build Recipe + +If a package needs a new version or checksum update (e.g., a new upstream release of `nco` or `cdo`), you can update the recipe and install the new version by hand without running the full pipeline. + +1. Update the Spack package recipe (e.g., in `var/spack/repos/` or the upstream Spack packages repo) to add the new version and checksum. + +2. Activate spack-stack (required for interactive Spack commands): + + ```bash + . ./setup.sh + ``` + +3. Activate the relevant environment: + + ```bash + cd envs/ge-gcc-15.2.0 + spack env activate -p . + ``` + +4. If the version spec in `spack.yaml` needs to change, edit it now. + +5. Re-concretize: + + ```bash + spack concretize 2>&1 | tee log.concretize + ``` + +6. Install the updated package (Spack will only build what has changed): + + ```bash + spack install @ 2>&1 | tee log.install. + ``` + +7. Refresh module files: + + ```bash + spack module lmod refresh -y --delete-tree + spack stack setup-meta-modules + ``` + +8. Deactivate: + + ```bash + spack env deactivate + ``` + +--- + +## Additional Options Reference + +The following flags are less commonly used. The descriptions below reflect our best current understanding; confirm with the SI Team if you need to use them. + +| Flag | Description | +|---|---| +| `-c BUILDCACHE_DIR` | Use a custom build cache directory. Required with `-r ops -m build` for operational partners who do not have write access to the shared mirrors. | +| `-d ENV_DIRS` | Override the default environment directory (default: `envs/` in the repo root). | +| `-C COMPILERS` | Comma-separated list of compilers to build (e.g., `gcc@=15.2.0,oneapi@=2025.3.0`). Overrides the site defaults. | +| `-p` / `--partition` | Override the SLURM partition. Defaults to `preops` when account is `s1873`, otherwise unset. | +| `-q` / `--qos` | Override the SLURM QOS. Defaults to `benchmark` when account is `s1873`, otherwise unset. | +| `--constraint` | Override the SLURM node constraint. Defaults to `mil` (Milan nodes). No short form (`-c`/`-C` are taken). | +| `-t` | Run tests for specific third-party dependencies after installation. The list of packages to test is hardcoded in `batch_install.sh`. | +| `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. Only needed on first run or when refreshing caches. | + +--- + +## Debugging Package Builds + +If a specific package fails to build, drop into Spack's build environment to debug it manually: + +```bash +spack clean +spack stage +cd $(spack location -s ) +spack build-env -- bash --norc --noprofile +``` + +This drops you into a clean bash shell with the exact environment variables, compiler wrappers, and dependencies loaded that Spack uses during the build. diff --git a/configs/sites/tier2/macos.gmao/README.md b/configs/sites/tier2/macos.gmao/README.md index f67a80d46..78f3661b3 100644 --- a/configs/sites/tier2/macos.gmao/README.md +++ b/configs/sites/tier2/macos.gmao/README.md @@ -99,7 +99,7 @@ The `batch_install.sh` script automates the creation of environments (e.g., `gcc ### Compiler Selection and Auto-Detection -By default, `batch_install.sh` builds the `gcc@=15.2.0` stack. +By default, `batch_install.sh` builds the `gcc@=15.3.0` stack. The script will automatically detect your currently active version of Apple Clang (e.g. `apple-clang@21.0.0`) and substitute it into the templates. If the NAG Fortran compiler (`nagfor`) is found in your `PATH`, the script will automatically detect its version as well and add it to the build queue (e.g., `nag@=7.2.7243`). @@ -112,7 +112,7 @@ If your NAG compiler is installed in a non-standard location and not in your `PA If you want to explicitly override the compilers built by the script entirely, use the `-C` flag with a comma-separated list of Spack compiler specs: ```bash -./util/gmao/batch_install.sh -C "gcc@=15.2.0,nag@=7.2.7243" -r dev -m build -H macos.gmao -e +./util/gmao/batch_install.sh -C "gcc@=15.3.0,nag@=7.2.7243" -r dev -m build -H macos.gmao -e ``` *Note: For every compiler you specify, you must have a corresponding `packages_-.yaml.template` file in this site directory.* @@ -161,7 +161,7 @@ Depending on which compiler you built, you will need to add the correct `install #### Loading the GCC Stack ```bash -module use -a /path/to/spack-stack/envs/ge-gcc-15.2.0-build/install/modulefiles/Core +module use -a /path/to/spack-stack/envs/ge-gcc-15.3.0-build/install/modulefiles/Core module load stack-gcc stack-openmpi geos-gcm-env ``` @@ -187,7 +187,7 @@ You only need to create each environment once. Our `macos.gmao` site configurati #### GCC Environment ```bash -spack stack create env --name ge-gcc-15.2.0 --template geos-dev --site macos.gmao --compiler=gcc-15.2.0 +spack stack create env --name ge-gcc-15.3.0 --template geos-dev --site macos.gmao --compiler=gcc-15.3.0 ``` #### NAG Environment @@ -203,11 +203,11 @@ spack stack create env --name genag-nag-7.2.7243 --template geos-dev-nag --site Navigate to the environment directory you wish to work on. For example, if you built the GCC stack: ```bash -cd envs/ge-gcc-15.2.0-build +cd envs/ge-gcc-15.3.0-build spack env activate -p . ``` -> **Note:** If you are building or debugging a different compiler stack, be sure to `cd` into that specific environment directory instead (e.g., `envs/ge-nag-7.2.7243-build` or `envs/ge-clang-22.1.3-build`). +> **Note:** If you are building or debugging a different compiler stack, be sure to `cd` into that specific environment directory instead (e.g., `envs/ge-nag-7.2.7243-build` or `envs/ge-clang-22.1.8-build`). > **Important:** Run this in *every* terminal where you plan to run Spack commands for this environment. diff --git a/configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template b/configs/sites/tier2/macos.gmao/packages_clang-22.1.8.yaml.template similarity index 89% rename from configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template rename to configs/sites/tier2/macos.gmao/packages_clang-22.1.8.yaml.template index 49fa8d5ef..fdbcbdc59 100644 --- a/configs/sites/tier2/macos.gmao/packages_clang-22.1.3.yaml.template +++ b/configs/sites/tier2/macos.gmao/packages_clang-22.1.8.yaml.template @@ -4,7 +4,7 @@ packages: cxx: require: [apple-clang@@APPLE_CLANG_VERSION@] fortran: - require: [llvm@21.1.3 ~clang +flang] + require: [llvm@22.1.8 ~clang +flang] mpi: require: [openmpi@=5.0.10] apple-clang: @@ -18,7 +18,7 @@ packages: llvm: buildable: False externals: - - spec: llvm@21.1.3 ~clang +flang languages:='fortran' + - spec: llvm@22.1.8 ~clang +flang languages:='fortran' prefix: @BREW_PREFIX@ extra_attributes: compilers: diff --git a/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template b/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template similarity index 94% rename from configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template rename to configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template index 4014ae07c..cb8e98040 100644 --- a/configs/sites/tier2/macos.gmao/packages_gcc-15.2.0.yaml.template +++ b/configs/sites/tier2/macos.gmao/packages_gcc-15.3.0.yaml.template @@ -4,7 +4,7 @@ packages: cxx: require: [apple-clang@@APPLE_CLANG_VERSION@] fortran: - require: [gcc@15.2.0] + require: [gcc@15.3.0] mpi: require: [openmpi@=5.0.10] apple-clang: @@ -18,7 +18,7 @@ packages: gcc: buildable: False externals: - - spec: gcc@15.2.0 languages:='fortran' + - spec: gcc@15.3.0 languages:='fortran' prefix: @BREW_PREFIX@ extra_attributes: compilers: diff --git a/configs/templates/geos-dev-nag/spack.yaml b/configs/templates/geos-dev-nag/spack.yaml index 5137c6ac5..ea3dc0d33 100644 --- a/configs/templates/geos-dev-nag/spack.yaml +++ b/configs/templates/geos-dev-nag/spack.yaml @@ -13,8 +13,10 @@ spack: # (see geos_gcm_env/package.py). On platforms where only +debug is supported # (e.g., macOS), the site packages.yaml requires geos-gcm-env +debug, which # causes both specs to resolve to +debug (only one unique build). - - geos-gcm-env ~debug - - geos-gcm-env +debug + # + # Also, NAG cannot build FMS yet (due to use of cray pointers) + - geos-gcm-env ~debug ~fms ~cdo + - geos-gcm-env +debug ~fms ~cdo packages: diff --git a/configs/templates/geos-dev/spack.yaml b/configs/templates/geos-dev/spack.yaml index c313a64cc..0cd32011a 100644 --- a/configs/templates/geos-dev/spack.yaml +++ b/configs/templates/geos-dev/spack.yaml @@ -13,8 +13,8 @@ spack: # (see geos_gcm_env/package.py). On platforms where only +debug is supported # (e.g., macOS), the site packages.yaml requires geos-gcm-env +debug, which # causes both specs to resolve to +debug (only one unique build). - - geos-gcm-env ~debug - - geos-gcm-env +debug + - geos-gcm-env ~debug +cdo + - geos-gcm-env +debug +cdo # Various fms builds - fms@=2024.03 ~gfs_phys constants=GEOS diff --git a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py index 8985ce217..b1a98f0a1 100644 --- a/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py +++ b/repos/spack_stack/spack_repo/spack_stack/packages/geos_gcm_env/package.py @@ -16,9 +16,11 @@ class GeosGcmEnv(BundlePackage): maintainers("climbfuji", "mathomp4", "Dooruk") # Current version - version("1.1.0") + version("11.10.0") variant("debug", default=False, description="Build debug version of selected dependencies") + variant("fms", default=True, description="Build FMS as part of the environment") + variant("cdo", default=True, description="Build CDO as part of the environment") depends_on("base-env", type="run") @@ -29,6 +31,12 @@ class GeosGcmEnv(BundlePackage): depends_on("esmf ~debug", type="run", when="~debug") depends_on("esmf +debug", type="run", when="+debug") + depends_on( + "fms@2024.03 precision=32,64 ~gfs_phys +openmp +pic constants=GEOS +deprecated_io +yaml build_type=Release", #noqa: E501 + type="run", + when="@11.10: +fms", + ) + # mapl is built as part of GEOS, don't load; # needs external gftl-shared/fargparse/pflogger # depends_on("mapl", type="run") @@ -46,4 +54,8 @@ class GeosGcmEnv(BundlePackage): depends_on("py-cmocean", type="run") + depends_on("nco", type="run") + depends_on("nccmp", type="run") + depends_on("cdo", type="run", when="+cdo") + # There is no need for install() since there is no code. diff --git a/util/gmao/README.md b/util/gmao/README.md new file mode 100644 index 000000000..0f4d6b45c --- /dev/null +++ b/util/gmao/README.md @@ -0,0 +1,153 @@ +GMAO Spack-Stack Utilities + +--- + +## Table of Contents + +- [Overview](#overview) +- [batch\_install.sh](#batch_installsh) +- [monitor\_install.py](#monitor_installpy) +- [patch\_ecbuild\_ectrans.py](#patch_ecbuild_ectranspy) + +--- + +## Overview + +This directory contains GMAO SI Team utility scripts for building and managing spack-stack environments. + +--- + +## batch_install.sh + +The primary script for building and installing spack-stack environments. It automates environment creation, cache population, and the Spack installation pipeline across all GMAO-supported sites. + +### Usage + +``` +batch_install.sh -r -m [-d ] [-c ] [-H ] +``` + +Run with `-h` for full usage: + +```bash +./util/gmao/batch_install.sh -h +``` + +### Key flags + +| Flag | Description | +|---|---| +| `-r ROLE` | Role: `dev` (SI Team developers with write access to shared mirrors, writes to default build cache) or `ops` (operational partners without write access, requires `-c` to specify their own build cache). | +| `-m MODE` | Mode: `build` (compile + populate build cache) or `install` (install from cache + generate modules). | +| `-H HOSTNAME` | Override hostname autodetection. Usually required (e.g. `-H discover-gmao`, `-H macos.gmao`). | +| `-s` | Submit the install step to the batch scheduler (SLURM/PBS) instead of running interactively. | +| `-e` | Continue if environment directories already exist. Required on all runs after the first. | +| `-u` | Populate bootstrap, source, and cargo mirrors. Requires `-r dev -m build`. First run only. | +| `-a ACCOUNT` | Override the scheduler account (default: `s1873`). | +| `-n` | Dry-run: print what would be executed without running anything. | +| `-p` / `--partition` | Override SLURM partition. **NCCS hosts only** (discover, discover-gmao). | +| `-q` / `--qos` | Override SLURM QOS. **NCCS hosts only** (discover, discover-gmao). | +| `--constraint` | Override SLURM node constraint (e.g. `--constraint=mil`). **NCCS hosts only**. No short form. | +| `-c BUILDCACHE_DIR` | Custom build cache path. Required with `-r ops -m build`. | +| `-d ENV_DIRS` | Override the default environment directory. | +| `-C COMPILERS` | Comma-separated compiler list, overrides site defaults. | +| `-N NAGFOR_PATH` | Path to `nagfor` executable for NAG compiler detection (macOS only). | +| `-t` | Run tests for hardcoded third-party packages after installation. | + +### Supported sites + +The script detects the current site from `$HOSTNAME` (stripping digits and domain), or you can override with `-H`: + +| Site key | System | +|---|---| +| `discover` | NASA NCCS Discover (JCSDA account/partition) | +| `discover-gmao` | NASA NCCS Discover (GMAO SI Team account/partition) | +| `nas` | NASA NAS Pleiades (TOSS 4) | +| `nas-toss5` | NASA NAS Pleiades (TOSS 5) | +| `macos.gmao` | macOS with Homebrew (GMAO SI Team) | + +### Site-specific READMEs + +For detailed workflow instructions for each site, see: + +- `configs/sites/tier2/discover-gmao/README.md` — Discover GMAO workflow +- `configs/sites/tier2/macos.gmao/README.md` — macOS GMAO workflow + +--- + +## monitor_install.py + +A Python script that monitors a spack install log file and reports progress with a progress bar. Useful for watching a build submitted via `-s` without having to tail the raw log. + +### Usage + +```bash +# Monitor a live log from its current end (attach to a running build) +python3 util/gmao/monitor_install.py + +# Monitor a live log from the beginning of the file +python3 util/gmao/monitor_install.py -f + +# Read the full log, print a progress report, then exit +python3 util/gmao/monitor_install.py -r +``` + +The log file is the `.log` file written by `batch_install.sh -s`, named: + +``` +spack...log +``` + +For example, after running: + +```bash +./util/gmao/batch_install.sh -r dev -m build -H discover-gmao -s -e +``` + +The log files will be named `spack.discover-gmao.ge-gcc-15.2.0-build.log`, etc. in the current directory. + +### Options + +| Flag | Description | +|---|---| +| `-f` / `--from-start` | Scan from the beginning of the file rather than tailing live. | +| `-r` / `--report` | Read full file, print progress summary, then exit (non-interactive). | +| `-t SECONDS` / `--timeout` | Seconds of log inactivity before exiting live mode (default: 300). | + +### Exit conditions (live mode) + +The script exits automatically when: +- All packages are installed (progress counter reaches total) +- A spack error is detected in the log +- No new log output for `--timeout` seconds (default: 5 minutes) + +--- + +## patch_ecbuild_ectrans.py + +A targeted patch script that works around an ectrans build failure with the Intel oneAPI compiler (`ifx`) at NAS (nas, nas-toss5). + +**Problem:** `ecbuild`'s Fortran flag checker incorrectly determines that `-march=core-avx2 -no-fma` is not supported by `ifx` and calls `ecbuild_critical()`, aborting the build. The flags are actually valid — the check is a false negative. + +**Fix:** Replaces the `ecbuild_critical()` call with the same logic as the success path, so the flags are always added regardless of the check result. + +See: https://github.com/JCSDA/spack-stack/issues/1775#issuecomment-3898802720 + +### Usage + +```bash +# Apply the patch +python3 util/gmao/patch_ecbuild_ectrans.py --patch /path/to/ecbuild_add_lang_flags.cmake + +# Revert the patch (restores from the .orig backup created at patch time) +python3 util/gmao/patch_ecbuild_ectrans.py --revert /path/to/ecbuild_add_lang_flags.cmake +``` + +The `ecbuild_add_lang_flags.cmake` file is typically found inside the staged or installed ectrans source tree within your spack environment. To locate it: + +```bash +spack env activate -p envs/ue-oneapi-2024.2.0-build +find $(spack location -i ecbuild) -name ecbuild_add_lang_flags.cmake +``` + +A `.orig` backup is created automatically when `--patch` is applied. `--revert` restores from this backup and removes it. If the patch has already been applied, the script will print a message and skip. diff --git a/util/gmao/batch_install.sh b/util/gmao/batch_install.sh index 92813329a..ab61af130 100755 --- a/util/gmao/batch_install.sh +++ b/util/gmao/batch_install.sh @@ -66,12 +66,25 @@ usage() { echo " requires role 'dev' and mode 'build'" echo " -e Continue builds/install in existing environments;" echo " by default, exit with an error if already exist" - echo " -C Set a comma-separated list of compilers to use (e.g. gcc@=15.2.0,nag@=7.2.7243);" + echo " -C Set a comma-separated list of compilers to use (e.g. gcc@=15.3.0,nag@=7.2.7243);" echo " overrides the default compilers for the site" echo " -N Path to nagfor executable (e.g. /opt/nag/bin/nagfor);" echo " forces NAG stack to be built using this specific compiler" echo " -a Set PBS/SLURM account (default: s1873);" echo " overrides the ACCOUNT environment variable" + echo " -p Set SLURM partition (e.g. preops, normal, compute);" + echo " overrides the default partition for the site;" + echo " also accepted as --partition=;" + echo " NOTE: only applies to NCCS hosts (discover, discover-gmao)" + echo " -q Set SLURM QOS (e.g. benchmark, high, normal);" + echo " overrides the default QOS for the site;" + echo " also accepted as --qos=;" + echo " NOTE: only applies to NCCS hosts (discover, discover-gmao)" + echo " --constraint=" + echo " Set SLURM node constraint (e.g. mil, cas, sky);" + echo " overrides the default constraint for the site;" + echo " no short form available (-c and -C are already used);" + echo " NOTE: only applies to NCCS hosts (discover, discover-gmao)" echo " -s Submit 'spack install' to batch scheduler" echo " -t Run tests for specific thirdparty dependencies;" echo " these are currently hardcoded in batch_install.sh" @@ -82,7 +95,7 @@ usage() { echo } -while getopts r:m:d:c:C:N:H:a:nuesth flag +while getopts r:m:d:c:C:N:H:a:p:q:nuesth flag do case "${flag}" in r) @@ -109,6 +122,12 @@ do a) ACCOUNT=${OPTARG} ;; + p) + SPACK_STACK_SLURM_PARTITION=${OPTARG} + ;; + q) + SPACK_STACK_SLURM_QOS=${OPTARG} + ;; n) SPACK_STACK_DRY_RUN="true" ;; @@ -130,6 +149,40 @@ do ;; esac done +shift $((OPTIND - 1)) + +# Parse any remaining long options (e.g. --constraint, --partition, --qos) +while [[ $# -gt 0 ]]; do + case $1 in + --constraint=*) + SPACK_STACK_SLURM_CONSTRAINT="${1#*=}" + ;; + --constraint) + SPACK_STACK_SLURM_CONSTRAINT="$2" + shift + ;; + --partition=*) + SPACK_STACK_SLURM_PARTITION="${1#*=}" + ;; + --partition) + SPACK_STACK_SLURM_PARTITION="$2" + shift + ;; + --qos=*) + SPACK_STACK_SLURM_QOS="${1#*=}" + ;; + --qos) + SPACK_STACK_SLURM_QOS="$2" + shift + ;; + *) + echo "ERROR, unknown argument: $1" + usage + exit 1 + ;; + esac + shift +done echo "INFO: $0 options:" echo " SPACK_STACK_ROLE: ${SPACK_STACK_ROLE:-not set}" @@ -143,6 +196,9 @@ echo " SPACK_STACK_IGNORE_ENV_EXIST: ${SPACK_STACK_IGNORE_ENV_EX echo " SPACK_STACK_SUBMIT_TO_SCHEDULER: ${SPACK_STACK_SUBMIT_TO_SCHEDULER:-false}" echo " SPACK_STACK_RUN_TESTS: ${SPACK_STACK_RUN_TESTS:-false}" echo " ACCOUNT: ${ACCOUNT:-s1873 (default)}" +echo " SPACK_STACK_SLURM_PARTITION: ${SPACK_STACK_SLURM_PARTITION:-use site default}" +echo " SPACK_STACK_SLURM_QOS: ${SPACK_STACK_SLURM_QOS:-use site default}" +echo " SPACK_STACK_SLURM_CONSTRAINT: ${SPACK_STACK_SLURM_CONSTRAINT:-use site default}" # Set default account if not provided via -a or environment ACCOUNT=${ACCOUNT:-s1873} @@ -235,11 +291,15 @@ case ${SPACK_STACK_BATCH_HOST} in export MAC_GMAO_NAG_PREFIX=$(dirname $(dirname "${MAC_GMAO_NAG_PATH}")) fi - SPACK_STACK_BATCH_COMPILERS=("gcc@=15.2.0") + # Note: clang (aka flang) is on hold for macOS until we move to + # 1. FMS 2025 (for GEOS purposes) + # 2. ESMF PR https://github.com/esmf-org/esmf/pull/558 is merged and released/tagged + #SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0" "clang@=22.1.8") + SPACK_STACK_BATCH_COMPILERS=("gcc@=15.3.0") if [[ -n "${MAC_GMAO_NAG_VERSION}" ]]; then SPACK_STACK_BATCH_COMPILERS+=("nag@=${MAC_GMAO_NAG_VERSION}") fi - + # Auto-detect Apple Clang version if command -v clang &> /dev/null; then export MAC_GMAO_APPLE_CLANG_VERSION=$(clang --version | grep "Apple clang version" | awk '{print $4}') @@ -394,32 +454,52 @@ function run_interactive_job() { ${script} ;; discover) - slurm_constraint="--constraint=mil" - if [[ "${ACCOUNT}" == "s1873" ]]; then - slurm_partition="--partition=preops --qos=benchmark" + slurm_constraint="${SPACK_STACK_SLURM_CONSTRAINT:+--constraint=${SPACK_STACK_SLURM_CONSTRAINT}}" + slurm_constraint="${slurm_constraint:---constraint=mil}" + if [[ -n "${SPACK_STACK_SLURM_PARTITION}" ]]; then + slurm_partition="--partition=${SPACK_STACK_SLURM_PARTITION}" + elif [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_partition="--partition=preops" else slurm_partition="" fi + if [[ -n "${SPACK_STACK_SLURM_QOS}" ]]; then + slurm_qos="--qos=${SPACK_STACK_SLURM_QOS}" + elif [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_qos="--qos=benchmark" + else + slurm_qos="" + fi slurm_log="${job_name}.log" echo "INFO: salloc output redirected to ${slurm_log}" salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ - ${slurm_constraint} ${slurm_partition} \ + ${slurm_constraint} ${slurm_partition} ${slurm_qos} \ --job-name=${job_name} \ --account=${ACCOUNT} bash ${script} \ > "${slurm_log}" 2>&1 echo "INFO: salloc job complete, log: ${slurm_log}" ;; discover-gmao) - slurm_constraint="--constraint=mil" - if [[ "${ACCOUNT}" == "s1873" ]]; then - slurm_partition="--partition=preops --qos=benchmark" + slurm_constraint="${SPACK_STACK_SLURM_CONSTRAINT:+--constraint=${SPACK_STACK_SLURM_CONSTRAINT}}" + slurm_constraint="${slurm_constraint:---constraint=mil}" + if [[ -n "${SPACK_STACK_SLURM_PARTITION}" ]]; then + slurm_partition="--partition=${SPACK_STACK_SLURM_PARTITION}" + elif [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_partition="--partition=preops" else slurm_partition="" fi + if [[ -n "${SPACK_STACK_SLURM_QOS}" ]]; then + slurm_qos="--qos=${SPACK_STACK_SLURM_QOS}" + elif [[ "${ACCOUNT}" == "s1873" ]]; then + slurm_qos="--qos=benchmark" + else + slurm_qos="" + fi slurm_log="${job_name}.log" echo "INFO: salloc output redirected to ${slurm_log}" salloc --nodes=1 --ntasks-per-node=${tpn} --time=${walltime} \ - ${slurm_constraint} ${slurm_partition} \ + ${slurm_constraint} ${slurm_partition} ${slurm_qos} \ --job-name=${job_name} \ --account=${ACCOUNT} bash ${script} \ > "${slurm_log}" 2>&1 @@ -820,7 +900,7 @@ for compiler in "${SPACK_STACK_BATCH_COMPILERS[@]}"; do nag_version=${MAC_GMAO_NAG_VERSION} nag_path=${MAC_GMAO_NAG_PATH} nag_prefix=${MAC_GMAO_NAG_PREFIX} - + apple_clang_version=${MAC_GMAO_APPLE_CLANG_VERSION:-"21.0.0"} for template_file in "${macos_site_dir}"/*.yaml.template; do diff --git a/util/gmao/monitor_install.py b/util/gmao/monitor_install.py index e46b3dea3..086c21b61 100755 --- a/util/gmao/monitor_install.py +++ b/util/gmao/monitor_install.py @@ -6,6 +6,11 @@ python3 monitor_install.py # monitor from current end (live) python3 monitor_install.py -f # scan from beginning of file (live) python3 monitor_install.py -r # read full file, print report, exit + +Live mode exits automatically when: + - All packages are installed (current == total in the Installing line) + - A spack error is detected + - No new log output for --timeout seconds (default: 300) """ import sys @@ -26,19 +31,36 @@ def format_bar(current, total, bar_len=30): return f"[{bar}] {current:>4}/{total} {pct:5.1f}%" -def monitor(logfile, from_start=False): +def monitor(logfile, from_start=False, timeout=300): + idle_seconds = 0 + last_total = None + with open(logfile) as f: if not from_start: f.seek(0, 2) # seek to end for live monitoring while True: line = f.readline() if not line: + idle_seconds += 1 + if idle_seconds >= timeout: + print(f"\nNo new output for {timeout}s - assuming build finished or crashed.") + break time.sleep(1) continue + + idle_seconds = 0 m = pattern_installing.search(line) if m: pkg, current, total = m.group(1), m.group(2), m.group(3) + last_total = total print(f"{format_bar(current, total)} {pkg}", flush=True) + if current == total: + print("\nAll packages installed successfully.") + break + + if pattern_error.search(line): + print("\nBuild error detected - exiting monitor.") + break def report(logfile): @@ -93,13 +115,15 @@ def main(): help="Scan from beginning of file instead of tailing live") parser.add_argument("-r", "--report", action="store_true", help="Read full file, show progress, print summary, then exit") + parser.add_argument("-t", "--timeout", type=int, default=300, + help="Seconds of log inactivity before exiting live mode (default: 300)") args = parser.parse_args() try: if args.report: report(args.logfile) else: - monitor(args.logfile, from_start=args.from_start) + monitor(args.logfile, from_start=args.from_start, timeout=args.timeout) except KeyboardInterrupt: print("\nDone monitoring.") except FileNotFoundError: