From 5a80172d5b48c114366e4f31bf9a5dea80fc6c55 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Wed, 18 Mar 2026 14:14:00 +0100 Subject: [PATCH 01/22] added compasUtils file --- compas_python_utils/compasUtils.py | 355 +++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 compas_python_utils/compasUtils.py diff --git a/compas_python_utils/compasUtils.py b/compas_python_utils/compasUtils.py new file mode 100644 index 000000000..2cc126c68 --- /dev/null +++ b/compas_python_utils/compasUtils.py @@ -0,0 +1,355 @@ +import h5py as h5 +import numpy as np +import pandas as pd +from numpy.dtypes import StringDType + + +######################################################################## +# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template + +def printCompasDetails(data, *seeds, mask=()): + """ + Function to print the full Compas output for given seeds, optionally with an additional mask + """ + list_of_keys = list(data.keys()) + + # Check if seed parameter exists - if not, just print without (e.g RunDetails) + if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): # Most output files + #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + + # Set the seed name parameter, mask on seeds as needed, and set the index + seedVariableName='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' + list_of_keys.remove(seedVariableName) # this is the index above, don't want to include it + + allSeeds = data[seedVariableName][()] + seedsMask = np.isin(allSeeds, seeds) + if len(seeds) == 0: # if any seed is included, do not reset the mask + seedsMask = np.ones_like(allSeeds).astype(bool) + if mask == (): + mask = np.ones_like(allSeeds).astype(bool) + mask &= seedsMask + + df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list(data.keys())}).set_index(seedVariableName).T + + else: # No seed parameter, so do custom print for Run Details + + # Get just the keys without the -Derivation suffix - those will be a second column + keys_not_derivations = [] + for key in list_of_keys: + if '-Derivation' not in key: + keys_not_derivations.append(key) + + # Some parameter values are string types, formatted as np.bytes_, need to convert back + def convert_strings(param_array): + if isinstance(param_array[0], np.bytes_): + return param_array.astype(str) + else: + return param_array + + df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T + nCols = df_keys.shape[1] # Required only because if we combine RDs, we get many columns (should fix later) + df_keys.columns = ['Parameter']*nCols + df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T + df_drvs.columns = ['Derivation']*nCols + df = pd.concat([df_keys, df_drvs], axis=1) + + # Add units as first col + units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) + return df + + + +######################################################################## +# ## Get event histories of MT data, SN data, and combined MT, SN data + +def getMtEvents(MT): + """ + This function takes in the `BSE_RLOF` output category from COMPAS, and returns the information + on the Mass Transfer (MT) events that happen for each seed. The events do not have to be in order, + either chronologically or by seed, this function will reorder them as required. + + OUT: + returnedSeeds (list): ordered list of the unique seeds in the MT file + returnedEvents (list): list of sublists, where each sublist contains all the MT events for a given seed. + MT event tuples take the form : + (stellarTypePrimary, stellarTypeSecondary, isRlof1, isRlof2, isCEE, isMrg) + returnTimes (list): is a list of sublists of times of each of the MT events + """ + mtSeeds = MT['SEED'][()] + mtTimes = MT['TimeMT'][()] == 1 + mtIsRlof2 = MT['RLOF(2)>MT'][()] == 1 + mtIsCEE = MT['CEE>MT'][()] == 1 + mtIsMrg = MT['Merger'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + mtSeedsInds = np.lexsort((mtTimes, mtSeeds)) # sort by seeds then times - lexsort sorts by the last column first... + mtSeeds = mtSeeds[mtSeedsInds] + mtTimes = mtTimes[mtSeedsInds] + mtPrimaryStype = mtPrimaryStype[mtSeedsInds] + mtSecondaryStype = mtSecondaryStype[mtSeedsInds] + mtIsRlof1 = mtIsRlof1[mtSeedsInds] + mtIsRlof2 = mtIsRlof2[mtSeedsInds] + mtIsCEE = mtIsCEE[mtSeedsInds] + mtIsMrg = mtIsMrg[mtSeedsInds] + + # Process the MT events + returnedSeeds = [] # array of seeds - will only contain seeds that have MT events + returnedEvents = [] # array of MT events for each seed in returnedSeeds + returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) + + lastSeed = -1 # initialize most recently processed seed + for seedIndex, thisSeed in enumerate(mtSeeds): # iterate over all RLOF file entries + thisTime = mtTimes[seedIndex] # time for this RLOF file entry + thisEvent = (mtPrimaryStype[seedIndex], mtSecondaryStype[seedIndex], + mtIsRlof1[seedIndex], mtIsRlof2[seedIndex], mtIsCEE[seedIndex], mtIsMrg[seedIndex]) # construct event tuple + + # If this is an entirely new seed: + if thisSeed != lastSeed: # same seed as last seed processed? + returnedSeeds.append(thisSeed) # no - new seed, record it + returnedTimes.append([thisTime]) # initialize the list of event times for this seed + returnedEvents.append([thisEvent]) # initialize the list of events for this seed + lastSeed = thisSeed # update the latest seed + + # Add event, if it is not a duplicate + try: + eventIndex = returnedEvents[-1].index(thisEvent) # find eventIndex of this particular event tuple in the array of events for this seed + if thisTime > returnedTimes[-1][eventIndex]: # ^ if event is not a duplicate, this will throw a ValueError + returnedTimes[-1][eventIndex] = thisTime # if event is duplicate, update time to the later of the duplicates + except ValueError: # event is not a duplicate: + returnedEvents[-1].append(thisEvent) # record new event tuple for this seed + returnedTimes[-1].append(thisTime) # record new event time for this seed + + return returnedSeeds, returnedEvents, returnedTimes # see above for description + + +def getSnEvents(SN): + """ + This function takes in the `BSE_Supernovae` output category from COMPAS, and returns the information + on the Supernova (SN) events that happen for each seed. The events do not have to be in order chronologically, + this function will reorder them as required. + + OUT: + returnedSeeds (list): ordered list of all the unique seeds in the SN file + returnedEvents (list): list of sublists, where each sublist contains all the SN events for a given seed. + SN event tuples take the form : + (stellarTypeProgenitor, stellarTypeRemnant, whichStarIsProgenitor, isBinaryUnbound) + returnedTimes (list): is a list of sublists of times of each of the SN events + """ + snSeeds = SN['SEED'][()] + snTimes = SN['Time'][()] + snProgStype = SN['Stellar_Type_Prev(SN)'][()] + snRemnStype = SN['Stellar_Type(SN)'][()] + snWhichProg = SN['Supernova_State'][()] + snIsUnbound = SN['Unbound'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + snSeedsInds = np.lexsort((snTimes, snSeeds)) # sort by seeds then times - lexsort sorts by the last column first... + snSeeds = snSeeds[snSeedsInds] + snTimes = snTimes[snSeedsInds] + snProgStype = snProgStype[snSeedsInds] + snRemnStype = snRemnStype[snSeedsInds] + snWhichProg = snWhichProg[snSeedsInds] + snIsUnbound = snIsUnbound[snSeedsInds] + + # Process the SN events + returnedSeeds = [] # array of seeds - will only contain seeds that have SN events + returnedEvents = [] # array of SN events for each seed in returnedSeeds + returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) + + lastSeed = -1 # initialize most recently processed seed + for seedIndex, thisSeed in enumerate(snSeeds): # iterate over all SN file entries + thisTime = snTimes[seedIndex] # time for this SN file entry + thisEvent = (snProgStype[seedIndex], snRemnStype[seedIndex], + snWhichProg[seedIndex], snIsUnbound[seedIndex]) # construct event tuple + + # If this is an entirely new seed: + if thisSeed != lastSeed: # same seed as last seed processed? + returnedSeeds.append(thisSeed) # no - new seed, record it + returnedTimes.append([thisTime]) # initialize the list of event times for this seed + returnedEvents.append([thisEvent]) # initialize the list of events for this seed + lastSeed = thisSeed # update the latest seed + else: # yes - second SN event for this seed + returnedTimes[-1].append(thisTime) # append time at end of array + returnedEvents[-1].append(thisEvent) # append event at end of array + + return returnedSeeds, returnedEvents, returnedTimes # see above for description + + +def getEventHistory(h5file, exclude_null=False): + """ + Get the event history for all seeds, including both RLOF and SN events, in chronological order. + IN: + h5file (h5.File() type): COMPAS HDF5 output file + exclude_null (bool): whether or not to exclude seeds which undergo no RLOF or SN events + OUT: + returnedSeeds (list): ordered list of all seeds in the output + returnedEvents (list): a list (corresponding to the ordered seeds above) of the + collected SN and MT events from the getMtEvents and getSnEvents functions above, + themselves ordered chronologically + """ + SP = h5file['BSE_System_Parameters'] + MT = h5file['BSE_RLOF'] + SN = h5file['BSE_Supernovae'] + allSeeds = SP['SEED'][()] # get all seeds + mtSeeds, mtEvents, mtTimes = getMtEvents(MT) # get MT events + snSeeds, snEvents, snTimes = getSnEvents(SN) # get SN events + + numMtSeeds = len(mtSeeds) # number of MT events + numSnSeeds = len(snSeeds) # number of SN events + + if numMtSeeds < 1 and numSnSeeds < 1: return [] # no events - return empty history + + returnedSeeds = [] # array of seeds - will only contain seeds that have events (of any type) + returnedEvents = [] # array of events - same size as returnedSeeds (includes event times) + + eventOrdering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events + + mtIndex = 0 # index into MT events arrays + snIndex = 0 # index into SN events arrays + + if exclude_null: + seedsToIterate = np.sort(np.unique(np.append(mtSeeds, snSeeds))) # iterate over all the seeds that have either MT or SN events + else: + seedsToIterate = allSeeds + + idxOrdered = np.argsort(seedsToIterate) + returnedSeeds = [None] * np.size(seedsToIterate) # array of seeds - will only contain seeds that have events (of any type) + returnedEvents = [None] * np.size(seedsToIterate) # array of events - same size as returnedSeeds (includes event times) + + for idx in idxOrdered: + seed = seedsToIterate[idx] + seedEvents = [] # initialise the events for the seed being processed + + # Collect any MT events for this seed, add the time of the event and the event type + while mtIndex < numMtSeeds and mtSeeds[mtIndex] == seed: + for eventIndex, event in enumerate(mtEvents[mtIndex]): + _, _, isRL1, isRL2, isCEE, isMrg = event + eventKey = 'MG' if isMrg else 'CE' if isCEE else 'RL' # event type: Mrg, CEE, RLOF 1->2, RLOF 2->1 + seedEvents.append((eventKey, mtTimes[mtIndex][eventIndex], *mtEvents[mtIndex][eventIndex])) + mtIndex += 1 + + # Collect any SN events for this seed, add the time of the event and the event type + while snIndex < numSnSeeds and snSeeds[snIndex] == seed: + for eventIndex, event in enumerate(snEvents[snIndex]): + eventKey = 'SN' + seedEvents.append((eventKey, snTimes[snIndex][eventIndex], *snEvents[snIndex][eventIndex])) + snIndex += 1 + + seedEvents.sort(key=lambda ev:(ev[1], eventOrdering.index(ev[0]))) # sort the events by time and event type (MT before SN if at the same time) + + returnedSeeds[idx] = seed # record the seed in the seeds array being returned + returnedEvents[idx] = seedEvents # record the events for this seed in the events array being returned + + return returnedSeeds, returnedEvents # see above for details + + + +########################################### +# ## Produce strings of the event histories + +stellarTypeDict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', + 10: 'WD', + 11: 'WD', + 12: 'WD', + 13: 'NS', + 14: 'BH', + 15: 'MR', + 16: 'MS', +} + +def buildEventString(events, useIntStypes=False): + """ + Function to produce a string representing the event history of a single binary for quick readability. + NOTE: unvectorized, do the vectorization later for a speed up + IN: + events (list of tuples): events output from getEventHistory() + OUT: + eventString (string): string representing the event history of the binary + + MT strings look like: + P>S, P, < is RLOF (1->2 or 1<-2) or otherwise = for CEE or & for Merger + + SN strings look like: + P*SR for star1 the SN progenitor,or + R*SP for star2 the SN progenitor, + where P is progenitor type, R is remnant type, + S is state (I for intact, U for unbound) + + Event strings for the same seed are separated by the undesrcore character ('_') + """ + def _remap_stype(int_stype): + if useIntStypes: + return str(int_stype) + else: + return stellarTypeDict[int_stype] + + # Empty event + if len(events) == 0: + return 'NA' + + eventStr = '' + for event in events: + if event[0] == 'SN': # SN event + _, time, stypeP, stypeC, whichSN, isUnbound = event + if whichSN == 1: # Progenitor or Remnant depending upon which star is the SN + charL = _remap_stype(stypeP) + charR = _remap_stype(stypeC) + else: + charL = _remap_stype(stypeC) + charR = _remap_stype(stypeP) + charM = '*u' if isUnbound else '*i' # unbound or intact + else: # Any of the MT events + _, time, stype1, stype2, isRL1, isRL2, isCEE, isMrg = event + charL = _remap_stype(stype1) # primary stellar type + charR = _remap_stype(stype2) # secondary stellar type + charM = '&' if isMrg \ + else '=' if isCEE \ + else '<' if isRL2 \ + else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 + eventStr += "{}{}{}_".format(charL, charM, charR) # event string for this star, _ is event separator + + return np.array(eventStr[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + +def getEventStrings(h5file=None, allEvents=None, useIntStypes=False): + """ + Function to calculate the event history strings for either the entire Compas output, or some list of events + IN: One of + h5file (h5.File() type): COMPAS HDF5 output file + allEvents (list of tuples) + OUT: + eventStrings (list): list of strings of the event history of each seed + """ + + # If output is + if (h5file == None) & (allEvents == None): + return + elif (allEvents == None): + _, allEvents = getEventHistory(h5file) + + eventStrings = np.zeros(len(allEvents), dtype=StringDType()) + for ii, eventsForGivenSeed in enumerate(allEvents): + eventString = buildEventString(eventsForGivenSeed, useIntStypes=useIntStypes) + eventStrings[ii] = eventString # append event string for this star (pop the last underscore first) + + return eventStrings + +"" + + +"" + From 6047ff0c78b3e2e1fbcaf04ce13578761880e1df Mon Sep 17 00:00:00 2001 From: Nicolas Rodriguez-Segovia Date: Fri, 3 Apr 2026 15:29:20 +0900 Subject: [PATCH 02/22] Updated COWD.cpp to properly log HeSDs --- src/COWD.cpp | 2 +- src/changelog.h | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/COWD.cpp b/src/COWD.cpp index b923f1cb8..11b4b310d 100755 --- a/src/COWD.cpp +++ b/src/COWD.cpp @@ -16,7 +16,7 @@ STELLAR_TYPE COWD::EvolveToNextPhase() { stellarType = STELLAR_TYPE::OXYGEN_NEON_WHITE_DWARF; } else { - stellarType = ResolveSNIa(); + stellarType = m_HeShellDetonation ? ResolveHeSD() : ResolveSNIa(); } return stellarType; } diff --git a/src/changelog.h b/src/changelog.h index 8de7208fb..8392f10b6 100644 --- a/src/changelog.h +++ b/src/changelog.h @@ -1699,6 +1699,8 @@ // distribution (Manjaro), possibly C++ version specific. See issue 1441 for description of defect and repair details. // 03.29.02 AB - March 16, 2026 - Defect repair: // - Fix for issue 1463: sign error in the Claeys+2014 common-envelope lambda prescription +// 03.29.03 NRS - April 3, 2026 - Defect repair: +// - Fixed HeSDs not being recorded in the Supernovae logs (mentioned in issue 1350). // // // Version string format is MM.mm.rr, where @@ -1710,7 +1712,7 @@ // if MM is incremented, set mm and rr to 00, even if defect repairs and minor enhancements were also made // if mm is incremented, set rr to 00, even if defect repairs were also made -const std::string VERSION_STRING = "03.29.02"; +const std::string VERSION_STRING = "03.29.03"; # endif // __changelog_h__ From dc3dc2c1c485a9a607ae1d8b125e85826e59e7c6 Mon Sep 17 00:00:00 2001 From: Avi Vajpeyi Date: Wed, 8 Apr 2026 22:28:10 -0400 Subject: [PATCH 03/22] Ship COMPAS C++ through PyPI with bundled runtime (pip install compas-popsynth) on Linux + Macs (#1473) * add CI runner * add more ci test * add hacky py-runner from CI linux * add python runner * add pypi release * fix release yml * fix build name * add verbose * add verbose * add more notes * add more notes * add mac * add mac * add mac * add mac * add mac * add some docs * temp hack to run dry mode every commit * try to fix mac * fix main CI runner * fixes to ci * make CI commenter run on main fork --- .github/workflows/comment-evolution-plot.yml | 98 +++++++++ .github/workflows/compas-compile-ci.yml | 53 +---- .github/workflows/native-linux-bundle.yml | 199 ++++++++++++++++++ .github/workflows/publish-pypi.yml | 193 +++++++++++++++++ .gitignore | 4 + README.md | 2 - compas_python_utils/__init__.py | 28 ++- compas_python_utils/compas_runner.py | 137 ++++++++++++ .../preprocessing/runSubmit.py | 7 +- misc/cicd-scripts/linux-dependencies | 61 ++++-- misc/cicd-scripts/macos-wheel-dependencies | 14 ++ misc/cicd-scripts/manylinux-dependencies | 24 +++ misc/cicd-scripts/package-linux-bundle.sh | 102 +++++++++ misc/cicd-scripts/package-macos-bundle.sh | 190 +++++++++++++++++ misc/cicd-scripts/run_compas.sh | 19 ++ .../pages/Getting started/getting-started.rst | 30 ++- .../Running COMPAS/running-via-python.rst | 34 ++- py_tests/test_compas_runner.py | 52 +++++ setup.py | 99 ++++++--- src/Makefile | 2 +- 20 files changed, 1227 insertions(+), 121 deletions(-) create mode 100644 .github/workflows/comment-evolution-plot.yml create mode 100644 .github/workflows/native-linux-bundle.yml create mode 100644 .github/workflows/publish-pypi.yml create mode 100644 compas_python_utils/compas_runner.py mode change 100644 => 100755 misc/cicd-scripts/linux-dependencies create mode 100755 misc/cicd-scripts/macos-wheel-dependencies create mode 100755 misc/cicd-scripts/manylinux-dependencies create mode 100755 misc/cicd-scripts/package-linux-bundle.sh create mode 100755 misc/cicd-scripts/package-macos-bundle.sh create mode 100755 misc/cicd-scripts/run_compas.sh create mode 100644 py_tests/test_compas_runner.py diff --git a/.github/workflows/comment-evolution-plot.yml b/.github/workflows/comment-evolution-plot.yml new file mode 100644 index 000000000..ed7208168 --- /dev/null +++ b/.github/workflows/comment-evolution-plot.yml @@ -0,0 +1,98 @@ +name: Comment PR with Evolution Plot + +on: + workflow_run: + workflows: + - COMPAS compile test + types: + - completed + +permissions: + actions: read + contents: read + issues: write + pull-requests: write + +jobs: + comment-with-plot: + name: Post evolution plot comment + runs-on: ubuntu-22.04 + if: > + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.pull_requests[0].number + + env: + ARTIFACT_NAME: detailedEvolutionPlot.png + + steps: + - name: Download evolution plot artifact from triggering run + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + const runId = context.payload.workflow_run.id; + const runNumber = context.payload.workflow_run.run_number; + const artifactName = `evolution-plot-${runNumber}`; + const { owner, repo } = context.repo; + + const artifacts = await github.paginate( + github.rest.actions.listWorkflowRunArtifacts, + { owner, repo, run_id: runId, per_page: 100 } + ); + const artifact = artifacts.find((entry) => entry.name === artifactName); + + if (!artifact) { + core.setFailed(`Artifact '${artifactName}' not found for workflow run ${runId}`); + return; + } + + const download = await github.rest.actions.downloadArtifact({ + owner, + repo, + artifact_id: artifact.id, + archive_format: 'zip', + }); + + fs.writeFileSync( + path.join(process.env.GITHUB_WORKSPACE, 'evolution-plot.zip'), + Buffer.from(download.data) + ); + + - name: Unpack evolution plot artifact + run: | + mkdir -p evolution-plot + unzip -o evolution-plot.zip -d evolution-plot + test -f "evolution-plot/${ARTIFACT_NAME}" + + - name: Create report with evolution plot + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + echo "## ✅ COMPAS Build Successful!" >> report.md + echo "" >> report.md + echo "| Item | Value |" >> report.md + echo "|------|-------|" >> report.md + echo "| **Commit** | [\`$(echo "$HEAD_SHA" | cut -c1-7)\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${HEAD_SHA}) |" >> report.md + echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}) |" >> report.md + echo "" >> report.md + + if [ -f "evolution-plot/${ARTIFACT_NAME}" ]; then + echo "### Detailed Evolution Plot" >> report.md + echo "
Click to view evolution plot" >> report.md + echo "" >> report.md + echo "![](./evolution-plot/${ARTIFACT_NAME})" >> report.md + echo "
" >> report.md + else + echo "### ⚠️ Evolution plot not found" >> report.md + fi + + echo "" >> report.md + echo "---" >> report.md + echo "Generated by COMPAS CI " >> report.md + + npx -y @dvcorg/cml comment create --pr "$PR_NUMBER" report.md diff --git a/.github/workflows/compas-compile-ci.yml b/.github/workflows/compas-compile-ci.yml index 4ee1eb40d..e11255923 100644 --- a/.github/workflows/compas-compile-ci.yml +++ b/.github/workflows/compas-compile-ci.yml @@ -19,6 +19,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: + contents: read + jobs: compas: env: @@ -40,7 +43,7 @@ jobs: echo "Building COMPAS from source..." cd src make clean 2>/dev/null || echo 'No existing build to clean' - make -j $(nproc) -f Makefile + make DOCKER_BUILD=1 CPP="g++" -j "$(nproc)" -f Makefile ./COMPAS -v echo "COMPAS build completed successfully!" @@ -106,51 +109,3 @@ jobs: echo "- Python Dependencies: Success" >> $GITHUB_STEP_SUMMARY echo "- Example COMPAS Job: Success" >> $GITHUB_STEP_SUMMARY echo "- Pytest Suite: Success" >> $GITHUB_STEP_SUMMARY - - comment-with-plot: - name: Comment PR with Evolution Plot - runs-on: ubuntu-22.04 - container: docker://ghcr.io/iterative/cml:0-dvc2-base1 - needs: compas - if: github.event_name == 'pull_request' - - env: - ARTIFACT_NAME: detailedEvolutionPlot.png - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Download evolution plot - uses: actions/download-artifact@v4 - with: - name: evolution-plot-${{ github.run_number }} - - - name: Create report with evolution plot - env: - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "## ✅ COMPAS Build Successful!" >> report.md - echo "" >> report.md - echo "| Item | Value |" >> report.md - echo "|------|-------|" >> report.md - echo "| **Commit** | [\`$(echo $GITHUB_SHA | cut -c1-7)\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}) |" >> report.md - echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}) |" >> report.md - echo "" >> report.md - - if [ -f "${{ env.ARTIFACT_NAME }}" ]; then - echo "### Detailed Evolution Plot" >> report.md - echo "
Click to view evolution plot" >> report.md - echo "" >> report.md - echo "![](./${{ env.ARTIFACT_NAME }})" >> report.md - echo "
" >> report.md - else - echo "### ⚠️ Evolution plot not found" >> report.md - fi - - echo "" >> report.md - echo "---" >> report.md - echo "Generated by COMPAS CI " >> report.md - - # Post the report using CML - cml comment create report.md \ No newline at end of file diff --git a/.github/workflows/native-linux-bundle.yml b/.github/workflows/native-linux-bundle.yml new file mode 100644 index 000000000..564731330 --- /dev/null +++ b/.github/workflows/native-linux-bundle.yml @@ -0,0 +1,199 @@ +name: Native Linux bundle + +on: + workflow_dispatch: + pull_request: + branches: + - dev + paths: + - src/** + - misc/cicd-scripts/** + - .github/workflows/native-linux-bundle.yml + push: + branches: + - dev + paths: + - src/** + - misc/cicd-scripts/** + - .github/workflows/native-linux-bundle.yml + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + native-linux-bundle: + name: Build native Linux bundle + runs-on: ubuntu-22.04 + + env: + BUNDLE_DIR: dist/COMPAS-linux-x86_64 + BUNDLE_TARBALL: dist/COMPAS-linux-x86_64.tar.gz + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_BASEDIR: ${{ github.workspace }} + CCACHE_COMPILERCHECK: content + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Restore compiler cache + uses: actions/cache@v4 + with: + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-native-linux-bundle-ccache-${{ hashFiles('src/Makefile', 'misc/cicd-scripts/linux-dependencies', '.github/workflows/native-linux-bundle.yml') }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-native-linux-bundle-ccache-${{ hashFiles('src/Makefile', 'misc/cicd-scripts/linux-dependencies', '.github/workflows/native-linux-bundle.yml') }}- + + - name: Install native build dependencies + run: | + # Keep this workflow lean: install only the native compiler and link dependencies. + bash misc/cicd-scripts/linux-dependencies native-build + + - name: Build COMPAS from source + working-directory: src + run: | + ccache --zero-stats || true + ccache --max-size=500M || true + make DOCKER_BUILD=1 CPP="ccache g++" -j "$(nproc)" -f Makefile + ./COMPAS -v + ccache --show-stats || true + + - name: Run native smoke test + working-directory: src + run: | + rm -rf smoke-output + ./COMPAS \ + -n 1 \ + --initial-mass-1 35 \ + --initial-mass-2 31 \ + -a 3.5 \ + --random-seed 0 \ + --metallicity 0.001 \ + --quiet \ + -o smoke-output + test -d smoke-output + find smoke-output -maxdepth 2 -type f | sort | sed -n '1,40p' + + - name: Inspect runtime dependencies + run: | + echo "Runtime dependencies for native build:" + ldd src/COMPAS + + - name: Package portable Linux bundle + run: | + bash misc/cicd-scripts/package-linux-bundle.sh src/COMPAS "$BUNDLE_DIR" + + - name: Test bundled launcher + run: | + "$BUNDLE_DIR/run_compas.sh" -v + + BUNDLE_SMOKE_DIR="${RUNNER_TEMP}/compas-bundle-smoke" + rm -rf "$BUNDLE_SMOKE_DIR" + + "$BUNDLE_DIR/run_compas.sh" \ + -n 1 \ + --initial-mass-1 35 \ + --initial-mass-2 31 \ + -a 3.5 \ + --random-seed 0 \ + --metallicity 0.001 \ + --quiet \ + -o "$BUNDLE_SMOKE_DIR" + + test -d "$BUNDLE_SMOKE_DIR" + + echo "Runtime dependencies for bundled binary with bundled lib/:" + LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/$BUNDLE_DIR/lib" ldd "$BUNDLE_DIR/bin/COMPAS" + + - name: Archive bundle tarball + run: | + tar -C dist -czf "$BUNDLE_TARBALL" COMPAS-linux-x86_64 + + - name: Upload bundle artifact + uses: actions/upload-artifact@v4 + with: + name: COMPAS-linux-x86_64 + path: ${{ env.BUNDLE_TARBALL }} + if-no-files-found: error + + verify-native-linux-bundle: + name: Verify bundled artifact on fresh runner + runs-on: ubuntu-22.04 + needs: native-linux-bundle + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Download bundle artifact + uses: actions/download-artifact@v4 + with: + name: COMPAS-linux-x86_64 + path: downloaded-artifact + + - name: Install Python runner only + run: | + python -m pip install --no-deps -e . + + - name: Unpack bundled artifact + run: | + mkdir -p bundle-test + tar -xzf downloaded-artifact/COMPAS-linux-x86_64.tar.gz -C bundle-test + test -x bundle-test/COMPAS-linux-x86_64/run_compas.sh + + - name: Inspect downloaded bundle dependencies + run: | + LD_LIBRARY_PATH="${GITHUB_WORKSPACE}/bundle-test/COMPAS-linux-x86_64/lib" \ + ldd bundle-test/COMPAS-linux-x86_64/bin/COMPAS | tee bundle-test/ldd.txt + + ! grep -q 'not found' bundle-test/ldd.txt + grep -q 'bundle-test/COMPAS-linux-x86_64/lib/libhdf5_serial' bundle-test/ldd.txt + grep -q 'bundle-test/COMPAS-linux-x86_64/lib/libgsl' bundle-test/ldd.txt + grep -q 'bundle-test/COMPAS-linux-x86_64/lib/libboost_program_options' bundle-test/ldd.txt + + - name: Run downloaded bundle smoke test + run: | + ./bundle-test/COMPAS-linux-x86_64/run_compas.sh -v + + ARTIFACT_SMOKE_DIR="${RUNNER_TEMP}/compas-downloaded-bundle-smoke" + rm -rf "$ARTIFACT_SMOKE_DIR" + + ./bundle-test/COMPAS-linux-x86_64/run_compas.sh \ + -n 1 \ + --initial-mass-1 35 \ + --initial-mass-2 31 \ + -a 3.5 \ + --random-seed 0 \ + --metallicity 0.001 \ + --quiet \ + -o "$ARTIFACT_SMOKE_DIR" + + test -d "$ARTIFACT_SMOKE_DIR" + + - name: Run Python runner against downloaded bundle + env: + COMPAS_BUNDLE_ROOT: ${{ github.workspace }}/bundle-test/COMPAS-linux-x86_64 + run: | + compas_run --print-path + compas_run -v + + PYTHON_RUNNER_SMOKE_DIR="${RUNNER_TEMP}/compas-python-runner-smoke" + rm -rf "$PYTHON_RUNNER_SMOKE_DIR" + + compas_run \ + -n 1 \ + --initial-mass-1 35 \ + --initial-mass-2 31 \ + -a 3.5 \ + --random-seed 0 \ + --metallicity 0.001 \ + --quiet \ + -o "$PYTHON_RUNNER_SMOKE_DIR" + + test -d "$PYTHON_RUNNER_SMOKE_DIR" diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 000000000..5c6f5a72f --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,193 @@ +name: Publish compas-popsynth to PyPI + +on: + push: + branches: + - dev + tags: + - 'v*' + workflow_dispatch: + inputs: + mode: + description: Build only or publish to PyPI + required: true + default: dry-run + type: choice + options: + - dry-run + - publish + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-linux-wheel: + name: Build and verify Linux wheel + runs-on: ubuntu-22.04 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Build Linux wheel with cibuildwheel + uses: pypa/cibuildwheel@v3.1.2 + env: + CIBW_BUILD: cp311-manylinux_x86_64 + CIBW_ARCHS_LINUX: x86_64 + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_ENVIRONMENT_LINUX: COMPAS_BINARY_WHEEL=1 + CIBW_BEFORE_ALL_LINUX: > + bash {project}/misc/cicd-scripts/manylinux-dependencies && + make -C {project}/src DOCKER_BUILD=1 CPP=g++ -j "$(nproc)" -f Makefile && + {project}/src/COMPAS -v && + bash {project}/misc/cicd-scripts/package-linux-bundle.sh {project}/src/COMPAS {project}/dist/COMPAS-linux-x86_64 && + rm -rf {project}/compas_python_utils/bundled/COMPAS-linux-x86_64 && + mkdir -p {project}/compas_python_utils/bundled && + cp -a {project}/dist/COMPAS-linux-x86_64 {project}/compas_python_utils/bundled/ && + test -f {project}/compas_python_utils/bundled/COMPAS-linux-x86_64/run_compas.sh + CIBW_TEST_COMMAND_LINUX: > + compas_run --print-path && + compas_run -v && + WHEEL_SMOKE_DIR="$(mktemp -d)" && + compas_run -n 1 --initial-mass-1 35 --initial-mass-2 31 -a 3.5 --random-seed 0 --metallicity 0.001 --quiet -o "$WHEEL_SMOKE_DIR" && + test -d "$WHEEL_SMOKE_DIR" + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: compas-popsynth-wheel-linux + path: wheelhouse/*.whl + if-no-files-found: error + + build-macos-arm64-wheel: + name: Build and verify macOS arm64 wheel + runs-on: macos-14 + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Build macOS arm64 wheel with cibuildwheel + uses: pypa/cibuildwheel@v3.1.2 + env: + CIBW_BUILD: cp311-macosx_arm64 + CIBW_ARCHS_MACOS: arm64 + CIBW_ENVIRONMENT_MACOS: COMPAS_BINARY_WHEEL=1 MACOSX_DEPLOYMENT_TARGET=14.0 + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + CIBW_BEFORE_ALL_MACOS: > + bash {project}/misc/cicd-scripts/macos-wheel-dependencies && + make -C {project}/src CPP=clang++ -j "$(sysctl -n hw.logicalcpu)" -f Makefile && + test -x {project}/src/COMPAS && + bash {project}/misc/cicd-scripts/package-macos-bundle.sh {project}/src/COMPAS arm64 {project}/dist/COMPAS-macos-arm64 && + rm -rf {project}/compas_python_utils/bundled/COMPAS-macos-arm64 && + mkdir -p {project}/compas_python_utils/bundled && + cp -R {project}/dist/COMPAS-macos-arm64 {project}/compas_python_utils/bundled/ && + test -f {project}/compas_python_utils/bundled/COMPAS-macos-arm64/run_compas.sh + CIBW_TEST_COMMAND_MACOS: > + compas_run --print-path && + compas_run -v && + WHEEL_SMOKE_DIR="$(mktemp -d)" && + compas_run -n 1 --initial-mass-1 35 --initial-mass-2 31 -a 3.5 --random-seed 0 --metallicity 0.001 --quiet -o "$WHEEL_SMOKE_DIR" && + test -d "$WHEEL_SMOKE_DIR" + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: compas-popsynth-wheel-macos-arm64 + path: wheelhouse/*.whl + if-no-files-found: error + + build-macos-x86_64-wheel: + name: Build and verify macOS x86_64 wheel + runs-on: macos-15-intel + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Build macOS x86_64 wheel with cibuildwheel + uses: pypa/cibuildwheel@v3.1.2 + env: + CIBW_BUILD: cp311-macosx_x86_64 + CIBW_ARCHS_MACOS: x86_64 + CIBW_ENVIRONMENT_MACOS: COMPAS_BINARY_WHEEL=1 MACOSX_DEPLOYMENT_TARGET=14.0 + CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + CIBW_BEFORE_ALL_MACOS: > + bash {project}/misc/cicd-scripts/macos-wheel-dependencies && + make -C {project}/src CPP=clang++ -j "$(sysctl -n hw.logicalcpu)" -f Makefile && + test -x {project}/src/COMPAS && + bash {project}/misc/cicd-scripts/package-macos-bundle.sh {project}/src/COMPAS x86_64 {project}/dist/COMPAS-macos-x86_64 && + rm -rf {project}/compas_python_utils/bundled/COMPAS-macos-x86_64 && + mkdir -p {project}/compas_python_utils/bundled && + cp -R {project}/dist/COMPAS-macos-x86_64 {project}/compas_python_utils/bundled/ && + test -f {project}/compas_python_utils/bundled/COMPAS-macos-x86_64/run_compas.sh + CIBW_TEST_COMMAND_MACOS: > + compas_run --print-path && + compas_run -v && + WHEEL_SMOKE_DIR="$(mktemp -d)" && + compas_run -n 1 --initial-mass-1 35 --initial-mass-2 31 -a 3.5 --random-seed 0 --metallicity 0.001 --quiet -o "$WHEEL_SMOKE_DIR" && + test -d "$WHEEL_SMOKE_DIR" + + - name: Upload wheel artifact + uses: actions/upload-artifact@v4 + with: + name: compas-popsynth-wheel-macos-x86_64 + path: wheelhouse/*.whl + if-no-files-found: error + + publish-pypi: + name: Publish wheel to PyPI + runs-on: ubuntu-22.04 + needs: + - build-linux-wheel + - build-macos-arm64-wheel + - build-macos-x86_64-wheel + if: ${{ (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')) || (github.event_name == 'workflow_dispatch' && github.event.inputs.mode == 'publish') }} + environment: pypi + permissions: + id-token: write + + steps: + - name: Download Linux wheel artifact + uses: actions/download-artifact@v4 + with: + name: compas-popsynth-wheel-linux + path: dist + + - name: Download macOS arm64 wheel artifact + uses: actions/download-artifact@v4 + with: + name: compas-popsynth-wheel-macos-arm64 + path: dist + + - name: Download macOS x86_64 wheel artifact + uses: actions/download-artifact@v4 + with: + name: compas-popsynth-wheel-macos-x86_64 + path: dist + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + verbose: true + attestations: false diff --git a/.gitignore b/.gitignore index 519579538..14eb83f46 100644 --- a/.gitignore +++ b/.gitignore @@ -21,10 +21,12 @@ Thumbs.db # Others .ipynb_checkpoints/ __pycache__/ +.ccache/ COMPAS_Output*/ output*/ *.coverage test_artifacts/ +build/ COMPAS *.log @@ -39,6 +41,8 @@ COMPAS *.synctex(busy) *.synctex.gz *.synctex.gz(busy) +dist/ +compas_python_utils/bundled/COMPAS-linux-x86_64/ # docs html *_build diff --git a/README.md b/README.md index 11ff49b4f..bdba8e989 100644 --- a/README.md +++ b/README.md @@ -105,5 +105,3 @@ Furthermore, ### Highlighted papers that have made use of COMPAS are listed at https://compas.science/science.html ; see https://ui.adsabs.harvard.edu/public-libraries/gzRk1qpbRUy4cP2GydR36Q for a full ADS library - - diff --git a/compas_python_utils/__init__.py b/compas_python_utils/__init__.py index bb4b3c31a..3756caee1 100644 --- a/compas_python_utils/__init__.py +++ b/compas_python_utils/__init__.py @@ -1,11 +1,35 @@ +import re +from pathlib import Path + +try: + from importlib.metadata import PackageNotFoundError, version as package_version +except ImportError: # pragma: no cover + from importlib_metadata import PackageNotFoundError, version as package_version + + +def _version_from_changelog() -> str: + changelog_path = Path(__file__).resolve().parent.parent / "src" / "changelog.h" + version_match = re.search( + r'VERSION_STRING = ["\']([^"\']+)["\']', + changelog_path.read_text(encoding="utf-8"), + ) + if not version_match: + return "0.0.0" + return ".".join(str(int(part)) for part in version_match.group(1).split(".")) + + __all__ = [] __author__ = "Team COMPAS" -__email__ = "compas email" +__email__ = "teamcompas@users.noreply.github.com" __uri__ = "https://github.com/TeamCOMPAS/COMPAS" __license__ = "MIT" __description__ = "COMPAS" __copyright__ = "Copyright 2022 COMPAS developers" __contributors__ = "https://github.com/TeamCOMPAS/COMPAS/graphs/contributors" -__version__ = "0.0.1" \ No newline at end of file + +try: + __version__ = package_version("compas-popsynth") +except PackageNotFoundError: + __version__ = _version_from_changelog() diff --git a/compas_python_utils/compas_runner.py b/compas_python_utils/compas_runner.py new file mode 100644 index 000000000..052b440fc --- /dev/null +++ b/compas_python_utils/compas_runner.py @@ -0,0 +1,137 @@ +import argparse +import os +import platform +import subprocess +import sys +from pathlib import Path +from typing import Iterable, Optional, Sequence + + +PACKAGE_ROOT = Path(__file__).resolve().parent +REPO_ROOT = PACKAGE_ROOT.parent +BUNDLED_DIRECTORIES = ( + "COMPAS-linux-x86_64", + "COMPAS-macos-arm64", + "COMPAS-macos-x86_64", +) + + +def _is_runnable_file(path: Path) -> bool: + return path.is_file() and (os.access(path, os.X_OK) or path.suffix == ".sh") + + +def _validate_explicit_path(path: Path, variable_name: str) -> str: + if not _is_runnable_file(path): + raise FileNotFoundError( + f"{variable_name} points to a non-runnable path: {path}" + ) + return str(path) + + +def _normalized_machine() -> str: + machine = platform.machine().lower() + if machine in {"amd64", "x86_64"}: + return "x86_64" + if machine in {"arm64", "aarch64"}: + return "arm64" + return machine + + +def _preferred_bundle_directories() -> Sequence[str]: + system = platform.system() + machine = _normalized_machine() + + if system == "Linux": + preferred = [f"COMPAS-linux-{machine}"] + elif system == "Darwin": + preferred = [f"COMPAS-macos-{machine}"] + else: + preferred = [] + + return tuple(dict.fromkeys([*preferred, *BUNDLED_DIRECTORIES])) + + +def _candidate_paths() -> Iterable[Path]: + bundle_root = os.environ.get("COMPAS_BUNDLE_ROOT") + if bundle_root: + bundle_path = Path(bundle_root) + yield bundle_path / "run_compas.sh" + yield bundle_path / "bin" / "COMPAS" + + for bundle_directory in _preferred_bundle_directories(): + yield PACKAGE_ROOT / "bundled" / bundle_directory / "run_compas.sh" + yield PACKAGE_ROOT / "bundled" / bundle_directory / "bin" / "COMPAS" + + compas_root = Path(os.environ.get("COMPAS_ROOT_DIR", REPO_ROOT)) + yield compas_root / "src" / "COMPAS" + yield compas_root / "bin" / "COMPAS" + + yield REPO_ROOT / "src" / "COMPAS" + yield REPO_ROOT / "bin" / "COMPAS" + + +def resolve_compas_executable() -> str: + explicit_path = os.environ.get("COMPAS_EXECUTABLE_PATH") + if explicit_path: + return _validate_explicit_path(Path(explicit_path), "COMPAS_EXECUTABLE_PATH") + + bundle_root = os.environ.get("COMPAS_BUNDLE_ROOT") + if bundle_root: + candidates = [Path(bundle_root) / "run_compas.sh", Path(bundle_root) / "bin" / "COMPAS"] + for candidate in candidates: + if _is_runnable_file(candidate): + return str(candidate) + raise FileNotFoundError( + "COMPAS_BUNDLE_ROOT is set, but neither run_compas.sh nor bin/COMPAS " + f"exists under {bundle_root}" + ) + + for candidate in _candidate_paths(): + if _is_runnable_file(candidate): + return str(candidate) + + raise FileNotFoundError( + "Unable to locate a COMPAS executable. Set COMPAS_EXECUTABLE_PATH to an " + "existing executable, or set COMPAS_BUNDLE_ROOT to an extracted bundle directory. " + f"Detected platform: {platform.system()} ({_normalized_machine()})." + ) + + +def run_compas( + compas_args: Optional[Sequence[str]] = None, + executable: Optional[str] = None, + check: bool = True, + **kwargs, +) -> subprocess.CompletedProcess: + resolved_executable = executable or resolve_compas_executable() + executable_path = Path(resolved_executable) + if executable_path.suffix == ".sh": + command = ["bash", resolved_executable, *(compas_args or [])] + else: + command = [resolved_executable, *(compas_args or [])] + return subprocess.run(command, check=check, **kwargs) + + +def main(argv: Optional[Sequence[str]] = None) -> int: + parser = argparse.ArgumentParser( + description="Run the COMPAS executable from Python.", + ) + parser.add_argument( + "--print-path", + action="store_true", + help="Print the resolved COMPAS executable path and exit.", + ) + args, compas_args = parser.parse_known_args(argv) + + executable = resolve_compas_executable() + + if args.print_path: + print(executable) + return 0 + + completed_process = run_compas(compas_args=compas_args, executable=executable, check=False) + return completed_process.returncode + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/compas_python_utils/preprocessing/runSubmit.py b/compas_python_utils/preprocessing/runSubmit.py index 7df4c46cd..68ed4c00d 100644 --- a/compas_python_utils/preprocessing/runSubmit.py +++ b/compas_python_utils/preprocessing/runSubmit.py @@ -6,6 +6,8 @@ import argparse import warnings +from compas_python_utils.compas_runner import resolve_compas_executable + # Check if we are using python 3 python_version = sys.version_info[0] print("python_version =", python_version) @@ -43,15 +45,14 @@ def __init__(self, config_file=DEFAULT_CONFIG_FILE, grid_filename=None, self.stringChoices = config['stringChoices'] if config['stringChoices'] else {} self.listChoices = config['listChoices'] if config['listChoices'] else {} - compas_root_dir = os.environ.get('COMPAS_ROOT_DIR', REPO_ROOT) + compas_root_dir = os.environ.get('COMPAS_ROOT_DIR') if compas_root_dir is None: warnings.warn( 'COMPAS_ROOT_DIR environment variable not set. Setting ' f'`export COMPAS_ROOT_DIR={REPO_ROOT}`' ) os.environ['COMPAS_ROOT_DIR'] = REPO_ROOT - compas_exe = os.path.join(compas_root_dir, 'src/COMPAS') - compas_executable_override = os.environ.get('COMPAS_EXECUTABLE_PATH', compas_exe) + compas_executable_override = resolve_compas_executable() print('compas_executable_override', compas_executable_override) self.compas_executable = compas_executable_override diff --git a/misc/cicd-scripts/linux-dependencies b/misc/cicd-scripts/linux-dependencies old mode 100644 new mode 100755 index 7accaa532..610dcf7a1 --- a/misc/cicd-scripts/linux-dependencies +++ b/misc/cicd-scripts/linux-dependencies @@ -1,17 +1,48 @@ -#! /bin/sh +#!/usr/bin/env bash +set -euo pipefail -sudo apt-get install -y \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - texlive-xetex \ - latexmk \ - xindy \ - dvipng \ - cm-super -sudo apt install texlive-latex-extra --fix-missing -sudo apt update && sudo apt install g++ libboost-all-dev libgsl-dev libhdf5-serial-dev -sudo apt-get install gcc libpq-dev -y -sudo apt-get install python3-dev python3-pip python3-venv python3-wheel -y \ No newline at end of file +PROFILE="${1:-full}" + +COMMON_BUILD_PACKAGES=( + ccache + g++ + libboost-all-dev + libgsl-dev + libhdf5-serial-dev +) + +FULL_ONLY_PACKAGES=( + texlive-latex-recommended + texlive-latex-extra + texlive-fonts-recommended + texlive-fonts-extra + texlive-xetex + latexmk + xindy + dvipng + cm-super + gcc + libpq-dev + python3-dev + python3-pip + python3-venv + python3-wheel +) + +sudo apt-get update + +case "$PROFILE" in + native-build) + sudo apt-get install -y "${COMMON_BUILD_PACKAGES[@]}" + ;; + full) + sudo apt-get install -y "${FULL_ONLY_PACKAGES[@]}" + sudo apt-get install -y --fix-missing texlive-latex-extra + sudo apt-get install -y "${COMMON_BUILD_PACKAGES[@]}" + ;; + *) + echo "Usage: $0 [full|native-build]" >&2 + exit 1 + ;; +esac diff --git a/misc/cicd-scripts/macos-wheel-dependencies b/misc/cicd-scripts/macos-wheel-dependencies new file mode 100755 index 000000000..b594fd195 --- /dev/null +++ b/misc/cicd-scripts/macos-wheel-dependencies @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Install the Homebrew packages required to compile and bundle the macOS PyPI +# wheel on GitHub's macOS runners. + +set -euo pipefail + +if ! command -v brew >/dev/null 2>&1; then + echo "Homebrew is required for macOS wheel builds" >&2 + exit 1 +fi + +brew update +brew install boost gsl hdf5 diff --git a/misc/cicd-scripts/manylinux-dependencies b/misc/cicd-scripts/manylinux-dependencies new file mode 100755 index 000000000..9bd7eda9c --- /dev/null +++ b/misc/cicd-scripts/manylinux-dependencies @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Install the native system packages needed to compile and repair the Linux +# PyPI wheel inside the manylinux container used by cibuildwheel. + +set -euo pipefail + +if command -v dnf >/dev/null 2>&1; then + PKG_MANAGER="dnf" +elif command -v yum >/dev/null 2>&1; then + PKG_MANAGER="yum" +else + echo "Expected dnf or yum in the manylinux build environment" >&2 + exit 1 +fi + +"$PKG_MANAGER" install -y \ + boost-devel \ + gcc-c++ \ + gsl-devel \ + hdf5-devel \ + make \ + patchelf \ + which diff --git a/misc/cicd-scripts/package-linux-bundle.sh b/misc/cicd-scripts/package-linux-bundle.sh new file mode 100755 index 000000000..063ea0166 --- /dev/null +++ b/misc/cicd-scripts/package-linux-bundle.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +# Create the redistributable Linux COMPAS bundle used by the native tarball +# workflow and by the Linux PyPI wheel build. + +set -euo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +REPO_ROOT="$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)" + +BINARY_PATH="${1:-$REPO_ROOT/src/COMPAS}" +BUNDLE_DIR="${2:-$REPO_ROOT/dist/COMPAS-linux-x86_64}" +BIN_DIR="$BUNDLE_DIR/bin" +LIB_DIR="$BUNDLE_DIR/lib" +README_PATH="$BUNDLE_DIR/README.txt" +TMP_LDD="$(mktemp)" + +cleanup() { + rm -f "$TMP_LDD" +} + +trap cleanup EXIT + +maybe_strip() { + local path="$1" + + if ! command -v strip >/dev/null 2>&1; then + return 0 + fi + + # Best-effort size reduction only. Some binaries or shared libraries may + # already be stripped or may not support the requested mode. + strip --strip-unneeded "$path" 2>/dev/null || strip "$path" 2>/dev/null || true +} + +if [ ! -x "$BINARY_PATH" ]; then + echo "Expected executable COMPAS binary at '$BINARY_PATH'" >&2 + exit 1 +fi + +if ! command -v ldd >/dev/null 2>&1; then + echo "ldd is required to package the Linux bundle" >&2 + exit 1 +fi + +rm -rf "$BUNDLE_DIR" +mkdir -p "$BIN_DIR" "$LIB_DIR" + +cp -Lf "$BINARY_PATH" "$BIN_DIR/COMPAS" +cp "$SCRIPT_DIR/run_compas.sh" "$BUNDLE_DIR/run_compas.sh" +chmod 755 "$BIN_DIR/COMPAS" "$BUNDLE_DIR/run_compas.sh" +maybe_strip "$BIN_DIR/COMPAS" + +ldd "$BIN_DIR/COMPAS" | tee "$TMP_LDD" + +if grep -q 'not found' "$TMP_LDD"; then + echo "Cannot package bundle with unresolved runtime dependencies" >&2 + exit 1 +fi + +awk ' + /=>/ && $3 ~ /^\// { print $3; next } + $1 ~ /^\// { print $1; next } +' "$TMP_LDD" | sort -u | while IFS= read -r lib; do + case "$lib" in + /lib64/ld-linux-*|/lib/x86_64-linux-gnu/ld-linux-*|*/libc.so.*|*/libm.so.*|*/libpthread.so.*|*/libdl.so.*|*/librt.so.*|*/libresolv.so.*|*/libnss_*.so.*|*/libcrypt.so.*|*/libutil.so.*|*/libanl.so.*) + echo "Skipping system runtime: $lib" + ;; + *) + cp -Ln "$lib" "$LIB_DIR/" + maybe_strip "$LIB_DIR/$(basename "$lib")" + ;; + esac +done + +cat > "$README_PATH" <<'EOF' +COMPAS bundled Linux artifact + +Supported platform: +- Ubuntu 22.04 +- Linux x86_64 + +Contents: +- bin/COMPAS: bundled COMPAS executable +- lib/: shared libraries packaged with this build +- run_compas.sh: launcher that sets LD_LIBRARY_PATH for the bundled libs + +How to run: +1. Unpack this directory on a supported Linux x86_64 machine. +2. From inside the unpacked directory, run: + ./run_compas.sh -v + +Notes and caveats: +- This bundle is intended to run without separately installing Boost, GSL, or HDF5. +- It still relies on the target machine's glibc and other base system libraries being compatible with Ubuntu 22.04. +- If you invoke bin/COMPAS directly, the bundled lib/ directory will not be added automatically; use run_compas.sh instead. +EOF + +echo +echo "Created bundle at: $BUNDLE_DIR" +echo "Bundled shared libraries:" +find "$LIB_DIR" -maxdepth 1 -type f | sort diff --git a/misc/cicd-scripts/package-macos-bundle.sh b/misc/cicd-scripts/package-macos-bundle.sh new file mode 100755 index 000000000..57df6f2a8 --- /dev/null +++ b/misc/cicd-scripts/package-macos-bundle.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# Create the redistributable macOS COMPAS bundle embedded in the platform- +# specific macOS wheels built by cibuildwheel. + +set -euo pipefail + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)" +REPO_ROOT="$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd)" + +BINARY_PATH="${1:-$REPO_ROOT/src/COMPAS}" +ARCH_NAME="${2:-$(uname -m)}" +BUNDLE_DIR="${3:-$REPO_ROOT/dist/COMPAS-macos-$ARCH_NAME}" +BIN_DIR="$BUNDLE_DIR/bin" +LIB_DIR="$BUNDLE_DIR/lib" +README_PATH="$BUNDLE_DIR/README.txt" +SEEN_DEPS="$(mktemp)" + +cleanup() { + rm -f "$SEEN_DEPS" +} + +trap cleanup EXIT + +if [ ! -x "$BINARY_PATH" ]; then + echo "Expected executable COMPAS binary at '$BINARY_PATH'" >&2 + exit 1 +fi + +for required_command in codesign install_name_tool otool; do + if ! command -v "$required_command" >/dev/null 2>&1; then + echo "$required_command is required to package the macOS bundle" >&2 + exit 1 + fi +done + +maybe_strip() { + local path="$1" + + if ! command -v strip >/dev/null 2>&1; then + return 0 + fi + + strip -x "$path" 2>/dev/null || true +} + +is_system_library() { + case "$1" in + /System/*|/usr/lib/*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +dependencies_for() { + otool -L "$1" | tail -n +2 | awk '{print $1}' +} + +resolve_dependency_path() { + local dependency="$1" + local referencing_file="$2" + local reference_dir + + case "$dependency" in + @loader_path/*) + reference_dir="$(CDPATH= cd -- "$(dirname -- "$referencing_file")" && pwd)" + echo "$reference_dir/${dependency#@loader_path/}" + ;; + @executable_path/*) + echo "$BIN_DIR/${dependency#@executable_path/}" + ;; + @rpath/*) + return 1 + ;; + *) + echo "$dependency" + ;; + esac +} + +copy_dependency() { + local source_path="$1" + local dest_path="$LIB_DIR/$(basename "$source_path")" + + if [ ! -f "$dest_path" ]; then + cp -f "$source_path" "$dest_path" + chmod 755 "$dest_path" + maybe_strip "$dest_path" + fi +} + +rewrite_references() { + local target="$1" + local target_dir_mode="$2" + local dependency + local dependency_name + + while IFS= read -r dependency; do + dependency_name="$(basename "$dependency")" + if [ -f "$LIB_DIR/$dependency_name" ]; then + if [ "$target_dir_mode" = "binary" ]; then + install_name_tool -change "$dependency" "@loader_path/../lib/$dependency_name" "$target" || true + else + install_name_tool -change "$dependency" "@loader_path/$dependency_name" "$target" || true + fi + fi + done < <(dependencies_for "$target") +} + +rm -rf "$BUNDLE_DIR" +mkdir -p "$BIN_DIR" "$LIB_DIR" + +cp -f "$BINARY_PATH" "$BIN_DIR/COMPAS" +cp "$SCRIPT_DIR/run_compas.sh" "$BUNDLE_DIR/run_compas.sh" +chmod 755 "$BIN_DIR/COMPAS" "$BUNDLE_DIR/run_compas.sh" +maybe_strip "$BIN_DIR/COMPAS" + +declare -a queue=("$BINARY_PATH") + +while [ "${#queue[@]}" -gt 0 ]; do + current_file="${queue[0]}" + queue=("${queue[@]:1}") + + while IFS= read -r dependency; do + resolved_dependency="$(resolve_dependency_path "$dependency" "$current_file" || true)" + + case "$dependency" in + @rpath/*|"") + continue + ;; + esac + + if [ -z "$resolved_dependency" ] || [ ! -e "$resolved_dependency" ]; then + continue + fi + + if is_system_library "$resolved_dependency"; then + continue + fi + + dependency_name="$(basename "$resolved_dependency")" + copy_dependency "$resolved_dependency" + + if ! grep -qx "$dependency_name" "$SEEN_DEPS" 2>/dev/null; then + echo "$dependency_name" >> "$SEEN_DEPS" + queue+=("$resolved_dependency") + fi + done < <(dependencies_for "$current_file") +done + +for library_path in "$LIB_DIR"/*; do + [ -f "$library_path" ] || continue + library_name="$(basename "$library_path")" + install_name_tool -id "@loader_path/$library_name" "$library_path" || true + rewrite_references "$library_path" "library" + codesign --force --sign - "$library_path" >/dev/null 2>&1 || true +done + +rewrite_references "$BIN_DIR/COMPAS" "binary" +codesign --force --sign - "$BIN_DIR/COMPAS" >/dev/null 2>&1 || true + +cat > "$README_PATH" <` for more details. +Running COMPAS via Python +========================= + +A convenient method of managing the many program options provided by COMPAS is to run COMPAS via Python, using a script to manage and +specify the values of the program options. + +Bundled PyPI executable +----------------------- + +On supported platforms, the PyPI package can include the native COMPAS +executable directly. After installing:: + + pip install compas-popsynth + +you can run the packaged executable with:: + + compas_run -v + +and a minimal smoke test with:: + + compas_run -n 1 --initial-mass-1 35 --initial-mass-2 31 -a 3.5 --random-seed 0 --metallicity 0.001 --quiet -o compas-smoke-output + +``compas_run`` resolves to the bundled native executable when a supported wheel +is installed. If you want to point the Python launcher at a separate local build +or an extracted bundle, set ``COMPAS_EXECUTABLE_PATH`` or ``COMPAS_BUNDLE_ROOT``. + +An example Python script is provided in the COMPAS suite on github: ``runSubmit.py``. Additionally, the default COMPAS options are specified on ``compasConfigDefault.yaml``. Users should copy the ``runSubmit.py`` and ``runSubmit.py`` scripts and modify the ``compasConfigDefault.yaml`` copy to match their experimental requirements. Refer to the :doc:`Getting started guide <../../Getting started/getting-started>` for more details. To run COMPAS via Python using the ``runSubmit.py`` script provided, set the shell environment variable ``COMPAS_ROOT_DIR`` to the parent directory of the directory in which the COMPAS executable resides, then type `python /path-to-runSubmit/runSubmit.py`. diff --git a/py_tests/test_compas_runner.py b/py_tests/test_compas_runner.py new file mode 100644 index 000000000..174a0bf1a --- /dev/null +++ b/py_tests/test_compas_runner.py @@ -0,0 +1,52 @@ +import os + +from compas_python_utils.compas_runner import main, resolve_compas_executable, run_compas + + +def _make_fake_executable(path, contents=None): + executable_contents = contents or "#!/usr/bin/env bash\nprintf 'fake-compas %s\\n' \"$*\"\n" + path.write_text(executable_contents) + os.chmod(path, 0o755) + + +def test_resolve_compas_executable_from_env(monkeypatch, tmp_path): + fake_executable = tmp_path / "COMPAS" + _make_fake_executable(fake_executable) + + monkeypatch.setenv("COMPAS_EXECUTABLE_PATH", str(fake_executable)) + monkeypatch.delenv("COMPAS_BUNDLE_ROOT", raising=False) + + assert resolve_compas_executable() == str(fake_executable) + + +def test_resolve_compas_executable_from_bundle_root(monkeypatch, tmp_path): + bundle_root = tmp_path / "COMPAS-linux-x86_64" + bundle_root.mkdir() + launcher = bundle_root / "run_compas.sh" + _make_fake_executable(launcher) + + monkeypatch.delenv("COMPAS_EXECUTABLE_PATH", raising=False) + monkeypatch.setenv("COMPAS_BUNDLE_ROOT", str(bundle_root)) + + assert resolve_compas_executable() == str(launcher) + + +def test_run_compas_executes_resolved_binary(monkeypatch, tmp_path): + fake_executable = tmp_path / "COMPAS" + _make_fake_executable(fake_executable) + + monkeypatch.setenv("COMPAS_EXECUTABLE_PATH", str(fake_executable)) + completed_process = run_compas(["-v"], capture_output=True, text=True) + + assert completed_process.returncode == 0 + assert "fake-compas -v" in completed_process.stdout + + +def test_main_print_path(monkeypatch, tmp_path, capsys): + fake_executable = tmp_path / "COMPAS" + _make_fake_executable(fake_executable) + + monkeypatch.setenv("COMPAS_EXECUTABLE_PATH", str(fake_executable)) + + assert main(["--print-path"]) == 0 + assert str(fake_executable) in capsys.readouterr().out diff --git a/setup.py b/setup.py index 8cf5abe20..928f0df5d 100644 --- a/setup.py +++ b/setup.py @@ -3,44 +3,62 @@ import re import sys -from setuptools import find_packages, setup +from setuptools import Distribution, find_packages, setup + +try: + from wheel.bdist_wheel import bdist_wheel as _bdist_wheel +except ImportError: + _bdist_wheel = None python_version = sys.version_info if python_version < (3, 8): sys.exit("Python < 3.8 is not supported, aborting setup") NAME = "compas_python_utils" +DIST_NAME = "compas-popsynth" PACKAGES = find_packages() HERE = os.path.dirname(os.path.realpath(__file__)) META_PATH = os.path.join(NAME, "__init__.py") CPP_VERSION_FILE = os.path.join("src", "changelog.h") +BUILD_BINARY_WHEEL = os.environ.get("COMPAS_BINARY_WHEEL") == "1" CLASSIFIERS = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", "Programming Language :: Python", "Programming Language :: Python :: 3", ] -INSTALL_REQUIRES = [ +CORE_RUNTIME_REQUIRES = [ "numpy>=1.16", + "PyYAML", +] +ANALYSIS_REQUIRES = [ "h5py", - "argparse", "stroopwafel", - "pytest>=3.6", - "pre-commit", - "flake8", - "black==22.10.0", - "isort", "matplotlib>=3.3.2", "pandas", "astropy>=4.0", "scipy>=1.5.0", - "latex", - "PyYAML", "tqdm", - "corner" + "corner", +] +DEV_ONLY_REQUIRES = [ + "pytest>=3.6", + "pytest-cov", + "pre-commit", + "flake8", + "black==22.10.0", + "isort", + "coverage-badge", + "deepdiff", + "jupytext", + "jupyter-autotime", + "memory_profiler", + "nbconvert", + "ipykernel", ] EXTRA_REQUIRE = dict( docs=[ @@ -58,24 +76,31 @@ "sphinx-togglebutton", "linuxdoc>=20210324" ], - dev=[ - "pytest-cov", - "pre-commit", - "flake8", - "black==22.10.0", - "isort", - "coverage-badge", - "deepdiff", - "jupytext", - "jupyter-autotime", - "memory_profiler", - "nbconvert", - "ipykernel", - ], + full=[], + dev=DEV_ONLY_REQUIRES, gpu=["cupy"], ) +class COMPASDistribution(Distribution): + def has_ext_modules(self): + return BUILD_BINARY_WHEEL + + +cmdclass = {} +if BUILD_BINARY_WHEEL and _bdist_wheel is not None: + class COMPASBdistWheel(_bdist_wheel): + def finalize_options(self): + super().finalize_options() + self.root_is_pure = False + + def get_tag(self): + _, _, platform_tag = super().get_tag() + return "py3", "none", platform_tag + + cmdclass["bdist_wheel"] = COMPASBdistWheel + + def read(*parts): with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f: return f.read() @@ -95,14 +120,16 @@ def find_version(version_file=read(CPP_VERSION_FILE)): r"VERSION_STRING = ['\"]([^'\"]*)['\"]", version_file, re.M ) if version_match: - return version_match.group(1) + raw_version = version_match.group(1) + normalized_parts = [str(int(part)) for part in raw_version.split(".")] + return ".".join(normalized_parts) raise RuntimeError("Unable to find version string.") if __name__ == "__main__": setup( - name=NAME, - version=find_meta("version"), + name=DIST_NAME, + version=find_version(), author=find_meta("author"), author_email=find_meta("email"), maintainer=find_meta("author"), @@ -113,22 +140,32 @@ def find_version(version_file=read(CPP_VERSION_FILE)): long_description=read("README.md"), long_description_content_type="text/markdown", packages=PACKAGES, + python_requires=">=3.8", package_data={ + NAME: [ + "bundled/*", + "bundled/*/*.sh", + "bundled/*/bin/*", + "bundled/*/lib/*", + ], f"{NAME}.preprocessing": ["*.txt", "*.yaml"], f"{NAME}.detailed_evolution_plotter": ["van_den_heuvel_figures/*"], f"{NAME}.cosmic_integration": ["SNR_Grid*"], }, include_package_data=True, - install_requires=INSTALL_REQUIRES, + install_requires=CORE_RUNTIME_REQUIRES + ANALYSIS_REQUIRES, extras_require=EXTRA_REQUIRE, classifiers=CLASSIFIERS, - zip_safe=True, + zip_safe=not BUILD_BINARY_WHEEL, + distclass=COMPASDistribution, + cmdclass=cmdclass, entry_points={ "console_scripts": [ f"compas_h5view= {NAME}.h5view:main", f"compas_h5copy= {NAME}.h5copy:main", f"compas_h5sample= {NAME}.h5sample:main", f"compas_plot_detailed_evolution={NAME}.detailed_evolution_plotter.plot_detailed_evolution:main", + f"compas_run={NAME}.compas_runner:main", f"compas_run_submit={NAME}.preprocessing.runSubmit:main", f"compas_sample_stroopwafel={NAME}.preprocessing.stroopwafelInterface:main", f"compas_sample_moe_di_stefano={NAME}.preprocessing.sampleMoeDiStefano:main", diff --git a/src/Makefile b/src/Makefile index ddab836d3..ae3d83836 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,4 +1,4 @@ -CPP := g++ +CPP ?= g++ UNAME_S := $(shell uname -s) HOMEBREW_PREFIX := $(shell brew --prefix 2>/dev/null) From 758e7a49b55960a220783c4dedc68c6f0008911d Mon Sep 17 00:00:00 2001 From: Avi Vajpeyi Date: Thu, 16 Apr 2026 17:50:42 -0400 Subject: [PATCH 04/22] docs: add notes on GH images --- .../Developer build/docker-developer.rst | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/online-docs/pages/Developer guide/Developer build/docker-developer.rst b/online-docs/pages/Developer guide/Developer build/docker-developer.rst index 1f8d496f1..ab7e3ba09 100644 --- a/online-docs/pages/Developer guide/Developer build/docker-developer.rst +++ b/online-docs/pages/Developer guide/Developer build/docker-developer.rst @@ -2,16 +2,29 @@ Creating a Docker image ======================= +Overview +-------- + +Docker simplifies COMPAS deployment by containerizing all dependencies and build tools, +eliminating manual environment configuration across different operating systems and platforms. +This enables reliable deployment on cloud platforms and consistent development environments. + +For information on how to use Docker to run COMPAS, see the :doc:`../../User guide/docker` page. + + GitHub CI/CD ------------ -``Docker`` images of all versions of COMPAS (``dev`` branch) are available at the -`TeamCOMPAS dockerHub page `__. Building of these images is performed automatically by the -``GitHub`` CI/CD process. +``Docker`` images of all versions of COMPAS (``dev`` branch) are available at: + + - `TeamCOMPAS dockerHub page `__ + - `GitHub Container Registry `__ + +Building of these images is performed automatically by the ``GitHub`` CI/CD process. Whenever a push to `TeamCOMPAS/dev `__ occurs, a continuous deployment process automatically -builds\ [#f1]_ a new image and deploys it to ``dockerHub`` with a `tag`\ [#f2]_ that corresponds to the value of ``VERSION_STRING`` +builds\ [#f1]_ a new image and deploys it to both ``dockerHub`` and the ``GitHub Container Registry`` with a `tag`\ [#f2]_ that corresponds to the value of ``VERSION_STRING`` in ``changelog.h`` (see :doc:`../changelog` for detailed information regarding ``changelog.h``). At time of writing, `GitHub Actions`\ [#f3]_ facilitates the above process. While this is convenient (because it's free and well @@ -24,6 +37,25 @@ See the `Atlassian CI/CD `__. + + Dockerfile ---------- From 364241fc8a8c80c32a5bd666a0f9d8a29871ac6d Mon Sep 17 00:00:00 2001 From: Ilya Mandel Date: Sun, 19 Apr 2026 19:23:45 +1000 Subject: [PATCH 05/22] Corrected the M&M NS remnant mass prescription to never return a remnant mass larger than the CO core mass (see issue #1468) --- src/GiantBranch.cpp | 12 ++++++------ src/changelog.h | 4 +++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/GiantBranch.cpp b/src/GiantBranch.cpp index 9d25e1650..244a9d2bc 100644 --- a/src/GiantBranch.cpp +++ b/src/GiantBranch.cpp @@ -1471,31 +1471,31 @@ double GiantBranch::CalculateRemnantNSMassMullerMandel(const double p_COCoreMass while (iterations++ < MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS && (utils::Compare(remnantMass, MULLERMANDEL_MINNS) < 0 || utils::Compare(remnantMass, OPTIONS->MaximumNeutronStarMass()) > 0 || - utils::Compare(remnantMass, p_HeCoreMass) > 0)) { + utils::Compare(remnantMass, p_COCoreMass) > 0)) { remnantMass = MULLERMANDEL_MU1 + RAND->RandomGaussian(MULLERMANDEL_SIGMA1); } if (iterations >= MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS) // failure to find a solution implies a narrow range; just pick a midpoint in this case - remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_HeCoreMass) + MULLERMANDEL_MINNS) / 2.0; + remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_COCoreMass) + MULLERMANDEL_MINNS) / 2.0; } else if (utils::Compare(p_COCoreMass, MULLERMANDEL_M2) < 0) { while (iterations++ < MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS && (utils::Compare(remnantMass, MULLERMANDEL_MINNS) < 0 || utils::Compare(remnantMass, OPTIONS->MaximumNeutronStarMass()) > 0 || - utils::Compare(remnantMass, p_HeCoreMass) > 0)) { + utils::Compare(remnantMass, p_COCoreMass) > 0)) { remnantMass = MULLERMANDEL_MU2A + MULLERMANDEL_MU2B / (MULLERMANDEL_M2 - MULLERMANDEL_M1) * (p_COCoreMass - MULLERMANDEL_M1) + RAND->RandomGaussian(MULLERMANDEL_SIGMA2); } if (iterations >= MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS) // failure to find a solution implies a narrow range; just pick a midpoint in this case - remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_HeCoreMass) + MULLERMANDEL_MINNS) / 2.0; + remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_COCoreMass) + MULLERMANDEL_MINNS) / 2.0; } else { while (iterations++ < MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS && (utils::Compare(remnantMass, MULLERMANDEL_MINNS) < 0 || utils::Compare(remnantMass, OPTIONS->MaximumNeutronStarMass()) > 0 || - utils::Compare(remnantMass, p_HeCoreMass) > 0)) { + utils::Compare(remnantMass, p_COCoreMass) > 0)) { remnantMass = MULLERMANDEL_MU3A + MULLERMANDEL_MU3B / (MULLERMANDEL_M3 - MULLERMANDEL_M2) * (p_COCoreMass - MULLERMANDEL_M2) + RAND->RandomGaussian(MULLERMANDEL_SIGMA3); } if (iterations >= MULLERMANDEL_REMNANT_MASS_MAX_ITERATIONS) // failure to find a solution implies a narrow range; just pick a midpoint in this case - remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_HeCoreMass) + MULLERMANDEL_MINNS) / 2.0; + remnantMass = (std::min(OPTIONS->MaximumNeutronStarMass(), p_COCoreMass) + MULLERMANDEL_MINNS) / 2.0; } return remnantMass; } diff --git a/src/changelog.h b/src/changelog.h index 8392f10b6..665ae8aa8 100644 --- a/src/changelog.h +++ b/src/changelog.h @@ -1701,6 +1701,8 @@ // - Fix for issue 1463: sign error in the Claeys+2014 common-envelope lambda prescription // 03.29.03 NRS - April 3, 2026 - Defect repair: // - Fixed HeSDs not being recorded in the Supernovae logs (mentioned in issue 1350). +// 03.29.04 IM - April 19, 2026 - Enhancement: +// - Corrected the M&M NS remnant mass prescription to never return a remnant mass larger than the CO core mass (see issue #1468) // // // Version string format is MM.mm.rr, where @@ -1712,7 +1714,7 @@ // if MM is incremented, set mm and rr to 00, even if defect repairs and minor enhancements were also made // if mm is incremented, set rr to 00, even if defect repairs were also made -const std::string VERSION_STRING = "03.29.03"; +const std::string VERSION_STRING = "03.29.04"; # endif // __changelog_h__ From 0aa9e76e36b7f78674789da335913621aaef2bd3 Mon Sep 17 00:00:00 2001 From: Avi Vajpeyi Date: Wed, 22 Apr 2026 13:19:45 -0400 Subject: [PATCH 06/22] CPP ?= g++ to CPP = g++ --- src/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index ae3d83836..0cf30f8ff 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,4 +1,5 @@ -CPP ?= g++ +CPP = g++ + UNAME_S := $(shell uname -s) HOMEBREW_PREFIX := $(shell brew --prefix 2>/dev/null) From effdbcdbb85c10ea2c24f65c478085f278210417 Mon Sep 17 00:00:00 2001 From: Avi Vajpeyi Date: Wed, 22 Apr 2026 23:14:55 -0400 Subject: [PATCH 07/22] Fix pr commenter bug (#1479) * fix command due to updates in @dvcorg/cml * fix node version, check for the saved evolution plot before trying to comment * simligy setup * add installs * update command * Revert "add installs" This reverts commit c272217764987eb48bb3aa316e741aded1e66d81. * Fix missing newline at end of compas-compile-ci.yml --- .github/workflows/comment-evolution-plot.yml | 49 ++++++++++---------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/comment-evolution-plot.yml b/.github/workflows/comment-evolution-plot.yml index ed7208168..1f996ee9c 100644 --- a/.github/workflows/comment-evolution-plot.yml +++ b/.github/workflows/comment-evolution-plot.yml @@ -62,37 +62,36 @@ jobs: - name: Unpack evolution plot artifact run: | + set -e mkdir -p evolution-plot unzip -o evolution-plot.zip -d evolution-plot test -f "evolution-plot/${ARTIFACT_NAME}" - - name: Create report with evolution plot + - name: Post PR comment env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} PR_NUMBER: ${{ github.event.workflow_run.pull_requests[0].number }} HEAD_SHA: ${{ github.event.workflow_run.head_sha }} RUN_ID: ${{ github.event.workflow_run.id }} run: | - echo "## ✅ COMPAS Build Successful!" >> report.md - echo "" >> report.md - echo "| Item | Value |" >> report.md - echo "|------|-------|" >> report.md - echo "| **Commit** | [\`$(echo "$HEAD_SHA" | cut -c1-7)\`](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${HEAD_SHA}) |" >> report.md - echo "| **Logs** | [View workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}) |" >> report.md - echo "" >> report.md - - if [ -f "evolution-plot/${ARTIFACT_NAME}" ]; then - echo "### Detailed Evolution Plot" >> report.md - echo "
Click to view evolution plot" >> report.md - echo "" >> report.md - echo "![](./evolution-plot/${ARTIFACT_NAME})" >> report.md - echo "
" >> report.md - else - echo "### ⚠️ Evolution plot not found" >> report.md - fi - - echo "" >> report.md - echo "---" >> report.md - echo "Generated by COMPAS CI " >> report.md - - npx -y @dvcorg/cml comment create --pr "$PR_NUMBER" report.md + set -e + + cat < report.md + ## ✅ COMPAS Build Successful! + + | Item | Value | + |------|-------| + | **Commit** | [\`$(echo "$HEAD_SHA" | cut -c1-7)\`]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/commit/$HEAD_SHA) | + | **Logs** | [View workflow]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$RUN_ID) | + + ### Detailed Evolution Plot +
Click to view evolution plot + + ![](./evolution-plot/${ARTIFACT_NAME}) +
+ + --- + Generated by COMPAS CI + EOF + + npx -y @dvcorg/cml@0.20.6 comment create --target=pr/"$PR_NUMBER" report.md From 3b9e63248d67a85cb0ff13f3025ac3909fd2d33d Mon Sep 17 00:00:00 2001 From: Ilya Mandel Date: Mon, 4 May 2026 00:57:37 +1000 Subject: [PATCH 08/22] Matlab cosmic integration updates --- compas_matlab_utils/CosmicHistoryIntegrator.m | 183 ++++++++++++------ 1 file changed, 125 insertions(+), 58 deletions(-) diff --git a/compas_matlab_utils/CosmicHistoryIntegrator.m b/compas_matlab_utils/CosmicHistoryIntegrator.m index 8b2dd4bfb..1d3ca62ba 100644 --- a/compas_matlab_utils/CosmicHistoryIntegrator.m +++ b/compas_matlab_utils/CosmicHistoryIntegrator.m @@ -1,24 +1,34 @@ -function [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, ... - MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, Rdetections, DetectableMergerRate]=... - CosmicHistoryIntegrator(filename, zlistformation, zmaxdetection, Msimulated, makeplots) +function [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByBinary, ... + FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, MergerRateByRedshiftByBinary,... + MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, pdetectionByRedshiftByMtByEta, ... + DetectableMergerRateByRedshiftByBinary, DetectableMergerRateByRedshiftByMtByEta, ... + RdetectionsByRedshiftByBinary, RdetectionsByRedshiftByMtByEta, RdetectionsPerfectDetectorByRedshiftByBinary]=... + CosmicHistoryIntegrator(filename, noisefile, zlistformation, zmaxdetection, Msimulated, makeplots, data) % Integrator for the binary black hole merger rate over cosmic history % COMPAS (Compact Object Mergers: Population Astrophysics and Statistics) % software package % % USAGE: -% [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, ... -% MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, Rdetections, DetectableMergerRate]]=... -% CosmicHistoryIntegrator(filename, zlistformation, zmaxdetection, Msimulated [,makeplots]) +% [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByBinary, ... +% FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, MergerRateByRedshiftByBinary,... +% MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, pdetectionByRedshiftByMtByEta, ... +% DetectableMergerRateByRedshiftByBinary, DetectableMergerRateByRedshiftByMtByEta, ... +% RdetectionsByRedshiftByBinary, RdetectionsByRedshiftByMtByEta, RdetectionsPerfectDetectorByRedshiftByBinary]=... +% CosmicHistoryIntegrator(filename, noisefile, zlistformation, zmaxdetection, Msimulated, Msimulated [,makeplots, data]) % % INPUTS: % filename: name of population synthesis input file % should be in COMPAS output h5 format +% noisefile: file containing noise ASD (first column frequency, second ASD) % zlistformation: vector of redshifts at which the formation rate is % computed % zmaxdetection: maximum redshift to which the detection rate is computed % Msimulated: total star forming mass represented by the simulation (for % normalisation) % makeplots: if set to 1, generates a set of useful plots (default = 0) +% data: if provided, this contains a list of simulated binaries as a matrix, +% with each row containing a binary and the columns containing M1, M2, Z and +% tdelay in that order; the filename is then ignored % % OUTPUTS: % SFR is a vector of size length(zlistformation) containing the star formation rate @@ -26,14 +36,20 @@ % Zlist is a vector of metallicities, taken from the COMPAS run input file % Mtlist is a list of total mass bins % etalist is a list of symmetric mass ratio bins +% FormationRateByRedshiftByBinary is a matrix of size length(zformationlist) X length(M1) +% which contains a formation rate of merging double compact objects just +% like this binary, in units of formed DCOs per Mpc^3 of comoving volume per year of source time % FormationRateByRedshiftByZ is a matrix of size length(zformationlist) X length(Zlist) -% which contains a formation rate of merging double compact objects in the given redshift +% which contains the formation rate of merging double compact objects in the given redshift % and metallicity bin, in units of formed DCOs per Mpc^3 of comoving volume per % year of source time % FormationRateByRedshiftByMtByEta is a matrix of size length(zformationlist) % X length(Mtlist) X length(etalist) which contains a formation rate of merging double compact objects % in the given redshift, total mass and eta bin, in units of formed DCOs per Mpc^3 % of comoving volume per year of source time +% MergerRateByRedshiftByBinary is a matrix of size length(zformationlist) X length(M1) +% which contains the merger rate of merging double compact objects just +% like this binary, in units of mergers per Mpc^3 of comoving volume per year of source time % MergerRateByRedshiftByZ is a matrix of size length(zformationlist) X length(Zlist) % which contains a merger rate of double compact objects in the given redshift % and metallicity bin, in units of mergers per Mpc^3 of comoving volume per @@ -44,21 +60,39 @@ % of comoving volume per year of source time % zlistdetection is a vector of redshifts at which detection rates are % computed (a subset of zlistformation going up to zmaxdetection) -% Rdetection is a matrix of size length(zlistdetection) X length(Mtlist) X -% length(etalist) containing the detection rate per year of observer time -% from a given redshift bin and total mass and symmetric mass ratio pixel -% DetectableMergerRate is a matrix of the same size as Rdetection but -% containing the intrinsic rate of detectable mergers per Mpc^3 of comoving +% pdetectionByRedshiftByMtByEta is a matrix of size length(zlistdetection) X length(Mtlist) X length(etalist) +% containing the probability that a binary of a given mass and mass ratio +% is detectable for a given merger redshift +% DetectableMergerRateByRedshiftByBinary is a matrix of size +% length(zlistdetection) X length(M1), containing the intrinsic rate of mergers of +% binaries just like this one per Mpc^3 of comoving % volume per year of source time +% DetectableMergerRateByRedshiftByMtByEta is a matrix of size +% length(zlistdetection) X length(Mtlist) X length(etalist) containing the detection rate per year of observer time +% from a given redshift bin and total mass and symmetric mass ratio pixel +% RdetectionsByRedshiftByBinary is a matrix of the same size as +% DetectableMergerRateByRedshiftByBinary containing the detection rate +% for binaries just like this one per year of observer time +% from a given redshift bin +% RdetectionsByRedshiftByMtByEta is a matrix of the same size as +% DetectableMergerRateByRedshiftByMtByEta containing the detection rate per +% year of observer time from a given redshift bin and total mass and symmetric mass ratio pixel +% RdetectionsPerfectDetectorByRedshiftByBinary is a matrix of the same size +% as RdetectionsByRedshiftByBinary containing the detection rate per year +% of observer time from a given redshift bin for binaries just like this one +% assuming an imaginary perfectly sensitive detector % % EXAMPLE: % zlist=0:0.01:10; -% [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, ... -% MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, Rdetections, DetectableMergerRate]=... -% CosmicHistoryIntegrator('~/Work/COMPASresults/runs/Zdistalpha1-031803.h5', zlist, 1.5, 90e6, 1); +% [SFR, Zlist, Mtlist, etalist, FormationRateByRedshiftByBinary, ... +% FormationRateByRedshiftByZ, FormationRateByRedshiftByMtByEta, MergerRateByRedshiftByBinary,... +% MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, pdetectionByRedshiftByMtByEta, ... +% DetectableMergerRateByRedshiftByBinary, DetectableMergerRateByRedshiftByMtByEta, ... +% RdetectionsByRedshiftByBinary, RdetectionsByRedshiftByMtByEta, RdetectionsPerfectDetectorByRedshiftByBinary]=... +% CosmicHistoryIntegrator('~/Work/COMPASresults/runs/Zdistalpha1-031803.h5', '~/Work/Rai/aligo_O4high.txt', zlist, 1.5, 90e6, 1); % figure(10), semilogy(zlist, sum(MergerRateByRedshiftByZ,2)*1e9,'LineWidth',3), set(gca,'FontSize',20), -% xlabel('Redshift z'), ylabel('Formation rate of merging DCO per Gpc^3 per yr') +% xlabel('Redshift z'), ylabel('Merger rate of DCO per Gpc^3 per yr') % @@ -71,15 +105,26 @@ Mpc=Mpcm/c; %Mpc in seconds yr=3.15569e7; %year in seconds -if (nargin<4) +if (nargin<5) error('Not enough input arguments.'); end; if (nargin<5), makeplots=0; end; +if (nargin<7), + %load COMPAS data + [M1,M2,Z,Tdelay]=DataRead(filename); +else + if(size(data,2)~=4), + error('The data must have 4 columns: M1, M2, Z, Tdelay'); + end; + M1=data(:,1); M2=data(:,2); Z=data(:,3); Tdelay=data(:,4); + %maxNS=0.0; %pretend all are BBH if reading from file +end; + + %cosmology calculator [tL,Dl,dVc]=Cosmology(zlistformation); -%load COMPAS data -[M1,M2,Z,Tdelay,maxNS]=DataRead(filename); + %metallicity-specific SFR [SFR,Zlist,Zweight]=Metallicity(zlistformation,min(Z),max(Z)); @@ -90,44 +135,72 @@ dz=zlistformation(2)-zlistformation(1); etalist=0.01:0.01:0.25; Mtlist=1:1:ceil(max(M1+M2)); +FormationRateByRedshiftByBinary=zeros(length(zlistformation),length(M1)); FormationRateByRedshiftByZ=zeros(length(zlistformation),length(Zlist)); FormationRateByRedshiftByMtByEta=zeros(length(zlistformation),length(Mtlist),length(etalist)); +MergerRateByRedshiftByBinary=zeros(length(zlistformation),length(M1)); MergerRateByRedshiftByZ=zeros(length(zlistformation),length(Zlist)); MergerRateByRedshiftByMtByEta=zeros(length(zlistformation),length(Mtlist),length(etalist)); -x=zeros(size(M1)); +etaindexByBinary=zeros(size(M1)); MtindexByBinary=zeros(size(M1)); for(i=1:length(M1)), Zcounter=find(Zlist>=Z(i),1); eta=M1(i)*M2(i)/(M1(i)+M2(i))^2; - etaindex=ceil(eta*100); - Mtindex=ceil(M1(i)+M2(i)); - FormationRateByRedshiftByZ(:,Zcounter)=transpose(SFR).*Zweight(:,Zcounter)/Msimulated; - FormationRateByRedshiftByMtByEta(:,Mtindex,etaindex)=transpose(SFR).*Zweight(:,Zcounter)/Msimulated; + etaindex=ceil(eta*100); etaindexByBinary(i)=etaindex; + Mtindex=ceil(M1(i)+M2(i)); MtindexByBinary(i)=Mtindex; + FormationRateByRedshiftByBinary(:,i)=transpose(SFR).*Zweight(:,Zcounter)/Msimulated; + FormationRateByRedshiftByZ(:,Zcounter)=FormationRateByRedshiftByZ(:,Zcounter)+... + FormationRateByRedshiftByBinary(:,i); + FormationRateByRedshiftByMtByEta(:,Mtindex,etaindex)=FormationRateByRedshiftByMtByEta(:,Mtindex,etaindex)+... + FormationRateByRedshiftByBinary(:,i); tLform=tL+Tdelay(i); %lookback time of when binary would have to form in order to merge at lookback time tL firsttooearlyindex=find((tLform)>max(tL),1); if(isempty(firsttooearlyindex)), firsttooearlyindex=length(tL)+1; end; zForm=interp1(tL,zlistformation,tLform(1:firsttooearlyindex-1)); zFormindex=ceil((zForm-zlistformation(1))./dz)+1; if(~isempty(zFormindex)) - x(i)=SFR(zFormindex(1))*Zweight(zFormindex(1),Zcounter)/Msimulated; + MergerRateByRedshiftByBinary(1:firsttooearlyindex-1,i)=... + transpose(SFR(zFormindex)).*Zweight(zFormindex,Zcounter)/Msimulated; MergerRateByRedshiftByZ(1:firsttooearlyindex-1,Zcounter)=... MergerRateByRedshiftByZ(1:firsttooearlyindex-1,Zcounter)+... - transpose(SFR(zFormindex)).*Zweight(zFormindex,Zcounter)/Msimulated; + MergerRateByRedshiftByBinary(1:firsttooearlyindex-1,i); MergerRateByRedshiftByMtByEta(1:firsttooearlyindex-1,Mtindex,etaindex) =... MergerRateByRedshiftByMtByEta(1:firsttooearlyindex-1,Mtindex,etaindex) + ... - transpose(SFR(zFormindex)).*Zweight(zFormindex,Zcounter)/Msimulated; + MergerRateByRedshiftByBinary(1:firsttooearlyindex-1,i); end; end; zlistdetection=zlistformation(1:find(zlistformation<=zmaxdetection,1,"last")); -fin=load('~/Work/Rai/LIGOfuture_data/freqVector.txt'); -%noise=load('~/Work/Rai/LIGOfuture_data/dataNomaLIGO.txt'); -noise=load('~/Work/Rai/LIGOfuture_data/dataEarly_low.txt'); -[Rdetections,DetectableMergerRate]=... - DetectionRate(zlistformation,Mtlist,etalist,MergerRateByRedshiftByMtByEta,zlistdetection,fin,noise,Dl,dVc); + +%detection probability +pdetectionByRedshiftByMtByEta=... + DetectionProbability(zlistdetection,Mtlist,etalist,noisefile,Dl); +pdetectionsByZByBinary=zeros(length(zlistdetection),length(M1)); +for(i=1:length(M1)), + pdetectionByRedshiftByBinary(:,i)=pdetectionByRedshiftByMtByEta(:,MtindexByBinary(i),etaindexByBinary(i)); +end; +%Detections per unit source time per unit Vc in each Mt and eta bin +DetectableMergerRateByRedshiftByMtByEta=... + MergerRateByRedshiftByMtByEta(1:length(zlistdetection),:,:).*pdetectionByRedshiftByMtByEta; +%Detections per unit source time per unit Vc for each binary +DetectableMergerRateByRedshiftByBinary=... + MergerRateByRedshiftByBinary(1:length(zlistdetection),:).*pdetectionByRedshiftByBinary; +%Detections per unit observer time in each Mt and eta bin +RdetectionsByRedshiftByMtByEta=... + DetectableMergerRateByRedshiftByMtByEta.*transpose(dVc(1:length(zlistdetection)))./(1+zlistdetection'); +%Detections per unit observer time by binary +RdetectionsByRedshiftByBinary=... + DetectableMergerRateByRedshiftByBinary.*transpose(dVc(1:length(zlistdetection)))./(1+zlistdetection'); +%Detections per unit observer time per binary by an imaginary perfect detector +RdetectionsPerfectDetectorByRedshiftByBinary=... + MergerRateByRedshiftByBinary(1:length(zlistdetection),:).*... + transpose(dVc(1:length(zlistdetection)))./(1+zlistdetection'); +%total detection rate +disp(['Total detection rate: ', ... + num2str(sum(sum(RdetectionsByRedshiftByBinary))), ' per year']); if(makeplots==1), %make a set of default plots MakePlots(M1,M2,Z,Tdelay,zlistformation,Zlist,SFR,Zweight,... - MergerRateByRedshiftByZ, Rdetections, DetectableMergerRate, zlistdetection, Mtlist, etalist, 1); + MergerRateByRedshiftByZ, RdetectionsByRedshiftByMtByEta, DetectableMergerRateByRedshiftByMtByEta, zlistdetection, Mtlist, etalist, 1); end; end %end of CosmicHistoryIntegrator @@ -136,7 +209,7 @@ %Load the data stored in COMPAS .h5 output format from a file %Select only double compact object mergers of interest, and return the %component masses, metallicities, and star formation to merger delay times -function [M1,M2,Z,Tdelay, maxNS]=DataRead(file) +function [M1,M2,Z,Tdelay]=DataRead(file) if(exist(file, 'file')~=2), error('Input file does not exist'); end; @@ -157,7 +230,7 @@ %NSBH=(((type1==13) & (type2==14)) | ((type1==14) & (type2==13))); %mergingDCO=mergingBNS | mergingNSBH | mergingBBH; %BNScount=sum(mergingBNS); NSBHcount=sum(mergingNSBH); BBHcount=sum(mergingBBH); - maxNS=max(max(mass1(type1==13)), max(mass2(type2==13))); + %maxNS=max(max(mass1(type1==13)), max(mass2(type2==13))); chirpmass=mass1.^0.6.*mass2.^0.6./(mass1+mass2).^0.2; q=mass2./mass1; seedCE=h5read(file,'/BSE_Common_Envelopes/SEED'); @@ -235,16 +308,17 @@ %Compute detection rates per year of observer time and per year of source time %per Mpc^3 of comoving volume as a function of total mass and eta -function [Rdetections, DetectableMergerRate]=... - DetectionRate(zlistformation,Mtlist,etalist,MergerRateByRedshiftByMtByEta,zlistdetection,freqfile,noisefile,Dl,dVc) +%Compute the detection probability as a function of detection redshift, +%total mass and eta +function [pdetection]=... + DetectionProbability(zlistdetection,Mtlist,etalist,noisefile,Dl) - fin=load('~/Work/Rai/LIGOfuture_data/freqVector.txt'); - noise=load('~/Work/Rai/LIGOfuture_data/dataMid_low.txt'); + noise=load(noisefile); - flow=10; + flow=max(10,ceil(min(noise(:,1)))); df=1; f=flow:df:500; %BBH focussed - Sf=interp1(fin, noise.^2, f); + Sf=interp1(noise(:,1), noise(:,2).^2, f); Ntheta=1e6; psi=rand(1,Ntheta)*pi; @@ -275,26 +349,19 @@ for(i=1:length(zlistdetection)), for(j=1:length(Mtlist)), - SNR(i,j,:)=SNRat1Mpc(ceil(j*(zlistdetection(i)+1)),:)./Dl(i); + SNR(i,j,:)=SNRat1Mpc(ceil(Mtlist(j)*(zlistdetection(i)+1)),:)./Dl(i); end; end; - %for(i=1:length(zlistdetection)), SNR(i,:,:)=SNRat1Mpc./Dl(i); end; - SNR8pre=1:0.1:1000; + SNR8pre=1:0.01:100; theta=1./SNR8pre; pdetect=1-interp1([0,Thetas,1],[(0:Ntheta)/Ntheta,1],theta); pdetect(1)=0; %set of measure zero to exceed threshold, but enforce just in case - Rdetections=zeros(length(zlistdetection),length(Mtlist),length(etalist)); %Detections per unit observer time - DetectableMergerRate=zeros(length(zlistdetection),length(Mtlist),length(etalist)); %Detections per unit source time per unit Vc SNR8=SNR/8; - pdetection=zeros(size(Rdetections)); - pdetection=pdetect(max(min(floor(SNR8*10),length(pdetect)),1)); - - DetectableMergerRate=MergerRateByRedshiftByMtByEta(1:length(zlistdetection),:,:).*pdetection; - Rdetections=DetectableMergerRate.*transpose(dVc(1:length(zlistdetection)))./(1+zlistdetection'); - -end %end of DetectionRate + pdetection=zeros(length(zlistdetection),length(Mtlist),length(etalist)); + pdetection=pdetect(max(min(floor((SNR8-1)*100),length(pdetect)),1)); +end %end of DetectionProbability %Make a set of default plots function MakePlots(M1,M2,Z,Tdelay,zlistformation,Zlist,SFR,Zweight,... @@ -341,11 +408,11 @@ function MakePlots(M1,M2,Z,Tdelay,zlistformation,Zlist,SFR,Zweight,... figure(fignumber+4), clf(fignumber+4); - RdetectionsByzMt=sum(Rdetections,3); %sum across eta - semilogy(zlistdetection, cumsum(sum(RdetectionsByzMt,2)), 'LineWidth', 3), hold on; - semilogy(zlistdetection, cumsum(sum(RdetectionsByzMt(:,Mtlist<=5),2)), 'LineWidth', 1); - semilogy(zlistdetection, cumsum(sum(RdetectionsByzMt(:,Mtlist>5 & Mtlist<20),2)), 'LineWidth', 1); - semilogy(zlistdetection, cumsum(sum(RdetectionsByzMt(:,Mtlist>=20),2)), 'LineWidth', 1); hold off; + RdetectionsByRedshiftMt=sum(Rdetections,3); %sum across eta + semilogy(zlistdetection, cumsum(sum(RdetectionsByRedshiftMt,2)), 'LineWidth', 3), hold on; + semilogy(zlistdetection, cumsum(sum(RdetectionsByRedshiftMt(:,Mtlist<=5),2)), 'LineWidth', 1); + semilogy(zlistdetection, cumsum(sum(RdetectionsByRedshiftMt(:,Mtlist>5 & Mtlist<20),2)), 'LineWidth', 1); + semilogy(zlistdetection, cumsum(sum(RdetectionsByRedshiftMt(:,Mtlist>=20),2)), 'LineWidth', 1); hold off; legend('Total rate', 'From M_t<=5 M_o', 'From 5=20 M_o'), set(gca, 'FontSize', 20); %for labels xlabel('z'), From 34efe339475ee7fac96db2f0248a42d04ac2cbc4 Mon Sep 17 00:00:00 2001 From: Ilya Mandel Date: Fri, 8 May 2026 16:33:56 +1000 Subject: [PATCH 09/22] Changed ASD to PSD --- compas_matlab_utils/CosmicHistoryIntegrator.m | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compas_matlab_utils/CosmicHistoryIntegrator.m b/compas_matlab_utils/CosmicHistoryIntegrator.m index 1d3ca62ba..e6ad67f72 100644 --- a/compas_matlab_utils/CosmicHistoryIntegrator.m +++ b/compas_matlab_utils/CosmicHistoryIntegrator.m @@ -19,7 +19,7 @@ % INPUTS: % filename: name of population synthesis input file % should be in COMPAS output h5 format -% noisefile: file containing noise ASD (first column frequency, second ASD) +% noisefile: file containing noise PSD (first column frequency, second PSD) % zlistformation: vector of redshifts at which the formation rate is % computed % zmaxdetection: maximum redshift to which the detection rate is computed @@ -90,7 +90,7 @@ % MergerRateByRedshiftByZ, MergerRateByRedshiftByMtByEta, zlistdetection, pdetectionByRedshiftByMtByEta, ... % DetectableMergerRateByRedshiftByBinary, DetectableMergerRateByRedshiftByMtByEta, ... % RdetectionsByRedshiftByBinary, RdetectionsByRedshiftByMtByEta, RdetectionsPerfectDetectorByRedshiftByBinary]=... -% CosmicHistoryIntegrator('~/Work/COMPASresults/runs/Zdistalpha1-031803.h5', '~/Work/Rai/aligo_O4high.txt', zlist, 1.5, 90e6, 1); +% CosmicHistoryIntegrator('~/Work/COMPASresults/runs/Zdistalpha1-031803.h5', '~/Work/Rai/psd-O4-2023_06_v1-L.txt', zlist, 1.5, 90e6, 1); % figure(10), semilogy(zlist, sum(MergerRateByRedshiftByZ,2)*1e9,'LineWidth',3), set(gca,'FontSize',20), % xlabel('Redshift z'), ylabel('Merger rate of DCO per Gpc^3 per yr') % @@ -318,7 +318,8 @@ flow=max(10,ceil(min(noise(:,1)))); df=1; f=flow:df:500; %BBH focussed - Sf=interp1(noise(:,1), noise(:,2).^2, f); + Sf=interp1(noise(:,1), noise(:,2), f); + %Sf=interp1(noise(:,1), noise(:,2).^2, f); %for ASDs Ntheta=1e6; psi=rand(1,Ntheta)*pi; From 0aac620ea1a70217f3c5cd331fe79990926a3a62 Mon Sep 17 00:00:00 2001 From: Aldana Grichener <110453835+AldanaGrichener@users.noreply.github.com> Date: Mon, 25 May 2026 12:34:59 -0700 Subject: [PATCH 10/22] Fix to non-random supernova kick angles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ran into another bug similar to issue #1378 (while working on the CBD PR, still forthcoming 🙂), applied the analogous fix to PR #1387. (Missed it back then because I bypassed #1378 by using a bash script which sent each seed individually) --- src/Options.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Options.h b/src/Options.h index b22eafe02..a780f7d74 100755 --- a/src/Options.h +++ b/src/Options.h @@ -1748,12 +1748,12 @@ class Options { void ShowHelp() { PrintOptionHelp(!m_CmdLine.optionValues.m_ShortHelp); } - double SN_MeanAnomaly1() const { return OPT_VALUE("kick-mean-anomaly-1", m_KickMeanAnomaly1, true); } - double SN_MeanAnomaly2() const { return OPT_VALUE("kick-mean-anomaly-2", m_KickMeanAnomaly2, true); } - double SN_Phi1() const { return OPT_VALUE("kick-phi-1", m_KickPhi1, true); } - double SN_Phi2() const { return OPT_VALUE("kick-phi-2", m_KickPhi2, true); } - double SN_Theta1() const { return OPT_VALUE("kick-theta-1", m_KickTheta1, true); } - double SN_Theta2() const { return OPT_VALUE("kick-theta-2", m_KickTheta2, true); } + double SN_MeanAnomaly1() const { return OPT_VALUE("kick-mean-anomaly-1", m_KickMeanAnomaly1, false); } + double SN_MeanAnomaly2() const { return OPT_VALUE("kick-mean-anomaly-2", m_KickMeanAnomaly2, false); } + double SN_Phi1() const { return OPT_VALUE("kick-phi-1", m_KickPhi1, false); } + double SN_Phi2() const { return OPT_VALUE("kick-phi-2", m_KickPhi2, false); } + double SN_Theta1() const { return OPT_VALUE("kick-theta-1", m_KickTheta1, false); } + double SN_Theta2() const { return OPT_VALUE("kick-theta-2", m_KickTheta2, false); } ZETA_PRESCRIPTION StellarZetaPrescription() const { return OPT_VALUE("stellar-zeta-prescription", m_StellarZetaPrescription.type, true); } bool StoreInputFiles() const { return m_CmdLine.optionValues.m_StoreInputFiles; } From b5568f125bdb4ff34d2cd5097bcb38a408b88ac1 Mon Sep 17 00:00:00 2001 From: jeffriley Date: Tue, 26 May 2026 13:08:40 +1000 Subject: [PATCH 11/22] Update Options.h - fix typo --- src/Options.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Options.h b/src/Options.h index a780f7d74..a9c1eb162 100755 --- a/src/Options.h +++ b/src/Options.h @@ -72,7 +72,7 @@ const std::string NOT_PROVIDED_STR(1, static_cast(NOT_PROVIDED_CHAR)); // 2. if 'fallback' is 'true': // the value specified on the commandline if the user did not specify the // option on the grid line (regardless of whether they specified the option -// on the commandline). In this case, if the user did not speify a value on +// on the commandline). In this case, if the user did not specify a value on // the commandline, the commandline value is set according to the default // behaviour for the option, and the grid line value is set from that. Note // that for options whose default behaviours is to draw a random number, this From defc1ae75acbce7519d935edb1a80ae47cbe5a99 Mon Sep 17 00:00:00 2001 From: Aldana Grichener <110453835+AldanaGrichener@users.noreply.github.com> Date: Tue, 26 May 2026 12:15:51 -0700 Subject: [PATCH 12/22] Updated changelog.h --- src/changelog.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/changelog.h b/src/changelog.h index 665ae8aa8..20cd4638c 100644 --- a/src/changelog.h +++ b/src/changelog.h @@ -1703,7 +1703,8 @@ // - Fixed HeSDs not being recorded in the Supernovae logs (mentioned in issue 1350). // 03.29.04 IM - April 19, 2026 - Enhancement: // - Corrected the M&M NS remnant mass prescription to never return a remnant mass larger than the CO core mass (see issue #1468) -// +// 03.29.05 AG - May 26, 2026 - Defect repair: +// - Fix for generalized issue #1378: reinstate "false" fallback option for SN kick angle options (mistakenly changed to "true" in v03.00.00) // // Version string format is MM.mm.rr, where // @@ -1714,7 +1715,7 @@ // if MM is incremented, set mm and rr to 00, even if defect repairs and minor enhancements were also made // if mm is incremented, set rr to 00, even if defect repairs were also made -const std::string VERSION_STRING = "03.29.04"; +const std::string VERSION_STRING = "03.29.05"; # endif // __changelog_h__ From b401059f3f64db65a701890e2b0b984b5aeb011f Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Fri, 26 Jun 2026 17:52:09 +0200 Subject: [PATCH 13/22] big cleanup of post processing utils --- compas_python_utils/compasUtils.py | 355 -------------------- compas_python_utils/debugging_utils.py | 440 +++++++++++++++++++++++++ 2 files changed, 440 insertions(+), 355 deletions(-) delete mode 100644 compas_python_utils/compasUtils.py create mode 100644 compas_python_utils/debugging_utils.py diff --git a/compas_python_utils/compasUtils.py b/compas_python_utils/compasUtils.py deleted file mode 100644 index 2cc126c68..000000000 --- a/compas_python_utils/compasUtils.py +++ /dev/null @@ -1,355 +0,0 @@ -import h5py as h5 -import numpy as np -import pandas as pd -from numpy.dtypes import StringDType - - -######################################################################## -# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template - -def printCompasDetails(data, *seeds, mask=()): - """ - Function to print the full Compas output for given seeds, optionally with an additional mask - """ - list_of_keys = list(data.keys()) - - # Check if seed parameter exists - if not, just print without (e.g RunDetails) - if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): # Most output files - #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility - - # Set the seed name parameter, mask on seeds as needed, and set the index - seedVariableName='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' - list_of_keys.remove(seedVariableName) # this is the index above, don't want to include it - - allSeeds = data[seedVariableName][()] - seedsMask = np.isin(allSeeds, seeds) - if len(seeds) == 0: # if any seed is included, do not reset the mask - seedsMask = np.ones_like(allSeeds).astype(bool) - if mask == (): - mask = np.ones_like(allSeeds).astype(bool) - mask &= seedsMask - - df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list(data.keys())}).set_index(seedVariableName).T - - else: # No seed parameter, so do custom print for Run Details - - # Get just the keys without the -Derivation suffix - those will be a second column - keys_not_derivations = [] - for key in list_of_keys: - if '-Derivation' not in key: - keys_not_derivations.append(key) - - # Some parameter values are string types, formatted as np.bytes_, need to convert back - def convert_strings(param_array): - if isinstance(param_array[0], np.bytes_): - return param_array.astype(str) - else: - return param_array - - df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T - nCols = df_keys.shape[1] # Required only because if we combine RDs, we get many columns (should fix later) - df_keys.columns = ['Parameter']*nCols - df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T - df_drvs.columns = ['Derivation']*nCols - df = pd.concat([df_keys, df_drvs], axis=1) - - # Add units as first col - units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} - df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) - return df - - - -######################################################################## -# ## Get event histories of MT data, SN data, and combined MT, SN data - -def getMtEvents(MT): - """ - This function takes in the `BSE_RLOF` output category from COMPAS, and returns the information - on the Mass Transfer (MT) events that happen for each seed. The events do not have to be in order, - either chronologically or by seed, this function will reorder them as required. - - OUT: - returnedSeeds (list): ordered list of the unique seeds in the MT file - returnedEvents (list): list of sublists, where each sublist contains all the MT events for a given seed. - MT event tuples take the form : - (stellarTypePrimary, stellarTypeSecondary, isRlof1, isRlof2, isCEE, isMrg) - returnTimes (list): is a list of sublists of times of each of the MT events - """ - mtSeeds = MT['SEED'][()] - mtTimes = MT['TimeMT'][()] == 1 - mtIsRlof2 = MT['RLOF(2)>MT'][()] == 1 - mtIsCEE = MT['CEE>MT'][()] == 1 - mtIsMrg = MT['Merger'][()] == 1 - - # We want the return arrays sorted by seed, so sort here. - mtSeedsInds = np.lexsort((mtTimes, mtSeeds)) # sort by seeds then times - lexsort sorts by the last column first... - mtSeeds = mtSeeds[mtSeedsInds] - mtTimes = mtTimes[mtSeedsInds] - mtPrimaryStype = mtPrimaryStype[mtSeedsInds] - mtSecondaryStype = mtSecondaryStype[mtSeedsInds] - mtIsRlof1 = mtIsRlof1[mtSeedsInds] - mtIsRlof2 = mtIsRlof2[mtSeedsInds] - mtIsCEE = mtIsCEE[mtSeedsInds] - mtIsMrg = mtIsMrg[mtSeedsInds] - - # Process the MT events - returnedSeeds = [] # array of seeds - will only contain seeds that have MT events - returnedEvents = [] # array of MT events for each seed in returnedSeeds - returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) - - lastSeed = -1 # initialize most recently processed seed - for seedIndex, thisSeed in enumerate(mtSeeds): # iterate over all RLOF file entries - thisTime = mtTimes[seedIndex] # time for this RLOF file entry - thisEvent = (mtPrimaryStype[seedIndex], mtSecondaryStype[seedIndex], - mtIsRlof1[seedIndex], mtIsRlof2[seedIndex], mtIsCEE[seedIndex], mtIsMrg[seedIndex]) # construct event tuple - - # If this is an entirely new seed: - if thisSeed != lastSeed: # same seed as last seed processed? - returnedSeeds.append(thisSeed) # no - new seed, record it - returnedTimes.append([thisTime]) # initialize the list of event times for this seed - returnedEvents.append([thisEvent]) # initialize the list of events for this seed - lastSeed = thisSeed # update the latest seed - - # Add event, if it is not a duplicate - try: - eventIndex = returnedEvents[-1].index(thisEvent) # find eventIndex of this particular event tuple in the array of events for this seed - if thisTime > returnedTimes[-1][eventIndex]: # ^ if event is not a duplicate, this will throw a ValueError - returnedTimes[-1][eventIndex] = thisTime # if event is duplicate, update time to the later of the duplicates - except ValueError: # event is not a duplicate: - returnedEvents[-1].append(thisEvent) # record new event tuple for this seed - returnedTimes[-1].append(thisTime) # record new event time for this seed - - return returnedSeeds, returnedEvents, returnedTimes # see above for description - - -def getSnEvents(SN): - """ - This function takes in the `BSE_Supernovae` output category from COMPAS, and returns the information - on the Supernova (SN) events that happen for each seed. The events do not have to be in order chronologically, - this function will reorder them as required. - - OUT: - returnedSeeds (list): ordered list of all the unique seeds in the SN file - returnedEvents (list): list of sublists, where each sublist contains all the SN events for a given seed. - SN event tuples take the form : - (stellarTypeProgenitor, stellarTypeRemnant, whichStarIsProgenitor, isBinaryUnbound) - returnedTimes (list): is a list of sublists of times of each of the SN events - """ - snSeeds = SN['SEED'][()] - snTimes = SN['Time'][()] - snProgStype = SN['Stellar_Type_Prev(SN)'][()] - snRemnStype = SN['Stellar_Type(SN)'][()] - snWhichProg = SN['Supernova_State'][()] - snIsUnbound = SN['Unbound'][()] == 1 - - # We want the return arrays sorted by seed, so sort here. - snSeedsInds = np.lexsort((snTimes, snSeeds)) # sort by seeds then times - lexsort sorts by the last column first... - snSeeds = snSeeds[snSeedsInds] - snTimes = snTimes[snSeedsInds] - snProgStype = snProgStype[snSeedsInds] - snRemnStype = snRemnStype[snSeedsInds] - snWhichProg = snWhichProg[snSeedsInds] - snIsUnbound = snIsUnbound[snSeedsInds] - - # Process the SN events - returnedSeeds = [] # array of seeds - will only contain seeds that have SN events - returnedEvents = [] # array of SN events for each seed in returnedSeeds - returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) - - lastSeed = -1 # initialize most recently processed seed - for seedIndex, thisSeed in enumerate(snSeeds): # iterate over all SN file entries - thisTime = snTimes[seedIndex] # time for this SN file entry - thisEvent = (snProgStype[seedIndex], snRemnStype[seedIndex], - snWhichProg[seedIndex], snIsUnbound[seedIndex]) # construct event tuple - - # If this is an entirely new seed: - if thisSeed != lastSeed: # same seed as last seed processed? - returnedSeeds.append(thisSeed) # no - new seed, record it - returnedTimes.append([thisTime]) # initialize the list of event times for this seed - returnedEvents.append([thisEvent]) # initialize the list of events for this seed - lastSeed = thisSeed # update the latest seed - else: # yes - second SN event for this seed - returnedTimes[-1].append(thisTime) # append time at end of array - returnedEvents[-1].append(thisEvent) # append event at end of array - - return returnedSeeds, returnedEvents, returnedTimes # see above for description - - -def getEventHistory(h5file, exclude_null=False): - """ - Get the event history for all seeds, including both RLOF and SN events, in chronological order. - IN: - h5file (h5.File() type): COMPAS HDF5 output file - exclude_null (bool): whether or not to exclude seeds which undergo no RLOF or SN events - OUT: - returnedSeeds (list): ordered list of all seeds in the output - returnedEvents (list): a list (corresponding to the ordered seeds above) of the - collected SN and MT events from the getMtEvents and getSnEvents functions above, - themselves ordered chronologically - """ - SP = h5file['BSE_System_Parameters'] - MT = h5file['BSE_RLOF'] - SN = h5file['BSE_Supernovae'] - allSeeds = SP['SEED'][()] # get all seeds - mtSeeds, mtEvents, mtTimes = getMtEvents(MT) # get MT events - snSeeds, snEvents, snTimes = getSnEvents(SN) # get SN events - - numMtSeeds = len(mtSeeds) # number of MT events - numSnSeeds = len(snSeeds) # number of SN events - - if numMtSeeds < 1 and numSnSeeds < 1: return [] # no events - return empty history - - returnedSeeds = [] # array of seeds - will only contain seeds that have events (of any type) - returnedEvents = [] # array of events - same size as returnedSeeds (includes event times) - - eventOrdering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events - - mtIndex = 0 # index into MT events arrays - snIndex = 0 # index into SN events arrays - - if exclude_null: - seedsToIterate = np.sort(np.unique(np.append(mtSeeds, snSeeds))) # iterate over all the seeds that have either MT or SN events - else: - seedsToIterate = allSeeds - - idxOrdered = np.argsort(seedsToIterate) - returnedSeeds = [None] * np.size(seedsToIterate) # array of seeds - will only contain seeds that have events (of any type) - returnedEvents = [None] * np.size(seedsToIterate) # array of events - same size as returnedSeeds (includes event times) - - for idx in idxOrdered: - seed = seedsToIterate[idx] - seedEvents = [] # initialise the events for the seed being processed - - # Collect any MT events for this seed, add the time of the event and the event type - while mtIndex < numMtSeeds and mtSeeds[mtIndex] == seed: - for eventIndex, event in enumerate(mtEvents[mtIndex]): - _, _, isRL1, isRL2, isCEE, isMrg = event - eventKey = 'MG' if isMrg else 'CE' if isCEE else 'RL' # event type: Mrg, CEE, RLOF 1->2, RLOF 2->1 - seedEvents.append((eventKey, mtTimes[mtIndex][eventIndex], *mtEvents[mtIndex][eventIndex])) - mtIndex += 1 - - # Collect any SN events for this seed, add the time of the event and the event type - while snIndex < numSnSeeds and snSeeds[snIndex] == seed: - for eventIndex, event in enumerate(snEvents[snIndex]): - eventKey = 'SN' - seedEvents.append((eventKey, snTimes[snIndex][eventIndex], *snEvents[snIndex][eventIndex])) - snIndex += 1 - - seedEvents.sort(key=lambda ev:(ev[1], eventOrdering.index(ev[0]))) # sort the events by time and event type (MT before SN if at the same time) - - returnedSeeds[idx] = seed # record the seed in the seeds array being returned - returnedEvents[idx] = seedEvents # record the events for this seed in the events array being returned - - return returnedSeeds, returnedEvents # see above for details - - - -########################################### -# ## Produce strings of the event histories - -stellarTypeDict = { - 0: 'MS', - 1: 'MS', - 2: 'HG', - 3: 'GB', - 4: 'GB', - 5: 'GB', - 6: 'GB', - 7: 'HE', - 8: 'HE', - 9: 'HE', - 10: 'WD', - 11: 'WD', - 12: 'WD', - 13: 'NS', - 14: 'BH', - 15: 'MR', - 16: 'MS', -} - -def buildEventString(events, useIntStypes=False): - """ - Function to produce a string representing the event history of a single binary for quick readability. - NOTE: unvectorized, do the vectorization later for a speed up - IN: - events (list of tuples): events output from getEventHistory() - OUT: - eventString (string): string representing the event history of the binary - - MT strings look like: - P>S, P, < is RLOF (1->2 or 1<-2) or otherwise = for CEE or & for Merger - - SN strings look like: - P*SR for star1 the SN progenitor,or - R*SP for star2 the SN progenitor, - where P is progenitor type, R is remnant type, - S is state (I for intact, U for unbound) - - Event strings for the same seed are separated by the undesrcore character ('_') - """ - def _remap_stype(int_stype): - if useIntStypes: - return str(int_stype) - else: - return stellarTypeDict[int_stype] - - # Empty event - if len(events) == 0: - return 'NA' - - eventStr = '' - for event in events: - if event[0] == 'SN': # SN event - _, time, stypeP, stypeC, whichSN, isUnbound = event - if whichSN == 1: # Progenitor or Remnant depending upon which star is the SN - charL = _remap_stype(stypeP) - charR = _remap_stype(stypeC) - else: - charL = _remap_stype(stypeC) - charR = _remap_stype(stypeP) - charM = '*u' if isUnbound else '*i' # unbound or intact - else: # Any of the MT events - _, time, stype1, stype2, isRL1, isRL2, isCEE, isMrg = event - charL = _remap_stype(stype1) # primary stellar type - charR = _remap_stype(stype2) # secondary stellar type - charM = '&' if isMrg \ - else '=' if isCEE \ - else '<' if isRL2 \ - else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 - eventStr += "{}{}{}_".format(charL, charM, charR) # event string for this star, _ is event separator - - return np.array(eventStr[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) - -def getEventStrings(h5file=None, allEvents=None, useIntStypes=False): - """ - Function to calculate the event history strings for either the entire Compas output, or some list of events - IN: One of - h5file (h5.File() type): COMPAS HDF5 output file - allEvents (list of tuples) - OUT: - eventStrings (list): list of strings of the event history of each seed - """ - - # If output is - if (h5file == None) & (allEvents == None): - return - elif (allEvents == None): - _, allEvents = getEventHistory(h5file) - - eventStrings = np.zeros(len(allEvents), dtype=StringDType()) - for ii, eventsForGivenSeed in enumerate(allEvents): - eventString = buildEventString(eventsForGivenSeed, useIntStypes=useIntStypes) - eventStrings[ii] = eventString # append event string for this star (pop the last underscore first) - - return eventStrings - -"" - - -"" - diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py new file mode 100644 index 000000000..76498c96c --- /dev/null +++ b/compas_python_utils/debugging_utils.py @@ -0,0 +1,440 @@ +import h5py as h5 +import numpy as np +import pandas as pd +from numpy.dtypes import StringDType +from typing import NewType + +"" +# New Types +MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) +H5File = NewType('H5File', h5._hl.files.File) +H5Group = NewType('H5Group', h5._hl.group.Group) +MtEventTuple = NewType('MtEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool]]) +SnEventTuple = NewType('SnEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool]]) +EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) + + +######################################################################## +# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template + +def print_compas_details_dataframe(data: H5Group, + *seeds: int, + mask: MaskNdarray=()) -> pd.DataFrame: + """Return a pd.DataFrame for the COMPAS output contained in an H5Group. + + Parameters + ---------- + data : H5Group + an H5Group containing COMPAS data + seeds : int or array_like of ints, optional + seeds of specific systems to include (default is to include all seeds) + mask : array_like of booleans, optional + boolean mask to filter out specific systems (default is no mask) + Must be same length as data['SEED'] + + Returns + ------- + compas_details : pd.DataFrame + The values of each parameter in the H5Group, excluding those removed by `seeds` or `mask`. + + Notes + ----- + Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. + If both `seeds` and `mask` are supplied, resulting systems must pass both filters. + + Examples + -------- + >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF']) + >>> print_compas_details_dataframe(mt_data) + [output of all BSE_RLOF events] + + >>> mt_seeds = mt_data['SEED'][()] + >>> print_compas_details_dataframe(mt_data, mt_seeds[:50]) + [output of all BSE_RLOF events occuring in the first 50 seeds] + + >>> cee_events = mt_data['CEE>MT'][()] == 1 # needed to convert to boolean mask + >>> print_compas_details_dataframe(mt_data, mt_seeds[:50], mask=cee_events) + [output of all Common Envelope events occuring in the first 50 seeds] + """ + list_of_keys = list(data.keys()) + + # Check if SEED parameter exists in data + if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): + seed_variable_name='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + + # If `seeds` or `mask` arguments supplied, create the relevant mask + all_seeds = data[seed_variable_name][()] + seeds_mask = np.isin(all_seeds, seeds) + if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask + seeds_mask = np.ones_like(all_seeds).astype(bool) + if mask == (): + mask = np.ones_like(all_seeds).astype(bool) + mask &= seeds_mask + df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list_of_keys}).set_index(seed_variable_name).T + + else: # No seed parameter, currently only applies to the RunDetails H5Group + keys_not_derivations = [key for key in list_of_keys if '-Derivation' not in key] # Get just the keys without the -Derivation suffix + + # Some parameter values are string types, formatted as np.bytes_, need to convert back + def convert_strings(param_array): + if isinstance(param_array[0], np.bytes_): + return param_array.astype(str) + else: + return param_array + + df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T + nCols = df_keys.shape[1] # Required only because if we have combined RunDetails output from a previous h5copy, we get many columns (should fix later) + df_keys.columns = ['Parameter']*nCols + df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T + df_drvs.columns = ['Derivation']*nCols + df = pd.concat([df_keys, df_drvs], axis=1) + + # Add units as first col + units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) + return df + + + +######################################################################## +# ## Get event histories of MT data, SN data, and combined MT, SN data + +def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: + """Calculates the EventTuple for the BSE_RLOF output H5Group. + + Parameters + ---------- + mt_data : H5Group + the COMPAS output H5Group corresponding to BSE_RLOF + + Returns + ------- + returned_seeds : list + an ordered list of the unique seeds in the mt_data file, + returned_events : list + a list of sublists, one sublist per seed, where each sublist + contains all the MtEventTuples for the given seed + returned_times : list + a list of sublists of times of each of the mt_data events. + + Notes + ----- + MtEventTuples take the form: (stellar_type_primary, stellar_type_secondary, is_rlof1, is_rlof2, is_cee, is_mrg). + The events in the input do not have to be ordered chronologically, this function orders them, in the event + that the input was coallated from a previous h5copy command. The output is not meant to be used generally, + it is just a feeder for get_event_history below. + """ + mt_seeds = mt_data['SEED'][()] + + mt_times = mt_data['TimeMT'][()] == 1 + mt_is_rlof2 = mt_data['RLOF(2)>MT'][()] == 1 + mt_is_cee = mt_data['CEE>MT'][()] == 1 + mt_is_mrg = mt_data['Merger'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) # sort by seeds then times - lexsort sorts by the last column first... + mt_seeds = mt_seeds[mt_seeds_idx] + mt_times = mt_times[mt_seeds_idx] + mt_primary_stype = mt_primary_stype[mt_seeds_idx] + mt_secondary_stype = mt_secondary_stype[mt_seeds_idx] + mt_is_rlof1 = mt_is_rlof1[mt_seeds_idx] + mt_is_rlof2 = mt_is_rlof2[mt_seeds_idx] + mt_is_cee = mt_is_cee[mt_seeds_idx] + mt_is_mrg = mt_is_mrg[mt_seeds_idx] + + # Process the mt_data events + returned_seeds = [] # array of seeds - will only contain seeds that have mt_data events + returned_events = [] # array of mt_data events for each seed in returned_seeds + returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) + + last_seed = -1 # initialize most recently processed seed + for seed_index, this_seed in enumerate(mt_seeds): # iterate over all RLOF file entries + this_time = mt_times[seed_index] # time for this RLOF file entry + this_event = (mt_primary_stype[seed_index], mt_secondary_stype[seed_index], + mt_is_rlof1[seed_index], mt_is_rlof2[seed_index], mt_is_cee[seed_index], mt_is_mrg[seed_index]) # construct event tuple + + # If this is an entirely new seed: + if this_seed != last_seed: # same seed as last seed processed? + returned_seeds.append(this_seed) # no - new seed, record it + returned_times.append([this_time]) # initialize the list of event times for this seed + returned_events.append([this_event]) # initialize the list of events for this seed + last_seed = this_seed # update the latest seed + + # Add event, if it is not a duplicate + try: + event_index = returned_events[-1].index(this_event) # find event_index of this particular event tuple in the array of events for this seed + if this_time > returned_times[-1][event_index]: # ^ if event is not a duplicate, this will throw a ValueError + returned_times[-1][event_index] = this_time # if event is duplicate, update time to the later of the duplicates + except ValueError: # event is not a duplicate: + returned_events[-1].append(this_event) # record new event tuple for this seed + returned_times[-1].append(this_time) # record new event time for this seed + + return returned_seeds, returned_events, returned_times # see above for description + + +def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: + """Calculates the EventTuple for the BSE_Supernovae output H5Group. + + Parameters + ---------- + sn_data : H5Group + the COMPAS output H5Group corresponding to BSE_Supernovae + + Returns + ------- + returned_seeds : list + an ordered list of the unique seeds in the sn_data file, + returned_events : list + a list of sublists, one sublist per seed, where each sublist + contains all the SnEventTuples for the given seed + returned_times : list + a list of sublists of times of each of the sn_data events. + + Notes + ----- + SnEventTuples take the form: (stellar_type_progenitor, stellar_type_remnant, which_starIsProgenitor, is_binary_unbound). + The events in the input do not have to be ordered chronologically, this function orders them, in the event + that the input was coallated from a previous h5copy command. The output is not meant to be used generally, + it is just a feeder for get_event_history below. + """ + sn_seeds = sn_data['SEED'][()] + sn_times = sn_data['Time'][()] + sn_prog_stype = sn_data['Stellar_Type_Prev(SN)'][()] + sn_remn_stype = sn_data['Stellar_Type(SN)'][()] + sn_which_prog = sn_data['Supernova_State'][()] + sn_is_unbound = sn_data['Unbound'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) # sort by seeds then times - lexsort sorts by the last column first... + sn_seeds = sn_seeds[sn_seeds_idx] + sn_times = sn_times[sn_seeds_idx] + sn_prog_stype = sn_prog_stype[sn_seeds_idx] + sn_remn_stype = sn_remn_stype[sn_seeds_idx] + sn_which_prog = sn_which_prog[sn_seeds_idx] + sn_is_unbound = sn_is_unbound[sn_seeds_idx] + + # Process the sn_data events + returned_seeds = [] # array of seeds - will only contain seeds that have sn_data events + returned_events = [] # array of sn_data events for each seed in returned_seeds + returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) + + last_seed = -1 # initialize most recently processed seed + for seed_index, this_seed in enumerate(sn_seeds): # iterate over all sn_data file entries + this_time = sn_times[seed_index] # time for this sn_data file entry + this_event = (sn_prog_stype[seed_index], sn_remn_stype[seed_index], + sn_which_prog[seed_index], sn_is_unbound[seed_index]) # construct event tuple + + # If this is an entirely new seed: + if this_seed != last_seed: # same seed as last seed processed? + returned_seeds.append(this_seed) # no - new seed, record it + returned_times.append([this_time]) # initialize the list of event times for this seed + returned_events.append([this_event]) # initialize the list of events for this seed + last_seed = this_seed # update the latest seed + else: # yes - second sn_data event for this seed + returned_times[-1].append(this_time) # append time at end of array + returned_events[-1].append(this_event) # append event at end of array + + return returned_seeds, returned_events, returned_times # see above for description + + +def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list]: + """Get the event history for all seeds, including both MT and SN events, in chronological order. + + Parameters + ---------- + data : H5File + the COMPAS output H5file + include_null : bool + whether to include seeds which undergo no MT or SN events (default is True) + + Returns + ------- + returned_seeds : list + an ordered list of all the unique seeds in the output file + returned_events : list + a list where each element is a chronological set of events corresponding to the associated seed + """ + sp_data = data['BSE_System_Parameters'] + mt_data = data['BSE_RLOF'] + sn_data = data['BSE_Supernovae'] + all_seeds = sp_data['SEED'][()] # get all seeds + mt_seeds, mt_events, mt_times = get_mt_data_tuple(mt_data) # get mt data tuple + sn_seeds, sn_events, sn_times = get_sn_data_tuple(sn_data) # get sn data tuple + + num_mt_seeds = len(mt_seeds) # number of mt_data events + num_sn_seeds = len(sn_seeds) # number of sn_data events + + if num_mt_seeds < 1 and num_sn_seeds < 1: return [] # no events - return empty history + + returned_seeds = [] # array of seeds - will only contain seeds that have events (of any type) + returned_events = [] # array of events - same size as returned_seeds (includes event times) + + event_ordering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events + + mt_index = 0 # index into mt_data events arrays + sn_index = 0 # index into sn_data events arrays + + if include_null: + seeds_to_iterate = all_seeds + else: + seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) # iterate over all the seeds that have either mt_data or sn_data events + + idx_ordered = np.argsort(seeds_to_iterate) + returned_seeds = [None] * np.size(seeds_to_iterate) # array of seeds - will only contain seeds that have events (of any type) + returned_events = [None] * np.size(seeds_to_iterate) # array of events - same size as returned_seeds (includes event times) + + for idx in idx_ordered: + seed = seeds_to_iterate[idx] + seed_events = [] # initialise the events for the seed being processed + + # Collect any mt_data events for this seed, add the time of the event and the event type + while mt_index < num_mt_seeds and mt_seeds[mt_index] == seed: + for event_index, event in enumerate(mt_events[mt_index]): + _, _, is_rl1, is_rl2, is_cee, is_mrg = event + event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 + seed_events.append((event_key, mt_times[mt_index][event_index], *mt_events[mt_index][event_index])) + mt_index += 1 + + # Collect any sn_data events for this seed, add the time of the event and the event type + while sn_index < num_sn_seeds and sn_seeds[sn_index] == seed: + for event_index, event in enumerate(sn_events[sn_index]): + event_key = 'SN' + seed_events.append((event_key, sn_times[sn_index][event_index], *sn_events[sn_index][event_index])) + sn_index += 1 + + seed_events.sort(key=lambda ev:(ev[1], event_ordering.index(ev[0]))) # sort the events by time and event type (mt_data before sn_data if at the same time) + returned_seeds[idx] = seed # record the seed in the seeds array being returned + returned_events[idx] = seed_events # record the events for this seed in the events array being returned + return returned_seeds, returned_events + + +########################################### +# ## Produce strings of the event histories + +stellar_type_dict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', + 10: 'WD', + 11: 'WD', + 12: 'WD', + 13: 'NS', + 14: 'BH', + 15: 'MR', + 16: 'MS', +} + + +def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistoryString: + """Produce a string representation of the event history of a single binary. + + Parameters + ---------- + events : list + list of mixed MtEventTuple and SnEventTuple's + use_int_stypes : bool, optional + whether to report stellar types as integers + (default is False, report them as 2-char stellar types) + + Returns + ------- + event_str : EventHistoryString + condensed string representation of the event history of a binary, in array form + + Notes + ----- + MT strings look like: + P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) + or otherwise '=' for CEE or '&' for Merger. + SN strings look like: + P*SR if star1 is the SN progenitor, or + R*SP if star2 is the SN progenitor, + where P is progenitor type, R is remnant type, + S is state ('i' for intact, 'u' for unbound) + Event strings for the same seed are separated by an undesrcore '_' + + TODO + ---- + Currently unvectorized, do the vectorization later for a speed up + """ + def _remap_stype(int_stype): + if use_int_stypes: + return str(int_stype) + else: + return stellar_type_dict[int_stype] + + # Empty event + if len(events) == 0: + return 'NA' + + event_str = '' + for event in events: + if event[0] == 'SN': # SN event + _, time, stype_p, stype_c, which_sn, is_unbound = event + if which_sn == 1: # Progenitor or Remnant depending upon which star is the SN + char_l = _remap_stype(stype_p) + char_r = _remap_stype(stype_c) + else: + char_l = _remap_stype(stype_c) + char_r = _remap_stype(stype_p) + char_m = '*u' if is_unbound else '*i' # unbound or intact + else: # Any of the MT events + _, time, stype_1, stype_2, is_rl1, is_rl2, is_cee, is_mrg = event + char_l = _remap_stype(stype_1) # primary stellar type + char_r = _remap_stype(stype_2) # secondary stellar type + char_m = '&' if is_mrg \ + else '=' if is_cee \ + else '<' if is_rl2 \ + else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 + event_str += "{}{}{}_".format(char_l, char_m, char_r) # event string for this star, _ is event separator + event_str = np.array(event_str[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + return event_str + + +def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: bool=False): + """Calculate the event history strings for a COMPAS population. + + Parameters + ---------- + data : H5File, optional + the COMPAS output H5file + all_events : list, optional + a list where each element is a chronological set of events corresponding to the associated seed, + one of the outputs of get_event_history(data) + use_int_stypes : bool, optional + + Returns + ------- + event_strings : array of EventHistoryString's + strings of the event history of each system + + Notes + ----- + Exactly one of data or all_events must be included. If neither, nothign is returned. + """ + + # If output is + if (data == None) & (all_events == None): + return + elif (all_events == None): + _, all_events = get_event_history(data) + + event_strings = np.zeros(len(all_events), dtype=StringDType()) + for ii, events_for_given_seed in enumerate(all_events): + event_string = build_event_string(events_for_given_seed, use_int_stypes=use_int_stypes) + event_strings[ii] = event_string # append event string for this star (pop the last underscore first) + return event_strings + +"" + From 11caeb8785a4091af80003e6290caa1607a4ad53 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 29 Jun 2026 12:05:36 +0200 Subject: [PATCH 14/22] addressed nicolas' concerns --- compas_python_utils/debugging_utils.py | 436 +++++++++++++++---------- 1 file changed, 272 insertions(+), 164 deletions(-) diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py index 76498c96c..15a9826a2 100644 --- a/compas_python_utils/debugging_utils.py +++ b/compas_python_utils/debugging_utils.py @@ -9,17 +9,46 @@ MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) H5File = NewType('H5File', h5._hl.files.File) H5Group = NewType('H5Group', h5._hl.group.Group) -MtEventTuple = NewType('MtEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool]]) -SnEventTuple = NewType('SnEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool]]) +MtEventTuple = NewType('MtEventTuple', + tuple[np.ndarray[int], + np.ndarray[int], + np.ndarray[bool], + np.ndarray[bool], + np.ndarray[bool], + np.ndarray[bool]]) +SnEventTuple = NewType('SnEventTuple', + tuple[np.ndarray[int], + np.ndarray[int], + np.ndarray[bool], + np.ndarray[bool]]) EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) ######################################################################## # ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template -def print_compas_details_dataframe(data: H5Group, - *seeds: int, - mask: MaskNdarray=()) -> pd.DataFrame: +def convert_bytes_array_to_strings(param_array): + """Check and convert np.bytes_ array to strings. + + Parameters + ---------- + param_array : np.ndarray[np.bytes_ | str] + a numpy array containing either np.bytes_ or strings + + Returns + ------- + param_array : np.ndarray[str] + the same array, but converted to strings if not already + """ + if np.issubdtype(param_array.dtype, np.bytes_): + return param_array.astype(str) + else: + return param_array + + +def print_compas_details_dataframe(data: H5Group, + *seeds: int, + mask: MaskNdarray = ()) -> pd.DataFrame: """Return a pd.DataFrame for the COMPAS output contained in an H5Group. Parameters @@ -32,70 +61,69 @@ def print_compas_details_dataframe(data: H5Group, boolean mask to filter out specific systems (default is no mask) Must be same length as data['SEED'] - Returns + Returns ------- compas_details : pd.DataFrame The values of each parameter in the H5Group, excluding those removed by `seeds` or `mask`. Notes ----- - Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. + Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. If both `seeds` and `mask` are supplied, resulting systems must pass both filters. Examples -------- - >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF']) + >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF'] >>> print_compas_details_dataframe(mt_data) [output of all BSE_RLOF events] - + >>> mt_seeds = mt_data['SEED'][()] >>> print_compas_details_dataframe(mt_data, mt_seeds[:50]) [output of all BSE_RLOF events occuring in the first 50 seeds] - + >>> cee_events = mt_data['CEE>MT'][()] == 1 # needed to convert to boolean mask >>> print_compas_details_dataframe(mt_data, mt_seeds[:50], mask=cee_events) [output of all Common Envelope events occuring in the first 50 seeds] """ - list_of_keys = list(data.keys()) - # Check if SEED parameter exists in data - if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): - seed_variable_name='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + if ('SEED' or 'SEED>MT') in data: + # SEED>MT is a relic from older versions, but we leave this in for + # backwards compatibility + seed_variable_name = 'SEED' if ('SEED' in data) else 'SEED>MT' # If `seeds` or `mask` arguments supplied, create the relevant mask all_seeds = data[seed_variable_name][()] seeds_mask = np.isin(all_seeds, seeds) - if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask + if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask seeds_mask = np.ones_like(all_seeds).astype(bool) if mask == (): mask = np.ones_like(all_seeds).astype(bool) mask &= seeds_mask - df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list_of_keys}).set_index(seed_variable_name).T - - else: # No seed parameter, currently only applies to the RunDetails H5Group - keys_not_derivations = [key for key in list_of_keys if '-Derivation' not in key] # Get just the keys without the -Derivation suffix - - # Some parameter values are string types, formatted as np.bytes_, need to convert back - def convert_strings(param_array): - if isinstance(param_array[0], np.bytes_): - return param_array.astype(str) - else: - return param_array - - df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T - nCols = df_keys.shape[1] # Required only because if we have combined RunDetails output from a previous h5copy, we get many columns (should fix later) - df_keys.columns = ['Parameter']*nCols - df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T - df_drvs.columns = ['Derivation']*nCols + df = pd.DataFrame.from_dict( + {param: data[param][()][mask] for param in data}).set_index(seed_variable_name).T + + else: # No seed parameter, currently only applies to the RunDetails H5Group + # Get just the keys without the -Derivation suffix + keys_not_derivations = [ + key for key in data if '-Derivation' not in key] + + df_keys = pd.DataFrame.from_dict({param: convert_bytes_array_to_strings( + data[param][()]) for param in keys_not_derivations}).T + # Required only because if we have combined RunDetails output from a + # previous h5copy, we get many columns (should fix later) + nCols = df_keys.shape[1] + df_keys.columns = ['Parameter'] * nCols + df_drvs = pd.DataFrame.from_dict({param: convert_bytes_array_to_strings( + data[param + '-Derivation'][()]) for param in keys_not_derivations}).T + df_drvs.columns = ['Derivation'] * nCols df = pd.concat([df_keys, df_drvs], axis=1) # Add units as first col - units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + units_dict = {key: data[key].attrs['units'].astype(str) for key in data} df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) return df - ######################################################################## # ## Get event histories of MT data, SN data, and combined MT, SN data @@ -109,10 +137,10 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: Returns ------- - returned_seeds : list - an ordered list of the unique seeds in the mt_data file, + returned_seeds : list + an ordered list of the unique seeds in the mt_data file returned_events : list - a list of sublists, one sublist per seed, where each sublist + a list of sublists, one sublist per seed, where each sublist contains all the MtEventTuples for the given seed returned_times : list a list of sublists of times of each of the mt_data events. @@ -135,9 +163,10 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: mt_is_mrg = mt_data['Merger'][()] == 1 # We want the return arrays sorted by seed, so sort here. - mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) # sort by seeds then times - lexsort sorts by the last column first... - mt_seeds = mt_seeds[mt_seeds_idx] - mt_times = mt_times[mt_seeds_idx] + # sort by seeds then times - lexsort sorts by the last column first... + mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) + mt_seeds = mt_seeds[mt_seeds_idx] + mt_times = mt_times[mt_seeds_idx] mt_primary_stype = mt_primary_stype[mt_seeds_idx] mt_secondary_stype = mt_secondary_stype[mt_seeds_idx] mt_is_rlof1 = mt_is_rlof1[mt_seeds_idx] @@ -146,33 +175,56 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: mt_is_mrg = mt_is_mrg[mt_seeds_idx] # Process the mt_data events - returned_seeds = [] # array of seeds - will only contain seeds that have mt_data events - returned_events = [] # array of mt_data events for each seed in returned_seeds - returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) - - last_seed = -1 # initialize most recently processed seed - for seed_index, this_seed in enumerate(mt_seeds): # iterate over all RLOF file entries - this_time = mt_times[seed_index] # time for this RLOF file entry - this_event = (mt_primary_stype[seed_index], mt_secondary_stype[seed_index], - mt_is_rlof1[seed_index], mt_is_rlof2[seed_index], mt_is_cee[seed_index], mt_is_mrg[seed_index]) # construct event tuple + # array of seeds - will only contain seeds that have mt_data events + returned_seeds = [] + # array of mt_data events for each seed in returned_seeds + returned_events = [] + # array of times for each event in returned_events (for each seed in + # returned_seeds) + returned_times = [] + + # initialize most recently processed seed + last_seed = -1 + for seed_index, this_seed in enumerate( + mt_seeds): # iterate over all RLOF file entries + # time for this RLOF file entry + this_time = mt_times[seed_index] + this_event = ( + mt_primary_stype[seed_index], + mt_secondary_stype[seed_index], + mt_is_rlof1[seed_index], + mt_is_rlof2[seed_index], + mt_is_cee[seed_index], + mt_is_mrg[seed_index]) # construct event tuple # If this is an entirely new seed: if this_seed != last_seed: # same seed as last seed processed? - returned_seeds.append(this_seed) # no - new seed, record it - returned_times.append([this_time]) # initialize the list of event times for this seed - returned_events.append([this_event]) # initialize the list of events for this seed + # no - new seed, record it + returned_seeds.append(this_seed) + # initialize the list of event times for this seed + returned_times.append([this_time]) + # initialize the list of events for this seed + returned_events.append([this_event]) last_seed = this_seed # update the latest seed # Add event, if it is not a duplicate try: - event_index = returned_events[-1].index(this_event) # find event_index of this particular event tuple in the array of events for this seed - if this_time > returned_times[-1][event_index]: # ^ if event is not a duplicate, this will throw a ValueError - returned_times[-1][event_index] = this_time # if event is duplicate, update time to the later of the duplicates + # find event_index of this particular event tuple in the array of + # events for this seed + event_index = returned_events[-1].index(this_event) + # ^ if event is not a duplicate, this will throw a ValueError + if this_time > returned_times[-1][event_index]: + # if event is duplicate, update time to the later of the + # duplicates + returned_times[-1][event_index] = this_time except ValueError: # event is not a duplicate: - returned_events[-1].append(this_event) # record new event tuple for this seed - returned_times[-1].append(this_time) # record new event time for this seed + # record new event tuple for this seed + returned_events[-1].append(this_event) + # record new event time for this seed + returned_times[-1].append(this_time) - return returned_seeds, returned_events, returned_times # see above for description + # see above for description + return returned_seeds, returned_events, returned_times def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: @@ -185,10 +237,10 @@ def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: Returns ------- - returned_seeds : list - an ordered list of the unique seeds in the sn_data file, + returned_seeds : list + an ordered list of the unique seeds in the sn_data file, returned_events : list - a list of sublists, one sublist per seed, where each sublist + a list of sublists, one sublist per seed, where each sublist contains all the SnEventTuples for the given seed returned_times : list a list of sublists of times of each of the sn_data events. @@ -208,123 +260,168 @@ def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: sn_is_unbound = sn_data['Unbound'][()] == 1 # We want the return arrays sorted by seed, so sort here. - sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) # sort by seeds then times - lexsort sorts by the last column first... - sn_seeds = sn_seeds[sn_seeds_idx] - sn_times = sn_times[sn_seeds_idx] + # sort by seeds then times - lexsort sorts by the last column first... + sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) + sn_seeds = sn_seeds[sn_seeds_idx] + sn_times = sn_times[sn_seeds_idx] sn_prog_stype = sn_prog_stype[sn_seeds_idx] sn_remn_stype = sn_remn_stype[sn_seeds_idx] sn_which_prog = sn_which_prog[sn_seeds_idx] sn_is_unbound = sn_is_unbound[sn_seeds_idx] - + # Process the sn_data events - returned_seeds = [] # array of seeds - will only contain seeds that have sn_data events - returned_events = [] # array of sn_data events for each seed in returned_seeds - returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) - - last_seed = -1 # initialize most recently processed seed - for seed_index, this_seed in enumerate(sn_seeds): # iterate over all sn_data file entries - this_time = sn_times[seed_index] # time for this sn_data file entry - this_event = (sn_prog_stype[seed_index], sn_remn_stype[seed_index], - sn_which_prog[seed_index], sn_is_unbound[seed_index]) # construct event tuple - + # array of seeds - will only contain seeds that have sn_data events + returned_seeds = [] + # array of sn_data events for each seed in returned_seeds + returned_events = [] + # array of times for each event in returned_events (for each seed in + # returned_seeds) + returned_times = [] + + # initialize most recently processed seed + last_seed = -1 + for seed_index, this_seed in enumerate( + sn_seeds): # iterate over all sn_data file entries + # time for this sn_data file entry + this_time = sn_times[seed_index] + this_event = ( + sn_prog_stype[seed_index], + sn_remn_stype[seed_index], + sn_which_prog[seed_index], + sn_is_unbound[seed_index]) # construct event tuple + # If this is an entirely new seed: if this_seed != last_seed: # same seed as last seed processed? - returned_seeds.append(this_seed) # no - new seed, record it - returned_times.append([this_time]) # initialize the list of event times for this seed - returned_events.append([this_event]) # initialize the list of events for this seed - last_seed = this_seed # update the latest seed + # no - new seed, record it + returned_seeds.append(this_seed) + # initialize the list of event times for this seed + returned_times.append([this_time]) + # initialize the list of events for this seed + returned_events.append([this_event]) + last_seed = this_seed # update the latest seed else: # yes - second sn_data event for this seed - returned_times[-1].append(this_time) # append time at end of array - returned_events[-1].append(this_event) # append event at end of array - - return returned_seeds, returned_events, returned_times # see above for description + returned_times[-1].append(this_time) # append time at end of array + # append event at end of array + returned_events[-1].append(this_event) + # see above for description + return returned_seeds, returned_events, returned_times -def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list]: + +def get_event_history( + data: H5File, include_null: bool = True) -> tuple[list, list]: """Get the event history for all seeds, including both MT and SN events, in chronological order. Parameters ---------- data : H5File - the COMPAS output H5file + the COMPAS output H5file include_null : bool whether to include seeds which undergo no MT or SN events (default is True) - Returns + Returns ------- returned_seeds : list an ordered list of all the unique seeds in the output file - returned_events : list + returned_events : list a list where each element is a chronological set of events corresponding to the associated seed """ sp_data = data['BSE_System_Parameters'] mt_data = data['BSE_RLOF'] sn_data = data['BSE_Supernovae'] - all_seeds = sp_data['SEED'][()] # get all seeds - mt_seeds, mt_events, mt_times = get_mt_data_tuple(mt_data) # get mt data tuple - sn_seeds, sn_events, sn_times = get_sn_data_tuple(sn_data) # get sn data tuple - - num_mt_seeds = len(mt_seeds) # number of mt_data events - num_sn_seeds = len(sn_seeds) # number of sn_data events - - if num_mt_seeds < 1 and num_sn_seeds < 1: return [] # no events - return empty history - - returned_seeds = [] # array of seeds - will only contain seeds that have events (of any type) - returned_events = [] # array of events - same size as returned_seeds (includes event times) - - event_ordering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events - - mt_index = 0 # index into mt_data events arrays - sn_index = 0 # index into sn_data events arrays + # get all seeds + all_seeds = sp_data['SEED'][()] + mt_seeds, mt_events, mt_times = get_mt_data_tuple( + mt_data) # get mt data tuple + sn_seeds, sn_events, sn_times = get_sn_data_tuple( + sn_data) # get sn data tuple + + # number of mt_data events + num_mt_seeds = len(mt_seeds) + # number of sn_data events + num_sn_seeds = len(sn_seeds) + + if num_mt_seeds < 1 and num_sn_seeds < 1: + return [] # no events - return empty history + + # array of seeds - will only contain seeds that have events (of any type) + returned_seeds = [] + # array of events - same size as returned_seeds (includes event times) + returned_events = [] + + # order of preference for simultaneous events + event_ordering = ['RL', 'CE', 'SN', 'MG'] + + # index into mt_data events arrays + mt_index = 0 + # index into sn_data events arrays + sn_index = 0 if include_null: seeds_to_iterate = all_seeds else: - seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) # iterate over all the seeds that have either mt_data or sn_data events - + # iterate over all the seeds that have either mt_data or sn_data events + seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) + idx_ordered = np.argsort(seeds_to_iterate) - returned_seeds = [None] * np.size(seeds_to_iterate) # array of seeds - will only contain seeds that have events (of any type) - returned_events = [None] * np.size(seeds_to_iterate) # array of events - same size as returned_seeds (includes event times) + # array of seeds - will only contain seeds that have events (of any type) + returned_seeds = [None] * np.size(seeds_to_iterate) + # array of events - same size as returned_seeds (includes event times) + returned_events = [None] * np.size(seeds_to_iterate) for idx in idx_ordered: seed = seeds_to_iterate[idx] - seed_events = [] # initialise the events for the seed being processed + # initialise the events for the seed being processed + seed_events = [] - # Collect any mt_data events for this seed, add the time of the event and the event type + # Collect any mt_data events for this seed, add the time of the event + # and the event type while mt_index < num_mt_seeds and mt_seeds[mt_index] == seed: for event_index, event in enumerate(mt_events[mt_index]): _, _, is_rl1, is_rl2, is_cee, is_mrg = event - event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 - seed_events.append((event_key, mt_times[mt_index][event_index], *mt_events[mt_index][event_index])) + # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 + event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' + seed_events.append( + (event_key, + mt_times[mt_index][event_index], + *mt_events[mt_index][event_index])) mt_index += 1 - # Collect any sn_data events for this seed, add the time of the event and the event type + # Collect any sn_data events for this seed, add the time of the event + # and the event type while sn_index < num_sn_seeds and sn_seeds[sn_index] == seed: for event_index, event in enumerate(sn_events[sn_index]): - event_key = 'SN' - seed_events.append((event_key, sn_times[sn_index][event_index], *sn_events[sn_index][event_index])) + event_key = 'SN' + seed_events.append( + (event_key, + sn_times[sn_index][event_index], + *sn_events[sn_index][event_index])) sn_index += 1 - seed_events.sort(key=lambda ev:(ev[1], event_ordering.index(ev[0]))) # sort the events by time and event type (mt_data before sn_data if at the same time) - returned_seeds[idx] = seed # record the seed in the seeds array being returned - returned_events[idx] = seed_events # record the events for this seed in the events array being returned - return returned_seeds, returned_events + # sort the events by time and event type (mt_data before sn_data if at + # the same time) + seed_events.sort(key=lambda ev: (ev[1], event_ordering.index(ev[0]))) + # record the seed in the seeds array being returned + returned_seeds[idx] = seed + # record the events for this seed in the events array being returned + returned_events[idx] = seed_events + return returned_seeds, returned_events ########################################### # ## Produce strings of the event histories -stellar_type_dict = { - 0: 'MS', - 1: 'MS', - 2: 'HG', - 3: 'GB', - 4: 'GB', - 5: 'GB', - 6: 'GB', - 7: 'HE', - 8: 'HE', - 9: 'HE', +simplified_stellar_type_dict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', 10: 'WD', 11: 'WD', 12: 'WD', @@ -335,7 +432,9 @@ def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list } -def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistoryString: +def build_event_string( + events: list, + use_int_stypes: bool = False) -> EventHistoryString: """Produce a string representation of the event history of a single binary. Parameters @@ -343,27 +442,27 @@ def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistory events : list list of mixed MtEventTuple and SnEventTuple's use_int_stypes : bool, optional - whether to report stellar types as integers + whether to report stellar types as integers (default is False, report them as 2-char stellar types) - Returns + Returns ------- event_str : EventHistoryString condensed string representation of the event history of a binary, in array form Notes ----- - MT strings look like: - P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) + MT strings look like: + P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) or otherwise '=' for CEE or '&' for Merger. SN strings look like: - P*SR if star1 is the SN progenitor, or + P*SR if star1 is the SN progenitor, or R*SP if star2 is the SN progenitor, - where P is progenitor type, R is remnant type, + where P is progenitor type, R is remnant type, S is state ('i' for intact, 'u' for unbound) Event strings for the same seed are separated by an undesrcore '_' - + TODO ---- Currently unvectorized, do the vectorization later for a speed up @@ -372,44 +471,50 @@ def _remap_stype(int_stype): if use_int_stypes: return str(int_stype) else: - return stellar_type_dict[int_stype] - + return simplified_stellar_type_dict[int_stype] + # Empty event if len(events) == 0: return 'NA' - + event_str = '' for event in events: if event[0] == 'SN': # SN event _, time, stype_p, stype_c, which_sn, is_unbound = event - if which_sn == 1: # Progenitor or Remnant depending upon which star is the SN + # Progenitor or Remnant depending upon which star is the SN + if which_sn == 1: char_l = _remap_stype(stype_p) char_r = _remap_stype(stype_c) else: char_l = _remap_stype(stype_c) char_r = _remap_stype(stype_p) char_m = '*u' if is_unbound else '*i' # unbound or intact - else: # Any of the MT events + else: # Any of the MT events _, time, stype_1, stype_2, is_rl1, is_rl2, is_cee, is_mrg = event - char_l = _remap_stype(stype_1) # primary stellar type - char_r = _remap_stype(stype_2) # secondary stellar type - char_m = '&' if is_mrg \ - else '=' if is_cee \ - else '<' if is_rl2 \ - else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 - event_str += "{}{}{}_".format(char_l, char_m, char_r) # event string for this star, _ is event separator - event_str = np.array(event_str[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + # primary stellar type + char_l = _remap_stype(stype_1) + # secondary stellar type + char_r = _remap_stype(stype_2) + # event type: CEE, RLOF 2->1, RLOF 1->2 + char_m = '&' if is_mrg else '=' if is_cee else '<' if is_rl2 else '>' + # event string for this star, _ is event separator + event_str += "{}{}{}_".format(char_l, char_m, char_r) + # return event string for this star (pop the last underscore first) + event_str = np.array(event_str[:-1], dtype=np.str_) return event_str -def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: bool=False): - """Calculate the event history strings for a COMPAS population. +def get_event_strings( + data: H5File = None, + all_events: list = None, + use_int_stypes: bool = False): + """Calculate the event history strings for a COMPAS population. Parameters ---------- data : H5File, optional - the COMPAS output H5file - all_events : list, optional + the COMPAS output H5file + all_events : list, optional a list where each element is a chronological set of events corresponding to the associated seed, one of the outputs of get_event_history(data) use_int_stypes : bool, optional @@ -421,20 +526,23 @@ def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: Notes ----- - Exactly one of data or all_events must be included. If neither, nothign is returned. + Exactly one of data or all_events must be included. If neither, nothing is returned. """ - # If output is - if (data == None) & (all_events == None): - return - elif (all_events == None): + # If output is + if (data is None) & (all_events is None): + return + elif (all_events is None): _, all_events = get_event_history(data) event_strings = np.zeros(len(all_events), dtype=StringDType()) - for ii, events_for_given_seed in enumerate(all_events): - event_string = build_event_string(events_for_given_seed, use_int_stypes=use_int_stypes) - event_strings[ii] = event_string # append event string for this star (pop the last underscore first) + for ii, events_for_given_seed in enumerate(all_events): + event_string = build_event_string( + events_for_given_seed, + use_int_stypes=use_int_stypes) + # append event string for this star (pop the last underscore first) + event_strings[ii] = event_string return event_strings -"" +"" From ec95efcbb19aa89494842e72f72a17d450c1b669 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 29 Jun 2026 12:13:42 +0200 Subject: [PATCH 15/22] starting the process of adding unit testing --- compas_python_utils/debugging_utils.py | 14 +++++++------- setup.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py index 15a9826a2..4047d10f8 100644 --- a/compas_python_utils/debugging_utils.py +++ b/compas_python_utils/debugging_utils.py @@ -4,8 +4,7 @@ from numpy.dtypes import StringDType from typing import NewType -"" -# New Types +### New Types MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) H5File = NewType('H5File', h5._hl.files.File) H5Group = NewType('H5Group', h5._hl.group.Group) @@ -24,8 +23,7 @@ EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) -######################################################################## -# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template +### Function to print the data from a given COMPAS HDF5 group in a readable pandas template def convert_bytes_array_to_strings(param_array): """Check and convert np.bytes_ array to strings. @@ -124,8 +122,7 @@ def print_compas_details_dataframe(data: H5Group, return df -######################################################################## -# ## Get event histories of MT data, SN data, and combined MT, SN data +### Get event histories of MT data, SN data, and combined MT, SN data def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: """Calculates the EventTuple for the BSE_RLOF output H5Group. @@ -544,5 +541,8 @@ def get_event_strings( event_strings[ii] = event_string return event_strings +def main(): + return -"" +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 8cf5abe20..29eef5c43 100644 --- a/setup.py +++ b/setup.py @@ -128,6 +128,7 @@ def find_version(version_file=read(CPP_VERSION_FILE)): f"compas_h5view= {NAME}.h5view:main", f"compas_h5copy= {NAME}.h5copy:main", f"compas_h5sample= {NAME}.h5sample:main", + f"compas_h5sample= {NAME}.debugging_utils:main", f"compas_plot_detailed_evolution={NAME}.detailed_evolution_plotter.plot_detailed_evolution:main", f"compas_run_submit={NAME}.preprocessing.runSubmit:main", f"compas_sample_stroopwafel={NAME}.preprocessing.stroopwafelInterface:main", From 164791fa7270cae5bfdbf4b9e7aceb4719e518db Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Wed, 18 Mar 2026 14:14:00 +0100 Subject: [PATCH 16/22] added compasUtils file --- compas_python_utils/compasUtils.py | 355 +++++++++++++++++++++++++++++ 1 file changed, 355 insertions(+) create mode 100644 compas_python_utils/compasUtils.py diff --git a/compas_python_utils/compasUtils.py b/compas_python_utils/compasUtils.py new file mode 100644 index 000000000..2cc126c68 --- /dev/null +++ b/compas_python_utils/compasUtils.py @@ -0,0 +1,355 @@ +import h5py as h5 +import numpy as np +import pandas as pd +from numpy.dtypes import StringDType + + +######################################################################## +# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template + +def printCompasDetails(data, *seeds, mask=()): + """ + Function to print the full Compas output for given seeds, optionally with an additional mask + """ + list_of_keys = list(data.keys()) + + # Check if seed parameter exists - if not, just print without (e.g RunDetails) + if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): # Most output files + #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + + # Set the seed name parameter, mask on seeds as needed, and set the index + seedVariableName='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' + list_of_keys.remove(seedVariableName) # this is the index above, don't want to include it + + allSeeds = data[seedVariableName][()] + seedsMask = np.isin(allSeeds, seeds) + if len(seeds) == 0: # if any seed is included, do not reset the mask + seedsMask = np.ones_like(allSeeds).astype(bool) + if mask == (): + mask = np.ones_like(allSeeds).astype(bool) + mask &= seedsMask + + df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list(data.keys())}).set_index(seedVariableName).T + + else: # No seed parameter, so do custom print for Run Details + + # Get just the keys without the -Derivation suffix - those will be a second column + keys_not_derivations = [] + for key in list_of_keys: + if '-Derivation' not in key: + keys_not_derivations.append(key) + + # Some parameter values are string types, formatted as np.bytes_, need to convert back + def convert_strings(param_array): + if isinstance(param_array[0], np.bytes_): + return param_array.astype(str) + else: + return param_array + + df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T + nCols = df_keys.shape[1] # Required only because if we combine RDs, we get many columns (should fix later) + df_keys.columns = ['Parameter']*nCols + df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T + df_drvs.columns = ['Derivation']*nCols + df = pd.concat([df_keys, df_drvs], axis=1) + + # Add units as first col + units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) + return df + + + +######################################################################## +# ## Get event histories of MT data, SN data, and combined MT, SN data + +def getMtEvents(MT): + """ + This function takes in the `BSE_RLOF` output category from COMPAS, and returns the information + on the Mass Transfer (MT) events that happen for each seed. The events do not have to be in order, + either chronologically or by seed, this function will reorder them as required. + + OUT: + returnedSeeds (list): ordered list of the unique seeds in the MT file + returnedEvents (list): list of sublists, where each sublist contains all the MT events for a given seed. + MT event tuples take the form : + (stellarTypePrimary, stellarTypeSecondary, isRlof1, isRlof2, isCEE, isMrg) + returnTimes (list): is a list of sublists of times of each of the MT events + """ + mtSeeds = MT['SEED'][()] + mtTimes = MT['TimeMT'][()] == 1 + mtIsRlof2 = MT['RLOF(2)>MT'][()] == 1 + mtIsCEE = MT['CEE>MT'][()] == 1 + mtIsMrg = MT['Merger'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + mtSeedsInds = np.lexsort((mtTimes, mtSeeds)) # sort by seeds then times - lexsort sorts by the last column first... + mtSeeds = mtSeeds[mtSeedsInds] + mtTimes = mtTimes[mtSeedsInds] + mtPrimaryStype = mtPrimaryStype[mtSeedsInds] + mtSecondaryStype = mtSecondaryStype[mtSeedsInds] + mtIsRlof1 = mtIsRlof1[mtSeedsInds] + mtIsRlof2 = mtIsRlof2[mtSeedsInds] + mtIsCEE = mtIsCEE[mtSeedsInds] + mtIsMrg = mtIsMrg[mtSeedsInds] + + # Process the MT events + returnedSeeds = [] # array of seeds - will only contain seeds that have MT events + returnedEvents = [] # array of MT events for each seed in returnedSeeds + returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) + + lastSeed = -1 # initialize most recently processed seed + for seedIndex, thisSeed in enumerate(mtSeeds): # iterate over all RLOF file entries + thisTime = mtTimes[seedIndex] # time for this RLOF file entry + thisEvent = (mtPrimaryStype[seedIndex], mtSecondaryStype[seedIndex], + mtIsRlof1[seedIndex], mtIsRlof2[seedIndex], mtIsCEE[seedIndex], mtIsMrg[seedIndex]) # construct event tuple + + # If this is an entirely new seed: + if thisSeed != lastSeed: # same seed as last seed processed? + returnedSeeds.append(thisSeed) # no - new seed, record it + returnedTimes.append([thisTime]) # initialize the list of event times for this seed + returnedEvents.append([thisEvent]) # initialize the list of events for this seed + lastSeed = thisSeed # update the latest seed + + # Add event, if it is not a duplicate + try: + eventIndex = returnedEvents[-1].index(thisEvent) # find eventIndex of this particular event tuple in the array of events for this seed + if thisTime > returnedTimes[-1][eventIndex]: # ^ if event is not a duplicate, this will throw a ValueError + returnedTimes[-1][eventIndex] = thisTime # if event is duplicate, update time to the later of the duplicates + except ValueError: # event is not a duplicate: + returnedEvents[-1].append(thisEvent) # record new event tuple for this seed + returnedTimes[-1].append(thisTime) # record new event time for this seed + + return returnedSeeds, returnedEvents, returnedTimes # see above for description + + +def getSnEvents(SN): + """ + This function takes in the `BSE_Supernovae` output category from COMPAS, and returns the information + on the Supernova (SN) events that happen for each seed. The events do not have to be in order chronologically, + this function will reorder them as required. + + OUT: + returnedSeeds (list): ordered list of all the unique seeds in the SN file + returnedEvents (list): list of sublists, where each sublist contains all the SN events for a given seed. + SN event tuples take the form : + (stellarTypeProgenitor, stellarTypeRemnant, whichStarIsProgenitor, isBinaryUnbound) + returnedTimes (list): is a list of sublists of times of each of the SN events + """ + snSeeds = SN['SEED'][()] + snTimes = SN['Time'][()] + snProgStype = SN['Stellar_Type_Prev(SN)'][()] + snRemnStype = SN['Stellar_Type(SN)'][()] + snWhichProg = SN['Supernova_State'][()] + snIsUnbound = SN['Unbound'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + snSeedsInds = np.lexsort((snTimes, snSeeds)) # sort by seeds then times - lexsort sorts by the last column first... + snSeeds = snSeeds[snSeedsInds] + snTimes = snTimes[snSeedsInds] + snProgStype = snProgStype[snSeedsInds] + snRemnStype = snRemnStype[snSeedsInds] + snWhichProg = snWhichProg[snSeedsInds] + snIsUnbound = snIsUnbound[snSeedsInds] + + # Process the SN events + returnedSeeds = [] # array of seeds - will only contain seeds that have SN events + returnedEvents = [] # array of SN events for each seed in returnedSeeds + returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) + + lastSeed = -1 # initialize most recently processed seed + for seedIndex, thisSeed in enumerate(snSeeds): # iterate over all SN file entries + thisTime = snTimes[seedIndex] # time for this SN file entry + thisEvent = (snProgStype[seedIndex], snRemnStype[seedIndex], + snWhichProg[seedIndex], snIsUnbound[seedIndex]) # construct event tuple + + # If this is an entirely new seed: + if thisSeed != lastSeed: # same seed as last seed processed? + returnedSeeds.append(thisSeed) # no - new seed, record it + returnedTimes.append([thisTime]) # initialize the list of event times for this seed + returnedEvents.append([thisEvent]) # initialize the list of events for this seed + lastSeed = thisSeed # update the latest seed + else: # yes - second SN event for this seed + returnedTimes[-1].append(thisTime) # append time at end of array + returnedEvents[-1].append(thisEvent) # append event at end of array + + return returnedSeeds, returnedEvents, returnedTimes # see above for description + + +def getEventHistory(h5file, exclude_null=False): + """ + Get the event history for all seeds, including both RLOF and SN events, in chronological order. + IN: + h5file (h5.File() type): COMPAS HDF5 output file + exclude_null (bool): whether or not to exclude seeds which undergo no RLOF or SN events + OUT: + returnedSeeds (list): ordered list of all seeds in the output + returnedEvents (list): a list (corresponding to the ordered seeds above) of the + collected SN and MT events from the getMtEvents and getSnEvents functions above, + themselves ordered chronologically + """ + SP = h5file['BSE_System_Parameters'] + MT = h5file['BSE_RLOF'] + SN = h5file['BSE_Supernovae'] + allSeeds = SP['SEED'][()] # get all seeds + mtSeeds, mtEvents, mtTimes = getMtEvents(MT) # get MT events + snSeeds, snEvents, snTimes = getSnEvents(SN) # get SN events + + numMtSeeds = len(mtSeeds) # number of MT events + numSnSeeds = len(snSeeds) # number of SN events + + if numMtSeeds < 1 and numSnSeeds < 1: return [] # no events - return empty history + + returnedSeeds = [] # array of seeds - will only contain seeds that have events (of any type) + returnedEvents = [] # array of events - same size as returnedSeeds (includes event times) + + eventOrdering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events + + mtIndex = 0 # index into MT events arrays + snIndex = 0 # index into SN events arrays + + if exclude_null: + seedsToIterate = np.sort(np.unique(np.append(mtSeeds, snSeeds))) # iterate over all the seeds that have either MT or SN events + else: + seedsToIterate = allSeeds + + idxOrdered = np.argsort(seedsToIterate) + returnedSeeds = [None] * np.size(seedsToIterate) # array of seeds - will only contain seeds that have events (of any type) + returnedEvents = [None] * np.size(seedsToIterate) # array of events - same size as returnedSeeds (includes event times) + + for idx in idxOrdered: + seed = seedsToIterate[idx] + seedEvents = [] # initialise the events for the seed being processed + + # Collect any MT events for this seed, add the time of the event and the event type + while mtIndex < numMtSeeds and mtSeeds[mtIndex] == seed: + for eventIndex, event in enumerate(mtEvents[mtIndex]): + _, _, isRL1, isRL2, isCEE, isMrg = event + eventKey = 'MG' if isMrg else 'CE' if isCEE else 'RL' # event type: Mrg, CEE, RLOF 1->2, RLOF 2->1 + seedEvents.append((eventKey, mtTimes[mtIndex][eventIndex], *mtEvents[mtIndex][eventIndex])) + mtIndex += 1 + + # Collect any SN events for this seed, add the time of the event and the event type + while snIndex < numSnSeeds and snSeeds[snIndex] == seed: + for eventIndex, event in enumerate(snEvents[snIndex]): + eventKey = 'SN' + seedEvents.append((eventKey, snTimes[snIndex][eventIndex], *snEvents[snIndex][eventIndex])) + snIndex += 1 + + seedEvents.sort(key=lambda ev:(ev[1], eventOrdering.index(ev[0]))) # sort the events by time and event type (MT before SN if at the same time) + + returnedSeeds[idx] = seed # record the seed in the seeds array being returned + returnedEvents[idx] = seedEvents # record the events for this seed in the events array being returned + + return returnedSeeds, returnedEvents # see above for details + + + +########################################### +# ## Produce strings of the event histories + +stellarTypeDict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', + 10: 'WD', + 11: 'WD', + 12: 'WD', + 13: 'NS', + 14: 'BH', + 15: 'MR', + 16: 'MS', +} + +def buildEventString(events, useIntStypes=False): + """ + Function to produce a string representing the event history of a single binary for quick readability. + NOTE: unvectorized, do the vectorization later for a speed up + IN: + events (list of tuples): events output from getEventHistory() + OUT: + eventString (string): string representing the event history of the binary + + MT strings look like: + P>S, P, < is RLOF (1->2 or 1<-2) or otherwise = for CEE or & for Merger + + SN strings look like: + P*SR for star1 the SN progenitor,or + R*SP for star2 the SN progenitor, + where P is progenitor type, R is remnant type, + S is state (I for intact, U for unbound) + + Event strings for the same seed are separated by the undesrcore character ('_') + """ + def _remap_stype(int_stype): + if useIntStypes: + return str(int_stype) + else: + return stellarTypeDict[int_stype] + + # Empty event + if len(events) == 0: + return 'NA' + + eventStr = '' + for event in events: + if event[0] == 'SN': # SN event + _, time, stypeP, stypeC, whichSN, isUnbound = event + if whichSN == 1: # Progenitor or Remnant depending upon which star is the SN + charL = _remap_stype(stypeP) + charR = _remap_stype(stypeC) + else: + charL = _remap_stype(stypeC) + charR = _remap_stype(stypeP) + charM = '*u' if isUnbound else '*i' # unbound or intact + else: # Any of the MT events + _, time, stype1, stype2, isRL1, isRL2, isCEE, isMrg = event + charL = _remap_stype(stype1) # primary stellar type + charR = _remap_stype(stype2) # secondary stellar type + charM = '&' if isMrg \ + else '=' if isCEE \ + else '<' if isRL2 \ + else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 + eventStr += "{}{}{}_".format(charL, charM, charR) # event string for this star, _ is event separator + + return np.array(eventStr[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + +def getEventStrings(h5file=None, allEvents=None, useIntStypes=False): + """ + Function to calculate the event history strings for either the entire Compas output, or some list of events + IN: One of + h5file (h5.File() type): COMPAS HDF5 output file + allEvents (list of tuples) + OUT: + eventStrings (list): list of strings of the event history of each seed + """ + + # If output is + if (h5file == None) & (allEvents == None): + return + elif (allEvents == None): + _, allEvents = getEventHistory(h5file) + + eventStrings = np.zeros(len(allEvents), dtype=StringDType()) + for ii, eventsForGivenSeed in enumerate(allEvents): + eventString = buildEventString(eventsForGivenSeed, useIntStypes=useIntStypes) + eventStrings[ii] = eventString # append event string for this star (pop the last underscore first) + + return eventStrings + +"" + + +"" + From 3d2135e52520407aeb2a05c2bbf7702679110d0a Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Fri, 26 Jun 2026 17:52:09 +0200 Subject: [PATCH 17/22] big cleanup of post processing utils --- compas_python_utils/compasUtils.py | 355 -------------------- compas_python_utils/debugging_utils.py | 440 +++++++++++++++++++++++++ 2 files changed, 440 insertions(+), 355 deletions(-) delete mode 100644 compas_python_utils/compasUtils.py create mode 100644 compas_python_utils/debugging_utils.py diff --git a/compas_python_utils/compasUtils.py b/compas_python_utils/compasUtils.py deleted file mode 100644 index 2cc126c68..000000000 --- a/compas_python_utils/compasUtils.py +++ /dev/null @@ -1,355 +0,0 @@ -import h5py as h5 -import numpy as np -import pandas as pd -from numpy.dtypes import StringDType - - -######################################################################## -# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template - -def printCompasDetails(data, *seeds, mask=()): - """ - Function to print the full Compas output for given seeds, optionally with an additional mask - """ - list_of_keys = list(data.keys()) - - # Check if seed parameter exists - if not, just print without (e.g RunDetails) - if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): # Most output files - #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility - - # Set the seed name parameter, mask on seeds as needed, and set the index - seedVariableName='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' - list_of_keys.remove(seedVariableName) # this is the index above, don't want to include it - - allSeeds = data[seedVariableName][()] - seedsMask = np.isin(allSeeds, seeds) - if len(seeds) == 0: # if any seed is included, do not reset the mask - seedsMask = np.ones_like(allSeeds).astype(bool) - if mask == (): - mask = np.ones_like(allSeeds).astype(bool) - mask &= seedsMask - - df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list(data.keys())}).set_index(seedVariableName).T - - else: # No seed parameter, so do custom print for Run Details - - # Get just the keys without the -Derivation suffix - those will be a second column - keys_not_derivations = [] - for key in list_of_keys: - if '-Derivation' not in key: - keys_not_derivations.append(key) - - # Some parameter values are string types, formatted as np.bytes_, need to convert back - def convert_strings(param_array): - if isinstance(param_array[0], np.bytes_): - return param_array.astype(str) - else: - return param_array - - df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T - nCols = df_keys.shape[1] # Required only because if we combine RDs, we get many columns (should fix later) - df_keys.columns = ['Parameter']*nCols - df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T - df_drvs.columns = ['Derivation']*nCols - df = pd.concat([df_keys, df_drvs], axis=1) - - # Add units as first col - units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} - df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) - return df - - - -######################################################################## -# ## Get event histories of MT data, SN data, and combined MT, SN data - -def getMtEvents(MT): - """ - This function takes in the `BSE_RLOF` output category from COMPAS, and returns the information - on the Mass Transfer (MT) events that happen for each seed. The events do not have to be in order, - either chronologically or by seed, this function will reorder them as required. - - OUT: - returnedSeeds (list): ordered list of the unique seeds in the MT file - returnedEvents (list): list of sublists, where each sublist contains all the MT events for a given seed. - MT event tuples take the form : - (stellarTypePrimary, stellarTypeSecondary, isRlof1, isRlof2, isCEE, isMrg) - returnTimes (list): is a list of sublists of times of each of the MT events - """ - mtSeeds = MT['SEED'][()] - mtTimes = MT['TimeMT'][()] == 1 - mtIsRlof2 = MT['RLOF(2)>MT'][()] == 1 - mtIsCEE = MT['CEE>MT'][()] == 1 - mtIsMrg = MT['Merger'][()] == 1 - - # We want the return arrays sorted by seed, so sort here. - mtSeedsInds = np.lexsort((mtTimes, mtSeeds)) # sort by seeds then times - lexsort sorts by the last column first... - mtSeeds = mtSeeds[mtSeedsInds] - mtTimes = mtTimes[mtSeedsInds] - mtPrimaryStype = mtPrimaryStype[mtSeedsInds] - mtSecondaryStype = mtSecondaryStype[mtSeedsInds] - mtIsRlof1 = mtIsRlof1[mtSeedsInds] - mtIsRlof2 = mtIsRlof2[mtSeedsInds] - mtIsCEE = mtIsCEE[mtSeedsInds] - mtIsMrg = mtIsMrg[mtSeedsInds] - - # Process the MT events - returnedSeeds = [] # array of seeds - will only contain seeds that have MT events - returnedEvents = [] # array of MT events for each seed in returnedSeeds - returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) - - lastSeed = -1 # initialize most recently processed seed - for seedIndex, thisSeed in enumerate(mtSeeds): # iterate over all RLOF file entries - thisTime = mtTimes[seedIndex] # time for this RLOF file entry - thisEvent = (mtPrimaryStype[seedIndex], mtSecondaryStype[seedIndex], - mtIsRlof1[seedIndex], mtIsRlof2[seedIndex], mtIsCEE[seedIndex], mtIsMrg[seedIndex]) # construct event tuple - - # If this is an entirely new seed: - if thisSeed != lastSeed: # same seed as last seed processed? - returnedSeeds.append(thisSeed) # no - new seed, record it - returnedTimes.append([thisTime]) # initialize the list of event times for this seed - returnedEvents.append([thisEvent]) # initialize the list of events for this seed - lastSeed = thisSeed # update the latest seed - - # Add event, if it is not a duplicate - try: - eventIndex = returnedEvents[-1].index(thisEvent) # find eventIndex of this particular event tuple in the array of events for this seed - if thisTime > returnedTimes[-1][eventIndex]: # ^ if event is not a duplicate, this will throw a ValueError - returnedTimes[-1][eventIndex] = thisTime # if event is duplicate, update time to the later of the duplicates - except ValueError: # event is not a duplicate: - returnedEvents[-1].append(thisEvent) # record new event tuple for this seed - returnedTimes[-1].append(thisTime) # record new event time for this seed - - return returnedSeeds, returnedEvents, returnedTimes # see above for description - - -def getSnEvents(SN): - """ - This function takes in the `BSE_Supernovae` output category from COMPAS, and returns the information - on the Supernova (SN) events that happen for each seed. The events do not have to be in order chronologically, - this function will reorder them as required. - - OUT: - returnedSeeds (list): ordered list of all the unique seeds in the SN file - returnedEvents (list): list of sublists, where each sublist contains all the SN events for a given seed. - SN event tuples take the form : - (stellarTypeProgenitor, stellarTypeRemnant, whichStarIsProgenitor, isBinaryUnbound) - returnedTimes (list): is a list of sublists of times of each of the SN events - """ - snSeeds = SN['SEED'][()] - snTimes = SN['Time'][()] - snProgStype = SN['Stellar_Type_Prev(SN)'][()] - snRemnStype = SN['Stellar_Type(SN)'][()] - snWhichProg = SN['Supernova_State'][()] - snIsUnbound = SN['Unbound'][()] == 1 - - # We want the return arrays sorted by seed, so sort here. - snSeedsInds = np.lexsort((snTimes, snSeeds)) # sort by seeds then times - lexsort sorts by the last column first... - snSeeds = snSeeds[snSeedsInds] - snTimes = snTimes[snSeedsInds] - snProgStype = snProgStype[snSeedsInds] - snRemnStype = snRemnStype[snSeedsInds] - snWhichProg = snWhichProg[snSeedsInds] - snIsUnbound = snIsUnbound[snSeedsInds] - - # Process the SN events - returnedSeeds = [] # array of seeds - will only contain seeds that have SN events - returnedEvents = [] # array of SN events for each seed in returnedSeeds - returnedTimes = [] # array of times for each event in returnedEvents (for each seed in returnedSeeds) - - lastSeed = -1 # initialize most recently processed seed - for seedIndex, thisSeed in enumerate(snSeeds): # iterate over all SN file entries - thisTime = snTimes[seedIndex] # time for this SN file entry - thisEvent = (snProgStype[seedIndex], snRemnStype[seedIndex], - snWhichProg[seedIndex], snIsUnbound[seedIndex]) # construct event tuple - - # If this is an entirely new seed: - if thisSeed != lastSeed: # same seed as last seed processed? - returnedSeeds.append(thisSeed) # no - new seed, record it - returnedTimes.append([thisTime]) # initialize the list of event times for this seed - returnedEvents.append([thisEvent]) # initialize the list of events for this seed - lastSeed = thisSeed # update the latest seed - else: # yes - second SN event for this seed - returnedTimes[-1].append(thisTime) # append time at end of array - returnedEvents[-1].append(thisEvent) # append event at end of array - - return returnedSeeds, returnedEvents, returnedTimes # see above for description - - -def getEventHistory(h5file, exclude_null=False): - """ - Get the event history for all seeds, including both RLOF and SN events, in chronological order. - IN: - h5file (h5.File() type): COMPAS HDF5 output file - exclude_null (bool): whether or not to exclude seeds which undergo no RLOF or SN events - OUT: - returnedSeeds (list): ordered list of all seeds in the output - returnedEvents (list): a list (corresponding to the ordered seeds above) of the - collected SN and MT events from the getMtEvents and getSnEvents functions above, - themselves ordered chronologically - """ - SP = h5file['BSE_System_Parameters'] - MT = h5file['BSE_RLOF'] - SN = h5file['BSE_Supernovae'] - allSeeds = SP['SEED'][()] # get all seeds - mtSeeds, mtEvents, mtTimes = getMtEvents(MT) # get MT events - snSeeds, snEvents, snTimes = getSnEvents(SN) # get SN events - - numMtSeeds = len(mtSeeds) # number of MT events - numSnSeeds = len(snSeeds) # number of SN events - - if numMtSeeds < 1 and numSnSeeds < 1: return [] # no events - return empty history - - returnedSeeds = [] # array of seeds - will only contain seeds that have events (of any type) - returnedEvents = [] # array of events - same size as returnedSeeds (includes event times) - - eventOrdering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events - - mtIndex = 0 # index into MT events arrays - snIndex = 0 # index into SN events arrays - - if exclude_null: - seedsToIterate = np.sort(np.unique(np.append(mtSeeds, snSeeds))) # iterate over all the seeds that have either MT or SN events - else: - seedsToIterate = allSeeds - - idxOrdered = np.argsort(seedsToIterate) - returnedSeeds = [None] * np.size(seedsToIterate) # array of seeds - will only contain seeds that have events (of any type) - returnedEvents = [None] * np.size(seedsToIterate) # array of events - same size as returnedSeeds (includes event times) - - for idx in idxOrdered: - seed = seedsToIterate[idx] - seedEvents = [] # initialise the events for the seed being processed - - # Collect any MT events for this seed, add the time of the event and the event type - while mtIndex < numMtSeeds and mtSeeds[mtIndex] == seed: - for eventIndex, event in enumerate(mtEvents[mtIndex]): - _, _, isRL1, isRL2, isCEE, isMrg = event - eventKey = 'MG' if isMrg else 'CE' if isCEE else 'RL' # event type: Mrg, CEE, RLOF 1->2, RLOF 2->1 - seedEvents.append((eventKey, mtTimes[mtIndex][eventIndex], *mtEvents[mtIndex][eventIndex])) - mtIndex += 1 - - # Collect any SN events for this seed, add the time of the event and the event type - while snIndex < numSnSeeds and snSeeds[snIndex] == seed: - for eventIndex, event in enumerate(snEvents[snIndex]): - eventKey = 'SN' - seedEvents.append((eventKey, snTimes[snIndex][eventIndex], *snEvents[snIndex][eventIndex])) - snIndex += 1 - - seedEvents.sort(key=lambda ev:(ev[1], eventOrdering.index(ev[0]))) # sort the events by time and event type (MT before SN if at the same time) - - returnedSeeds[idx] = seed # record the seed in the seeds array being returned - returnedEvents[idx] = seedEvents # record the events for this seed in the events array being returned - - return returnedSeeds, returnedEvents # see above for details - - - -########################################### -# ## Produce strings of the event histories - -stellarTypeDict = { - 0: 'MS', - 1: 'MS', - 2: 'HG', - 3: 'GB', - 4: 'GB', - 5: 'GB', - 6: 'GB', - 7: 'HE', - 8: 'HE', - 9: 'HE', - 10: 'WD', - 11: 'WD', - 12: 'WD', - 13: 'NS', - 14: 'BH', - 15: 'MR', - 16: 'MS', -} - -def buildEventString(events, useIntStypes=False): - """ - Function to produce a string representing the event history of a single binary for quick readability. - NOTE: unvectorized, do the vectorization later for a speed up - IN: - events (list of tuples): events output from getEventHistory() - OUT: - eventString (string): string representing the event history of the binary - - MT strings look like: - P>S, P, < is RLOF (1->2 or 1<-2) or otherwise = for CEE or & for Merger - - SN strings look like: - P*SR for star1 the SN progenitor,or - R*SP for star2 the SN progenitor, - where P is progenitor type, R is remnant type, - S is state (I for intact, U for unbound) - - Event strings for the same seed are separated by the undesrcore character ('_') - """ - def _remap_stype(int_stype): - if useIntStypes: - return str(int_stype) - else: - return stellarTypeDict[int_stype] - - # Empty event - if len(events) == 0: - return 'NA' - - eventStr = '' - for event in events: - if event[0] == 'SN': # SN event - _, time, stypeP, stypeC, whichSN, isUnbound = event - if whichSN == 1: # Progenitor or Remnant depending upon which star is the SN - charL = _remap_stype(stypeP) - charR = _remap_stype(stypeC) - else: - charL = _remap_stype(stypeC) - charR = _remap_stype(stypeP) - charM = '*u' if isUnbound else '*i' # unbound or intact - else: # Any of the MT events - _, time, stype1, stype2, isRL1, isRL2, isCEE, isMrg = event - charL = _remap_stype(stype1) # primary stellar type - charR = _remap_stype(stype2) # secondary stellar type - charM = '&' if isMrg \ - else '=' if isCEE \ - else '<' if isRL2 \ - else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 - eventStr += "{}{}{}_".format(charL, charM, charR) # event string for this star, _ is event separator - - return np.array(eventStr[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) - -def getEventStrings(h5file=None, allEvents=None, useIntStypes=False): - """ - Function to calculate the event history strings for either the entire Compas output, or some list of events - IN: One of - h5file (h5.File() type): COMPAS HDF5 output file - allEvents (list of tuples) - OUT: - eventStrings (list): list of strings of the event history of each seed - """ - - # If output is - if (h5file == None) & (allEvents == None): - return - elif (allEvents == None): - _, allEvents = getEventHistory(h5file) - - eventStrings = np.zeros(len(allEvents), dtype=StringDType()) - for ii, eventsForGivenSeed in enumerate(allEvents): - eventString = buildEventString(eventsForGivenSeed, useIntStypes=useIntStypes) - eventStrings[ii] = eventString # append event string for this star (pop the last underscore first) - - return eventStrings - -"" - - -"" - diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py new file mode 100644 index 000000000..76498c96c --- /dev/null +++ b/compas_python_utils/debugging_utils.py @@ -0,0 +1,440 @@ +import h5py as h5 +import numpy as np +import pandas as pd +from numpy.dtypes import StringDType +from typing import NewType + +"" +# New Types +MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) +H5File = NewType('H5File', h5._hl.files.File) +H5Group = NewType('H5Group', h5._hl.group.Group) +MtEventTuple = NewType('MtEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool]]) +SnEventTuple = NewType('SnEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool]]) +EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) + + +######################################################################## +# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template + +def print_compas_details_dataframe(data: H5Group, + *seeds: int, + mask: MaskNdarray=()) -> pd.DataFrame: + """Return a pd.DataFrame for the COMPAS output contained in an H5Group. + + Parameters + ---------- + data : H5Group + an H5Group containing COMPAS data + seeds : int or array_like of ints, optional + seeds of specific systems to include (default is to include all seeds) + mask : array_like of booleans, optional + boolean mask to filter out specific systems (default is no mask) + Must be same length as data['SEED'] + + Returns + ------- + compas_details : pd.DataFrame + The values of each parameter in the H5Group, excluding those removed by `seeds` or `mask`. + + Notes + ----- + Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. + If both `seeds` and `mask` are supplied, resulting systems must pass both filters. + + Examples + -------- + >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF']) + >>> print_compas_details_dataframe(mt_data) + [output of all BSE_RLOF events] + + >>> mt_seeds = mt_data['SEED'][()] + >>> print_compas_details_dataframe(mt_data, mt_seeds[:50]) + [output of all BSE_RLOF events occuring in the first 50 seeds] + + >>> cee_events = mt_data['CEE>MT'][()] == 1 # needed to convert to boolean mask + >>> print_compas_details_dataframe(mt_data, mt_seeds[:50], mask=cee_events) + [output of all Common Envelope events occuring in the first 50 seeds] + """ + list_of_keys = list(data.keys()) + + # Check if SEED parameter exists in data + if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): + seed_variable_name='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + + # If `seeds` or `mask` arguments supplied, create the relevant mask + all_seeds = data[seed_variable_name][()] + seeds_mask = np.isin(all_seeds, seeds) + if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask + seeds_mask = np.ones_like(all_seeds).astype(bool) + if mask == (): + mask = np.ones_like(all_seeds).astype(bool) + mask &= seeds_mask + df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list_of_keys}).set_index(seed_variable_name).T + + else: # No seed parameter, currently only applies to the RunDetails H5Group + keys_not_derivations = [key for key in list_of_keys if '-Derivation' not in key] # Get just the keys without the -Derivation suffix + + # Some parameter values are string types, formatted as np.bytes_, need to convert back + def convert_strings(param_array): + if isinstance(param_array[0], np.bytes_): + return param_array.astype(str) + else: + return param_array + + df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T + nCols = df_keys.shape[1] # Required only because if we have combined RunDetails output from a previous h5copy, we get many columns (should fix later) + df_keys.columns = ['Parameter']*nCols + df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T + df_drvs.columns = ['Derivation']*nCols + df = pd.concat([df_keys, df_drvs], axis=1) + + # Add units as first col + units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) + return df + + + +######################################################################## +# ## Get event histories of MT data, SN data, and combined MT, SN data + +def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: + """Calculates the EventTuple for the BSE_RLOF output H5Group. + + Parameters + ---------- + mt_data : H5Group + the COMPAS output H5Group corresponding to BSE_RLOF + + Returns + ------- + returned_seeds : list + an ordered list of the unique seeds in the mt_data file, + returned_events : list + a list of sublists, one sublist per seed, where each sublist + contains all the MtEventTuples for the given seed + returned_times : list + a list of sublists of times of each of the mt_data events. + + Notes + ----- + MtEventTuples take the form: (stellar_type_primary, stellar_type_secondary, is_rlof1, is_rlof2, is_cee, is_mrg). + The events in the input do not have to be ordered chronologically, this function orders them, in the event + that the input was coallated from a previous h5copy command. The output is not meant to be used generally, + it is just a feeder for get_event_history below. + """ + mt_seeds = mt_data['SEED'][()] + + mt_times = mt_data['TimeMT'][()] == 1 + mt_is_rlof2 = mt_data['RLOF(2)>MT'][()] == 1 + mt_is_cee = mt_data['CEE>MT'][()] == 1 + mt_is_mrg = mt_data['Merger'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) # sort by seeds then times - lexsort sorts by the last column first... + mt_seeds = mt_seeds[mt_seeds_idx] + mt_times = mt_times[mt_seeds_idx] + mt_primary_stype = mt_primary_stype[mt_seeds_idx] + mt_secondary_stype = mt_secondary_stype[mt_seeds_idx] + mt_is_rlof1 = mt_is_rlof1[mt_seeds_idx] + mt_is_rlof2 = mt_is_rlof2[mt_seeds_idx] + mt_is_cee = mt_is_cee[mt_seeds_idx] + mt_is_mrg = mt_is_mrg[mt_seeds_idx] + + # Process the mt_data events + returned_seeds = [] # array of seeds - will only contain seeds that have mt_data events + returned_events = [] # array of mt_data events for each seed in returned_seeds + returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) + + last_seed = -1 # initialize most recently processed seed + for seed_index, this_seed in enumerate(mt_seeds): # iterate over all RLOF file entries + this_time = mt_times[seed_index] # time for this RLOF file entry + this_event = (mt_primary_stype[seed_index], mt_secondary_stype[seed_index], + mt_is_rlof1[seed_index], mt_is_rlof2[seed_index], mt_is_cee[seed_index], mt_is_mrg[seed_index]) # construct event tuple + + # If this is an entirely new seed: + if this_seed != last_seed: # same seed as last seed processed? + returned_seeds.append(this_seed) # no - new seed, record it + returned_times.append([this_time]) # initialize the list of event times for this seed + returned_events.append([this_event]) # initialize the list of events for this seed + last_seed = this_seed # update the latest seed + + # Add event, if it is not a duplicate + try: + event_index = returned_events[-1].index(this_event) # find event_index of this particular event tuple in the array of events for this seed + if this_time > returned_times[-1][event_index]: # ^ if event is not a duplicate, this will throw a ValueError + returned_times[-1][event_index] = this_time # if event is duplicate, update time to the later of the duplicates + except ValueError: # event is not a duplicate: + returned_events[-1].append(this_event) # record new event tuple for this seed + returned_times[-1].append(this_time) # record new event time for this seed + + return returned_seeds, returned_events, returned_times # see above for description + + +def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: + """Calculates the EventTuple for the BSE_Supernovae output H5Group. + + Parameters + ---------- + sn_data : H5Group + the COMPAS output H5Group corresponding to BSE_Supernovae + + Returns + ------- + returned_seeds : list + an ordered list of the unique seeds in the sn_data file, + returned_events : list + a list of sublists, one sublist per seed, where each sublist + contains all the SnEventTuples for the given seed + returned_times : list + a list of sublists of times of each of the sn_data events. + + Notes + ----- + SnEventTuples take the form: (stellar_type_progenitor, stellar_type_remnant, which_starIsProgenitor, is_binary_unbound). + The events in the input do not have to be ordered chronologically, this function orders them, in the event + that the input was coallated from a previous h5copy command. The output is not meant to be used generally, + it is just a feeder for get_event_history below. + """ + sn_seeds = sn_data['SEED'][()] + sn_times = sn_data['Time'][()] + sn_prog_stype = sn_data['Stellar_Type_Prev(SN)'][()] + sn_remn_stype = sn_data['Stellar_Type(SN)'][()] + sn_which_prog = sn_data['Supernova_State'][()] + sn_is_unbound = sn_data['Unbound'][()] == 1 + + # We want the return arrays sorted by seed, so sort here. + sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) # sort by seeds then times - lexsort sorts by the last column first... + sn_seeds = sn_seeds[sn_seeds_idx] + sn_times = sn_times[sn_seeds_idx] + sn_prog_stype = sn_prog_stype[sn_seeds_idx] + sn_remn_stype = sn_remn_stype[sn_seeds_idx] + sn_which_prog = sn_which_prog[sn_seeds_idx] + sn_is_unbound = sn_is_unbound[sn_seeds_idx] + + # Process the sn_data events + returned_seeds = [] # array of seeds - will only contain seeds that have sn_data events + returned_events = [] # array of sn_data events for each seed in returned_seeds + returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) + + last_seed = -1 # initialize most recently processed seed + for seed_index, this_seed in enumerate(sn_seeds): # iterate over all sn_data file entries + this_time = sn_times[seed_index] # time for this sn_data file entry + this_event = (sn_prog_stype[seed_index], sn_remn_stype[seed_index], + sn_which_prog[seed_index], sn_is_unbound[seed_index]) # construct event tuple + + # If this is an entirely new seed: + if this_seed != last_seed: # same seed as last seed processed? + returned_seeds.append(this_seed) # no - new seed, record it + returned_times.append([this_time]) # initialize the list of event times for this seed + returned_events.append([this_event]) # initialize the list of events for this seed + last_seed = this_seed # update the latest seed + else: # yes - second sn_data event for this seed + returned_times[-1].append(this_time) # append time at end of array + returned_events[-1].append(this_event) # append event at end of array + + return returned_seeds, returned_events, returned_times # see above for description + + +def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list]: + """Get the event history for all seeds, including both MT and SN events, in chronological order. + + Parameters + ---------- + data : H5File + the COMPAS output H5file + include_null : bool + whether to include seeds which undergo no MT or SN events (default is True) + + Returns + ------- + returned_seeds : list + an ordered list of all the unique seeds in the output file + returned_events : list + a list where each element is a chronological set of events corresponding to the associated seed + """ + sp_data = data['BSE_System_Parameters'] + mt_data = data['BSE_RLOF'] + sn_data = data['BSE_Supernovae'] + all_seeds = sp_data['SEED'][()] # get all seeds + mt_seeds, mt_events, mt_times = get_mt_data_tuple(mt_data) # get mt data tuple + sn_seeds, sn_events, sn_times = get_sn_data_tuple(sn_data) # get sn data tuple + + num_mt_seeds = len(mt_seeds) # number of mt_data events + num_sn_seeds = len(sn_seeds) # number of sn_data events + + if num_mt_seeds < 1 and num_sn_seeds < 1: return [] # no events - return empty history + + returned_seeds = [] # array of seeds - will only contain seeds that have events (of any type) + returned_events = [] # array of events - same size as returned_seeds (includes event times) + + event_ordering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events + + mt_index = 0 # index into mt_data events arrays + sn_index = 0 # index into sn_data events arrays + + if include_null: + seeds_to_iterate = all_seeds + else: + seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) # iterate over all the seeds that have either mt_data or sn_data events + + idx_ordered = np.argsort(seeds_to_iterate) + returned_seeds = [None] * np.size(seeds_to_iterate) # array of seeds - will only contain seeds that have events (of any type) + returned_events = [None] * np.size(seeds_to_iterate) # array of events - same size as returned_seeds (includes event times) + + for idx in idx_ordered: + seed = seeds_to_iterate[idx] + seed_events = [] # initialise the events for the seed being processed + + # Collect any mt_data events for this seed, add the time of the event and the event type + while mt_index < num_mt_seeds and mt_seeds[mt_index] == seed: + for event_index, event in enumerate(mt_events[mt_index]): + _, _, is_rl1, is_rl2, is_cee, is_mrg = event + event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 + seed_events.append((event_key, mt_times[mt_index][event_index], *mt_events[mt_index][event_index])) + mt_index += 1 + + # Collect any sn_data events for this seed, add the time of the event and the event type + while sn_index < num_sn_seeds and sn_seeds[sn_index] == seed: + for event_index, event in enumerate(sn_events[sn_index]): + event_key = 'SN' + seed_events.append((event_key, sn_times[sn_index][event_index], *sn_events[sn_index][event_index])) + sn_index += 1 + + seed_events.sort(key=lambda ev:(ev[1], event_ordering.index(ev[0]))) # sort the events by time and event type (mt_data before sn_data if at the same time) + returned_seeds[idx] = seed # record the seed in the seeds array being returned + returned_events[idx] = seed_events # record the events for this seed in the events array being returned + return returned_seeds, returned_events + + +########################################### +# ## Produce strings of the event histories + +stellar_type_dict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', + 10: 'WD', + 11: 'WD', + 12: 'WD', + 13: 'NS', + 14: 'BH', + 15: 'MR', + 16: 'MS', +} + + +def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistoryString: + """Produce a string representation of the event history of a single binary. + + Parameters + ---------- + events : list + list of mixed MtEventTuple and SnEventTuple's + use_int_stypes : bool, optional + whether to report stellar types as integers + (default is False, report them as 2-char stellar types) + + Returns + ------- + event_str : EventHistoryString + condensed string representation of the event history of a binary, in array form + + Notes + ----- + MT strings look like: + P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) + or otherwise '=' for CEE or '&' for Merger. + SN strings look like: + P*SR if star1 is the SN progenitor, or + R*SP if star2 is the SN progenitor, + where P is progenitor type, R is remnant type, + S is state ('i' for intact, 'u' for unbound) + Event strings for the same seed are separated by an undesrcore '_' + + TODO + ---- + Currently unvectorized, do the vectorization later for a speed up + """ + def _remap_stype(int_stype): + if use_int_stypes: + return str(int_stype) + else: + return stellar_type_dict[int_stype] + + # Empty event + if len(events) == 0: + return 'NA' + + event_str = '' + for event in events: + if event[0] == 'SN': # SN event + _, time, stype_p, stype_c, which_sn, is_unbound = event + if which_sn == 1: # Progenitor or Remnant depending upon which star is the SN + char_l = _remap_stype(stype_p) + char_r = _remap_stype(stype_c) + else: + char_l = _remap_stype(stype_c) + char_r = _remap_stype(stype_p) + char_m = '*u' if is_unbound else '*i' # unbound or intact + else: # Any of the MT events + _, time, stype_1, stype_2, is_rl1, is_rl2, is_cee, is_mrg = event + char_l = _remap_stype(stype_1) # primary stellar type + char_r = _remap_stype(stype_2) # secondary stellar type + char_m = '&' if is_mrg \ + else '=' if is_cee \ + else '<' if is_rl2 \ + else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 + event_str += "{}{}{}_".format(char_l, char_m, char_r) # event string for this star, _ is event separator + event_str = np.array(event_str[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + return event_str + + +def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: bool=False): + """Calculate the event history strings for a COMPAS population. + + Parameters + ---------- + data : H5File, optional + the COMPAS output H5file + all_events : list, optional + a list where each element is a chronological set of events corresponding to the associated seed, + one of the outputs of get_event_history(data) + use_int_stypes : bool, optional + + Returns + ------- + event_strings : array of EventHistoryString's + strings of the event history of each system + + Notes + ----- + Exactly one of data or all_events must be included. If neither, nothign is returned. + """ + + # If output is + if (data == None) & (all_events == None): + return + elif (all_events == None): + _, all_events = get_event_history(data) + + event_strings = np.zeros(len(all_events), dtype=StringDType()) + for ii, events_for_given_seed in enumerate(all_events): + event_string = build_event_string(events_for_given_seed, use_int_stypes=use_int_stypes) + event_strings[ii] = event_string # append event string for this star (pop the last underscore first) + return event_strings + +"" + From 0f4452acea1f15026b8ae79dbab1b80ea40a18f2 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 29 Jun 2026 12:05:36 +0200 Subject: [PATCH 18/22] addressed nicolas' concerns --- compas_python_utils/debugging_utils.py | 436 +++++++++++++++---------- 1 file changed, 272 insertions(+), 164 deletions(-) diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py index 76498c96c..15a9826a2 100644 --- a/compas_python_utils/debugging_utils.py +++ b/compas_python_utils/debugging_utils.py @@ -9,17 +9,46 @@ MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) H5File = NewType('H5File', h5._hl.files.File) H5Group = NewType('H5Group', h5._hl.group.Group) -MtEventTuple = NewType('MtEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool], np.ndarray[bool]]) -SnEventTuple = NewType('SnEventTuple', tuple[ np.ndarray[int], np.ndarray[int], np.ndarray[bool], np.ndarray[bool]]) +MtEventTuple = NewType('MtEventTuple', + tuple[np.ndarray[int], + np.ndarray[int], + np.ndarray[bool], + np.ndarray[bool], + np.ndarray[bool], + np.ndarray[bool]]) +SnEventTuple = NewType('SnEventTuple', + tuple[np.ndarray[int], + np.ndarray[int], + np.ndarray[bool], + np.ndarray[bool]]) EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) ######################################################################## # ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template -def print_compas_details_dataframe(data: H5Group, - *seeds: int, - mask: MaskNdarray=()) -> pd.DataFrame: +def convert_bytes_array_to_strings(param_array): + """Check and convert np.bytes_ array to strings. + + Parameters + ---------- + param_array : np.ndarray[np.bytes_ | str] + a numpy array containing either np.bytes_ or strings + + Returns + ------- + param_array : np.ndarray[str] + the same array, but converted to strings if not already + """ + if np.issubdtype(param_array.dtype, np.bytes_): + return param_array.astype(str) + else: + return param_array + + +def print_compas_details_dataframe(data: H5Group, + *seeds: int, + mask: MaskNdarray = ()) -> pd.DataFrame: """Return a pd.DataFrame for the COMPAS output contained in an H5Group. Parameters @@ -32,70 +61,69 @@ def print_compas_details_dataframe(data: H5Group, boolean mask to filter out specific systems (default is no mask) Must be same length as data['SEED'] - Returns + Returns ------- compas_details : pd.DataFrame The values of each parameter in the H5Group, excluding those removed by `seeds` or `mask`. Notes ----- - Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. + Designed for easier visualization in a jupyter notebook, for inspection / debugging purposes. If both `seeds` and `mask` are supplied, resulting systems must pass both filters. Examples -------- - >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF']) + >>> mt_data = h5.File('COMPAS_Output.h5')['BSE_RLOF'] >>> print_compas_details_dataframe(mt_data) [output of all BSE_RLOF events] - + >>> mt_seeds = mt_data['SEED'][()] >>> print_compas_details_dataframe(mt_data, mt_seeds[:50]) [output of all BSE_RLOF events occuring in the first 50 seeds] - + >>> cee_events = mt_data['CEE>MT'][()] == 1 # needed to convert to boolean mask >>> print_compas_details_dataframe(mt_data, mt_seeds[:50], mask=cee_events) [output of all Common Envelope events occuring in the first 50 seeds] """ - list_of_keys = list(data.keys()) - # Check if SEED parameter exists in data - if ('SEED' in list_of_keys) | ('SEED>MT' in list_of_keys): - seed_variable_name='SEED' if ('SEED' in list_of_keys) else 'SEED>MT' #SEED>MT is a relic from older versions, but we leave this in for backwards compatibility + if ('SEED' or 'SEED>MT') in data: + # SEED>MT is a relic from older versions, but we leave this in for + # backwards compatibility + seed_variable_name = 'SEED' if ('SEED' in data) else 'SEED>MT' # If `seeds` or `mask` arguments supplied, create the relevant mask all_seeds = data[seed_variable_name][()] seeds_mask = np.isin(all_seeds, seeds) - if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask + if len(seeds) == 0: # If `seeds` argument is not supplied, set the default mask seeds_mask = np.ones_like(all_seeds).astype(bool) if mask == (): mask = np.ones_like(all_seeds).astype(bool) mask &= seeds_mask - df = pd.DataFrame.from_dict({param: data[param][()][mask] for param in list_of_keys}).set_index(seed_variable_name).T - - else: # No seed parameter, currently only applies to the RunDetails H5Group - keys_not_derivations = [key for key in list_of_keys if '-Derivation' not in key] # Get just the keys without the -Derivation suffix - - # Some parameter values are string types, formatted as np.bytes_, need to convert back - def convert_strings(param_array): - if isinstance(param_array[0], np.bytes_): - return param_array.astype(str) - else: - return param_array - - df_keys = pd.DataFrame.from_dict({param: convert_strings(data[param][()]) for param in keys_not_derivations }).T - nCols = df_keys.shape[1] # Required only because if we have combined RunDetails output from a previous h5copy, we get many columns (should fix later) - df_keys.columns = ['Parameter']*nCols - df_drvs = pd.DataFrame.from_dict({param: convert_strings(data[param+'-Derivation'][()]) for param in keys_not_derivations }).T - df_drvs.columns = ['Derivation']*nCols + df = pd.DataFrame.from_dict( + {param: data[param][()][mask] for param in data}).set_index(seed_variable_name).T + + else: # No seed parameter, currently only applies to the RunDetails H5Group + # Get just the keys without the -Derivation suffix + keys_not_derivations = [ + key for key in data if '-Derivation' not in key] + + df_keys = pd.DataFrame.from_dict({param: convert_bytes_array_to_strings( + data[param][()]) for param in keys_not_derivations}).T + # Required only because if we have combined RunDetails output from a + # previous h5copy, we get many columns (should fix later) + nCols = df_keys.shape[1] + df_keys.columns = ['Parameter'] * nCols + df_drvs = pd.DataFrame.from_dict({param: convert_bytes_array_to_strings( + data[param + '-Derivation'][()]) for param in keys_not_derivations}).T + df_drvs.columns = ['Derivation'] * nCols df = pd.concat([df_keys, df_drvs], axis=1) # Add units as first col - units_dict = {key:data[key].attrs['units'].astype(str) for key in list_of_keys} + units_dict = {key: data[key].attrs['units'].astype(str) for key in data} df.insert(loc=0, column='(units)', value=pd.Series(units_dict)) return df - ######################################################################## # ## Get event histories of MT data, SN data, and combined MT, SN data @@ -109,10 +137,10 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: Returns ------- - returned_seeds : list - an ordered list of the unique seeds in the mt_data file, + returned_seeds : list + an ordered list of the unique seeds in the mt_data file returned_events : list - a list of sublists, one sublist per seed, where each sublist + a list of sublists, one sublist per seed, where each sublist contains all the MtEventTuples for the given seed returned_times : list a list of sublists of times of each of the mt_data events. @@ -135,9 +163,10 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: mt_is_mrg = mt_data['Merger'][()] == 1 # We want the return arrays sorted by seed, so sort here. - mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) # sort by seeds then times - lexsort sorts by the last column first... - mt_seeds = mt_seeds[mt_seeds_idx] - mt_times = mt_times[mt_seeds_idx] + # sort by seeds then times - lexsort sorts by the last column first... + mt_seeds_idx = np.lexsort((mt_times, mt_seeds)) + mt_seeds = mt_seeds[mt_seeds_idx] + mt_times = mt_times[mt_seeds_idx] mt_primary_stype = mt_primary_stype[mt_seeds_idx] mt_secondary_stype = mt_secondary_stype[mt_seeds_idx] mt_is_rlof1 = mt_is_rlof1[mt_seeds_idx] @@ -146,33 +175,56 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: mt_is_mrg = mt_is_mrg[mt_seeds_idx] # Process the mt_data events - returned_seeds = [] # array of seeds - will only contain seeds that have mt_data events - returned_events = [] # array of mt_data events for each seed in returned_seeds - returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) - - last_seed = -1 # initialize most recently processed seed - for seed_index, this_seed in enumerate(mt_seeds): # iterate over all RLOF file entries - this_time = mt_times[seed_index] # time for this RLOF file entry - this_event = (mt_primary_stype[seed_index], mt_secondary_stype[seed_index], - mt_is_rlof1[seed_index], mt_is_rlof2[seed_index], mt_is_cee[seed_index], mt_is_mrg[seed_index]) # construct event tuple + # array of seeds - will only contain seeds that have mt_data events + returned_seeds = [] + # array of mt_data events for each seed in returned_seeds + returned_events = [] + # array of times for each event in returned_events (for each seed in + # returned_seeds) + returned_times = [] + + # initialize most recently processed seed + last_seed = -1 + for seed_index, this_seed in enumerate( + mt_seeds): # iterate over all RLOF file entries + # time for this RLOF file entry + this_time = mt_times[seed_index] + this_event = ( + mt_primary_stype[seed_index], + mt_secondary_stype[seed_index], + mt_is_rlof1[seed_index], + mt_is_rlof2[seed_index], + mt_is_cee[seed_index], + mt_is_mrg[seed_index]) # construct event tuple # If this is an entirely new seed: if this_seed != last_seed: # same seed as last seed processed? - returned_seeds.append(this_seed) # no - new seed, record it - returned_times.append([this_time]) # initialize the list of event times for this seed - returned_events.append([this_event]) # initialize the list of events for this seed + # no - new seed, record it + returned_seeds.append(this_seed) + # initialize the list of event times for this seed + returned_times.append([this_time]) + # initialize the list of events for this seed + returned_events.append([this_event]) last_seed = this_seed # update the latest seed # Add event, if it is not a duplicate try: - event_index = returned_events[-1].index(this_event) # find event_index of this particular event tuple in the array of events for this seed - if this_time > returned_times[-1][event_index]: # ^ if event is not a duplicate, this will throw a ValueError - returned_times[-1][event_index] = this_time # if event is duplicate, update time to the later of the duplicates + # find event_index of this particular event tuple in the array of + # events for this seed + event_index = returned_events[-1].index(this_event) + # ^ if event is not a duplicate, this will throw a ValueError + if this_time > returned_times[-1][event_index]: + # if event is duplicate, update time to the later of the + # duplicates + returned_times[-1][event_index] = this_time except ValueError: # event is not a duplicate: - returned_events[-1].append(this_event) # record new event tuple for this seed - returned_times[-1].append(this_time) # record new event time for this seed + # record new event tuple for this seed + returned_events[-1].append(this_event) + # record new event time for this seed + returned_times[-1].append(this_time) - return returned_seeds, returned_events, returned_times # see above for description + # see above for description + return returned_seeds, returned_events, returned_times def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: @@ -185,10 +237,10 @@ def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: Returns ------- - returned_seeds : list - an ordered list of the unique seeds in the sn_data file, + returned_seeds : list + an ordered list of the unique seeds in the sn_data file, returned_events : list - a list of sublists, one sublist per seed, where each sublist + a list of sublists, one sublist per seed, where each sublist contains all the SnEventTuples for the given seed returned_times : list a list of sublists of times of each of the sn_data events. @@ -208,123 +260,168 @@ def get_sn_data_tuple(sn_data: H5Group) -> tuple[list, list, list]: sn_is_unbound = sn_data['Unbound'][()] == 1 # We want the return arrays sorted by seed, so sort here. - sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) # sort by seeds then times - lexsort sorts by the last column first... - sn_seeds = sn_seeds[sn_seeds_idx] - sn_times = sn_times[sn_seeds_idx] + # sort by seeds then times - lexsort sorts by the last column first... + sn_seeds_idx = np.lexsort((sn_times, sn_seeds)) + sn_seeds = sn_seeds[sn_seeds_idx] + sn_times = sn_times[sn_seeds_idx] sn_prog_stype = sn_prog_stype[sn_seeds_idx] sn_remn_stype = sn_remn_stype[sn_seeds_idx] sn_which_prog = sn_which_prog[sn_seeds_idx] sn_is_unbound = sn_is_unbound[sn_seeds_idx] - + # Process the sn_data events - returned_seeds = [] # array of seeds - will only contain seeds that have sn_data events - returned_events = [] # array of sn_data events for each seed in returned_seeds - returned_times = [] # array of times for each event in returned_events (for each seed in returned_seeds) - - last_seed = -1 # initialize most recently processed seed - for seed_index, this_seed in enumerate(sn_seeds): # iterate over all sn_data file entries - this_time = sn_times[seed_index] # time for this sn_data file entry - this_event = (sn_prog_stype[seed_index], sn_remn_stype[seed_index], - sn_which_prog[seed_index], sn_is_unbound[seed_index]) # construct event tuple - + # array of seeds - will only contain seeds that have sn_data events + returned_seeds = [] + # array of sn_data events for each seed in returned_seeds + returned_events = [] + # array of times for each event in returned_events (for each seed in + # returned_seeds) + returned_times = [] + + # initialize most recently processed seed + last_seed = -1 + for seed_index, this_seed in enumerate( + sn_seeds): # iterate over all sn_data file entries + # time for this sn_data file entry + this_time = sn_times[seed_index] + this_event = ( + sn_prog_stype[seed_index], + sn_remn_stype[seed_index], + sn_which_prog[seed_index], + sn_is_unbound[seed_index]) # construct event tuple + # If this is an entirely new seed: if this_seed != last_seed: # same seed as last seed processed? - returned_seeds.append(this_seed) # no - new seed, record it - returned_times.append([this_time]) # initialize the list of event times for this seed - returned_events.append([this_event]) # initialize the list of events for this seed - last_seed = this_seed # update the latest seed + # no - new seed, record it + returned_seeds.append(this_seed) + # initialize the list of event times for this seed + returned_times.append([this_time]) + # initialize the list of events for this seed + returned_events.append([this_event]) + last_seed = this_seed # update the latest seed else: # yes - second sn_data event for this seed - returned_times[-1].append(this_time) # append time at end of array - returned_events[-1].append(this_event) # append event at end of array - - return returned_seeds, returned_events, returned_times # see above for description + returned_times[-1].append(this_time) # append time at end of array + # append event at end of array + returned_events[-1].append(this_event) + # see above for description + return returned_seeds, returned_events, returned_times -def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list]: + +def get_event_history( + data: H5File, include_null: bool = True) -> tuple[list, list]: """Get the event history for all seeds, including both MT and SN events, in chronological order. Parameters ---------- data : H5File - the COMPAS output H5file + the COMPAS output H5file include_null : bool whether to include seeds which undergo no MT or SN events (default is True) - Returns + Returns ------- returned_seeds : list an ordered list of all the unique seeds in the output file - returned_events : list + returned_events : list a list where each element is a chronological set of events corresponding to the associated seed """ sp_data = data['BSE_System_Parameters'] mt_data = data['BSE_RLOF'] sn_data = data['BSE_Supernovae'] - all_seeds = sp_data['SEED'][()] # get all seeds - mt_seeds, mt_events, mt_times = get_mt_data_tuple(mt_data) # get mt data tuple - sn_seeds, sn_events, sn_times = get_sn_data_tuple(sn_data) # get sn data tuple - - num_mt_seeds = len(mt_seeds) # number of mt_data events - num_sn_seeds = len(sn_seeds) # number of sn_data events - - if num_mt_seeds < 1 and num_sn_seeds < 1: return [] # no events - return empty history - - returned_seeds = [] # array of seeds - will only contain seeds that have events (of any type) - returned_events = [] # array of events - same size as returned_seeds (includes event times) - - event_ordering = ['RL', 'CE', 'SN', 'MG'] # order of preference for simultaneous events - - mt_index = 0 # index into mt_data events arrays - sn_index = 0 # index into sn_data events arrays + # get all seeds + all_seeds = sp_data['SEED'][()] + mt_seeds, mt_events, mt_times = get_mt_data_tuple( + mt_data) # get mt data tuple + sn_seeds, sn_events, sn_times = get_sn_data_tuple( + sn_data) # get sn data tuple + + # number of mt_data events + num_mt_seeds = len(mt_seeds) + # number of sn_data events + num_sn_seeds = len(sn_seeds) + + if num_mt_seeds < 1 and num_sn_seeds < 1: + return [] # no events - return empty history + + # array of seeds - will only contain seeds that have events (of any type) + returned_seeds = [] + # array of events - same size as returned_seeds (includes event times) + returned_events = [] + + # order of preference for simultaneous events + event_ordering = ['RL', 'CE', 'SN', 'MG'] + + # index into mt_data events arrays + mt_index = 0 + # index into sn_data events arrays + sn_index = 0 if include_null: seeds_to_iterate = all_seeds else: - seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) # iterate over all the seeds that have either mt_data or sn_data events - + # iterate over all the seeds that have either mt_data or sn_data events + seeds_to_iterate = np.sort(np.unique(np.append(mt_seeds, sn_seeds))) + idx_ordered = np.argsort(seeds_to_iterate) - returned_seeds = [None] * np.size(seeds_to_iterate) # array of seeds - will only contain seeds that have events (of any type) - returned_events = [None] * np.size(seeds_to_iterate) # array of events - same size as returned_seeds (includes event times) + # array of seeds - will only contain seeds that have events (of any type) + returned_seeds = [None] * np.size(seeds_to_iterate) + # array of events - same size as returned_seeds (includes event times) + returned_events = [None] * np.size(seeds_to_iterate) for idx in idx_ordered: seed = seeds_to_iterate[idx] - seed_events = [] # initialise the events for the seed being processed + # initialise the events for the seed being processed + seed_events = [] - # Collect any mt_data events for this seed, add the time of the event and the event type + # Collect any mt_data events for this seed, add the time of the event + # and the event type while mt_index < num_mt_seeds and mt_seeds[mt_index] == seed: for event_index, event in enumerate(mt_events[mt_index]): _, _, is_rl1, is_rl2, is_cee, is_mrg = event - event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 - seed_events.append((event_key, mt_times[mt_index][event_index], *mt_events[mt_index][event_index])) + # event type: Merger, CEE, RLOF 1->2, RLOF 2->1 + event_key = 'MG' if is_mrg else 'CE' if is_cee else 'RL' + seed_events.append( + (event_key, + mt_times[mt_index][event_index], + *mt_events[mt_index][event_index])) mt_index += 1 - # Collect any sn_data events for this seed, add the time of the event and the event type + # Collect any sn_data events for this seed, add the time of the event + # and the event type while sn_index < num_sn_seeds and sn_seeds[sn_index] == seed: for event_index, event in enumerate(sn_events[sn_index]): - event_key = 'SN' - seed_events.append((event_key, sn_times[sn_index][event_index], *sn_events[sn_index][event_index])) + event_key = 'SN' + seed_events.append( + (event_key, + sn_times[sn_index][event_index], + *sn_events[sn_index][event_index])) sn_index += 1 - seed_events.sort(key=lambda ev:(ev[1], event_ordering.index(ev[0]))) # sort the events by time and event type (mt_data before sn_data if at the same time) - returned_seeds[idx] = seed # record the seed in the seeds array being returned - returned_events[idx] = seed_events # record the events for this seed in the events array being returned - return returned_seeds, returned_events + # sort the events by time and event type (mt_data before sn_data if at + # the same time) + seed_events.sort(key=lambda ev: (ev[1], event_ordering.index(ev[0]))) + # record the seed in the seeds array being returned + returned_seeds[idx] = seed + # record the events for this seed in the events array being returned + returned_events[idx] = seed_events + return returned_seeds, returned_events ########################################### # ## Produce strings of the event histories -stellar_type_dict = { - 0: 'MS', - 1: 'MS', - 2: 'HG', - 3: 'GB', - 4: 'GB', - 5: 'GB', - 6: 'GB', - 7: 'HE', - 8: 'HE', - 9: 'HE', +simplified_stellar_type_dict = { + 0: 'MS', + 1: 'MS', + 2: 'HG', + 3: 'GB', + 4: 'GB', + 5: 'GB', + 6: 'GB', + 7: 'HE', + 8: 'HE', + 9: 'HE', 10: 'WD', 11: 'WD', 12: 'WD', @@ -335,7 +432,9 @@ def get_event_history(data: H5File, include_null: bool=True) -> tuple[list, list } -def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistoryString: +def build_event_string( + events: list, + use_int_stypes: bool = False) -> EventHistoryString: """Produce a string representation of the event history of a single binary. Parameters @@ -343,27 +442,27 @@ def build_event_string(events: list, use_int_stypes: bool=False) -> EventHistory events : list list of mixed MtEventTuple and SnEventTuple's use_int_stypes : bool, optional - whether to report stellar types as integers + whether to report stellar types as integers (default is False, report them as 2-char stellar types) - Returns + Returns ------- event_str : EventHistoryString condensed string representation of the event history of a binary, in array form Notes ----- - MT strings look like: - P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) + MT strings look like: + P>S, P', '<' is RLOF (star1 -> star2 or star1 <- star2) or otherwise '=' for CEE or '&' for Merger. SN strings look like: - P*SR if star1 is the SN progenitor, or + P*SR if star1 is the SN progenitor, or R*SP if star2 is the SN progenitor, - where P is progenitor type, R is remnant type, + where P is progenitor type, R is remnant type, S is state ('i' for intact, 'u' for unbound) Event strings for the same seed are separated by an undesrcore '_' - + TODO ---- Currently unvectorized, do the vectorization later for a speed up @@ -372,44 +471,50 @@ def _remap_stype(int_stype): if use_int_stypes: return str(int_stype) else: - return stellar_type_dict[int_stype] - + return simplified_stellar_type_dict[int_stype] + # Empty event if len(events) == 0: return 'NA' - + event_str = '' for event in events: if event[0] == 'SN': # SN event _, time, stype_p, stype_c, which_sn, is_unbound = event - if which_sn == 1: # Progenitor or Remnant depending upon which star is the SN + # Progenitor or Remnant depending upon which star is the SN + if which_sn == 1: char_l = _remap_stype(stype_p) char_r = _remap_stype(stype_c) else: char_l = _remap_stype(stype_c) char_r = _remap_stype(stype_p) char_m = '*u' if is_unbound else '*i' # unbound or intact - else: # Any of the MT events + else: # Any of the MT events _, time, stype_1, stype_2, is_rl1, is_rl2, is_cee, is_mrg = event - char_l = _remap_stype(stype_1) # primary stellar type - char_r = _remap_stype(stype_2) # secondary stellar type - char_m = '&' if is_mrg \ - else '=' if is_cee \ - else '<' if is_rl2 \ - else '>' # event type: CEE, RLOF 2->1, RLOF 1->2 - event_str += "{}{}{}_".format(char_l, char_m, char_r) # event string for this star, _ is event separator - event_str = np.array(event_str[:-1], dtype=np.str_) # return event string for this star (pop the last underscore first) + # primary stellar type + char_l = _remap_stype(stype_1) + # secondary stellar type + char_r = _remap_stype(stype_2) + # event type: CEE, RLOF 2->1, RLOF 1->2 + char_m = '&' if is_mrg else '=' if is_cee else '<' if is_rl2 else '>' + # event string for this star, _ is event separator + event_str += "{}{}{}_".format(char_l, char_m, char_r) + # return event string for this star (pop the last underscore first) + event_str = np.array(event_str[:-1], dtype=np.str_) return event_str -def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: bool=False): - """Calculate the event history strings for a COMPAS population. +def get_event_strings( + data: H5File = None, + all_events: list = None, + use_int_stypes: bool = False): + """Calculate the event history strings for a COMPAS population. Parameters ---------- data : H5File, optional - the COMPAS output H5file - all_events : list, optional + the COMPAS output H5file + all_events : list, optional a list where each element is a chronological set of events corresponding to the associated seed, one of the outputs of get_event_history(data) use_int_stypes : bool, optional @@ -421,20 +526,23 @@ def get_event_strings(data: H5File=None, all_events: list=None, use_int_stypes: Notes ----- - Exactly one of data or all_events must be included. If neither, nothign is returned. + Exactly one of data or all_events must be included. If neither, nothing is returned. """ - # If output is - if (data == None) & (all_events == None): - return - elif (all_events == None): + # If output is + if (data is None) & (all_events is None): + return + elif (all_events is None): _, all_events = get_event_history(data) event_strings = np.zeros(len(all_events), dtype=StringDType()) - for ii, events_for_given_seed in enumerate(all_events): - event_string = build_event_string(events_for_given_seed, use_int_stypes=use_int_stypes) - event_strings[ii] = event_string # append event string for this star (pop the last underscore first) + for ii, events_for_given_seed in enumerate(all_events): + event_string = build_event_string( + events_for_given_seed, + use_int_stypes=use_int_stypes) + # append event string for this star (pop the last underscore first) + event_strings[ii] = event_string return event_strings -"" +"" From 4625d966f30a843198d9e4b2561f8092fe5a773a Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 29 Jun 2026 12:13:42 +0200 Subject: [PATCH 19/22] starting the process of adding unit testing --- compas_python_utils/debugging_utils.py | 14 +++++++------- setup.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py index 15a9826a2..4047d10f8 100644 --- a/compas_python_utils/debugging_utils.py +++ b/compas_python_utils/debugging_utils.py @@ -4,8 +4,7 @@ from numpy.dtypes import StringDType from typing import NewType -"" -# New Types +### New Types MaskNdarray = NewType('MaskNdarray', np.ndarray[bool]) H5File = NewType('H5File', h5._hl.files.File) H5Group = NewType('H5Group', h5._hl.group.Group) @@ -24,8 +23,7 @@ EventHistoryString = NewType('EventHistoryString', np.array(np.str_)) -######################################################################## -# ## Function to print the data from a given COMPAS HDF5 group in a readable pandas template +### Function to print the data from a given COMPAS HDF5 group in a readable pandas template def convert_bytes_array_to_strings(param_array): """Check and convert np.bytes_ array to strings. @@ -124,8 +122,7 @@ def print_compas_details_dataframe(data: H5Group, return df -######################################################################## -# ## Get event histories of MT data, SN data, and combined MT, SN data +### Get event histories of MT data, SN data, and combined MT, SN data def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: """Calculates the EventTuple for the BSE_RLOF output H5Group. @@ -544,5 +541,8 @@ def get_event_strings( event_strings[ii] = event_string return event_strings +def main(): + return -"" +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 928f0df5d..f37f891ff 100644 --- a/setup.py +++ b/setup.py @@ -164,6 +164,7 @@ def find_version(version_file=read(CPP_VERSION_FILE)): f"compas_h5view= {NAME}.h5view:main", f"compas_h5copy= {NAME}.h5copy:main", f"compas_h5sample= {NAME}.h5sample:main", + f"compas_h5sample= {NAME}.debugging_utils:main", f"compas_plot_detailed_evolution={NAME}.detailed_evolution_plotter.plot_detailed_evolution:main", f"compas_run={NAME}.compas_runner:main", f"compas_run_submit={NAME}.preprocessing.runSubmit:main", From 5f5b0fa32076671757681b05426126be4103eac4 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Tue, 30 Jun 2026 10:10:09 +0200 Subject: [PATCH 20/22] minor textual fixes --- compas_python_utils/debugging_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compas_python_utils/debugging_utils.py b/compas_python_utils/debugging_utils.py index 4047d10f8..9f8fbb431 100644 --- a/compas_python_utils/debugging_utils.py +++ b/compas_python_utils/debugging_utils.py @@ -140,7 +140,7 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: a list of sublists, one sublist per seed, where each sublist contains all the MtEventTuples for the given seed returned_times : list - a list of sublists of times of each of the mt_data events. + a list of sublists of times of each of the mt_data events Notes ----- @@ -150,7 +150,6 @@ def get_mt_data_tuple(mt_data: H5Group) -> tuple[list, list, list]: it is just a feeder for get_event_history below. """ mt_seeds = mt_data['SEED'][()] - mt_times = mt_data['Time tuple[list, list, list]: Returns ------- returned_seeds : list - an ordered list of the unique seeds in the sn_data file, + an ordered list of the unique seeds in the sn_data file returned_events : list a list of sublists, one sublist per seed, where each sublist contains all the SnEventTuples for the given seed returned_times : list - a list of sublists of times of each of the sn_data events. + a list of sublists of times of each of the sn_data events Notes ----- From dfcf1efedf087aaf4ebea276bf9f3ac3532d6a65 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 6 Jul 2026 16:51:33 +0200 Subject: [PATCH 21/22] Remove duplicate compas_h5sample entry point from setup.py. The debugging_utils module is library-only and its entry point was overwriting the real compas_h5sample CLI. Co-authored-by: Cursor --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f37f891ff..928f0df5d 100644 --- a/setup.py +++ b/setup.py @@ -164,7 +164,6 @@ def find_version(version_file=read(CPP_VERSION_FILE)): f"compas_h5view= {NAME}.h5view:main", f"compas_h5copy= {NAME}.h5copy:main", f"compas_h5sample= {NAME}.h5sample:main", - f"compas_h5sample= {NAME}.debugging_utils:main", f"compas_plot_detailed_evolution={NAME}.detailed_evolution_plotter.plot_detailed_evolution:main", f"compas_run={NAME}.compas_runner:main", f"compas_run_submit={NAME}.preprocessing.runSubmit:main", From f5a6dd291d0cc6c0d16045cff000d5c97ac080b0 Mon Sep 17 00:00:00 2001 From: Reinhold Willcox Date: Mon, 6 Jul 2026 16:55:20 +0200 Subject: [PATCH 22/22] removed line from setup.py that was causing issues --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index f37f891ff..928f0df5d 100644 --- a/setup.py +++ b/setup.py @@ -164,7 +164,6 @@ def find_version(version_file=read(CPP_VERSION_FILE)): f"compas_h5view= {NAME}.h5view:main", f"compas_h5copy= {NAME}.h5copy:main", f"compas_h5sample= {NAME}.h5sample:main", - f"compas_h5sample= {NAME}.debugging_utils:main", f"compas_plot_detailed_evolution={NAME}.detailed_evolution_plotter.plot_detailed_evolution:main", f"compas_run={NAME}.compas_runner:main", f"compas_run_submit={NAME}.preprocessing.runSubmit:main",