diff --git a/lib/entry-points.js b/lib/entry-points.js index dd77444e02..37cb3b3c8c 100644 --- a/lib/entry-points.js +++ b/lib/entry-points.js @@ -147382,9 +147382,34 @@ var featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN", minimumVersion: void 0 }, - ["overlay_analysis_resource_checks_v2" /* OverlayAnalysisResourceChecksV2 */]: { + ["overlay_analysis_min_disk_8_gb" /* OverlayAnalysisMinDisk8Gb */]: { defaultValue: false, - envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2", + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_9_gb" /* OverlayAnalysisMinDisk9Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_10_gb" /* OverlayAnalysisMinDisk10Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_11_gb" /* OverlayAnalysisMinDisk11Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_12_gb" /* OverlayAnalysisMinDisk12Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB", + minimumVersion: void 0 + }, + ["overlay_analysis_min_disk_13_gb" /* OverlayAnalysisMinDisk13Gb */]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB", minimumVersion: void 0 }, ["overlay_analysis_status_check" /* OverlayAnalysisStatusCheck */]: { @@ -149617,10 +149642,15 @@ async function cachePrefix(codeql, language) { } // src/config-utils.ts -var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 2e4; -var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1e6; -var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14e3; -var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1e6; +var OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14e3; +var OVERLAY_MINIMUM_DISK_SPACE_FEATURES = [ + ["overlay_analysis_min_disk_8_gb" /* OverlayAnalysisMinDisk8Gb */, 8e3], + ["overlay_analysis_min_disk_9_gb" /* OverlayAnalysisMinDisk9Gb */, 9e3], + ["overlay_analysis_min_disk_10_gb" /* OverlayAnalysisMinDisk10Gb */, 1e4], + ["overlay_analysis_min_disk_11_gb" /* OverlayAnalysisMinDisk11Gb */, 11e3], + ["overlay_analysis_min_disk_12_gb" /* OverlayAnalysisMinDisk12Gb */, 12e3], + ["overlay_analysis_min_disk_13_gb" /* OverlayAnalysisMinDisk13Gb */, 13e3] +]; var OVERLAY_MINIMUM_MEMORY_MB = 5 * 1024; var CODEQL_VERSION_REDUCED_OVERLAY_MEMORY_USAGE = "2.24.3"; async function getSupportedLanguageMap(codeql, logger) { @@ -149870,16 +149900,26 @@ async function checkOverlayAnalysisFeatureEnabled(features, codeql, languages, c } return new Success(void 0); } -function runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks) { - const minimumDiskSpaceBytes = useV2ResourceChecks ? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES : OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES; - if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) { - const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1e6); - const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1e6); +async function getMinimumDiskSpaceMb(features) { + let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB; + for (const [feature, thresholdMb] of OVERLAY_MINIMUM_DISK_SPACE_FEATURES) { + if (await features.getValue(feature)) { + minimumMb = Math.min(minimumMb, thresholdMb); + } + } + return minimumMb; +} +function runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb) { + const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1e6); + if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1e6) { logger.info( `Setting overlay database mode to ${"none" /* None */} due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).` ); return false; } + logger.debug( + `Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the minimum of ${minimumDiskSpaceMb} MB.` + ); return true; } async function runnerHasSufficientMemory(codeql, ramInput, logger) { @@ -149900,12 +149940,13 @@ async function runnerHasSufficientMemory(codeql, ramInput, logger) { return false; } logger.debug( - `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.` + `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.` ); return true; } -async function checkRunnerResources(codeql, diskUsage, ramInput, logger, useV2ResourceChecks) { - if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) { +async function checkRunnerResources(codeql, features, diskUsage, ramInput, logger) { + const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features); + if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) { return new Failure("insufficient-disk-space" /* InsufficientDiskSpace */); } if (!await runnerHasSufficientMemory(codeql, ramInput, logger)) { @@ -149953,9 +149994,6 @@ async function checkOverlayEnablement(codeql, features, languages, sourceRoot, b "overlay_analysis_skip_resource_checks" /* OverlayAnalysisSkipResourceChecks */, codeql ); - const useV2ResourceChecks = await features.getValue( - "overlay_analysis_resource_checks_v2" /* OverlayAnalysisResourceChecksV2 */ - ); const checkOverlayStatus = await features.getValue( "overlay_analysis_status_check" /* OverlayAnalysisStatusCheck */ ); @@ -149969,10 +150007,10 @@ async function checkOverlayEnablement(codeql, features, languages, sourceRoot, b } const resourceResult = performResourceChecks && diskUsage !== void 0 ? await checkRunnerResources( codeql, + features, diskUsage, ramInput, - logger, - useV2ResourceChecks + logger ) : new Success(void 0); if (resourceResult.isFailure()) { return resourceResult; diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index 84c709e72a..bedf219efd 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -1295,17 +1295,36 @@ checkOverlayEnablementMacro.serial( ); checkOverlayEnablementMacro.serial( - "No overlay-base database on default branch if runner disk space is below v2 limit and v2 resource checks enabled", + "Overlay-base database on default branch if runner disk space is above the default limit", { languages: [BuiltInLanguage.javascript], features: [ Feature.OverlayAnalysis, Feature.OverlayAnalysisCodeScanningJavascript, - Feature.OverlayAnalysisResourceChecksV2, ], isDefaultBranch: true, diskUsage: { - numAvailableBytes: 5_000_000_000, + numAvailableBytes: 15_000_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + useOverlayDatabaseCaching: true, + }, +); + +checkOverlayEnablementMacro.serial( + "No overlay-base database on default branch if runner disk space is below the default limit", + { + languages: [BuiltInLanguage.javascript], + features: [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisCodeScanningJavascript, + ], + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: 10_000_000_000, numTotalBytes: 100_000_000_000, }, }, @@ -1314,18 +1333,71 @@ checkOverlayEnablementMacro.serial( }, ); +// Check that each feature flag lowers the limit to the threshold that its name +// declares. Both sides of the boundary are needed to pin the threshold down: a +// mapping to a lower value would still pass the case at the limit, and one to a +// higher value would still fail the case below it. +for (const [feature, thresholdGb] of [ + [Feature.OverlayAnalysisMinDisk8Gb, 8], + [Feature.OverlayAnalysisMinDisk9Gb, 9], + [Feature.OverlayAnalysisMinDisk10Gb, 10], + [Feature.OverlayAnalysisMinDisk11Gb, 11], + [Feature.OverlayAnalysisMinDisk12Gb, 12], + [Feature.OverlayAnalysisMinDisk13Gb, 13], +] as Array<[Feature, number]>) { + const features = [ + Feature.OverlayAnalysis, + Feature.OverlayAnalysisCodeScanningJavascript, + feature, + ]; + + checkOverlayEnablementMacro.serial( + `Overlay-base database on default branch if ${feature} is enabled and runner disk space is at its limit`, + { + languages: [BuiltInLanguage.javascript], + features, + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: thresholdGb * 1_000_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + overlayDatabaseMode: OverlayDatabaseMode.OverlayBase, + useOverlayDatabaseCaching: true, + }, + ); + + checkOverlayEnablementMacro.serial( + `No overlay-base database on default branch if ${feature} is enabled and runner disk space is below its limit`, + { + languages: [BuiltInLanguage.javascript], + features, + isDefaultBranch: true, + diskUsage: { + numAvailableBytes: thresholdGb * 1_000_000_000 - 1_000_000, + numTotalBytes: 100_000_000_000, + }, + }, + { + disabledReason: OverlayDisabledReason.InsufficientDiskSpace, + }, + ); +} + checkOverlayEnablementMacro.serial( - "Overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks enabled", + "Overlay-base database on default branch if runner disk space is exactly at the lowest limit enabled by a feature flag", { languages: [BuiltInLanguage.javascript], features: [ Feature.OverlayAnalysis, Feature.OverlayAnalysisCodeScanningJavascript, - Feature.OverlayAnalysisResourceChecksV2, + Feature.OverlayAnalysisMinDisk9Gb, + Feature.OverlayAnalysisMinDisk12Gb, ], isDefaultBranch: true, diskUsage: { - numAvailableBytes: 15_000_000_000, + numAvailableBytes: 9_000_000_000, numTotalBytes: 100_000_000_000, }, }, @@ -1336,16 +1408,18 @@ checkOverlayEnablementMacro.serial( ); checkOverlayEnablementMacro.serial( - "No overlay-base database on default branch if runner disk space is between v2 and v1 limits and v2 resource checks not enabled", + "No overlay-base database on default branch if runner disk space is below the lowest limit enabled by a feature flag", { languages: [BuiltInLanguage.javascript], features: [ Feature.OverlayAnalysis, Feature.OverlayAnalysisCodeScanningJavascript, + Feature.OverlayAnalysisMinDisk9Gb, + Feature.OverlayAnalysisMinDisk12Gb, ], isDefaultBranch: true, diskUsage: { - numAvailableBytes: 15_000_000_000, + numAvailableBytes: 8_500_000_000, numTotalBytes: 100_000_000_000, }, }, diff --git a/src/config-utils.ts b/src/config-utils.ts index b5a880ba7b..6b9c41e3b4 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -48,7 +48,7 @@ import { import { prepareDiffInformedAnalysis } from "./diff-informed-analysis-utils"; import { EnvVar } from "./environment"; import * as errorMessages from "./error-messages"; -import { Feature, FeatureEnablement } from "./feature-flags"; +import { Feature, FeatureEnablement, FeatureWithoutCLI } from "./feature-flags"; import { RepositoryProperties, RepositoryPropertyName, @@ -101,19 +101,28 @@ export { type Config } from "./config/action-config"; * whether to perform overlay analysis, then the action will not perform overlay * analysis unless overlay analysis has been explicitly enabled via environment * variable. + * + * This threshold can be lowered by the feature flags in + * `OVERLAY_MINIMUM_DISK_SPACE_FEATURES`. */ -const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 20000; -const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES = - OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB * 1_000_000; +const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB = 14000; /** - * The v2 minimum available disk space (in MB) required to perform overlay - * analysis. This is a lower threshold than the v1 limit, allowing overlay - * analysis to run on runners with less available disk space. + * Feature flags that lower the minimum available disk space required to perform + * overlay analysis, paired with the threshold (in MB) that each one enables. + * + * If several of these are enabled, the lowest threshold takes effect. */ -const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB = 14000; -const OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES = - OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_MB * 1_000_000; +const OVERLAY_MINIMUM_DISK_SPACE_FEATURES: ReadonlyArray< + [FeatureWithoutCLI, number] +> = [ + [Feature.OverlayAnalysisMinDisk8Gb, 8000], + [Feature.OverlayAnalysisMinDisk9Gb, 9000], + [Feature.OverlayAnalysisMinDisk10Gb, 10000], + [Feature.OverlayAnalysisMinDisk11Gb, 11000], + [Feature.OverlayAnalysisMinDisk12Gb, 12000], + [Feature.OverlayAnalysisMinDisk13Gb, 13000], +]; /** * The minimum memory (in MB) that must be available for CodeQL to perform overlay analysis. If @@ -588,24 +597,42 @@ async function checkOverlayAnalysisFeatureEnabled( return new Success(undefined); } +/** + * Returns the minimum available disk space (in MB) required to perform overlay + * analysis, which is the lowest threshold enabled by a feature flag, or the + * default threshold if no such feature flag is enabled. + */ +async function getMinimumDiskSpaceMb( + features: FeatureEnablement, +): Promise { + let minimumMb = OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_MB; + for (const [feature, thresholdMb] of OVERLAY_MINIMUM_DISK_SPACE_FEATURES) { + if (await features.getValue(feature)) { + minimumMb = Math.min(minimumMb, thresholdMb); + } + } + return minimumMb; +} + /** Checks if the runner has enough disk space for overlay analysis. */ function runnerHasSufficientDiskSpace( diskUsage: DiskUsage, logger: Logger, - useV2ResourceChecks: boolean, + minimumDiskSpaceMb: number, ): boolean { - const minimumDiskSpaceBytes = useV2ResourceChecks - ? OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_V2_BYTES - : OVERLAY_MINIMUM_AVAILABLE_DISK_SPACE_BYTES; - if (diskUsage.numAvailableBytes < minimumDiskSpaceBytes) { - const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000); - const minimumDiskSpaceMb = Math.round(minimumDiskSpaceBytes / 1_000_000); + const diskSpaceMb = Math.round(diskUsage.numAvailableBytes / 1_000_000); + if (diskUsage.numAvailableBytes < minimumDiskSpaceMb * 1_000_000) { logger.info( `Setting overlay database mode to ${OverlayDatabaseMode.None} ` + `due to insufficient disk space (${diskSpaceMb} MB, needed ${minimumDiskSpaceMb} MB).`, ); return false; } + + logger.debug( + `Disk space available for CodeQL analysis is ${diskSpaceMb} MB, which is at or above the ` + + `minimum of ${minimumDiskSpaceMb} MB.`, + ); return true; } @@ -637,7 +664,7 @@ async function runnerHasSufficientMemory( } logger.debug( - `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`, + `Memory available for CodeQL analysis is ${memoryFlagValue} MB, which is at or above the minimum of ${OVERLAY_MINIMUM_MEMORY_MB} MB.`, ); return true; } @@ -648,12 +675,13 @@ async function runnerHasSufficientMemory( */ async function checkRunnerResources( codeql: CodeQL, + features: FeatureEnablement, diskUsage: DiskUsage, ramInput: string | undefined, logger: Logger, - useV2ResourceChecks: boolean, ): Promise> { - if (!runnerHasSufficientDiskSpace(diskUsage, logger, useV2ResourceChecks)) { + const minimumDiskSpaceMb = await getMinimumDiskSpaceMb(features); + if (!runnerHasSufficientDiskSpace(diskUsage, logger, minimumDiskSpaceMb)) { return new Failure(OverlayDisabledReason.InsufficientDiskSpace); } if (!(await runnerHasSufficientMemory(codeql, ramInput, logger))) { @@ -752,9 +780,6 @@ export async function checkOverlayEnablement( Feature.OverlayAnalysisSkipResourceChecks, codeql, )); - const useV2ResourceChecks = await features.getValue( - Feature.OverlayAnalysisResourceChecksV2, - ); const checkOverlayStatus = await features.getValue( Feature.OverlayAnalysisStatusCheck, ); @@ -770,10 +795,10 @@ export async function checkOverlayEnablement( performResourceChecks && diskUsage !== undefined ? await checkRunnerResources( codeql, + features, diskUsage, ramInput, logger, - useV2ResourceChecks, ) : new Success(undefined); if (resourceResult.isFailure()) { diff --git a/src/feature-flags.ts b/src/feature-flags.ts index b3107af962..66532cd850 100644 --- a/src/feature-flags.ts +++ b/src/feature-flags.ts @@ -121,12 +121,19 @@ export enum Feature { * `OverlayAnalysisMatchCodeqlVersion` overrides this flag. */ OverlayAnalysisMatchCodeqlVersionDryRun = "overlay_analysis_match_codeql_version_dry_run", - OverlayAnalysisPython = "overlay_analysis_python", /** - * Controls whether lower disk space requirements are used for overlay hardware checks. - * Has no effect if `OverlayAnalysisSkipResourceChecks` is enabled. + * Feature flags that lower the amount of available disk space that the overlay hardware check + * requires. The lowest threshold that is enabled takes effect; if none are enabled, the default + * threshold applies. These flags have no effect if `OverlayAnalysisSkipResourceChecks` is + * enabled. */ - OverlayAnalysisResourceChecksV2 = "overlay_analysis_resource_checks_v2", + OverlayAnalysisMinDisk8Gb = "overlay_analysis_min_disk_8_gb", + OverlayAnalysisMinDisk9Gb = "overlay_analysis_min_disk_9_gb", + OverlayAnalysisMinDisk10Gb = "overlay_analysis_min_disk_10_gb", + OverlayAnalysisMinDisk11Gb = "overlay_analysis_min_disk_11_gb", + OverlayAnalysisMinDisk12Gb = "overlay_analysis_min_disk_12_gb", + OverlayAnalysisMinDisk13Gb = "overlay_analysis_min_disk_13_gb", + OverlayAnalysisPython = "overlay_analysis_python", OverlayAnalysisRuby = "overlay_analysis_ruby", /** Controls whether hardware checks are skipped for overlay analysis. */ OverlayAnalysisSkipResourceChecks = "overlay_analysis_skip_resource_checks", @@ -354,9 +361,34 @@ export const featureConfig = { envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MATCH_CODEQL_VERSION_DRY_RUN", minimumVersion: undefined, }, - [Feature.OverlayAnalysisResourceChecksV2]: { + [Feature.OverlayAnalysisMinDisk8Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_8_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk9Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_9_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk10Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_10_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk11Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_11_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk12Gb]: { + defaultValue: false, + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_12_GB", + minimumVersion: undefined, + }, + [Feature.OverlayAnalysisMinDisk13Gb]: { defaultValue: false, - envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_RESOURCE_CHECKS_V2", + envVar: "CODEQL_ACTION_OVERLAY_ANALYSIS_MIN_DISK_13_GB", minimumVersion: undefined, }, [Feature.OverlayAnalysisStatusCheck]: {