From c88e34e38432474e861e4c4df74f53a3f1214214 Mon Sep 17 00:00:00 2001 From: Kevin Brightwell Date: Mon, 6 Jan 2025 10:32:10 -0500 Subject: [PATCH] MultiCI: Add class wrapping cache-key generation In order to inject environment content, this abstracts some free-functions into a single class to be injected. This has no functional changes, only moving code around. --- sources/src/caching/cache-key.ts | 121 +++++++++--------- sources/src/caching/caches.ts | 7 +- .../caching/gradle-home-extry-extractor.ts | 32 +++-- sources/src/caching/gradle-user-home-cache.ts | 16 ++- sources/test/jest/cache-debug.test.ts | 5 +- 5 files changed, 107 insertions(+), 74 deletions(-) diff --git a/sources/src/caching/cache-key.ts b/sources/src/caching/cache-key.ts index e4a42a3..c9bda66 100644 --- a/sources/src/caching/cache-key.ts +++ b/sources/src/caching/cache-key.ts @@ -27,74 +27,79 @@ export class CacheKey { } /** - * Generates a cache key specific to the current job execution. - * The key is constructed from the following inputs (with some user overrides): - * - The cache key prefix: defaults to 'gradle-' but can be overridden by the user - * - The cache protocol version - * - The runner operating system - * - The name of the workflow and Job being executed - * - The matrix values for the Job being executed (job context) - * - The SHA of the commit being executed - * - * Caches are restored by trying to match the these key prefixes in order: - * - The full key with SHA - * - A previous key for this Job + matrix - * - Any previous key for this Job (any matrix) - * - Any previous key for this cache on the current OS + * Provides generation fascilities for [CacheKey] values. */ -export function generateCacheKey(cacheName: string, config: CacheConfig): CacheKey { - const prefix = process.env[CACHE_KEY_PREFIX_VAR] || '' +export class CacheKeyGenerator { + /** + * Generates a cache key specific to the current job execution. + * The key is constructed from the following inputs (with some user overrides): + * - The cache key prefix: defaults to 'gradle-' but can be overridden by the user + * - The cache protocol version + * - The runner operating system + * - The name of the workflow and Job being executed + * - The matrix values for the Job being executed (job context) + * - The SHA of the commit being executed + * + * Caches are restored by trying to match the these key prefixes in order: + * - The full key with SHA + * - A previous key for this Job + matrix + * - Any previous key for this Job (any matrix) + * - Any previous key for this cache on the current OS + */ + generateCacheKey(cacheName: string, config: CacheConfig): CacheKey { + const prefix = process.env[CACHE_KEY_PREFIX_VAR] || '' - const cacheKeyBase = `${prefix}${getCacheKeyBase(cacheName, CACHE_PROTOCOL_VERSION)}` + const cacheKeyBase = `${prefix}${this.getCacheKeyBase(cacheName, CACHE_PROTOCOL_VERSION)}` - // At the most general level, share caches for all executions on the same OS - const cacheKeyForEnvironment = `${cacheKeyBase}|${getCacheKeyEnvironment()}` + // At the most general level, share caches for all executions on the same OS + const cacheKeyForEnvironment = `${cacheKeyBase}|${this.getCacheKeyEnvironment()}` - // Then prefer caches that run job with the same ID - const cacheKeyForJob = `${cacheKeyForEnvironment}|${getCacheKeyJob()}` + // Then prefer caches that run job with the same ID + const cacheKeyForJob = `${cacheKeyForEnvironment}|${this.getCacheKeyJob()}` - // Prefer (even more) jobs that run this job in the same workflow with the same context (matrix) - const cacheKeyForJobContext = `${cacheKeyForJob}[${getCacheKeyJobInstance()}]` + // Prefer (even more) jobs that run this job in the same workflow with the same context (matrix) + const cacheKeyForJobContext = `${cacheKeyForJob}[${this.getCacheKeyJobInstance()}]` - // Exact match on Git SHA - const cacheKey = `${cacheKeyForJobContext}-${getCacheKeyJobExecution()}` + // Exact match on Git SHA + const cacheKey = `${cacheKeyForJobContext}-${this.getCacheKeyJobExecution()}` - if (config.isCacheStrictMatch()) { - return new CacheKey(cacheKey, [cacheKeyForJobContext]) + if (config.isCacheStrictMatch()) { + return new CacheKey(cacheKey, [cacheKeyForJobContext]) + } + + return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment]) } - return new CacheKey(cacheKey, [cacheKeyForJobContext, cacheKeyForJob, cacheKeyForEnvironment]) -} - -export function getCacheKeyBase(cacheName: string, cacheProtocolVersion: string): string { - // Prefix can be used to force change all cache keys (defaults to cache protocol version) - return `gradle-${cacheName}-${cacheProtocolVersion}` -} - -function getCacheKeyEnvironment(): string { - const runnerOs = process.env['RUNNER_OS'] || '' - const runnerArch = process.env['RUNNER_ARCH'] || '' - return process.env[CACHE_KEY_OS_VAR] || `${runnerOs}-${runnerArch}` -} - -function getCacheKeyJob(): string { - return process.env[CACHE_KEY_JOB_VAR] || github.context.job -} - -function getCacheKeyJobInstance(): string { - const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR] - if (override) { - return override + getCacheKeyBase(cacheName: string, cacheProtocolVersion: string): string { + // Prefix can be used to force change all cache keys (defaults to cache protocol version) + return `gradle-${cacheName}-${cacheProtocolVersion}` } - // By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation - // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. - const workflowName = github.context.workflow - const workflowJobContext = getJobMatrix() - return hashStrings([workflowName, workflowJobContext]) -} + private getCacheKeyEnvironment(): string { + const runnerOs = process.env['RUNNER_OS'] || '' + const runnerArch = process.env['RUNNER_ARCH'] || '' + return process.env[CACHE_KEY_OS_VAR] || `${runnerOs}-${runnerArch}` + } -function getCacheKeyJobExecution(): string { - // Used to associate a cache key with a particular execution (default is bound to the git commit sha) - return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha + private getCacheKeyJob(): string { + return process.env[CACHE_KEY_JOB_VAR] || github.context.job + } + + private getCacheKeyJobInstance(): string { + const override = process.env[CACHE_KEY_JOB_INSTANCE_VAR] + if (override) { + return override + } + + // By default, we hash the workflow name and the full `matrix` data for the run, to uniquely identify this job invocation + // The only way we can obtain the `matrix` data is via the `workflow-job-context` parameter in action.yml. + const workflowName = github.context.workflow + const workflowJobContext = getJobMatrix() + return hashStrings([workflowName, workflowJobContext]) + } + + private getCacheKeyJobExecution(): string { + // Used to associate a cache key with a particular execution (default is bound to the git commit sha) + return process.env[CACHE_KEY_JOB_EXECUTION_VAR] || github.context.sha + } } diff --git a/sources/src/caching/caches.ts b/sources/src/caching/caches.ts index d3f4356..37b9736 100644 --- a/sources/src/caching/caches.ts +++ b/sources/src/caching/caches.ts @@ -10,6 +10,7 @@ import {CacheCleaner} from './cache-cleaner' import {DaemonController} from '../daemon-controller' import {CacheConfig} from '../configuration' import {BuildResults} from '../build-results' +import {CacheKeyGenerator} from './cache-key' const CACHE_RESTORED_VAR = 'GRADLE_BUILD_ACTION_CACHE_RESTORED' @@ -26,7 +27,8 @@ export async function restore( } core.exportVariable(CACHE_RESTORED_VAR, true) - const gradleStateCache = new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig) + // TODO(Nava2): Move `new CacheKeyGenerator()` to a class property. + const gradleStateCache = new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig, new CacheKeyGenerator()) if (cacheConfig.isCacheDisabled()) { core.info('Cache is disabled: will not restore state from previous builds.') @@ -110,7 +112,8 @@ export async function save( } await core.group('Caching Gradle state', async () => { - return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig).save(cacheListener) + const cacheKeyGenerator = new CacheKeyGenerator() + return new GradleUserHomeCache(userHome, gradleUserHome, cacheConfig, cacheKeyGenerator).save(cacheListener) }) } diff --git a/sources/src/caching/gradle-home-extry-extractor.ts b/sources/src/caching/gradle-home-extry-extractor.ts index ffb15ac..9099ba7 100644 --- a/sources/src/caching/gradle-home-extry-extractor.ts +++ b/sources/src/caching/gradle-home-extry-extractor.ts @@ -8,7 +8,7 @@ import {cacheDebug, hashFileNames, isCacheDebuggingEnabled, restoreCache, saveCa import {BuildResult, loadBuildResults} from '../build-results' import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' -import {getCacheKeyBase} from './cache-key' +import {CacheKeyGenerator} from './cache-key' import {versionIsAtLeast} from '../execution/gradle' const SKIP_RESTORE_VAR = 'GRADLE_BUILD_ACTION_SKIP_RESTORE' @@ -85,10 +85,18 @@ abstract class AbstractEntryExtractor { protected readonly gradleUserHome: string private extractorName: string - constructor(gradleUserHome: string, extractorName: string, cacheConfig: CacheConfig) { + private readonly cacheKeyGenerator: CacheKeyGenerator + + constructor( + gradleUserHome: string, + extractorName: string, + cacheConfig: CacheConfig, + cacheKeyGenerator: CacheKeyGenerator + ) { this.gradleUserHome = gradleUserHome this.extractorName = extractorName this.cacheConfig = cacheConfig + this.cacheKeyGenerator = cacheKeyGenerator } /** @@ -247,7 +255,7 @@ abstract class AbstractEntryExtractor { cacheDebug(`Generating cache key for ${artifactType} from file names: ${relativeFiles}`) - return `${getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}` + return `${this.cacheKeyGenerator.getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}` } protected async createCacheKeyFromFileContents(artifactType: string, pattern: string): Promise { @@ -255,7 +263,7 @@ abstract class AbstractEntryExtractor { cacheDebug(`Generating cache key for ${artifactType} from files matching: ${pattern}`) - return `${getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}` + return `${this.cacheKeyGenerator.getCacheKeyBase(artifactType, CACHE_PROTOCOL_VERSION)}-${key}` } // Run actions sequentially if debugging is enabled @@ -305,8 +313,12 @@ abstract class AbstractEntryExtractor { } export class GradleHomeEntryExtractor extends AbstractEntryExtractor { - constructor(gradleUserHome: string, cacheConfig: CacheConfig) { - super(gradleUserHome, 'gradle-home', cacheConfig) + constructor( + gradleUserHome: string, + cacheConfig: CacheConfig, + cacheKeyGenerator: CacheKeyGenerator = new CacheKeyGenerator() + ) { + super(gradleUserHome, 'gradle-home', cacheConfig, cacheKeyGenerator) } async extract(listener: CacheListener): Promise { @@ -364,8 +376,12 @@ export class GradleHomeEntryExtractor extends AbstractEntryExtractor { } export class ConfigurationCacheEntryExtractor extends AbstractEntryExtractor { - constructor(gradleUserHome: string, cacheConfig: CacheConfig) { - super(gradleUserHome, 'configuration-cache', cacheConfig) + constructor( + gradleUserHome: string, + cacheConfig: CacheConfig, + cacheKeyGenerator: CacheKeyGenerator = new CacheKeyGenerator() + ) { + super(gradleUserHome, 'configuration-cache', cacheConfig, cacheKeyGenerator) } /** diff --git a/sources/src/caching/gradle-user-home-cache.ts b/sources/src/caching/gradle-user-home-cache.ts index 55fc650..38013aa 100644 --- a/sources/src/caching/gradle-user-home-cache.ts +++ b/sources/src/caching/gradle-user-home-cache.ts @@ -4,7 +4,7 @@ import * as glob from '@actions/glob' import path from 'path' import fs from 'fs' -import {generateCacheKey} from './cache-key' +import {CacheKeyGenerator} from './cache-key' import {CacheListener} from './cache-reporting' import {saveCache, restoreCache, cacheDebug, isCacheDebuggingEnabled, tryDelete} from './cache-utils' import {CacheConfig, ACTION_METADATA_DIR} from '../configuration' @@ -21,10 +21,18 @@ export class GradleUserHomeCache { private readonly gradleUserHome: string private readonly cacheConfig: CacheConfig - constructor(userHome: string, gradleUserHome: string, cacheConfig: CacheConfig) { + private readonly cacheKeyGenerator: CacheKeyGenerator + + constructor( + userHome: string, + gradleUserHome: string, + cacheConfig: CacheConfig, + cacheKeyGenerator: CacheKeyGenerator + ) { this.userHome = userHome this.gradleUserHome = gradleUserHome this.cacheConfig = cacheConfig + this.cacheKeyGenerator = cacheKeyGenerator } init(): void { @@ -52,7 +60,7 @@ export class GradleUserHomeCache { async restore(listener: CacheListener): Promise { const entryListener = listener.entry(this.cacheDescription) - const cacheKey = generateCacheKey(this.cacheName, this.cacheConfig) + const cacheKey = this.cacheKeyGenerator.generateCacheKey(this.cacheName, this.cacheConfig) cacheDebug( `Requesting ${this.cacheDescription} with @@ -95,7 +103,7 @@ export class GradleUserHomeCache { * it is saved with the exact key. */ async save(listener: CacheListener): Promise { - const cacheKey = generateCacheKey(this.cacheName, this.cacheConfig).key + const cacheKey = this.cacheKeyGenerator.generateCacheKey(this.cacheName, this.cacheConfig).key const restoredCacheKey = core.getState(RESTORED_CACHE_KEY_KEY) const gradleHomeEntryListener = listener.entry(this.cacheDescription) diff --git a/sources/test/jest/cache-debug.test.ts b/sources/test/jest/cache-debug.test.ts index 141d51b..6f4dbbc 100644 --- a/sources/test/jest/cache-debug.test.ts +++ b/sources/test/jest/cache-debug.test.ts @@ -2,6 +2,7 @@ import * as path from 'path' import * as fs from 'fs' import {GradleUserHomeCache} from "../../src/caching/gradle-user-home-cache" import {CacheConfig} from "../../src/configuration" +import { CacheKeyGenerator } from '../../src/caching/cache-key' const testTmp = 'test/jest/tmp' fs.rmSync(testTmp, {recursive: true, force: true}) @@ -12,7 +13,7 @@ describe("--info and --stacktrace", () => { const emptyGradleHome = `${testTmp}/empty-gradle-home` fs.mkdirSync(emptyGradleHome, {recursive: true}) - const stateCache = new GradleUserHomeCache("ignored", emptyGradleHome, new CacheConfig()) + const stateCache = new GradleUserHomeCache("ignored", emptyGradleHome, new CacheConfig(), new CacheKeyGenerator()) stateCache.configureInfoLogLevel() expect(fs.readFileSync(path.resolve(emptyGradleHome, "gradle.properties"), 'utf-8')) @@ -25,7 +26,7 @@ describe("--info and --stacktrace", () => { fs.mkdirSync(existingGradleHome, {recursive: true}) fs.writeFileSync(path.resolve(existingGradleHome, "gradle.properties"), "org.gradle.logging.level=debug\n") - const stateCache = new GradleUserHomeCache("ignored", existingGradleHome, new CacheConfig()) + const stateCache = new GradleUserHomeCache("ignored", existingGradleHome, new CacheConfig(), new CacheKeyGenerator()) stateCache.configureInfoLogLevel() expect(fs.readFileSync(path.resolve(existingGradleHome, "gradle.properties"), 'utf-8'))