Skip to content

Commit 1279e8d

Browse files
henrymercerhenrymercer
committed
Mitigate caches being evicted before they can be downloaded
1 parent af1f613 commit 1279e8d

5 files changed

Lines changed: 91 additions & 12 deletions

File tree

‎lib/analyze-action.js‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎lib/init-action.js‎

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/api-client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ export interface ActionsCacheItem {
249249
created_at?: string;
250250
id?: number;
251251
key?: string;
252+
last_accessed_at?: string;
252253
size_in_bytes?: number;
253254
}
254255

‎src/overlay/caching.test.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -417,3 +417,37 @@ test.serial(
417417
t.deepEqual(result, ["2.25.0", "2.24.0"]);
418418
},
419419
);
420+
421+
test.serial(
422+
"getCodeQlVersionsForOverlayBaseDatabases ignores cache entries close to eviction",
423+
async (t) => {
424+
const logger = getRunnerLogger(true);
425+
426+
const now = Date.now();
427+
const isoDaysAgo = (days: number) =>
428+
new Date(now - days * 24 * 60 * 60 * 1000).toISOString();
429+
430+
sinon.stub(apiClient, "getAutomationID").resolves("test-automation-id/");
431+
sinon.stub(apiClient, "listActionsCaches").resolves([
432+
{
433+
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.25.0-abc123-1-1",
434+
last_accessed_at: isoDaysAgo(1),
435+
},
436+
{
437+
// Older than the 6-day threshold; close to the 7-day eviction window.
438+
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.26.0-def456-2-1",
439+
last_accessed_at: isoDaysAgo(6.5),
440+
},
441+
{
442+
key: "codeql-overlay-base-database-1-c5666c509a2d9895-python-2.24.0-ghi789-3-1",
443+
last_accessed_at: isoDaysAgo(3),
444+
},
445+
]);
446+
447+
const result = await getCodeQlVersionsForOverlayBaseDatabases(
448+
["python"],
449+
logger,
450+
);
451+
t.deepEqual(result, ["2.25.0", "2.24.0"]);
452+
},
453+
);

‎src/overlay/caching.ts‎

Lines changed: 52 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ import {
88
getWorkflowRunAttempt,
99
getWorkflowRunID,
1010
} from "../actions-util";
11-
import { getAutomationID, listActionsCaches } from "../api-client";
11+
import {
12+
type ActionsCacheItem,
13+
getAutomationID,
14+
listActionsCaches,
15+
} from "../api-client";
1216
import { createCacheKeyHash } from "../caching-utils";
1317
import { type CodeQL } from "../codeql";
1418
import { type Config } from "../config-utils";
@@ -48,6 +52,12 @@ const OVERLAY_BASE_DATABASE_MAX_UPLOAD_SIZE_BYTES =
4852
const CACHE_VERSION = 1;
4953
const CACHE_PREFIX = "codeql-overlay-base-database";
5054

55+
// The Actions cache evicts entries that have not been accessed in the past 7
56+
// days. We conservatively set a limit of 6 days to avoid using a cached base DB
57+
// that may be evicted before we can download it.
58+
const CACHE_ENTRY_MAX_AGE_DAYS = 6;
59+
const CACHE_ENTRY_MAX_AGE_MS = CACHE_ENTRY_MAX_AGE_DAYS * 24 * 60 * 60 * 1000;
60+
5161
// The purpose of this ten-minute limit is to guard against the possibility
5262
// that the cache service is unresponsive, which would otherwise cause the
5363
// entire action to hang. Normally we expect cache operations to complete
@@ -435,6 +445,39 @@ async function getCacheKeyPrefixBase(
435445
return `${CACHE_PREFIX}-${CACHE_VERSION}-${componentsHash}-${languagesComponent}-`;
436446
}
437447

448+
/**
449+
* Lists overlay-base database cache entries with the given key prefix, ignoring entries that are
450+
* old enough that they may be evicted by the Actions cache before we attempt to download them.
451+
*/
452+
async function listRecentOverlayBaseDatabaseCaches(
453+
cacheKeyPrefix: string,
454+
logger: Logger,
455+
): Promise<ActionsCacheItem[]> {
456+
const allCaches = await listActionsCaches(cacheKeyPrefix);
457+
458+
if (allCaches.length === 0) {
459+
logger.info("No overlay-base databases found in Actions cache.");
460+
return [];
461+
}
462+
463+
const cutoffMs = Date.now() - CACHE_ENTRY_MAX_AGE_MS;
464+
const recentCaches = allCaches.filter((cache) => {
465+
if (!cache.last_accessed_at) return true;
466+
const lastAccessedMs = Date.parse(cache.last_accessed_at);
467+
return Number.isNaN(lastAccessedMs) || lastAccessedMs >= cutoffMs;
468+
});
469+
const numTooOldDatabases = allCaches.length - recentCaches.length;
470+
const tooOldSuffix =
471+
numTooOldDatabases > 0
472+
? ` (ignoring ${numTooOldDatabases} that may be evicted soon)`
473+
: "";
474+
logger.info(
475+
`Found ${allCaches.length} overlay-base ${allCaches.length === 1 ? "database" : "databases"} in the Actions cache${tooOldSuffix}.`,
476+
);
477+
478+
return recentCaches;
479+
}
480+
438481
/**
439482
* Searches the GitHub Actions cache for overlay-base databases matching the given languages, and
440483
* returns all stable CodeQL versions found across matching cache entries.
@@ -448,7 +491,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
448491
): Promise<string[] | undefined> {
449492
const languages = rawLanguages.map(parseBuiltInLanguage);
450493
if (languages.includes(undefined)) {
451-
logger.warning(
494+
logger.info(
452495
"One or more provided languages are not recognized as built-in languages. " +
453496
"Skipping searching for overlay-base databases in cache.",
454497
);
@@ -463,22 +506,19 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
463506
`prefix ${cacheKeyPrefix}`,
464507
);
465508

466-
const caches = await listActionsCaches(cacheKeyPrefix);
509+
const caches = await listRecentOverlayBaseDatabaseCaches(
510+
cacheKeyPrefix,
511+
logger,
512+
);
467513

468514
if (caches.length === 0) {
469-
logger.info("No overlay-base databases found in Actions cache.");
470515
return [];
471516
}
472517

473-
logger.info(
474-
`Found ${caches.length} overlay-base ` +
475-
`${caches.length === 1 ? "database" : "databases"} in the Actions cache.`,
476-
);
477-
478518
// Parse CodeQL versions from cache keys, matching only stable releases.
479519
//
480-
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies will have
481-
// a suffix like `+202604201548` that will break the match.
520+
// After the prefix, the remaining key format starts with `${codeQlVersion}-`. Nightlies have a
521+
// suffix like `+202604201548` that will prevent a match.
482522
//
483523
// Caveat: this relies on the fact that we haven't released any CodeQL bundles with the
484524
// `x.y.z-<pre-release>` semver format which does not interact well with the current overlay base
@@ -506,7 +546,7 @@ export async function getCodeQlVersionsForOverlayBaseDatabases(
506546
const versions = [...versionSet].sort(semver.rcompare);
507547

508548
logger.info(
509-
`Found overlay databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
549+
`Found overlay-base databases for the following CodeQL versions in the Actions cache: ${versions.join(", ")}`,
510550
);
511551

512552
return versions;

0 commit comments

Comments
 (0)