build cache index file

This commit is contained in:
Vladimir Safonkin 2022-10-12 16:21:41 +02:00
parent 7e403af8c5
commit de93cfa81c

View file

@ -525,15 +525,9 @@ function resolvePaths(patterns) {
.replace(new RegExp(`\\${path.sep}`, 'g'), '/'); .replace(new RegExp(`\\${path.sep}`, 'g'), '/');
core.debug(`Matched: ${relativeFile}`); core.debug(`Matched: ${relativeFile}`);
// Paths are made relative so the tar entries are all relative to the root of the workspace. // Paths are made relative so the tar entries are all relative to the root of the workspace.
if (relativeFile === '') {
// path.relative returns empty string if workspace and file are equal
paths.push('.');
}
else {
paths.push(`${relativeFile}`); paths.push(`${relativeFile}`);
} }
} }
}
catch (e_1_1) { e_1 = { error: e_1_1 }; } catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally { finally {
try { try {
@ -689,7 +683,6 @@ const util = __importStar(__nccwpck_require__(3837));
const utils = __importStar(__nccwpck_require__(1518)); const utils = __importStar(__nccwpck_require__(1518));
const constants_1 = __nccwpck_require__(8840); const constants_1 = __nccwpck_require__(8840);
const requestUtils_1 = __nccwpck_require__(3981); const requestUtils_1 = __nccwpck_require__(3981);
const abort_controller_1 = __nccwpck_require__(2557);
/** /**
* Pipes the body of a HTTP response to a stream * Pipes the body of a HTTP response to a stream
* *
@ -873,26 +866,17 @@ function downloadCacheStorageSDK(archiveLocation, archivePath, options) {
const fd = fs.openSync(archivePath, 'w'); const fd = fs.openSync(archivePath, 'w');
try { try {
downloadProgress.startDisplayTimer(); downloadProgress.startDisplayTimer();
const controller = new abort_controller_1.AbortController();
const abortSignal = controller.signal;
while (!downloadProgress.isDone()) { while (!downloadProgress.isDone()) {
const segmentStart = downloadProgress.segmentOffset + downloadProgress.segmentSize; const segmentStart = downloadProgress.segmentOffset + downloadProgress.segmentSize;
const segmentSize = Math.min(maxSegmentSize, contentLength - segmentStart); const segmentSize = Math.min(maxSegmentSize, contentLength - segmentStart);
downloadProgress.nextSegment(segmentSize); downloadProgress.nextSegment(segmentSize);
const result = yield promiseWithTimeout(options.segmentTimeoutInMs || 3600000, client.downloadToBuffer(segmentStart, segmentSize, { const result = yield client.downloadToBuffer(segmentStart, segmentSize, {
abortSignal,
concurrency: options.downloadConcurrency, concurrency: options.downloadConcurrency,
onProgress: downloadProgress.onProgress() onProgress: downloadProgress.onProgress()
})); });
if (result === 'timeout') {
controller.abort();
throw new Error('Aborting cache download as the download time exceeded the timeout.');
}
else if (Buffer.isBuffer(result)) {
fs.writeFileSync(fd, result); fs.writeFileSync(fd, result);
} }
} }
}
finally { finally {
downloadProgress.stopDisplayTimer(); downloadProgress.stopDisplayTimer();
fs.closeSync(fd); fs.closeSync(fd);
@ -901,16 +885,6 @@ function downloadCacheStorageSDK(archiveLocation, archivePath, options) {
}); });
} }
exports.downloadCacheStorageSDK = downloadCacheStorageSDK; exports.downloadCacheStorageSDK = downloadCacheStorageSDK;
const promiseWithTimeout = (timeoutMs, promise) => __awaiter(void 0, void 0, void 0, function* () {
let timeoutHandle;
const timeoutPromise = new Promise(resolve => {
timeoutHandle = setTimeout(() => resolve('timeout'), timeoutMs);
});
return Promise.race([promise, timeoutPromise]).then(result => {
clearTimeout(timeoutHandle);
return result;
});
});
//# sourceMappingURL=downloadUtils.js.map //# sourceMappingURL=downloadUtils.js.map
/***/ }), /***/ }),
@ -1070,7 +1044,6 @@ const fs_1 = __nccwpck_require__(7147);
const path = __importStar(__nccwpck_require__(1017)); const path = __importStar(__nccwpck_require__(1017));
const utils = __importStar(__nccwpck_require__(1518)); const utils = __importStar(__nccwpck_require__(1518));
const constants_1 = __nccwpck_require__(8840); const constants_1 = __nccwpck_require__(8840);
const IS_WINDOWS = process.platform === 'win32';
function getTarPath(args, compressionMethod) { function getTarPath(args, compressionMethod) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
switch (process.platform) { switch (process.platform) {
@ -1118,43 +1091,26 @@ function getWorkingDirectory() {
var _a; var _a;
return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd(); return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd();
} }
// Common function for extractTar and listTar to get the compression method
function getCompressionProgram(compressionMethod) {
// -d: Decompress.
// unzstd is equivalent to 'zstd -d'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
switch (compressionMethod) {
case constants_1.CompressionMethod.Zstd:
return [
'--use-compress-program',
IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30'
];
case constants_1.CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'];
default:
return ['-z'];
}
}
function listTar(archivePath, compressionMethod) {
return __awaiter(this, void 0, void 0, function* () {
const args = [
...getCompressionProgram(compressionMethod),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
];
yield execTar(args, compressionMethod);
});
}
exports.listTar = listTar;
function extractTar(archivePath, compressionMethod) { function extractTar(archivePath, compressionMethod) {
return __awaiter(this, void 0, void 0, function* () { return __awaiter(this, void 0, void 0, function* () {
// Create directory to extract tar into // Create directory to extract tar into
const workingDirectory = getWorkingDirectory(); const workingDirectory = getWorkingDirectory();
yield io.mkdirP(workingDirectory); yield io.mkdirP(workingDirectory);
// --d: Decompress.
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram() {
switch (compressionMethod) {
case constants_1.CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -d --long=30'];
case constants_1.CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -d'];
default:
return ['-z'];
}
}
const args = [ const args = [
...getCompressionProgram(compressionMethod), ...getCompressionProgram(),
'-xf', '-xf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P', '-P',
@ -1173,19 +1129,15 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) {
fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n')); fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n'));
const workingDirectory = getWorkingDirectory(); const workingDirectory = getWorkingDirectory();
// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores.
// zstdmt is equivalent to 'zstd -T0'
// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners. // Using 30 here because we also support 32-bit self-hosted runners.
// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd.
function getCompressionProgram() { function getCompressionProgram() {
switch (compressionMethod) { switch (compressionMethod) {
case constants_1.CompressionMethod.Zstd: case constants_1.CompressionMethod.Zstd:
return [ return ['--use-compress-program', 'zstd -T0 --long=30'];
'--use-compress-program',
IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30'
];
case constants_1.CompressionMethod.ZstdWithoutLong: case constants_1.CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']; return ['--use-compress-program', 'zstd -T0'];
default: default:
return ['-z']; return ['-z'];
} }
@ -1207,6 +1159,32 @@ function createTar(archiveFolder, sourceDirectories, compressionMethod) {
}); });
} }
exports.createTar = createTar; exports.createTar = createTar;
function listTar(archivePath, compressionMethod) {
return __awaiter(this, void 0, void 0, function* () {
// --d: Decompress.
// --long=#: Enables long distance matching with # bits.
// Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit.
// Using 30 here because we also support 32-bit self-hosted runners.
function getCompressionProgram() {
switch (compressionMethod) {
case constants_1.CompressionMethod.Zstd:
return ['--use-compress-program', 'zstd -d --long=30'];
case constants_1.CompressionMethod.ZstdWithoutLong:
return ['--use-compress-program', 'zstd -d'];
default:
return ['-z'];
}
}
const args = [
...getCompressionProgram(),
'-tf',
archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'),
'-P'
];
yield execTar(args, compressionMethod);
});
}
exports.listTar = listTar;
//# sourceMappingURL=tar.js.map //# sourceMappingURL=tar.js.map
/***/ }), /***/ }),
@ -1257,8 +1235,7 @@ function getDownloadOptions(copy) {
const result = { const result = {
useAzureSdk: true, useAzureSdk: true,
downloadConcurrency: 8, downloadConcurrency: 8,
timeoutInMs: 30000, timeoutInMs: 30000
segmentTimeoutInMs: 3600000
}; };
if (copy) { if (copy) {
if (typeof copy.useAzureSdk === 'boolean') { if (typeof copy.useAzureSdk === 'boolean') {
@ -1270,21 +1247,10 @@ function getDownloadOptions(copy) {
if (typeof copy.timeoutInMs === 'number') { if (typeof copy.timeoutInMs === 'number') {
result.timeoutInMs = copy.timeoutInMs; result.timeoutInMs = copy.timeoutInMs;
} }
if (typeof copy.segmentTimeoutInMs === 'number') {
result.segmentTimeoutInMs = copy.segmentTimeoutInMs;
}
}
const segmentDownloadTimeoutMins = process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS'];
if (segmentDownloadTimeoutMins &&
!isNaN(Number(segmentDownloadTimeoutMins)) &&
isFinite(Number(segmentDownloadTimeoutMins))) {
result.segmentTimeoutInMs = Number(segmentDownloadTimeoutMins) * 60 * 1000;
} }
core.debug(`Use Azure SDK: ${result.useAzureSdk}`); core.debug(`Use Azure SDK: ${result.useAzureSdk}`);
core.debug(`Download concurrency: ${result.downloadConcurrency}`); core.debug(`Download concurrency: ${result.downloadConcurrency}`);
core.debug(`Request timeout (ms): ${result.timeoutInMs}`); core.debug(`Request timeout (ms): ${result.timeoutInMs}`);
core.debug(`Cache segment download timeout mins env var: ${process.env['SEGMENT_DOWNLOAD_TIMEOUT_MINS']}`);
core.debug(`Segment download timeout (ms): ${result.segmentTimeoutInMs}`);
return result; return result;
} }
exports.getDownloadOptions = getDownloadOptions; exports.getDownloadOptions = getDownloadOptions;
@ -4865,6 +4831,7 @@ const file_command_1 = __nccwpck_require__(717);
const utils_1 = __nccwpck_require__(5278); const utils_1 = __nccwpck_require__(5278);
const os = __importStar(__nccwpck_require__(2037)); const os = __importStar(__nccwpck_require__(2037));
const path = __importStar(__nccwpck_require__(1017)); const path = __importStar(__nccwpck_require__(1017));
const uuid_1 = __nccwpck_require__(8974);
const oidc_utils_1 = __nccwpck_require__(8041); const oidc_utils_1 = __nccwpck_require__(8041);
/** /**
* The code to exit an action * The code to exit an action
@ -4894,10 +4861,21 @@ function exportVariable(name, val) {
process.env[name] = convertedVal; process.env[name] = convertedVal;
const filePath = process.env['GITHUB_ENV'] || ''; const filePath = process.env['GITHUB_ENV'] || '';
if (filePath) { if (filePath) {
return file_command_1.issueFileCommand('ENV', file_command_1.prepareKeyValueMessage(name, val)); const delimiter = `ghadelimiter_${uuid_1.v4()}`;
// These should realistically never happen, but just in case someone finds a way to exploit uuid generation let's not allow keys or values that contain the delimiter.
if (name.includes(delimiter)) {
throw new Error(`Unexpected input: name should not contain the delimiter "${delimiter}"`);
} }
if (convertedVal.includes(delimiter)) {
throw new Error(`Unexpected input: value should not contain the delimiter "${delimiter}"`);
}
const commandValue = `${name}<<${delimiter}${os.EOL}${convertedVal}${os.EOL}${delimiter}`;
file_command_1.issueCommand('ENV', commandValue);
}
else {
command_1.issueCommand('set-env', { name }, convertedVal); command_1.issueCommand('set-env', { name }, convertedVal);
} }
}
exports.exportVariable = exportVariable; exports.exportVariable = exportVariable;
/** /**
* Registers a secret which will get masked from logs * Registers a secret which will get masked from logs
@ -4914,7 +4892,7 @@ exports.setSecret = setSecret;
function addPath(inputPath) { function addPath(inputPath) {
const filePath = process.env['GITHUB_PATH'] || ''; const filePath = process.env['GITHUB_PATH'] || '';
if (filePath) { if (filePath) {
file_command_1.issueFileCommand('PATH', inputPath); file_command_1.issueCommand('PATH', inputPath);
} }
else { else {
command_1.issueCommand('add-path', {}, inputPath); command_1.issueCommand('add-path', {}, inputPath);
@ -4954,11 +4932,8 @@ function getMultilineInput(name, options) {
const inputs = getInput(name, options) const inputs = getInput(name, options)
.split('\n') .split('\n')
.filter(x => x !== ''); .filter(x => x !== '');
if (options && options.trimWhitespace === false) {
return inputs; return inputs;
} }
return inputs.map(input => input.trim());
}
exports.getMultilineInput = getMultilineInput; exports.getMultilineInput = getMultilineInput;
/** /**
* Gets the input value of the boolean type in the YAML 1.2 "core schema" specification. * Gets the input value of the boolean type in the YAML 1.2 "core schema" specification.
@ -4990,12 +4965,8 @@ exports.getBooleanInput = getBooleanInput;
*/ */
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
function setOutput(name, value) { function setOutput(name, value) {
const filePath = process.env['GITHUB_OUTPUT'] || '';
if (filePath) {
return file_command_1.issueFileCommand('OUTPUT', file_command_1.prepareKeyValueMessage(name, value));
}
process.stdout.write(os.EOL); process.stdout.write(os.EOL);
command_1.issueCommand('set-output', { name }, utils_1.toCommandValue(value)); command_1.issueCommand('set-output', { name }, value);
} }
exports.setOutput = setOutput; exports.setOutput = setOutput;
/** /**
@ -5124,11 +5095,7 @@ exports.group = group;
*/ */
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
function saveState(name, value) { function saveState(name, value) {
const filePath = process.env['GITHUB_STATE'] || ''; command_1.issueCommand('save-state', { name }, value);
if (filePath) {
return file_command_1.issueFileCommand('STATE', file_command_1.prepareKeyValueMessage(name, value));
}
command_1.issueCommand('save-state', { name }, utils_1.toCommandValue(value));
} }
exports.saveState = saveState; exports.saveState = saveState;
/** /**
@ -5194,14 +5161,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result; return result;
}; };
Object.defineProperty(exports, "__esModule", ({ value: true })); Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.prepareKeyValueMessage = exports.issueFileCommand = void 0; exports.issueCommand = void 0;
// We use any as a valid input type // We use any as a valid input type
/* eslint-disable @typescript-eslint/no-explicit-any */ /* eslint-disable @typescript-eslint/no-explicit-any */
const fs = __importStar(__nccwpck_require__(7147)); const fs = __importStar(__nccwpck_require__(7147));
const os = __importStar(__nccwpck_require__(2037)); const os = __importStar(__nccwpck_require__(2037));
const uuid_1 = __nccwpck_require__(8974);
const utils_1 = __nccwpck_require__(5278); const utils_1 = __nccwpck_require__(5278);
function issueFileCommand(command, message) { function issueCommand(command, message) {
const filePath = process.env[`GITHUB_${command}`]; const filePath = process.env[`GITHUB_${command}`];
if (!filePath) { if (!filePath) {
throw new Error(`Unable to find environment variable for file command ${command}`); throw new Error(`Unable to find environment variable for file command ${command}`);
@ -5213,22 +5179,7 @@ function issueFileCommand(command, message) {
encoding: 'utf8' encoding: 'utf8'
}); });
} }
exports.issueFileCommand = issueFileCommand; exports.issueCommand = issueCommand;
function prepareKeyValueMessage(key, value) {
const delimiter = `ghadelimiter_${uuid_1.v4()}`;
const convertedValue = utils_1.toCommandValue(value);
// These should realistically never happen, but just in case someone finds a
// way to exploit uuid generation let's not allow keys or values that contain
// the delimiter.
if (key.includes(delimiter)) {
throw new Error(`Unexpected input: name should not contain the delimiter "${delimiter}"`);
}
if (convertedValue.includes(delimiter)) {
throw new Error(`Unexpected input: value should not contain the delimiter "${delimiter}"`);
}
return `${key}<<${delimiter}${os.EOL}${convertedValue}${os.EOL}${delimiter}`;
}
exports.prepareKeyValueMessage = prepareKeyValueMessage;
//# sourceMappingURL=file-command.js.map //# sourceMappingURL=file-command.js.map
/***/ }), /***/ }),