diff --git a/dist/post_run/index.js b/dist/post_run/index.js index 276e74b..40c8157 100644 --- a/dist/post_run/index.js +++ b/dist/post_run/index.js @@ -74,9 +74,10 @@ exports.isFeatureAvailable = isFeatureAvailable; * @param primaryKey an explicit key for restoring the cache * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key * @param downloadOptions cache download options + * @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform * @returns string returns the key for the cache hit, otherwise returns undefined */ -function restoreCache(paths, primaryKey, restoreKeys, options) { +function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive = false) { return __awaiter(this, void 0, void 0, function* () { checkPaths(paths); restoreKeys = restoreKeys || []; @@ -94,7 +95,8 @@ function restoreCache(paths, primaryKey, restoreKeys, options) { try { // path are needed to compute version const cacheEntry = yield cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod + compressionMethod, + enableCrossOsArchive }); if (!(cacheEntry === null || cacheEntry === void 0 ? void 0 : cacheEntry.archiveLocation)) { // Cache not found @@ -141,10 +143,11 @@ exports.restoreCache = restoreCache; * * @param paths a list of file paths to be cached * @param key an explicit key for restoring the cache + * @param enableCrossOsArchive an optional boolean enabled to save cache on windows which could be restored on any platform * @param options cache upload options * @returns number returns cacheId if the cache was saved successfully and throws an error if save fails */ -function saveCache(paths, key, options) { +function saveCache(paths, key, options, enableCrossOsArchive = false) { var _a, _b, _c, _d, _e; return __awaiter(this, void 0, void 0, function* () { checkPaths(paths); @@ -175,6 +178,7 @@ function saveCache(paths, key, options) { core.debug('Reserving Cache'); const reserveCacheResponse = yield cacheHttpClient.reserveCache(key, paths, { compressionMethod, + enableCrossOsArchive, cacheSize: archiveFileSize }); if ((_a = reserveCacheResponse === null || reserveCacheResponse === void 0 ? void 0 : reserveCacheResponse.result) === null || _a === void 0 ? void 0 : _a.cacheId) { @@ -247,7 +251,6 @@ const crypto = __importStar(__nccwpck_require__(6113)); const fs = __importStar(__nccwpck_require__(7147)); const url_1 = __nccwpck_require__(7310); const utils = __importStar(__nccwpck_require__(1518)); -const constants_1 = __nccwpck_require__(8840); const downloadUtils_1 = __nccwpck_require__(5500); const options_1 = __nccwpck_require__(6215); const requestUtils_1 = __nccwpck_require__(3981); @@ -277,10 +280,17 @@ function createHttpClient() { const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token); return new http_client_1.HttpClient('actions/cache', [bearerCredentialHandler], getRequestOptions()); } -function getCacheVersion(paths, compressionMethod) { - const components = paths.concat(!compressionMethod || compressionMethod === constants_1.CompressionMethod.Gzip - ? [] - : [compressionMethod]); +function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { + const components = paths; + // Add compression method to cache version to restore + // compressed cache as per compression method + if (compressionMethod) { + components.push(compressionMethod); + } + // Only check for windows platforms if enableCrossOsArchive is false + if (process.platform === 'win32' && !enableCrossOsArchive) { + components.push('windows-only'); + } // Add salt to cache version to support breaking changes in cache entry components.push(versionSalt); return crypto @@ -292,9 +302,10 @@ exports.getCacheVersion = getCacheVersion; function getCacheEntry(keys, paths, options) { return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); - const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod); + const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod, options === null || options === void 0 ? void 0 : options.enableCrossOsArchive); const resource = `cache?keys=${encodeURIComponent(keys.join(','))}&version=${version}`; const response = yield requestUtils_1.retryTypedResponse('getCacheEntry', () => __awaiter(this, void 0, void 0, function* () { return httpClient.getJson(getCacheApiUrl(resource)); })); + // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -308,6 +319,7 @@ function getCacheEntry(keys, paths, options) { const cacheResult = response.result; const cacheDownloadUrl = cacheResult === null || cacheResult === void 0 ? void 0 : cacheResult.archiveLocation; if (!cacheDownloadUrl) { + // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.'); } core.setSecret(cacheDownloadUrl); @@ -353,7 +365,7 @@ exports.downloadCache = downloadCache; function reserveCache(key, paths, options) { return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); - const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod); + const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod, options === null || options === void 0 ? void 0 : options.enableCrossOsArchive); const reserveCacheRequest = { key, version, @@ -596,10 +608,6 @@ function getVersion(app) { // Use zstandard if possible to maximize cache performance function getCompressionMethod() { return __awaiter(this, void 0, void 0, function* () { - if (process.platform === 'win32' && !(yield isGnuTarInstalled())) { - // Disable zstd due to bug https://github.com/actions/cache/issues/301 - return constants_1.CompressionMethod.Gzip; - } const versionOutput = yield getVersion('zstd'); const version = semver.clean(versionOutput); if (!versionOutput.toLowerCase().includes('zstd command line interface')) { @@ -623,13 +631,16 @@ function getCacheFileName(compressionMethod) { : constants_1.CacheFilename.Zstd; } exports.getCacheFileName = getCacheFileName; -function isGnuTarInstalled() { +function getGnuTarPathOnWindows() { return __awaiter(this, void 0, void 0, function* () { + if (fs.existsSync(constants_1.GnuTarPathOnWindows)) { + return constants_1.GnuTarPathOnWindows; + } const versionOutput = yield getVersion('tar'); - return versionOutput.toLowerCase().includes('gnu tar'); + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''; }); } -exports.isGnuTarInstalled = isGnuTarInstalled; +exports.getGnuTarPathOnWindows = getGnuTarPathOnWindows; function assertDefined(name, value) { if (value === undefined) { throw Error(`Expected ${name} but value was undefiend`); @@ -665,6 +676,11 @@ var CompressionMethod; CompressionMethod["ZstdWithoutLong"] = "zstd-without-long"; CompressionMethod["Zstd"] = "zstd"; })(CompressionMethod = exports.CompressionMethod || (exports.CompressionMethod = {})); +var ArchiveToolType; +(function (ArchiveToolType) { + ArchiveToolType["GNU"] = "gnu"; + ArchiveToolType["BSD"] = "bsd"; +})(ArchiveToolType = exports.ArchiveToolType || (exports.ArchiveToolType = {})); // The default number of retry attempts. exports.DefaultRetryAttempts = 2; // The default delay in milliseconds between retry attempts. @@ -673,6 +689,12 @@ exports.DefaultRetryDelay = 5000; // over the socket during this period, the socket is destroyed and the download // is aborted. exports.SocketTimeout = 5000; +// The default path of GNUtar on hosted Windows runners +exports.GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`; +// The default path of BSDtar on hosted Windows runners +exports.SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`; +exports.TarFilename = 'cache.tar'; +exports.ManifestFilename = 'manifest.txt'; //# sourceMappingURL=constants.js.map /***/ }), @@ -1085,27 +1107,27 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", ({ value: true })); const exec_1 = __nccwpck_require__(1514); +const core_1 = __nccwpck_require__(2186); const io = __importStar(__nccwpck_require__(7436)); const fs_1 = __nccwpck_require__(7147); const path = __importStar(__nccwpck_require__(1017)); const utils = __importStar(__nccwpck_require__(1518)); const constants_1 = __nccwpck_require__(8840); const IS_WINDOWS = process.platform === 'win32'; -function getTarPath(args, compressionMethod) { +core_1.exportVariable('MSYS', 'winsymlinks:nativestrict'); +// Returns tar path and type: BSD or GNU +function getTarPath() { return __awaiter(this, void 0, void 0, function* () { switch (process.platform) { case 'win32': { - const systemTar = `${process.env['windir']}\\System32\\tar.exe`; - if (compressionMethod !== constants_1.CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd - args.push('--force-local'); + const gnuTar = yield utils.getGnuTarPathOnWindows(); + const systemTar = constants_1.SystemTarPathOnWindows; + if (gnuTar) { + // Use GNUtar as default on windows + return { path: gnuTar, type: constants_1.ArchiveToolType.GNU }; } else if (fs_1.existsSync(systemTar)) { - return systemTar; - } - else if (yield utils.isGnuTarInstalled()) { - args.push('--force-local'); + return { path: systemTar, type: constants_1.ArchiveToolType.BSD }; } break; } @@ -1113,25 +1135,92 @@ function getTarPath(args, compressionMethod) { const gnuTar = yield io.which('gtar', false); if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - args.push('--delay-directory-restore'); - return gnuTar; + return { path: gnuTar, type: constants_1.ArchiveToolType.GNU }; + } + else { + return { + path: yield io.which('tar', true), + type: constants_1.ArchiveToolType.BSD + }; } - break; } default: break; } - return yield io.which('tar', true); + // Default assumption is GNU tar is present in path + return { + path: yield io.which('tar', true), + type: constants_1.ArchiveToolType.GNU + }; }); } -function execTar(args, compressionMethod, cwd) { +// Return arguments for tar as per tarPath, compressionMethod, method type and os +function getTarArgs(tarPath, compressionMethod, type, archivePath = '') { return __awaiter(this, void 0, void 0, function* () { - try { - yield exec_1.exec(`"${yield getTarPath(args, compressionMethod)}"`, args, { cwd }); + const args = [`"${tarPath.path}"`]; + const cacheFileName = utils.getCacheFileName(compressionMethod); + const tarFile = 'cache.tar'; + const workingDirectory = getWorkingDirectory(); + // Speficic args for BSD tar on windows for workaround + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + // Method specific args + switch (type) { + case 'create': + args.push('--posix', '-cf', BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--exclude', BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', constants_1.ManifestFilename); + break; + case 'extract': + args.push('-xf', BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')); + break; + case 'list': + args.push('-tf', BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P'); + break; } - catch (error) { - throw new Error(`Tar failed with error: ${error === null || error === void 0 ? void 0 : error.message}`); + // Platform specific args + if (tarPath.type === constants_1.ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': + args.push('--force-local'); + break; + case 'darwin': + args.push('--delay-directory-restore'); + break; + } } + return args; + }); +} +// Returns commands to run tar and compression program +function getCommands(compressionMethod, type, archivePath = '') { + return __awaiter(this, void 0, void 0, function* () { + let args; + const tarPath = yield getTarPath(); + const tarArgs = yield getTarArgs(tarPath, compressionMethod, type, archivePath); + const compressionArgs = type !== 'create' + ? yield getDecompressionProgram(tarPath, compressionMethod, archivePath) + : yield getCompressionProgram(tarPath, compressionMethod); + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + if (BSD_TAR_ZSTD && type !== 'create') { + args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')]; + } + else { + args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')]; + } + if (BSD_TAR_ZSTD) { + return args; + } + return [args.join(' ')]; }); } function getWorkingDirectory() { @@ -1139,91 +1228,116 @@ function getWorkingDirectory() { return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd(); } // Common function for extractTar and listTar to get the compression method -function getCompressionProgram(compressionMethod) { - // -d: Decompress. - // unzstd is equivalent to 'zstd -d' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ]; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']; - default: - return ['-z']; - } +function getDecompressionProgram(tarPath, compressionMethod, archivePath) { + return __awaiter(this, void 0, void 0, function* () { + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -d --long=30 --force -o', + constants_1.TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -d --force -o', + constants_1.TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']; + default: + return ['-z']; + } + }); } +// Used for creating the archive +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +function getCompressionProgram(tarPath, compressionMethod) { + return __awaiter(this, void 0, void 0, function* () { + const cacheFileName = utils.getCacheFileName(compressionMethod); + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --long=30 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + constants_1.TarFilename + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + constants_1.TarFilename + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']; + default: + return ['-z']; + } + }); +} +// Executes all commands as separate processes +function execCommands(commands, cwd) { + return __awaiter(this, void 0, void 0, function* () { + for (const command of commands) { + try { + yield exec_1.exec(command, undefined, { cwd }); + } + catch (error) { + throw new Error(`${command.split(' ')[0]} failed with error: ${error === null || error === void 0 ? void 0 : error.message}`); + } + } + }); +} +// List the contents of a tar function listTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - const args = [ - ...getCompressionProgram(compressionMethod), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ]; - yield execTar(args, compressionMethod); + const commands = yield getCommands(compressionMethod, 'list', archivePath); + yield execCommands(commands); }); } exports.listTar = listTar; +// Extract a tar function extractTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into const workingDirectory = getWorkingDirectory(); yield io.mkdirP(workingDirectory); - const args = [ - ...getCompressionProgram(compressionMethod), - '-xf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ]; - yield execTar(args, compressionMethod); + const commands = yield getCommands(compressionMethod, 'extract', archivePath); + yield execCommands(commands); }); } exports.extractTar = extractTar; +// Create a tar function createTar(archiveFolder, sourceDirectories, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = 'manifest.txt'; - const cacheFileName = utils.getCacheFileName(compressionMethod); - fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n')); - const workingDirectory = getWorkingDirectory(); - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ]; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']; - default: - return ['-z']; - } - } - const args = [ - '--posix', - ...getCompressionProgram(), - '-cf', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - manifestFilename - ]; - yield execTar(args, compressionMethod, archiveFolder); + fs_1.writeFileSync(path.join(archiveFolder, constants_1.ManifestFilename), sourceDirectories.join('\n')); + const commands = yield getCommands(compressionMethod, 'create'); + yield execCommands(commands, archiveFolder); }); } exports.createTar = createTar; diff --git a/dist/run/index.js b/dist/run/index.js index 61bd30d..6ebddf4 100644 --- a/dist/run/index.js +++ b/dist/run/index.js @@ -74,9 +74,10 @@ exports.isFeatureAvailable = isFeatureAvailable; * @param primaryKey an explicit key for restoring the cache * @param restoreKeys an optional ordered list of keys to use for restoring the cache if no cache hit occurred for key * @param downloadOptions cache download options + * @param enableCrossOsArchive an optional boolean enabled to restore on windows any cache created on any platform * @returns string returns the key for the cache hit, otherwise returns undefined */ -function restoreCache(paths, primaryKey, restoreKeys, options) { +function restoreCache(paths, primaryKey, restoreKeys, options, enableCrossOsArchive = false) { return __awaiter(this, void 0, void 0, function* () { checkPaths(paths); restoreKeys = restoreKeys || []; @@ -94,7 +95,8 @@ function restoreCache(paths, primaryKey, restoreKeys, options) { try { // path are needed to compute version const cacheEntry = yield cacheHttpClient.getCacheEntry(keys, paths, { - compressionMethod + compressionMethod, + enableCrossOsArchive }); if (!(cacheEntry === null || cacheEntry === void 0 ? void 0 : cacheEntry.archiveLocation)) { // Cache not found @@ -141,10 +143,11 @@ exports.restoreCache = restoreCache; * * @param paths a list of file paths to be cached * @param key an explicit key for restoring the cache + * @param enableCrossOsArchive an optional boolean enabled to save cache on windows which could be restored on any platform * @param options cache upload options * @returns number returns cacheId if the cache was saved successfully and throws an error if save fails */ -function saveCache(paths, key, options) { +function saveCache(paths, key, options, enableCrossOsArchive = false) { var _a, _b, _c, _d, _e; return __awaiter(this, void 0, void 0, function* () { checkPaths(paths); @@ -175,6 +178,7 @@ function saveCache(paths, key, options) { core.debug('Reserving Cache'); const reserveCacheResponse = yield cacheHttpClient.reserveCache(key, paths, { compressionMethod, + enableCrossOsArchive, cacheSize: archiveFileSize }); if ((_a = reserveCacheResponse === null || reserveCacheResponse === void 0 ? void 0 : reserveCacheResponse.result) === null || _a === void 0 ? void 0 : _a.cacheId) { @@ -247,7 +251,6 @@ const crypto = __importStar(__nccwpck_require__(6113)); const fs = __importStar(__nccwpck_require__(7147)); const url_1 = __nccwpck_require__(7310); const utils = __importStar(__nccwpck_require__(1518)); -const constants_1 = __nccwpck_require__(8840); const downloadUtils_1 = __nccwpck_require__(5500); const options_1 = __nccwpck_require__(6215); const requestUtils_1 = __nccwpck_require__(3981); @@ -277,10 +280,17 @@ function createHttpClient() { const bearerCredentialHandler = new auth_1.BearerCredentialHandler(token); return new http_client_1.HttpClient('actions/cache', [bearerCredentialHandler], getRequestOptions()); } -function getCacheVersion(paths, compressionMethod) { - const components = paths.concat(!compressionMethod || compressionMethod === constants_1.CompressionMethod.Gzip - ? [] - : [compressionMethod]); +function getCacheVersion(paths, compressionMethod, enableCrossOsArchive = false) { + const components = paths; + // Add compression method to cache version to restore + // compressed cache as per compression method + if (compressionMethod) { + components.push(compressionMethod); + } + // Only check for windows platforms if enableCrossOsArchive is false + if (process.platform === 'win32' && !enableCrossOsArchive) { + components.push('windows-only'); + } // Add salt to cache version to support breaking changes in cache entry components.push(versionSalt); return crypto @@ -292,9 +302,10 @@ exports.getCacheVersion = getCacheVersion; function getCacheEntry(keys, paths, options) { return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); - const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod); + const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod, options === null || options === void 0 ? void 0 : options.enableCrossOsArchive); const resource = `cache?keys=${encodeURIComponent(keys.join(','))}&version=${version}`; const response = yield requestUtils_1.retryTypedResponse('getCacheEntry', () => __awaiter(this, void 0, void 0, function* () { return httpClient.getJson(getCacheApiUrl(resource)); })); + // Cache not found if (response.statusCode === 204) { // List cache for primary key only if cache miss occurs if (core.isDebug()) { @@ -308,6 +319,7 @@ function getCacheEntry(keys, paths, options) { const cacheResult = response.result; const cacheDownloadUrl = cacheResult === null || cacheResult === void 0 ? void 0 : cacheResult.archiveLocation; if (!cacheDownloadUrl) { + // Cache achiveLocation not found. This should never happen, and hence bail out. throw new Error('Cache not found.'); } core.setSecret(cacheDownloadUrl); @@ -353,7 +365,7 @@ exports.downloadCache = downloadCache; function reserveCache(key, paths, options) { return __awaiter(this, void 0, void 0, function* () { const httpClient = createHttpClient(); - const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod); + const version = getCacheVersion(paths, options === null || options === void 0 ? void 0 : options.compressionMethod, options === null || options === void 0 ? void 0 : options.enableCrossOsArchive); const reserveCacheRequest = { key, version, @@ -596,10 +608,6 @@ function getVersion(app) { // Use zstandard if possible to maximize cache performance function getCompressionMethod() { return __awaiter(this, void 0, void 0, function* () { - if (process.platform === 'win32' && !(yield isGnuTarInstalled())) { - // Disable zstd due to bug https://github.com/actions/cache/issues/301 - return constants_1.CompressionMethod.Gzip; - } const versionOutput = yield getVersion('zstd'); const version = semver.clean(versionOutput); if (!versionOutput.toLowerCase().includes('zstd command line interface')) { @@ -623,13 +631,16 @@ function getCacheFileName(compressionMethod) { : constants_1.CacheFilename.Zstd; } exports.getCacheFileName = getCacheFileName; -function isGnuTarInstalled() { +function getGnuTarPathOnWindows() { return __awaiter(this, void 0, void 0, function* () { + if (fs.existsSync(constants_1.GnuTarPathOnWindows)) { + return constants_1.GnuTarPathOnWindows; + } const versionOutput = yield getVersion('tar'); - return versionOutput.toLowerCase().includes('gnu tar'); + return versionOutput.toLowerCase().includes('gnu tar') ? io.which('tar') : ''; }); } -exports.isGnuTarInstalled = isGnuTarInstalled; +exports.getGnuTarPathOnWindows = getGnuTarPathOnWindows; function assertDefined(name, value) { if (value === undefined) { throw Error(`Expected ${name} but value was undefiend`); @@ -665,6 +676,11 @@ var CompressionMethod; CompressionMethod["ZstdWithoutLong"] = "zstd-without-long"; CompressionMethod["Zstd"] = "zstd"; })(CompressionMethod = exports.CompressionMethod || (exports.CompressionMethod = {})); +var ArchiveToolType; +(function (ArchiveToolType) { + ArchiveToolType["GNU"] = "gnu"; + ArchiveToolType["BSD"] = "bsd"; +})(ArchiveToolType = exports.ArchiveToolType || (exports.ArchiveToolType = {})); // The default number of retry attempts. exports.DefaultRetryAttempts = 2; // The default delay in milliseconds between retry attempts. @@ -673,6 +689,12 @@ exports.DefaultRetryDelay = 5000; // over the socket during this period, the socket is destroyed and the download // is aborted. exports.SocketTimeout = 5000; +// The default path of GNUtar on hosted Windows runners +exports.GnuTarPathOnWindows = `${process.env['PROGRAMFILES']}\\Git\\usr\\bin\\tar.exe`; +// The default path of BSDtar on hosted Windows runners +exports.SystemTarPathOnWindows = `${process.env['SYSTEMDRIVE']}\\Windows\\System32\\tar.exe`; +exports.TarFilename = 'cache.tar'; +exports.ManifestFilename = 'manifest.txt'; //# sourceMappingURL=constants.js.map /***/ }), @@ -1085,27 +1107,27 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", ({ value: true })); const exec_1 = __nccwpck_require__(1514); +const core_1 = __nccwpck_require__(2186); const io = __importStar(__nccwpck_require__(7436)); const fs_1 = __nccwpck_require__(7147); const path = __importStar(__nccwpck_require__(1017)); const utils = __importStar(__nccwpck_require__(1518)); const constants_1 = __nccwpck_require__(8840); const IS_WINDOWS = process.platform === 'win32'; -function getTarPath(args, compressionMethod) { +core_1.exportVariable('MSYS', 'winsymlinks:nativestrict'); +// Returns tar path and type: BSD or GNU +function getTarPath() { return __awaiter(this, void 0, void 0, function* () { switch (process.platform) { case 'win32': { - const systemTar = `${process.env['windir']}\\System32\\tar.exe`; - if (compressionMethod !== constants_1.CompressionMethod.Gzip) { - // We only use zstandard compression on windows when gnu tar is installed due to - // a bug with compressing large files with bsdtar + zstd - args.push('--force-local'); + const gnuTar = yield utils.getGnuTarPathOnWindows(); + const systemTar = constants_1.SystemTarPathOnWindows; + if (gnuTar) { + // Use GNUtar as default on windows + return { path: gnuTar, type: constants_1.ArchiveToolType.GNU }; } else if (fs_1.existsSync(systemTar)) { - return systemTar; - } - else if (yield utils.isGnuTarInstalled()) { - args.push('--force-local'); + return { path: systemTar, type: constants_1.ArchiveToolType.BSD }; } break; } @@ -1113,25 +1135,92 @@ function getTarPath(args, compressionMethod) { const gnuTar = yield io.which('gtar', false); if (gnuTar) { // fix permission denied errors when extracting BSD tar archive with GNU tar - https://github.com/actions/cache/issues/527 - args.push('--delay-directory-restore'); - return gnuTar; + return { path: gnuTar, type: constants_1.ArchiveToolType.GNU }; + } + else { + return { + path: yield io.which('tar', true), + type: constants_1.ArchiveToolType.BSD + }; } - break; } default: break; } - return yield io.which('tar', true); + // Default assumption is GNU tar is present in path + return { + path: yield io.which('tar', true), + type: constants_1.ArchiveToolType.GNU + }; }); } -function execTar(args, compressionMethod, cwd) { +// Return arguments for tar as per tarPath, compressionMethod, method type and os +function getTarArgs(tarPath, compressionMethod, type, archivePath = '') { return __awaiter(this, void 0, void 0, function* () { - try { - yield exec_1.exec(`"${yield getTarPath(args, compressionMethod)}"`, args, { cwd }); + const args = [`"${tarPath.path}"`]; + const cacheFileName = utils.getCacheFileName(compressionMethod); + const tarFile = 'cache.tar'; + const workingDirectory = getWorkingDirectory(); + // Speficic args for BSD tar on windows for workaround + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + // Method specific args + switch (type) { + case 'create': + args.push('--posix', '-cf', BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--exclude', BSD_TAR_ZSTD + ? tarFile + : cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '--files-from', constants_1.ManifestFilename); + break; + case 'extract': + args.push('-xf', BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', '-C', workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/')); + break; + case 'list': + args.push('-tf', BSD_TAR_ZSTD + ? tarFile + : archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P'); + break; } - catch (error) { - throw new Error(`Tar failed with error: ${error === null || error === void 0 ? void 0 : error.message}`); + // Platform specific args + if (tarPath.type === constants_1.ArchiveToolType.GNU) { + switch (process.platform) { + case 'win32': + args.push('--force-local'); + break; + case 'darwin': + args.push('--delay-directory-restore'); + break; + } } + return args; + }); +} +// Returns commands to run tar and compression program +function getCommands(compressionMethod, type, archivePath = '') { + return __awaiter(this, void 0, void 0, function* () { + let args; + const tarPath = yield getTarPath(); + const tarArgs = yield getTarArgs(tarPath, compressionMethod, type, archivePath); + const compressionArgs = type !== 'create' + ? yield getDecompressionProgram(tarPath, compressionMethod, archivePath) + : yield getCompressionProgram(tarPath, compressionMethod); + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + if (BSD_TAR_ZSTD && type !== 'create') { + args = [[...compressionArgs].join(' '), [...tarArgs].join(' ')]; + } + else { + args = [[...tarArgs].join(' '), [...compressionArgs].join(' ')]; + } + if (BSD_TAR_ZSTD) { + return args; + } + return [args.join(' ')]; }); } function getWorkingDirectory() { @@ -1139,91 +1228,116 @@ function getWorkingDirectory() { return (_a = process.env['GITHUB_WORKSPACE']) !== null && _a !== void 0 ? _a : process.cwd(); } // Common function for extractTar and listTar to get the compression method -function getCompressionProgram(compressionMethod) { - // -d: Decompress. - // unzstd is equivalent to 'zstd -d' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' - ]; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd']; - default: - return ['-z']; - } +function getDecompressionProgram(tarPath, compressionMethod, archivePath) { + return __awaiter(this, void 0, void 0, function* () { + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -d --long=30 --force -o', + constants_1.TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -d --long=30"' : 'unzstd --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -d --force -o', + constants_1.TarFilename, + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/') + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -d"' : 'unzstd']; + default: + return ['-z']; + } + }); } +// Used for creating the archive +// -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. +// zstdmt is equivalent to 'zstd -T0' +// --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. +// Using 30 here because we also support 32-bit self-hosted runners. +// Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. +function getCompressionProgram(tarPath, compressionMethod) { + return __awaiter(this, void 0, void 0, function* () { + const cacheFileName = utils.getCacheFileName(compressionMethod); + const BSD_TAR_ZSTD = tarPath.type === constants_1.ArchiveToolType.BSD && + compressionMethod !== constants_1.CompressionMethod.Gzip && + IS_WINDOWS; + switch (compressionMethod) { + case constants_1.CompressionMethod.Zstd: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --long=30 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + constants_1.TarFilename + ] + : [ + '--use-compress-program', + IS_WINDOWS ? '"zstd -T0 --long=30"' : 'zstdmt --long=30' + ]; + case constants_1.CompressionMethod.ZstdWithoutLong: + return BSD_TAR_ZSTD + ? [ + 'zstd -T0 --force -o', + cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + constants_1.TarFilename + ] + : ['--use-compress-program', IS_WINDOWS ? '"zstd -T0"' : 'zstdmt']; + default: + return ['-z']; + } + }); +} +// Executes all commands as separate processes +function execCommands(commands, cwd) { + return __awaiter(this, void 0, void 0, function* () { + for (const command of commands) { + try { + yield exec_1.exec(command, undefined, { cwd }); + } + catch (error) { + throw new Error(`${command.split(' ')[0]} failed with error: ${error === null || error === void 0 ? void 0 : error.message}`); + } + } + }); +} +// List the contents of a tar function listTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { - const args = [ - ...getCompressionProgram(compressionMethod), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ]; - yield execTar(args, compressionMethod); + const commands = yield getCommands(compressionMethod, 'list', archivePath); + yield execCommands(commands); }); } exports.listTar = listTar; +// Extract a tar function extractTar(archivePath, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Create directory to extract tar into const workingDirectory = getWorkingDirectory(); yield io.mkdirP(workingDirectory); - const args = [ - ...getCompressionProgram(compressionMethod), - '-xf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/') - ]; - yield execTar(args, compressionMethod); + const commands = yield getCommands(compressionMethod, 'extract', archivePath); + yield execCommands(commands); }); } exports.extractTar = extractTar; +// Create a tar function createTar(archiveFolder, sourceDirectories, compressionMethod) { return __awaiter(this, void 0, void 0, function* () { // Write source directories to manifest.txt to avoid command length limits - const manifestFilename = 'manifest.txt'; - const cacheFileName = utils.getCacheFileName(compressionMethod); - fs_1.writeFileSync(path.join(archiveFolder, manifestFilename), sourceDirectories.join('\n')); - const workingDirectory = getWorkingDirectory(); - // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. - // zstdmt is equivalent to 'zstd -T0' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. - function getCompressionProgram() { - switch (compressionMethod) { - case constants_1.CompressionMethod.Zstd: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' - ]; - case constants_1.CompressionMethod.ZstdWithoutLong: - return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt']; - default: - return ['-z']; - } - } - const args = [ - '--posix', - ...getCompressionProgram(), - '-cf', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--exclude', - cacheFileName.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P', - '-C', - workingDirectory.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '--files-from', - manifestFilename - ]; - yield execTar(args, compressionMethod, archiveFolder); + fs_1.writeFileSync(path.join(archiveFolder, constants_1.ManifestFilename), sourceDirectories.join('\n')); + const commands = yield getCommands(compressionMethod, 'create'); + yield execCommands(commands, archiveFolder); }); } exports.createTar = createTar; diff --git a/package-lock.json b/package-lock.json index 7d01a5f..3af99ba 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "3.1.0", "license": "MIT", "dependencies": { - "@actions/cache": "^3.1.1", + "@actions/cache": "^3.1.2", "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", "@actions/github": "^5.1.1", @@ -34,9 +34,9 @@ } }, "node_modules/@actions/cache": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.1.1.tgz", - "integrity": "sha512-gOUdNap8FvlpoQAMYWiNPi9Ltt7jKWv9RuUVKg9cp/vQA9qTXoKiBkTioUAgIejh/qf7jrojYn3lCyIRIsoSeQ==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.1.2.tgz", + "integrity": "sha512-3XeKcXIonfIbqvW7gPm/VLOhv1RHQ1dtTgSBCH6OUhCgSTii9bEVgu0PIms7UbLnXeMCKFzECfpbud8fJEvBbQ==", "dependencies": { "@actions/core": "^1.10.0", "@actions/exec": "^1.0.1", @@ -3236,9 +3236,9 @@ }, "dependencies": { "@actions/cache": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.1.1.tgz", - "integrity": "sha512-gOUdNap8FvlpoQAMYWiNPi9Ltt7jKWv9RuUVKg9cp/vQA9qTXoKiBkTioUAgIejh/qf7jrojYn3lCyIRIsoSeQ==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@actions/cache/-/cache-3.1.2.tgz", + "integrity": "sha512-3XeKcXIonfIbqvW7gPm/VLOhv1RHQ1dtTgSBCH6OUhCgSTii9bEVgu0PIms7UbLnXeMCKFzECfpbud8fJEvBbQ==", "requires": { "@actions/core": "^1.10.0", "@actions/exec": "^1.0.1", diff --git a/package.json b/package.json index ec6c1b1..3ff2a7c 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "author": "golangci", "license": "MIT", "dependencies": { - "@actions/cache": "^3.1.1", + "@actions/cache": "^3.1.2", "@actions/core": "^1.10.0", "@actions/exec": "^1.1.1", "@actions/github": "^5.1.1",