improve caching

Dockerfile

@@ -1,5 +0,0 @@
-FROM golangci/golangci-lint:v1.26
-
-COPY entrypoint.sh /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]

README.md

@@ -49,6 +49,14 @@ The restrictions of annotations are the following:
 1. Currently, they don't support markdown formatting (see the [feature request](https://github.community/t5/GitHub-API-Development-and/Checks-Ability-to-include-Markdown-in-line-annotations/m-p/56704))
 2. They aren't shown in list of comments like it was with [golangci.com](https://golangci.com). If you would like to have comments - please, up-vote [the issue](https://github.com/golangci/golangci-lint-action/issues/5).
 
+## Performance
+
+The action was implemented with performance in mind:
+
+1. We cache data by [@actions/cache](https://github.com/actions/cache) between builds: Go build cache, Go modules cache, golangci-lint analysis cache.
+2. We don't use Docker because image pulling is slow.
+3. We do as much as we can in parallel, e.g. we download cache, go and golangci-lint binary in parallel.
+
 ## Internals
 
 We use JavaScript-based action. We don't use Docker-based action because:
@@ -59,12 +67,21 @@ We use JavaScript-based action. We don't use Docker-based action because:
 Inside our action we perform 3 steps:
 
 1. Setup environment running in parallel:
-  * restore [cache](https://github.com/actions/cache) of previous analyzes into `$HOME/.cache/golangci-lint`
+  * restore [cache](https://github.com/actions/cache) of previous analyzes
   * list [releases of golangci-lint](https://github.com/golangci/golangci-lint/releases) and find the latest patch version
     for needed version (users of this action can specify only minor version). After that install [golangci-lint](https://github.com/golangci/golangci-lint) using [@actions/tool-cache](https://github.com/actions/toolkit/tree/master/packages/tool-cache)
   * install the latest Go 1.x version using [@actions/setup-go](https://github.com/actions/setup-go)
 2. Run `golangci-lint` with specified by user `args`
-3. Save cache from `$HOME/.cache/golangci-lint` for later builds
+3. Save cache for later builds
+
+### Caching internals
+
+1. We save and restore the following directories: `~/.cache/golangci-lint`, `~/.cache/go-build`, `~/go/pkg`.
+2. The primary caching key looks like `golangci-lint.cache-{interval_number}-{go.mod_hash}`. Interval number ensures that we periodically invalidate
+   our cache (every 7 days). `go.mod` hash ensures that we invalidate the cache early - as soon as dependencies have changed.
+3. We use [restore keys](https://help.github.com/en/actions/configuring-and-managing-workflows/caching-dependencies-to-speed-up-workflows#matching-a-cache-key): `golangci-lint.cache-{interval_number}-`, `golangci-lint.cache-`. GitHub matches keys by prefix if we have no exact match for the primary cache.
+
+This scheme is basic and needs improvements. Pull requests and ideas are welcome.
 
 ## Development of this action
 

dist/post_run/index.js

@@ -27870,29 +27870,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const core = __importStar(__webpack_require__(470));
 const restore_1 = __importDefault(__webpack_require__(206));
 const save_1 = __importDefault(__webpack_require__(781));
-// TODO: ensure dir exists, have access, etc
+const crypto = __importStar(__webpack_require__(417));
-const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
+const fs = __importStar(__webpack_require__(747));
-const setCacheInputs = () => {
+function checksumFile(hashName, path) {
-    process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
+    return new Promise((resolve, reject) => {
-    process.env.INPUT_PATH = getCacheDir();
+        const hash = crypto.createHash(hashName);
+        const stream = fs.createReadStream(path);
+        stream.on("error", err => reject(err));
+        stream.on("data", chunk => hash.update(chunk));
+        stream.on("end", () => resolve(hash.digest("hex")));
+    });
+}
+const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
+const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
+const getCacheDirs = () => {
+    // Not existing dirs are ok here: it works.
+    return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
 };
+const getIntervalKey = (invalidationIntervalDays) => {
+    const now = new Date();
+    const secondsSinceEpoch = now.getTime() / 1000;
+    const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
+    return intervalNumber.toString();
+};
+function buildCacheKeys() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const keys = [];
+        let cacheKey = `golangci-lint.cache-`;
+        keys.push(cacheKey);
+        // Periodically invalidate a cache because a new code being added.
+        // TODO: configure it via inputs.
+        cacheKey += `${getIntervalKey(7)}-`;
+        keys.push(cacheKey);
+        if (yield pathExists(`go.mod`)) {
+            // Add checksum to key to invalidate a cache when dependencies change.
+            cacheKey += yield checksumFile(`cache-key`, `go.mod`);
+        }
+        else {
+            cacheKey += `nogomod`;
+        }
+        keys.push(cacheKey);
+        return keys;
+    });
+}
 function restoreCache() {
     return __awaiter(this, void 0, void 0, function* () {
         const startedAt = Date.now();
-        setCacheInputs();
+        const keys = yield buildCacheKeys();
+        const primaryKey = keys.pop();
+        const restoreKeys = keys.reverse();
+        core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
+        process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
+        process.env.INPUT_KEY = primaryKey;
+        process.env.INPUT_PATH = getCacheDirs().join(`\n`);
         // Tell golangci-lint to use our cache directory.
-        process.env.GOLANGCI_LINT_CACHE = getCacheDir();
+        process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
         yield restore_1.default();
-        core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
+        core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
     });
 }
 exports.restoreCache = restoreCache;
 function saveCache() {
     return __awaiter(this, void 0, void 0, function* () {
         const startedAt = Date.now();
-        setCacheInputs();
+        const cacheDirs = getCacheDirs();
+        process.env.INPUT_PATH = cacheDirs.join(`\n`);
         yield save_1.default();
-        core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
+        core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
     });
 }
 exports.saveCache = saveCache;

dist/run/index.js

@@ -27882,29 +27882,73 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const core = __importStar(__webpack_require__(470));
 const restore_1 = __importDefault(__webpack_require__(206));
 const save_1 = __importDefault(__webpack_require__(781));
-// TODO: ensure dir exists, have access, etc
+const crypto = __importStar(__webpack_require__(417));
-const getCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
+const fs = __importStar(__webpack_require__(747));
-const setCacheInputs = () => {
+function checksumFile(hashName, path) {
-    process.env.INPUT_KEY = `golangci-lint.analysis-cache`;
+    return new Promise((resolve, reject) => {
-    process.env.INPUT_PATH = getCacheDir();
+        const hash = crypto.createHash(hashName);
+        const stream = fs.createReadStream(path);
+        stream.on("error", err => reject(err));
+        stream.on("data", chunk => hash.update(chunk));
+        stream.on("end", () => resolve(hash.digest("hex")));
+    });
+}
+const pathExists = (path) => __awaiter(void 0, void 0, void 0, function* () { return !!(yield fs.promises.stat(path).catch(e => false)); });
+const getLintCacheDir = () => `${process.env.HOME}/.cache/golangci-lint`;
+const getCacheDirs = () => {
+    // Not existing dirs are ok here: it works.
+    return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`];
 };
+const getIntervalKey = (invalidationIntervalDays) => {
+    const now = new Date();
+    const secondsSinceEpoch = now.getTime() / 1000;
+    const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400));
+    return intervalNumber.toString();
+};
+function buildCacheKeys() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const keys = [];
+        let cacheKey = `golangci-lint.cache-`;
+        keys.push(cacheKey);
+        // Periodically invalidate a cache because a new code being added.
+        // TODO: configure it via inputs.
+        cacheKey += `${getIntervalKey(7)}-`;
+        keys.push(cacheKey);
+        if (yield pathExists(`go.mod`)) {
+            // Add checksum to key to invalidate a cache when dependencies change.
+            cacheKey += yield checksumFile(`cache-key`, `go.mod`);
+        }
+        else {
+            cacheKey += `nogomod`;
+        }
+        keys.push(cacheKey);
+        return keys;
+    });
+}
 function restoreCache() {
     return __awaiter(this, void 0, void 0, function* () {
         const startedAt = Date.now();
-        setCacheInputs();
+        const keys = yield buildCacheKeys();
+        const primaryKey = keys.pop();
+        const restoreKeys = keys.reverse();
+        core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`);
+        process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`);
+        process.env.INPUT_KEY = primaryKey;
+        process.env.INPUT_PATH = getCacheDirs().join(`\n`);
         // Tell golangci-lint to use our cache directory.
-        process.env.GOLANGCI_LINT_CACHE = getCacheDir();
+        process.env.GOLANGCI_LINT_CACHE = getLintCacheDir();
         yield restore_1.default();
-        core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
+        core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`);
     });
 }
 exports.restoreCache = restoreCache;
 function saveCache() {
     return __awaiter(this, void 0, void 0, function* () {
         const startedAt = Date.now();
-        setCacheInputs();
+        const cacheDirs = getCacheDirs();
+        process.env.INPUT_PATH = cacheDirs.join(`\n`);
         yield save_1.default();
-        core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`);
+        core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`);
     });
 }
 exports.saveCache = saveCache;

entrypoint.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-echo 'golangci-lint-action: start'
-echo " flags:  ${INPUT_FLAGS}"
-echo " format: ${INPUT_FORMAT}"
-
-cd "${GITHUB_WORKSPACE}/${INPUT_DIRECTORY}" || exit 1
-
-# shellcheck disable=SC2086
-golangci-lint run --out-format ${INPUT_FORMAT} ${INPUT_FLAGS}

src/cache.ts

@@ -1,29 +1,81 @@
 import * as core from "@actions/core"
 import restore from "cache/lib/restore"
 import save from "cache/lib/save"
+import * as crypto from "crypto"
+import * as fs from "fs"
 
-// TODO: ensure dir exists, have access, etc
+function checksumFile(hashName: string, path: string) {
-const getCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
+  return new Promise((resolve, reject) => {
+    const hash = crypto.createHash(hashName)
+    const stream = fs.createReadStream(path)
+    stream.on("error", err => reject(err))
+    stream.on("data", chunk => hash.update(chunk))
+    stream.on("end", () => resolve(hash.digest("hex")))
+  })
+}
 
-const setCacheInputs = (): void => {
+const pathExists = async (path: string) => !!(await fs.promises.stat(path).catch(e => false))
-  process.env.INPUT_KEY = `golangci-lint.analysis-cache`
+
-  process.env.INPUT_PATH = getCacheDir()
+const getLintCacheDir = (): string => `${process.env.HOME}/.cache/golangci-lint`
+
+const getCacheDirs = (): string[] => {
+  // Not existing dirs are ok here: it works.
+  return [getLintCacheDir(), `${process.env.HOME}/.cache/go-build`, `${process.env.HOME}/go/pkg`]
+}
+
+const getIntervalKey = (invalidationIntervalDays: number): string => {
+  const now = new Date()
+  const secondsSinceEpoch = now.getTime() / 1000
+  const intervalNumber = Math.floor(secondsSinceEpoch / (invalidationIntervalDays * 86400))
+  return intervalNumber.toString()
+}
+
+async function buildCacheKeys(): Promise<string[]> {
+  const keys = []
+  let cacheKey = `golangci-lint.cache-`
+  keys.push(cacheKey)
+
+  // Periodically invalidate a cache because a new code being added.
+  // TODO: configure it via inputs.
+  cacheKey += `${getIntervalKey(7)}-`
+  keys.push(cacheKey)
+
+  if (await pathExists(`go.mod`)) {
+    // Add checksum to key to invalidate a cache when dependencies change.
+    cacheKey += await checksumFile(`cache-key`, `go.mod`)
+  } else {
+    cacheKey += `nogomod`
+  }
+  keys.push(cacheKey)
+
+  return keys
 }
 
 export async function restoreCache(): Promise<void> {
   const startedAt = Date.now()
-  setCacheInputs()
+
+  const keys = await buildCacheKeys()
+  const primaryKey = keys.pop()
+  const restoreKeys = keys.reverse()
+  core.info(`Primary analysis cache key is '${primaryKey}', restore keys are '${restoreKeys.join(` | `)}'`)
+  process.env[`INPUT_RESTORE-KEYS`] = restoreKeys.join(`\n`)
+  process.env.INPUT_KEY = primaryKey
+
+  process.env.INPUT_PATH = getCacheDirs().join(`\n`)
 
   // Tell golangci-lint to use our cache directory.
-  process.env.GOLANGCI_LINT_CACHE = getCacheDir()
+  process.env.GOLANGCI_LINT_CACHE = getLintCacheDir()
 
   await restore()
-  core.info(`Restored golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
+  core.info(`Restored cache for golangci-lint from key '${primaryKey}' in ${Date.now() - startedAt}ms`)
 }
 
 export async function saveCache(): Promise<void> {
   const startedAt = Date.now()
-  setCacheInputs()
+
+  const cacheDirs = getCacheDirs()
+  process.env.INPUT_PATH = cacheDirs.join(`\n`)
+
   await save()
-  core.info(`Saved golangci-lint analysis cache in ${Date.now() - startedAt}ms`)
+  core.info(`Saved cache for golangci-lint from paths '${cacheDirs.join(`, `)}' in ${Date.now() - startedAt}ms`)
 }