From 5e5fbe724b2b620d7cbabd8e0d7bcf6b65a4180c Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Thu, 21 May 2020 13:10:26 +0300 Subject: [PATCH 1/5] examples: Reuse cargo registry cache across OSes The registry filesystem format and the git repositories work across platforms, no need to maintain runner-specific caches for these. --- examples.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples.md b/examples.md index 8ead3cf..21a5f6c 100644 --- a/examples.md +++ b/examples.md @@ -251,7 +251,7 @@ Esy allows you to export built dependencies and import pre-built dependencies. ...(Build job)... # Re-export dependencies if anything has changed or if it is the first time - - name: Setting dependency cache + - name: Setting dependency cache run: | esy export-dependencies if: steps.restore-cache.outputs.cache-hit != 'true' @@ -428,13 +428,18 @@ When dependencies are installed later in the workflow, we must specify the same ## Rust - Cargo ```yaml -- uses: actions/cache@v2 +- name: Cache cargo dependencies + uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + key: cargo-deps-${{ hashFiles('**/Cargo.lock') }} +- name: Cache cargo build + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }} ``` ## Scala - SBT From f478dc2492cb3bb400aeafa26ac10e2987d3d9cb Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Thu, 21 May 2020 16:22:09 +0300 Subject: [PATCH 2/5] examples: Only cache non-redundant cargo state As the packed crate sources are stored under ~/.cargo/registry/cache, it's redundant to archive the unpacked sources. The sufficient local state of the git dependencies is in ~/.cargo/git/db. Cargo automatically checks out the source trees if missing. --- examples.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples.md b/examples.md index 21a5f6c..5572988 100644 --- a/examples.md +++ b/examples.md @@ -432,8 +432,9 @@ When dependencies are installed later in the workflow, we must specify the same uses: actions/cache@v2 with: path: | - ~/.cargo/registry - ~/.cargo/git + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db key: cargo-deps-${{ hashFiles('**/Cargo.lock') }} - name: Cache cargo build uses: actions/cache@v2 From 25c295a81d91613b13d0ff56920010300760dcd2 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Fri, 22 May 2020 01:31:59 +0300 Subject: [PATCH 3/5] Added a more elaborate Rust example This example could be used by projects that build Rust end product crates on an OS matrix, but share the cargo cache. To avoid fetching the dependencies from the network in each matrix job in case of a cache miss, a dedicated job pre-populates the cache on Linux, to be reused by the jobs in the matrix. Also demonstrate separate proximate caching of the registry index. --- examples.md | 93 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/examples.md b/examples.md index 5572988..66e27a7 100644 --- a/examples.md +++ b/examples.md @@ -427,6 +427,14 @@ When dependencies are installed later in the workflow, we must specify the same ## Rust - Cargo +### Simple end product build + +If `Cargo.lock` is checked into git, its hash can be used as a key +to cache filesystem state suitable for the build. Use the `--locked` option +with cargo build and test commands to ensure that the state cached at the +post step corresponds to the contents of `Cargo.lock` that were hashed for +the key. + ```yaml - name: Cache cargo dependencies uses: actions/cache@v2 @@ -443,6 +451,91 @@ When dependencies are installed later in the workflow, we must specify the same key: ${{ runner.os }}-cargo-build-${{ hashFiles('**/Cargo.lock') }} ``` +### A separate job to fetch and cache the dependencies + +The files cached from `$CARGO_HOME` are platform-independent. +If cargo build/test jobs are run on a matrix and `Cargo.lock` changes often, +it might make sense to populate the cache with the matching state in one job, +then reuse it in the matrix jobs. + +This example also uses a separate cache to avoid expensive syncs with the +`crates.io-index` repository. + +```yaml +jobs: + update-deps: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - id: cargo-deps + name: Cache cargo dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + key: cargo-deps-${{ hashFiles('**/Cargo.lock') }} + + - if: ${{ steps.cargo-deps.outputs.cache-hit != 'true' }} + id: ls-crates-io-index + name: Get head commit hash of crates.io registry index + shell: bash + run: | + commit=$( + git ls-remote --heads https://github.com/rust-lang/crates.io-index.git master | + cut -f 1 + ) + echo "::set-output name=head::$commit" + - if: ${{ steps.cargo-deps.outputs.cache-hit != 'true' }} + name: Cache cargo registry index + uses: actions/cache@v2 + with: + path: ~/.cargo/registry/index + key: cargo-index-${{ steps.ls-crates-io-index.outputs.head }} + restore-keys: cargo-index- + + - if: ${{ steps.cargo-deps.outputs.cache-hit != 'true' }} + name: Fetch dependencies and update registry index + run: cargo fetch --locked + + test: + needs: update-deps + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + # https://github.com/actions/runner/issues/498 + - if: ${{ runner.os == 'Windows' }} + name: Fix up Cargo.lock hash + shell: powershell + run: | + Get-ChildItem . -Recurse -Filter Cargo.lock | + Foreach-Object { + ((Get-Content $_.FullName) -join "`n") + "`n" | + Set-Content -NoNewline $_.FullName + } + + - name: Restore cargo dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry/index + ~/.cargo/registry/cache + ~/.cargo/git/db + key: cargo-deps-${{ hashFiles('**/Cargo.lock') }} + + - name: Build and test + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked +``` + ## Scala - SBT ```yaml From 8d9c7e9644dd63330064c4862fc66faf2cd0eb68 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Sun, 24 May 2020 06:10:04 +0300 Subject: [PATCH 4/5] Add an example for testing a Rust library No Cargo.lock in the checkout, so the workflow has to generate it first. --- examples.md | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/examples.md b/examples.md index 66e27a7..7a32993 100644 --- a/examples.md +++ b/examples.md @@ -536,6 +536,103 @@ jobs: args: --locked ``` +### Testing a library + +Rust library projects typically [do not put][cargo-faq] `Cargo.lock` under +version control. It can be generated with the `cargo generate-lockfile` command +assisted by the registry index cache that is used and possibly updated +for every workflow run. The same index cache is reused by the test jobs. + +[cargo-faq]: https://doc.rust-lang.org/cargo/faq.html#why-do-binaries-have-cargolock-in-version-control-but-not-libraries + +```yaml +jobs: + update-deps: + runs-on: ubuntu-latest + outputs: + crates-io-index-head: ${{ steps.ls-crates-io-index.outputs.head }} + steps: + - uses: actions/checkout@v2 + + - id: ls-crates-io-index + name: Get head commit hash of crates.io registry index + shell: bash + run: | + commit=$( + git ls-remote --heads https://github.com/rust-lang/crates.io-index.git master | + cut -f 1 + ) + echo "::set-output name=head::$commit" + + - name: Cache cargo registry index + uses: actions/cache@v2 + with: + path: ~/.cargo/registry/index + key: cargo-index-${{ steps.ls-crates-io-index.outputs.head }} + restore-keys: cargo-index- + + - name: Generate Cargo.lock + run: cargo generate-lockfile + + - id: cargo-deps + name: Cache dependency crates + uses: actions/cache@v2 + with: + path: ~/.cargo/registry/cache + key: cargo-deps-${{ hashFiles('Cargo.lock') }} + + - if: ${{ steps.cargo-deps.outputs.cache-hit != 'true' }} + name: Fetch dependencies + run: cargo fetch --locked + + - name: Upload Cargo.lock + uses: actions/upload-artifact@v2 + with: + name: lockfile + path: Cargo.lock + + test: + needs: update-deps + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + + - name: Download Cargo.lock + uses: actions/download-artifact@v2 + with: + name: lockfile + + - name: Restore cargo registry index + uses: actions/cache@v2 + with: + path: ~/.cargo/registry/index + key: cargo-index-${{ needs.update-deps.outputs.crates-io-index-head }} + # May miss on Windows: + # https://github.com/actions/cache/issues/330#issuecomment-637701649 + restore-keys: cargo-index- + + - name: Restore dependency crates + uses: actions/cache@v2 + with: + path: ~/.cargo/registry/cache + key: cargo-deps-${{ hashFiles('Cargo.lock') }} + + - name: Build + uses: actions-rs/cargo@v1 + with: + command: build + args: --all-targets --locked + + - name: Test + uses: actions-rs/cargo@v1 + with: + command: test + args: --locked +``` + ## Scala - SBT ```yaml From 15978f289c74724905dd00c3b91f2591882d475a Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Sat, 13 Jun 2020 07:03:49 +0300 Subject: [PATCH 5/5] Set CARGO_INCREMENTAL to 0 in examples This speeds up builds unless the target directory is also cached. Conveniently, the simple example which features such caching omits the build step. --- examples.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples.md b/examples.md index 7a32993..31fd605 100644 --- a/examples.md +++ b/examples.md @@ -531,6 +531,8 @@ jobs: - name: Build and test uses: actions-rs/cargo@v1 + env: + CARGO_INCREMENTAL: 0 with: command: test args: --locked @@ -597,6 +599,8 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} + env: + CARGO_INCREMENTAL: 0 steps: - uses: actions/checkout@v2