monorepo¶
A monorepo: test only the packages a commit touched.
flowchart LR
s0["test api"]
s1["test web"]
s2["test worker"]
The steps this script plans for a push about which nothing is known. An arrow is a needs.
# A monorepo: test only the packages a commit touched.
#
# Each package's step is given that package's directory and the shared library
# as inputs, so a result is also reused whenever those are unchanged, even if
# the commit touched something else entirely.
PACKAGES = ["api", "web", "worker"]
SHARED = ["libs/common", "package.json", "package-lock.json"]
def test(package):
return step(
"test " + package,
image = "node:22-bookworm-slim",
run = "npm ci && npm test --workspace packages/" + package,
inputs = ["packages/" + package] + SHARED,
caches = {"npm": "/root/.npm"},
)
def ci(ctx):
touched = [f.split("/")[1] for f in ctx.changed_files if f.startswith("packages/") and f.count("/") >= 2]
# A change to something every package depends on means all of them.
shared_changed = any([f.startswith("libs/") or f in SHARED for f in ctx.changed_files])
if shared_changed or not ctx.changed_files:
touched = PACKAGES
return [test(p) for p in PACKAGES if p in touched]
# Generated by `starci compile` from monorepo.ci.star. Edit that, not this.
#
# Each job is one step of the plan: the same image, command, tools and
# packages, given only the step's inputs. A job skips its work when it has
# already passed with the same inputs.
#
# What does not carry over:
# - The script decides by which files a commit changed. A workflow file
# cannot, so it does what the script does when nothing is known about the
# changes, on every run.
# - Passes are remembered in this repository's Actions cache, which GitHub
# scopes by branch and evicts when old. starci's own store has neither
# limit.
name: CI
on:
push:
branches: ['**']
tags: ['**']
pull_request:
workflow_dispatch:
jobs:
test-api:
name: "test api"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
sparse-checkout-cone-mode: false
sparse-checkout: |
/libs/common
/package-lock.json
/package.json
/packages/api
- name: Has this already passed with these inputs?
id: passed
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-passed
key: starci-passed-test-api-f3340bab756f5a08-${{ hashFiles('libs/common', 'libs/common/**', 'package-lock.json', 'package-lock.json/**', 'package.json', 'package.json/**', 'packages/api', 'packages/api/**') }}
- name: "Cache: npm"
if: steps.passed.outputs.cache-hit != 'true'
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-cache/npm
key: starci-cache-npm-${{ github.run_id }}
restore-keys: starci-cache-npm-
- name: "test api"
if: steps.passed.outputs.cache-hit != 'true'
env:
STARCI_RUN: |-
npm ci && npm test --workspace packages/api
run: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$RUNNER_TEMP/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- name: Remember this pass
if: steps.passed.outputs.cache-hit != 'true'
run: mkdir -p "$RUNNER_TEMP/starci-passed" && date > "$RUNNER_TEMP/starci-passed/at"
test-web:
name: "test web"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
sparse-checkout-cone-mode: false
sparse-checkout: |
/libs/common
/package-lock.json
/package.json
/packages/web
- name: Has this already passed with these inputs?
id: passed
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-passed
key: starci-passed-test-web-85d544780b9ce3be-${{ hashFiles('libs/common', 'libs/common/**', 'package-lock.json', 'package-lock.json/**', 'package.json', 'package.json/**', 'packages/web', 'packages/web/**') }}
- name: "Cache: npm"
if: steps.passed.outputs.cache-hit != 'true'
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-cache/npm
key: starci-cache-npm-${{ github.run_id }}
restore-keys: starci-cache-npm-
- name: "test web"
if: steps.passed.outputs.cache-hit != 'true'
env:
STARCI_RUN: |-
npm ci && npm test --workspace packages/web
run: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$RUNNER_TEMP/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- name: Remember this pass
if: steps.passed.outputs.cache-hit != 'true'
run: mkdir -p "$RUNNER_TEMP/starci-passed" && date > "$RUNNER_TEMP/starci-passed/at"
test-worker:
name: "test worker"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
sparse-checkout-cone-mode: false
sparse-checkout: |
/libs/common
/package-lock.json
/package.json
/packages/worker
- name: Has this already passed with these inputs?
id: passed
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-passed
key: starci-passed-test-worker-39e3d8cb7f23d06f-${{ hashFiles('libs/common', 'libs/common/**', 'package-lock.json', 'package-lock.json/**', 'package.json', 'package.json/**', 'packages/worker', 'packages/worker/**') }}
- name: "Cache: npm"
if: steps.passed.outputs.cache-hit != 'true'
uses: actions/cache@v4
with:
path: ${{ runner.temp }}/starci-cache/npm
key: starci-cache-npm-${{ github.run_id }}
restore-keys: starci-cache-npm-
- name: "test worker"
if: steps.passed.outputs.cache-hit != 'true'
env:
STARCI_RUN: |-
npm ci && npm test --workspace packages/worker
run: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$RUNNER_TEMP/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- name: Remember this pass
if: steps.passed.outputs.cache-hit != 'true'
run: mkdir -p "$RUNNER_TEMP/starci-passed" && date > "$RUNNER_TEMP/starci-passed/at"
# Generated by `starci compile --target circleci` from monorepo.ci.star. Edit that, not this.
#
# Each job is one step of the plan: the same image, command, tools and
# packages, given only the step's inputs. A job skips its work when it has
# already passed with the same inputs.
#
# What does not carry over:
# - The script reads ctx.changed_files, so its plan depends on the commit.
# This file is the plan for a commit about which nothing is known, and is
# the same for every push: steps the script would add or drop for a
# particular branch or set of changed files are not represented.
# - Passes are remembered in this project's CircleCI cache, which expires.
# starci's own store does not.
version: 2.1
jobs:
test-api:
machine:
image: ubuntu-2404:current
resource_class: medium
steps:
- checkout
- run:
name: "Keep only this step's inputs"
command: |
git sparse-checkout set --no-cone '/libs/common' '/package-lock.json' '/package.json' '/packages/api'
- run:
name: "Identify the inputs"
command: |
git ls-files -s -- ':(top,literal)libs/common' ':(top,literal)package-lock.json' ':(top,literal)package.json' ':(top,literal)packages/api' > /tmp/starci-inputs
- restore_cache:
name: Has this already passed with these inputs?
keys:
- starci-passed-test-api-f3340bab756f5a08-{{ checksum "/tmp/starci-inputs" }}
- run:
name: "Stop here if it has"
command: |
if [ -e ~/starci-passed/at ]; then
echo "Passed before with the same inputs, on $(cat ~/starci-passed/at). Nothing to do."
circleci-agent step halt
fi
- restore_cache:
name: "Cache: npm"
keys:
- starci-cache-npm-
- run:
name: "test api"
environment:
STARCI_RUN: |-
npm ci && npm test --workspace packages/api
command: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$HOME/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- save_cache:
name: "Save cache: npm"
key: starci-cache-npm-{{ .Revision }}
paths:
- ~/starci-cache/npm
- run:
name: "Remember this pass"
command: |
mkdir -p ~/starci-passed && date > ~/starci-passed/at
- save_cache:
key: starci-passed-test-api-f3340bab756f5a08-{{ checksum "/tmp/starci-inputs" }}
paths:
- ~/starci-passed
test-web:
machine:
image: ubuntu-2404:current
resource_class: medium
steps:
- checkout
- run:
name: "Keep only this step's inputs"
command: |
git sparse-checkout set --no-cone '/libs/common' '/package-lock.json' '/package.json' '/packages/web'
- run:
name: "Identify the inputs"
command: |
git ls-files -s -- ':(top,literal)libs/common' ':(top,literal)package-lock.json' ':(top,literal)package.json' ':(top,literal)packages/web' > /tmp/starci-inputs
- restore_cache:
name: Has this already passed with these inputs?
keys:
- starci-passed-test-web-85d544780b9ce3be-{{ checksum "/tmp/starci-inputs" }}
- run:
name: "Stop here if it has"
command: |
if [ -e ~/starci-passed/at ]; then
echo "Passed before with the same inputs, on $(cat ~/starci-passed/at). Nothing to do."
circleci-agent step halt
fi
- restore_cache:
name: "Cache: npm"
keys:
- starci-cache-npm-
- run:
name: "test web"
environment:
STARCI_RUN: |-
npm ci && npm test --workspace packages/web
command: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$HOME/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- save_cache:
name: "Save cache: npm"
key: starci-cache-npm-{{ .Revision }}
paths:
- ~/starci-cache/npm
- run:
name: "Remember this pass"
command: |
mkdir -p ~/starci-passed && date > ~/starci-passed/at
- save_cache:
key: starci-passed-test-web-85d544780b9ce3be-{{ checksum "/tmp/starci-inputs" }}
paths:
- ~/starci-passed
test-worker:
machine:
image: ubuntu-2404:current
resource_class: medium
steps:
- checkout
- run:
name: "Keep only this step's inputs"
command: |
git sparse-checkout set --no-cone '/libs/common' '/package-lock.json' '/package.json' '/packages/worker'
- run:
name: "Identify the inputs"
command: |
git ls-files -s -- ':(top,literal)libs/common' ':(top,literal)package-lock.json' ':(top,literal)package.json' ':(top,literal)packages/worker' > /tmp/starci-inputs
- restore_cache:
name: Has this already passed with these inputs?
keys:
- starci-passed-test-worker-39e3d8cb7f23d06f-{{ checksum "/tmp/starci-inputs" }}
- run:
name: "Stop here if it has"
command: |
if [ -e ~/starci-passed/at ]; then
echo "Passed before with the same inputs, on $(cat ~/starci-passed/at). Nothing to do."
circleci-agent step halt
fi
- restore_cache:
name: "Cache: npm"
keys:
- starci-cache-npm-
- run:
name: "test worker"
environment:
STARCI_RUN: |-
npm ci && npm test --workspace packages/worker
command: |
docker run --rm \
-v "$PWD:/workspace" \
-w /workspace \
-v "$HOME/starci-cache/npm:/root/.npm" \
node:22-bookworm-slim sh -c "$STARCI_RUN"
- save_cache:
name: "Save cache: npm"
key: starci-cache-npm-{{ .Revision }}
paths:
- ~/starci-cache/npm
- run:
name: "Remember this pass"
command: |
mkdir -p ~/starci-passed && date > ~/starci-passed/at
- save_cache:
key: starci-passed-test-worker-39e3d8cb7f23d06f-{{ checksum "/tmp/starci-inputs" }}
paths:
- ~/starci-passed
workflows:
ci:
jobs:
- test-api
- test-web
- test-worker
Download: monorepo.ci.star, GitHub Actions, CircleCI