Skip to content

Commit

Permalink
workflows/eval.yml: Run on dev branch pushes and compute outpath diff…
Browse files Browse the repository at this point in the history
…erence
  • Loading branch information
infinisil committed Nov 27, 2024
1 parent 0892479 commit 13717ef
Show file tree
Hide file tree
Showing 4 changed files with 234 additions and 16 deletions.
111 changes: 95 additions & 16 deletions .github/workflows/eval.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
name: Eval

on: pull_request_target
on:
pull_request_target:
push:
# Keep this synced with ci/request-reviews/dev-branches.txt
branches:
- master
- staging
- release-*
- staging-*
- haskell-updates

permissions:
contents: read
Expand All @@ -11,6 +20,7 @@ jobs:
runs-on: ubuntu-latest
outputs:
mergedSha: ${{ steps.merged.outputs.mergedSha }}
baseSha: ${{ steps.baseSha.outputs.baseSha }}
systems: ${{ steps.systems.outputs.systems }}
steps:
# Important: Because of `pull_request_target`, this doesn't check out the PR,
Expand All @@ -24,23 +34,39 @@ jobs:
id: merged
env:
GH_TOKEN: ${{ github.token }}
GH_EVENT: ${{ github.event_name }}
run: |
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
echo "Checking the merge commit $mergedSha"
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
else
# Skipping so that no notifications are sent
echo "Skipping the rest..."
fi
case "$GH_EVENT" in
push)
echo "mergedSha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
;;
pull_request_target)
if mergedSha=$(base/ci/get-merge-commit.sh ${{ github.repository }} ${{ github.event.number }}); then
echo "Checking the merge commit $mergedSha"
echo "mergedSha=$mergedSha" >> "$GITHUB_OUTPUT"
else
# Skipping so that no notifications are sent
echo "Skipping the rest..."
fi
;;
esac
rm -rf base
- name: Check out the PR at the test merge commit
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
# Add this to _all_ subsequent steps to skip them
if: steps.merged.outputs.mergedSha
with:
ref: ${{ steps.merged.outputs.mergedSha }}
fetch-depth: 2
path: nixpkgs

- name: Determine base commit
if: github.event_name == 'pull_request_target' && steps.merged.outputs.mergedSha
id: baseSha
run: |
baseSha=$(git -C nixpkgs rev-parse HEAD^1)
echo "baseSha=$baseSha" >> "$GITHUB_OUTPUT"
- name: Install Nix
uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30
if: steps.merged.outputs.mergedSha
Expand Down Expand Up @@ -124,18 +150,71 @@ jobs:
- name: Combine all output paths and eval stats
run: |
nix-build nixpkgs/ci -A eval.combine \
--arg resultsDir ./intermediate
--arg resultsDir ./intermediate \
-o prResult
- name: Upload the combined results
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: result
path: result/*
path: prResult/*

- name: Get base run id
if: needs.attrs.outputs.baseSha
id: baseRunId
run: |
set -e
# TODO: Wait until it's done
# Get the latest eval.yml workflow run for the PR's base commit
if ! run=$(gh api --method GET /repos/"$REPOSITORY"/actions/workflows/eval.yml/runs \
-f head_sha="$BASE_SHA" \
--jq '.workflow_runs | sort_by(.run_started_at) | .[-1]')
|| [[ -z "$run" ]]; then
echo "Could not find an eval.yml workflow run for $BASE_SHA, cannot make comparison"
exit 0
fi
echo "Comparing against $(jq .html_url <<< "$run")"
runId=$(jq .id <<< "$run")
conclusion=$(jq .conclusion <<< "$run")
while [[ "$conclusion" == null ]]; do
echo "Workflow not done, waiting 10 seconds before checking again"
sleep 10
conclusion=$(gh api /repos/"$REPOSITORY"/actions/runs/"$runId" --jq '.conclusion')
done
if [[ "$conclusion" != "success" ]]; then
echo "Workflow was not successful, cannot make comparison"
exit 0
fi
echo "baseRunId=$runId" >> "$GITHUB_OUTPUT"
env:
REPOSITORY: ${{ github.repository }}
BASE_SHA: ${{ needs.attrs.outputs.baseSha }}

- uses: actions/download-artifact@v4
if: steps.baseRunId.outputs.baseRunId
with:
name: result
path: baseResult
github-token: ${{ github.token }}
run-id: ${{ steps.baseRunId.outputs.baseRunId }}

- name: Compare against the base branch
if: steps.baseRunId.outputs.baseRunId
run: |
nix-build nixpkgs/ci -A eval.compare \
--arg beforeResultDir ./baseResult \
--arg afterResultDir ./prResult \
-o comparison
# TODO: Run this workflow also on `push` (on at least the main development branches)
# Then add an extra step here that waits for the base branch (not the merge base, because that could be very different)
# to have completed the eval, then use
# gh api --method GET /repos/NixOS/nixpkgs/actions/workflows/eval.yml/runs -f head_sha=<BASE>
# and follow it to the artifact results, where you can then download the outpaths.json from the base branch
# That can then be used to compare the number of changed paths, get evaluation stats and ping appropriate reviewers
# TODO: Request reviews from maintainers for packages whose files are modified in the PR
- name: Upload the combined results
if: steps.baseRunId.outputs.baseRunId
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: comparison
path: comparison/*
18 changes: 18 additions & 0 deletions ci/eval/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,22 @@ let
jq -s from_entries > $out/stats.json
'';

compare =
{ beforeResultDir, afterResultDir }:
runCommand "compare" {
nativeBuildInputs = [
jq
];
} ''
mkdir $out
jq -n -f ${./filter.jq} \
--slurpfile before ${beforeResultDir}/outpaths.json \
--slurpfile after ${afterResultDir}/outpaths.json \
> $out/changed-paths.json
# TODO: Compare eval stats
'';

full =
{
# Whether to evaluate just a single system, by default all are evaluated
Expand All @@ -262,12 +278,14 @@ let
resultsDir = results;
};


in
{
inherit
attrpathsSuperset
singleSystem
combine
compare
# The above three are used by separate VMs in a GitHub workflow,
# while the below is intended for testing on a single local machine
full
Expand Down
120 changes: 120 additions & 0 deletions ci/eval/filter.jq
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# Turns
#
# {
# "hello.aarch64-linux": "a",
# "hello.x86_64-linux": "b",
# "hello.aarch64-darwin": "c",
# "hello.x86_64-darwin": "d"
# }
#
# into
#
# {
# "hello": {
# "linux": {
# "aarch64": "a",
# "x86_64": "b"
# },
# "darwin": {
# "aarch64": "c",
# "x86_64": "d"
# }
# }
# }
#
# while filtering out any attribute paths that don't match this pattern
def expand_system:
to_entries
| map(
.key |= split(".")
| select(.key | length > 1)
| .double = (.key[-1] | split("-"))
| select(.double | length == 2)
)
| group_by(.key[0:-1])
| map(
{
key: .[0].key[0:-1] | join("."),
value:
group_by(.double[1])
| map(
{
key: .[0].double[1],
value: map(.key = .double[0]) | from_entries
}
)
| from_entries
})
| from_entries
;

# Transposes
#
# {
# "a": [ "x", "y" ],
# "b": [ "x" ],
# }
#
# into
#
# {
# "x": [ "a", "b" ],
# "y": [ "a" ]
# }
def transpose:
[
to_entries[]
| {
key: .key,
value: .value[]
}
]
| group_by(.value)
| map({
key: .[0].value,
value: map(.key)
})
| from_entries
;

# Computes the key difference for two objects:
# {
# added: [ <keys only in the second object> ],
# removed: [ <keys only in the first object> ],
# changed: [ <keys with different values between the two objects> ],
# }
#
def diff($before; $after):
{
added: $after | delpaths($before | keys | map([.])) | keys,
removed: $before | delpaths($after | keys | map([.])) | keys,
changed:
$before
| to_entries
| map(
$after."\(.key)" as $after2
| select(
# Filter out attributes that don't exist anymore
($after2 != null)
and
# Filter out attributes that are the same as the new value
(.value != $after2)
)
| .key
)
}
;

($before[0] | expand_system) as $before
| ($after[0] | expand_system) as $after
| .attrdiff = diff($before; $after)
| .rebuildsByKernel = (
.attrdiff.changed
| map({
key: .,
value: diff($before."\(.)"; $after."\(.)").changed
})
| from_entries
| transpose
)
| .rebuildCountByKernel = (.rebuildsByKernel | with_entries(.value |= length))
1 change: 1 addition & 0 deletions ci/request-reviews/dev-branches.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# Trusted development branches:
# These generally require PRs to update and are built by Hydra.
# Keep this synced with the branches in .github/workflows/eval.yml
master
staging
release-*
Expand Down

0 comments on commit 13717ef

Please sign in to comment.