[Draft, for CI] Take Julia's LLVM passes from JuliaIR.jl, as an opt-in

.buildkite/downstream.yml

@@ -4,6 +4,14 @@
 # `src/` and `ext/`, `Project.toml` and the CI configuration, except the other back-ends'
 # target files. The `if:` conditions implement the commit message tags documented in the
 # Development section of the README; keep both in sync when adding a step.
+#
+# Every package also gets a "(ports)" step,
+# which runs its tests with the ports of Julia's passes (the `julia_passes = "ports"`
+# preference, with JuliaIR.jl set up by `.ci/juliair.jl` in an environment of its own,
+# as the default one persists on some agents) and fails unless JuliaIR.jl's
+# ledger shows that they ran. `[only ports]` selects only the ports steps and
+# `[skip ports]` skips them; a package's own tags (e.g. `[only cuda]`) select both of its
+# steps. Steps are skipped on draft pull requests unless a tag selects them.
 steps:
   - label: "CUDA.jl"
     plugins:
@@ -34,6 +42,7 @@ steps:
         - "ext/**"
         - "Project.toml"
         - ".buildkite/**"
+        - ".ci/**"
       exclude:
         - "src/{gcn,spirv,metal,bpf,native}.jl"
     if: |
@@ -44,6 +53,39 @@ steps:
     soft_fail:
       - exit_status: 3
 
+  - label: "CUDA.jl (ports)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.13"
+    command: |
+      unset LD_LIBRARY_PATH
+      echo "--- :julia: Instantiating project"
+      # an environment of its own, so that the preference doesn't reach the other steps
+      export JULIA_PROJECT="$$(mktemp -d)"
+      julia .ci/juliair.jl downstream CUDA
+
+      export JULIAIR_LEDGER="$$PWD/juliair-ledger.tsv"
+      rm -f "$$JULIAIR_LEDGER"
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("CUDA")'
+
+      echo "+++ :julia: Checking that the ports ran"
+      julia .ci/juliair.jl check-ledger "$$JULIAIR_LEDGER"
+    agents:
+      queue: "cuda"
+    env:
+      JULIA_CPU_THREADS: "3"
+    if_changed:
+      include: *sources
+      exclude:
+        - "src/{gcn,spirv,metal,bpf,native}.jl"
+    if: |
+      build.message !~ /\[skip [^\]]*(tests|cuda|ptx|ports)/ &&
+      (build.message =~ /\[only [^\]]*(cuda|ptx|ports)/ ||
+       build.message !~ /\[only / && !build.pull_request.draft)
+    timeout_in_minutes: 120
+
   - label: "oneAPI.jl"
     plugins:
       - JuliaCI/julia#v1:
@@ -77,6 +119,39 @@ steps:
     soft_fail:
       - exit_status: 3
 
+  - label: "oneAPI.jl (ports)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.13"
+    command: |
+      echo "--- :julia: Instantiating project"
+      # an environment of its own, so that the preference doesn't reach the other steps
+      export JULIA_PROJECT="$$(mktemp -d)"
+      julia .ci/juliair.jl downstream oneAPI
+      julia -e 'using Pkg;
+                println("+++ :julia: Building support library")
+                include(joinpath(Pkg.devdir(), "oneAPI", "deps", "build_ci.jl"))'
+
+      export JULIAIR_LEDGER="$$PWD/juliair-ledger.tsv"
+      rm -f "$$JULIAIR_LEDGER"
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("oneAPI")'
+
+      echo "+++ :julia: Checking that the ports ran"
+      julia .ci/juliair.jl check-ledger "$$JULIAIR_LEDGER"
+    agents:
+      queue: "oneapi"
+    if_changed:
+      include: *sources
+      exclude:
+        - "src/{ptx,gcn,metal,bpf,native}.jl"
+    if: |
+      build.message !~ /\[skip [^\]]*(tests|oneapi|spirv|ports)/ &&
+      (build.message =~ /\[only [^\]]*(oneapi|spirv|ports)/ ||
+       build.message !~ /\[only / && !build.pull_request.draft)
+    timeout_in_minutes: 60
+
   - label: "AMDGPU.jl"
     plugins:
       - JuliaCI/julia#v1:
@@ -106,6 +181,36 @@ steps:
     soft_fail:
       - exit_status: 3
 
+  - label: "AMDGPU.jl (ports)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.13"
+    command: |
+      echo "--- :julia: Instantiating project"
+      # an environment of its own, so that the preference doesn't reach the other steps
+      export JULIA_PROJECT="$$(mktemp -d)"
+      julia .ci/juliair.jl downstream AMDGPU
+
+      export JULIAIR_LEDGER="$$PWD/juliair-ledger.tsv"
+      rm -f "$$JULIAIR_LEDGER"
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("AMDGPU")'
+
+      echo "+++ :julia: Checking that the ports ran"
+      julia .ci/juliair.jl check-ledger "$$JULIAIR_LEDGER"
+    agents:
+      queue: "rocm"
+    if_changed:
+      include: *sources
+      exclude:
+        - "src/{ptx,spirv,metal,bpf,native}.jl"
+    if: |
+      build.message !~ /\[skip [^\]]*(tests|amdgpu|gcn|ports)/ &&
+      (build.message =~ /\[only [^\]]*(amdgpu|gcn|ports)/ ||
+       build.message !~ /\[only / && !build.pull_request.draft)
+    timeout_in_minutes: 60
+
   - label: "Metal.jl"
     plugins:
       - JuliaCI/julia#v1:
@@ -146,6 +251,36 @@ steps:
     soft_fail:
       - exit_status: 3
 
+  - label: "Metal.jl (ports)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.13"
+    command: |
+      echo "--- :julia: Instantiating project"
+      # an environment of its own, so that the preference doesn't reach the other steps
+      export JULIA_PROJECT="$$(mktemp -d)"
+      JULIA_PKG_PRECOMPILE_AUTO=0 julia .ci/juliair.jl downstream Metal --workspace
+
+      export JULIAIR_LEDGER="$$PWD/juliair-ledger.tsv"
+      rm -f "$$JULIAIR_LEDGER"
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("Metal")'
+
+      echo "+++ :julia: Checking that the ports ran"
+      julia .ci/juliair.jl check-ledger "$$JULIAIR_LEDGER"
+    agents:
+      queue: "metal"
+    if_changed:
+      include: *sources
+      exclude:
+        - "src/{ptx,gcn,spirv,bpf,native}.jl"
+    if: |
+      build.message !~ /\[skip [^\]]*(tests|metal|ports)/ &&
+      (build.message =~ /\[only [^\]]*(metal|ports)/ ||
+       build.message !~ /\[only / && !build.pull_request.draft)
+    timeout_in_minutes: 120
+
   - label: "OpenCL.jl"
     plugins:
       - JuliaCI/julia#v1:
@@ -176,5 +311,35 @@ steps:
     soft_fail:
       - exit_status: 3
 
+  - label: "OpenCL.jl (ports)"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: "1.13"
+    command: |
+      echo "--- :julia: Instantiating project"
+      # an environment of its own, so that the preference doesn't reach the other steps
+      export JULIA_PROJECT="$$(mktemp -d)"
+      julia .ci/juliair.jl downstream OpenCL SPIRVIntrinsics
+
+      export JULIAIR_LEDGER="$$PWD/juliair-ledger.tsv"
+      rm -f "$$JULIAIR_LEDGER"
+      julia -e 'println("+++ :julia: Running tests")
+                using Pkg
+                Pkg.test("OpenCL")'
+
+      echo "+++ :julia: Checking that the ports ran"
+      julia .ci/juliair.jl check-ledger "$$JULIAIR_LEDGER"
+    agents:
+      queue: "cuda"
+    if_changed:
+      include: *sources
+      exclude:
+        - "src/{ptx,gcn,metal,bpf,native}.jl"
+    if: |
+      build.message !~ /\[skip [^\]]*(tests|opencl|spirv|ports)/ &&
+      (build.message =~ /\[only [^\]]*(opencl|spirv|ports)/ ||
+       build.message !~ /\[only / && !build.pull_request.draft)
+    timeout_in_minutes: 120
+
 env:
   JULIA_PKG_SERVER_REGISTRY_PREFERENCE: "eager" # see freshly-registered versions without cloning the registry

.ci/juliair.jl

@@ -0,0 +1,127 @@
+# Set up CI environments for the selected Julia pass provider.
+#
+#     julia .ci/juliair.jl test (cpp|ports)
+#
+# Prepares GPUCompiler's test environment (`test/`), for `julia --project=test
+# test/runtests.jl`, with the `julia_passes` preference set to the given provider.
+#
+#     julia .ci/juliair.jl downstream PACKAGE [PACKAGES...] [--workspace]
+#
+# Develops GPUCompiler and the downstream PACKAGE (and further PACKAGES, e.g. packages of
+# PACKAGE's repository) into the active environment, for `Pkg.test(PACKAGE)` with the
+# ports: JuliaIR.jl is added too, and to PACKAGE's test environment, where GPUCompiler
+# loads it. The preference goes into the active environment (in CI, a temporary one),
+# which `Pkg.test` passes on to the test environment; GPUCompiler is added to PACKAGE's
+# test dependencies so that it applies there. `--workspace` also instantiates PACKAGE's
+# workspace against GPUCompiler (Metal.jl's tests run examples in its workspace
+# environments).
+#
+#     julia .ci/juliair.jl check-ledger FILE
+#
+# Fails unless FILE, the JuliaIR.jl ledger (`JULIAIR_LEDGER`) of a run with the ports,
+# shows that the core ports ran.
+
+using Pkg, TOML
+
+const ROOT = dirname(@__DIR__)
+const GPUCOMPILER_UUID = "61eb1bfa-7361-4325-ad38-22787b887f55"
+const JULIAIR_UUID = "3913dba9-7ba9-44aa-bf76-1cfa3b3073b1"
+
+function write_preferences(dir, mode)
+    # an environment that doesn't exist yet (Pkg creates it on the first change)
+    mkpath(dir)
+    file = joinpath(dir, "LocalPreferences.toml")
+    open(io -> TOML.print(io, Dict("GPUCompiler" => Dict("julia_passes" => mode))), file, "w")
+    println("$file:")
+    print(read(file, String))
+end
+
+function test(mode)
+    mode in ("cpp", "ports") || error("Unknown provider $mode; expected cpp or ports")
+    write_preferences(joinpath(ROOT, "test"), mode)
+    Pkg.activate(joinpath(ROOT, "test"))
+    # Julia 1.10 and 1.11 have no workspaces, so the test environment needs GPUCompiler
+    VERSION < v"1.12" && Pkg.develop(path=ROOT)
+    Pkg.instantiate()
+    Pkg.status()
+end
+
+function downstream(packages, workspace)
+    package = first(packages)
+    env = dirname(Base.active_project())
+    write_preferences(env, "ports")
+
+    Pkg.develop(path=ROOT)
+    Pkg.add(PackageSpec(name="JuliaIR", version="0.1"))
+    Pkg.develop([PackageSpec(name=name) for name in packages])
+
+    checkout = joinpath(Pkg.devdir(), package)
+    if workspace
+        # before adding JuliaIR.jl to the test environment, which is part of the workspace
+        # and could then not be resolved; the examples that need the workspace run in the
+        # test process, which has loaded GPUCompiler and JuliaIR.jl already
+        Pkg.activate(checkout)
+        Pkg.develop(path=ROOT)
+        Pkg.instantiate(workspace=true)
+        Pkg.activate(env)
+    end
+    test_project = joinpath(checkout, "test", "Project.toml")
+    isfile(test_project) || error("$package has no test/Project.toml")
+    add = Dict("GPUCompiler" => GPUCOMPILER_UUID, "JuliaIR" => JULIAIR_UUID)
+    # by editing the text, as rewriting it with TOML.print reorders the file
+    deps = get(TOML.parsefile(test_project), "deps", Dict())
+    lines = [k * " = \"" * v * "\"" for (k, v) in add if !haskey(deps, k)]
+    if !isempty(lines)
+        text = read(test_project, String)
+        occursin(r"^\[deps\]$"m, text) || (text = "[deps]\n\n" * text)
+        text = replace(text, r"^\[deps\]$"m => join(["[deps]"; lines], "\n"); count=1)
+        write(test_project, text)
+    end
+
+    Pkg.status()
+end
+
+# the ports that run for every kernel on every target and Julia version, and JuliaLICM,
+# which runs for every loop (that test suites have)
+const CORE_PORTS = ["jl-alloc-opt", "jl-late-lower-gc-frame", "jl-propagate-addrspaces",
+                    "jl-remove-ni", "jl-demote-float16", "jl-remove-addrspaces", "jl-licm"]
+
+function check_ledger(file)
+    isfile(file) || error("No JuliaIR.jl ledger at $file: the ports did not run")
+    counts = Dict{String,Int}()
+    pids = Set{String}()
+    for (i, line) in enumerate(eachline(file))
+        # PID, "pass", the pass, the Julia dialect and the LLVM version
+        fields = split(line, '\t')
+        if !(length(fields) == 5 && all(isdigit, fields[1]) && fields[2] == "pass" &&
+             startswith(fields[3], "jl-") && startswith(fields[4], "julia=") &&
+             startswith(fields[5], "llvm="))
+            error("Malformed line $i in the ledger at $file: $(repr(line))")
+        end
+        counts[fields[3]] = get(counts, fields[3], 0) + 1
+        push!(pids, fields[1])
+    end
+    for (name, count) in sort!(collect(counts))
+        println(rpad(name, 32), count)
+    end
+    println("in $(length(pids)) processes")
+    missing_ports = setdiff(CORE_PORTS, keys(counts))
+    isempty(missing_ports) ||
+        error("The ports $(join(missing_ports, ", ")) did not run; is GPUCompiler using the C++ passes?")
+end
+
+function main(args)
+    isempty(args) && error("Usage: julia .ci/juliair.jl (test MODE | downstream PACKAGE... [--workspace] | check-ledger FILE)")
+    command = args[1]
+    if command == "test"
+        test(args[2])
+    elseif command == "downstream"
+        downstream(filter(!=("--workspace"), args[2:end]), "--workspace" in args)
+    elseif command == "check-ledger"
+        check_ledger(args[2])
+    else
+        error("Unknown command $command")
+    end
+end
+
+main(ARGS)