Mark kernel entry points norecurse

src/driver.jl

@@ -362,6 +362,15 @@ const __llvm_initialized = Ref(false)
         # mark the kernel entry-point functions (optimization may need it)
         if job.config.kernel
             mark_kernel!(entry)
+
+            # device code cannot call a kernel, so it cannot recurse (clang does the same
+            # for CUDA and HIP kernels). this lets `ReversePostOrderFunctionAttrsPass` mark
+            # the functions it calls, which the AMDGPU back-end otherwise assumes may
+            # recurse, giving the kernel a dynamic stack. on the CPU, a kernel is an
+            # ordinary function that could be called back from code outside the module.
+            if !(job.config.target isa NativeCompilerTarget) && isempty(entry.uses)
+                push!(entry.function_attributes, EnumAttribute("norecurse", 0))
+            end
         end
 
         if job.config.toplevel

src/optim.jl

@@ -421,6 +421,9 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op
 end
 
 function buildCleanupPipeline(mpm, @nospecialize(job::CompilerJob), opt_level)
+    # propagate `norecurse` from kernels to the functions they call, after the runtime
+    # has been linked and the last inliner has run
+    add!(mpm, ReversePostOrderFunctionAttrsPass())
     if opt_level >= 2
         add!(mpm, FunctionPassManager()) do fpm
             if VERSION < v"1.12.0-DEV.1390"

test/gcn.jl

@@ -709,6 +709,30 @@ end
     end
 end
 
+@testset "calls without dynamic stack" begin
+    # the back-end assumes a callee without `norecurse` may recurse, and gives the kernel a
+    # dynamic stack, even if the call is never made
+    mod = @eval module $(gensym())
+        import ..sink_gcn
+        @noinline child(i) = sink_gcn(i)
+        function entry(i)
+            child(i)
+            return
+        end
+    end
+
+    @test @filecheck begin
+        @check "define internal {{.*}} @{{(julia|j)_child_[0-9]+}}({{.*}}) {{.*}} #[[ATTRS:[0-9]+]]"
+        @check "attributes #[[ATTRS]] = { {{.*}}norecurse"
+        GCN.code_llvm(mod.entry, Tuple{Int64}; dump_module=true, kernel=true)
+    end
+
+    @test @filecheck begin
+        @check ".amdhsa_uses_dynamic_stack 0"
+        GCN.code_native(mod.entry, Tuple{Int64}; dump_module=true, kernel=true)
+    end
+end
+
 @testset "child function reuse" begin
     # bug: depending on a child function from multiple parents resulted in
     #      the child only being present once

test/native.jl

@@ -917,6 +917,23 @@ end
     @test_throws KernelError Native.code_llvm(devnull, mod.invalid_kernel, Tuple{}; kernel=true) == nothing
 end
 
+@testset "kernels may recurse" begin
+    # unlike on GPUs, a kernel is an ordinary function that may be called from elsewhere
+    mod = @eval module $(gensym())
+        import ..sink
+        @noinline child(i) = sink(i)
+        function kernel(i)
+            child(i)
+            return
+        end
+    end
+
+    @test @filecheck begin
+        @check_not "norecurse"
+        Native.code_llvm(mod.kernel, Tuple{Int64}; dump_module=true, kernel=true)
+    end
+end
+
 @testset "unbound typevars" begin
     # suppress the warning Julia emits when defining a method with an unbound typevar
     mod = redirect_stderr(devnull) do