Periodically recycle workers to avoid handle exhaustion.

Author: maleadt

src/compiler/compilation.jl

@@ -53,11 +53,13 @@ end
 end
 
 # compile to executable machine code
+const compilations = Threads.Atomic{Int}(0)
 function compile(@nospecialize(job::CompilerJob))
     # TODO: this creates a context; cache those.
     obj, meta = JuliaContext() do ctx
         GPUCompiler.compile(:obj, job)
     end
+    compilations[] += 1
 
     (obj, entry=LLVM.name(meta.entry))
 end

test/runtests.jl

@@ -280,6 +280,14 @@ try
                         p = recycle_worker(p)
                     else
                         print_testworker_stats(test, wrkr, resp)
+
+                        compilations = resp[7]
+                        if Sys.iswindows() && compilations > 100
+                            # XXX: restart to avoid handle exhaustion
+                            #      (see pocl/pocl#1941)
+                            @warn "Restarting worker $wrkr to avoid handle exhaustion"
+                            p = recycle_worker(p)
+                        end
                     end
                 end
 

test/setup.jl

@@ -114,6 +114,7 @@ function runtests(f, name, platform_filter)
 
         # process results
         cpu_rss = Sys.maxrss()
+        compilations = OpenCL.compilations[]
         if VERSION >= v"1.11.0-DEV.1529"
             tc = Test.get_test_counts(data[1])
             passes,fails,error,broken,c_passes,c_fails,c_errors,c_broken =
@@ -130,7 +131,7 @@ function runtests(f, name, platform_filter)
                     data[4],
                     data[5])
         end
-        res = vcat(collect(data), cpu_rss)
+        res = vcat(collect(data), cpu_rss, compilations)
 
         GC.gc(true)
         res