diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml index 5c9a889a5..3bc227743 100644 --- a/.JuliaFormatter.toml +++ b/.JuliaFormatter.toml @@ -22,5 +22,4 @@ align_matrix = false join_lines_based_on_source = true trailing_comma = true indent_submodule = true -# jobs.jl is ignored due to a parsing error: https://github.com/julia-vscode/CSTParser.jl/issues/371 -ignore = [".git", "test/jobs.jl"] +ignore = [".git"] diff --git a/test/jobs.jl b/test/jobs.jl index 5be6b9bfb..176afc615 100644 --- a/test/jobs.jl +++ b/test/jobs.jl @@ -26,7 +26,8 @@ end @test s.code == "1" @test s.environment.manifest_toml === "name=1" @test s.sysimage === true - @test JuliaHub._sysimage_manifest_sha(s.environment) == "b3be53dd7b40e92821b39188b37a70bb81d47d1db3818703744544efece3538c" + @test JuliaHub._sysimage_manifest_sha(s.environment) == + "b3be53dd7b40e92821b39188b37a70bb81d47d1db3818703744544efece3538c" end @test_throws ArgumentError JuliaHub.script(; code="1", project=".") @test_throws ArgumentError JuliaHub.script(; code="1", artifacts=".") @@ -43,7 +44,7 @@ end let s = JuliaHub.script( jobfile("script.jl"); project_directory=jobfile(), - sysimage = true, + sysimage=true, ) @test s.code == read(jobfile("script.jl"), String) @test s.environment.project_toml == read(jobfile("Project.toml"), String) @@ -61,7 +62,7 @@ end @test s.environment.artifacts_toml === nothing @test s.sysimage === false - s = JuliaHub.BatchJob(s, sysimage=true) + s = JuliaHub.BatchJob(s; sysimage=true) @test s.code == "test()" @test s.environment.project_toml == read(jobfile("Project.toml"), String) @test s.environment.manifest_toml == read(jobfile("Manifest.toml"), String) @@ -96,18 +97,19 @@ end @test isfile(bundle.environment.tarball_path) @test bundle.code == "test()" @test bundle.sysimage === true - @test JuliaHub._sysimage_manifest_sha(bundle.environment) == "631fc619c1d04e525872df2779fa95a0dc47edd9558af629af88c493daa6300d" + @test JuliaHub._sysimage_manifest_sha(bundle.environment) == + "631fc619c1d04e525872df2779fa95a0dc47edd9558af629af88c493daa6300d" mktempdir() do path bigfile_path = joinpath(path, "bigfile") - open(bigfile_path, write=true) do io - chunk = '\0' ^ (2^20) + open(bigfile_path; write=true) do io + chunk = '\0'^(2^20) for _ in 1:3000 write(io, chunk) end end @test_throws JuliaHub.AppBundleSizeError JuliaHub.appbundle(path; code="") - rm(bigfile_path, force=true) + rm(bigfile_path; force=true) end # Testing relative paths to the appbundle directory @@ -165,7 +167,9 @@ end end # Nodes with requirements that can't be met @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; ncpu=100, memory=5) - @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; ncpu=100, memory=5, throw=true) + @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; + ncpu=100, memory=5, throw=true + ) @test JuliaHub.nodespec(; ncpu=100, memory=5, throw=false) === nothing @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; ncpu=4, memory=50_000) @@ -174,7 +178,9 @@ end @test n.vcores == 2 @test n.mem == 8 end - @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; ncpu=1, memory=5, exactmatch=true) + @test_throws JuliaHub.InvalidRequestError JuliaHub.nodespec(; + ncpu=1, memory=5, exactmatch=true + ) @test JuliaHub.nodespec(; ncpu=1, memory=5, exactmatch=true, throw=false) === nothing let n = JuliaHub.nodespec(; ncpu=2, memory=8, exactmatch=true) @test n.vcores == 2 @@ -314,14 +320,14 @@ end @test cc.elastic === false ComputeConfig_eval_tests(cc) end - let cc = JuliaHub.ComputeConfig(ns_cheapest; nnodes = 5, elastic=true) + let cc = JuliaHub.ComputeConfig(ns_cheapest; nnodes=5, elastic=true) @test cc.nnodes_max === 5 @test cc.nnodes_min === nothing @test cc.process_per_node === true @test cc.elastic === true ComputeConfig_eval_tests(cc) end - let cc = JuliaHub.ComputeConfig(ns_cheapest; nnodes = (10, 20)) + let cc = JuliaHub.ComputeConfig(ns_cheapest; nnodes=(10, 20)) @test cc.nnodes_max === 20 @test cc.nnodes_min === 10 @test cc.process_per_node === true @@ -385,7 +391,7 @@ end @testset "JuliaHub.submit_job/s()" begin Mocking.apply(mocking_patch) do s = JuliaHub.script"run()" - let jc = JuliaHub.submit_job(s, dryrun = true) + let jc = JuliaHub.submit_job(s; dryrun=true) @test jc isa JuliaHub.WorkloadConfig @test jc.app isa JuliaHub.BatchJob @test jc.compute.node == ns_cheapest @@ -399,18 +405,18 @@ end end @test JuliaHub.submit_job(s) isa JuliaHub.Job # Test passing valid parameters - kwargs_ns = (; ncpu = 4, memory = 16, ngpu = 1) + kwargs_ns = (; ncpu=4, memory=16, ngpu=1) ns = JuliaHub.nodespec(; kwargs_ns...) @test ns != ns_cheapest - kwargs_cc = (; process_per_node = false, nnodes=(3, 10)) + kwargs_cc = (; process_per_node=false, nnodes=(3, 10)) kwargs_rt = (; - alias = "test-job", - project = "e1d9d1d4-814c-4f0c-a3c1-5e063cd2b02b", - env = Dict("MY_ARGUMENT" => "value"), - timelimit = 5 + alias="test-job", + project="e1d9d1d4-814c-4f0c-a3c1-5e063cd2b02b", + env=Dict("MY_ARGUMENT" => "value"), + timelimit=5, ) kwargs = (; kwargs_ns..., kwargs_cc..., kwargs_rt...) - let jc = JuliaHub.submit_job(s; kwargs..., dryrun = true) + let jc = JuliaHub.submit_job(s; kwargs..., dryrun=true) @test jc.compute.node == ns @test jc.compute.process_per_node === false @test jc.compute.nnodes_max == 10 @@ -421,8 +427,8 @@ end @test jc.env == kwargs.env end cc = JuliaHub.ComputeConfig(ns; kwargs_cc...) - kwargs_rt = (; kwargs_rt..., timelimit = JuliaHub.Unlimited()) - let jc = JuliaHub.submit_job(s, cc; kwargs_rt..., dryrun = true) + kwargs_rt = (; kwargs_rt..., timelimit=JuliaHub.Unlimited()) + let jc = JuliaHub.submit_job(s, cc; kwargs_rt..., dryrun=true) @test jc.compute.node == ns @test jc.compute.process_per_node === false @test jc.compute.nnodes_max == 10 @@ -437,11 +443,14 @@ end @test_throws MethodError JuliaHub.submit_job(s, cc; kwargs...) @test_throws ArgumentError JuliaHub.submit_job(s, timelimit=-20) @test_throws ArgumentError JuliaHub.submit_job(s, project="123") - @test_throws ArgumentError JuliaHub.submit_job(s; env = (; jobname = "foo"), alias = "bar") - @test_logs (:warn,) JuliaHub.submit_job(s; env = (; jobname = "foo")) + @test_throws ArgumentError JuliaHub.submit_job(s; env=(; jobname="foo"), alias="bar") + @test_logs (:warn,) JuliaHub.submit_job(s; env=(; jobname="foo")) # DEPRECATED: test the name -> alias deprecation logic @test_throws ArgumentError JuliaHub.submit_job(s, cc; name="foo", alias="bar") - let jc = @test_logs (:warn, "The `name` argument to `submit_job` is deprecated and will be removed in 0.2.0") JuliaHub.submit_job(s, cc; name="foo", dryrun=true) + let jc = @test_logs ( + :warn, + "The `name` argument to `submit_job` is deprecated and will be removed in 0.2.0", + ) JuliaHub.submit_job(s, cc; name="foo", dryrun=true) @test jc isa JuliaHub.WorkloadConfig @test jc.alias == "foo" end @@ -505,26 +514,28 @@ end empty!(MOCK_JULIAHUB_STATE) Mocking.apply(mocking_patch) do @test JuliaHub.job("jr-eezd3arpcj") isa JuliaHub.Job - @test JuliaHub.job("jr-eezd3arpcj", throw=false) isa JuliaHub.Job + @test JuliaHub.job("jr-eezd3arpcj"; throw=false) isa JuliaHub.Job @test_throws JuliaHub.InvalidRequestError JuliaHub.job("jr-nonexistent-id") - @test JuliaHub.job("jr-nonexistent-id", throw=false) === nothing + @test JuliaHub.job("jr-nonexistent-id"; throw=false) === nothing let job = JuliaHub.job("jr-eezd3arpcj") @test JuliaHub.job("jr-eezd3arpcj") isa JuliaHub.Job - @test JuliaHub.job("jr-eezd3arpcj", throw=false) isa JuliaHub.Job - end - let job = JuliaHub.Job(Dict( - "jobname" => "jr-nonexistent-id", - "outputs" => "", - "status" => "Running", - "inputs" => nothing, - "jobname_alias" => nothing, - "submittimestamp" => nothing, - "starttimestamp" => nothing, - "endtimestamp" => nothing, - )) + @test JuliaHub.job("jr-eezd3arpcj"; throw=false) isa JuliaHub.Job + end + let job = JuliaHub.Job( + Dict( + "jobname" => "jr-nonexistent-id", + "outputs" => "", + "status" => "Running", + "inputs" => nothing, + "jobname_alias" => nothing, + "submittimestamp" => nothing, + "starttimestamp" => nothing, + "endtimestamp" => nothing, + ), + ) @test_throws JuliaHub.InvalidRequestError JuliaHub.job(job) - @test JuliaHub.job(job, throw=false) === nothing + @test JuliaHub.job(job; throw=false) === nothing end # Handling of invalid job files: @@ -536,17 +547,17 @@ end "hash" => Dict("algorithm" => nothing, "value" => nothing), "upload_timestamp" => "2022-06-27T19:47:45.37875+00:00", "size" => nothing, - "type" => "source" + "type" => "source", ), Dict( "name" => "jr-eezd3arpcj-test", "hash" => Dict{String, Any}("algorithm" => nothing, "value" => nothing), "upload_timestamp" => "2022-06-27T19:47:45.37875+00:00", "size" => nothing, - "type" => "result" - ) - ] - ) + "type" => "result", + ), + ], + ), ) let j = JuliaHub.job("jr-eezd3arpcj") @test j isa JuliaHub.Job @@ -591,7 +602,7 @@ end function logging_mocking_wrapper(f::Base.Callable, testset_name::AbstractString; legacy=false) global MOCK_JULIAHUB_STATE - logengine = LogEngine(kafkalogging=!legacy) + logengine = LogEngine(; kafkalogging=!legacy) MOCK_JULIAHUB_STATE[:logengine] = logengine try Mocking.apply(mocking_patch) do @@ -605,7 +616,7 @@ function logging_mocking_wrapper(f::Base.Callable, testset_name::AbstractString; end JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC -@testset "Job logs: legacy = $legacy" for legacy in [true , false] +@testset "Job logs: legacy = $legacy" for legacy in [true, false] logging_mocking_wrapper("Basic logging"; legacy=legacy) do logengine @testset "Invalid requests" begin # Negative offsets are not allowed @@ -666,8 +677,12 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test JuliaHub.hasfirst(lb) end # Requesting a non-existent offset should throw an error - @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered("jr-test1"; offset=1) - @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered("jr-test1"; offset=1_000_000) + @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered( + "jr-test1"; offset=1 + ) + @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered( + "jr-test1"; offset=1_000_000 + ) end # Now, let's test with a single log message @@ -680,17 +695,19 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test length(lb.logs) == 0 @test JuliaHub.hasfirst(lb) # Because the first and last are found, these operations should succeed, but be no-ops. - JuliaHub.job_logs_newer!(lb, count=1) + JuliaHub.job_logs_newer!(lb; count=1) @test length(lb.logs) == 1 @test lb.logs[1].message == "SINGLE" @test JuliaHub.hasfirst(lb) @test JuliaHub.haslast(lb) end # As there is only one log (offset=0), requesting offset=1 should throw - @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered("jr-test1"; offset=1) + @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered( + "jr-test1"; offset=1 + ) let lb = JuliaHub.job_logs_buffered("jr-test1") @test length(lb.logs) == 0 - JuliaHub.job_logs_newer!(lb, count=1) + JuliaHub.job_logs_newer!(lb; count=1) @test length(lb.logs) == 0 @test JuliaHub.haslast(lb) JuliaHub.job_logs_older!(lb) @@ -699,7 +716,9 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test JuliaHub.hasfirst(lb) @test JuliaHub.haslast(lb) end - @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered("jr-test1"; offset=2) + @test_throws JuliaHub.InvalidRequestError JuliaHub.job_logs_buffered( + "jr-test1"; offset=2 + ) end # Now let's try the case with a bunch of log messages, which will @@ -717,7 +736,7 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test JuliaHub.hasfirst(lb) @test !JuliaHub.haslast(lb) # Because the first and last are found, these operations should succeed, but be no-ops. - JuliaHub.job_logs_newer!(lb, count=5) + JuliaHub.job_logs_newer!(lb; count=5) @test length(lb.logs) == 5 @test lb.logs[1].message == "LOG 1" @test lb.logs[end].message == "LOG 5" @@ -725,7 +744,7 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test !JuliaHub.haslast(lb) # Let's now fetch the next 19 logs one by one for i = 6:24 - JuliaHub.job_logs_newer!(lb, count=1) + JuliaHub.job_logs_newer!(lb; count=1) @test length(lb.logs) == i @test lb.logs[1].message == "LOG 1" @test lb.logs[end].message == "LOG $i" @@ -733,7 +752,7 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test !JuliaHub.haslast(lb) end # And let's now fetch a bunch more, but we should only receive one. - JuliaHub.job_logs_newer!(lb, count=100) + JuliaHub.job_logs_newer!(lb; count=100) @test length(lb.logs) == 25 @test lb.logs[1].message == "LOG 1" @test lb.logs[end].message == "LOG 25" @@ -746,14 +765,14 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test !JuliaHub.hasfirst(lb) @test JuliaHub.haslast(lb) # because it's a finished job # Fetch 15 older logs - JuliaHub.job_logs_older!(lb, count=15) + JuliaHub.job_logs_older!(lb; count=15) @test length(lb.logs) == 15 @test lb.logs[1].message == "LOG 11" @test lb.logs[end].message == "LOG 25" @test !JuliaHub.hasfirst(lb) @test JuliaHub.haslast(lb) # And another one.. - JuliaHub.job_logs_older!(lb, count=1) + JuliaHub.job_logs_older!(lb; count=1) @test length(lb.logs) == 16 @test lb.logs[1].message == "LOG 10" @test lb.logs[end].message == "LOG 25" @@ -768,8 +787,8 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test JuliaHub.haslast(lb) # newer! is still a no-op JuliaHub.job_logs_newer!(lb) - JuliaHub.job_logs_newer!(lb, count=1) - JuliaHub.job_logs_newer!(lb, count=1_000_000) + JuliaHub.job_logs_newer!(lb; count=1) + JuliaHub.job_logs_newer!(lb; count=1_000_000) @test length(lb.logs) == 25 @test lb.logs[1].message == "LOG 1" @test lb.logs[end].message == "LOG 25" @@ -779,26 +798,26 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC # And starting from the middle. Conceptually, at the start, the cursor # should be just before offset=17, so the first newer log will be offset=17 # log.. which is LOG 18, since the indexing starts at 1. - let lb = JuliaHub.job_logs_buffered("jr-test1", offset=17) + let lb = JuliaHub.job_logs_buffered("jr-test1"; offset=17) @test length(lb.logs) == 0 @test !JuliaHub.hasfirst(lb) @test !JuliaHub.haslast(lb) # because it's a finished job # Fetch older and newer longs mixedly - JuliaHub.job_logs_older!(lb, count=1) + JuliaHub.job_logs_older!(lb; count=1) @test length(lb.logs) == 1 # Since we're moving backwards, we are fetching LOG 17 with older! @test lb.logs[1].message == "LOG 17" @test !JuliaHub.hasfirst(lb) @test !JuliaHub.haslast(lb) - JuliaHub.job_logs_newer!(lb, count=3) + JuliaHub.job_logs_newer!(lb; count=3) @test length(lb.logs) == 4 @test lb.logs[1].message == "LOG 17" @test lb.logs[end].message == "LOG 20" @test !JuliaHub.hasfirst(lb) @test !JuliaHub.haslast(lb) - JuliaHub.job_logs_older!(lb, count=2) + JuliaHub.job_logs_older!(lb; count=2) @test length(lb.logs) == 6 @test lb.logs[1].message == "LOG 15" @test lb.logs[end].message == "LOG 20" @@ -820,17 +839,17 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @test JuliaHub.haslast(lb) end # Explicitly checking that offset=1 leads to LOG 2 - let lb = JuliaHub.job_logs_buffered("jr-test1", offset=1) - JuliaHub.job_logs_newer!(lb, count=1) + let lb = JuliaHub.job_logs_buffered("jr-test1"; offset=1) + JuliaHub.job_logs_newer!(lb; count=1) @test length(lb.logs) == 1 @test lb.logs[1].message == "LOG 2" end # Also checking that starting by moving forward makes sense at higher offset - let lb = JuliaHub.job_logs_buffered("jr-test1", offset=17) + let lb = JuliaHub.job_logs_buffered("jr-test1"; offset=17) @test length(lb.logs) == 0 @test !JuliaHub.hasfirst(lb) @test !JuliaHub.haslast(lb) # because it's a finished job - JuliaHub.job_logs_newer!(lb, count=1) + JuliaHub.job_logs_newer!(lb; count=1) @test length(lb.logs) == 1 @test lb.logs[1].message == "LOG 18" @test !JuliaHub.hasfirst(lb) @@ -842,11 +861,14 @@ JuliaHub._OPTION_LoggingMode[] = JuliaHub._LoggingMode.AUTOMATIC @testset "Callback" begin logengine.jobs["jr-test1"].logs = ["LOG $i" for i = 1:25] cb_results = [] - lb = JuliaHub.job_logs_buffered("jr-test1", offset=6) do buffer::JuliaHub.AbstractJobLogsBuffer, logs::AbstractVector{JuliaHub.JobLogMessage} + lb = JuliaHub.job_logs_buffered( + "jr-test1"; offset=6 + ) do buffer::JuliaHub.AbstractJobLogsBuffer, + logs::AbstractVector{JuliaHub.JobLogMessage} push!(cb_results, (first(logs).message, length(logs), last(logs).message)) end - JuliaHub.job_logs_newer!(lb, count=12) - JuliaHub.job_logs_older!(lb, count=1) + JuliaHub.job_logs_newer!(lb; count=12) + JuliaHub.job_logs_older!(lb; count=1) JuliaHub.job_logs_newer!(lb) JuliaHub.job_logs_older!(lb) @test length(cb_results) == 4