Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: logging for preset comms device #3459

Merged
merged 1 commit into from
Nov 27, 2024
+33 −16
Merged

fix: logging for preset comms device #3459

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 21 additions & 12 deletions src/solver/type_getters.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,18 +589,7 @@ function get_comms_context(parsed_args)
end
comms_ctx = ClimaComms.context(device)
ClimaComms.init(comms_ctx)
if ClimaComms.iamroot(comms_ctx)
Logging.global_logger(Logging.ConsoleLogger(stderr, Logging.Info))
else
Logging.global_logger(Logging.NullLogger())
end
@info "Running on $(nameof(typeof(device)))."
if comms_ctx isa ClimaComms.SingletonCommsContext
@info "Setting up single-process ClimaAtmos run"
else
@info "Setting up distributed ClimaAtmos run" nprocs =
ClimaComms.nprocs(comms_ctx)
end

if NVTX.isactive()
# makes output on buildkite a bit nicer
if ClimaComms.iamroot(comms_ctx)
Expand All @@ -613,6 +602,26 @@ function get_comms_context(parsed_args)
return comms_ctx
end

"""
silence_non_root_processes(comms_ctx)

Set the logging behavior based on the process rank within the given communication context `comms_ctx`.
If the process is the root process, logging is set to display messages to the console with `Info` level.
For all other processes, logging is silenced by setting it to a `NullLogger`.

# Arguments
- `comms_ctx`: The communication context used to determine the rank of the process.

"""
function silence_non_root_processes(comms_ctx)
# Set logging to only display for the root process
if ClimaComms.iamroot(comms_ctx)
Logging.global_logger(Logging.ConsoleLogger(stderr, Logging.Info))
else
Logging.global_logger(Logging.NullLogger())
end
end

function get_simulation(config::AtmosConfig)
params = create_parameter_set(config)
atmos = get_atmos(config, params)
Expand Down
16 changes: 12 additions & 4 deletions src/solver/types.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -647,17 +647,25 @@ function AtmosConfig(
override_file = CP.merge_toml_files(config["toml"]),
)
comms_ctx = isnothing(comms_ctx) ? get_comms_context(config) : comms_ctx
device = ClimaComms.device(comms_ctx)
silence_non_root_processes(comms_ctx)
@info "Running on $(nameof(typeof(device)))"
if comms_ctx isa ClimaComms.SingletonCommsContext
@info "Setting up single-process ClimaAtmos run"
else
@info "Setting up distributed ClimaAtmos run" nprocs =
ClimaComms.nprocs(comms_ctx)
end

config = config_with_resolved_and_acquired_artifacts(config, comms_ctx)
device = ClimaComms.device(comms_ctx)
if device isa ClimaComms.CPUMultiThreaded
@info "Running ClimaCore in threaded mode, with $(Threads.nthreads()) threads."
@info "Running ClimaCore in threaded mode, with $(Threads.nthreads()) threads"
else
@info "Running ClimaCore in unthreaded mode."
@info "Running ClimaCore in unthreaded mode"
end

isempty(job_id) &&
@warn "`job_id` is empty and likely not passed to AtmosConfig."
@warn "`job_id` is empty and likely not passed to AtmosConfig"

@info "Making AtmosConfig with config files: $(sprint(config_summary, config_files))"

Expand Down
Loading