From 80eefe413c0226d1304d3636d73a39becbcf33f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Str=C3=B6mer?= <8915976+sstroemer@users.noreply.github.com> Date: Wed, 20 Nov 2024 11:36:33 +0100 Subject: [PATCH 1/5] ci: add a windows runner to the test CI job (DuckDB failed on windows which we did not catch immediately) --- .github/workflows/CI.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2ed1841..451464e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -28,6 +28,7 @@ jobs: - '1.11' os: - ubuntu-latest + - windows-latest arch: - x64 steps: From f5567cb589de5df5f7ac4bc247263fcc7e5723c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Str=C3=B6mer?= <8915976+sstroemer@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:15:27 +0100 Subject: [PATCH 2/5] fix: fixes #32 and precompilation of DuckDB under Windows --- Project.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index bdc9d3f..f0cdb38 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DuckDB = "d2f5444f-75bc-4fdf-ac35-56f514c445e1" +DuckDB_jll = "2cbbab25-fc8b-58cf-88d4-687a02676033" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819" JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" @@ -54,7 +55,8 @@ CSV = "0.10" Cbc = "1.2.0" DataFrames = "1.6" Dates = "<0.0.1,1" -DuckDB = "1.1.0" +DuckDB = "0.10, ~1.0" # ~1.0 until fixed, see https://github.com/duckdb/duckdb/issues/13911 +DuckDB_jll = 0.10, ~1.0" # DuckDB 1.0.0 still allows the faulty (under Windows) DuckDB_jll 1.1.0 GLPK = "1.2.1" Gurobi = "1.3.0" HiGHS = "1.9" From fc07702312d9226be2df8f6f0ee05018565fa7f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Str=C3=B6mer?= Date: Wed, 20 Nov 2024 12:18:08 +0100 Subject: [PATCH 3/5] fix: random " being gone after commit --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index f0cdb38..b6fa539 100644 --- a/Project.toml +++ b/Project.toml @@ -55,8 +55,8 @@ CSV = "0.10" Cbc = "1.2.0" DataFrames = "1.6" Dates = "<0.0.1,1" -DuckDB = "0.10, ~1.0" # ~1.0 until fixed, see https://github.com/duckdb/duckdb/issues/13911 -DuckDB_jll = 0.10, ~1.0" # DuckDB 1.0.0 still allows the faulty (under Windows) DuckDB_jll 1.1.0 +DuckDB = "0.10, ~1.0" # ~1.0 until fixed, see https://github.com/duckdb/duckdb/issues/13911 +DuckDB_jll = "0.10, ~1.0" # DuckDB 1.0.0 still allows the faulty (under Windows) DuckDB_jll 1.1.0 GLPK = "1.2.1" Gurobi = "1.3.0" HiGHS = "1.9" From 3275025f8c21caa5fbdced31239fa85546575e46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Str=C3=B6mer?= Date: Wed, 20 Nov 2024 12:33:31 +0100 Subject: [PATCH 4/5] fix: properly close log files during precompile, which might fix issues under Windows --- src/precompile/precompile_tools.jl | 52 ++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/precompile/precompile_tools.jl b/src/precompile/precompile_tools.jl index 340a524..60bc41f 100644 --- a/src/precompile/precompile_tools.jl +++ b/src/precompile/precompile_tools.jl @@ -7,9 +7,10 @@ config = Dict("optimization.snapshots.count" => 3, "general.verbosity.core" => "error") fn = String(normpath(__dir, "01_basic_single_node.iesopt.yaml")) - generate!(fn; config=Dict("general.verbosity.core" => "debug")) - generate!(fn; config=Dict("general.verbosity.core" => "info")) - generate!(fn; config=Dict("general.verbosity.core" => "warn")) + safe_close_filelogger(generate!(fn; config=Dict("general.verbosity.core" => "debug"))) + safe_close_filelogger(generate!(fn; config=Dict("general.verbosity.core" => "info"))) + safe_close_filelogger(generate!(fn; config=Dict("general.verbosity.core" => "warn"))) + model = generate!( fn; config=Dict( @@ -19,6 +20,8 @@ ), ) optimize!(model) + safe_close_filelogger(model) + model = generate!( fn; config=Dict( @@ -28,6 +31,8 @@ ), ) optimize!(model) + safe_close_filelogger(model) + model = generate!( fn; config=Dict( @@ -37,31 +42,48 @@ ), ) optimize!(model) - IESopt.run(fn; config, skip_validation=true) - generate!(normpath(__dir, "02_advanced_single_node.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger(model) + + safe_close_filelogger(IESopt.run(fn; config, skip_validation=true)) + + safe_close_filelogger( + generate!(normpath(__dir, "02_advanced_single_node.iesopt.yaml"); config, skip_validation=true), + ) # generate!(normpath(__dir, "03_basic_two_nodes.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "04_soft_constraints.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "05_basic_two_nodes_1y.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "06_recursion_h2.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "07_csv_filestorage.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "08_basic_investment.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "09_csv_only.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger( + generate!(normpath(__dir, "07_csv_filestorage.iesopt.yaml"); config, skip_validation=true), + ) + safe_close_filelogger( + generate!(normpath(__dir, "08_basic_investment.iesopt.yaml"); config, skip_validation=true), + ) + safe_close_filelogger(generate!(normpath(__dir, "09_csv_only.iesopt.yaml"); config, skip_validation=true)) # generate!(normpath(__dir, "10_basic_load_shedding.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "11_basic_unit_commitment.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger( + generate!(normpath(__dir, "11_basic_unit_commitment.iesopt.yaml"); config, skip_validation=true), + ) # generate!(normpath(__dir, "12_incremental_efficiency.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "15_varying_efficiency.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "16_noncore_components.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger( + generate!(normpath(__dir, "16_noncore_components.iesopt.yaml"); config, skip_validation=true), + ) # generate!(normpath(__dir, "17_varying_connection_capacity.iesopt.yaml"); config, skip_validation=true) model = generate!(normpath(__dir, "18_addons.iesopt.yaml"); config, skip_validation=true) get_components(model; tagged=["ModifyMe"]) + safe_close_filelogger(model) model = generate!(normpath(__dir, "20_chp.iesopt.yaml"); config, skip_validation=true) get_component(model, "chp") + safe_close_filelogger(model) # generate!(normpath(__dir, "22_snapshot_weights.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "23_snapshots_from_csv.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "25_global_parameters.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger( + generate!(normpath(__dir, "25_global_parameters.iesopt.yaml"); config, skip_validation=true), + ) # generate!( # normpath(__dir, "26_initial_states.iesopt.yaml"); # config, @@ -72,8 +94,12 @@ # generate!(normpath(__dir, "29_advanced_unit_commitment.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "31_exclusive_operation.iesopt.yaml"); config, skip_validation=true) # generate!(normpath(__dir, "37_certificates.iesopt.yaml"); config=Dict("general.verbosity.core" => "error")) - generate!(normpath(__dir, "44_lossy_connections.iesopt.yaml"); config, skip_validation=true) - generate!(normpath(__dir, "47_disable_components.iesopt.yaml"); config, skip_validation=true) + safe_close_filelogger( + generate!(normpath(__dir, "44_lossy_connections.iesopt.yaml"); config, skip_validation=true), + ) + safe_close_filelogger( + generate!(normpath(__dir, "47_disable_components.iesopt.yaml"); config, skip_validation=true), + ) end # Clean up output files after testing is done. From 304f87af0e20a15b636cd15f458a39a49de99c94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Str=C3=B6mer?= Date: Wed, 20 Nov 2024 13:41:53 +0100 Subject: [PATCH 5/5] fix: catch exceptions in windows CI runner related to file removal (for files that may still be in use) --- src/IESopt.jl | 8 +++++++- src/precompile/precompile_tools.jl | 9 +++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/IESopt.jl b/src/IESopt.jl index dbacc08..40cd080 100644 --- a/src/IESopt.jl +++ b/src/IESopt.jl @@ -609,7 +609,13 @@ function _optimize!(model::JuMP.Model; @nospecialize(kwargs...)) lcsn = lowercase(JuMP.solver_name(model)) scenario_name = @config(model, general.name.scenario) log_file = abspath(@config(model, paths.results), "$(scenario_name).$(lcsn).log") - rm(log_file; force=true) + + try + rm(log_file; force=true) + catch + @warn "Failed to cleanup solver log file; maybe it appends, maybe it overwrites, maybe it fails - we do not know" log_file + end + if JuMP.solver_name(model) == "Gurobi" @info "Logging solver output" log_file JuMP.set_attribute(model, "LogFile", log_file) diff --git a/src/precompile/precompile_tools.jl b/src/precompile/precompile_tools.jl index 60bc41f..00fcedc 100644 --- a/src/precompile/precompile_tools.jl +++ b/src/precompile/precompile_tools.jl @@ -102,8 +102,13 @@ ) end - # Clean up output files after testing is done. - rm(normpath(__dir, "out"); force=true, recursive=true) + try + # Clean up output files after testing is done. + rm(normpath(__dir, "out"); force=true, recursive=true) + catch + @warn "Failed to cleanup output files after precompilation, left-overs might be present" path = + normpath(__dir, "out") + end end precompile(_attach_optimizer, (JuMP.Model,))