From 4eb67be7f48c67b9916c4b4c601eea072375a17f Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sat, 3 Feb 2024 20:19:30 +0000 Subject: [PATCH 1/6] Migrate to PythonCall and drop PyCall --- .vscode/settings.json | 2 ++ CondaPkg.toml | 3 +++ Project.toml | 2 ++ src/Andes.jl | 23 +++++++++-------------- src/kvxopt_pythoncall.jl | 22 ++++++++++++++++++++++ test/runtests.jl | 2 +- test/test_andes_py.jl | 12 ------------ test/test_andes_pythoncall.jl | 21 +++++++++++++++++++++ 8 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 CondaPkg.toml create mode 100644 src/kvxopt_pythoncall.jl delete mode 100644 test/test_andes_py.jl create mode 100644 test/test_andes_pythoncall.jl diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..7a73a41 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/CondaPkg.toml b/CondaPkg.toml new file mode 100644 index 0000000..27db262 --- /dev/null +++ b/CondaPkg.toml @@ -0,0 +1,3 @@ +[deps] +andes = "" +kvxopt = "" diff --git a/Project.toml b/Project.toml index 5b22454..c076cb9 100644 --- a/Project.toml +++ b/Project.toml @@ -5,8 +5,10 @@ version = "0.1.2" [deps] Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" +CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [compat] diff --git a/src/Andes.jl b/src/Andes.jl index 9095744..269cd42 100644 --- a/src/Andes.jl +++ b/src/Andes.jl @@ -2,24 +2,19 @@ __precompile__() module Andes -using Conda -using PyCall +using PythonCall -import SparseArrays +const py = PythonCall.pynew() -const py = PyNULL() - -include("kvxopt.jl") +include("kvxopt_pythoncall.jl") function __init__() - copy!(py, pyimport_conda("andes", "andes", "conda-forge")) + PythonCall.pycopy!(py, pyimport("andes")) - pytype_mapping(pyimport("kvxopt").spmatrix, SparseArrays.SparseMatrixCSC) + PythonCall.pyconvert_add_rule("kvxopt.base:spmatrix", + AbstractSparseMatrixCSC, + spmatrix_to_julia, + ) end - -# --- export --- -export convert; - -end # module - +end \ No newline at end of file diff --git a/src/kvxopt_pythoncall.jl b/src/kvxopt_pythoncall.jl new file mode 100644 index 0000000..f5d49ca --- /dev/null +++ b/src/kvxopt_pythoncall.jl @@ -0,0 +1,22 @@ +using CondaPkg +using SparseArrays: AbstractSparseMatrixCSC, SparseMatrixCSC +using PythonCall: pyconvert_add_rule + +""" +Convert KVXOPT.spmatrix to SparseMatrixCSC +""" +function spmatrix_to_julia(S::Type{T}, x::Py) where {T<:AbstractSparseMatrixCSC} + # Ensure x is a KVXOPT spmatrix + if Bool(pytype(x) != pyimport("kvxopt").spmatrix) + throw(ArgumentError("x must be a KVXOPT spmatrix")) + end + + # Extract the size, I, J, and V arrays from the spmatrix and explicitly convert them + m, n = pyconvert(Tuple{Int64,Int64}, x.size) + I = pyconvert(Vector{Int64}, x.CCS[0]) .+ 1 + J = pyconvert(Vector{Int64}, x.CCS[1]) .+ 1 + V = pyconvert(Vector{Float64}, x.CCS[2]) + + # Create and return the SparseMatrixCSC + return PythonCall.pyconvert_return(SparseMatrixCSC(m, n, I, J, V)) +end diff --git a/test/runtests.jl b/test/runtests.jl index 1133f66..bac8983 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,3 +1,3 @@ using Test -include("test_andes_py.jl") +include("test_andes_pythoncall.jl") diff --git a/test/test_andes_py.jl b/test/test_andes_py.jl deleted file mode 100644 index ba728c0..0000000 --- a/test/test_andes_py.jl +++ /dev/null @@ -1,12 +0,0 @@ -using Andes - -@testset "Andes.jl run power flow" begin - kundur = Andes.py.utils.paths.get_case("kundur/kundur_full.xlsx") - system = Andes.py.run(kundur, no_output=true) - - @test system.PFlow.converged == true - - # test sparse matrix conversion - @test size(system.PFlow.A) == (28, 28) - -end diff --git a/test/test_andes_pythoncall.jl b/test/test_andes_pythoncall.jl new file mode 100644 index 0000000..440fd0b --- /dev/null +++ b/test/test_andes_pythoncall.jl @@ -0,0 +1,21 @@ +using Test +using Andes +using PythonCall +using SparseArrays + +@testset "Test Andes functionalities" begin + @testset "SparseMatrixCSC conversion from Andes system example" begin + ss = Andes.py.system.example() + converted_matrix = pyconvert(SparseMatrixCSC, ss.dae.gy) + @test converted_matrix isa SparseMatrixCSC + @test size(converted_matrix) == (34, 34) + end + + @testset "Power flow run" begin + Andes.py.config_logger(40) + kundur = Andes.py.utils.paths.get_case("kundur/kundur_full.xlsx") + system = Andes.py.run(kundur, no_output=true) + + @test Bool(system.PFlow.converged == true) + end +end From b2225cac03a7cc0bcba28d9a546a7e755647f5e6 Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sun, 4 Feb 2024 20:59:44 +0000 Subject: [PATCH 2/6] Clean up. --- src/Andes.jl | 2 ++ src/kvxopt.jl | 45 --------------------------------------------- 2 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 src/kvxopt.jl diff --git a/src/Andes.jl b/src/Andes.jl index 269cd42..1a495ea 100644 --- a/src/Andes.jl +++ b/src/Andes.jl @@ -17,4 +17,6 @@ function __init__() ) end +export pyconvert, pytype + end \ No newline at end of file diff --git a/src/kvxopt.jl b/src/kvxopt.jl deleted file mode 100644 index 0a906c5..0000000 --- a/src/kvxopt.jl +++ /dev/null @@ -1,45 +0,0 @@ -import Base:convert -import SparseArrays: AbstractSparseMatrixCSC, SparseMatrixCSC - - -""" -Convert Julia array to KVXOPT matrix or spmatrix -""" -function julia_to_kvxopt(A) - if issparse(A) - J = zeros(Int64, length(A.rowval)); - for i = 1:size(A, 2) - J[A.colptr[i]:A.colptr[i + 1] - 1] .= i - 1; - end - I = A.rowval .- 1 - V = A.nzval - Ap = @pycall kvxopt.spmatrix(PyCall.array2py(vec(V)), PyCall.array2py(vec(I)), PyCall.array2py(vec(J)), (size(A, 1), size(A, 2)))::PyObject; - elseif isempty(A) - Ap = pybuiltin("None"); - else - sA = size(A) - if length(sA) == 1 - sA = (sA[1], 1) - end - Ap = @pycall kvxopt.matrix(PyCall.array2py(vec(A)), sA)::PyObject; - end - return Ap; -end - -""" -Convert KVXOPT spmatrix to SparseMatrixCSC -""" -function convert(::Type{T}, A::PyObject) where T <: AbstractSparseMatrixCSC - if A.typecode == "d" - r = SparseMatrixCSC(A.size[1], A.size[2], - vec(A.CCS[1])::Vector{Int64} .+ 1, - vec(A.CCS[2])::Vector{Int64} .+ 1, - vec(A.CCS[3])::Vector{Float64})::SparseMatrixCSC{Float64,Int64} - elseif A.typecode == "z" - r = SparseMatrixCSC(A.size[1], A.size[2], - vec(A.CCS[1])::Vector{Int64} .+ 1, - vec(A.CCS[2])::Vector{Int64} .+ 1, - vec(A.CCS[3])::Vector{ComplexF64})::SparseMatrixCSC{ComplexF64,Int64} - end - return r -end From 209c69382eaa966055751d391520ce0fd53f63df Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sun, 4 Feb 2024 21:15:28 +0000 Subject: [PATCH 3/6] Setting package numbers and more clean up. --- CondaPkg.toml | 4 +++- Project.toml | 5 +---- deps/build.jl | 6 +++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/CondaPkg.toml b/CondaPkg.toml index 27db262..7186f70 100644 --- a/CondaPkg.toml +++ b/CondaPkg.toml @@ -1,3 +1,5 @@ +channels = ["conda-forge"] + [deps] andes = "" -kvxopt = "" +kvxopt = ">1.3.0.0" diff --git a/Project.toml b/Project.toml index c076cb9..d320d90 100644 --- a/Project.toml +++ b/Project.toml @@ -4,17 +4,14 @@ authors = ["Hantao Cui "] version = "0.1.2" [deps] -Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [compat] -Conda = "1.4.1" Pkg = "1.3.0" -PyCall = "1.19.4" +PythonCall = "0.9.15" julia = "1.3.1" [extras] diff --git a/deps/build.jl b/deps/build.jl index 5484d34..b6b5d6c 100644 --- a/deps/build.jl +++ b/deps/build.jl @@ -1,9 +1,9 @@ -using Pkg -using PyCall -using Conda +using PythonCall +using CondaPkg try pyimport("andes") + pyimport("kvxopt") catch @warn "PyCall is not configured to an existing Python env." @warn "Andes.jl will use Conda for PyCall and install andes." From 0e21a4b204e25b30a98797e48757750982e6c14f Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sun, 4 Feb 2024 21:17:27 +0000 Subject: [PATCH 4/6] Update --- .github/workflows/documentation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b09ec97..6f62028 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: julia-actions/setup-julia@latest with: - version: '1.6' + version: '1.9' - name: Install dependencies run: julia --project=docs/ -e 'using Pkg; Pkg.develop(PackageSpec(path=pwd())); Pkg.instantiate()' - name: Build and deploy From a6e7cff4f7bc1ad8cf9c2f8715cdbe172914b1fe Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sun, 4 Feb 2024 23:06:35 +0000 Subject: [PATCH 5/6] Update CI. --- .github/workflows/CI.yml | 24 +----------------------- Project.toml | 1 - 2 files changed, 1 insertion(+), 24 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 29a63d3..1bc84cb 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -18,7 +18,7 @@ jobs: fail-fast: false matrix: version: - - '1.8' + - '1.10' - 'nightly' os: - ubuntu-latest @@ -33,28 +33,6 @@ jobs: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - uses: julia-actions/cache@v1 - - name: Fix weird Conda.jl/PyCall.jl build error - env: - PYTHON: "" - shell: julia --color=yes {0} - run: | - using Pkg - try - Pkg.instantiate() - println("Successfully instantiated the test environment") - catch e - display(e) - end - ENV["PYTHON"] = "" - Pkg.add("Conda") - println("Try building Conda and PyCall") - try - Pkg.build("Conda") - using Conda - println("Successfully built Conda") - catch e - display(e) - end - uses: julia-actions/julia-buildpkg@v1 env: PYTHON: "" # for conda packages diff --git a/Project.toml b/Project.toml index 123a587..c06fe0c 100644 --- a/Project.toml +++ b/Project.toml @@ -10,7 +10,6 @@ PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" [compat] -Pkg = "1.3.0" PythonCall = "0.9.15" julia = "1.3.1" From 2d68d8dcac2948458103110bb3205e8df79504ea Mon Sep 17 00:00:00 2001 From: Hantao Cui Date: Sun, 4 Feb 2024 23:12:53 +0000 Subject: [PATCH 6/6] Update tag version. --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index c06fe0c..e001d08 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Andes" uuid = "93a26e3f-343a-4ab9-b467-a68c67574964" -authors = ["Hantao Cui "] -version = "0.2.0" +authors = ["Hantao Cui "] +version = "1.0.0" [deps] CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"