Refactor and unify binding generators (#37)

Co-authored-by: Valentin Churavy <v.churavy@gmail.com>

Manifest.toml

@@ -0,0 +1,183 @@
+# This file is machine-generated - editing it directly is not advised
+
+[[ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
+
+[[Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[CEnum]]
+git-tree-sha1 = "eb4cb44a499229b3b8426dcfb5dd85333951ff90"
+uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
+version = "0.4.2"
+
+[[Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[Downloads]]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
+
+[[FileWatching]]
+uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
+
+[[InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[JLLWrappers]]
+deps = ["Artifacts", "Preferences"]
+git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.5.0"
+
+[[LLVM]]
+deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Printf", "Unicode"]
+git-tree-sha1 = "5007c1421563108110bbd57f63d8ad4565808818"
+uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
+version = "5.2.0"
+
+[[LLVMExtra_jll]]
+deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"]
+git-tree-sha1 = "1222116d7313cdefecf3d45a2bc1a89c4e7c9217"
+uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab"
+version = "0.0.22+0"
+
+[[LazyArtifacts]]
+deps = ["Artifacts", "Pkg"]
+uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+
+[[LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.4"
+
+[[LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "8.4.0+0"
+
+[[LibGit2]]
+deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[LibGit2_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
+uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.6.4+0"
+
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.11.0+1"
+
+[[Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[MLIR_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl", "TOML", "Zlib_jll", "libLLVM_jll"]
+git-tree-sha1 = "5a321efba65796a42f64be069875fca0973ee89a"
+uuid = "a70bccb4-a5c0-5e2e-a329-e731972457e8"
+version = "15.0.7+9"
+
+[[Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.2+1"
+
+[[MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2023.1.10"
+
+[[NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
+
+[[Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.10.0"
+
+[[Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.4.1"
+
+[[Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.3"
+
+[[Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
+
+[[UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.13+1"
+
+[[libLLVM_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8f36deef-c2a5-5394-99ed-8e07531fb29a"
+version = "15.0.7+10"
+
+[[mlir_jl_tblgen_jll]]
+deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"]
+git-tree-sha1 = "ed5e9e896e64afd8f94f9b579f27507253b2c520"
+uuid = "3dcf74f3-6bc3-597e-9d13-043388c3118a"
+version = "0.0.1+0"
+
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.52.0+1"
+
+[[p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "17.4.0+2"

Project.toml

@@ -12,5 +12,5 @@ mlir_jl_tblgen_jll = "3dcf74f3-6bc3-597e-9d13-043388c3118a"
 [compat]
 CEnum = "0.4"
 LLVM = "5"
-MLIR_jll = "14,15"
+MLIR_jll = "14,15,16"
 julia = "1.9"

res/Project.toml → bindings/Project.toml

@@ -1,4 +1,4 @@
 [deps]
 BinaryBuilderBase = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 Clang = "40e3b903-d033-50b4-a0cc-940c62c95e31"
-Clang_jll = "0ee61d77-7f21-5576-8119-9fcc46b10100"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"

bindings/make.jl

@@ -0,0 +1,160 @@
+if VERSION >= v"1.8" || VERSION < v"1.7"
+    error("This script only supports 1.7")
+end
+
+using Pkg
+import BinaryBuilderBase: PkgSpec, Prefix, temp_prefix, setup_dependencies, cleanup_dependencies, destdir
+using Clang.Generators
+
+function mlir_dialects(version::VersionNumber)
+    dialects = Tuple{String,String}[
+        ("Builtin.jl", "../IR/BuiltinOps.td"),
+    ]
+
+    if version >= v"14"
+        append!(dialects, [
+            ("AMX.jl", "AMX/AMX.td"),
+            ("Affine.jl", "Affine/IR/AffineOps.td"),
+            ("ArmNeon.jl", "ArmNeon/ArmNeon.td"),
+            ("ArmSVE.jl", "ArmSVE/ArmSVE.td"),
+            ("Async.jl", "Async/IR/AsyncOps.td"),
+            ("Bufferization.jl", "Bufferization/IR/BufferizationOps.td"),
+            ("Complex.jl", "Complex/IR/ComplexOps.td"),
+            # ("DLTI.jl", "DLTI/DLTI.td"), fails on v15
+            ("EmitC.jl", "EmitC/IR/EmitC.td"),
+            ("LLVMIR.jl", "LLVMIR/LLVMOps.td"),
+            ("Linalg.jl", "Linalg/IR/LinalgOps.td"), # TODO include LinalgStructuredOps.td
+            ("Math.jl", "Math/IR/MathOps.td"),
+            ("MemRef.jl", "MemRef/IR/MemRefOps.td"),
+            ("OpenACC.jl", "OpenACC/OpenACCOps.td"),
+            ("OpenMP.jl", "OpenMP/OpenMPOps.td"),
+            ("PDL.jl", "PDL/IR/PDLOps.td"),
+            ("PDLInterp.jl", "PDLInterp/IR/PDLInterpOps.td"),
+            ("Quant.jl", "Quant/QuantOps.td"),
+            ("SPIRV.jl", "SPIRV/IR/SPIRVOps.td"),
+            ("Shape.jl", "Shape/IR/ShapeOps.td"),
+            ("SparseTensor.jl", "SparseTensor/IR/SparseTensorOps.td"),
+            ("Tensor.jl", "Tensor/IR/TensorOps.td"),
+            ("Tosa.jl", "Tosa/IR/TosaOps.td"),
+            ("Vector.jl", "Vector/IR/VectorOps.td"),
+            ("X86Vector.jl", "X86Vector/X86Vector.td"),
+        ])
+    end
+
+    if v"14" <= version < v"15"
+        append!(dialects, [
+            ("GPU.jl", "GPU/GPUOps.td"),
+            ("SCF.jl", "SCF/SCFOps.td"),
+            ("StandardOps.jl", "StandardOps/IR/Ops.td"),
+        ])
+    end
+
+    if v"14" <= version < v"16"
+        append!(dialects, [
+            ("Arithmetic.jl", "Arithmetic/IR/ArithmeticOps.td"), # renamed to 'Arith' in v16
+        ])
+    end
+
+    if version >= v"15"
+        append!(dialects, [
+            ("GPU.jl", "GPU/IR/GPUOps.td"),
+            ("SCF.jl", "SCF/IR/SCFOps.td"),
+            ("AMDGPU.jl", "AMDGPU/AMDGPU.td"),
+            ("ControlFlow.jl", "ControlFlow/IR/ControlFlowOps.td"),
+            ("Func.jl", "Func/IR/FuncOps.td"),
+            ("MLProgram.jl", "MLProgram/IR/MLProgramOps.td"),
+            ("NVGPU.jl", "NVGPU/IR/NVGPU.td"),
+            ("Transform.jl", "Transform/IR/TransformOps.td"),
+        ])
+    end
+
+    if version >= v"16"
+        append!(dialects, [
+            ("Arith.jl", "Arith/IR/ArithOps.td"),
+            ("Index.jl", "Index/IR/IndexOps.td"),
+        ])
+    end
+
+    if version >= v"17"
+        append!(dialects, [
+            ("ArmSME.jl", "ArmSME/IR/ArmSME.td"),
+            ("IRDL.jl", "IRDL/IR/IRDLOps.td"),
+            ("UB.jl", "UB/IR/UBOps.td"),
+        ])
+    end
+
+    if version >= v"18"
+        append!(dialects, [
+            ("Mesh.jl", "Mesh/IR/MeshOps.td"),
+        ])
+    end
+
+    return dialects
+end
+
+function rewrite!(dag::ExprDAG) end
+
+julia_llvm = Dict([v"1.9" => v"14.0.5", v"1.10" => v"15.0.7", v"1.11" => v"16.0.6"])
+options = load_options(joinpath(@__DIR__, "wrap.toml"))
+
+@add_def off_t
+@add_def MlirTypesCallback
+
+for (julia_version, llvm_version) in julia_llvm
+    println("Generating... julia version: $julia_version, llvm version: $llvm_version")
+
+    temp_prefix() do prefix
+        platform = Pkg.BinaryPlatforms.HostPlatform()
+        platform["llvm_version"] = string(llvm_version.major)
+        platform["julia_version"] = string(julia_version)
+
+        # Note: 1.10
+        dependencies = PkgSpec[
+            PkgSpec(; name="LLVM_full_jll", version=llvm_version),
+            PkgSpec(; name="mlir_jl_tblgen_jll")
+            ]
+
+        artifact_paths = setup_dependencies(prefix, dependencies, platform; verbose=true)
+
+        mlir_jl_tblgen = joinpath(destdir(prefix, platform), "bin", "mlir-jl-tblgen")
+        include_dir = joinpath(destdir(prefix, platform), "include")
+
+        # generate MLIR API bindings
+        mkpath(joinpath(@__DIR__, "..", "src", "API", string(llvm_version.major)))
+
+        let options = deepcopy(options)
+            output_file_path = joinpath(@__DIR__, "..", "src", "API", string(llvm_version.major), options["general"]["output_file_path"])
+            isdir(dirname(output_file_path)) || mkpath(dirname(output_file_path))
+            options["general"]["output_file_path"] = output_file_path
+
+            libmlir_header_dir = joinpath(include_dir, "mlir-c")
+            args = Generators.get_default_args()
+            append!(args, ["-I", include_dir, "-x", "c++"])
+
+            headers = detect_headers(libmlir_header_dir, args, Dict(), endswith("Python/Interop.h"))
+            ctx = create_context(headers, args, options)
+
+            # build without printing so we can do custom rewriting
+            build!(ctx, BUILDSTAGE_NO_PRINTING)
+
+            rewrite!(ctx.dag)
+
+            # print
+            build!(ctx, BUILDSTAGE_PRINTING_ONLY)
+        end
+
+        # generate MLIR dialect bindings
+        mkpath(joinpath(@__DIR__, "..", "src", "Dialects", string(llvm_version.major)))
+
+        for (binding, td) in mlir_dialects(llvm_version)
+            flags = [
+                "--generator=jl-op-defs",
+                joinpath(include_dir, "mlir", "Dialect", td),
+                "-I", include_dir,
+                "-o", joinpath(@__DIR__, "..", "src", "Dialects", string(llvm_version.major), binding),
+            ]
+            run(`$mlir_jl_tblgen $flags`)
+            println("- Generated \"$binding\" from \"$td\"")
+        end
+    end
+end