Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RFC: Initialize runtime globals during __init__ #703

Draft
wants to merge 8 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
333 changes: 169 additions & 164 deletions src/HDF5.jl

Large diffs are not rendered by default.

501 changes: 301 additions & 200 deletions src/api_types.jl

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions src/blosc_filter.jl
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ function blosc_set_local(dcpl::hid_t, htype::hid_t, space::hid_t)
end

htypesize = h5t_get_size(htype)
if h5t_get_class(htype) == H5T_ARRAY
if h5t_get_class(htype) == H5T.ARRAY
hsuper = h5t_get_super(htype)
basetypesize = h5t_get_size(hsuper)
h5t_close(hsuper)
Expand Down Expand Up @@ -61,7 +61,7 @@ function blosc_filter(flags::Cuint, cd_nelmts::Csize_t,
doshuffle = cd_nelmts >= 6 ? unsafe_load(cd_values, 6) != 0 : true
# to do: set compressor based on compcode in unsafe_load(cd_values, 7)?

if (flags & H5Z_FLAG_REVERSE) == 0 # compressing
if (flags & H5Z.FLAG_REVERSE) == 0 # compressing
# Allocate an output buffer exactly as long as the input data; if
# the result is larger, we simply return 0. The filter is flagged
# as optional, so HDF5 marks the chunk as uncompressed and proceeds.
Expand Down Expand Up @@ -100,7 +100,7 @@ function register_blosc()
c_blosc_filter = @cfunction(blosc_filter, Csize_t,
(Cuint, Csize_t, Ptr{Cuint}, Csize_t,
Ptr{Csize_t}, Ptr{Ptr{Cvoid}}))
h5z_register(H5Z_class_t(H5Z_CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter))
h5z_register(H5Z_class_t(H5Z.CLASS_T_VERS, FILTER_BLOSC, 1, 1, pointer(blosc_name), C_NULL, c_blosc_set_local, c_blosc_filter))

return nothing
end
Expand All @@ -109,7 +109,7 @@ const set_blosc_values = Cuint[0,0,0,0,5,1,0]
function h5p_set_blosc(p::Properties, level::Integer=5)
0 <= level <= 9 || throw(ArgumentError("blosc compression $level not in [0,9]"))
set_blosc_values[5] = level
h5p_set_filter(p.id, FILTER_BLOSC, H5Z_FLAG_OPTIONAL, length(set_blosc_values), set_blosc_values)
h5p_set_filter(p.id, FILTER_BLOSC, H5Z.FLAG_OPTIONAL, length(set_blosc_values), set_blosc_values)

return nothing
end
10 changes: 9 additions & 1 deletion src/deprecated.jl
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ end

### Changed in PR#689
# - switch from H5Rdereference1 to H5Rdereference2
@deprecate h5r_dereference(obj_id, ref_type, ref) h5r_dereference(obj_id, H5P_DEFAULT, ref_type, ref) false
@deprecate h5r_dereference(obj_id, ref_type, ref) h5r_dereference(obj_id, H5P.DEFAULT, ref_type, ref) false

### Changed in PR#690, PR#695
# - rename exported bindings
Expand Down Expand Up @@ -222,3 +222,11 @@ import Base: names
### Changed in PR#694
@deprecate has(parent::Union{File,Group,Dataset}, path::AbstractString) Base.haskey(parent, path)
@deprecate exists(parent::Union{File,Group,Dataset,Datatype,Attributes}, path::AbstractString) Base.haskey(parent, path) false

for pseudomod in (H5, H5D, H5E, H5F, H5FD, H5I, H5L, H5O, H5P, H5R, H5S, H5T, H5Z)
modsym = nameof(typeof(pseudomod))
for C in propertynames(pseudomod)
constname = Symbol(modsym, :_, C)
@eval @deprecate_binding($constname, $modsym.$C, false)
end
end
124 changes: 124 additions & 0 deletions src/macros.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
import Base.Meta: isexpr

"""
```julia
@defconstants H5PREFIX begin
STATIC_CONSTANT::hid_t = 1
DYNAMIC_CONSTANT::hid_t
...
end
```

Defines a collection of "constants" (`STATIC_CONSTANT`, `DYANMIC_CONSTANT`, ...) within
the namespace `H5PREFIX` wherein the distinction between runtime and static values is
abstracted away from the user's view.

For truly constant values such as `STATIC_CONSTANT`, the syntax is given as in the first
line above where the value is assigned to a name with a type assertion. The value may also
be an expression which is evaluated the current module context. Note that the value
evaluation occurs before the constants are assigned, so the expression cannot refer to
any previous constants within the same block; e.g. the following will not work:
```julia
julia> @defconstants BITMASK begin
BIT_ONE::UInt = 0x1
BIT_TWO::UInt = 0x2
BIT_BOTH::Uint = BIT_ONE | BIT_TWO
end
ERROR: LoadError: UndefVarError: BIT_ONE not defined
...
```

The second permitted syntax is for runtime constants which require initialization.
In this form, only the name and value type are given, and an appropriately-typed,
uninitialized `Ref` container is assigned to the constant binding. These values must be
assigned at runtime from an `__init__()` function:
```julia
function __init__()
H5PREFIX.DYANMIC_CONSTANT = my_runtime_value()
end
```
"""
macro defconstants(prefix::Symbol, expr::Expr)
isexpr(expr, :block) || error("Expected block expression")
stmts = expr.args
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe better to Base.remove_linenums!(expr1) first ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Possibly — I don't know which is faster to execute. I was guessing the isa-continue might be since it wouldn't have to modify any arrays.


symbols = Vector{Symbol}(undef, 0)
imports = Vector{Union{Symbol,Expr}}(undef, 0)
defbody = Vector{Expr}(undef, 0)
getbody = Vector{Expr}(undef, 0)
setbody = Vector{Expr}(undef, 0)

innermod = Symbol(:_, prefix)
for line in stmts
line isa LineNumberNode && continue

isruntime = isexpr(line, :(=)) ? false :
isexpr(line, :(::)) ? true :
error("Unexpected statement: ", line)

# Get the name and type pair
nametype = isruntime ? line : line.args[1]
isexpr(nametype, :(::)) || error("Expected `name::type`, got: ", nametype)
name = nametype.args[1]::Symbol
type = nametype.args[2]::Union{Symbol,Expr}
# Save type for later use
push!(imports, type)

if isruntime
value = esc(:(Ref{$type}()))
else
valexpr = :(convert($type, $(line.args[2])))
value = QuoteNode(Core.eval(__module__, valexpr))
end
fullname = esc(:($innermod.$name))
getexpr = isruntime ? :($(fullname)[]) : fullname

push!(symbols, name)
push!(defbody, :(const $(esc(name)) = $value))
push!(getbody, :(sym === $(QuoteNode(name)) && return $getexpr))
if isruntime
setexpr = quote
sym === $(QuoteNode(name)) && begin
$fullname[] = value
return value
end
end
append!(setbody, Base.remove_linenums!(setexpr).args)
end
end

# Build expressions to import all necessary types from the parent module.
imports = Expr[Expr(:(.), :(.), :(.), sym) for sym in unique!(imports)]
# Push the imports into the definition body
pushfirst!(defbody, Expr(:import, imports...))

eprefix = esc(prefix)
einnermod = esc(innermod)
block = quote
baremodule $einnermod
struct $eprefix end
$(defbody...)
end
const $eprefix = $einnermod.$prefix()
function Base.propertynames(::$einnermod.$prefix)
return $((symbols...,))
end
function Base.getproperty(::$einnermod.$prefix, sym::Symbol)
$(getbody...)
error($(string(prefix) * " has no constant "), sym)
end
end
if !isempty(setbody)
setfn = quote
function Base.setproperty!(::$einnermod.$prefix, sym::Symbol, value)
$(setbody...)
error($(string(prefix) * "."), sym, " cannot be set")
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably change the error message to be consistent with what is printed in Base. Either:

Assigning a variable to in an other module:
"ERROR: cannot assign variables in other modules"

Trying to redefine a constant:
"ERROR: cannot declare x constant; it already has a value"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I like the second one because it's still technically correct.

end
end
append!(block.args, Base.remove_linenums!(setfn).args)
end

Base.remove_linenums!(block)
block.head = :toplevel
return block
end
20 changes: 10 additions & 10 deletions test/compound.jl
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,18 @@ function unsafe_convert(::Type{foo_hdf5}, x::foo)
end

function datatype(::Type{foo_hdf5})
dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(foo_hdf5))
dtype = HDF5.h5t_create(HDF5.H5T.COMPOUND, sizeof(foo_hdf5))
HDF5.h5t_insert(dtype, "a", fieldoffset(foo_hdf5, 1), datatype(Float64))

vlenstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1)
HDF5.h5t_set_size(vlenstr_dtype, HDF5.H5T_VARIABLE)
HDF5.h5t_set_cset(vlenstr_dtype, HDF5.H5T_CSET_UTF8)
vlenstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1)
HDF5.h5t_set_size(vlenstr_dtype, HDF5.H5T.VARIABLE)
HDF5.h5t_set_cset(vlenstr_dtype, HDF5.H5T.CSET_UTF8)
HDF5.h5t_insert(dtype, "b", fieldoffset(foo_hdf5, 2), vlenstr_dtype)

fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1)
fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1)
HDF5.h5t_set_size(fixedstr_dtype, 20 * sizeof(UInt8))
HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T_CSET_UTF8)
HDF5.h5t_set_strpad(fixedstr_dtype, HDF5.H5T_STR_NULLPAD)
HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T.CSET_UTF8)
HDF5.h5t_set_strpad(fixedstr_dtype, HDF5.H5T.STR_NULLPAD)
HDF5.h5t_insert(dtype, "c", fieldoffset(foo_hdf5, 3), fixedstr_dtype)

hsz = HDF5.hsize_t[3,3]
Expand All @@ -63,11 +63,11 @@ struct bar_hdf5
end

function datatype(::Type{bar_hdf5})
dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(bar_hdf5))
dtype = HDF5.h5t_create(HDF5.H5T.COMPOUND, sizeof(bar_hdf5))

fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T_C_S1)
fixedstr_dtype = HDF5.h5t_copy(HDF5.H5T.C_S1)
HDF5.h5t_set_size(fixedstr_dtype, 20 * sizeof(UInt8))
HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T_CSET_UTF8)
HDF5.h5t_set_cset(fixedstr_dtype, HDF5.H5T.CSET_UTF8)

hsz = HDF5.hsize_t[2]
array_dtype = HDF5.h5t_array_create(fixedstr_dtype, 1, hsz)
Expand Down
2 changes: 1 addition & 1 deletion test/custom.jl
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ struct Simple
end

function datatype(::Type{Simple})
dtype = HDF5.h5t_create(HDF5.H5T_COMPOUND, sizeof(Simple))
dtype = HDF5.h5t_create(H5T.COMPOUND, sizeof(Simple))
HDF5.h5t_insert(dtype, "a", fieldoffset(Simple, 1), datatype(Float64))
HDF5.h5t_insert(dtype, "b", fieldoffset(Simple, 2), datatype(Int))
HDF5.Datatype(dtype)
Expand Down
6 changes: 3 additions & 3 deletions test/extend_test.jl
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,11 @@ b = d_create(fid, "b", Int, ((1000,), (-1,)), chunk=(100,)) #-1 is equivalent to
b[1:200] = ones(200)
dims, max_dims = HDF5.get_dims(b)
@test dims == (UInt64(1000),)
@test max_dims == (HDF5.H5S_UNLIMITED,)
@test max_dims == (H5S.UNLIMITED,)
set_dims!(b, (10000,))
dims, max_dims = HDF5.get_dims(b)
@test dims == (UInt64(10000),)
@test max_dims == (HDF5.H5S_UNLIMITED,)
@test max_dims == (H5S.UNLIMITED,)
#println("b is size current $(map(int,HDF5.get_dims(b)[1])) max $(map(int,HDF5.get_dims(b)[2]))")
# b[:] = [1:10000] # gave error no method lastindex(HDF5.Dataset{PlainHDF5File},),
# so I defined lastindex(dset::HDF5.Dataset) = length(dset), and exported lastindex
Expand All @@ -71,7 +71,7 @@ dims, max_dims = HDF5.get_dims(d_again)
b_again = fid["b"]
dims, max_dims = HDF5.get_dims(b_again)
@test dims == (UInt64(10000),)
@test max_dims == (HDF5.H5S_UNLIMITED,)
@test max_dims == (H5S.UNLIMITED,)
#println("b is size current $(map(int,HDF5.get_dims(b)[1])) max $(map(int,HDF5.get_dims(b)[2]))")

close(fid)
Expand Down
4 changes: 2 additions & 2 deletions test/gc.jl
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ GC.enable(false)
fn = tempname()
for i = 1:10
file = h5open(fn, "w")
memtype_id = HDF5.h5t_create(HDF5.H5T_COMPOUND, 2*sizeof(Float64))
memtype_id = HDF5.h5t_create(HDF5.H5T.COMPOUND, 2*sizeof(Float64))
HDF5.h5t_insert(memtype_id, "real", 0, HDF5.hdf5_type_id(Float64))
HDF5.h5t_insert(memtype_id, "imag", sizeof(Float64), HDF5.hdf5_type_id(Float64))
dt = HDF5.Datatype(memtype_id)
Expand Down Expand Up @@ -64,7 +64,7 @@ for i = 1:10
end
GC.enable(true)

let plist = p_create(HDF5.H5P_FILE_ACCESS) # related to issue #620
let plist = p_create(HDF5.H5P.FILE_ACCESS) # related to issue #620
HDF5.h5p_close(plist)
@test_nowarn finalize(plist)
end
Expand Down
2 changes: 1 addition & 1 deletion test/mmap.jl
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ f = h5open(fn, "w")
# datasets) and the other with explicit early allocation.
hdf5_A = d_create(f, "A", datatype(Int64), dataspace(3,3))
hdf5_B = d_create(f, "B", datatype(Float64), dataspace(3,3);
alloc_time = HDF5.H5D_ALLOC_TIME_EARLY)
alloc_time = H5D.ALLOC_TIME_EARLY)
# The late case cannot be mapped yet.
@test_throws ErrorException("Error getting offset") readmmap(f["A"])
# Then write and fill dataset A, making it mappable. B was filled with 0.0 at
Expand Down
2 changes: 1 addition & 1 deletion test/mpio.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ myrank = MPI.Comm_rank(comm)

@test HDF5.has_parallel()

let fileprop = p_create(HDF5.H5P_FILE_ACCESS)
let fileprop = p_create(HDF5.H5P.FILE_ACCESS)
HDF5.h5p_set_fapl_mpio(fileprop, comm, info)
h5comm, h5info = HDF5.h5p_get_fapl_mpio(fileprop)

Expand Down
26 changes: 13 additions & 13 deletions test/plain.jl
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,13 @@ write(f, "salut", salut)
write(f, "ucode", ucode)
# Manually write a variable-length string (issue #187)
let
dtype = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_C_S1))
HDF5.h5t_set_size(dtype.id, HDF5.H5T_VARIABLE)
dtype = HDF5.Datatype(HDF5.h5t_copy(H5T.C_S1))
HDF5.h5t_set_size(dtype.id, H5T.VARIABLE)
HDF5.h5t_set_cset(dtype.id, HDF5.cset(typeof(salut)))
dspace = HDF5.dataspace(salut)
dset = HDF5.d_create(f, "salut-vlen", dtype, dspace)
GC.@preserve salut begin
HDF5.h5d_write(dset, dtype, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, [pointer(salut)])
HDF5.h5d_write(dset, dtype, H5S.ALL, H5S.ALL, H5P.DEFAULT, [pointer(salut)])
end
end
# Arrays of strings
Expand Down Expand Up @@ -355,14 +355,14 @@ close(fd)
rm(fn)

# File creation and access property lists
cpl = p_create(HDF5.H5P_FILE_CREATE)
cpl = p_create(H5P.FILE_CREATE)
cpl[:userblock] = 1024
apl = p_create(HDF5.H5P_FILE_ACCESS)
apl[:libver_bounds] = (HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST)
apl = p_create(H5P.FILE_ACCESS)
apl[:libver_bounds] = (H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST)
h5open(fn, false, true, true, true, false, cpl, apl) do fid
write(fid, "intarray", [1, 2, 3])
end
h5open(fn, "r", libver_bounds=(HDF5.H5F_LIBVER_EARLIEST, HDF5.H5F_LIBVER_LATEST)) do fid
h5open(fn, "r", libver_bounds=(H5F.LIBVER_EARLIEST, H5F.LIBVER_LATEST)) do fid
intarray = read(fid, "intarray")
@test intarray == [1, 2, 3]
end
Expand Down Expand Up @@ -626,8 +626,8 @@ end # empty and 0-size arrays
fn = tempname()
hfile = h5open(fn, "w")

dtype_varstring = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_C_S1))
HDF5.h5t_set_size(dtype_varstring, HDF5.H5T_VARIABLE)
dtype_varstring = HDF5.Datatype(HDF5.h5t_copy(H5T.C_S1))
HDF5.h5t_set_size(dtype_varstring, H5T.VARIABLE)

write(hfile, "uint8_array", UInt8[(1:8)...])
write(hfile, "bool_scalar", true)
Expand All @@ -637,7 +637,7 @@ varstring = "var"
write(hfile, "fixed_string", fixstring)
vardset = d_create(hfile, "variable_string", dtype_varstring, dataspace(varstring))
GC.@preserve varstring begin
HDF5.h5d_write(vardset, dtype_varstring, HDF5.H5S_ALL, HDF5.H5S_ALL, HDF5.H5P_DEFAULT, [pointer(varstring)])
HDF5.h5d_write(vardset, dtype_varstring, H5S.ALL, H5S.ALL, H5P.DEFAULT, [pointer(varstring)])
end
flush(hfile)
close(dtype_varstring)
Expand Down Expand Up @@ -699,10 +699,10 @@ dset = d_create(group, "dset", datatype(Int), dataspace((1,)))
meta = a_create(dset, "meta", datatype(Bool), dataspace((1,)))
@test sprint(show, meta) == "HDF5 attribute: meta"

prop = p_create(HDF5.H5P_DATASET_CREATE)
prop = p_create(H5P.DATASET_CREATE)
@test sprint(show, prop) == "HDF5 property: dataset create class"

dtype = HDF5.Datatype(HDF5.h5t_copy(HDF5.H5T_IEEE_F64LE))
dtype = HDF5.Datatype(HDF5.h5t_copy(H5T.IEEE_F64LE))
@test sprint(show, dtype) == "HDF5 datatype: H5T_IEEE_F64LE"

dspace = dataspace((1,))
Expand Down Expand Up @@ -839,7 +839,7 @@ hfile[GenericString("test")] = 17.2
@test_nowarn a_delete(dset1, GenericString("meta1"))

# transient types
memtype_id = HDF5.h5t_copy(HDF5.H5T_NATIVE_DOUBLE)
memtype_id = HDF5.h5t_copy(H5T.NATIVE_DOUBLE)
dt = HDF5.Datatype(memtype_id)
@test !HDF5.h5t_committed(dt)
t_commit(hfile, GenericString("dt"), dt)
Expand Down
Loading