Skip to content

Commit

Permalink
Provide high level syntax for dealing with HDF5 files in memory (#1077)
Browse files Browse the repository at this point in the history
* Fix get_driver for Core

* Enhance creating a file from memory using the Core driver

* Check for file image callbacks

* Formatting

* Use h5_free_memory for h5p_get_file_image helper
  • Loading branch information
mkitti authored Sep 4, 2023
1 parent 897c476 commit 72f39f3
Show file tree
Hide file tree
Showing 8 changed files with 169 additions and 6 deletions.
67 changes: 67 additions & 0 deletions src/api/helpers.jl
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,27 @@ function h5f_get_dset_no_attrs_hint(file_id)::Bool
return minimize[]
end

"""
h5f_get_file_image(file_id)
Return a `Vector{UInt8}` containing the file image. Does not include the user block.
"""
function h5f_get_file_image(file_id)
buffer_length = h5f_get_file_image(file_id, C_NULL, 0)
buffer = Vector{UInt8}(undef, buffer_length)
h5f_get_file_image(file_id, buffer, buffer_length)
return buffer
end

"""
h5f_get_file_image(file_id, buffer::Vector{UInt8})
Store the file image in the provided buffer.
"""
function h5f_get_file_image(file_id, buffer::Vector{UInt8})
h5f_get_file_image(fild_id, buffer, length(buffer))
end

###
### Attribute Interface
###
Expand Down Expand Up @@ -790,6 +811,52 @@ function h5p_get_virtual_view(dapl_id)
return view[]
end

"""
h5p_get_file_image(fapl_id)::Vector{UInt8}
Retrieve a file image of the appropriate size in a `Vector{UInt8}`.
"""
function h5p_get_file_image(fapl_id)::Vector{UInt8}
cb = h5p_get_file_image_callbacks(fapl_id)
if cb.image_free != C_NULL
# The user has configured their own memory deallocation routines.
# The user should use a lower level call to properly handle deallocation
error(
"File image callback image_free is not C_NULL. Use the three argument method of h5p_get_file_image when setting file image callbacks."
)
end
buf_ptr_ref = Ref{Ptr{Nothing}}()
buf_len_ref = Ref{Csize_t}(0)
h5p_get_file_image(fapl_id, buf_ptr_ref, buf_len_ref)
image = unsafe_wrap(Array{UInt8}, Ptr{UInt8}(buf_ptr_ref[]), buf_len_ref[]; own=false)
finalizer(image) do image
# Use h5_free_memory to ensure we are using the correct free
h5_free_memory(image)
end
return image
end

"""
h5p_set_file_image(fapl_id, image::Vector{UInt8})
Set the file image from a `Vector{UInt8}`.
"""
function h5p_set_file_image(fapl_id, image::Vector{UInt8})
h5p_set_file_image(fapl_id, image, length(image))
end

"""
h5p_get_file_image_callbacks(fapl_id)
Retrieve the file image callbacks for memory operations
"""
function h5p_get_file_image_callbacks(fapl_id)
cb = H5FD_file_image_callbacks_t(C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL, C_NULL)
r = Ref(cb)
h5p_get_file_image_callbacks(fapl_id, r)
return r[]
end

# Note: The following function(s) implement direct ccalls because the binding generator
# cannot (yet) do the string wrapping and memory freeing.

Expand Down
4 changes: 2 additions & 2 deletions src/drivers/drivers.jl
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ Core(; increment=8192, backing_store=true, write_tracking=false, page_size=52428

function get_driver(p::Properties, ::Type{Core})
r_increment = Ref{Csize_t}(0)
r_backing_store = Ref{Cuint}(0)
r_write_tracking = Ref{Cuint}(0)
r_backing_store = Ref{Bool}(0)
r_write_tracking = Ref{Bool}(0)
r_page_size = Ref{Csize_t}(0)
API.h5p_get_fapl_core(p, r_increment, r_backing_store)
API.h5p_get_core_write_tracking(p, r_write_tracking, r_page_size)
Expand Down
2 changes: 1 addition & 1 deletion src/drivers/ros3.jl
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ function Base.convert(::Type{API.H5FD_ros3_fapl_t}, driver::ROS3)
end

function get_driver(fapl::Properties, ::Type{ROS3})
r_fa = Ref{H5FD_ros3_fapl_t}()
r_fa = Ref{API.H5FD_ros3_fapl_t}()
API.h5p_get_fapl_ros3(fapl, r_fa)
return ROS3(r_fa[])
end
Expand Down
57 changes: 56 additions & 1 deletion src/file.jl
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ function h5open(
if cr && (tr || !isfile(filename))
flag = swmr ? API.H5F_ACC_TRUNC | API.H5F_ACC_SWMR_WRITE : API.H5F_ACC_TRUNC
fid = API.h5f_create(filename, flag, fcpl, fapl)
elseif fapl.driver isa Drivers.Core && fapl.driver.backing_store == 0
flag = wr ? API.H5F_ACC_RDWR : API.H5F_ACC_RDONLY
fid = API.h5f_open(filename, flag, fapl)
else
occursin(r"(s3a?|https?)://", filename) ||
ishdf5(filename) ||
Expand Down Expand Up @@ -77,7 +80,7 @@ function h5open(
end

"""
function h5open(f::Function, args...; pv...)
h5open(f::Function, args...; pv...)
Apply the function f to the result of `h5open(args...; kwargs...)` and close the resulting
`HDF5.File` upon completion.
Expand All @@ -103,6 +106,46 @@ function h5open(f::Function, args...; context=copy(CONTEXT), pv...)
end
end

"""
h5open(file_image::Vector{UInt8}, mode::AbstractString="r";
name=nothing,
fapl=FileAccessProperties(),
increment=8192,
backing_store=false,
write_tracking=false,
page_size=524288,
pv...
)
Open a file image contained in a `Vector{UInt8}`. See [`API.h5p_set_file_image`](@ref).
Unlike [`Drivers.Core`](@ref) the default here is not to use a backing store.
"""
function h5open(
file_image::Vector{UInt8},
mode::AbstractString="r";
name=nothing,
fapl=FileAccessProperties(),
increment=8192,
backing_store=false,
write_tracking=false,
page_size=524288,
pv...
)
fapl.driver = Drivers.Core(; increment, backing_store, write_tracking, page_size)
fapl.file_image = file_image
if isnothing(name)
if fapl.driver.backing_store != 0
# The temporary file will be used as a backing store
name = tempname()
else
# Provide it with a unique name based on the objectid
name = "<memory objectid>: " * repr(objectid(file_image))
end
end
h5open(name, mode; fapl, pv...)
end

function h5rewrite(f::Function, filename::AbstractString, args...)
tmppath, tmpio = mktemp(dirname(filename))
close(tmpio)
Expand Down Expand Up @@ -169,3 +212,15 @@ start_swmr_write(h5::File) = API.h5f_start_swmr_write(h5)
# Flush buffers
Base.flush(f::Union{Object,Attribute,Datatype,File}, scope=API.H5F_SCOPE_GLOBAL) =
API.h5f_flush(checkvalid(f), scope)

# File image conversion

function Vector{UInt8}(h5f::File)
flush(h5f)
API.h5f_get_file_image(h5f)
end
function File(file_image::Vector{UInt8}, name=nothing)
h5open(file_image; name)
end
Base.convert(::Type{Vector{UInt8}}, h5f::File) = Vector{UInt8}(h5f)
Base.convert(::Type{File}, file_image::Vector{UInt8}) = File(file_image)
3 changes: 3 additions & 0 deletions src/properties.jl
Original file line number Diff line number Diff line change
Expand Up @@ -745,6 +745,7 @@ class_propertynames(::Type{FileAccessProperties}) = (
:file_locking,
:libver_bounds,
:meta_block_size,
:file_image,
)

function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Symbol)
Expand All @@ -755,6 +756,7 @@ function class_getproperty(::Type{FileAccessProperties}, p::Properties, name::Sy
name === :file_locking ? API.h5p_get_file_locking(p) :
name === :libver_bounds ? get_libver_bounds(p) :
name === :meta_block_size ? API.h5p_get_meta_block_size(p) :
name === :file_image ? API.h5p_get_file_image(p) :
# deprecated
name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); drv = get_driver(p, MPIO); (drv.comm, drv.info)) :
class_getproperty(superclass(FileAccessProperties), p, name)
Expand All @@ -766,6 +768,7 @@ function class_setproperty!(::Type{FileAccessProperties}, p::Properties, name::S
name === :file_locking ? API.h5p_set_file_locking(p, val...) :
name === :libver_bounds ? set_libver_bounds!(p, val) :
name === :meta_block_size ? API.h5p_set_meta_block_size(p, val) :
name === :file_image ? API.h5p_set_file_image(p, val) :
# deprecated
name === :fapl_mpio ? (depwarn("The `fapl_mpio` property is deprecated, use `driver=HDF5.Drivers.MPIO(...)` instead.", :fapl_mpio); Drivers.set_driver!(p, Drivers.MPIO(val...))) :
class_setproperty!(superclass(FileAccessProperties), p, name, val)
Expand Down
15 changes: 14 additions & 1 deletion src/show.jl
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,20 @@ function Base.show(io::IO, prop::Properties)
# or always well-defined (e.g. chunk if layout != :chunked, dxpl_mpio if no MPI)
try
val = getproperty(prop, name)
print(io, "\n ", rpad(name, 15), " = ", repr(val), ",")
if name == :file_image
print(
io,
"\n ",
rpad(name, 15),
" = ",
repr(typeof(val)),
", length = ",
length(val),
","
)
else
print(io, "\n ", rpad(name, 15), " = ", repr(val), ",")
end
catch e
end
end
Expand Down
12 changes: 11 additions & 1 deletion test/drivers.jl
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,18 @@ using Test
end

fn = tempname()
h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f
file_image = h5open(fn, "w"; driver=Drivers.Core(; backing_store=false)) do f
ds = write_dataset(f, "core_dataset", A)
Vector{UInt8}(f)
end
@test !isfile(fn)

h5open(file_image) do f
@test f["core_dataset"][] == A
end

fn = tempname()
h5open(fn, "r"; driver=Drivers.Core(; backing_store=false), file_image) do f
@test f["core_dataset"][] == A
end
end
15 changes: 15 additions & 0 deletions test/properties.jl
Original file line number Diff line number Diff line change
Expand Up @@ -121,5 +121,20 @@ using Test
nothing
end

file_image = read(fn)

rm(fn; force=true)

fapl = HDF5.FileAccessProperties()
fapl.driver = HDF5.Drivers.Core(; backing_store=false)
fapl.file_image = copy(file_image)
@test fapl.file_image == file_image
file_image2 = h5open(fn, "r"; fapl) do f
@test haskey(f, "group")
convert(Vector{UInt8}, f)
end
# file_image2 does not include user block
@test file_image2 == @view(file_image[1025:end])

rm(fn; force=true)
end

0 comments on commit 72f39f3

Please sign in to comment.