WIP: Nested named tuples

Project.toml

@@ -4,7 +4,10 @@ version = "0.4.4"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+BangBang = "198e06fe-97b7-11e9-32a5-e1d131e6ad66"
 DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+NamedTupleTools = "d9ec5142-1e00-5aa0-9d6a-321866360f50"
+Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]

src/StructArrays.jl

@@ -6,6 +6,8 @@ export StructArray, StructVector, LazyRow, LazyRows
 export collect_structarray, fieldarrays
 export replace_storage
 
+include("typelevel.jl")
+include("lenses.jl")
 include("interface.jl")
 include("structarray.jl")
 include("utils.jl")

src/collect.jl

@@ -127,6 +127,8 @@ function _widenarray(dest::AbstractArray, i, ::Type{T}) where T
  new
 end
 
+import BangBang
+
 """
 `append!!(dest, itr) -> dest′`
 
@@ -141,7 +143,7 @@ holds. Note that `dest′` may or may not be the same object as `dest`.
 The state of `dest` is unpredictable after `append!!`
 is called (e.g., it may contain just half of the elements from `itr`).
 """
-append!!(dest::AbstractVector, itr) =
+BangBang.append!!(dest::AbstractVector, itr) =
  _append!!(dest, itr, Base.IteratorSize(itr))
 
 function _append!!(dest::AbstractVector, itr, ::Union{Base.HasShape, Base.HasLength})
@@ -161,9 +163,9 @@ _append!!(dest::AbstractVector, itr, ::Base.SizeUnknown) =
 
 # Optimized version when element collection is an `AbstractVector`
 # This only works for julia 1.3 or greater, which has `append!` for `AbstractVector`
-@static if VERSION ≥ v"1.3.0"
- function append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
- new = iscompatible(T, V) ? dest : widen_from_type(dest, length(dest) + 1, T)
- return append!(new, v)
- end
-end
+# @static if VERSION ≥ v"1.3.0"
+#  function BangBang.append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
+#  new = iscompatible(T, V) ? dest : widen_from_type(dest, length(dest) + 1, T)
+#  return append!(new, v)
+#  end
+# end

src/lenses.jl

@@ -0,0 +1,56 @@
+using BangBang
+using Setfield
+
+"""
+ lenses(t::Tuple)
+ lenses(nt::NamedTuple)
+ lenses(NT::Type{NamedTuple{K,V}})
+
+Build a Tuple of lenses for a given value or type
+
+Example:
+ julia> nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
+
+ julia> lenses(nt)
+ ((@lens _.a.b), (@lens _.a.c.d), (@lens _.a.c.e), (@lens _.f))
+
+ julia> lenses(typeof(nt))
+ ((@lens _.a.b), (@lens _.a.c.d), (@lens _.a.c.e), (@lens _.f))
+"""
+function lenses end
+
+lenses(t::Tuple) = _lenses(t, ())
+
+lenses(nt::NamedTuple) = _lenses(nt, ())
+lenses(NT::Type{NamedTuple{K,V}}) where {K,V} = lenses(fromtype(NT))
+
+function _lenses(t::Tuple, acc)
+ result = ()
+ for (k,v) in enumerate(t)
+ acc_k = push!!(acc, Setfield.IndexLens((k,)))
+ ℓ = _lenses(v, acc_k)
+ result = append!!(result, ℓ)
+ end
+ return result
+end
+
+function _lenses(nt::NamedTuple, acc)
+ result = ()
+ for k in keys(nt)
+ nt_k = getproperty(nt, k)
+ # Add "breadcrumb" steps to the accumulator as we descend into the tree
+ acc_k = push!!(acc, Setfield.PropertyLens{k}())
+ ℓ = _lenses(nt_k, acc_k)
+ result = append!!(result, ℓ)
+ end
+ return result
+end
+
+# When we reach a leaf node (an array), compose the steps to get a lens
+function _lenses(a::AbstractArray, acc)
+ return (Setfield.compose(acc...),)
+end
+
+function _lenses(::Type{T}, acc) where {T}
+ return (Setfield.compose(acc...),)
+end

src/structarray.jl

@@ -1,20 +1,27 @@
+using BangBang
+
 """
 A type that stores an array of structures as a structure of arrays.
 # Fields:
 - `fieldarrays`: a (named) tuple of arrays. Also `fieldarrays(x)`
 """
-struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
+struct StructArray{T, N, C<:Tup, I, L} <: AbstractArray{T, N}
  fieldarrays::C
+ lenses::L
 
  function StructArray{T, N, C}(c) where {T, N, C<:Tup}
- if length(c) > 0
- ax = axes(c[1])
+ ℓ = lenses(c)
+ L = typeof(ℓ)
+ arrays = [get(c, ℓⱼ) for ℓⱼ in ℓ]
+ if length(arrays) > 0
+ ax = axes(arrays[1])
  length(ax) == N || error("wrong number of dimensions")
- for i = 2:length(c)
- axes(c[i]) == ax || error("all field arrays must have same shape")
+
+ for i = 2:length(arrays)
+ axes(arrays[i]) == ax || error("all field arrays must have same shape")
  end
  end
- new{T, N, C, index_type(C)}(c)
+ new{T, N, C, index_type(C), L}(c, ℓ)
  end
 end
 
@@ -33,6 +40,7 @@ array_types(::Type{TT}) where {TT<:Tuple} = TT
 
 function StructArray{T}(c::C) where {T, C<:Tup}
  cols = strip_params(staticschema(T))(c)
+ array1 = get(c, lenses(c)[1])
  N = isempty(cols) ? 1 : ndims(cols[1])
  StructArray{T, N, typeof(cols)}(cols)
 end

src/typelevel.jl

@@ -0,0 +1,31 @@
+using NamedTupleTools: namedtuple
+
+ntkeys(::Type{NamedTuple{K,V}}) where {K, V} = K
+ntvaltype(::Type{NamedTuple{K,V}}) where {K, V} = V
+
+"""
+ fromtype(::Type)
+
+`fromtype` turns a type into a value that's easier to work with.
+
+Example:
+
+ julia> nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
+
+ julia> NT = typeof(nt)
+ NamedTuple{(:a, :f),Tuple{NamedTuple{(:b, :c),Tuple{Array{Int64,1},NamedTuple{(:d, :e),Tuple{Array{Int64,1},Array{Int64,1}}}}},Array{Int64,1}}}
+
+ julia> fromtype(NT)
+ (a = (b = Array{Int64,1}, c = (d = Array{Int64,1}, e = Array{Int64,1})), f = Array{Int64,1})
+"""
+function fromtype end
+
+function fromtype(NT::Type{NamedTuple{names, T}}) where {names, T}
+ return namedtuple(ntkeys(NT), fromtype(ntvaltype(NT)))
+end
+
+function fromtype(TT::Type{T}) where {T <: Tuple} 
+ return fromtype.(Tuple(TT.types))
+end
+
+fromtype(T) = T