From 7cc6fb4c4b0fddf9d4903cc8651e29c69acfe445 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 09:01:43 -0800
Subject: [PATCH 01/11] ]add BangBang, Setfield

---
 Project.toml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/Project.toml b/Project.toml
index ff8cfb13..4eeafea5 100644
--- a/Project.toml
+++ b/Project.toml
@@ -4,7 +4,9 @@ version = "0.4.4"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+BangBang = "198e06fe-97b7-11e9-32a5-e1d131e6ad66"
 DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]

From a2e16bc71dbe4f836b2f8efcf09730012eb83174 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 09:02:07 -0800
Subject: [PATCH 02/11] append!! => BangBang.append!!

---
 src/collect.jl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/collect.jl b/src/collect.jl
index 8c644007..399352f4 100644
--- a/src/collect.jl
+++ b/src/collect.jl
@@ -127,6 +127,8 @@ function _widenarray(dest::AbstractArray, i, ::Type{T}) where T
     new
 end
 
+import BangBang
+
 """
 `append!!(dest, itr) -> dest′`
 
@@ -141,7 +143,7 @@ holds.  Note that `dest′` may or may not be the same object as `dest`.
 The state of `dest` is unpredictable after `append!!`
 is called (e.g., it may contain just half of the elements from `itr`).
 """
-append!!(dest::AbstractVector, itr) =
+BangBang.append!!(dest::AbstractVector, itr) =
     _append!!(dest, itr, Base.IteratorSize(itr))
 
 function _append!!(dest::AbstractVector, itr, ::Union{Base.HasShape, Base.HasLength})
@@ -162,7 +164,7 @@ _append!!(dest::AbstractVector, itr, ::Base.SizeUnknown) =
 # Optimized version when element collection is an `AbstractVector`
 # This only works for julia 1.3 or greater, which has `append!` for `AbstractVector`
 @static if VERSION ≥ v"1.3.0"
-    function append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
+    function BangBang.append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
         new = iscompatible(T, V) ? dest : widen_from_type(dest, length(dest) + 1, T)
         return append!(new, v)
     end

From e87197c7d8176d32b5f7f3d59708e60b410c96b3 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 09:02:26 -0800
Subject: [PATCH 03/11] lenses.jl

---
 src/StructArrays.jl |  1 +
 src/lenses.jl       | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+)
 create mode 100644 src/lenses.jl

diff --git a/src/StructArrays.jl b/src/StructArrays.jl
index 8a219a18..65ebe937 100644
--- a/src/StructArrays.jl
+++ b/src/StructArrays.jl
@@ -6,6 +6,7 @@ export StructArray, StructVector, LazyRow, LazyRows
 export collect_structarray, fieldarrays
 export replace_storage
 
+include("lenses.jl")
 include("interface.jl")
 include("structarray.jl")
 include("utils.jl")
diff --git a/src/lenses.jl b/src/lenses.jl
new file mode 100644
index 00000000..bc13c1ee
--- /dev/null
+++ b/src/lenses.jl
@@ -0,0 +1,25 @@
+using BangBang
+using Setfield
+
+lenses(nt::NamedTuple) = _lenses(nt, ())
+
+function _lenses(nt::NamedTuple, acc)
+    result = ()
+    for k in keys(nt)
+        nt_k = getproperty(nt, k)
+        # Add "breadcrumb" steps to the accumulator as we descend into the tree
+        acc_k = push!!(acc, Setfield.PropertyLens{k}())
+        ℓ = _lenses(nt_k, acc_k)
+        result = append!!(result, ℓ)
+    end
+    return result
+end
+
+# When we reach a leaf node (an array), compose the steps to get a lens
+function _lenses(a::AbstractArray, acc)
+    return (Setfield.compose(acc...),)
+end
+
+nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
+
+lenses(nt)

From 30e2608fed8355a0fa68879e2e0c9f31803a0c3f Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 09:03:49 -0800
Subject: [PATCH 04/11] First crack at getting nesting to work

---
 src/structarray.jl | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/structarray.jl b/src/structarray.jl
index 34fe3bd1..0b75c523 100644
--- a/src/structarray.jl
+++ b/src/structarray.jl
@@ -1,3 +1,5 @@
+using BangBang
+
 """
 A type that stores an array of structures as a structure of arrays.
 # Fields:
@@ -7,11 +9,13 @@ struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
     fieldarrays::C
 
     function StructArray{T, N, C}(c) where {T, N, C<:Tup}
-        if length(c) > 0
-            ax = axes(c[1])
+        arrays = [get(nt, ℓ) for ℓ in lenses(nt)]
+        if length(arrays) > 0
+            ax = axes(arrays[1])
             length(ax) == N || error("wrong number of dimensions")
-            for i = 2:length(c)
-                axes(c[i]) == ax || error("all field arrays must have same shape")
+
+            for i = 2:length(arrays)
+                axes(arrays[i]) == ax || error("all field arrays must have same shape")
             end
         end
         new{T, N, C, index_type(C)}(c)

From 788c569409c3c497302a3f637c32a969aa8fed58 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 12:27:52 -0800
Subject: [PATCH 05/11] typelevel.jl

---
 src/lenses.jl    |  6 ++++++
 src/typelevel.jl | 31 +++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 src/typelevel.jl

diff --git a/src/lenses.jl b/src/lenses.jl
index bc13c1ee..373fd66a 100644
--- a/src/lenses.jl
+++ b/src/lenses.jl
@@ -2,6 +2,7 @@ using BangBang
 using Setfield
 
 lenses(nt::NamedTuple) = _lenses(nt, ())
+lenses(NT::Type{NamedTuple{K,V}}) where {K,V} = lenses(fromtype(NT))
 
 function _lenses(nt::NamedTuple, acc)
     result = ()
@@ -20,6 +21,11 @@ function _lenses(a::AbstractArray, acc)
     return (Setfield.compose(acc...),)
 end
 
+function _lenses(::Type{A}, acc) where {A <: AbstractArray}
+    return (Setfield.compose(acc...),)
+end
+
 nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
 
 lenses(nt)
+lenses(typeof(nt))
diff --git a/src/typelevel.jl b/src/typelevel.jl
new file mode 100644
index 00000000..7c63e3fd
--- /dev/null
+++ b/src/typelevel.jl
@@ -0,0 +1,31 @@
+using NamedTupleTools: namedtuple
+
+ntkeys(::Type{NamedTuple{K,V}}) where {K, V} = K
+ntvaltype(::Type{NamedTuple{K,V}}) where {K, V} = V
+
+"""
+    fromtype(::Type)
+    
+`fromtype` turns a type into a value that's easier to work with.
+
+Example:
+
+    julia> nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
+
+    julia> NT = typeof(nt)
+    NamedTuple{(:a, :f),Tuple{NamedTuple{(:b, :c),Tuple{Array{Int64,1},NamedTuple{(:d, :e),Tuple{Array{Int64,1},Array{Int64,1}}}}},Array{Int64,1}}}
+
+    julia> fromtype(NT)
+    (a = (b = Array{Int64,1}, c = (d = Array{Int64,1}, e = Array{Int64,1})), f = Array{Int64,1})
+"""
+function fromtype end
+
+function fromtype(NT::Type{NamedTuple{names, T}}) where {names, T}
+    return namedtuple(ntkeys(NT), fromtype(ntvaltype(NT)))
+end
+
+function fromtype(TT::Type{T}) where {T <: Tuple} 
+    return fromtype.(Tuple(TT.types))
+end
+
+fromtype(T) = T

From 15b4184cb7823649bff882620591ee8266fe0f94 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 4 Dec 2020 12:28:14 -0800
Subject: [PATCH 06/11] more progress

---
 Project.toml        | 1 +
 src/StructArrays.jl | 1 +
 src/structarray.jl  | 7 +++++--
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/Project.toml b/Project.toml
index 4eeafea5..c0876548 100644
--- a/Project.toml
+++ b/Project.toml
@@ -6,6 +6,7 @@ version = "0.4.4"
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
 BangBang = "198e06fe-97b7-11e9-32a5-e1d131e6ad66"
 DataAPI = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+NamedTupleTools = "d9ec5142-1e00-5aa0-9d6a-321866360f50"
 Setfield = "efcf1570-3423-57d1-acb7-fd33fddbac46"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
diff --git a/src/StructArrays.jl b/src/StructArrays.jl
index 65ebe937..5cd891b7 100644
--- a/src/StructArrays.jl
+++ b/src/StructArrays.jl
@@ -6,6 +6,7 @@ export StructArray, StructVector, LazyRow, LazyRows
 export collect_structarray, fieldarrays
 export replace_storage
 
+include("typelevel.jl")
 include("lenses.jl")
 include("interface.jl")
 include("structarray.jl")
diff --git a/src/structarray.jl b/src/structarray.jl
index 0b75c523..33b4d3ac 100644
--- a/src/structarray.jl
+++ b/src/structarray.jl
@@ -6,10 +6,12 @@ A type that stores an array of structures as a structure of arrays.
 - `fieldarrays`: a (named) tuple of arrays. Also `fieldarrays(x)`
 """
 struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
+    lenses
     fieldarrays::C
 
     function StructArray{T, N, C}(c) where {T, N, C<:Tup}
-        arrays = [get(nt, ℓ) for ℓ in lenses(nt)]
+        ℓ = lenses(c)
+        arrays = [get(c, ℓⱼ) for ℓⱼ in ℓ]
         if length(arrays) > 0
             ax = axes(arrays[1])
             length(ax) == N || error("wrong number of dimensions")
@@ -18,7 +20,7 @@ struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
                 axes(arrays[i]) == ax || error("all field arrays must have same shape")
             end
         end
-        new{T, N, C, index_type(C)}(c)
+        new{T, N, C, index_type(C)}(ℓ, c)
     end
 end
 
@@ -37,6 +39,7 @@ array_types(::Type{TT}) where {TT<:Tuple} = TT
 
 function StructArray{T}(c::C) where {T, C<:Tup}
     cols = strip_params(staticschema(T))(c)
+    array1 = get(nt, lenses(c)[1])
     N = isempty(cols) ? 1 : ndims(cols[1])
     StructArray{T, N, typeof(cols)}(cols)
 end

From a1ce58729bc204a8f6df42c62699ef8abb90cfa9 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Sat, 5 Dec 2020 15:45:45 -0800
Subject: [PATCH 07/11] bugfix

---
 src/structarray.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/structarray.jl b/src/structarray.jl
index 33b4d3ac..81273093 100644
--- a/src/structarray.jl
+++ b/src/structarray.jl
@@ -39,7 +39,7 @@ array_types(::Type{TT}) where {TT<:Tuple} = TT
 
 function StructArray{T}(c::C) where {T, C<:Tup}
     cols = strip_params(staticschema(T))(c)
-    array1 = get(nt, lenses(c)[1])
+    array1 = get(c, lenses(c)[1])
     N = isempty(cols) ? 1 : ndims(cols[1])
     StructArray{T, N, typeof(cols)}(cols)
 end

From f4babaf9ecd7459761b718121cd14dad8c823269 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Sat, 5 Dec 2020 15:46:01 -0800
Subject: [PATCH 08/11] lenses for tuples

---
 src/lenses.jl | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/lenses.jl b/src/lenses.jl
index 373fd66a..9f4b4ed7 100644
--- a/src/lenses.jl
+++ b/src/lenses.jl
@@ -1,9 +1,21 @@
 using BangBang
 using Setfield
 
+lenses(t::Tuple) = _lenses(t, ())
+
 lenses(nt::NamedTuple) = _lenses(nt, ())
 lenses(NT::Type{NamedTuple{K,V}}) where {K,V} = lenses(fromtype(NT))
 
+function _lenses(t::Tuple, acc)
+    result = ()
+    for (k,v) in enumerate(t)
+        acc_k = push!!(acc, Setfield.IndexLens((k,)))
+        ℓ = _lenses(v, acc_k)
+        result = append!!(result, ℓ)
+    end
+    return result
+end
+
 function _lenses(nt::NamedTuple, acc)
     result = ()
     for k in keys(nt)

From ef83223c738e07bc397cb01720efc041f107cda6 Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Sat, 5 Dec 2020 15:59:16 -0800
Subject: [PATCH 09/11] comment out type piracy

---
 src/collect.jl | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/collect.jl b/src/collect.jl
index 399352f4..5d12352f 100644
--- a/src/collect.jl
+++ b/src/collect.jl
@@ -163,9 +163,9 @@ _append!!(dest::AbstractVector, itr, ::Base.SizeUnknown) =
 
 # Optimized version when element collection is an `AbstractVector`
 # This only works for julia 1.3 or greater, which has `append!` for `AbstractVector`
-@static if VERSION ≥ v"1.3.0"
-    function BangBang.append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
-        new = iscompatible(T, V) ? dest : widen_from_type(dest, length(dest) + 1, T)
-        return append!(new, v)
-    end
-end
+# @static if VERSION ≥ v"1.3.0"
+#     function BangBang.append!!(dest::V, v::AbstractVector{T}) where {V<:AbstractVector, T}
+#         new = iscompatible(T, V) ? dest : widen_from_type(dest, length(dest) + 1, T)
+#         return append!(new, v)
+#     end
+# end

From 04549d5f97e8a96afbd2901da4e2b0b77f79edac Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Sat, 5 Dec 2020 16:09:49 -0800
Subject: [PATCH 10/11] lenses docstring

---
 src/lenses.jl | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/lenses.jl b/src/lenses.jl
index 9f4b4ed7..c43a1126 100644
--- a/src/lenses.jl
+++ b/src/lenses.jl
@@ -1,6 +1,24 @@
 using BangBang
 using Setfield
 
+"""
+    lenses(t::Tuple)
+    lenses(nt::NamedTuple)
+    lenses(NT::Type{NamedTuple{K,V}})
+
+Build a Tuple of lenses for a given value or type
+
+Example:
+    julia> nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
+
+    julia> lenses(nt)
+    ((@lens _.a.b), (@lens _.a.c.d), (@lens _.a.c.e), (@lens _.f))
+
+    julia> lenses(typeof(nt))
+    ((@lens _.a.b), (@lens _.a.c.d), (@lens _.a.c.e), (@lens _.f))
+"""
+function lenses end
+
 lenses(t::Tuple) = _lenses(t, ())
 
 lenses(nt::NamedTuple) = _lenses(nt, ())
@@ -36,8 +54,3 @@ end
 function _lenses(::Type{A}, acc) where {A <: AbstractArray}
     return (Setfield.compose(acc...),)
 end
-
-nt = (a=(b=[1,2],c=(d=[3,4],e=[5,6])),f=[7,8]);
-
-lenses(nt)
-lenses(typeof(nt))

From 77bc8f9aedd224fdeada40628db19da68a095acc Mon Sep 17 00:00:00 2001
From: Chad Scherrer <chad.scherrer@gmail.com>
Date: Fri, 11 Dec 2020 07:38:53 -0800
Subject: [PATCH 11/11] catching up

---
 src/lenses.jl      | 2 +-
 src/structarray.jl | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/lenses.jl b/src/lenses.jl
index c43a1126..e8fafa02 100644
--- a/src/lenses.jl
+++ b/src/lenses.jl
@@ -51,6 +51,6 @@ function _lenses(a::AbstractArray, acc)
     return (Setfield.compose(acc...),)
 end
 
-function _lenses(::Type{A}, acc) where {A <: AbstractArray}
+function _lenses(::Type{T}, acc) where {T}
     return (Setfield.compose(acc...),)
 end
diff --git a/src/structarray.jl b/src/structarray.jl
index 81273093..2acf74de 100644
--- a/src/structarray.jl
+++ b/src/structarray.jl
@@ -5,12 +5,13 @@ A type that stores an array of structures as a structure of arrays.
 # Fields:
 - `fieldarrays`: a (named) tuple of arrays. Also `fieldarrays(x)`
 """
-struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
-    lenses
+struct StructArray{T, N, C<:Tup, I, L} <: AbstractArray{T, N}
     fieldarrays::C
+    lenses::L
 
     function StructArray{T, N, C}(c) where {T, N, C<:Tup}
         ℓ = lenses(c)
+        L = typeof(ℓ)
         arrays = [get(c, ℓⱼ) for ℓⱼ in ℓ]
         if length(arrays) > 0
             ax = axes(arrays[1])
@@ -20,7 +21,7 @@ struct StructArray{T, N, C<:Tup, I} <: AbstractArray{T, N}
                 axes(arrays[i]) == ax || error("all field arrays must have same shape")
             end
         end
-        new{T, N, C, index_type(C)}(ℓ, c)
+        new{T, N, C, index_type(C), L}(c, ℓ)
     end
 end