Skip to content

Commit

Permalink
Batch load arrow record batches
Browse files Browse the repository at this point in the history
  • Loading branch information
jmao-denver committed Dec 14, 2023
1 parent c649758 commit d8a2df2
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,13 @@ public synchronized void setSchema(final ByteBuffer ipcMessage) {
parseSchema((Schema) mi.header.header(new Schema()));
}

@ScriptApi
public synchronized void addRecordBatches(final ByteBuffer... ipcMessages) {
for (final ByteBuffer ipcMessage : ipcMessages) {
addRecordBatch(ipcMessage);
}
}

@ScriptApi
public synchronized void addRecordBatch(final ByteBuffer ipcMessage) {
if (completed) {
Expand Down
4 changes: 1 addition & 3 deletions py/server/deephaven/arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,7 @@ def to_table(pa_table: pa.Table, cols: List[str] = None) -> Table:
j_barrage_table_builder.setSchema(pa_buffer)

record_batches = pa_table.to_batches()
for rb in record_batches:
pa_buffer = rb.serialize()
j_barrage_table_builder.addRecordBatch(pa_buffer)
j_barrage_table_builder.addRecordBatches([jpy.byte_buffer(rb.serialize()) for rb in record_batches])
j_barrage_table_builder.onCompleted()

return Table(j_table=j_barrage_table_builder.getResultTable())
Expand Down

0 comments on commit d8a2df2

Please sign in to comment.