Fix Python tests for FunctionGeneratedTable (#4511)

engine/table/src/main/java/io/deephaven/engine/table/impl/util/FunctionGeneratedTableFactory.java

@@ -53,8 +53,17 @@ public static Table create(Supplier<Table> tableGenerator, int refreshIntervalMs
     }
 
     /**
-     * Create a table that refreshes based on the value of your function, automatically called when any of the
-     * sourceTables tick.
+     * Create a table that refreshes based on the value of your function, automatically called in a
+     * dependency-respecting way when at least one of the {@code sourceTables} tick.
+     * <p>
+     * <em>Note</em> that the {@code tableGenerator} may access data in the {@code sourceTables} but should not perform
+     * further table operations on them without careful handling. Table operations may be memoized, and it is possible
+     * that a table operation will return a table created by a previous invocation of the same operation. Since that
+     * result will not have been included in the {@code sourceTables}, it's not automatically treated as a dependency
+     * for purposes of determining when it's safe to invoke {@code tableGenerator}, allowing races to exist between
+     * accessing the operation result and that result's own update processing. It's best to include all dependencies
+     * directly in {@code sourceTables}, or only compute on-demand inputs under a
+     * {@link io.deephaven.engine.liveness.LivenessScope}.
      *
      * @param tableGenerator a function returning a table to copy into the output table
      * @param sourceTables The query engine does not know the details of your function inputs. If you are dependent on a

py/server/deephaven/table_factory.py

@@ -358,8 +358,13 @@ def function_generated_table(table_generator: Callable[[], Table],
 
     The table definition must not change between invocations of the 'table_generator' function, or an exception will be raised.
 
-    Note: any tables used by the 'table_generator' function *MUST* be specified in the 'source_tables'. This
-    ensures that the 'table_generator' is rerun only *after* any changes to the 'source_tables' have been processed.
+   Note that the 'table_generator' may access data in the sourceTables but should not perform further table operations
+   on them without careful handling. Table operations may be memoized, and it is possible that a table operation will
+   return a table created by a previous invocation of the same operation. Since that result will not have been included
+   in the 'source_table', it's not automatically treated as a dependency for purposes of determining when it's safe to
+   invoke 'table_generator', allowing races to exist between accessing the operation result and that result's own update
+   processing. It's best to include all dependencies directly in 'source_table', or only compute on-demand inputs under
+   a LivenessScope.
 
     Args:
         table_generator (Callable[[], Table]): The table generator function. This function must return a Table.

py/server/tests/test_function_generated_table.py

@@ -1,10 +1,10 @@
-from datetime import datetime
 from typing import Any
 
 import deephaven.dtypes as dht
 from deephaven import empty_table, input_table, new_table, update_graph, function_generated_table
 from deephaven.column import string_col, int_col
 from deephaven.execution_context import get_exec_ctx
+from deephaven.liveness_scope import liveness_scope
 from deephaven.table import Table
 from tests.testbase import BaseTestCase
 
@@ -44,20 +44,18 @@ def test_generated_table_1trigger(self):
             "MyStr": dht.string
         }
         append_only_input_table = input_table(col_defs=col_defs)
+        input_table_lastby = append_only_input_table.last_by()
 
         def table_generator_function():
-            print("Running table_generator_function() at time: " + str(datetime.now()))
-            return append_only_input_table.last_by().update('ResultStr = MyStr')
+            with liveness_scope():
+                return input_table_lastby.update('ResultStr = MyStr')
 
-        result_table = function_generated_table(table_generator_function, source_tables=append_only_input_table)
+        result_table = function_generated_table(table_generator_function, source_tables=input_table_lastby)
 
         self.assertEqual(result_table.size, 0)
 
-        print("Adding row at time: " + str(datetime.now()))
         append_only_input_table.add(new_table([string_col(name='MyStr', data=['test string'])]))
-        print("add() returned at time: " + str(datetime.now()))
         self.wait_ticking_table_update(result_table, row_count=1, timeout=30)
-        print("result_table has 1 row at time: " + str(datetime.now()))
 
         first_row_key = get_row_key(0, result_table)
         result_str = result_table.j_table.getColumnSource("ResultStr").get(first_row_key)
@@ -71,16 +69,17 @@ def test_generated_table_2triggers(self):
         append_only_input_table2 = input_table(col_defs={"MyInt": dht.int32})
 
         def table_generator_function():
-            t1 = append_only_input_table1.last_by()
-            t2 = append_only_input_table2.last_by()
+            with liveness_scope():
+                t1 = append_only_input_table1.last_by()
+                t2 = append_only_input_table2.last_by()
 
-            my_str = t1.j_table.getColumnSource('MyStr').get(get_row_key(0, t1))
-            my_int = t2.j_table.getColumnSource('MyInt').getInt(get_row_key(0, t2))
+                my_str = None if t1.size == 0 else t1.j_table.getColumnSource('MyStr').get(get_row_key(0, t1))
+                my_int = None if t2.size == 0 else t2.j_table.getColumnSource('MyInt').getInt(get_row_key(0, t2))
 
-            return new_table([
-                string_col('ResultStr', [my_str]),
-                int_col('ResultInt', [my_int]),
-            ])
+                return new_table([
+                    string_col('ResultStr', [my_str]),
+                    int_col('ResultInt', [my_int]),
+                ])
 
         result_table = function_generated_table(table_generator_function,
                                                 source_tables=[append_only_input_table1, append_only_input_table2])