Expose and add some docs

Signed-off-by: Krzysztof Lecki <klecki@nvidia.com>

dali/python/nvidia/dali/_autograph/__init__.py

@@ -41,6 +41,7 @@
 from nvidia.dali._autograph.impl.api import convert
 from nvidia.dali._autograph.impl.api import converted_call
 from nvidia.dali._autograph.impl.api import do_not_convert
+from nvidia.dali._autograph.impl.api import is_autograph_artifact as _is_autograph_artifact
 
 # from nvidia.dali._autograph.impl.api import StackTraceMapper
 from nvidia.dali._autograph.impl.api import to_code

dali/python/nvidia/dali/_autograph/impl/api.py

@@ -446,11 +446,11 @@ def _fall_back_unconverted(f, args, kwargs, options, exc):
     """Falls back to calling the function unconverted, in case of error."""
     # TODO(mdan): Consider adding an internal metric.
     warning_template = (
-        "AutoGraph could not transform %s and will run it as-is.\n" "%s" "Cause: %s\n"
+        "AutoGraph could not transform %s and will run it as-is.\n"
+        "%s"
+        "Cause: %s\n"
+        "To silence this warning, decorate the function with @nvidia.dali.pipeline.do_not_convert"
     )
-    # TODO(klecki): Expose the do_not_convert in DALI
-    #   'To silence this warning, decorate the function with'
-    #   ' @tf.autograph.experimental.do_not_convert')
     if isinstance(exc, errors.InaccessibleSourceCodeError):
         if ag_ctx.INSPECT_SOURCE_SUPPORTED:
             logging.warning(warning_template, f, "", exc)
@@ -598,7 +598,6 @@ def _log_callargs(f, args, kwargs):
 #
 
 
-@export_symbol("autograph.experimental.do_not_convert")
 def do_not_convert(func=None):
     """Decorator that suppresses the conversion of a function.
 

dali/python/nvidia/dali/external_source.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
 from nvidia.dali import backend as _b
 from nvidia.dali import tensors as _tensors
 from nvidia.dali import types as _types
+from nvidia.dali import _conditionals
 from nvidia.dali._multiproc.messages import TaskArgs as _TaskArgs, SampleRange as _SampleRange
 import nvidia.dali.types
 from nvidia.dali._utils.external_source_impl import (
@@ -783,6 +784,23 @@ def __call__(
                     "(specify `batch=True` in the external source definition and make sure "
                     "your source returns batches)".format(what)
                 )
+            if _conditionals._is_autograph_artifact(source_desc.source):
+                raise ValueError(
+                    "The `source` parameter that was passed to external source was created "
+                    "in a scope that was converted with AutoGraph. To allow the `source` to be "
+                    "correctly used with parallel external source, it must remain unconverted. "
+                    "To prevent conversion, two steps may need to be taken:\n"
+                    "1. The `source` or a factory creating the parameter passed to `source` must "
+                    "be defined at global scope (or at least outside of `pipeline_def` scope).\n"
+                    "2. If the `source` is created by some factory function that is invoked within "
+                    "pipeline definition, that function must be decorated with "
+                    "`@nvidia.dali.pipeline.do_not_convert`, otherwise it will be recursively "
+                    "converted when the pipeline definition is traced.\n"
+                    "You can read more details and see examples in the "
+                    "`@nvidia.dali.pipeline.do_not_convert` decorator documentation. The AutoGraph "
+                    "conversion is part of conditional execution in DALI."
+                )
+
         else:
             for kwarg_value, kwarg_name in (
                 (prefetch_queue_depth, "prefetch_queue_depth"),

dali/python/nvidia/dali/pipeline.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2017-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright (c) 2017-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -1884,6 +1884,49 @@ def create_pipeline(*args, **kwargs):
     return actual_decorator(fn) if fn else actual_decorator
 
 
+def do_not_convert(func=None):
+    """Decorator that suppresses the conversion of a function by AutoGraph.
+
+    In conditional mode DALI uses fork of [TensorFlow's AutoGraph](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/autograph/g3doc/reference/index.md)
+    to rewrite `if` statements, so they can be detected and used in processing DALI graph.
+
+    When used with [Parallel External Source](...), this may interfere with the serialization of
+    the provided callback. To prevent this, functions that are used to create the `source`
+    parameter, should be marked with @do_not_convert.
+
+    The AutoGraph conversion is applied to any top-level function or method called within the
+    pipeline definition (as well as the pipeline definition itself).
+    When a function is converted, all functions defined within its syntactical scope are also
+    converted.
+
+    To prevent a function from being converted, its top-level encompassing function must be marked
+    with this decorator.
+
+    Note that typically only functions that do not process DataNodes (so don't use DALI operators
+    directly) should be marked with this decorator.
+
+    Parameters
+    ----------
+    func : _type_, optional
+        _description_, by default None
+
+    Returns
+    -------
+    _type_
+        _description_
+    """
+
+    if func is None:
+        return do_not_convert
+
+    if getattr(func, "_is_pipeline_def", False):
+        # TODO(klecki): The other way round as well?
+        raise ValueError("Pipeline definition cannot be marked with @do_not_convert.")
+
+    # TODO(klecki): Verify if we don't have any DataNodes in the returned structure?
+    return _conditionals.do_not_convert(func)
+
+
 def _collect_ops(output_nodes):
     """
     Traverses the pipeline graph starting from the outputs to collect all reachable operators.