[components] Rename build_defs_from_toplevel_components_folder to bui…

…ld_component_defs and make it less magical (#26618)

## Summary & Motivation

Does a couple things

* Renames the top-level entry point from `build_defs_from_toplevel_components_folder` to `build_component_defs`
* Changes the scaffolding to make it a top-level export of `dagster_components`
* Removes upwards traversal in the folder hiearchy. Instead requires targeting of the exact directory. This avoids magic and makes errors more explicit

## How I Tested These Changes

Generated empty code location from scaffolding and loaded it.

## Changelog

BK

examples/experimental/components/examples/hello_world_deployment/code_locations/hello_world/hello_world/definitions.py

@@ -1,13 +1,13 @@
 from pathlib import Path
 
 import dagster as dg
-from dagster_components import ComponentRegistry, build_defs_from_toplevel_components_folder
+from dagster_components import ComponentRegistry, build_component_defs
 from dagster_components.lib.pipes_subprocess_script_collection import (
     PipesSubprocessScriptCollection,
 )
 
-defs = build_defs_from_toplevel_components_folder(
-    path=Path(__file__).parent,
+defs = build_component_defs(
+    code_location_root=Path(__file__).parent.parent,
     registry=ComponentRegistry(
         {"pipes_subprocess_script_collection": PipesSubprocessScriptCollection}
     ),

python_modules/libraries/dagster-components/dagster_components/__init__.py

@@ -6,6 +6,6 @@
     component as component,
 )
 from dagster_components.core.component_defs_builder import (
-    build_defs_from_toplevel_components_folder as build_defs_from_toplevel_components_folder,
+    build_component_defs as build_component_defs,
 )
 from dagster_components.version import __version__ as __version__

python_modules/libraries/dagster-components/dagster_components/cli/generate.py

@@ -8,6 +8,7 @@
 from dagster_components import ComponentRegistry
 from dagster_components.core.deployment import (
     CodeLocationProjectContext,
+    find_enclosing_code_location_root_path,
     is_inside_code_location_project,
 )
 from dagster_components.generate import generate_component_instance
@@ -39,8 +40,8 @@ def generate_component_command(
         )
         sys.exit(1)
 
-    context = CodeLocationProjectContext.from_path(
-        Path.cwd(),
+    context = CodeLocationProjectContext.from_code_location_path(
+        find_enclosing_code_location_root_path(Path.cwd()),
         ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
     )
     if not context.has_component_type(component_type):

python_modules/libraries/dagster-components/dagster_components/cli/list.py

@@ -8,6 +8,7 @@
 from dagster_components.core.component import ComponentMetadata, ComponentRegistry
 from dagster_components.core.deployment import (
     CodeLocationProjectContext,
+    find_enclosing_code_location_root_path,
     is_inside_code_location_project,
 )
 from dagster_components.utils import CLI_BUILTIN_COMPONENT_LIB_KEY
@@ -31,8 +32,8 @@ def list_component_types_command(ctx: click.Context) -> None:
         )
         sys.exit(1)
 
-    context = CodeLocationProjectContext.from_path(
-        Path.cwd(),
+    context = CodeLocationProjectContext.from_code_location_path(
+        find_enclosing_code_location_root_path(Path.cwd()),
         ComponentRegistry.from_entry_point_discovery(builtin_component_lib=builtin_component_lib),
     )
     output: Dict[str, Any] = {}

python_modules/libraries/dagster-components/dagster_components/core/component_decl_builder.py

@@ -47,4 +47,4 @@ def path_to_decl_node(path: Path) -> Optional[ComponentDeclNode]:
         if component:
             subs.append(component)
 
-    return ComponentFolder(path=path, sub_decls=subs) if subs else None
+    return ComponentFolder(path=path, sub_decls=subs)

python_modules/libraries/dagster-components/dagster_components/core/component_defs_builder.py

@@ -125,16 +125,21 @@ def defs_from_components(
 
 # Public method so optional Nones are fine
 @suppress_dagster_warnings
-def build_defs_from_toplevel_components_folder(
-    path: Path,
+def build_component_defs(
+    code_location_root: Path,
     resources: Optional[Mapping[str, object]] = None,
     registry: Optional["ComponentRegistry"] = None,
 ) -> "Definitions":
-    """Build a Definitions object from an entire component hierarchy."""
+    """Build a Definitions object for all the component instances in a given code location.
+
+    Args:
+        code_location_root (Path): The path to the code location root.
+            The path must be a code location directory that has a pyproject.toml with a [dagster] section.
+    """
     from dagster._core.definitions.definitions_class import Definitions
 
-    context = CodeLocationProjectContext.from_path(
-        path, registry or ComponentRegistry.from_entry_point_discovery()
+    context = CodeLocationProjectContext.from_code_location_path(
+        code_location_root, registry or ComponentRegistry.from_entry_point_discovery()
     )
 
     all_defs: List[Definitions] = []

python_modules/libraries/dagster-components/dagster_components/core/deployment.py

@@ -15,13 +15,19 @@
 
 def is_inside_code_location_project(path: Path) -> bool:
     try:
-        _resolve_code_location_root_path(path)
+        find_enclosing_code_location_root_path(path)
         return True
     except DagsterError:
         return False
 
 
-def _resolve_code_location_root_path(path: Path) -> Path:
+def find_enclosing_code_location_root_path(path: Path) -> Path:
+    """Given a path, locate the code location root directory that contains it. It
+    determines this by finding a pyproject.toml with a [tool.dagster] section.
+
+    Searches parent directory recursively until it finds it. It if navigates
+    to the root directory, it throws an error.
+    """
     current_path = path.absolute()
     while not _is_code_location_root(current_path):
         current_path = current_path.parent
@@ -40,11 +46,15 @@ def _is_code_location_root(path: Path) -> bool:
 
 class CodeLocationProjectContext:
     @classmethod
-    def from_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self:
-        root_path = _resolve_code_location_root_path(path)
+    def from_code_location_path(cls, path: Path, component_registry: "ComponentRegistry") -> Self:
+        if not _is_code_location_root(path):
+            raise DagsterError(
+                f"Path {path} is not a code location root. Must have a pyproject.toml with a [tool.dagster] section."
+            )
+
         return cls(
-            root_path=str(root_path),
-            name=os.path.basename(root_path),
+            root_path=str(path),
+            name=os.path.basename(path),
             component_registry=component_registry,
         )
 

python_modules/libraries/dagster-components/dagster_components_tests/code_locations/custom_sling_location/custom_sling_location/definitions.py

@@ -1,11 +1,9 @@
 from pathlib import Path
 
 from dagster._core.definitions.definitions_class import Definitions
-from dagster_components.core.component_defs_builder import (
-    build_defs_from_toplevel_components_folder,
-)
+from dagster_components.core.component_defs_builder import build_component_defs
 
-defs = build_defs_from_toplevel_components_folder(path=Path(__file__).parent)
+defs = build_component_defs(code_location_root=Path(__file__).parent.parent)
 
 if __name__ == "__main__":
     Definitions.validate_loadable(defs)

python_modules/libraries/dagster-dg/dagster_dg/templates/CODE_LOCATION_NAME_PLACEHOLDER/CODE_LOCATION_NAME_PLACEHOLDER/definitions.py

@@ -1,9 +1,5 @@
 from pathlib import Path
 
-from dagster_components.core.component_defs_builder import (
-    build_defs_from_toplevel_components_folder,
-)
+from dagster_components import build_component_defs
 
-defs = build_defs_from_toplevel_components_folder(
-    path=Path(__file__).parent,
-)
+defs = build_component_defs(code_location_root=Path(__file__).parent.parent)

python_modules/libraries/dagster-dg/dagster_dg/templates/COMPONENT_TYPE/COMPONENT_TYPE_NAME_PLACEHOLDER.py.jinja

@@ -3,7 +3,6 @@ from dagster_components import (
     Component,
     ComponentRegistry,
     ComponentLoadContext,
-    build_defs_from_toplevel_components_folder,
     component,
 )