Initial support for source-only dependencies

.pylintrc

@@ -1,6 +1,6 @@
 [DESIGN]
 max-args=8
-max-attributes=10
+max-attributes=11
 
 [MESSAGES CONTROL]
 disable=too-few-public-methods

tests/data/valid/scenarios/native/src/test.cc

@@ -4,6 +4,7 @@
 #include "yambs-sample/example/sample.h"
 #include "yambs-sample2/example/sample.h"
 #include "yambs2/sample.h"
+#include "yambs3/sample2.h"
 
 namespace Example4 {
 
@@ -31,6 +32,10 @@ void method2(void) {
   Example3::method1();
   Example3::method2();
   Example3::method3();
+
+  Example5::method1();
+  Example5::method2();
+  Example5::method3();
 }
 
 } // namespace Example4

tests/data/valid/scenarios/native/yambs.yaml

@@ -30,3 +30,6 @@ variants:
 dependencies:
   - github: {repo: yambs-sample, owner: *self}
   - directory: ../native2
+
+  - directory: ../native3
+    as_source: true

tests/data/valid/scenarios/native3/src/sample2.cc

@@ -0,0 +1,32 @@
+#include "sample2.h"
+
+namespace Example5 {
+
+static void method4(void) {
+  int a = 0;
+  for (int i = 0; i < 1000; i++) {
+    a *= 2;
+  }
+
+  (void)a;
+}
+
+void method1(void) {
+  int a = 0;
+  for (int i = 0; i < 1000; i++) {
+    a *= 2;
+  }
+  method4();
+  (void)a;
+}
+
+void method2(void) {
+  int a = 0;
+  for (int i = 0; i < 1000; i++) {
+    a *= 2;
+  }
+  method4();
+  (void)a;
+}
+
+} // namespace Example5

tests/data/valid/scenarios/native3/src/sample2.h

@@ -0,0 +1,16 @@
+#pragma once
+
+namespace Example5 {
+
+void method1(void);
+void method2(void);
+
+inline void method3(void) {
+  int b = 0;
+  for (int i = 0; i < 1000; i++) {
+    b *= 10;
+  }
+  (void)b;
+}
+
+} // namespace Example5

tests/data/valid/scenarios/native3/yambs.yaml

@@ -0,0 +1,3 @@
+---
+project:
+  name: yambs3

yambs/aggregation/__init__.py

@@ -13,10 +13,13 @@
 BySuffixPaths = Dict[str, Set[Path]]
 
 
-def collect_files(root: Path, recurse: bool = True) -> BySuffixPaths:
+def collect_files(
+    root: Path, recurse: bool = True, files: BySuffixPaths = None
+) -> BySuffixPaths:
     """Collect files (by suffix) from a starting directory."""
 
-    files: BySuffixPaths = defaultdict(set)
+    if files is None:
+        files = defaultdict(set)
 
     for item in root.iterdir():
         if item.is_dir() and recurse:
@@ -44,12 +47,21 @@ def sources_headers(paths: BySuffixPaths) -> Set[Path]:
 
 
 def populate_sources(
-    paths: BySuffixPaths, src_root: Path, apps: Set[Path], regular: Set[Path]
+    paths: BySuffixPaths,
+    src_root: Path,
+    apps: Set[Path],
+    regular: Set[Path],
+    third_party: Set[Path],
 ) -> None:
     """Populate application and regular sources from a group of paths."""
 
     for source in compile_sources(paths):
-        if str(source.relative_to(src_root)).startswith(APP_ROOT):
-            apps.add(source)
-        else:
-            regular.add(source)
+        try:
+            if str(source.relative_to(src_root)).startswith(APP_ROOT):
+                apps.add(source)
+            else:
+                regular.add(source)
+
+        # Handle a source file outside of the main source tree.
+        except ValueError:
+            third_party.add(source)

yambs/data/schemas/Dependency.yaml

@@ -12,6 +12,10 @@ properties:
     enum: [github]
     default: github
 
+  as_source:
+    type: boolean
+    default: false
+
   target:
     type: string
     default: opt

yambs/data/templates/native_build.ninja.j2

@@ -1,5 +1,6 @@
 include_dir = {{ninja_out}}
 src_dir = {{src_root}}
+third_party_dir = {{third_party_root}}
 generated_dir = $src_dir/generated
 
 # Flags common to all builds, regardless of variant.

yambs/dependency/config.py

@@ -68,3 +68,4 @@ def init(self, data: _JsonObject) -> None:
         )
         self.target: str = data["target"]  # type: ignore
         self.version: str = data["version"]  # type: ignore
+        self.as_source: bool = data["as_source"]  # type: ignore

yambs/dependency/handlers/types.py

@@ -21,6 +21,7 @@ class DependencyTask(NamedTuple):
     static: Path
 
     build_commands: List[List[str]]
+    source_dirs: Set[Path]
 
     compile_flags: List[str]
     link_flags: List[str]

yambs/dependency/handlers/yambs/__init__.py

@@ -113,7 +113,8 @@ def yambs_handler(task: DependencyTask) -> DependencyState:
     directory = handle_source(task)
     config, directory = handle_config_load(task, directory)
 
-    handle_static_lib(directory, task)
+    if not task.dep.as_source:
+        handle_static_lib(directory, task)
 
     # Ensure the 'src' directory is linked within the include directory.
     src_include = task.include.joinpath(task.data["name"])
@@ -123,6 +124,9 @@ def yambs_handler(task: DependencyTask) -> DependencyState:
         else:
             src_include.symlink_to(directory.joinpath("src"))
 
+    if task.dep.as_source:
+        task.source_dirs.add(src_include)
+
     # Check if loading the project configuration data is necessary.
     # Read the project's configuration data to find any nested dependencies.
     check_nested_dependencies(

yambs/dependency/manager.py

@@ -64,6 +64,9 @@ def __init__(self, root: Path, project_root: Path) -> None:
         # A list of commands to run that should build dependencies.
         self.build_commands: List[List[str]] = []
 
+        # Directories to be treated as additional source directories.
+        self.source_dirs: Set[Path] = set()
+
         # Ensure absolute paths aren't generated into ninja configurations.
         base = self.root.parent
 
@@ -108,6 +111,7 @@ def _create_task(self, dep: Dependency) -> DependencyTask:
             self.include,
             self.static,
             self.build_commands,
+            self.source_dirs,
             self.compile_flags,
             self.link_flags,
             dep,

yambs/environment/native.py

@@ -45,9 +45,7 @@ def __init__(self, config: Native) -> None:
         self.sources = collect_files(config.src_root)
         self.apps: Set[Path] = set()
         self.regular: Set[Path] = set()
-        populate_sources(
-            self.sources, config.src_root, self.apps, self.regular
-        )
+        self.third_party: Set[Path] = set()
 
         self.jinja = get_jinja()
 
@@ -72,9 +70,33 @@ def write_compile_line(self, stream: TextIO, path: Path) -> Path:
 
         return out
 
+    def write_third_party_line(self, stream: TextIO, path: Path) -> Path:
+        """Write a single source-compile line for a third-party source."""
+
+        translator = get_translator(path)
+
+        # Get the relative part of the path from the third-party root.
+        rel_part = path.relative_to(self.config.third_party_root)
+
+        out = translator.translate(Path("$build_dir", "third-party", rel_part))
+
+        stream.write(
+            f"build {out}: {translator.rule} $third_party_dir/{rel_part}"
+        )
+        stream.write(linesep)
+
+        return out
+
     def write_source_rules(self, stream: TextIO) -> Set[Path]:
         """Write source rules."""
-        return {self.write_compile_line(stream, path) for path in self.regular}
+
+        # Add third-party sources.
+        return {
+            self.write_compile_line(stream, path) for path in self.regular
+        } | {
+            self.write_third_party_line(stream, path)
+            for path in self.third_party
+        }
 
     def write_static_library_rule(
         self, stream: TextIO, outputs: Set[Path]
@@ -200,6 +222,21 @@ def generate(self, sources_only: bool = False) -> None:
                 self.dependency_manager.link_flags
             )
 
+            # Handle additional source directories (belonging to dependencies).
+            for path in self.dependency_manager.source_dirs:
+                collect_files(path, files=self.sources)
+            populate_sources(
+                self.sources,
+                self.config.src_root,
+                self.apps,
+                self.regular,
+                self.third_party,
+            )
+
+            # Handle third-party sources.
+            # probably do like:
+            # $build_dir/third-party $src_dir/../ ?
+
             # Render templates.
             generate_variants(
                 self.jinja,
@@ -226,7 +263,11 @@ def generate(self, sources_only: bool = False) -> None:
         # Render format file.
         render_format(
             self.config,
-            sources_headers(self.sources),
+            {
+                x
+                for x in sources_headers(self.sources)
+                if self.config.src_root in x.parents
+            },
             suffix=self.config.data["variants"]["clang"]["suffix"],
         )
 

yambs/translation/__init__.py

@@ -23,9 +23,13 @@ class SourceTranslator(NamedTuple):
     output_extension: str = ".o"
     dest: Path = Path(BUILD_DIR_VAR)
 
+    def translate(self, path: Path) -> Path:
+        """Translate a path by changing its suffix."""
+        return path.with_suffix(self.output_extension)
+
     def output(self, path: Path) -> Path:
         """Get the output file from a given path."""
-        return self.dest.joinpath(path.with_suffix(self.output_extension))
+        return self.dest.joinpath(self.translate(path))
 
     @property
     def gets_linked(self) -> bool: