diff --git a/.github/workflows/scripts/build.py b/.github/workflows/scripts/build.py index ca3a0c7ef1..437d74bf16 100644 --- a/.github/workflows/scripts/build.py +++ b/.github/workflows/scripts/build.py @@ -51,6 +51,8 @@ "BLE_FreeRTOS" ] +console = Console(emoji=False, color_system="standard") + def build_project(project:Path, target, board, maxim_path:Path, distclean=False, extra_args=None) -> Tuple[int, tuple]: clean_cmd = "make clean" if not distclean else "make distclean" if "Bluetooth" in project.as_posix() or "BLE" in project.as_posix(): @@ -105,11 +107,102 @@ def build_project(project:Path, target, board, maxim_path:Path, distclean=False, return (return_code, project_info) - -def test(maxim_path : Path = None, targets=None, boards=None, projects=None, change_file=None): - - console = Console(emoji=False, color_system="standard") - +def query_build_variable(project:Path, variable:str) -> list: + result = run(f"make query QUERY_VAR={variable}", cwd=project, shell=True, capture_output=True, encoding="utf-8") + if result.returncode != 0: + return [] + + for line in result.stdout.splitlines(): + if variable in line: + # query output string will be "{variable}={item1, item2, ..., itemN}" + return str(line).split("=")[1].split(" ") + + raise Exception(f"Bad variable query for {variable} in {project}") + +""" +Create a dictionary mapping each target micro to its dependencies in the MSDK. +The dependency paths contain IPATH, VPATH, SRCS, and LIBS from all of the target's examples. + +It should be noted that any paths relative to the Libraries folder will be walked back to a +1-level deep sub-folder. + +Ex: Libraries/PeriphDrivers/Source/ADC/adc_me14.c ---> Libraries/PeriphDrivers + +This is so that any changes to source files will be caught by the dependency tracker. +TODO: Improve this so the dependency checks are more granular. (i.e. ME14 source change only + affects ME14). This is difficult because example projects do not have direct visibility + into library SRCS/VPATH for libraries that build a static file. + +""" +def create_dependency_map(maxim_path:Path, targets:list) -> dict: + dependency_map = dict() + + with Progress(console=console) as progress: + task_dependency_map = progress.add_task("Creating dependency map...", total=len(targets)) + for target in targets: + progress.update(task_dependency_map, description=f"Creating dependency map for {target}...") + dependency_map[target] = [] + examples_dir = Path(maxim_path / "Examples" / target) + if examples_dir.exists(): + projects = [Path(i).parent for i in examples_dir.rglob("**/project.mk")] + for project in projects: + console.print(f"\t- Checking dependencies: {project}") + IPATH = query_build_variable(project, "IPATH") + VPATH = query_build_variable(project, "VPATH") + SRCS = query_build_variable(project, "SRCS") + LIBS = query_build_variable(project, "SRCS") + dependencies = list(set(IPATH + VPATH + SRCS + LIBS)) + for i in dependencies: + if i == ".": + dependencies.remove(i) + i = project + + # Convert to absolute paths + if not Path(i).is_absolute(): + dependencies.remove(i) + corrected = Path(Path(project) / i).absolute() + i = corrected + + # Walk back library paths to their root library folder. + # This is so that any src changes will get caught, since + # usually the project does not have the src folders as direct + # dependencies on VPATH/SRCS. IPATH will be exposed to the project. + if "Libraries" in str(i): + path = Path(i) + while path.parent.stem != "Libraries" and path.exists(): + path = path.parent + i = str(path) + + if i not in dependency_map[target]: + dependency_map[target].append(str(i)) + + + if "." in dependency_map[target]: + dependency_map[target].remove(".") # Root project dir + if str(maxim_path) in dependency_map[target]: + dependency_map[target].remove(str(maxim_path)) # maxim_path gets added for "mxc_version.h" + + dependency_map[target] = sorted(list(set(dependency_map[target]))) + # console.print(f"\t- {target} dependencies:\n{dependency_map[target]}") + progress.update(task_dependency_map, advance=1) + + return dependency_map + +def get_affected_targets(dependency_map: dict, file: Path) -> list: + file = Path(file) + affected = [] + for target in dependency_map.keys(): + add = False + if target in str(file).upper(): add = True + + for dependency in dependency_map[target]: + if file.is_relative_to(dependency): add = True + + if add and target not in affected: affected.append(target) + + return affected + +def test(maxim_path : Path = None, targets=None, boards=None, projects=None, change_file=None): env = os.environ.copy() if maxim_path is None and "MAXIM_PATH" in env.keys(): maxim_path = Path(env['MAXIM_PATH']).absolute() @@ -153,17 +246,30 @@ def test(maxim_path : Path = None, targets=None, boards=None, projects=None, cha for i in targets: targets_to_skip.append(i) files:list = [] with open(args.change_file, "r") as change_file: - files = change_file.read().replace(" ", "\n").splitlines() + files = change_file.read().strip().replace(" ", "\n").splitlines() + files = [maxim_path / file for file in files] - console.print(f"Checking {len(files)} changed files...") - - for f in files: - for target in targets_to_skip: - if target in str(f).upper(): - targets_to_skip.remove(target) - console.print(f"\t- Testing {target} from change to: {f}") - - targets = [i for i in targets if i not in targets_to_skip] + if not files: + console.print("[red]Changed files is empty. Skipping dependency checks.[/red]") + else: + console.print("Creating dependency map...") + dependency_map = create_dependency_map(maxim_path, targets) + console.print(f"Checking {len(files)} changed files...") + + for f in files: + affected = get_affected_targets(dependency_map, f) + if affected: + for i in affected: + if i in targets_to_skip: + targets_to_skip.remove(i) + console.print(f"\t- Testing {i} from change to {f}") + else: + console.print(f"\t- Unknown effects from change to {f}, testing everything") + targets_to_skip.clear() + + if len(targets_to_skip) == 0: break + + targets = [i for i in targets if i not in targets_to_skip] if targets is not None: console.print(f"Testing: {targets}")