Merge pull request #5 from Jake-Carter/feat/build_test_optimizations

Test

.github/workflows/Build_Examples.yml

@@ -57,6 +57,27 @@ jobs:
           ref: '${{ github.event.pull_request.head.ref }}'
           repository: '${{ github.event.pull_request.head.repo.full_name }}'
 
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            **.c
+            **.h
+            **.S
+            **.s
+            **.a
+            **.cpp
+            **.mk
+            **/Makefile
+
+      - name: List all changed files
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          echo ${ALL_CHANGED_FILES} > changed_files
+          cat changed_files
+
       - name: Check watch files
         id: check_watch        
         run: |
@@ -222,7 +243,7 @@ jobs:
           # Set MAXIM_PATH to limit warnings
           export MAXIM_PATH=$(pwd)
 
-          python .github/workflows/scripts/build.py
+          python .github/workflows/scripts/build.py --change_file changed_files
 
   Regression_Test_Make_3_81:
     runs-on:

.github/workflows/Verify_Register_SVD.yml

@@ -655,5 +655,26 @@ jobs:
             fi
           done
           set -e
-          
+
+          # Remove generated register files for next time a runner picks up the job.
+          # The register files generated in a previous instance will remain and won't
+          # be regenerated because the Wall-E doesn't refresh its workspace and this 
+          # workflow only generates the register files for a specific part only once 
+          # (to save time) by checking whether the generated register files exist or not.
+          cd ${{ env.MSDK-INTERNAL_DIR }}/SVD/Devices/
+
+          # Enter each directory and remove ./chip_test/
+          for file_dir in *
+          do
+            # Don't mess with files.
+            [[ -d $file_dir ]] || continue
+
+            if [[ ${DIE_TO_CHIP_NAMES[$file_dir]+_} ]]; then
+              rm -rf ./${file_dir}/chip_test/
+            fi
+          done
+
+          # Return to workspace
+          cd ../../../
+
           exit $CHECK_FAIL

.github/workflows/scripts/build.py

@@ -51,18 +51,28 @@
     "BLE_FreeRTOS"
 ]
 
+console = Console(emoji=False, color_system="standard")
+
 def build_project(project:Path, target, board, maxim_path:Path, distclean=False, extra_args=None) -> Tuple[int, tuple]:
-    clean_cmd = "make clean" if not distclean else "make distclean"
-    if "Bluetooth" in project.as_posix() or "BLE" in project.as_posix():
-        # Clean cordio lib for BLE projects
-        clean_cmd += "&& make clean.cordio"
+    clean_cmd = "make clean" if not distclean else "make distclean"    
     res = run(clean_cmd, cwd=project, shell=True, capture_output=True, encoding="utf-8")
 
     # Test build
     build_cmd = f"make -r -j 8 TARGET={target} MAXIM_PATH={maxim_path.as_posix()} BOARD={board} FORCE_COLOR=1"
     if extra_args:
         build_cmd += f" {str(extra_args)}"
     res = run(build_cmd, cwd=project, shell=True, capture_output=True, encoding="utf-8")
+
+    if res.returncode != 0 and ("Bluetooth" in project.as_posix() or "BLE" in project.as_posix()):
+        # NOTE: Special case is required to handle some Cordio re-builds.
+        # TODO: Track Cordio options better across library re-builds
+        console.print(f"[red]BLE project {project} failed.[/red]")
+        console.print("[yellow]Trying a clean.cordio and rebuild...[/yellow]")
+        clean_cmd += "&& make clean.cordio"
+        run(clean_cmd, cwd=project, shell=True, capture_output=True, encoding="utf-8")
+        res = run(build_cmd, cwd=project, shell=True, capture_output=True, encoding="utf-8")
+        if res.returncode == 0:
+            console.print("[green]Pass[/green] after rebuild.")
 
     project_info = {
         "target":target,
@@ -105,11 +115,113 @@ def build_project(project:Path, target, board, maxim_path:Path, distclean=False,
 
     return (return_code, project_info)
 
-
-def test(maxim_path : Path = None, targets=None, boards=None, projects=None): 
-
-    console = Console(emoji=False, color_system="standard")
-
+def query_build_variable(project:Path, variable:str) -> list:
+    result = run(f"make query QUERY_VAR=\"{variable}\"", cwd=project, shell=True, capture_output=True, encoding="utf-8")
+    if result.returncode != 0:
+        return []
+
+    output = []
+    for v in variable.split(" "):
+        for line in result.stdout.splitlines():
+            if v in line:
+                # query output string will be "{variable}={item1, item2, ..., itemN}"
+                output += str(line).split("=")[1].split(" ")
+
+    return output
+
+"""
+Create a dictionary mapping each target micro to its dependencies in the MSDK.
+The dependency paths contain IPATH, VPATH, SRCS, and LIBS from all of the target's examples.
+
+It should be noted that any paths relative to the Libraries folder will be walked back to a
+1-level deep sub-folder.
+
+Ex: Libraries/PeriphDrivers/Source/ADC/adc_me14.c   --->   Libraries/PeriphDrivers
+
+This is so that any changes to source files will be caught by the dependency tracker.
+TODO: Improve this so the dependency checks are more granular.  (i.e. ME14 source change only 
+    affects ME14).  This is difficult because example projects do not have direct visibility 
+    into library SRCS/VPATH for libraries that build a static file.
+
+The format of the dependency map a dictionary:
+    dependency_map[TARGET] = [DEPENDENCIES]
+
+    where TARGET is the target micro's part number (ex: MAX78000)
+    and DEPENDENCIES is a list of absolute paths as strings (ex:   ['/home/jhcarter/repos/msdk/Examples/MAX32690/ADC', 
+                                                                    '/home/jhcarter/repos/msdk/Examples/MAX32690/ADC/board.c', 
+                                                                    '/home/jhcarter/repos/msdk/Examples/MAX32690/ADC/heap.c',
+                                                                    ...])
+    the paths may be a file or a folder.
+"""
+def create_dependency_map(maxim_path:Path, targets:list) -> dict:
+    dependency_map = dict()
+
+    with Progress(console=console) as progress:
+        task_dependency_map = progress.add_task("Creating dependency map...", total=len(targets))
+        for target in targets:
+            progress.update(task_dependency_map, description=f"Creating dependency map for {target}...")
+            dependency_map[target] = []
+            examples_dir = Path(maxim_path / "Examples" / target)
+            if examples_dir.exists():
+                projects = [Path(i).parent for i in examples_dir.rglob("**/project.mk")]
+                for project in projects:
+                    console.print(f"\t- Checking dependencies: {project}")
+                    dependencies = query_build_variable(project, "IPATH VPATH SRCS LIBS")
+                    dependencies = list(set(dependencies))
+                    for i in dependencies:
+                        if i == ".":
+                            dependencies.remove(i)
+                            i = project
+
+                        # Convert to absolute paths
+                        if not Path(i).is_absolute():
+                            dependencies.remove(i)
+                            corrected = Path(Path(project) / i).absolute()
+                            i = corrected
+
+                        # Walk back library paths to their root library folder.
+                        # This is so that any src changes will get caught, since
+                        # usually the project does not have the src folders as direct
+                        # dependencies on VPATH/SRCS.  IPATH will be exposed to the project.
+                        if "Libraries" in str(i):
+                            path = Path(i)
+                            while path.parent.stem != "Libraries" and path.exists():
+                                path = path.parent
+                            i = str(path)
+
+                        if i not in dependency_map[target]:
+                            dependency_map[target].append(str(i))
+
+
+            if "." in dependency_map[target]: 
+                dependency_map[target].remove(".") # Root project dir
+            if str(maxim_path) in dependency_map[target]:
+                dependency_map[target].remove(str(maxim_path)) # maxim_path gets added for "mxc_version.h"
+
+            dependency_map[target] = sorted(list(set(dependency_map[target])))
+            # console.print(f"\t- {target} dependencies:\n{dependency_map[target]}")
+            progress.update(task_dependency_map, advance=1)
+
+    return dependency_map
+
+"""
+Return a list of target microcontrollers that are affected by a change to the specified file.
+"""
+def get_affected_targets(dependency_map: dict, file: Path) -> list:
+    file = Path(file)
+    affected = []
+    for target in dependency_map.keys():
+        add = False
+        if target in str(file).upper(): add = True
+
+        for dependency in dependency_map[target]:
+            if file.is_relative_to(dependency): add = True
+
+        if add and target not in affected: affected.append(target)
+
+    return affected
+
+def test(maxim_path : Path = None, targets=None, boards=None, projects=None, change_file=None):
     env = os.environ.copy()
     if maxim_path is None and "MAXIM_PATH" in env.keys():
         maxim_path = Path(env['MAXIM_PATH']).absolute()
@@ -120,9 +232,16 @@ def test(maxim_path : Path = None, targets=None, boards=None, projects=None):
 
     env["FORCE_COLOR"] = 1
 
+    console.print(f"Blacklist: {blacklist}")
+    console.print(f"Project blacklist: {project_blacklist}")
+    console.print(f"Known errors: {known_errors}")
+    console.print(f"Hard FP whitelist: {hardfp_test_list}")
+
     # Remove the periphdrivers build directory
     console.print("Cleaning PeriphDrivers build directories...")
     shutil.rmtree(Path(maxim_path) / "Libraries" / "PeriphDrivers" / "bin", ignore_errors=True)
+    console.print("Cleaning Cordio build directories...")
+    shutil.rmtree(Path(maxim_path) / "Libraries" / "Cordio" / "bin", ignore_errors=True)
 
     # Get list of target micros if none is specified
     if targets is None:
@@ -142,6 +261,42 @@ def test(maxim_path : Path = None, targets=None, boards=None, projects=None):
     # Enforce alphabetical ordering
     targets = sorted(targets)
 
+    if (args.change_file is not None):
+        console.print(f"Reading '{args.change_file}'")
+        targets_to_skip = []
+        for i in targets: targets_to_skip.append(i)
+        files:list = []
+        with open(args.change_file, "r") as change_file:
+            files = change_file.read().strip().replace(" ", "\n").splitlines()
+            files = [maxim_path / file for file in files]
+
+        if not files:
+            console.print("[red]Changed files is empty.  Skipping dependency checks.[/red]")
+        else:
+            console.print("Creating dependency map...")
+            dependency_map = create_dependency_map(maxim_path, targets)
+            console.print(f"Checking {len(files)} changed files...")
+
+            for f in files:
+                affected = get_affected_targets(dependency_map, f)
+                if affected:
+                    for i in affected:
+                        if i in targets_to_skip:
+                            targets_to_skip.remove(i)
+                            console.print(f"\t- Testing {i} from change to {f}")
+                else:
+                    console.print(f"\t- Unknown effects from change to {f}, testing everything")
+                    targets_to_skip.clear()
+
+                if len(targets_to_skip) == 0: break
+
+            targets = [i for i in targets if i not in targets_to_skip]
+
+    if targets is not None:
+        console.print(f"Testing: {targets}")
+    else:
+        console.print("Nothing to be tested.")
+
     # Track failed projects for end summary
     failed = []
     warnings = []
@@ -344,15 +499,18 @@ def test(maxim_path : Path = None, targets=None, boards=None, projects=None):
 parser.add_argument("--targets", type=str, nargs="+", required=False, help="Target microcontrollers to test.")
 parser.add_argument("--boards", type=str, nargs="+", required=False, help="Boards to test.  Should match the BSP folder-name exactly.")
 parser.add_argument("--projects", type=str, nargs="+", required=False, help="Examples to populate.  Should match the example's folder name.")
+parser.add_argument("--change_file", type=str, required=False, help="(Optional) Pass a text file containing a list of space-separated or new-line separated changed files.  The build script will intelligently adjust which parts it tests based on this list.")
 
 if __name__ == "__main__":
     args = parser.parse_args()
     inspect(args, title="Script arguments:", )
+
     exit(
         test(
             maxim_path=args.maxim_path,
             targets=args.targets,
             boards=args.boards,
-            projects=args.projects
+            projects=args.projects,
+            change_file=args.change_file
         )
     )

Examples/MAX32520/AES/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_bayes_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_class_marks_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_convolution_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_dotproduct_example_f32/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_fft_bin_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_fir_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_graphic_equalizer_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_linear_interp_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_matrix_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph

Examples/MAX32520/ARM-DSP/arm_signal_converge_example/Makefile

@@ -370,7 +370,7 @@ endif
 
 all:
 # 	Extend the functionality of the "all" recipe here
-	arm-none-eabi-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
+	$(PREFIX)-size --format=berkeley $(BUILD_DIR)/$(PROJECT).elf
 
 libclean: 
 	$(MAKE)  -f ${PERIPH_DRIVER_DIR}/periphdriver.mk clean.periph