f4pga/flows: override user configuration with CLI options; allow over…

…riding with None (#609)

f4pga/flows/argparser.py

@@ -21,12 +21,12 @@
 from re import finditer as re_finditer
 
 
-def _add_flow_arg(parser: ArgumentParser):
+def p_add_flow_arg(parser: ArgumentParser):
     parser.add_argument("-f", "--flow", metavar="flow_path", type=str, help="Path to flow definition file")
 
 
-def _setup_build_parser(parser: ArgumentParser):
-    _add_flow_arg(parser)
+def p_setup_build_parser(parser: ArgumentParser):
+    p_add_flow_arg(parser)
 
     parser.add_argument(
         "-t", "--target", metavar="target_name", type=str, help="Perform stages necessary to acquire target"
@@ -51,7 +51,7 @@ def _setup_build_parser(parser: ArgumentParser):
     parser.add_argument("--val", "-V", action="append", default=[])
 
 
-def _setup_show_dep_parser(parser: ArgumentParser):
+def p_setup_show_dep_parser(parser: ArgumentParser):
     parser.add_argument(
         "-p", "--part", metavar="part_name", type=str, help="Name of the part (use to display part-specific values.)"
     )
@@ -64,7 +64,7 @@ def _setup_show_dep_parser(parser: ArgumentParser):
         help="Name of the stage (use if you want to set the value only for that stage). Requires `-p`.",
     )
 
-    _add_flow_arg(parser)
+    p_add_flow_arg(parser)
 
 
 def setup_argparser():
@@ -78,23 +78,23 @@ def setup_argparser():
     parser.add_argument("-s", "--silent", action="store_true")
 
     subparsers = parser.add_subparsers(dest="command")
-    _setup_build_parser(subparsers.add_parser("build"))
+    p_setup_build_parser(subparsers.add_parser("build"))
     show_dep = subparsers.add_parser("showd", description="Show the value(s) assigned to a dependency")
-    _setup_show_dep_parser(show_dep)
+    p_setup_show_dep_parser(show_dep)
 
     return parser
 
 
-def _parse_depval(depvalstr: str):
+def p_parse_depval(depvalstr: str):
     """
     Parse a dependency or value definition in form of:
     optional_stage_name.value_or_dependency_name=value
-    See `_parse_cli_value` for detail on how to pass different kinds of values.
+    See `p_parse_cli_value` for detail on how to pass different kinds of values.
     """
 
     d = {"name": None, "stage": None, "value": None}
 
-    splitted = list(_unescaped_separated("=", depvalstr))
+    splitted = list(p_unescaped_separated("=", depvalstr))
 
     if len(splitted) != 2:
         raise Exception("Too many components")
@@ -111,12 +111,12 @@ def _parse_depval(depvalstr: str):
     if len(path_components) > 0:
         raise Exception("Too many path components")
 
-    d["value"] = _parse_cli_value(valstr)
+    d["value"] = p_parse_cli_value(valstr)
 
     return d
 
 
-def _unescaped_matches(regexp: str, s: str, escape_chr="\\"):
+def p_unescaped_matches(regexp: str, s: str, escape_chr="\\"):
     """
     Find all occurences of a pattern in a string that contains escape sequences.
     Yields pairs of starting and ending indices of the pattern.
@@ -128,17 +128,23 @@ def _unescaped_matches(regexp: str, s: str, escape_chr="\\"):
     # unescaped characters, but to map the results back to the string containing the
     # escape sequences, we need to track the offsets by which the characters were
     # shifted.
+
+    # TODO: This doesn't handle self-escape case
     offsets = []
     offset = 0
     for sl in s.split(escape_chr):
-        if len(sl) <= 1:
-            continue
-        noescape = sl[(1 if offset != 0 else 0) :]
+        noescape = sl[(1 if offset != 0 else 0) :] if len(sl) > 1 else ""
         for _ in noescape:
             offsets.append(offset)
         offset += 2
         noescapes += noescape
 
+    if len(offsets) == 0:
+        return (item for item in [])
+
+    last_offset = offsets[-1]
+    offsets.append(last_offset)
+
     iter = re_finditer(regexp, noescapes)
 
     for m in iter:
@@ -149,13 +155,13 @@ def _unescaped_matches(regexp: str, s: str, escape_chr="\\"):
         yield off1, off2
 
 
-def _unescaped_separated(regexp: str, s: str, escape_chr="\\"):
+def p_unescaped_separated(regexp: str, s: str, escape_chr="\\"):
     """
     Yields substrings of a string that contains escape sequences.
     """
 
     last_end = 0
-    for start, end in _unescaped_matches(regexp, s, escape_chr=escape_chr):
+    for start, end in p_unescaped_matches(regexp, s, escape_chr=escape_chr):
         yield s[last_end:start]
         last_end = end
     if last_end < len(s):
@@ -164,7 +170,7 @@ def _unescaped_separated(regexp: str, s: str, escape_chr="\\"):
         yield ""
 
 
-def _parse_cli_value(s: str):
+def p_parse_cli_value(s: str):
     """
     Parse a value/dependency passed to CLI
     CLI values are generated by the following non-contextual grammar:
@@ -188,7 +194,7 @@ def _parse_cli_value(s: str):
     """
 
     if len(s) == 0:
-        return ""
+        return None
 
     # List
     if s[0] == "[":
@@ -197,7 +203,7 @@ def _parse_cli_value(s: str):
         inner = s[1 : (len(s) - 1)]
         if inner == "":
             return []
-        return [_parse_cli_value(v) for v in _unescaped_separated(",", inner)]
+        return [p_parse_cli_value(v) for v in p_unescaped_separated(",", inner)]
 
     # Dictionary
     if s[0] == "{":
@@ -207,14 +213,14 @@ def _parse_cli_value(s: str):
         inner = s[1 : (len(s) - 1)]
         if inner == "":
             return {}
-        for kv in _unescaped_separated(",", inner):
-            k_v = list(_unescaped_separated(":", kv))
+        for kv in p_unescaped_separated(",", inner):
+            k_v = list(p_unescaped_separated(":", kv))
             if len(k_v) < 2:
                 raise Exception("Missing value in dictionary entry")
             if len(k_v) > 2:
                 raise Exception("Unexpected ':' token")
             key = k_v[0]
-            value = _parse_cli_value(k_v[1])
+            value = p_parse_cli_value(k_v[1])
             d[key] = value
 
         return d
@@ -243,7 +249,7 @@ def create_defdict():
     part_flow_config = create_defdict()
 
     def add_entries(arglist: "list[str]", dict_name: str):
-        for value_def in (_parse_depval(cliv) for cliv in arglist):
+        for value_def in (p_parse_depval(cliv) for cliv in arglist):
             stage = value_def["stage"]
             if stage is None:
                 part_flow_config[dict_name][value_def["name"]] = value_def["value"]

f4pga/flows/commands.py

@@ -44,6 +44,7 @@
     FlowConfig,
     FlowDefinition,
     open_project_flow_cfg,
+    override_prj_flow_cfg_by_cli,
     verify_platform_name,
 )
 from f4pga.flows.flow import Flow
@@ -219,13 +220,14 @@ def cmd_build(args: Namespace):
         project_flow_cfg = open_project_flow_config(args.flow)
     elif part_name is not None:
         project_flow_cfg = ProjectFlowConfig(".temp.flow.json")
-        project_flow_cfg.flow_cfg = get_cli_flow_config(args, part_name)
     if part_name is None and project_flow_cfg is not None:
         part_name = project_flow_cfg.get_default_part()
 
-    if project_flow_cfg is None:
+    if (project_flow_cfg is None) and part_name is None:
         fatal(-1, "No configuration was provided. Use `--flow`, and/or " "`--part` to configure flow.")
 
+    override_prj_flow_cfg_by_cli(project_flow_cfg, get_cli_flow_config(args, part_name))
+
     flow_cfg = make_flow_config(project_flow_cfg, part_name)
 
     if args.info:

f4pga/flows/flow.py

@@ -70,7 +70,7 @@ def __init__(self, target: str, cfg: FlowConfig, f4cache: "F4Cache | None"):
         self.dep_paths = {
             n: p
             for n, p in cfg.get_dependency_overrides().items()
-            if p_req_exists(p)  # and not p_dep_differ(p, f4cache)
+            if (p is not None) and p_req_exists(p)  # and not p_dep_differ(p, f4cache)
         }
         if f4cache is not None:
             for dep in self.dep_paths.values():

f4pga/flows/flow_config.py

@@ -132,6 +132,54 @@ def get_dependency_platform_overrides(self, part: str):
         return platform_ovds
 
 
+def override_prj_flow_cfg_by_cli(cfg: ProjectFlowConfig, cli_d: "dict[str, dict[str, dict]]"):
+    for part_name, part_cfg in cli_d.items():
+        print(f"OVERRIDING CONFIG FOR {part_name}")
+        p_cfg = cfg.flow_cfg.get(part_name)
+        if p_cfg is None:
+            p_cfg = {}
+            cfg.flow_cfg[part_name] = p_cfg
+        cli_p_values = part_cfg.get("values")
+        cli_p_dependencies = part_cfg.get("dependencies")
+        p_values = p_cfg.get("values")
+        p_dependencies = p_cfg.get("dependencies")
+        if cli_p_values is not None:
+            if p_values is None:
+                p_values = {}
+                part_cfg["values"] = p_values
+            p_values.update(cli_p_values)
+        if cli_p_dependencies is not None:
+            if p_dependencies is None:
+                p_dependencies = {}
+                part_cfg["dependencies"] = p_dependencies
+            p_dependencies.update(cli_p_dependencies)
+
+        for stage_name, cli_stage_cfg in part_cfg.items():
+            if _is_kword(stage_name):
+                continue
+
+            stage_cfg = part_cfg.get(stage_name)
+            if stage_cfg is None:
+                stage_cfg = {}
+                part_cfg[stage_name] = stage_cfg
+
+            stage_values = stage_cfg.get("values")
+            stage_dependencies = stage_cfg.get("dependencies")
+            cli_stage_values = cli_stage_cfg.get("values")
+            cli_stage_dependencies = cli_stage_cfg.get("dependencies")
+
+            if cli_stage_values is not None:
+                if stage_values is None:
+                    stage_values = {}
+                    stage_cfg["values"] = stage_values
+                stage_values.update(cli_stage_values)
+            if cli_stage_dependencies is not None:
+                if stage_dependencies is None:
+                    stage_dependencies = {}
+                    stage_cfg["dependencies"] = stage_dependencies
+                stage_dependencies.update(cli_stage_dependencies)
+
+
 class FlowConfig:
     part: str
     r_env: ResolutionEnv
@@ -144,7 +192,7 @@ def __init__(self, project_config: ProjectFlowConfig, platform_def: FlowDefiniti
         self.stages = platform_def.stages
         self.part = part
 
-        self.dependencies_explicit = deep(lambda p: str(Path(p).resolve()))(
+        self.dependencies_explicit = deep(lambda p: str(Path(p).resolve()), allow_none=True)(
             self.r_env.resolve(project_config.get_dependencies_raw(part))
         )