extra patches path (#17520)

* extra patches path

* Update conan/tools/files/patches.py

Co-authored-by: Andrey Filipenkov <kambaladecapitator@gmail.com>

* fix docstring of Environment.vars() (#17521)

fix docstring of vars()

* GnuToolchain's make_args handle empty values correctly (#17532)

test

* Change replace_in_file to return True on success (#17531)

- was inconsistent before: returned False if the
  pattern was not found (with strict off), otherwise None

* upgrade bottle version (#17534)

* better traces and msgs (#17517)

* better traces and msgs

* fix tests

* Feature/workspace api (#17501)

* working in workspace_api

* fixed loader + run() error

* update webtest testing version (#17537)

* proposing new --format=<format> --out-file=<file.ext> feature (#17507)

* proposing new --format=file.ext feature

* change approach and some simplifications

* remove unused

* minor changes

* minor changes

* simplify a bit

* fix format

* minor changes

* fix

* move colorama deinit/reinit

* add out-file only if formatters

* revert

* wip

* do not deinit/reinit if not tty

* clean import

* minor changes

---------

Co-authored-by: czoido <mrgalleta@gmail.com>

* Test/remote login (#17552)

* new test for remote_login

* unittest->pytest

* Keep sessions around between each ConanRequester (#17455)

* Keep sessions around between each ConanRequester

Avoid creating a new session for each ConanRequester instance, this
helps a lot with performance, as it avoids the overhead duplicate handshakes

* Patch new cached value on ConanRequester tests

* Modify adapter as needed

* Fix tests

* Name

* Update conans/client/rest/conan_requester.py

* Refactor requester tests

* Reinit is not part of this PR

* Refactor request usage out of app

* Make max_retries a local variable

* Add rest of posible `ConanBasicApp` usages (#17555)

Add rest of posible ConanBasicApp usages

* Improve typing information in remove sub API (#17564)

As the API actually accepts the remote to be `None`, this should also be
reflected in the type information in order to support users.

* small refactor to PkgConfigDeps (#17568)

* Updated to use maxsplit=1 (#17571)

* Updated to use maxsplit=1

* Add tests, unquote paths with spaces

---------

Co-authored-by: Abril Rincón Blanco <git@rinconblanco.es>

* refactor moving model conans->conan (#17572)

* refactor moving model conans->conan

* fix tests

* allow non trim_conandata

* removed print

---------

Co-authored-by: Andrey Filipenkov <kambaladecapitator@gmail.com>
Co-authored-by: Andrey Filipenkov <decapitator@ukr.net>
Co-authored-by: Ernesto de Gracia Herranz <ernestodgh@jfrog.com>
Co-authored-by: mrbean-bremen <mrbean-bremen@users.noreply.github.com>
Co-authored-by: czoido <mrgalleta@gmail.com>
Co-authored-by: Abril Rincón Blanco <rubenrb@jfrog.com>
Co-authored-by: Martin Lambertsen <github@lambertsen.one>
Co-authored-by: maitrey <maitreymishra23@gmail.com>
Co-authored-by: Abril Rincón Blanco <git@rinconblanco.es>

conan/internal/model/conf.py

@@ -33,6 +33,7 @@
     "core.sources:download_urls": "List of URLs to download backup sources from",
     "core.sources:upload_url": "Remote URL to upload backup sources to",
     "core.sources:exclude_urls": "URLs which will not be backed up",
+    "core.sources.patch:extra_path": "Extra path to search for patch files for conan create",
     # Package ID
     "core.package_id:default_unknown_mode": "By default, 'semver_mode'",
     "core.package_id:default_non_embed_mode": "By default, 'minor_mode'",

conan/tools/files/conandata.py

@@ -69,5 +69,8 @@ def trim_conandata(conanfile, raise_if_missing=True):
         if version_data is not None:
             result[k] = {version: version_data}
 
+    # Update the internal conanfile data too
+    conanfile.conan_data = result
+
     new_conandata_yml = yaml.safe_dump(result, default_flow_style=False)
     save(path, new_conandata_yml)

conan/tools/files/patches.py

@@ -3,9 +3,11 @@
 import shutil
 
 import patch_ng
+import yaml
 
 from conan.errors import ConanException
-from conans.util.files import mkdir
+from conan.internal.paths import DATA_YML
+from conans.util.files import mkdir, load, save
 
 
 class PatchLogHandler(logging.Handler):
@@ -122,27 +124,60 @@ def export_conandata_patches(conanfile):
     if conanfile.conan_data is None:
         raise ConanException("conandata.yml not defined")
 
-    patches = conanfile.conan_data.get('patches')
-    if patches is None:
-        conanfile.output.info("export_conandata_patches(): No patches defined in conandata")
-        return
+    conanfile_patches = conanfile.conan_data.get('patches')
 
-    if isinstance(patches, dict):
-        assert conanfile.version, "Can only be exported if conanfile.version is already defined"
-        entries = patches.get(conanfile.version, [])
-        if entries is None:
-            conanfile.output.warning(f"export_conandata_patches(): No patches defined for version {conanfile.version} in conandata.yml")
+    def _handle_patches(patches, patches_folder):
+        if patches is None:
+            conanfile.output.info("export_conandata_patches(): No patches defined in conandata")
             return
-    elif isinstance(patches, list):
-        entries = patches
-    else:
-        raise ConanException("conandata.yml 'patches' should be a list or a dict {version: list}")
-    for it in entries:
-        patch_file = it.get("patch_file")
-        if patch_file:
-            src = os.path.join(conanfile.recipe_folder, patch_file)
-            dst = os.path.join(conanfile.export_sources_folder, patch_file)
-            if not os.path.exists(src):
-                raise ConanException(f"Patch file does not exist: '{src}'")
-            mkdir(os.path.dirname(dst))
-            shutil.copy2(src, dst)
+
+        if isinstance(patches, dict):
+            assert conanfile.version, "Can only be exported if conanfile.version is already defined"
+            entries = patches.get(conanfile.version, [])
+            if entries is None:
+                conanfile.output.warning("export_conandata_patches(): No patches defined for "
+                                         f"version {conanfile.version} in conandata.yml")
+                return
+        elif isinstance(patches, list):
+            entries = patches
+        else:
+            raise ConanException("conandata.yml 'patches' should be a list or a dict "
+                                 "{version: list}")
+        for it in entries:
+            patch_file = it.get("patch_file")
+            if patch_file:
+                src = os.path.join(patches_folder, patch_file)
+                dst = os.path.join(conanfile.export_sources_folder, patch_file)
+                if not os.path.exists(src):
+                    raise ConanException(f"Patch file does not exist: '{src}'")
+                mkdir(os.path.dirname(dst))
+                shutil.copy2(src, dst)
+        return entries
+
+    _handle_patches(conanfile_patches, conanfile.recipe_folder)
+
+    extra_path = conanfile.conf.get("core.sources.patch:extra_path")
+    if extra_path:
+        if not os.path.isdir(extra_path):
+            raise ConanException(f"Patches extra path '{extra_path}' does not exist")
+        pkg_path = os.path.join(extra_path, conanfile.name)
+        if not os.path.isdir(pkg_path):
+            return
+        data_path = os.path.join(pkg_path, DATA_YML)
+        try:
+            data = yaml.safe_load(load(data_path))
+        except Exception as e:
+            raise ConanException("Invalid yml format at {}: {}".format(data_path, e))
+        data = data or {}
+        conanfile.output.info(f"Applying extra patches 'core.sources.patch:extra_path': {data_path}")
+        new_patches = _handle_patches(data.get('patches'), pkg_path)
+
+        # Update the CONANDATA.YML
+        conanfile_patches = conanfile_patches or {}
+        conanfile_patches.setdefault(conanfile.version, []).extend(new_patches)
+
+        conanfile.conan_data['patches'] = conanfile_patches
+        # Saving in the EXPORT folder
+        conanfile_data_path = os.path.join(conanfile.export_folder, DATA_YML)
+        new_conandata_yml = yaml.safe_dump(conanfile.conan_data, default_flow_style=False)
+        save(conanfile_data_path, new_conandata_yml)

test/functional/tools/test_files.py

@@ -6,8 +6,9 @@
 
 from conan.test.assets.genconanfile import GenConanfile
 from conan.test.utils.file_server import TestFileServer
+from conan.test.utils.test_files import temp_folder
 from conan.test.utils.tools import TestClient
-from conans.util.files import save
+from conans.util.files import save, load
 
 
 class MockPatchset:
@@ -365,7 +366,7 @@ def build(self):
     assert mock_patch_ng.apply_args[1:] == (0, False)
 
 
-def test_export_conandata_patches(mock_patch_ng):
+def test_export_conandata_patches():
     conanfile = textwrap.dedent("""
         import os
         from conan import ConanFile
@@ -402,7 +403,7 @@ def source(self):
     # wrong patches
     client.save({"conandata.yml": "patches: 123"})
     client.run("create .", assert_error=True)
-    assert "conandata.yml 'patches' should be a list or a dict"  in client.out
+    assert "conandata.yml 'patches' should be a list or a dict" in client.out
 
     # No patch found
     client.save({"conandata.yml": conandata_yml})
@@ -447,3 +448,60 @@ def build(self):
     client.save({"conandata.yml": conandata_yml, "conanfile.py": conanfile})
     client.run("create .")
     assert "No patches defined for version 1.0 in conandata.yml" in client.out
+
+
+@pytest.mark.parametrize("trim", [True, False])
+def test_export_conandata_patches_extra_origin(trim):
+    conanfile = textwrap.dedent(f"""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import export_conandata_patches, load, trim_conandata
+
+        class Pkg(ConanFile):
+            name = "mypkg"
+            version = "1.0"
+
+            def export(self):
+                if {trim}:
+                    trim_conandata(self)
+
+            def layout(self):
+                self.folders.source = "source_subfolder"
+
+            def export_sources(self):
+                export_conandata_patches(self)
+
+            def source(self):
+                self.output.info(load(self, os.path.join(self.export_sources_folder,
+                                                         "patches/mypatch.patch")))
+        """)
+    client = TestClient(light=True)
+    patches_folder = temp_folder()
+    conandata_yml = textwrap.dedent("""
+        patches:
+            "1.0":
+                - patch_file: "patches/mypatch.patch"
+        """)
+    save(os.path.join(patches_folder, "mypkg", "conandata.yml"), conandata_yml)
+    save(os.path.join(patches_folder, "mypkg", "patches", "mypatch.patch"), "mypatch!!!")
+
+    pkg_conandata = textwrap.dedent("""\
+        patches:
+            "1.1":
+                - patch_file: "patches/mypatch2.patch"
+    """)
+    client.save({"conanfile.py": conanfile,
+                 "conandata.yml": pkg_conandata,
+                 "patches/mypatch2.patch": ""})
+    client.run(f'create . -cc core.sources.patch:extra_path="{patches_folder}"')
+    assert "mypkg/1.0: Applying extra patches" in client.out
+    assert "mypkg/1.0: mypatch!!!" in client.out
+
+    conandata = load(client.exported_layout().conandata())
+    assert "1.0" in conandata
+    assert "patch_file: patches/mypatch.patch" in conandata
+
+    if trim:
+        assert "1.1" not in conandata
+    else:
+        assert "1.1" in conandata