Package vendor new feature (#16073)

* wip

* wip

* wip

* fix

* fix test

* Updated graph serialization and display for re-package nodes

* Fix broken tests

* Make graph command check tools.graph:repackage is passed when trying to compute graph on a repackage dependency

* Fix integration test, changed error output

* renamed to bundle

* fixes

* fix test

* rename conf to build_bundle

* renames

* avoid propagate options

* Update conans/client/graph/graph.py

Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

---------

Co-authored-by: PerseoGI <perseog@jfrog.com>
Co-authored-by: Rubén Rincón Blanco <git@rinconblanco.es>

Author: 3 people

conan/cli/commands/create.py

@@ -95,6 +95,9 @@ def create(conan_api, parser, *args):
     else:
         requires = [ref] if not is_build else None
         tool_requires = [ref] if is_build else None
+        if conanfile.vendor:  # Automatically allow repackaging for conan create
+            pr = profile_build if is_build else profile_host
+            pr.conf.update("&:tools.graph:vendor", "build")
         deps_graph = conan_api.graph.load_graph_requires(requires, tool_requires,
                                                          profile_host=profile_host,
                                                          profile_build=profile_build,

conan/cli/formatters/graph/info_graph_html.py

@@ -144,6 +144,11 @@
                     if (node.recipe == "Platform") {
                         font.background = "Violet";
                     }
+                    if (node.vendor) {
+                        borderColor = "Red";
+                        shapeProperties = {borderDashes: [3,5]};
+                        borderWidth = 2;
+                    }
                     nodes.push({
                         id: node_id,
                         font: font,
@@ -225,6 +230,14 @@
                     font: {size: 35, color: "white"},
                     color: {border: "SkyBlue", background: "Black"}
                 });
+                counter++;
+
+                legend_nodes.push({x: x + counter*step, y: y, shape: "box",
+                    label: "vendor", font: {size: 35},
+                    color: {border: "Red"},
+                    shapeProperties: {borderDashes: [3,5]},
+                    borderWidth: 2
+                });
                 return {nodes: new vis.DataSet(legend_nodes)};
             }
             let error = document.getElementById("error");

conans/client/graph/compute_pid.py

@@ -38,6 +38,9 @@ def compute_package_id(node, new_config, config_version):
             else:
                 data[require] = req_info
 
+    if conanfile.vendor:  # Make the package_id fully independent of dependencies versions
+        data, build_data = OrderedDict(), OrderedDict()  # TODO, cleaner, now minimal diff
+
     reqs_info = RequirementsInfo(data)
     build_requires_info = RequirementsInfo(build_data)
     python_requires = PythonRequiresInfo(python_requires, python_mode)

conans/client/graph/graph.py

@@ -108,6 +108,8 @@ def propagate_downstream(self, require, node, src_node=None):
         self.transitive_deps.pop(require, None)
         self.transitive_deps[require] = TransitiveRequirement(require, node)
 
+        if self.conanfile.vendor:
+            return
         # Check if need to propagate downstream
         if not self.dependants:
             return
@@ -154,6 +156,8 @@ def check_downstream_exists(self, require):
         # Check if need to propagate downstream
         # Then propagate downstream
 
+        if self.conanfile.vendor:
+            return result
         # Seems the algrithm depth-first, would only have 1 dependant at most to propagate down
         # at any given time
         if not self.dependants:

conans/client/graph/graph_binaries.py

@@ -181,6 +181,15 @@ def _evaluate_node(self, node, build_mode, remotes, update):
             node.build_allowed = True
             node.binary = BINARY_BUILD if not node.cant_build else BINARY_INVALID
 
+        if node.binary == BINARY_BUILD:
+            conanfile = node.conanfile
+            if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)):
+                node.conanfile.info.invalid = f"The package '{conanfile.ref}' is a vendoring one, " \
+                                              f"needs to be built from source, but it " \
+                                              "didn't enable 'tools.graph:vendor=build' to compute " \
+                                              "its dependencies"
+                node.binary = BINARY_INVALID
+
     def _process_node(self, node, build_mode, remotes, update):
         # Check that this same reference hasn't already been checked
         if self._evaluate_is_cached(node):

conans/client/graph/graph_builder.py

@@ -5,7 +5,7 @@
 from conan.internal.cache.conan_reference_layout import BasicLayout
 from conans.client.conanfile.configure import run_configure_method
 from conans.client.graph.graph import DepsGraph, Node, CONTEXT_HOST, \
-    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL
+    CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE
 from conans.client.graph.graph import RECIPE_PLATFORM
 from conans.client.graph.graph_error import GraphLoopError, GraphConflictError, GraphMissingError, \
     GraphRuntimeError, GraphError
@@ -52,7 +52,10 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
                     continue
                 new_node = self._expand_require(require, node, dep_graph, profile_host,
                                                 profile_build, graph_lock)
-                if new_node:
+                if new_node and (not new_node.conanfile.vendor
+                                 or new_node.recipe == RECIPE_EDITABLE or
+                                 new_node.conanfile.conf.get("tools.graph:vendor",
+                                                             choices=("build",))):
                     self._initialize_requires(new_node, dep_graph, graph_lock, profile_build,
                                               profile_host)
                     open_requires.extendleft((r, new_node)
@@ -386,6 +389,7 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr
     @staticmethod
     def _compute_down_options(node, require, new_ref):
         # The consumer "up_options" are the options that come from downstream to this node
+        visible = require.visible and not node.conanfile.vendor
         if require.options is not None:
             # If the consumer has specified "requires(options=xxx)", we need to use it
             # It will have less priority than downstream consumers
@@ -395,11 +399,11 @@ def _compute_down_options(node, require, new_ref):
             # options["dep"].opt=value only propagate to visible and host dependencies
             # we will evaluate if necessary a potential "build_options", but recall that it is
             # now possible to do "self.build_requires(..., options={k:v})" to specify it
-            if require.visible:
+            if visible:
                 # Only visible requirements in the host context propagate options from downstream
                 down_options.update_options(node.conanfile.up_options)
         else:
-            if require.visible:
+            if visible:
                 down_options = node.conanfile.up_options
             elif not require.build:  # for requires in "host", like test_requires, pass myoptions
                 down_options = node.conanfile.private_up_options

conans/model/conan_file.py

@@ -47,8 +47,8 @@ class ConanFile:
     default_options = None
     default_build_options = None
     package_type = None
+    vendor = False
     languages = []
-
     implements = []
 
     provides = None
@@ -170,6 +170,7 @@ def serialize(self):
         result["label"] = self.display_name
         if self.info is not None:
             result["info"] = self.info.serialize()
+        result["vendor"] = self.vendor
         return result
 
     @property

conans/model/conf.py

@@ -83,6 +83,7 @@
     "tools.files.download:retry": "Number of retries in case of failure when downloading",
     "tools.files.download:retry_wait": "Seconds to wait between download attempts",
     "tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing",
+    "tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them",
     "tools.graph:skip_binaries": "Allow the graph to skip binaries not needed in the current configuration (True by default)",
     "tools.gnu:make_program": "Indicate path to make program",
     "tools.gnu:define_libcxx11_abi": "Force definition of GLIBCXX_USE_CXX11_ABI=1 for libstdc++11",

test/integration/command/info/info_test.py

@@ -4,7 +4,7 @@
 
 from conan.cli.exit_codes import ERROR_GENERAL
 from conans.model.recipe_ref import RecipeReference
-from conan.test.utils.tools import TestClient, GenConanfile, TurboTestClient
+from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, GenConanfile, TurboTestClient
 
 
 class TestBasicCliOutput:
@@ -425,3 +425,27 @@ class Pkg(ConanFile):
     assert "pkg/0.1@user" in c.out
     c.run("graph info . --channel=channel")
     assert "pkg/0.1@user/channel" in c.out
+
+
+def test_graph_info_bundle():
+    c = TestClient(light=True)
+    c.save({"subfolder/conanfile.py": GenConanfile("liba", "1.0")})
+    c.run("create ./subfolder")
+    conanfile = textwrap.dedent("""
+        from conan import ConanFile
+        class RepackageRecipe(ConanFile):
+            name = "lib"
+            version = "1.0"
+            def requirements(self):
+                self.requires("liba/1.0")
+            vendor = True
+        """)
+    c.save({"conanfile.py": conanfile})
+    c.run("create .")
+    c.save({"conanfile.py": GenConanfile("consumer", "1.0").with_requires("lib/1.0")})
+
+    c.run("graph info . --build='lib*'")
+    c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Invalid")})
+
+    c.run("graph info . -c tools.graph:vendor=build --build='lib*'")
+    c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Build")})

test/integration/command_v2/test_inspect.py

@@ -22,7 +22,9 @@ def test_basic_inspect():
                      "    shared: ['True', 'False']",
                      'package_type: None',
                      'requires: []',
-                     'revision_mode: hash']
+                     'revision_mode: hash',
+                     'vendor: False'
+                     ]
 
 
 def test_options_description():
@@ -92,6 +94,7 @@ def test_normal_inspect():
                                    'package_type: None',
                                    'requires: []',
                                    'revision_mode: hash',
+                                   'vendor: False',
                                    'version: 1.0']
 
 
@@ -137,7 +140,8 @@ class Pkg(ConanFile):
             "False, 'test': False, 'force': False, 'direct': True, 'build': "
             "False, 'transitive_headers': None, 'transitive_libs': None, 'headers': "
             "True, 'package_id_mode': None, 'visible': True}]",
-            'revision_mode: hash'] == tc.out.splitlines()
+            'revision_mode: hash',
+            'vendor: False'] == tc.out.splitlines()
 
 
 def test_pythonrequires_remote():

test/integration/test_package_vendor.py

@@ -0,0 +1,140 @@
+import os
+import textwrap
+
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.tools import TestClient
+
+
+def test_package_vendor():
+    c = TestClient()
+    app = textwrap.dedent("""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import copy, save
+
+        class App(ConanFile):
+            name = "app"
+            version = "0.1"
+            package_type = "application"
+            vendor = True
+            requires = "pkga/0.1"
+            def package(self):
+                copy(self, "*", src=self.dependencies["pkga"].package_folder,
+                     dst=self.package_folder)
+                save(self, os.path.join(self.package_folder, "app.exe"), "app")
+            """)
+
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+                                                            .with_package_file("pkga.dll", "dll"),
+            "app/conanfile.py": app
+            })
+    c.run("create pkga")
+    c.run("create app")  # -c tools.graph:vendor=build will be automatic
+    assert "app/0.1: package(): Packaged 1 '.dll' file: pkga.dll" in c.out
+
+    # we can safely remove pkga
+    c.run("remove pkg* -c")
+    c.run("list app:*")
+    assert "pkga" not in c.out  # The binary doesn't depend on pkga
+    c.run("install --requires=app/0.1 --deployer=full_deploy")
+    assert "pkga" not in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # we can create a modified pkga
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+           .with_package_file("pkga.dll", "newdll")})
+    c.run("create pkga")
+    # still using the re-packaged one
+    c.run("install --requires=app/0.1 --deployer=full_deploy")
+    assert "pkga" not in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # but we can force the expansion, still not the rebuild
+    c.run("install --requires=app/0.1 --deployer=full_deploy -c tools.graph:vendor=build")
+    assert "pkga" in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"
+
+    # and finally we can force the expansion and the rebuild
+    c.run("install --requires=app/0.1 --build=app* --deployer=full_deploy "
+          "-c tools.graph:vendor=build")
+    assert "pkga" in c.out
+    assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
+    assert c.load("full_deploy/host/app/0.1/pkga.dll") == "newdll"
+    # This shoulnd't happen, no visibility over transitive dependencies of app
+    assert not os.path.exists(os.path.join(c.current_folder, "full_deploy", "host", "pkga"))
+
+    # lets remove the binary
+    c.run("remove app:* -c")
+    c.run("install --requires=app/0.1", assert_error=True)
+    assert "Missing binary" in c.out
+    c.run("install --requires=app/0.1 --build=missing", assert_error=True)
+    assert "app/0.1: Invalid: The package 'app/0.1' is a vendoring one, needs to be built " \
+           "from source, but it didn't enable 'tools.graph:vendor=build'" in c.out
+
+    c.run("install --requires=app/0.1 --build=missing  -c tools.graph:vendor=build")
+    assert "pkga" in c.out  # it works
+
+
+def test_package_vendor_editable():
+    c = TestClient()
+    pkgb = textwrap.dedent("""
+        import os
+        from conan import ConanFile
+        from conan.tools.files import copy, save
+
+        class App(ConanFile):
+            name = "pkgb"
+            version = "0.1"
+            package_type = "shared-library"
+            vendor = True
+            requires = "pkga/0.1"
+            def layout(self):
+                self.folders.build = "build"
+                self.cpp.build.bindirs = ["build"]
+            def generate(self):
+                copy(self, "*", src=self.dependencies["pkga"].package_folder,
+                     dst=self.build_folder)
+            def build(self):
+                save(self, os.path.join(self.build_folder, "pkgb.dll"), "dll")
+            """)
+
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
+                                                            .with_package_file("bin/pkga.dll", "d"),
+            "pkgb/conanfile.py": pkgb,
+            "app/conanfile.py": GenConanfile("app", "0.1").with_settings("os")
+                                                          .with_requires("pkgb/0.1")
+            })
+    c.run("create pkga")
+    c.run("editable add pkgb")
+    c.run("install app -s os=Linux")
+    assert "pkga" in c.out
+    # The environment file of "app" doesn't have any visibility of the "pkga" paths
+    envfile_app = c.load("app/conanrunenv.sh")
+    assert "pkga" not in envfile_app
+    # But the environment file needed to build "pkgb" has visibility over the "pkga" paths
+    envfile_pkgb = c.load("pkgb/conanrunenv.sh")
+    assert "pkga" in envfile_pkgb
+
+
+def test_vendor_dont_propagate_options():
+    c = TestClient()
+    app = GenConanfile("app", "0.1").with_requires("pkga/0.1").with_class_attribute("vendor=True")
+    c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_shared_option(False),
+            "app/conanfile.py": app,
+            "consumer/conanfile.txt": "[requires]\napp/0.1",
+            "consumer_shared/conanfile.txt": "[requires]\napp/0.1\n[options]\n*:shared=True"
+            })
+    c.run("create pkga")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build")})
+    c.run("create app")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
+    c.run("install consumer --build=app/* -c tools.graph:vendor=build")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
+    c.run("install consumer_shared --build=app/* -c tools.graph:vendor=build")
+    c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
+                            "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})