Skip to content

Commit 2a5633b

Browse files
jneemtorhovland
andcommitted
Add source overlays
This adds a new mechanism for overlaying sources. An overlayed source returns packages from two (or more) sources. This functionality is not intended for public use, but it will be useful for packaging a workspace that contains inter-crate dependencies. Co-authored-by: Tor Hovland <55164+torhovland@users.noreply.github.com>
1 parent a9ee3e8 commit 2a5633b

File tree

3 files changed

+186
-4
lines changed

3 files changed

+186
-4
lines changed

src/cargo/sources/config.rs

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
//! sources to one another via the `replace-with` key in `.cargo/config`.
66
77
use crate::core::{GitReference, PackageId, SourceId};
8+
use crate::sources::overlay::DependencyConfusionThreatOverlaySource;
89
use crate::sources::source::Source;
910
use crate::sources::{ReplacedSource, CRATES_IO_REGISTRY};
1011
use crate::util::context::{self, ConfigRelativePath, OptValue};
@@ -24,6 +25,8 @@ pub struct SourceConfigMap<'gctx> {
2425
cfgs: HashMap<String, SourceConfig>,
2526
/// Mapping of [`SourceId`] to the source name.
2627
id2name: HashMap<SourceId, String>,
28+
/// Mapping of sources to local registries that will be overlaid on them.
29+
overlays: HashMap<SourceId, SourceId>,
2730
gctx: &'gctx GlobalContext,
2831
}
2932

@@ -81,6 +84,18 @@ impl<'gctx> SourceConfigMap<'gctx> {
8184
base.add_config(key, value)?;
8285
}
8386
}
87+
88+
Ok(base)
89+
}
90+
91+
/// Like [`SourceConfigMap::new`] but includes sources from source
92+
/// replacement configurations.
93+
pub fn new_with_overlays(
94+
gctx: &'gctx GlobalContext,
95+
overlays: impl IntoIterator<Item = (SourceId, SourceId)>,
96+
) -> CargoResult<SourceConfigMap<'gctx>> {
97+
let mut base = SourceConfigMap::new(gctx)?;
98+
base.overlays = overlays.into_iter().collect();
8499
Ok(base)
85100
}
86101

@@ -90,6 +105,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
90105
let mut base = SourceConfigMap {
91106
cfgs: HashMap::new(),
92107
id2name: HashMap::new(),
108+
overlays: HashMap::new(),
93109
gctx,
94110
};
95111
base.add(
@@ -136,7 +152,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
136152
debug!("loading: {}", id);
137153

138154
let Some(mut name) = self.id2name.get(&id) else {
139-
return id.load(self.gctx, yanked_whitelist);
155+
return self.load_overlaid(id, yanked_whitelist);
140156
};
141157
let mut cfg_loc = "";
142158
let orig_name = name;
@@ -161,7 +177,7 @@ impl<'gctx> SourceConfigMap<'gctx> {
161177
name = s;
162178
cfg_loc = c;
163179
}
164-
None if id == cfg.id => return id.load(self.gctx, yanked_whitelist),
180+
None if id == cfg.id => return self.load_overlaid(id, yanked_whitelist),
165181
None => {
166182
break cfg.id.with_precise_from(id);
167183
}
@@ -178,8 +194,8 @@ impl<'gctx> SourceConfigMap<'gctx> {
178194
}
179195
};
180196

181-
let new_src = new_id.load(
182-
self.gctx,
197+
let new_src = self.load_overlaid(
198+
new_id,
183199
&yanked_whitelist
184200
.iter()
185201
.map(|p| p.map_source(id, new_id))
@@ -215,6 +231,23 @@ restore the source replacement configuration to continue the build
215231
Ok(Box::new(ReplacedSource::new(id, new_id, new_src)))
216232
}
217233

234+
/// Gets the [`Source`] for a given [`SourceId`] without performing any source replacement.
235+
fn load_overlaid(
236+
&self,
237+
id: SourceId,
238+
yanked_whitelist: &HashSet<PackageId>,
239+
) -> CargoResult<Box<dyn Source + 'gctx>> {
240+
let src = id.load(self.gctx, yanked_whitelist)?;
241+
if let Some(overlay_id) = self.overlays.get(&id) {
242+
let overlay = overlay_id.load(self.gctx(), yanked_whitelist)?;
243+
Ok(Box::new(DependencyConfusionThreatOverlaySource::new(
244+
overlay, src,
245+
)))
246+
} else {
247+
Ok(src)
248+
}
249+
}
250+
218251
/// Adds a source config with an associated name.
219252
fn add(&mut self, name: &str, cfg: SourceConfig) -> CargoResult<()> {
220253
if let Some(old_name) = self.id2name.insert(cfg.id, name.to_string()) {

src/cargo/sources/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ pub use self::replaced::ReplacedSource;
3939
pub mod config;
4040
pub mod directory;
4141
pub mod git;
42+
pub mod overlay;
4243
pub mod path;
4344
pub mod registry;
4445
pub mod replaced;

src/cargo/sources/overlay.rs

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
use std::task::ready;
2+
3+
use tracing::debug;
4+
5+
use crate::sources::IndexSummary;
6+
7+
use super::source::{MaybePackage, Source};
8+
9+
/// A `Source` that overlays one source over another, pretending that the packages
10+
/// available in the overlay are actually available in the other one.
11+
///
12+
/// This is a massive footgun and a terrible idea, so we do not (and never will)
13+
/// expose this publicly. However, it is useful for some very specific private
14+
/// things, like locally verifying a bunch of packages at a time before any of
15+
/// them have been published.
16+
pub struct DependencyConfusionThreatOverlaySource<'gctx> {
17+
// The overlay source. The naming here comes from the main application of this,
18+
// where there is a remote registry that we overlay some local packages on.
19+
local: Box<dyn Source + 'gctx>,
20+
// The source we're impersonating.
21+
remote: Box<dyn Source + 'gctx>,
22+
}
23+
24+
impl<'gctx> DependencyConfusionThreatOverlaySource<'gctx> {
25+
pub fn new(local: Box<dyn Source + 'gctx>, remote: Box<dyn Source + 'gctx>) -> Self {
26+
debug!(
27+
"overlaying {} on {}",
28+
local.source_id().as_url(),
29+
remote.source_id().as_url()
30+
);
31+
Self { local, remote }
32+
}
33+
}
34+
35+
impl<'gctx> Source for DependencyConfusionThreatOverlaySource<'gctx> {
36+
fn source_id(&self) -> crate::core::SourceId {
37+
self.remote.source_id()
38+
}
39+
40+
fn supports_checksums(&self) -> bool {
41+
self.local.supports_checksums() && self.remote.supports_checksums()
42+
}
43+
44+
fn requires_precise(&self) -> bool {
45+
self.local.requires_precise() || self.remote.requires_precise()
46+
}
47+
48+
fn query(
49+
&mut self,
50+
dep: &crate::core::Dependency,
51+
kind: super::source::QueryKind,
52+
f: &mut dyn FnMut(super::IndexSummary),
53+
) -> std::task::Poll<crate::CargoResult<()>> {
54+
let local_source = self.local.source_id();
55+
let remote_source = self.remote.source_id();
56+
57+
let local_dep = dep.clone().map_source(remote_source, local_source);
58+
let mut local_packages = std::collections::HashSet::new();
59+
let mut local_callback = |index: IndexSummary| {
60+
let index = index.map_summary(|s| s.map_source(local_source, remote_source));
61+
local_packages.insert(index.as_summary().clone());
62+
f(index)
63+
};
64+
ready!(self.local.query(&local_dep, kind, &mut local_callback))?;
65+
66+
let mut package_collision = None;
67+
let mut remote_callback = |index: IndexSummary| {
68+
if local_packages.contains(index.as_summary()) {
69+
package_collision = Some(index.as_summary().clone());
70+
}
71+
f(index)
72+
};
73+
ready!(self.remote.query(dep, kind, &mut remote_callback))?;
74+
75+
if let Some(collision) = package_collision {
76+
std::task::Poll::Ready(Err(anyhow::anyhow!(
77+
"found a package in the remote registry and the local overlay: {}@{}",
78+
collision.name(),
79+
collision.version()
80+
)))
81+
} else {
82+
std::task::Poll::Ready(Ok(()))
83+
}
84+
}
85+
86+
fn invalidate_cache(&mut self) {
87+
self.local.invalidate_cache();
88+
self.remote.invalidate_cache();
89+
}
90+
91+
fn set_quiet(&mut self, quiet: bool) {
92+
self.local.set_quiet(quiet);
93+
self.remote.set_quiet(quiet);
94+
}
95+
96+
fn download(
97+
&mut self,
98+
package: crate::core::PackageId,
99+
) -> crate::CargoResult<super::source::MaybePackage> {
100+
let local_source = self.local.source_id();
101+
let remote_source = self.remote.source_id();
102+
103+
self.local
104+
.download(package.map_source(remote_source, local_source))
105+
.map(|maybe_pkg| match maybe_pkg {
106+
MaybePackage::Ready(pkg) => {
107+
MaybePackage::Ready(pkg.map_source(local_source, remote_source))
108+
}
109+
x => x,
110+
})
111+
.or_else(|_| self.remote.download(package))
112+
}
113+
114+
fn finish_download(
115+
&mut self,
116+
pkg_id: crate::core::PackageId,
117+
contents: Vec<u8>,
118+
) -> crate::CargoResult<crate::core::Package> {
119+
// The local registry should never return MaybePackage::Download from `download`, so any
120+
// downloads that need to be finished come from the remote registry.
121+
self.remote.finish_download(pkg_id, contents)
122+
}
123+
124+
fn fingerprint(&self, pkg: &crate::core::Package) -> crate::CargoResult<String> {
125+
Ok(pkg.package_id().version().to_string())
126+
}
127+
128+
fn describe(&self) -> String {
129+
self.remote.describe()
130+
}
131+
132+
fn add_to_yanked_whitelist(&mut self, pkgs: &[crate::core::PackageId]) {
133+
self.local.add_to_yanked_whitelist(pkgs);
134+
self.remote.add_to_yanked_whitelist(pkgs);
135+
}
136+
137+
fn is_yanked(
138+
&mut self,
139+
pkg: crate::core::PackageId,
140+
) -> std::task::Poll<crate::CargoResult<bool>> {
141+
self.remote.is_yanked(pkg)
142+
}
143+
144+
fn block_until_ready(&mut self) -> crate::CargoResult<()> {
145+
self.local.block_until_ready()?;
146+
self.remote.block_until_ready()
147+
}
148+
}

0 commit comments

Comments
 (0)