Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/uv-cache/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ uv-cache-key = { workspace = true }
uv-dirs = { workspace = true }
uv-distribution-types = { workspace = true }
uv-fs = { workspace = true, features = ["tokio"] }
uv-git-types = { workspace = true }
uv-normalize = { workspace = true }
uv-pypi-types = { workspace = true }
uv-redacted = { workspace = true }
Expand Down
11 changes: 8 additions & 3 deletions crates/uv-cache/src/wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf};

use uv_cache_key::{CanonicalUrl, cache_digest};
use uv_distribution_types::IndexUrl;
use uv_git_types::GitLfs;
use uv_redacted::DisplaySafeUrl;

/// Cache wheels and their metadata, both from remote wheels and built from source distributions.
Expand All @@ -15,11 +16,11 @@ pub enum WheelCache<'a> {
Path(&'a DisplaySafeUrl),
/// An editable dependency, which we key by URL.
Editable(&'a DisplaySafeUrl),
/// A Git dependency, which we key by URL and SHA.
/// A Git dependency, which we key by URL, SHA and LFS status.
///
/// Note that this variant only exists for source distributions; wheels can't be delivered
/// through Git.
Git(&'a DisplaySafeUrl, &'a str),
Git(&'a DisplaySafeUrl, &'a str, &'a GitLfs),
}

impl WheelCache<'_> {
Expand All @@ -39,10 +40,14 @@ impl WheelCache<'_> {
Self::Editable(url) => WheelCacheKind::Editable
.root()
.join(cache_digest(&CanonicalUrl::new(url))),
Self::Git(url, sha) => WheelCacheKind::Git
Self::Git(url, sha, GitLfs::Disabled) => WheelCacheKind::Git
.root()
.join(cache_digest(&CanonicalUrl::new(url)))
.join(sha),
Self::Git(url, sha, GitLfs::Enabled) => WheelCacheKind::Git
.root()
.join(cache_digest(&CanonicalUrl::new(url)))
.join(format!("{sha}_lfs")),
}
}

Expand Down
4 changes: 4 additions & 0 deletions crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3882,6 +3882,10 @@ pub struct AddArgs {
#[arg(long, group = "git-ref", action = clap::ArgAction::Set)]
pub branch: Option<String>,

/// Whether to use Git LFS when adding a dependency from Git.
#[arg(long, env = EnvVars::UV_GIT_LFS, value_parser = clap::builder::BoolishValueParser::new())]
pub lfs: bool,

/// Extras to enable for the dependency.
///
/// May be provided more than once.
Expand Down
13 changes: 12 additions & 1 deletion crates/uv-distribution-types/src/requirement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -856,6 +856,11 @@ impl From<RequirementSource> for RequirementSourceWire {
.append_pair("subdirectory", &subdirectory);
}

// Put lfs=true in the query when explicitly enabled.
if git.lfs().enabled() {
url.query_pairs_mut().append_pair("lfs", "true");
}

// Put the requested reference in the query.
match git.reference() {
GitReference::Branch(branch) => {
Expand Down Expand Up @@ -932,6 +937,7 @@ impl TryFrom<RequirementSourceWire> for RequirementSource {

let mut reference = GitReference::DefaultBranch;
let mut subdirectory: Option<PortablePathBuf> = None;
let mut lfs: bool = false;
for (key, val) in repository.query_pairs() {
match &*key {
"tag" => reference = GitReference::Tag(val.into_owned()),
Expand All @@ -940,6 +946,7 @@ impl TryFrom<RequirementSourceWire> for RequirementSource {
"subdirectory" => {
subdirectory = Some(PortablePathBuf::from(val.as_ref()));
}
"lfs" => lfs = matches!(val.to_lowercase().as_str(), "true"),
_ => {}
}
}
Expand All @@ -962,10 +969,14 @@ impl TryFrom<RequirementSourceWire> for RequirementSource {
if let Some(subdirectory) = subdirectory.as_ref() {
url.set_fragment(Some(&format!("subdirectory={subdirectory}")));
}
// Persist only when lfs support is explicitly requested
if lfs {
url.set_fragment(Some("lfs=true"));
}
let url = VerbatimUrl::from_url(url);

Ok(Self::Git {
git: GitUrl::from_fields(repository, reference, precise)?,
git: GitUrl::from_fields(repository, reference, precise, lfs.into())?,
subdirectory: subdirectory.map(Box::<Path>::from),
url,
})
Expand Down
2 changes: 2 additions & 0 deletions crates/uv-distribution/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ pub enum Error {
MissingPkgInfo,
#[error("The source distribution `{}` has no subdirectory `{}`", _0, _1.display())]
MissingSubdirectory(DisplaySafeUrl, PathBuf),
#[error("The source distribution is missing a Git LFS artifacts")]
MissingGitLfsArtifacts,
#[error("Failed to extract static metadata from `PKG-INFO`")]
PkgInfo(#[source] uv_pypi_types::MetadataError),
#[error("Failed to extract metadata from `requires.txt`")]
Expand Down
7 changes: 6 additions & 1 deletion crates/uv-distribution/src/index/built_wheel_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,12 @@ impl<'a> BuiltWheelIndex<'a> {

let cache_shard = self.cache.shard(
CacheBucket::SourceDistributions,
WheelCache::Git(&source_dist.url, git_sha.as_short_str()).root(),
WheelCache::Git(
&source_dist.url,
git_sha.as_short_str(),
&source_dist.git.lfs(),
)
.root(),
);

// If there are build settings, we need to scope to a cache shard.
Expand Down
11 changes: 10 additions & 1 deletion crates/uv-distribution/src/metadata/lowering.rs
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ impl LoweredRequirement {
rev,
tag,
branch,
lfs,
marker,
..
} => {
Expand All @@ -175,6 +176,7 @@ impl LoweredRequirement {
rev,
tag,
branch,
lfs,
)?;
(source, marker)
}
Expand Down Expand Up @@ -407,6 +409,7 @@ impl LoweredRequirement {
rev,
tag,
branch,
lfs,
marker,
..
} => {
Expand All @@ -416,6 +419,7 @@ impl LoweredRequirement {
rev,
tag,
branch,
lfs,
)?;
(source, marker)
}
Expand Down Expand Up @@ -580,6 +584,7 @@ fn git_source(
rev: Option<String>,
tag: Option<String>,
branch: Option<String>,
lfs: Option<bool>,
) -> Result<RequirementSource, LoweringError> {
let reference = match (rev, tag, branch) {
(None, None, None) => GitReference::DefaultBranch,
Expand All @@ -601,13 +606,17 @@ fn git_source(
.ok_or_else(|| LoweringError::NonUtf8Path(subdirectory.to_path_buf()))?;
url.set_fragment(Some(&format!("subdirectory={subdirectory}")));
}
// Persist only when lfs support is explicitly requested
if let Some(true) = lfs {
url.set_fragment(Some("lfs=true"));
}
let url = VerbatimUrl::from_url(url);

let repository = git.clone();

Ok(RequirementSource::Git {
url,
git: GitUrl::from_reference(repository, reference)?,
git: GitUrl::from_fields(repository, reference, None, lfs.into())?,
subdirectory,
})
}
Expand Down
6 changes: 3 additions & 3 deletions crates/uv-distribution/src/metadata/requires_dist.rs
Original file line number Diff line number Diff line change
Expand Up @@ -542,13 +542,13 @@ mod test {
tqdm = { git = "https://github.com/tqdm/tqdm", ref = "baaaaaab" }
"#};

assert_snapshot!(format_err(input).await, @r###"
assert_snapshot!(format_err(input).await, @r#"
error: TOML parse error at line 8, column 48
|
8 | tqdm = { git = "https://github.com/tqdm/tqdm", ref = "baaaaaab" }
| ^^^
unknown field `ref`, expected one of `git`, `subdirectory`, `rev`, `tag`, `branch`, `url`, `path`, `editable`, `package`, `index`, `workspace`, `marker`, `extra`, `group`
"###);
unknown field `ref`, expected one of `git`, `subdirectory`, `rev`, `tag`, `branch`, `lfs`, `url`, `path`, `editable`, `package`, `index`, `workspace`, `marker`, `extra`, `group`
"#);
}

#[tokio::test]
Expand Down
16 changes: 13 additions & 3 deletions crates/uv-distribution/src/source/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1565,10 +1565,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
}

// Validate that LFS artifacts were fully initialized
if resource.git.lfs().enabled() && !fetch.lfs_ready() {
return Err(Error::MissingGitLfsArtifacts);
}

let git_sha = fetch.git().precise().expect("Exact commit after checkout");
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
WheelCache::Git(resource.url, git_sha.as_short_str()).root(),
WheelCache::Git(resource.url, git_sha.as_short_str(), &resource.git.lfs()).root(),
);
let metadata_entry = cache_shard.entry(METADATA);

Expand Down Expand Up @@ -1667,7 +1672,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.map(|oid| {
self.build_context.cache().shard(
CacheBucket::SourceDistributions,
WheelCache::Git(resource.url, oid.as_short_str()).root(),
WheelCache::Git(resource.url, oid.as_short_str(), &resource.git.lfs()).root(),
)
});
if cache_shard
Expand Down Expand Up @@ -1769,10 +1774,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
}
}

// Validate that LFS artifacts were fully initialized
if resource.git.lfs().enabled() && !fetch.lfs_ready() {
return Err(Error::MissingGitLfsArtifacts);
}

let git_sha = fetch.git().precise().expect("Exact commit after checkout");
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
WheelCache::Git(resource.url, git_sha.as_short_str()).root(),
WheelCache::Git(resource.url, git_sha.as_short_str(), &resource.git.lfs()).root(),
);
let metadata_entry = cache_shard.entry(METADATA);

Expand Down
1 change: 1 addition & 0 deletions crates/uv-git-types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ workspace = true

[dependencies]
uv-redacted = { workspace = true }
uv-static = { workspace = true }

serde = { workspace = true }
thiserror = { workspace = true }
Expand Down
68 changes: 65 additions & 3 deletions crates/uv-git-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,54 @@ pub use crate::reference::GitReference;

use thiserror::Error;
use uv_redacted::DisplaySafeUrl;
use uv_static::EnvVars;

mod github;
mod oid;
mod reference;

/// Configuration for Git LFS (Large File Storage) support.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)]
pub enum GitLfs {
/// Git LFS is disabled (default).
#[default]
Disabled,
/// Git LFS is enabled.
Enabled,
}

impl GitLfs {
/// Create a `GitLfs` configuration from environment variables.
pub fn from_env() -> Self {
if std::env::var(EnvVars::UV_GIT_LFS).is_ok() {
Self::Enabled
} else {
Self::Disabled
}
}

/// Returns true if LFS is enabled.
pub fn enabled(self) -> bool {
matches!(self, Self::Enabled)
}
}

impl From<Option<bool>> for GitLfs {
fn from(value: Option<bool>) -> Self {
match value {
Some(true) => Self::Enabled,
Some(false) => Self::Disabled,
None => Self::from_env(),
}
}
}

impl From<bool> for GitLfs {
fn from(value: bool) -> Self {
if value { Self::Enabled } else { Self::Disabled }
}
}

#[derive(Debug, Error)]
pub enum GitUrlParseError {
#[error(
Expand All @@ -27,31 +70,36 @@ pub struct GitUrl {
reference: GitReference,
/// The precise commit to use, if known.
precise: Option<GitOid>,
/// Git LFS configuration for this repository.
lfs: GitLfs,
}

impl GitUrl {
/// Create a new [`GitUrl`] from a repository URL and a reference.
pub fn from_reference(
repository: DisplaySafeUrl,
reference: GitReference,
lfs: GitLfs,
) -> Result<Self, GitUrlParseError> {
Self::from_fields(repository, reference, None)
Self::from_fields(repository, reference, None, lfs)
}

/// Create a new [`GitUrl`] from a repository URL and a precise commit.
pub fn from_commit(
repository: DisplaySafeUrl,
reference: GitReference,
precise: GitOid,
lfs: GitLfs,
) -> Result<Self, GitUrlParseError> {
Self::from_fields(repository, reference, Some(precise))
Self::from_fields(repository, reference, Some(precise), lfs)
}

/// Create a new [`GitUrl`] from a repository URL and a precise commit, if known.
pub fn from_fields(
repository: DisplaySafeUrl,
reference: GitReference,
precise: Option<GitOid>,
lfs: GitLfs,
) -> Result<Self, GitUrlParseError> {
match repository.scheme() {
"http" | "https" | "ssh" | "file" => {}
Expand All @@ -66,6 +114,7 @@ impl GitUrl {
repository,
reference,
precise,
lfs,
})
}

Expand Down Expand Up @@ -97,6 +146,18 @@ impl GitUrl {
pub fn precise(&self) -> Option<GitOid> {
self.precise
}

/// Return the Git LFS configuration.
pub fn lfs(&self) -> GitLfs {
self.lfs
}

/// Set the Git LFS configuration.
#[must_use]
pub fn with_lfs(mut self, lfs: GitLfs) -> Self {
self.lfs = lfs;
self
}
}

impl TryFrom<DisplaySafeUrl> for GitUrl {
Expand All @@ -120,7 +181,8 @@ impl TryFrom<DisplaySafeUrl> for GitUrl {
url.set_path(&prefix);
}

Self::from_reference(url, reference)
// TODO(samypr100): GitLfs::from_env() for now unless we want to support additional query params
Self::from_reference(url, reference, GitLfs::from_env())
}
}

Expand Down
Loading
Loading