From 8726887da46086caec85a021595e86db404380b4 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 27 Aug 2025 15:17:20 -0500 Subject: [PATCH 1/7] Add `lfs = true` support in Git source declarations --- .../uv-distribution-types/src/requirement.rs | 4 +- .../uv-distribution/src/metadata/lowering.rs | 7 +- .../src/metadata/requires_dist.rs | 6 +- crates/uv-git-types/src/lib.rs | 62 +++++- crates/uv-git/src/git.rs | 10 +- crates/uv-git/src/source.rs | 1 + ...xt__test__line-endings-whitespace.txt.snap | 1 + ...ments_txt__test__parse-whitespace.txt.snap | 1 + crates/uv-resolver/src/lock/mod.rs | 7 +- crates/uv-workspace/src/pyproject.rs | 8 + crates/uv/src/commands/project/add.rs | 2 + crates/uv/tests/it/sync.rs | 207 ++++++++++++++++++ uv.schema.json | 7 + 13 files changed, 308 insertions(+), 15 deletions(-) diff --git a/crates/uv-distribution-types/src/requirement.rs b/crates/uv-distribution-types/src/requirement.rs index 25676d999ecd6..543c7e1558d16 100644 --- a/crates/uv-distribution-types/src/requirement.rs +++ b/crates/uv-distribution-types/src/requirement.rs @@ -7,7 +7,7 @@ use thiserror::Error; use uv_cache_key::{CacheKey, CacheKeyHasher}; use uv_distribution_filename::DistExtension; use uv_fs::{CWD, PortablePath, PortablePathBuf, relative_to}; -use uv_git_types::{GitOid, GitReference, GitUrl, GitUrlParseError, OidParseError}; +use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError, OidParseError}; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::VersionSpecifiers; use uv_pep508::{ @@ -965,7 +965,7 @@ impl TryFrom for RequirementSource { let url = VerbatimUrl::from_url(url); Ok(Self::Git { - git: GitUrl::from_fields(repository, reference, precise)?, + git: GitUrl::from_fields(repository, reference, precise, GitLfs::from_env())?, subdirectory: subdirectory.map(Box::::from), url, }) diff --git a/crates/uv-distribution/src/metadata/lowering.rs b/crates/uv-distribution/src/metadata/lowering.rs index 627e2d4dd14a6..5952eb54275a1 100644 --- a/crates/uv-distribution/src/metadata/lowering.rs +++ b/crates/uv-distribution/src/metadata/lowering.rs @@ -166,6 +166,7 @@ impl LoweredRequirement { rev, tag, branch, + lfs, marker, .. } => { @@ -175,6 +176,7 @@ impl LoweredRequirement { rev, tag, branch, + lfs, )?; (source, marker) } @@ -407,6 +409,7 @@ impl LoweredRequirement { rev, tag, branch, + lfs, marker, .. } => { @@ -416,6 +419,7 @@ impl LoweredRequirement { rev, tag, branch, + lfs, )?; (source, marker) } @@ -580,6 +584,7 @@ fn git_source( rev: Option, tag: Option, branch: Option, + lfs: Option, ) -> Result { let reference = match (rev, tag, branch) { (None, None, None) => GitReference::DefaultBranch, @@ -607,7 +612,7 @@ fn git_source( Ok(RequirementSource::Git { url, - git: GitUrl::from_reference(repository, reference)?, + git: GitUrl::from_fields(repository, reference, None, lfs.into())?, subdirectory, }) } diff --git a/crates/uv-distribution/src/metadata/requires_dist.rs b/crates/uv-distribution/src/metadata/requires_dist.rs index eb0c007f992e5..f9482d977818b 100644 --- a/crates/uv-distribution/src/metadata/requires_dist.rs +++ b/crates/uv-distribution/src/metadata/requires_dist.rs @@ -542,13 +542,13 @@ mod test { tqdm = { git = "https://github.com/tqdm/tqdm", ref = "baaaaaab" } "#}; - assert_snapshot!(format_err(input).await, @r###" + assert_snapshot!(format_err(input).await, @r#" error: TOML parse error at line 8, column 48 | 8 | tqdm = { git = "https://github.com/tqdm/tqdm", ref = "baaaaaab" } | ^^^ - unknown field `ref`, expected one of `git`, `subdirectory`, `rev`, `tag`, `branch`, `url`, `path`, `editable`, `package`, `index`, `workspace`, `marker`, `extra`, `group` - "###); + unknown field `ref`, expected one of `git`, `subdirectory`, `rev`, `tag`, `branch`, `lfs`, `url`, `path`, `editable`, `package`, `index`, `workspace`, `marker`, `extra`, `group` + "#); } #[tokio::test] diff --git a/crates/uv-git-types/src/lib.rs b/crates/uv-git-types/src/lib.rs index dbfa02ea31f25..b6c1a046debc1 100644 --- a/crates/uv-git-types/src/lib.rs +++ b/crates/uv-git-types/src/lib.rs @@ -9,6 +9,48 @@ mod github; mod oid; mod reference; +/// Configuration for Git LFS (Large File Storage) support. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Default)] +pub enum GitLfs { + /// Git LFS is disabled (default). + #[default] + Disabled, + /// Git LFS is enabled. + Enabled, +} + +impl GitLfs { + /// Create a `GitLfs` configuration from environment variables. + pub fn from_env() -> Self { + if std::env::var("UV_GIT_LFS").is_ok() { + Self::Enabled + } else { + Self::Disabled + } + } + + /// Returns true if LFS is enabled. + pub fn enabled(self) -> bool { + matches!(self, Self::Enabled) + } +} + +impl From> for GitLfs { + fn from(value: Option) -> Self { + match value { + Some(true) => Self::Enabled, + Some(false) => Self::Disabled, + None => Self::from_env(), + } + } +} + +impl From for GitLfs { + fn from(value: bool) -> Self { + if value { Self::Enabled } else { Self::Disabled } + } +} + #[derive(Debug, Error)] pub enum GitUrlParseError { #[error( @@ -27,6 +69,8 @@ pub struct GitUrl { reference: GitReference, /// The precise commit to use, if known. precise: Option, + /// Git LFS configuration for this repository. + lfs: GitLfs, } impl GitUrl { @@ -35,7 +79,7 @@ impl GitUrl { repository: DisplaySafeUrl, reference: GitReference, ) -> Result { - Self::from_fields(repository, reference, None) + Self::from_fields(repository, reference, None, GitLfs::from_env()) } /// Create a new [`GitUrl`] from a repository URL and a precise commit. @@ -44,7 +88,7 @@ impl GitUrl { reference: GitReference, precise: GitOid, ) -> Result { - Self::from_fields(repository, reference, Some(precise)) + Self::from_fields(repository, reference, Some(precise), GitLfs::from_env()) } /// Create a new [`GitUrl`] from a repository URL and a precise commit, if known. @@ -52,6 +96,7 @@ impl GitUrl { repository: DisplaySafeUrl, reference: GitReference, precise: Option, + lfs: GitLfs, ) -> Result { match repository.scheme() { "http" | "https" | "ssh" | "file" => {} @@ -66,6 +111,7 @@ impl GitUrl { repository, reference, precise, + lfs, }) } @@ -97,6 +143,18 @@ impl GitUrl { pub fn precise(&self) -> Option { self.precise } + + /// Return the Git LFS configuration. + pub fn lfs(&self) -> GitLfs { + self.lfs + } + + /// Set the Git LFS configuration. + #[must_use] + pub fn with_lfs(mut self, lfs: GitLfs) -> Self { + self.lfs = lfs; + self + } } impl TryFrom for GitUrl { diff --git a/crates/uv-git/src/git.rs b/crates/uv-git/src/git.rs index 4b37a52862946..691e00a5fb58d 100644 --- a/crates/uv-git/src/git.rs +++ b/crates/uv-git/src/git.rs @@ -1,7 +1,6 @@ //! Git support is derived from Cargo's implementation. //! Cargo is dual-licensed under either Apache 2.0 or MIT, at the user's choice. //! Source: -use std::env; use std::fmt::Display; use std::path::{Path, PathBuf}; use std::str::{self}; @@ -15,7 +14,7 @@ use tracing::{debug, warn}; use url::Url; use uv_fs::Simplified; -use uv_git_types::{GitHubRepository, GitOid, GitReference}; +use uv_git_types::{GitHubRepository, GitLfs, GitOid, GitReference}; use uv_redacted::DisplaySafeUrl; use uv_static::EnvVars; use uv_version::version; @@ -238,12 +237,11 @@ impl GitRemote { client: &ClientWithMiddleware, disable_ssl: bool, offline: bool, + lfs: GitLfs, ) -> Result<(GitDatabase, GitOid)> { let reference = locked_rev .map(ReferenceOrOid::Oid) .unwrap_or(ReferenceOrOid::Reference(reference)); - let enable_lfs_fetch = env::var(EnvVars::UV_GIT_LFS).is_ok(); - if let Some(mut db) = db { fetch( &mut db.repo, @@ -261,7 +259,7 @@ impl GitRemote { }; if let Some(rev) = resolved_commit_hash { - if enable_lfs_fetch { + if lfs.enabled() { fetch_lfs(&mut db.repo, &self.url, &rev, disable_ssl) .with_context(|| format!("failed to fetch LFS objects at {rev}"))?; } @@ -293,7 +291,7 @@ impl GitRemote { Some(rev) => rev, None => reference.resolve(&repo)?, }; - if enable_lfs_fetch { + if lfs.enabled() { fetch_lfs(&mut repo, &self.url, &rev, disable_ssl) .with_context(|| format!("failed to fetch LFS objects at {rev}"))?; } diff --git a/crates/uv-git/src/source.rs b/crates/uv-git/src/source.rs index cb6d0a24f3941..30958f74136e9 100644 --- a/crates/uv-git/src/source.rs +++ b/crates/uv-git/src/source.rs @@ -135,6 +135,7 @@ impl GitSource { &self.client, self.disable_ssl, self.offline, + self.git.lfs(), )?; Ok((db, actual_rev, task)) diff --git a/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__line-endings-whitespace.txt.snap b/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__line-endings-whitespace.txt.snap index 45b1cc43f3043..a01b3b7158edb 100644 --- a/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__line-endings-whitespace.txt.snap +++ b/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__line-endings-whitespace.txt.snap @@ -56,6 +56,7 @@ RequirementsTxt { }, reference: DefaultBranch, precise: None, + lfs: Disabled, }, subdirectory: None, }, diff --git a/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__parse-whitespace.txt.snap b/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__parse-whitespace.txt.snap index 45b1cc43f3043..a01b3b7158edb 100644 --- a/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__parse-whitespace.txt.snap +++ b/crates/uv-requirements-txt/src/snapshots/uv_requirements_txt__test__parse-whitespace.txt.snap @@ -56,6 +56,7 @@ RequirementsTxt { }, reference: DefaultBranch, precise: None, + lfs: Disabled, }, subdirectory: None, }, diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index 2543d178ff229..a47f37bc4321a 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -5061,7 +5061,12 @@ fn normalize_requirement( repository.set_fragment(None); repository.set_query(None); - GitUrl::from_fields(repository, git.reference().clone(), git.precise())? + GitUrl::from_fields( + repository, + git.reference().clone(), + git.precise(), + git.lfs(), + )? }; // Reconstruct the PEP 508 URL from the underlying data. diff --git a/crates/uv-workspace/src/pyproject.rs b/crates/uv-workspace/src/pyproject.rs index cd643d5fefe9e..f629b28e9ed24 100644 --- a/crates/uv-workspace/src/pyproject.rs +++ b/crates/uv-workspace/src/pyproject.rs @@ -1091,6 +1091,8 @@ pub enum Source { rev: Option, tag: Option, branch: Option, + /// Whether to use Git LFS when cloning the repository. + lfs: Option, #[serde( skip_serializing_if = "uv_pep508::marker::ser::is_empty", serialize_with = "uv_pep508::marker::ser::serialize", @@ -1189,6 +1191,7 @@ impl<'de> Deserialize<'de> for Source { rev: Option, tag: Option, branch: Option, + lfs: Option, url: Option, path: Option, editable: Option, @@ -1212,6 +1215,7 @@ impl<'de> Deserialize<'de> for Source { rev, tag, branch, + lfs, url, path, editable, @@ -1289,6 +1293,7 @@ impl<'de> Deserialize<'de> for Source { rev, tag, branch, + lfs, marker, extra, group, @@ -1598,6 +1603,7 @@ impl Source { rev, tag, branch, + lfs: None, marker: *marker, extra: extra.clone(), group: group.clone(), @@ -1707,6 +1713,7 @@ impl Source { rev: rev.cloned(), tag, branch, + lfs: None, git: git.repository().clone(), subdirectory: subdirectory.map(PortablePathBuf::from), marker: MarkerTree::TRUE, @@ -1718,6 +1725,7 @@ impl Source { rev, tag, branch, + lfs: None, git: git.repository().clone(), subdirectory: subdirectory.map(PortablePathBuf::from), marker: MarkerTree::TRUE, diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index c449416e41c61..ced42192b814b 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -843,6 +843,7 @@ fn edits( rev, tag, branch, + lfs, marker, extra, group, @@ -861,6 +862,7 @@ fn edits( rev, tag, branch, + lfs, marker, extra, group, diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index 7347d8fb6743b..8a4abffba4622 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -13473,6 +13473,213 @@ fn reject_unmatched_runtime() -> Result<()> { Ok(()) } +/// Test Git LFS configuration. +#[test] +#[cfg(feature = "git")] +fn sync_git_lfs() -> Result<()> { + let context = TestContext::new("3.13"); + + // Set `lfs = true` in the source + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "test-project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = ["test-lfs-repo"] + + [tool.uv.sources] + test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git", lfs = true } + "#, + )?; + + uv_snapshot!(context.filters(), context.sync(), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + "); + + // Verify that we can import the module and access LFS content + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + // `UV_GIT_LFS=false` should not override `lfs = true` + uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "false").arg("--reinstall").arg("--no-cache"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + // Set `lfs = false` in the source + pyproject_toml.write_str( + r#" + [project] + name = "test-project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = ["test-lfs-repo"] + + [tool.uv.sources] + test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git", lfs = false } + "#, + )?; + + uv_snapshot!(context.filters(), context.sync().arg("--reinstall").arg("--no-cache"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + "); + + // Verify that LFS content is missing (import should fail) + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + // `UV_GIT_LFS=true` should not override `lfs = false` + uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "true").arg("--reinstall").arg("--no-cache"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + // `UV_GIT_LFS = true` should work without explicit lfs flag + pyproject_toml.write_str( + r#" + [project] + name = "test-project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = ["test-lfs-repo"] + + [tool.uv.sources] + test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git" } + "#, + )?; + + uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "true").arg("--reinstall").arg("--no-cache"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + "); + + // Verify that we can import the module when UV_GIT_LFS is set + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module; print('LFS module imported via env var')"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module; print('LFS module imported via env var') + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + Ok(()) +} + #[test] fn match_runtime_optional() -> Result<()> { let context = TestContext::new("3.12").with_exclude_newer("2025-01-01T00:00Z"); diff --git a/uv.schema.json b/uv.schema.json index 221c630e2eef7..0f7a89a928dd3 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -1964,6 +1964,13 @@ } ] }, + "lfs": { + "description": "Whether to use Git LFS when cloning the repository.", + "type": [ + "boolean", + "null" + ] + }, "marker": { "$ref": "#/definitions/MarkerTree" }, From 9f43d2b4397e43afa2c689d3d5a222e91f2d14df Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:00:56 -0400 Subject: [PATCH 2/7] chore: basic cleanup --- Cargo.lock | 1 + crates/uv-git-types/Cargo.toml | 1 + crates/uv-git-types/src/lib.rs | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index ad2372778962f..e4e4555b6dd19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6024,6 +6024,7 @@ dependencies = [ "tracing", "url", "uv-redacted", + "uv-static", ] [[package]] diff --git a/crates/uv-git-types/Cargo.toml b/crates/uv-git-types/Cargo.toml index a374d7cbea5e5..66c1ee4551919 100644 --- a/crates/uv-git-types/Cargo.toml +++ b/crates/uv-git-types/Cargo.toml @@ -17,6 +17,7 @@ workspace = true [dependencies] uv-redacted = { workspace = true } +uv-static = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } diff --git a/crates/uv-git-types/src/lib.rs b/crates/uv-git-types/src/lib.rs index b6c1a046debc1..bfcf33130d2ce 100644 --- a/crates/uv-git-types/src/lib.rs +++ b/crates/uv-git-types/src/lib.rs @@ -4,6 +4,7 @@ pub use crate::reference::GitReference; use thiserror::Error; use uv_redacted::DisplaySafeUrl; +use uv_static::EnvVars; mod github; mod oid; @@ -22,7 +23,7 @@ pub enum GitLfs { impl GitLfs { /// Create a `GitLfs` configuration from environment variables. pub fn from_env() -> Self { - if std::env::var("UV_GIT_LFS").is_ok() { + if std::env::var(EnvVars::UV_GIT_LFS).is_ok() { Self::Enabled } else { Self::Disabled From 87d3fe344d3e61085bcd00e074f1335e29f9a4ee Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:03:15 -0400 Subject: [PATCH 3/7] fix: ensure lfs state is propagated in the git database and checkouts --- crates/uv-git/src/git.rs | 117 ++++++++++++++++++++++++------- crates/uv-git/src/resolver.rs | 13 ++-- crates/uv-git/src/source.rs | 35 ++++++--- crates/uv-static/src/env_vars.rs | 4 ++ 4 files changed, 132 insertions(+), 37 deletions(-) diff --git a/crates/uv-git/src/git.rs b/crates/uv-git/src/git.rs index 691e00a5fb58d..12b3c7a9f0ab5 100644 --- a/crates/uv-git/src/git.rs +++ b/crates/uv-git/src/git.rs @@ -14,7 +14,7 @@ use tracing::{debug, warn}; use url::Url; use uv_fs::Simplified; -use uv_git_types::{GitHubRepository, GitLfs, GitOid, GitReference}; +use uv_git_types::{GitHubRepository, GitOid, GitReference}; use uv_redacted::DisplaySafeUrl; use uv_static::EnvVars; use uv_version::version; @@ -25,6 +25,10 @@ use crate::rate_limit::{GITHUB_RATE_LIMIT_STATUS, is_github_rate_limited}; /// checkout is ready to go. See [`GitCheckout::reset`] for why we need this. const CHECKOUT_READY_LOCK: &str = ".ok"; +/// A file indicates that if present, `git lfs` artifacts have been smudged +/// as part of a revision. See [`GitCheckout::reset`] for why we need this. +const LFS_READY_INDICATOR: &str = ".ok_lfs"; + #[derive(Debug, thiserror::Error)] pub enum GitError { #[error("Git executable not found. Ensure that Git is installed and available.")] @@ -142,6 +146,8 @@ pub(crate) struct GitRemote { pub(crate) struct GitDatabase { /// Underlying Git repository instance for this database. repo: GitRepository, + /// Git LFS artifacts have been initialized (if requested). + lfs_ready: bool, } /// A local checkout of a particular revision from a [`GitRepository`]. @@ -203,6 +209,31 @@ impl GitRepository { result.truncate(result.trim_end().len()); Ok(result.parse()?) } + + /// Verifies LFS artifacts have been initialized for a given `refname`. + fn lfs_fsck_objects(&self, refname: &str) -> bool { + let mut cmd = if let Ok(lfs) = GIT_LFS.as_ref() { + lfs.clone() + } else { + warn!("Git LFS is not available, skipping LFS fetch"); + return false; + }; + + let result = cmd + .arg("fsck") + .arg("--objects") + .arg(refname) + .cwd(&self.path) + .exec_with_output(); + + match result { + Ok(_) => true, + Err(err) => { + debug!("Git LFS validation failed: {err}"); + false + } + } + } } impl GitRemote { @@ -237,7 +268,7 @@ impl GitRemote { client: &ClientWithMiddleware, disable_ssl: bool, offline: bool, - lfs: GitLfs, + with_lfs: bool, ) -> Result<(GitDatabase, GitOid)> { let reference = locked_rev .map(ReferenceOrOid::Oid) @@ -259,9 +290,10 @@ impl GitRemote { }; if let Some(rev) = resolved_commit_hash { - if lfs.enabled() { - fetch_lfs(&mut db.repo, &self.url, &rev, disable_ssl) + if with_lfs { + let lfs_ready = fetch_lfs(&mut db.repo, &self.url, &rev, disable_ssl) .with_context(|| format!("failed to fetch LFS objects at {rev}"))?; + db = db.with_lfs_ready(lfs_ready); } return Ok((db, rev)); } @@ -291,19 +323,25 @@ impl GitRemote { Some(rev) => rev, None => reference.resolve(&repo)?, }; - if lfs.enabled() { - fetch_lfs(&mut repo, &self.url, &rev, disable_ssl) - .with_context(|| format!("failed to fetch LFS objects at {rev}"))?; - } - - Ok((GitDatabase { repo }, rev)) + let lfs_ready = with_lfs + .then(|| { + fetch_lfs(&mut repo, &self.url, &rev, disable_ssl) + .with_context(|| format!("failed to fetch LFS objects at {rev}")) + }) + .transpose()? + .unwrap_or(false); + + Ok((GitDatabase { repo, lfs_ready }, rev)) } /// Creates a [`GitDatabase`] of this remote at `db_path`. #[allow(clippy::unused_self)] pub(crate) fn db_at(&self, db_path: &Path) -> Result { let repo = GitRepository::open(db_path)?; - Ok(GitDatabase { repo }) + Ok(GitDatabase { + repo, + lfs_ready: false, + }) } } @@ -317,7 +355,7 @@ impl GitDatabase { let checkout = match GitRepository::open(destination) .ok() .map(|repo| GitCheckout::new(rev, repo)) - .filter(GitCheckout::is_fresh) + .filter(|co| co.is_fresh(self.lfs_ready)) { Some(co) => co, None => GitCheckout::clone_into(destination, self, rev)?, @@ -343,6 +381,23 @@ impl GitDatabase { pub(crate) fn contains(&self, oid: GitOid) -> bool { self.repo.rev_parse(&format!("{oid}^0")).is_ok() } + + /// Checks if `oid` contains necessary LFS artifacts in this database. + pub(crate) fn contains_lfs_artifacts(&self, oid: GitOid) -> bool { + self.repo.lfs_fsck_objects(&format!("{oid}^0")) + } + + /// Indicates Git LFS artifacts have been initialized. + pub(crate) fn lfs_ready(&self) -> bool { + self.lfs_ready + } + + /// Set the Git LFS configuration. + #[must_use] + pub(crate) fn with_lfs_ready(mut self, lfs: bool) -> Self { + self.lfs_ready = lfs; + self + } } impl GitCheckout { @@ -391,16 +446,18 @@ impl GitCheckout { let repo = GitRepository::open(into)?; let checkout = Self::new(revision, repo); - checkout.reset()?; + checkout.reset(database.lfs_ready)?; Ok(checkout) } /// Checks if the `HEAD` of this checkout points to the expected revision. - fn is_fresh(&self) -> bool { + fn is_fresh(&self, with_lfs: bool) -> bool { match self.repo.rev_parse("HEAD") { Ok(id) if id == self.revision => { // See comments in reset() for why we check this - self.repo.path.join(CHECKOUT_READY_LOCK).exists() + let ok_exists = self.repo.path.join(CHECKOUT_READY_LOCK).exists(); + // We additionally check if LFS is initialized for this checkout (when requested) + ok_exists && (!with_lfs || self.repo.path.join(LFS_READY_INDICATOR).exists()) } _ => false, } @@ -410,18 +467,25 @@ impl GitCheckout { /// additional interrupt protection by a dummy file [`CHECKOUT_READY_LOCK`]. /// /// If we're interrupted while performing a `git reset` (e.g., we die - /// because of a signal) Cargo needs to be sure to try to check out this + /// because of a signal) uv needs to be sure to try to check out this /// repo again on the next go-round. /// - /// To enable this we have a dummy file in our checkout, [`.cargo-ok`], - /// which if present means that the repo has been successfully reset and is - /// ready to go. Hence if we start to do a reset, we make sure this file + /// To enable this we have a dummy file in our checkout, [`.ok`], + /// and optionally [`.ok_lfs`] for when Git LFS is enabled + /// which when present means that the repo has been successfully reset and is + /// ready to go. Hence, if we start to do a reset, we make sure these files /// *doesn't* exist, and then once we're done we create the file. /// - /// [`.cargo-ok`]: CHECKOUT_READY_LOCK - fn reset(&self) -> Result<()> { + /// [`.ok`]: CHECKOUT_READY_LOCK + /// [`.ok_lfs`]: LFS_READY_INDICATOR + fn reset(&self, with_lfs: bool) -> Result<()> { let ok_file = self.repo.path.join(CHECKOUT_READY_LOCK); let _ = paths::remove_file(&ok_file); + + // We want to skip smudge if lfs was disabled for that repository + // as smudge filters can trigger on a reset even if lfs artifacts + // were not originally "fetched". + let lfs_skip_smudge = if with_lfs { "0" } else { "1" }; debug!("Reset {} to {}", self.repo.path.display(), self.revision); // Perform the hard reset. @@ -429,6 +493,7 @@ impl GitCheckout { .arg("reset") .arg("--hard") .arg(self.revision.as_str()) + .env(EnvVars::GIT_LFS_SKIP_SMUDGE, lfs_skip_smudge) .cwd(&self.repo.path) .exec_with_output()?; @@ -438,6 +503,7 @@ impl GitCheckout { .arg("update") .arg("--recursive") .arg("--init") + .env(EnvVars::GIT_LFS_SKIP_SMUDGE, lfs_skip_smudge) .cwd(&self.repo.path) .exec_with_output() .map(drop)?; @@ -673,14 +739,14 @@ fn fetch_lfs( url: &Url, revision: &GitOid, disable_ssl: bool, -) -> Result<()> { +) -> Result { let mut cmd = if let Ok(lfs) = GIT_LFS.as_ref() { debug!("Fetching Git LFS objects"); lfs.clone() } else { // Since this feature is opt-in, warn if not available warn!("Git LFS is not available, skipping LFS fetch"); - return Ok(()); + return Ok(false); }; if disable_ssl { @@ -697,10 +763,13 @@ fn fetch_lfs( .env_remove(EnvVars::GIT_INDEX_FILE) .env_remove(EnvVars::GIT_OBJECT_DIRECTORY) .env_remove(EnvVars::GIT_ALTERNATE_OBJECT_DIRECTORIES) + // We should not support requesting LFS artifacts but skipping smudging artifacts. + .env_remove(EnvVars::GIT_LFS_SKIP_SMUDGE) .cwd(&repo.path); cmd.exec_with_output()?; - Ok(()) + + Ok(true) } /// The result of GitHub fast path check. See [`github_fast_path`] for more. diff --git a/crates/uv-git/src/resolver.rs b/crates/uv-git/src/resolver.rs index e6e6e5ccde60c..61f1f07a23ac6 100644 --- a/crates/uv-git/src/resolver.rs +++ b/crates/uv-git/src/resolver.rs @@ -167,11 +167,14 @@ impl GitResolver { let lock_dir = cache.join("locks"); fs::create_dir_all(&lock_dir).await?; let repository_url = RepositoryUrl::new(url.repository()); - let _lock = LockedFile::acquire( - lock_dir.join(cache_digest(&repository_url)), - &repository_url, - ) - .await?; + let lock_ident = { + let mut digest = cache_digest(&repository_url); + if url.lfs().enabled() { + digest.push_str("_lfs"); + } + digest + }; + let _lock = LockedFile::acquire(lock_dir.join(&lock_ident), &repository_url).await?; // Fetch the Git repository. let source = if let Some(reporter) = reporter { diff --git a/crates/uv-git/src/source.rs b/crates/uv-git/src/source.rs index 30958f74136e9..d75bad1dd2f00 100644 --- a/crates/uv-git/src/source.rs +++ b/crates/uv-git/src/source.rs @@ -74,9 +74,16 @@ impl GitSource { pub fn fetch(self) -> Result { // Compute the canonical URL for the repository. let canonical = RepositoryUrl::new(self.git.repository()); + let lfs_requested = self.git.lfs().enabled(); // The path to the repo, within the Git database. - let ident = cache_digest(&canonical); + let ident = { + let mut digest = cache_digest(&canonical); + if lfs_requested { + digest.push_str("_lfs"); + } + digest + }; let db_path = self.cache.join("db").join(&ident); // Authenticate the URL, if necessary. @@ -96,8 +103,12 @@ impl GitSource { // revision, then no update needs to happen. if let (Some(rev), Some(db)) = (self.git.precise(), &maybe_db) { if db.contains(rev) { - debug!("Using existing Git source `{}`", self.git.repository()); - return Ok((maybe_db.unwrap(), rev, None)); + // Check whether GitLFS was requested, but we have not fetched LFS artifacts. + let reinit_lfs = lfs_requested && !db.contains_lfs_artifacts(rev); + if !reinit_lfs { + debug!("Using existing Git source `{}`", self.git.repository()); + return Ok((maybe_db.unwrap().with_lfs_ready(lfs_requested), rev, None)); + } } } @@ -108,10 +119,18 @@ impl GitSource { if let GitReference::BranchOrTagOrCommit(maybe_commit) = self.git.reference() { if let Ok(oid) = maybe_commit.parse::() { if db.contains(oid) { - // This reference is an exact commit. Treat it like it's - // locked. - debug!("Using existing Git source `{}`", self.git.repository()); - return Ok((maybe_db.unwrap(), oid, None)); + // Check whether GitLFS was requested, but we have not fetched LFS artifacts. + let reinit_lfs = lfs_requested && !db.contains_lfs_artifacts(oid); + if !reinit_lfs { + // This reference is an exact commit. Treat it like it's + // locked. + debug!("Using existing Git source `{}`", self.git.repository()); + return Ok(( + maybe_db.unwrap().with_lfs_ready(lfs_requested), + oid, + None, + )); + } } } } @@ -135,7 +154,7 @@ impl GitSource { &self.client, self.disable_ssl, self.offline, - self.git.lfs(), + lfs_requested, )?; Ok((db, actual_rev, task)) diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index fc073cf922d05..41743569fe825 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -769,6 +769,10 @@ impl EnvVars { #[attr_added_in("0.6.4")] pub const GIT_TERMINAL_PROMPT: &'static str = "GIT_TERMINAL_PROMPT"; + /// Skip Smudge LFS Filter. + #[attr_hidden] + pub const GIT_LFS_SKIP_SMUDGE: &'static str = "GIT_LFS_SKIP_SMUDGE"; + /// Used in tests for better git isolation. /// /// For example, we run some tests in ~/.local/share/uv/tests. From 89c3e771509b9bf6028f7e9448502a8120a6b9b3 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:10:50 -0400 Subject: [PATCH 4/7] fix: support stateful lfs fragments and indicators --- .../uv-distribution-types/src/requirement.rs | 15 +++++++++++-- .../uv-distribution/src/metadata/lowering.rs | 4 ++++ crates/uv-git-types/src/lib.rs | 9 +++++--- crates/uv-installer/src/satisfies.rs | 20 +++++++++++++++++ crates/uv-pypi-types/src/direct_url.rs | 7 +++++- crates/uv-pypi-types/src/parsed_url.rs | 22 ++++++++++++++++++- .../src/lock/export/pylock_toml.rs | 5 +++-- .../src/lock/export/requirements_txt.rs | 1 + crates/uv-resolver/src/lock/mod.rs | 20 ++++++++++++++--- 9 files changed, 91 insertions(+), 12 deletions(-) diff --git a/crates/uv-distribution-types/src/requirement.rs b/crates/uv-distribution-types/src/requirement.rs index 543c7e1558d16..6f65647c2c7d5 100644 --- a/crates/uv-distribution-types/src/requirement.rs +++ b/crates/uv-distribution-types/src/requirement.rs @@ -7,7 +7,7 @@ use thiserror::Error; use uv_cache_key::{CacheKey, CacheKeyHasher}; use uv_distribution_filename::DistExtension; use uv_fs::{CWD, PortablePath, PortablePathBuf, relative_to}; -use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError, OidParseError}; +use uv_git_types::{GitOid, GitReference, GitUrl, GitUrlParseError, OidParseError}; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::VersionSpecifiers; use uv_pep508::{ @@ -856,6 +856,11 @@ impl From for RequirementSourceWire { .append_pair("subdirectory", &subdirectory); } + // Put lfs=true in the query when explicitly enabled. + if git.lfs().enabled() { + url.query_pairs_mut().append_pair("lfs", "true"); + } + // Put the requested reference in the query. match git.reference() { GitReference::Branch(branch) => { @@ -932,6 +937,7 @@ impl TryFrom for RequirementSource { let mut reference = GitReference::DefaultBranch; let mut subdirectory: Option = None; + let mut lfs: bool = false; for (key, val) in repository.query_pairs() { match &*key { "tag" => reference = GitReference::Tag(val.into_owned()), @@ -940,6 +946,7 @@ impl TryFrom for RequirementSource { "subdirectory" => { subdirectory = Some(PortablePathBuf::from(val.as_ref())); } + "lfs" => lfs = matches!(val.to_lowercase().as_str(), "true"), _ => {} } } @@ -962,10 +969,14 @@ impl TryFrom for RequirementSource { if let Some(subdirectory) = subdirectory.as_ref() { url.set_fragment(Some(&format!("subdirectory={subdirectory}"))); } + // Persist only when lfs support is explicitly requested + if lfs { + url.set_fragment(Some("lfs=true")); + } let url = VerbatimUrl::from_url(url); Ok(Self::Git { - git: GitUrl::from_fields(repository, reference, precise, GitLfs::from_env())?, + git: GitUrl::from_fields(repository, reference, precise, lfs.into())?, subdirectory: subdirectory.map(Box::::from), url, }) diff --git a/crates/uv-distribution/src/metadata/lowering.rs b/crates/uv-distribution/src/metadata/lowering.rs index 5952eb54275a1..9a24890505305 100644 --- a/crates/uv-distribution/src/metadata/lowering.rs +++ b/crates/uv-distribution/src/metadata/lowering.rs @@ -606,6 +606,10 @@ fn git_source( .ok_or_else(|| LoweringError::NonUtf8Path(subdirectory.to_path_buf()))?; url.set_fragment(Some(&format!("subdirectory={subdirectory}"))); } + // Persist only when lfs support is explicitly requested + if let Some(true) = lfs { + url.set_fragment(Some("lfs=true")); + } let url = VerbatimUrl::from_url(url); let repository = git.clone(); diff --git a/crates/uv-git-types/src/lib.rs b/crates/uv-git-types/src/lib.rs index bfcf33130d2ce..559bcc7a746da 100644 --- a/crates/uv-git-types/src/lib.rs +++ b/crates/uv-git-types/src/lib.rs @@ -79,8 +79,9 @@ impl GitUrl { pub fn from_reference( repository: DisplaySafeUrl, reference: GitReference, + lfs: GitLfs, ) -> Result { - Self::from_fields(repository, reference, None, GitLfs::from_env()) + Self::from_fields(repository, reference, None, lfs) } /// Create a new [`GitUrl`] from a repository URL and a precise commit. @@ -88,8 +89,9 @@ impl GitUrl { repository: DisplaySafeUrl, reference: GitReference, precise: GitOid, + lfs: GitLfs, ) -> Result { - Self::from_fields(repository, reference, Some(precise), GitLfs::from_env()) + Self::from_fields(repository, reference, Some(precise), lfs) } /// Create a new [`GitUrl`] from a repository URL and a precise commit, if known. @@ -179,7 +181,8 @@ impl TryFrom for GitUrl { url.set_path(&prefix); } - Self::from_reference(url, reference) + // TODO(samypr100): GitLfs::from_env() for now unless we want to support additional query params + Self::from_reference(url, reference, GitLfs::from_env()) } } diff --git a/crates/uv-installer/src/satisfies.rs b/crates/uv-installer/src/satisfies.rs index 9808bbdc4b246..1abaccf324936 100644 --- a/crates/uv-installer/src/satisfies.rs +++ b/crates/uv-installer/src/satisfies.rs @@ -173,6 +173,7 @@ impl RequirementSatisfaction { vcs: VcsKind::Git, requested_revision: _, commit_id: installed_precise, + git_lfs: installed_git_lfs, }, subdirectory: installed_subdirectory, } = direct_url.as_ref() @@ -188,6 +189,25 @@ impl RequirementSatisfaction { return Self::Mismatch; } + let requested_git_lfs = requested_git.lfs().enabled(); + let installed_git_lfs = installed_git_lfs.is_some_and(|lfs| lfs); + if requested_git_lfs != installed_git_lfs { + debug!( + "GitLFS mismatch: {} (installed) vs. {} (requested)", + if installed_git_lfs { + "enabled" + } else { + "disabled" + }, + if requested_git_lfs { + "enabled" + } else { + "disabled" + }, + ); + return Self::Mismatch; + } + if !RepositoryUrl::parse(installed_url).is_ok_and(|installed_url| { installed_url == RepositoryUrl::new(requested_git.repository()) }) { diff --git a/crates/uv-pypi-types/src/direct_url.rs b/crates/uv-pypi-types/src/direct_url.rs index 4e344d9228658..bcfa92f61daa4 100644 --- a/crates/uv-pypi-types/src/direct_url.rs +++ b/crates/uv-pypi-types/src/direct_url.rs @@ -36,7 +36,7 @@ pub enum DirectUrl { }, /// The direct URL is path to a VCS repository. For example: /// ```json - /// {"url": "https://github.com/pallets/flask.git", "vcs_info": {"commit_id": "8d9519df093864ff90ca446d4af2dc8facd3c542", "vcs": "git"}} + /// {"url": "https://github.com/pallets/flask.git", "vcs_info": {"commit_id": "8d9519df093864ff90ca446d4af2dc8facd3c542", "vcs": "git", "git_lfs": true }} /// ``` VcsUrl { url: String, @@ -70,6 +70,8 @@ pub struct VcsInfo { pub commit_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub requested_revision: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub git_lfs: Option, } #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -133,6 +135,9 @@ impl TryFrom<&DirectUrl> for Url { if let Some(subdirectory) = subdirectory { url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); } + if let Some(true) = vcs_info.git_lfs { + url.set_fragment(Some("lfs=true")); + } Ok(url) } } diff --git a/crates/uv-pypi-types/src/parsed_url.rs b/crates/uv-pypi-types/src/parsed_url.rs index 3b3b21f17cd88..5d721988d5989 100644 --- a/crates/uv-pypi-types/src/parsed_url.rs +++ b/crates/uv-pypi-types/src/parsed_url.rs @@ -6,7 +6,7 @@ use url::{ParseError, Url}; use uv_cache_key::{CacheKey, CacheKeyHasher}; use uv_distribution_filename::{DistExtension, ExtensionError}; -use uv_git_types::{GitUrl, GitUrlParseError}; +use uv_git_types::{GitLfs, GitUrl, GitUrlParseError}; use uv_pep508::{ Pep508Url, UnnamedRequirementUrl, VerbatimUrl, VerbatimUrlError, looks_like_git_repository, }; @@ -264,6 +264,7 @@ impl ParsedDirectoryUrl { /// Examples: /// * `git+https://git.example.com/MyProject.git` /// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir` +/// * `git+https://git.example.com/MyProject.git@v1.0#egg=pkg&subdirectory=pkg_dir&lfs=true` #[derive(Debug, Clone, Eq, PartialEq, PartialOrd, Hash, Ord)] pub struct ParsedGitUrl { pub url: GitUrl, @@ -286,6 +287,7 @@ impl TryFrom for ParsedGitUrl { /// excluded, it's presumed to come from `tool.uv.sources`. fn try_from(url_in: DisplaySafeUrl) -> Result { let subdirectory = get_subdirectory(&url_in).map(PathBuf::into_boxed_path); + let lfs = get_git_lfs(&url_in); let url = url_in .as_str() @@ -294,6 +296,7 @@ impl TryFrom for ParsedGitUrl { let url = DisplaySafeUrl::parse(url) .map_err(|err| ParsedUrlError::UrlParse(url.to_string(), err))?; let url = GitUrl::try_from(url)?; + let url = url.with_lfs(lfs); Ok(Self { url, subdirectory }) } } @@ -365,6 +368,18 @@ fn get_subdirectory(url: &Url) -> Option { Some(PathBuf::from(subdirectory)) } +/// Determine if the URL is GitLFS-enabled: +/// `git+https://git.example.com/MyProject.git@v1.0#lfs=true` +/// `git+https://git.example.com/MyProject.git@v1.0#subdirectory=pkg_dir&lfs=true` +fn get_git_lfs(url: &Url) -> GitLfs { + let fragment = url.fragment().unwrap_or_default(); + let lfs = fragment + .split('&') + .find_map(|fragment| fragment.strip_prefix("lfs=")) + .unwrap_or_default(); + matches!(lfs.to_lowercase().as_str(), "true").into() +} + impl TryFrom for ParsedUrl { type Error = ParsedUrlError; @@ -485,6 +500,7 @@ impl From<&ParsedGitUrl> for DirectUrl { vcs: VcsKind::Git, commit_id: value.url.precise().as_ref().map(ToString::to_string), requested_revision: value.url.reference().as_str().map(ToString::to_string), + git_lfs: value.url.lfs().enabled().then_some(true), }, subdirectory: value.subdirectory.clone(), } @@ -526,11 +542,15 @@ impl From for DisplaySafeUrl { impl From for DisplaySafeUrl { fn from(value: ParsedGitUrl) -> Self { + let lfs_enabled = value.url.lfs().enabled(); let mut url = Self::parse(&format!("{}{}", "git+", Self::from(value.url).as_str())) .expect("Git URL is invalid"); if let Some(subdirectory) = value.subdirectory { url.set_fragment(Some(&format!("subdirectory={}", subdirectory.display()))); } + if lfs_enabled { + url.set_fragment(Some("lfs=true")); + } url } } diff --git a/crates/uv-resolver/src/lock/export/pylock_toml.rs b/crates/uv-resolver/src/lock/export/pylock_toml.rs index 60df876420835..f30e86888d6b8 100644 --- a/crates/uv-resolver/src/lock/export/pylock_toml.rs +++ b/crates/uv-resolver/src/lock/export/pylock_toml.rs @@ -28,7 +28,7 @@ use uv_distribution_types::{ }; use uv_fs::{PortablePathBuf, relative_to}; use uv_git::{RepositoryReference, ResolvedRepositoryReference}; -use uv_git_types::{GitOid, GitReference, GitUrl, GitUrlParseError}; +use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError}; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::Version; use uv_pep508::{MarkerEnvironment, MarkerTree, VerbatimUrl}; @@ -1437,7 +1437,8 @@ impl PylockTomlVcs { .unwrap_or_else(|| GitReference::BranchOrTagOrCommit(self.commit_id.to_string())); let precise = self.commit_id; - GitUrl::from_commit(url, reference, precise)? + // TODO(samypr100): GitLfs::from_env() as pylock.toml spec doesn't specify how to label LFS support + GitUrl::from_commit(url, reference, precise, GitLfs::from_env())? }; // Reconstruct the PEP 508-compatible URL from the `GitSource`. diff --git a/crates/uv-resolver/src/lock/export/requirements_txt.rs b/crates/uv-resolver/src/lock/export/requirements_txt.rs index 61a8daa447181..31acca2533da5 100644 --- a/crates/uv-resolver/src/lock/export/requirements_txt.rs +++ b/crates/uv-resolver/src/lock/export/requirements_txt.rs @@ -91,6 +91,7 @@ impl std::fmt::Display for RequirementsTxtExport<'_> { url, GitReference::from(git.kind.clone()), git.precise, + git.lfs, ) .expect("Internal Git URLs must have supported schemes"); diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index a47f37bc4321a..a9c15cb3e6d2f 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -33,7 +33,7 @@ use uv_distribution_types::{ }; use uv_fs::{PortablePath, PortablePathBuf, relative_to}; use uv_git::{RepositoryReference, ResolvedRepositoryReference}; -use uv_git_types::{GitOid, GitReference, GitUrl, GitUrlParseError}; +use uv_git_types::{GitLfs, GitOid, GitReference, GitUrl, GitUrlParseError}; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::Version; use uv_pep508::{MarkerEnvironment, MarkerTree, VerbatimUrl, VerbatimUrlError, split_scheme}; @@ -2710,8 +2710,12 @@ impl Package { url.set_query(None); // Reconstruct the `GitUrl` from the `GitSource`. - let git_url = - GitUrl::from_commit(url, GitReference::from(git.kind.clone()), git.precise)?; + let git_url = GitUrl::from_commit( + url, + GitReference::from(git.kind.clone()), + git.precise, + git.lfs, + )?; // Reconstruct the PEP 508-compatible URL from the `GitSource`. let url = DisplaySafeUrl::from(ParsedGitUrl { @@ -3573,6 +3577,7 @@ impl Source { panic!("Git distribution is missing a precise hash: {git_dist}") }), subdirectory: git_dist.subdirectory.clone(), + lfs: git_dist.git.lfs(), }, ) } @@ -3892,6 +3897,7 @@ struct GitSource { precise: GitOid, subdirectory: Option>, kind: GitSourceKind, + lfs: GitLfs, } /// An error that occurs when a source string could not be parsed. @@ -3907,12 +3913,14 @@ impl GitSource { fn from_url(url: &Url) -> Result { let mut kind = GitSourceKind::DefaultBranch; let mut subdirectory = None; + let mut lfs = GitLfs::Disabled; for (key, val) in url.query_pairs() { match &*key { "tag" => kind = GitSourceKind::Tag(val.into_owned()), "branch" => kind = GitSourceKind::Branch(val.into_owned()), "rev" => kind = GitSourceKind::Rev(val.into_owned()), "subdirectory" => subdirectory = Some(PortablePathBuf::from(val.as_ref()).into()), + "lfs" => lfs = matches!(val.to_lowercase().as_str(), "true").into(), _ => {} } } @@ -3923,6 +3931,7 @@ impl GitSource { precise, subdirectory, kind, + lfs, }) } } @@ -4311,6 +4320,11 @@ fn locked_git_url(git_dist: &GitSourceDist) -> DisplaySafeUrl { .append_pair("subdirectory", &subdirectory); } + // Put lfs=true in the query when explicitly enabled. + if git_dist.git.lfs().enabled() { + url.query_pairs_mut().append_pair("lfs", "true"); + } + // Put the requested reference in the query. match git_dist.git.reference() { GitReference::Branch(branch) => { From c9de4739071e001efdaac67092bfeea3fc7fdafe Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:11:48 -0400 Subject: [PATCH 5/7] fix: proper caching lfs-based built wheels --- Cargo.lock | 1 + crates/uv-cache/Cargo.toml | 1 + crates/uv-cache/src/wheel.rs | 11 ++++++++--- crates/uv-distribution/src/error.rs | 2 ++ .../src/index/built_wheel_index.rs | 7 ++++++- crates/uv-distribution/src/source/mod.rs | 16 +++++++++++++--- crates/uv-git/src/source.rs | 7 +++++++ 7 files changed, 38 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e4e4555b6dd19..e28475b52f213 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5572,6 +5572,7 @@ dependencies = [ "uv-dirs", "uv-distribution-types", "uv-fs", + "uv-git-types", "uv-normalize", "uv-pypi-types", "uv-redacted", diff --git a/crates/uv-cache/Cargo.toml b/crates/uv-cache/Cargo.toml index 779309f0f7ab3..d22a5137c9bff 100644 --- a/crates/uv-cache/Cargo.toml +++ b/crates/uv-cache/Cargo.toml @@ -22,6 +22,7 @@ uv-cache-key = { workspace = true } uv-dirs = { workspace = true } uv-distribution-types = { workspace = true } uv-fs = { workspace = true, features = ["tokio"] } +uv-git-types = { workspace = true } uv-normalize = { workspace = true } uv-pypi-types = { workspace = true } uv-redacted = { workspace = true } diff --git a/crates/uv-cache/src/wheel.rs b/crates/uv-cache/src/wheel.rs index 406efd4ebca1e..e19d7fbe0cba6 100644 --- a/crates/uv-cache/src/wheel.rs +++ b/crates/uv-cache/src/wheel.rs @@ -2,6 +2,7 @@ use std::path::{Path, PathBuf}; use uv_cache_key::{CanonicalUrl, cache_digest}; use uv_distribution_types::IndexUrl; +use uv_git_types::GitLfs; use uv_redacted::DisplaySafeUrl; /// Cache wheels and their metadata, both from remote wheels and built from source distributions. @@ -15,11 +16,11 @@ pub enum WheelCache<'a> { Path(&'a DisplaySafeUrl), /// An editable dependency, which we key by URL. Editable(&'a DisplaySafeUrl), - /// A Git dependency, which we key by URL and SHA. + /// A Git dependency, which we key by URL, SHA and LFS status. /// /// Note that this variant only exists for source distributions; wheels can't be delivered /// through Git. - Git(&'a DisplaySafeUrl, &'a str), + Git(&'a DisplaySafeUrl, &'a str, &'a GitLfs), } impl WheelCache<'_> { @@ -39,10 +40,14 @@ impl WheelCache<'_> { Self::Editable(url) => WheelCacheKind::Editable .root() .join(cache_digest(&CanonicalUrl::new(url))), - Self::Git(url, sha) => WheelCacheKind::Git + Self::Git(url, sha, GitLfs::Disabled) => WheelCacheKind::Git .root() .join(cache_digest(&CanonicalUrl::new(url))) .join(sha), + Self::Git(url, sha, GitLfs::Enabled) => WheelCacheKind::Git + .root() + .join(cache_digest(&CanonicalUrl::new(url))) + .join(format!("{sha}_lfs")), } } diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index 12720eda6863d..025b53e3ad698 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -88,6 +88,8 @@ pub enum Error { MissingPkgInfo, #[error("The source distribution `{}` has no subdirectory `{}`", _0, _1.display())] MissingSubdirectory(DisplaySafeUrl, PathBuf), + #[error("The source distribution is missing a Git LFS artifacts")] + MissingGitLfsArtifacts, #[error("Failed to extract static metadata from `PKG-INFO`")] PkgInfo(#[source] uv_pypi_types::MetadataError), #[error("Failed to extract metadata from `requires.txt`")] diff --git a/crates/uv-distribution/src/index/built_wheel_index.rs b/crates/uv-distribution/src/index/built_wheel_index.rs index 15ac2fd897513..71c71c5bd27a3 100644 --- a/crates/uv-distribution/src/index/built_wheel_index.rs +++ b/crates/uv-distribution/src/index/built_wheel_index.rs @@ -202,7 +202,12 @@ impl<'a> BuiltWheelIndex<'a> { let cache_shard = self.cache.shard( CacheBucket::SourceDistributions, - WheelCache::Git(&source_dist.url, git_sha.as_short_str()).root(), + WheelCache::Git( + &source_dist.url, + git_sha.as_short_str(), + &source_dist.git.lfs(), + ) + .root(), ); // If there are build settings, we need to scope to a cache shard. diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index 1a22ada942701..8587604c1f466 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -1565,10 +1565,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } } + // Validate that LFS artifacts were fully initialized + if resource.git.lfs().enabled() && !fetch.lfs_ready() { + return Err(Error::MissingGitLfsArtifacts); + } + let git_sha = fetch.git().precise().expect("Exact commit after checkout"); let cache_shard = self.build_context.cache().shard( CacheBucket::SourceDistributions, - WheelCache::Git(resource.url, git_sha.as_short_str()).root(), + WheelCache::Git(resource.url, git_sha.as_short_str(), &resource.git.lfs()).root(), ); let metadata_entry = cache_shard.entry(METADATA); @@ -1667,7 +1672,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { .map(|oid| { self.build_context.cache().shard( CacheBucket::SourceDistributions, - WheelCache::Git(resource.url, oid.as_short_str()).root(), + WheelCache::Git(resource.url, oid.as_short_str(), &resource.git.lfs()).root(), ) }); if cache_shard @@ -1769,10 +1774,15 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { } } + // Validate that LFS artifacts were fully initialized + if resource.git.lfs().enabled() && !fetch.lfs_ready() { + return Err(Error::MissingGitLfsArtifacts); + } + let git_sha = fetch.git().precise().expect("Exact commit after checkout"); let cache_shard = self.build_context.cache().shard( CacheBucket::SourceDistributions, - WheelCache::Git(resource.url, git_sha.as_short_str()).root(), + WheelCache::Git(resource.url, git_sha.as_short_str(), &resource.git.lfs()).root(), ); let metadata_entry = cache_shard.entry(METADATA); diff --git a/crates/uv-git/src/source.rs b/crates/uv-git/src/source.rs index d75bad1dd2f00..a08b509c4e500 100644 --- a/crates/uv-git/src/source.rs +++ b/crates/uv-git/src/source.rs @@ -185,6 +185,7 @@ impl GitSource { Ok(Fetch { git: self.git.with_precise(actual_rev), path: checkout_path, + lfs_ready: db.lfs_ready(), }) } } @@ -194,6 +195,8 @@ pub struct Fetch { git: GitUrl, /// The path to the checked out repository. path: PathBuf, + /// Git LFS artifacts have been initialized (if requested). + lfs_ready: bool, } impl Fetch { @@ -205,6 +208,10 @@ impl Fetch { &self.path } + pub fn lfs_ready(&self) -> &bool { + &self.lfs_ready + } + pub fn into_git(self) -> GitUrl { self.git } From ebfd6de15e8cb1332bada25d4aee8dec45cb9bb4 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:12:30 -0400 Subject: [PATCH 6/7] tests: initial uv-sync test adjustments --- crates/uv/tests/it/sync.rs | 165 +++++++++++++++++++++++++++++-------- 1 file changed, 131 insertions(+), 34 deletions(-) diff --git a/crates/uv/tests/it/sync.rs b/crates/uv/tests/it/sync.rs index 8a4abffba4622..966b4b16d4c1c 100644 --- a/crates/uv/tests/it/sync.rs +++ b/crates/uv/tests/it/sync.rs @@ -13478,9 +13478,9 @@ fn reject_unmatched_runtime() -> Result<()> { #[cfg(feature = "git")] fn sync_git_lfs() -> Result<()> { let context = TestContext::new("3.13"); + let pyproject_toml = context.temp_dir.child("pyproject.toml"); // Set `lfs = true` in the source - let pyproject_toml = context.temp_dir.child("pyproject.toml"); pyproject_toml.write_str( r#" [project] @@ -13490,11 +13490,11 @@ fn sync_git_lfs() -> Result<()> { dependencies = ["test-lfs-repo"] [tool.uv.sources] - test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git", lfs = true } + test-lfs-repo = { git = "https://github.com/samypr100/test-lfs-repo.git", rev = "657500f0703dc173ac5d68dfa1d7e8c985c84424", lfs = true } "#, )?; - uv_snapshot!(context.filters(), context.sync(), @r" + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_GIT_LFS).arg("--no-cache"), @r" success: true exit_code: 0 ----- stdout ----- @@ -13503,29 +13503,22 @@ fn sync_git_lfs() -> Result<()> { Resolved 2 packages in [TIME] Prepared 1 package in [TIME] Installed 1 package in [TIME] - + test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) "); // Verify that we can import the module and access LFS content uv_snapshot!(context.filters(), context.python_command() .arg("-c") .arg("import test_lfs_repo.lfs_module"), @r#" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- - Traceback (most recent call last): - File "", line 1, in - import test_lfs_repo.lfs_module - File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 - version https://git-lfs.github.com/spec/v1 - ^^^^^ - SyntaxError: invalid syntax "#); // `UV_GIT_LFS=false` should not override `lfs = true` - uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "false").arg("--reinstall").arg("--no-cache"), @r" + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_GIT_LFS, "false").arg("--reinstall").arg("--no-cache"), @r" success: true exit_code: 0 ----- stdout ----- @@ -13535,24 +13528,17 @@ fn sync_git_lfs() -> Result<()> { Prepared 1 package in [TIME] Uninstalled 1 package in [TIME] Installed 1 package in [TIME] - ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + ~ test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) "); uv_snapshot!(context.filters(), context.python_command() .arg("-c") .arg("import test_lfs_repo.lfs_module"), @r#" - success: false - exit_code: 1 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- - Traceback (most recent call last): - File "", line 1, in - import test_lfs_repo.lfs_module - File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 - version https://git-lfs.github.com/spec/v1 - ^^^^^ - SyntaxError: invalid syntax "#); // Set `lfs = false` in the source @@ -13565,11 +13551,11 @@ fn sync_git_lfs() -> Result<()> { dependencies = ["test-lfs-repo"] [tool.uv.sources] - test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git", lfs = false } + test-lfs-repo = { git = "https://github.com/samypr100/test-lfs-repo.git", rev = "657500f0703dc173ac5d68dfa1d7e8c985c84424", lfs = false } "#, )?; - uv_snapshot!(context.filters(), context.sync().arg("--reinstall").arg("--no-cache"), @r" + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_GIT_LFS).arg("--reinstall").arg("--no-cache"), @r" success: true exit_code: 0 ----- stdout ----- @@ -13579,7 +13565,8 @@ fn sync_git_lfs() -> Result<()> { Prepared 1 package in [TIME] Uninstalled 1 package in [TIME] Installed 1 package in [TIME] - ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) "); // Verify that LFS content is missing (import should fail) @@ -13601,7 +13588,7 @@ fn sync_git_lfs() -> Result<()> { "#); // `UV_GIT_LFS=true` should not override `lfs = false` - uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "true").arg("--reinstall").arg("--no-cache"), @r" + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_GIT_LFS, "true").arg("--reinstall").arg("--no-cache"), @r" success: true exit_code: 0 ----- stdout ----- @@ -13611,7 +13598,7 @@ fn sync_git_lfs() -> Result<()> { Prepared 1 package in [TIME] Uninstalled 1 package in [TIME] Installed 1 package in [TIME] - ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + ~ test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) "); uv_snapshot!(context.filters(), context.python_command() @@ -13641,11 +13628,11 @@ fn sync_git_lfs() -> Result<()> { dependencies = ["test-lfs-repo"] [tool.uv.sources] - test-lfs-repo = { git = "https://github.com/zanieb/test-lfs-repo.git" } + test-lfs-repo = { git = "https://github.com/samypr100/test-lfs-repo.git", rev = "657500f0703dc173ac5d68dfa1d7e8c985c84424" } "#, )?; - uv_snapshot!(context.filters(), context.sync().env("UV_GIT_LFS", "true").arg("--reinstall").arg("--no-cache"), @r" + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_GIT_LFS, "true").arg("--reinstall").arg("--no-cache"), @r" success: true exit_code: 0 ----- stdout ----- @@ -13655,13 +13642,40 @@ fn sync_git_lfs() -> Result<()> { Prepared 1 package in [TIME] Uninstalled 1 package in [TIME] Installed 1 package in [TIME] - ~ test-lfs-repo==0.1.0 (from git+https://github.com/zanieb/test-lfs-repo.git@39b6b03dc0a301420b7b4e73311d799fb139ba2e) + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) "); // Verify that we can import the module when UV_GIT_LFS is set uv_snapshot!(context.filters(), context.python_command() .arg("-c") .arg("import test_lfs_repo.lfs_module; print('LFS module imported via env var')"), @r#" + success: true + exit_code: 0 + ----- stdout ----- + LFS module imported via env var + + ----- stderr ----- + "#); + + // Cache should be primed with non-LFS sources + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_GIT_LFS).arg("--reinstall"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" success: false exit_code: 1 ----- stdout ----- @@ -13669,14 +13683,97 @@ fn sync_git_lfs() -> Result<()> { ----- stderr ----- Traceback (most recent call last): File "", line 1, in - import test_lfs_repo.lfs_module; print('LFS module imported via env var') - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + import test_lfs_repo.lfs_module File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 version https://git-lfs.github.com/spec/v1 ^^^^^ SyntaxError: invalid syntax "#); + // Cache should be primed with LFS sources + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_GIT_LFS, "true").arg("--reinstall"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module; print('LFS module imported via env var')"), @r#" + success: true + exit_code: 0 + ----- stdout ----- + LFS module imported via env var + + ----- stderr ----- + "#); + + // Cache should hit non-LFS sources + uv_snapshot!(context.filters(), context.sync().env_remove(EnvVars::UV_GIT_LFS).arg("--reinstall"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module"), @r#" + success: false + exit_code: 1 + ----- stdout ----- + + ----- stderr ----- + Traceback (most recent call last): + File "", line 1, in + import test_lfs_repo.lfs_module + File "[SITE_PACKAGES]/test_lfs_repo/lfs_module.py", line 1 + version https://git-lfs.github.com/spec/v1 + ^^^^^ + SyntaxError: invalid syntax + "#); + + // Cache should hit LFS sources + uv_snapshot!(context.filters(), context.sync().env(EnvVars::UV_GIT_LFS, "true").arg("--reinstall"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo.git@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + "); + + uv_snapshot!(context.filters(), context.python_command() + .arg("-c") + .arg("import test_lfs_repo.lfs_module; print('LFS module imported via env var')"), @r#" + success: true + exit_code: 0 + ----- stdout ----- + LFS module imported via env var + + ----- stderr ----- + "#); + Ok(()) } From 52cbd5cd138661c6cd62e3d3bccbdccc136fac87 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Sun, 12 Oct 2025 15:37:57 -0400 Subject: [PATCH 7/7] feat: support --lfs in uv add --- crates/uv-cli/src/lib.rs | 4 + crates/uv-workspace/src/pyproject.rs | 16 +++- crates/uv/src/commands/project/add.rs | 14 ++++ crates/uv/src/lib.rs | 1 + crates/uv/src/settings.rs | 4 + crates/uv/tests/it/edit.rs | 107 ++++++++++++++++++++++++++ docs/reference/cli.md | 3 +- 7 files changed, 144 insertions(+), 5 deletions(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 4e4d6769403d2..ef6e55eaee09c 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -3882,6 +3882,10 @@ pub struct AddArgs { #[arg(long, group = "git-ref", action = clap::ArgAction::Set)] pub branch: Option, + /// Whether to use Git LFS when adding a dependency from Git. + #[arg(long, env = EnvVars::UV_GIT_LFS, value_parser = clap::builder::BoolishValueParser::new())] + pub lfs: bool, + /// Extras to enable for the dependency. /// /// May be provided more than once. diff --git a/crates/uv-workspace/src/pyproject.rs b/crates/uv-workspace/src/pyproject.rs index f629b28e9ed24..4334615a1ff61 100644 --- a/crates/uv-workspace/src/pyproject.rs +++ b/crates/uv-workspace/src/pyproject.rs @@ -1549,6 +1549,10 @@ pub enum SourceError { "`{0}` did not resolve to a Git repository, but a Git reference (`--branch {1}`) was provided." )] UnusedBranch(String, String), + #[error( + "`{0}` did not resolve to a Git repository, but a Git extension (`--lfs`) was provided." + )] + UnusedLfs(String), #[error( "`{0}` did not resolve to a local directory, but the `--editable` flag was provided. Editable installs are only supported for local directories." )] @@ -1578,12 +1582,13 @@ impl Source { rev: Option, tag: Option, branch: Option, + lfs: Option, root: &Path, existing_sources: Option<&BTreeMap>, ) -> Result, SourceError> { // If the user specified a Git reference for a non-Git source, try existing Git sources before erroring. if !matches!(source, RequirementSource::Git { .. }) - && (branch.is_some() || tag.is_some() || rev.is_some()) + && (branch.is_some() || tag.is_some() || rev.is_some() || lfs.is_some()) { if let Some(sources) = existing_sources { if let Some(package_sources) = sources.get(name) { @@ -1603,7 +1608,7 @@ impl Source { rev, tag, branch, - lfs: None, + lfs, marker: *marker, extra: extra.clone(), group: group.clone(), @@ -1621,6 +1626,9 @@ impl Source { if let Some(branch) = branch { return Err(SourceError::UnusedBranch(name.to_string(), branch)); } + if let Some(true) = lfs { + return Err(SourceError::UnusedLfs(name.to_string())); + } } // If we resolved a non-path source, and user specified an `--editable` flag, error. @@ -1713,7 +1721,7 @@ impl Source { rev: rev.cloned(), tag, branch, - lfs: None, + lfs, git: git.repository().clone(), subdirectory: subdirectory.map(PortablePathBuf::from), marker: MarkerTree::TRUE, @@ -1725,7 +1733,7 @@ impl Source { rev, tag, branch, - lfs: None, + lfs, git: git.repository().clone(), subdirectory: subdirectory.map(PortablePathBuf::from), marker: MarkerTree::TRUE, diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index ced42192b814b..fc42cbb1235b3 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -83,6 +83,7 @@ pub(crate) async fn add( rev: Option, tag: Option, branch: Option, + lfs: Option, extras_of_dependency: Vec, package: Option, python: Option, @@ -373,6 +374,7 @@ pub(crate) async fn add( rev.as_deref(), tag.as_deref(), branch.as_deref(), + lfs, marker, ) }) @@ -640,6 +642,7 @@ pub(crate) async fn add( rev.as_deref(), tag.as_deref(), branch.as_deref(), + lfs, &extras_of_dependency, index, &mut toml, @@ -771,6 +774,7 @@ fn edits( rev: Option<&str>, tag: Option<&str>, branch: Option<&str>, + lfs: Option, extras: &[ExtraName], index: Option<&IndexName>, toml: &mut PyProjectTomlMut, @@ -801,6 +805,7 @@ fn edits( rev.map(ToString::to_string), tag.map(ToString::to_string), branch.map(ToString::to_string), + lfs, script_dir, existing_sources, )? @@ -825,6 +830,7 @@ fn edits( rev.map(ToString::to_string), tag.map(ToString::to_string), branch.map(ToString::to_string), + lfs, project.root(), existing_sources, )? @@ -1183,6 +1189,7 @@ fn augment_requirement( rev: Option<&str>, tag: Option<&str>, branch: Option<&str>, + lfs: Option, marker: Option, ) -> UnresolvedRequirement { match requirement { @@ -1209,6 +1216,11 @@ fn augment_requirement( } else { git }; + let git = if let Some(lfs) = lfs { + git.with_lfs(lfs.into()) + } else { + git + }; RequirementSource::Git { git, subdirectory, @@ -1262,6 +1274,7 @@ fn resolve_requirement( rev: Option, tag: Option, branch: Option, + lfs: Option, root: &Path, existing_sources: Option<&BTreeMap>, ) -> Result<(uv_pep508::Requirement, Option), anyhow::Error> { @@ -1274,6 +1287,7 @@ fn resolve_requirement( rev, tag, branch, + lfs, root, existing_sources, ); diff --git a/crates/uv/src/lib.rs b/crates/uv/src/lib.rs index b370d6eacbbfe..23f99b83dcab3 100644 --- a/crates/uv/src/lib.rs +++ b/crates/uv/src/lib.rs @@ -2074,6 +2074,7 @@ async fn run_project( args.rev, args.tag, args.branch, + args.lfs, args.extras, args.package, args.python, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index e8d3470fa3018..77d70e8154421 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -1469,6 +1469,7 @@ pub(crate) struct AddSettings { pub(crate) rev: Option, pub(crate) tag: Option, pub(crate) branch: Option, + pub(crate) lfs: Option, pub(crate) package: Option, pub(crate) script: Option, pub(crate) python: Option, @@ -1506,6 +1507,7 @@ impl AddSettings { rev, tag, branch, + lfs, no_sync, locked, frozen, @@ -1600,6 +1602,7 @@ impl AddSettings { .unwrap_or_default(); let bounds = bounds.or(filesystem.as_ref().and_then(|fs| fs.add.add_bounds)); + let lfs = lfs.then_some(true); Self { locked, @@ -1619,6 +1622,7 @@ impl AddSettings { rev, tag, branch, + lfs, package, script, python: python.and_then(Maybe::into_option), diff --git a/crates/uv/tests/it/edit.rs b/crates/uv/tests/it/edit.rs index f051bf3c3bb49..a63b745288dac 100644 --- a/crates/uv/tests/it/edit.rs +++ b/crates/uv/tests/it/edit.rs @@ -630,6 +630,16 @@ fn add_git_error() -> Result<()> { error: `flask` did not resolve to a Git repository, but a Git reference (`--branch 0.0.1`) was provided. "###); + // Request lfs without a Git source. + uv_snapshot!(context.filters(), context.add().arg("flask").arg("--lfs"), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: `flask` did not resolve to a Git repository, but a Git extension (`--lfs`) was provided. + "###); + Ok(()) } @@ -662,6 +672,103 @@ fn add_git_branch() -> Result<()> { Ok(()) } +#[test] +#[cfg(feature = "git")] +fn add_git_lfs() -> Result<()> { + let context = TestContext::new("3.13"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str(indoc! {r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = [] + "#})?; + + uv_snapshot!(context.filters(), context.add().arg("test-lfs-repo @ git+https://github.com/samypr100/test-lfs-repo").arg("--rev").arg("657500f0703dc173ac5d68dfa1d7e8c985c84424").arg("--lfs"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + "); + + let pyproject_toml = context.read("pyproject.toml"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + pyproject_toml, @r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.13" + dependencies = [ + "test-lfs-repo", + ] + + [tool.uv.sources] + test-lfs-repo = { git = "https://github.com/samypr100/test-lfs-repo", rev = "657500f0703dc173ac5d68dfa1d7e8c985c84424", lfs = true } + "# + ); + }); + + let lock = context.read("uv.lock"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r#" + version = 1 + revision = 3 + requires-python = ">=3.13" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "test-lfs-repo" }, + ] + + [package.metadata] + requires-dist = [{ name = "test-lfs-repo", git = "https://github.com/samypr100/test-lfs-repo?lfs=true&rev=657500f0703dc173ac5d68dfa1d7e8c985c84424" }] + + [[package]] + name = "test-lfs-repo" + version = "0.1.0" + source = { git = "https://github.com/samypr100/test-lfs-repo?lfs=true&rev=657500f0703dc173ac5d68dfa1d7e8c985c84424#657500f0703dc173ac5d68dfa1d7e8c985c84424" } + "# + ); + }); + + uv_snapshot!(context.filters(), context.add().arg("test-lfs-repo @ git+https://github.com/samypr100/test-lfs-repo").arg("--rev").arg("4e82e85f6a8b8825d614ea23c550af55b2b7738c").arg("--lfs"), @r" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + Prepared 1 package in [TIME] + Uninstalled 1 package in [TIME] + Installed 1 package in [TIME] + - test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo@657500f0703dc173ac5d68dfa1d7e8c985c84424#lfs=true) + + test-lfs-repo==0.1.0 (from git+https://github.com/samypr100/test-lfs-repo@4e82e85f6a8b8825d614ea23c550af55b2b7738c#lfs=true) + "); + + Ok(()) +} + /// Add a Git requirement using the `--raw-sources` API. #[test] #[cfg(feature = "git")] diff --git a/docs/reference/cli.md b/docs/reference/cli.md index eff9f3214cb72..d8953d0af00da 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -865,7 +865,8 @@ uv add [OPTIONS] >
  • disabled: Do not use keyring for credential lookup
  • subprocess: Use the keyring command for credential lookup
    • -
--link-mode link-mode

The method to use when installing packages from the global cache.

+
--lfs

Whether to use Git LFS when adding a dependency from Git

+

May also be set with the UV_GIT_LFS environment variable.

--link-mode link-mode

The method to use when installing packages from the global cache.

Defaults to clone (also known as Copy-on-Write) on macOS, and hardlink on Linux and Windows.

WARNING: The use of symlink link mode is discouraged, as they create tight coupling between the cache and the target environment. For example, clearing the cache (uv cache clean) will break all installed packages by way of removing the underlying source files. Use symlinks with caution.

May also be set with the UV_LINK_MODE environment variable.

Possible values: