From 423cfc8cfaef54325b02d18379dbc309d346a6ac Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 04:25:55 +0000 Subject: [PATCH 1/8] Start fix for #645, #546 Assisted-by: Claude Code:claude-opus-5-5 From aa605f2159a648af0fdeb1f192b30a563ee47922 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 04:34:39 +0000 Subject: [PATCH 2/8] Find the Pipenv venv that .env and old Pipenv use Agent mode and hosted stale-install checks now find the venv Pipenv really uses in two cases where they used to miss it, patch the system interpreter instead, and let VEX attest not_affected: - WORKON_HOME, PIPENV_CUSTOM_VENV_NAME or PIPENV_VENV_IN_PROJECT set in the project's .env (or PIPENV_DOTENV_LOCATION), which every Pipenv command loads before it picks the venv (#546). - An explicit "not in project" setting next to a ./.venv directory. Only Pipenv 2023.11.14+ skips ./.venv then; 2018.11 to 2023.10.24 still use it, so both venvs are now patched (#645). Assisted-by: Claude Code:claude-opus-5-5 --- .../src/crawlers/python_crawler.rs | 360 ++++++++++++++++-- 1 file changed, 333 insertions(+), 27 deletions(-) diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index dfdee4f52..0f178f67d 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -693,17 +693,182 @@ fn pipenv_venv_in_project(cwd: &Path, var: &impl Fn(&str) -> Option) -> /// [`find_pipenv_virtualenv_site_packages`]). An explicit "in project" /// with no `./.venv` means Pipenv has no venv yet, so nothing. /// - A `./.venv` directory and an explicit "in project": `./.venv` only. -/// - A `./.venv` directory and an explicit "not in project": the -/// WORKON_HOME venv only (Pipenv 2023+ ignores `./.venv` then). +/// - A `./.venv` directory and an explicit "not in project": Pipenv +/// 2023.11.14+ ignores `./.venv` and uses WORKON_HOME, but 2018.11 +/// through 2023.10.24 use an existing `./.venv` directory whatever the +/// setting says (and only 2026.2+ reads the Pipfile key). /// - A `./.venv` directory and nothing explicit: Pipenv up to 2026.1 uses -/// it, 2026.2+ prefers an existing WORKON_HOME venv. Without running -/// Pipenv the version is unknown, so both are returned, WORKON_HOME -/// first, and whichever one the installed Pipenv uses gets patched. +/// it, 2026.2+ prefers an existing WORKON_HOME venv. +/// +/// In both of those cases the version is unknown without running Pipenv, +/// so both venvs are returned, WORKON_HOME first, and whichever one the +/// installed Pipenv uses gets patched. +/// +/// The settings come from the project's `.env` layered over the process +/// environment, as every Pipenv command loads it first (see +/// [`pipenv_dotenv`]), and then from the process environment alone (older +/// Pipenv read some settings before loading `.env`). Both views' venvs are +/// returned, the `.env` view first. /// /// Never `./venv`: no Pipenv release uses it. async fn pipenv_project_site_packages( cwd: &Path, var: &impl Fn(&str) -> Option, +) -> Vec { + let dotenv = pipenv_dotenv(cwd, var); + let mut results = Vec::new(); + if !dotenv.is_empty() { + let layered = |name: &str| { + dotenv + .iter() + .rev() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + .or_else(|| var(name)) + }; + results = pipenv_settings_site_packages(cwd, &layered).await; + } + for site in pipenv_settings_site_packages(cwd, var).await { + if !results.contains(&site) { + results.push(site); + } + } + results +} + +/// The `.env` variables Pipenv loads before it resolves the venv +/// (`load_dot_env`, unchanged from 2018 through 2026): `PIPENV_DOTENV_LOCATION` +/// (relative to the project) or `/.env`, unless +/// `bool(PIPENV_DONT_LOAD_ENV)`. Pipenv loads it with `override=True`, so +/// these beat the process environment. Empty when there is nothing to load. +fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Vec<(String, String)> { + let dont_load = match var("PIPENV_DONT_LOAD_ENV") { + Some(value) => match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => true, + "0" | "false" | "no" | "off" => false, + other => !other.is_empty(), + }, + None => false, + }; + if dont_load { + return Vec::new(); + } + let path = match var("PIPENV_DOTENV_LOCATION").filter(|v| !v.is_empty()) { + Some(location) => cwd.join(location), + None => cwd.join(".env"), + }; + // Regular files only, non-blocking: a FIFO `.env` must not wedge + // discovery. + match read_regular_to_string_sync(&path) { + Ok(text) => parse_dotenv(&text, var), + Err(_) => Vec::new(), + } +} + +/// `KEY=value` pairs of a `.env` file as python-dotenv (vendored by Pipenv) +/// reads them, in file order: blank lines and `#` comments skipped, an +/// optional `export ` prefix, single-quoted values literal, double-quoted +/// values with backslash escapes, unquoted values trimmed and cut at ` #`. +/// `${NAME}` / `${NAME:-default}` in unquoted and double-quoted values +/// expand from earlier keys in the file, then the environment (the +/// `override=True` order). A key without `=` sets nothing. +fn parse_dotenv(text: &str, var: &impl Fn(&str) -> Option) -> Vec<(String, String)> { + let mut pairs: Vec<(String, String)> = Vec::new(); + for line in text.lines() { + let line = line.trim_start(); + if line.is_empty() || line.starts_with('#') { + continue; + } + let line = line.strip_prefix("export ").map_or(line, str::trim_start); + let Some((key, raw)) = line.split_once('=') else { + continue; + }; + let key = key.trim(); + if key.is_empty() { + continue; + } + let raw = raw.trim_start(); + let (value, interpolate) = if let Some(rest) = raw.strip_prefix('\'') { + ( + rest.split('\'').next().unwrap_or_default().to_string(), + false, + ) + } else if let Some(rest) = raw.strip_prefix('"') { + let mut value = String::new(); + let mut chars = rest.chars(); + while let Some(c) = chars.next() { + match c { + '"' => break, + '\\' => match chars.next() { + Some('n') => value.push('\n'), + Some('t') => value.push('\t'), + Some('r') => value.push('\r'), + Some(other) => { + if !matches!(other, '"' | '\\' | '\'') { + value.push('\\'); + } + value.push(other); + } + None => value.push('\\'), + }, + c => value.push(c), + } + } + (value, true) + } else { + let unquoted = match raw.find(" #").or_else(|| raw.find("\t#")) { + Some(at) => &raw[..at], + None => raw, + }; + (unquoted.trim_end().to_string(), true) + }; + let value = if interpolate { + let lookup = |name: &str| { + pairs + .iter() + .rev() + .find(|(k, _)| k == name) + .map(|(_, v)| v.clone()) + .or_else(|| var(name)) + }; + dotenv_interpolate(&value, &lookup) + } else { + value + }; + pairs.push((key.to_string(), value)); + } + pairs +} + +/// python-dotenv's `${NAME}` / `${NAME:-default}` expansion (a bare +/// `$NAME` stays literal); an unset name takes the default, else "". +fn dotenv_interpolate(value: &str, lookup: &impl Fn(&str) -> Option) -> String { + let mut out = String::new(); + let mut rest = value; + while let Some(start) = rest.find("${") { + let Some(len) = rest[start + 2..].find('}') else { + break; + }; + out.push_str(&rest[..start]); + let inner = &rest[start + 2..start + 2 + len]; + let (name, default) = match inner.split_once(":-") { + Some((name, default)) => (name, Some(default)), + None => (inner, None), + }; + match lookup(name) { + Some(found) => out.push_str(&found), + None => out.push_str(default.unwrap_or_default()), + } + rest = &rest[start + 3 + len..]; + } + out.push_str(rest); + out +} + +/// [`pipenv_project_site_packages`] for one settings view. +async fn pipenv_settings_site_packages( + cwd: &Path, + var: &impl Fn(&str) -> Option, ) -> Vec { let in_project = pipenv_venv_in_project(cwd, var); let dot_venv = cwd.join(".venv"); @@ -718,9 +883,7 @@ async fn pipenv_project_site_packages( return in_tree; } let mut results = find_pipenv_virtualenv_site_packages_with(cwd, var).await; - if in_project.is_none() { - results.extend(in_tree); - } + results.extend(in_tree); results } @@ -3215,8 +3378,7 @@ mod tests { } /// #334: when Pipenv has no venv yet, discovery must not fall back to a - /// tree Pipenv will never use: a stray `venv/`, or a `./.venv` that an - /// explicit "not in project" rules out. + /// tree Pipenv will never use, such as a stray `venv/`. #[tokio::test] async fn pipenv_without_its_venv_does_not_fall_back_to_stray_trees() { let (_tmp, project, site, var) = pipenv_project_with_workon_venv(&[]); @@ -3225,36 +3387,44 @@ mod tests { assert!(find_local_venv_site_packages_with(&project, &var) .await .is_empty()); - - let (_tmp, project, site, var) = - pipenv_project_with_workon_venv(&[("PIPENV_VENV_IN_PROJECT", "0".to_string())]); - std::fs::remove_dir_all(site.ancestors().nth(3).unwrap()).unwrap(); - let _dot = fake_venv(&project, ".venv"); - assert!(find_local_venv_site_packages_with(&project, &var) - .await - .is_empty()); } - /// #334: an explicit "not in project" (`PIPENV_VENV_IN_PROJECT` falsy, + /// #645: an explicit "not in project" (`PIPENV_VENV_IN_PROJECT` falsy, /// `PIPENV_NO_VENV_IN_PROJECT` truthy, or Pipenv 2026.2+'s Pipfile - /// `[pipenv] venv_in_project = false`) makes Pipenv ignore a `./.venv` - /// directory. An explicit "in project" makes it use `./.venv` only, and - /// the environment variable beats the Pipfile. + /// `[pipenv] venv_in_project = false`) makes only Pipenv 2023.11.14+ + /// ignore a `./.venv` directory (#334); 2018.11 through 2023.10.24 use + /// an existing `./.venv` directory whatever the setting says. The + /// version is unknown, so both venvs are returned, WORKON_HOME first, + /// and `./.venv` alone when it is the only one. An explicit "in project" + /// makes every version use `./.venv` only, and the environment variable + /// beats the Pipfile. #[tokio::test] async fn pipenv_venv_in_project_settings_decide_about_dot_venv() { for env in [ ("PIPENV_VENV_IN_PROJECT", "0"), ("PIPENV_VENV_IN_PROJECT", "false"), ("PIPENV_VENV_IN_PROJECT", "Off"), + ("PIPENV_VENV_IN_PROJECT", "no"), ("PIPENV_NO_VENV_IN_PROJECT", "1"), ] { let (_tmp, project, site, var) = pipenv_project_with_workon_venv(&[(env.0, env.1.to_string())]); - let _dot = fake_venv(&project, ".venv"); + let dot = fake_venv(&project, ".venv"); assert_eq!( find_local_venv_site_packages_with(&project, &var).await, - vec![site], - "{}={:?} must skip ./.venv", + vec![site.clone(), dot.clone()], + "{}={:?}: Pipenv <= 2023.10.24 still uses ./.venv", + env.0, + env.1 + ); + // Only `./.venv` exists: that is the venv old Pipenv uses, so + // it is patched instead of falling through to the global + // interpreter. + std::fs::remove_dir_all(site.ancestors().nth(3).unwrap()).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![dot], + "{}={:?} with only ./.venv", env.0, env.1 ); @@ -3269,8 +3439,8 @@ mod tests { .unwrap(); assert_eq!( find_local_venv_site_packages_with(&project, &var).await, - vec![site.clone()], - "Pipfile venv_in_project = false must skip ./.venv" + vec![site.clone(), dot.clone()], + "Pipfile venv_in_project = false: only 2026.2+ reads it" ); std::fs::write( project.join("Pipfile"), @@ -3326,6 +3496,142 @@ mod tests { ); } + /// #546: every Pipenv command loads the project's `.env` (with + /// `override=True`) before it resolves the venv, so a + /// `PIPENV_CUSTOM_VENV_NAME` or `WORKON_HOME` set there moves the venv. + /// Discovery must find that venv instead of falling through to the + /// global interpreter. The process-environment venv is still returned + /// after it (older Pipenv cached some settings before loading `.env`). + #[tokio::test] + async fn pipenv_dotenv_settings_move_the_venv() { + // PIPENV_CUSTOM_VENV_NAME in .env, WORKON_HOME exported. + let (tmp, project, site, var) = pipenv_project_with_workon_venv(&[]); + let custom = fake_venv(&tmp.path().join("wh"), "myenv"); + std::fs::write(project.join(".env"), "PIPENV_CUSTOM_VENV_NAME=myenv\n").unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![custom.clone(), site.clone()] + ); + std::fs::remove_dir_all(site.ancestors().nth(3).unwrap()).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![custom.clone()], + "the .env-named venv alone" + ); + + // PIPENV_DONT_LOAD_ENV: Pipenv skips .env, so does discovery. + let dont = env_of(&[ + ( + "WORKON_HOME", + tmp.path().join("wh").to_string_lossy().into_owned(), + ), + ("PIPENV_DONT_LOAD_ENV", "1".to_string()), + ]); + assert!(find_local_venv_site_packages_with(&project, &dont) + .await + .is_empty()); + + // WORKON_HOME in .env with nothing exported (python-dotenv syntax: + // `export`, quotes, inline comments, `${VAR}` interpolation). + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("proj"); + std::fs::create_dir_all(&project).unwrap(); + std::fs::write(project.join("Pipfile"), "[packages]\nsix = \"==1.16.0\"\n").unwrap(); + let real = std::fs::canonicalize(&project).unwrap(); + let hash = pipenv_venv_hash(&pipenv_path_string(&real.join("Pipfile"))); + let base = tmp.path().to_string_lossy().into_owned(); + let elsewhere = fake_venv(&tmp.path().join("elsewhere"), &format!("proj-{hash}")); + let home = env_of(&[( + "HOME", + tmp.path().join("home").to_string_lossy().into_owned(), + )]); + for dotenv in [ + format!("WORKON_HOME={base}/elsewhere\n"), + format!("# venvs\nexport WORKON_HOME=\"{base}/elsewhere\" # here\n"), + format!("WORKON_HOME='{base}/elsewhere'\n"), + format!("BASE={base}\nWORKON_HOME=${{BASE}}/elsewhere # comment\n"), + ] { + std::fs::write(project.join(".env"), &dotenv).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &home).await, + vec![elsewhere.clone()], + ".env {dotenv:?}" + ); + } + + // PIPENV_DOTENV_LOCATION names the file (relative to the project). + std::fs::remove_file(project.join(".env")).unwrap(); + std::fs::write( + project.join("ci.env"), + format!("WORKON_HOME={base}/elsewhere\n"), + ) + .unwrap(); + assert!(find_local_venv_site_packages_with(&project, &home) + .await + .is_empty()); + let located = env_of(&[ + ( + "HOME", + tmp.path().join("home").to_string_lossy().into_owned(), + ), + ("PIPENV_DOTENV_LOCATION", "ci.env".to_string()), + ]); + assert_eq!( + find_local_venv_site_packages_with(&project, &located).await, + vec![elsewhere] + ); + } + + /// #546 / #645: `.env` also carries the in-project setting. + #[tokio::test] + async fn pipenv_dotenv_venv_in_project_is_honoured() { + // `.env` says "in project" and Pipenv has created `./.venv`, so + // `./.venv` comes first. The process-environment view (nothing + // explicit) still adds this project's own WORKON_HOME venv after it. + let (_tmp, project, site, var) = pipenv_project_with_workon_venv(&[]); + let dot = fake_venv(&project, ".venv"); + std::fs::write(project.join(".env"), "PIPENV_VENV_IN_PROJECT=1\n").unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![dot.clone(), site.clone()] + ); + // With no `./.venv` yet, the `.env` "in project" means Pipenv has + // no venv; only the process view's WORKON_HOME venv remains. + std::fs::remove_dir_all(project.join(".venv")).unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![site] + ); + } + + #[test] + fn dotenv_parsing_follows_python_dotenv() { + let var = env_of(&[("OUTER", "out".to_string())]); + let text = "\ +# comment +export A=1 +B = two words # trailing +C=\"quoted # not a comment\" +D='single ${OUTER}' +E=\"line\\nbreak\" +F=${A}-${OUTER}-${MISSING:-dflt} +NOVALUE +G= +=skipped +"; + let parsed = parse_dotenv(text, &var); + let get = |k: &str| parsed.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str()); + assert_eq!(get("A"), Some("1")); + assert_eq!(get("B"), Some("two words")); + assert_eq!(get("C"), Some("quoted # not a comment")); + assert_eq!(get("D"), Some("single ${OUTER}")); + assert_eq!(get("E"), Some("line\nbreak")); + assert_eq!(get("F"), Some("1-out-dflt")); + assert_eq!(get("NOVALUE"), None); + assert_eq!(get("G"), Some("")); + assert_eq!(parsed.len(), 7); + } + #[tokio::test] async fn pipenv_case_insensitive_fallback_matches_recased_directory() { // Pipenv on a case-insensitive filesystem reuses `-` From 6e2384974942307c90ac179decdd9ed3e8403955 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 04:36:31 +0000 Subject: [PATCH 3/8] Test Pipenv .env and .venv discovery end to end Scan-level regressions for both fixes: a .env-named venv is scanned (and PIPENV_DONT_LOAD_ENV turns that off), and an explicit "not in project" setting now scans ./.venv as well as the WORKON_HOME venv. The Pipenv compatibility doc describes the new discovery rules. Assisted-by: Claude Code:claude-opus-5-5 --- .../tests/in_process_python_envs.rs | 96 ++++++++++++++++--- docs/testing/pipenv-compatibility.md | 2 +- 2 files changed, 85 insertions(+), 13 deletions(-) diff --git a/crates/socket-patch-cli/tests/in_process_python_envs.rs b/crates/socket-patch-cli/tests/in_process_python_envs.rs index f9deaa42d..77b52a5e3 100644 --- a/crates/socket-patch-cli/tests/in_process_python_envs.rs +++ b/crates/socket-patch-cli/tests/in_process_python_envs.rs @@ -546,6 +546,8 @@ const PIPENV_VARS: &[&str] = &[ "PIPENV_NO_VENV_IN_PROJECT", "PIPENV_CUSTOM_VENV_NAME", "PIPENV_PIPFILE", + "PIPENV_DONT_LOAD_ENV", + "PIPENV_DOTENV_LOCATION", ]; /// Run `scan` with exactly `env` set among [`PIPENV_VARS`]. @@ -612,33 +614,103 @@ async fn pipenv_opt_outs_keep_activated_virtual_env_from_hijacking_scan() { } } -/// #334: Pipenv never uses `venv/`, and `PIPENV_VENV_IN_PROJECT=0` makes it -/// ignore a `./.venv` directory, so neither may shadow Pipenv's venv. +/// #334: Pipenv never uses `venv/`, so it may not shadow Pipenv's venv. #[tokio::test] #[serial] async fn pipenv_stray_venv_dirs_do_not_shadow_the_pipenv_venv() { - for (stray, opt_out) in [("venv", None), (".venv", Some("0"))] { + let (_tmp, project, workon) = pipenv_project(); + let stray_site = venv_site_packages(&project.join("venv"), "python3.12"); + std::fs::create_dir_all(&stray_site).unwrap(); + write_dist_info(&stray_site, "stray_decoy", "6.6.6"); + let server = MockServer::start().await; + mock_batch_empty(&server).await; + let env: Vec<(&str, &Path)> = vec![ + ("WORKON_HOME", &workon), + ("PIPENV_CUSTOM_VENV_NAME", Path::new("proj-env")), + ]; + let code = scan_with_pipenv_env(default_args(&project, server.uri()), &env).await; + assert_eq!(code, 0); + let bodies = batch_bodies(&server).await; + assert_discovered(&bodies, "pkg:pypi/pipenv-pkg@1.0.0"); + assert_not_discovered(&bodies, "pkg:pypi/stray-decoy@6.6.6"); +} + +/// #645: with `PIPENV_VENV_IN_PROJECT=0` (or `PIPENV_NO_VENV_IN_PROJECT=1`) +/// only Pipenv 2023.11.14+ ignores a `./.venv` directory; 2018.11 through +/// 2023.10.24 still use it. Both venvs are scanned, so whichever one the +/// installed Pipenv uses is patched. +#[tokio::test] +#[serial] +async fn pipenv_explicit_not_in_project_still_scans_dot_venv() { + for (name, value) in [ + ("PIPENV_VENV_IN_PROJECT", "0"), + ("PIPENV_NO_VENV_IN_PROJECT", "1"), + ] { let (_tmp, project, workon) = pipenv_project(); - let stray_site = venv_site_packages(&project.join(stray), "python3.12"); - std::fs::create_dir_all(&stray_site).unwrap(); - write_dist_info(&stray_site, "stray_decoy", "6.6.6"); + let dot_site = venv_site_packages(&project.join(".venv"), "python3.12"); + std::fs::create_dir_all(&dot_site).unwrap(); + write_dist_info(&dot_site, "dot_venv_pkg", "1.0.0"); let server = MockServer::start().await; mock_batch_empty(&server).await; - let mut env: Vec<(&str, &Path)> = vec![ + let env: Vec<(&str, &Path)> = vec![ ("WORKON_HOME", &workon), ("PIPENV_CUSTOM_VENV_NAME", Path::new("proj-env")), + (name, Path::new(value)), ]; - if let Some(value) = opt_out { - env.push(("PIPENV_VENV_IN_PROJECT", Path::new(value))); - } let code = scan_with_pipenv_env(default_args(&project, server.uri()), &env).await; - assert_eq!(code, 0, "{stray}"); + assert_eq!(code, 0, "{name}={value}"); let bodies = batch_bodies(&server).await; assert_discovered(&bodies, "pkg:pypi/pipenv-pkg@1.0.0"); - assert_not_discovered(&bodies, "pkg:pypi/stray-decoy@6.6.6"); + assert_discovered(&bodies, "pkg:pypi/dot-venv-pkg@1.0.0"); } } +/// #546: Pipenv loads the project's `.env` before it picks the venv, so a +/// `PIPENV_CUSTOM_VENV_NAME` or `WORKON_HOME` there decides which venv is +/// scanned (and `PIPENV_DONT_LOAD_ENV` turns that off). +#[tokio::test] +#[serial] +async fn pipenv_dotenv_settings_pick_the_scanned_venv() { + // Name in .env, WORKON_HOME exported. + let (_tmp, project, workon) = pipenv_project(); + std::fs::write(project.join(".env"), "PIPENV_CUSTOM_VENV_NAME=proj-env\n").unwrap(); + let server = MockServer::start().await; + mock_batch_empty(&server).await; + let code = scan_with_pipenv_env( + default_args(&project, server.uri()), + &[("WORKON_HOME", &workon)], + ) + .await; + assert_eq!(code, 0); + assert_discovered(&batch_bodies(&server).await, "pkg:pypi/pipenv-pkg@1.0.0"); + + // Both in .env, nothing exported. + std::fs::write( + project.join(".env"), + format!( + "export WORKON_HOME=\"{}\"\nPIPENV_CUSTOM_VENV_NAME=proj-env # named\n", + workon.display() + ), + ) + .unwrap(); + let server = MockServer::start().await; + mock_batch_empty(&server).await; + let code = scan_with_pipenv_env(default_args(&project, server.uri()), &[]).await; + assert_eq!(code, 0); + assert_discovered(&batch_bodies(&server).await, "pkg:pypi/pipenv-pkg@1.0.0"); + + // PIPENV_DONT_LOAD_ENV: Pipenv ignores .env, and so does discovery. + let server = MockServer::start().await; + mock_batch_empty(&server).await; + let code = scan_with_pipenv_env( + default_args(&project, server.uri()), + &[("PIPENV_DONT_LOAD_ENV", Path::new("1"))], + ) + .await; + assert_eq!(code, 0); + assert_not_discovered(&batch_bodies(&server).await, "pkg:pypi/pipenv-pkg@1.0.0"); +} + // --------------------------------------------------------------------------- // Package-manager-recorded envs: PDM's saved interpreter / PEP 582, and uv's // UV_PROJECT_ENVIRONMENT, ahead of a stray `./.venv` the manager never uses diff --git a/docs/testing/pipenv-compatibility.md b/docs/testing/pipenv-compatibility.md index da3499c08..6097807d4 100644 --- a/docs/testing/pipenv-compatibility.md +++ b/docs/testing/pipenv-compatibility.md @@ -17,7 +17,7 @@ requirements.txt lanes of the same ecosystem. | Input | Hosted | Vendored | Agent | |-------|--------|----------|-------| -| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). With an auto-detected `.venv` and an existing WORKON_HOME venv, both are patched, since Pipenv 2026.2+ uses the WORKON_HOME venv and older releases use `.venv`. | +| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). Settings come from the project's `.env` (or `PIPENV_DOTENV_LOCATION`, unless `PIPENV_DONT_LOAD_ENV`) layered over the environment, as Pipenv loads it first, and from the environment alone. With a `.venv` directory and an existing WORKON_HOME venv, both are patched unless the project is explicitly in-project: Pipenv 2026.2+ prefers the WORKON_HOME venv when nothing is set, 2023.11.14+ uses it when the project is explicitly not in-project, and older releases use `.venv` either way. | | `Pipfile.lock`, `pipfile-spec` < 6 (Pipenv 0–6) | Refused (`redirect_pipenv_skipped`), lock untouched. | Refused (`pypi_pipenv_spec_unsupported`). | Works. | | Lock-only checkout (nothing installed) | Discovered from the lock and redirected. | Discovered from the lock; the patched wheel or source distribution is downloaded and verified from the service without a local install. | Nothing to patch (no installed distribution); the lock's pins are listed as lockfile-only packages. | From bf91b102859907d613f0bb38c60b0f14dd6c3037 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 05:13:00 +0000 Subject: [PATCH 4/8] Fix Pipenv .env test and escapes on Windows The new .env test removed the whole WORKON_HOME on Windows, where site-packages sits one level shallower, so the .env-named venv went with it. .env values now decode exactly python-dotenv's escapes, so a backslash in a quoted Windows path is kept as written. Assisted-by: Claude Code:claude-opus-5-5 --- .../tests/in_process_python_envs.rs | 3 +- .../src/crawlers/python_crawler.rs | 85 +++++++++++++------ 2 files changed, 59 insertions(+), 29 deletions(-) diff --git a/crates/socket-patch-cli/tests/in_process_python_envs.rs b/crates/socket-patch-cli/tests/in_process_python_envs.rs index 77b52a5e3..de713bd7b 100644 --- a/crates/socket-patch-cli/tests/in_process_python_envs.rs +++ b/crates/socket-patch-cli/tests/in_process_python_envs.rs @@ -689,7 +689,8 @@ async fn pipenv_dotenv_settings_pick_the_scanned_venv() { project.join(".env"), format!( "export WORKON_HOME=\"{}\"\nPIPENV_CUSTOM_VENV_NAME=proj-env # named\n", - workon.display() + // python-dotenv decodes `\r`, `\t`... in double quotes. + workon.display().to_string().replace('\\', "/") ), ) .unwrap(); diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 0f178f67d..48b546003 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -789,32 +789,9 @@ fn parse_dotenv(text: &str, var: &impl Fn(&str) -> Option) -> Vec<(Strin } let raw = raw.trim_start(); let (value, interpolate) = if let Some(rest) = raw.strip_prefix('\'') { - ( - rest.split('\'').next().unwrap_or_default().to_string(), - false, - ) + (dotenv_unquote(rest, '\''), false) } else if let Some(rest) = raw.strip_prefix('"') { - let mut value = String::new(); - let mut chars = rest.chars(); - while let Some(c) = chars.next() { - match c { - '"' => break, - '\\' => match chars.next() { - Some('n') => value.push('\n'), - Some('t') => value.push('\t'), - Some('r') => value.push('\r'), - Some(other) => { - if !matches!(other, '"' | '\\' | '\'') { - value.push('\\'); - } - value.push(other); - } - None => value.push('\\'), - }, - c => value.push(c), - } - } - (value, true) + (dotenv_unquote(rest, '"'), true) } else { let unquoted = match raw.find(" #").or_else(|| raw.find("\t#")) { Some(at) => &raw[..at], @@ -840,6 +817,45 @@ fn parse_dotenv(text: &str, var: &impl Fn(&str) -> Option) -> Vec<(Strin pairs } +/// The body of a quoted python-dotenv value up to the closing `quote`, +/// decoding only the escapes python-dotenv decodes: `\\` and `\'` in single +/// quotes; those plus `\"` and `\a \b \f \n \r \t \v` in double quotes. Any +/// other backslash stays literal, so Windows paths survive. +fn dotenv_unquote(rest: &str, quote: char) -> String { + let mut value = String::new(); + let mut chars = rest.chars().peekable(); + while let Some(c) = chars.next() { + if c == quote { + break; + } + if c != '\\' { + value.push(c); + continue; + } + let decoded = match (quote, chars.peek()) { + (_, Some('\\')) => Some('\\'), + (_, Some('\'')) => Some('\''), + ('"', Some('"')) => Some('"'), + ('"', Some('a')) => Some('\u{7}'), + ('"', Some('b')) => Some('\u{8}'), + ('"', Some('f')) => Some('\u{c}'), + ('"', Some('n')) => Some('\n'), + ('"', Some('r')) => Some('\r'), + ('"', Some('t')) => Some('\t'), + ('"', Some('v')) => Some('\u{b}'), + _ => None, + }; + match decoded { + Some(decoded) => { + chars.next(); + value.push(decoded); + } + None => value.push('\\'), + } + } + value +} + /// python-dotenv's `${NAME}` / `${NAME:-default}` expansion (a bare /// `$NAME` stays literal); an unset name takes the default, else "". fn dotenv_interpolate(value: &str, lookup: &impl Fn(&str) -> Option) -> String { @@ -3512,7 +3528,11 @@ mod tests { find_local_venv_site_packages_with(&project, &var).await, vec![custom.clone(), site.clone()] ); - std::fs::remove_dir_all(site.ancestors().nth(3).unwrap()).unwrap(); + // Remove only the default venv (`wh/proj-`); the site-packages + // depth under it differs between POSIX and Windows. + let wh = tmp.path().join("wh"); + let default_root = site.ancestors().find(|a| a.parent() == Some(wh.as_path())); + std::fs::remove_dir_all(default_root.unwrap()).unwrap(); assert_eq!( find_local_venv_site_packages_with(&project, &var).await, vec![custom.clone()], @@ -3539,7 +3559,9 @@ mod tests { std::fs::write(project.join("Pipfile"), "[packages]\nsix = \"==1.16.0\"\n").unwrap(); let real = std::fs::canonicalize(&project).unwrap(); let hash = pipenv_venv_hash(&pipenv_path_string(&real.join("Pipfile"))); - let base = tmp.path().to_string_lossy().into_owned(); + // Forward slashes: python-dotenv decodes `\r`, `\t`... inside double + // quotes, so a quoted Windows path must not carry backslashes. + let base = tmp.path().to_string_lossy().replace('\\', "/"); let elsewhere = fake_venv(&tmp.path().join("elsewhere"), &format!("proj-{hash}")); let home = env_of(&[( "HOME", @@ -3615,6 +3637,9 @@ C=\"quoted # not a comment\" D='single ${OUTER}' E=\"line\\nbreak\" F=${A}-${OUTER}-${MISSING:-dflt} +H=\"C:\\Users\\dev\\Temp\" +I='it\\'s' +J='C:\\Temp\\x' NOVALUE G= =skipped @@ -3629,7 +3654,11 @@ G= assert_eq!(get("F"), Some("1-out-dflt")); assert_eq!(get("NOVALUE"), None); assert_eq!(get("G"), Some("")); - assert_eq!(parsed.len(), 7); + // Unknown escapes stay literal (Windows paths); `\'` and `\\` decode. + assert_eq!(get("H"), Some(r"C:\Users\dev\Temp")); + assert_eq!(get("I"), Some("it's")); + assert_eq!(get("J"), Some(r"C:\Temp\x")); + assert_eq!(parsed.len(), 10); } #[tokio::test] From a416025dcc28d083c114ef24e7c2b09128ed3302 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sat, 3 Oct 2026 02:39:28 -0400 Subject: [PATCH 5/8] fix(pipenv): honor complete dotenv settings views --- .../tests/in_process_redirect_pipenv.rs | 113 +++++ .../src/crawlers/python_crawler.rs | 394 +++++++++++++----- docs/testing/pipenv-compatibility.md | 2 +- 3 files changed, 414 insertions(+), 95 deletions(-) diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index ea25f497e..7b484a837 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -602,3 +602,116 @@ async fn warm_venv_with_the_upstream_release_is_not_attested() { roll_back(tmp.path(), &server).await; assert_eq!(read(&lock_path), LOCK); } + +/// Native Pipenv resolves these .env settings before choosing its env. +/// A healthy ambient interpreter or empty local env must not hide the +/// selected stale installation from the hosted-byte/VEX check. +#[tokio::test] +#[serial] +async fn dotenv_selected_pipenv_install_is_checked_before_vex() { + for case in [ + "single-quoted", + "multiline", + "ignore-active", + "override-active", + "relative-active", + ] { + let server = MockServer::start().await; + mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("project"); + std::fs::create_dir_all(&project).unwrap(); + write_project_with_upstream_install(&project); + let workon = tmp.path().join("workon"); + std::fs::create_dir_all(&workon).unwrap(); + let actual = workon.join("actual"); + std::fs::rename(project.join(".venv"), &actual).unwrap(); + std::fs::create_dir_all(site_packages(&project)).unwrap(); + let ambient_project = tmp.path().join("ambient-project"); + std::fs::create_dir_all(&ambient_project).unwrap(); + write_project_with_upstream_install(&ambient_project); + let ambient_file = site_packages(&ambient_project).join("urllib3/response.py"); + std::fs::write(&ambient_file, PATCHED).unwrap(); + let actual_file = if cfg!(windows) { + actual.join("Lib/site-packages/urllib3/response.py") + } else { + actual.join("lib/python3.12/site-packages/urllib3/response.py") + }; + let dotenv = match case { + "single-quoted" => "NAME=actual\nPIPENV_CUSTOM_VENV_NAME='${NAME}'\n".to_string(), + "multiline" => "PIPENV_CUSTOM_VENV_NAME=actual\nAPP_SETTINGS=\"first\nPIPENV_CUSTOM_VENV_NAME=decoy\nlast\"\n".to_string(), + "ignore-active" => "PIPENV_IGNORE_VIRTUALENVS=1\nPIPENV_CUSTOM_VENV_NAME=actual\n".to_string(), + "relative-active" => "VIRTUAL_ENV=../workon/actual\n".to_string(), + _ => format!("VIRTUAL_ENV='{}'\n", actual.to_string_lossy().replace('\\', "/")), + }; + std::fs::write(project.join(".env"), &dotenv).unwrap(); + let vex = project.join("out.vex.json"); + let mut cmd = tokio::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + for (key, _) in std::env::vars_os() { + let key_text = key.to_string_lossy(); + if key_text.starts_with("SOCKET_") + || key_text.starts_with("PIPENV_") + || matches!(key_text.as_ref(), "VIRTUAL_ENV" | "WORKON_HOME") + { + cmd.env_remove(key); + } + } + for key in [ + "SOCKET_OFFLINE", + "SOCKET_DEBUG", + "SOCKET_API_URL", + "SOCKET_PROXY_URL", + ] { + cmd.env_remove(key); + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1") + .env(MAJOR_ENV, "2026") + .env("WORKON_HOME", &workon) + .args(["scan", "--mode", "hosted", "--yes", "--json", "--cwd"]) + .arg(&project) + .args([ + "--api-url", + &server.uri(), + "--org", + ORG, + "--api-token", + "fake", + "--vex", + ]) + .arg(&vex) + .args(["--vex-product", VEX_PRODUCT]); + if case.ends_with("active") { + cmd.env("VIRTUAL_ENV", ambient_project.join(".venv")); + } + let out = cmd.output().await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap_or_else(|error| { + panic!( + "{case}: {error}: stdout={} stderr={}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) + }); + assert_eq!(out.status.code(), Some(1), "{case}: {json}"); + assert!( + json["redirect"]["warnings"] + .as_array() + .unwrap() + .iter() + .any(|warning| warning["code"] == "redirect_pypi_stale_install"), + "{case}: {json}" + ); + assert!(!vex.exists(), "{case}: stale bytes must not produce VEX"); + assert_eq!( + std::fs::read(&actual_file).unwrap(), + UPSTREAM, + "the probe is read-only" + ); + assert_eq!( + std::fs::read(&ambient_file).unwrap(), + PATCHED, + "the ambient env is unchanged" + ); + assert_eq!(read(&project.join("Pipfile")), PIPFILE); + assert!(read(&project.join("Pipfile.lock")).contains(HOSTED_URL)); + } +} diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 48b546003..811c07bd7 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -351,27 +351,24 @@ async fn find_local_venv_site_packages_with( return found; } - let pipenv = is_pipenv_project(cwd); - let poetry = if pipenv { - None - } else { - load_poetry_project(cwd, var).await - }; + // Pipenv loads .env before deciding whether an activated venv applies. + // Resolve that decision within each supported settings view, together + // with the placement settings. Pipenv never falls back to `venv/`. + if is_pipenv_project(cwd) { + return pipenv_project_site_packages(cwd, var).await; + } + let poetry = load_poetry_project(cwd, var).await; - // 1. Check VIRTUAL_ENV env var. Pipenv ignores it under `PIPENV_ACTIVE` - // (a `pipenv shell` started in another project) and - // `PIPENV_IGNORE_VIRTUALENVS`, so for a Pipenv project the activated venv - // then belongs to something else and must not be patched. Poetry ignores - // it once `poetry env use` recorded an env for the project (see + // 1. Check VIRTUAL_ENV env var. Poetry ignores it once `poetry env use` + // recorded an env for the project (see // [`poetry_active_prefix`]). PDM likewise skips an activated venv under // `PDM_IGNORE_ACTIVE_VENV`. - let pdm_ignores_active = pdm_env_flag(var, "PDM_IGNORE_ACTIVE_VENV") - && pdm_drives_project(cwd).await; + let pdm_ignores_active = + pdm_env_flag(var, "PDM_IGNORE_ACTIVE_VENV") && pdm_drives_project(cwd).await; let active_prefix = match &poetry { Some(project) => poetry_active_prefix(project, var), None if pdm_ignores_active => None, - None if !pipenv || pipenv_uses_virtual_env(var) => var("VIRTUAL_ENV"), - None => None, + None => var("VIRTUAL_ENV"), }; if let Some(virtual_env) = active_prefix { let venv_path = PathBuf::from(&virtual_env); @@ -382,15 +379,6 @@ async fn find_local_venv_site_packages_with( } } - // 2. A Pipenv project's venv is whatever Pipenv resolves, which is not - // the generic probe order below: Pipenv never uses `venv/`, and its - // in-project settings can rule out an existing `./.venv`. When Pipenv - // has no venv yet there is nothing to patch, so the generic probes must - // not fall back to a tree Pipenv will never use. - if pipenv { - return pipenv_project_site_packages(cwd, var).await; - } - // 3. Poetry decides for itself whether `./.venv` is the project's env // (`EnvManager.in_project_venv_exists`): only an existing `./.venv`, and // only when `virtualenvs.in-project` is not explicitly `false`. When @@ -708,7 +696,8 @@ fn pipenv_venv_in_project(cwd: &Path, var: &impl Fn(&str) -> Option) -> /// environment, as every Pipenv command loads it first (see /// [`pipenv_dotenv`]), and then from the process environment alone (older /// Pipenv read some settings before loading `.env`). Both views' venvs are -/// returned, the `.env` view first. +/// returned, the `.env` view first. Each view applies its own active-venv +/// decision before its in-project/WORKON_HOME placement. /// /// Never `./venv`: no Pipenv release uses it. async fn pipenv_project_site_packages( @@ -765,69 +754,152 @@ fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Vec<(Stri } } -/// `KEY=value` pairs of a `.env` file as python-dotenv (vendored by Pipenv) -/// reads them, in file order: blank lines and `#` comments skipped, an -/// optional `export ` prefix, single-quoted values literal, double-quoted -/// values with backslash escapes, unquoted values trimmed and cut at ` #`. -/// `${NAME}` / `${NAME:-default}` in unquoted and double-quoted values -/// expand from earlier keys in the file, then the environment (the -/// `override=True` order). A key without `=` sets nothing. +/// python-dotenv bindings are a stream, not independent lines: quoted +/// values may contain newlines and setting-like text. File reads normalize +/// CRLF/CR like Python's text mode. Every quoted/unquoted value is then +/// interpolated against preceding bindings before process variables. fn parse_dotenv(text: &str, var: &impl Fn(&str) -> Option) -> Vec<(String, String)> { - let mut pairs: Vec<(String, String)> = Vec::new(); - for line in text.lines() { - let line = line.trim_start(); - if line.is_empty() || line.starts_with('#') { - continue; - } - let line = line.strip_prefix("export ").map_or(line, str::trim_start); - let Some((key, raw)) = line.split_once('=') else { - continue; + let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + let mut rest = normalized.as_str(); + let mut values: Vec<(String, Option)> = Vec::new(); + while !rest.is_empty() { + let binding = match dotenv_binding(&mut rest) { + Ok(binding) => binding, + Err(()) => { + // Native parser recovery discards the remaining physical + // line at the failure, not the next valid binding. + rest = rest.split_once('\n').map_or("", |(_, tail)| tail); + continue; + } }; - let key = key.trim(); - if key.is_empty() { + let Some((key, value)) = binding else { continue; - } - let raw = raw.trim_start(); - let (value, interpolate) = if let Some(rest) = raw.strip_prefix('\'') { - (dotenv_unquote(rest, '\''), false) - } else if let Some(rest) = raw.strip_prefix('"') { - (dotenv_unquote(rest, '"'), true) - } else { - let unquoted = match raw.find(" #").or_else(|| raw.find("\t#")) { - Some(at) => &raw[..at], - None => raw, - }; - (unquoted.trim_end().to_string(), true) }; - let value = if interpolate { - let lookup = |name: &str| { - pairs - .iter() - .rev() - .find(|(k, _)| k == name) - .map(|(_, v)| v.clone()) - .or_else(|| var(name)) + let value = value.map(|value| { + let lookup = |name: &str| match values.iter().find(|(key, _)| key == name) { + // A valueless binding shadows the environment during + // interpolation, but is not exported after resolution. + Some((_, value)) => Some(value.clone().unwrap_or_default()), + None => var(name), }; dotenv_interpolate(&value, &lookup) + }); + if let Some((_, previous)) = values.iter_mut().find(|(name, _)| name == key) { + *previous = value; } else { - value - }; - pairs.push((key.to_string(), value)); + values.push((key.to_string(), value)); + } } - pairs + // Last binding wins even when it has no value: native load_dotenv + // first builds its mapping, then exports only its non-None entries. + values + .into_iter() + .filter_map(|(key, value)| value.map(|value| (key, value))) + .collect() +} + +fn dotenv_whitespace(c: char) -> bool { + // Python's str.isspace additionally recognizes these separators. + c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}') } -/// The body of a quoted python-dotenv value up to the closing `quote`, -/// decoding only the escapes python-dotenv decodes: `\\` and `\'` in single -/// quotes; those plus `\"` and `\a \b \f \n \r \t \v` in double quotes. Any -/// other backslash stays literal, so Windows paths survive. -fn dotenv_unquote(rest: &str, quote: char) -> String { +fn dotenv_inline_whitespace(c: char) -> bool { + c != '\n' && c != '\r' && dotenv_whitespace(c) +} + +/// Consume one native python-dotenv binding. Failure keeps the cursor at +/// the token that did not parse, matching its line-recovery semantics. +fn dotenv_binding<'a>(rest: &mut &'a str) -> Result)>, ()> { + *rest = rest.trim_start_matches(dotenv_whitespace); + if rest.is_empty() { + return Ok(None); + } + if let Some(after) = rest.strip_prefix("export") { + if after.chars().next().is_some_and(dotenv_inline_whitespace) { + *rest = after.trim_start_matches(dotenv_inline_whitespace); + } + } + let key = if rest.starts_with('#') { + None + } else if let Some(after) = rest.strip_prefix('\'') { + let end = after.find('\'').filter(|end| *end > 0).ok_or(())?; + let key = &after[..end]; + *rest = &after[end + 1..]; + Some(key) + } else { + let end = rest + .find(|c| c == '=' || c == '#' || dotenv_whitespace(c)) + .unwrap_or(rest.len()); + if end == 0 { + return Err(()); + } + let key = &rest[..end]; + *rest = &rest[end..]; + Some(key) + }; + *rest = rest.trim_start_matches(dotenv_inline_whitespace); + let value = if let Some(after) = rest.strip_prefix('=') { + *rest = after.trim_start_matches(dotenv_inline_whitespace); + Some(dotenv_value(rest)?) + } else { + None + }; + *rest = rest.trim_start_matches(dotenv_inline_whitespace); + if rest.starts_with('#') { + *rest = &rest[rest.find('\n').unwrap_or(rest.len())..]; + } + *rest = rest.trim_start_matches(dotenv_inline_whitespace); + if let Some(after) = rest.strip_prefix('\n') { + *rest = after; + } else if !rest.is_empty() { + return Err(()); + } + Ok(key.map(|key| (key, value))) +} + +fn dotenv_value(rest: &mut &str) -> Result { + static SINGLE: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"\A'((?:\\'|[^'])*)'").unwrap()); + static DOUBLE: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r#"\A"((?:\\"|[^"])*)""#).unwrap()); + let quote = rest.chars().next(); + if matches!(quote, Some('\'' | '"')) { + let parser = if quote == Some('\'') { + &*SINGLE + } else { + &*DOUBLE + }; + let captures = parser.captures(rest).ok_or(())?; + let body = captures.get(1).unwrap().as_str(); + let value = dotenv_decode_escapes(body, quote.unwrap()); + *rest = &rest[captures.get(0).unwrap().end()..]; + return Ok(value); + } + let end = rest.find('\n').unwrap_or(rest.len()); + let line = &rest[..end]; + *rest = &rest[end..]; + let comment = line + .char_indices() + .find_map(|(index, c)| { + (c == '#' + && line[..index] + .chars() + .next_back() + .is_some_and(dotenv_whitespace)) + .then_some(index) + }) + .unwrap_or(line.len()); + Ok(line[..comment] + .trim_end_matches(dotenv_whitespace) + .to_string()) +} + +/// Decode only python-dotenv's escape set after the complete quoted body +/// was parsed. Unknown escapes stay literal, including Windows paths. +fn dotenv_decode_escapes(body: &str, quote: char) -> String { let mut value = String::new(); - let mut chars = rest.chars().peekable(); + let mut chars = body.chars().peekable(); while let Some(c) = chars.next() { - if c == quote { - break; - } if c != '\\' { value.push(c); continue; @@ -856,28 +928,23 @@ fn dotenv_unquote(rest: &str, quote: char) -> String { value } -/// python-dotenv's `${NAME}` / `${NAME:-default}` expansion (a bare -/// `$NAME` stays literal); an unset name takes the default, else "". +/// python-dotenv's variable grammar. Unsupported `${NAME:...}` and bare +/// `$NAME` stay literal; defaults and expansions are not recursive. fn dotenv_interpolate(value: &str, lookup: &impl Fn(&str) -> Option) -> String { + static VARIABLES: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"\$\{([^}:]*)(?::-([^}]*))?\}").unwrap()); let mut out = String::new(); - let mut rest = value; - while let Some(start) = rest.find("${") { - let Some(len) = rest[start + 2..].find('}') else { - break; - }; - out.push_str(&rest[..start]); - let inner = &rest[start + 2..start + 2 + len]; - let (name, default) = match inner.split_once(":-") { - Some((name, default)) => (name, Some(default)), - None => (inner, None), - }; - match lookup(name) { + let mut end = 0; + for captures in VARIABLES.captures_iter(value) { + let whole = captures.get(0).unwrap(); + out.push_str(&value[end..whole.start()]); + match lookup(captures.get(1).unwrap().as_str()) { Some(found) => out.push_str(&found), - None => out.push_str(default.unwrap_or_default()), + None => out.push_str(captures.get(2).map_or("", |default| default.as_str())), } - rest = &rest[start + 3 + len..]; + end = whole.end(); } - out.push_str(rest); + out.push_str(&value[end..]); out } @@ -886,6 +953,16 @@ async fn pipenv_settings_site_packages( cwd: &Path, var: &impl Fn(&str) -> Option, ) -> Vec { + if pipenv_uses_virtual_env(var) { + if let Some(prefix) = var("VIRTUAL_ENV").filter(|prefix| !prefix.is_empty()) { + // Pipenv evaluates a relative active prefix from the project, + // which can differ from this process's cwd (`--cwd`). + let found = find_site_packages_under(&cwd.join(prefix), "site-packages").await; + if !found.is_empty() { + return found; + } + } + } let in_project = pipenv_venv_in_project(cwd, var); let dot_venv = cwd.join(".venv"); if !dot_venv.is_dir() { @@ -3626,6 +3703,135 @@ mod tests { ); } + #[test] + fn dotenv_bindings_match_native_quotes_and_record_boundaries() { + let var = env_of(&[("NAME", "ambient".to_string())]); + for (text, expected) in [ + ( + "NAME=actual\nCUSTOM='${NAME}'\n", + vec![("NAME", "actual"), ("CUSTOM", "actual")], + ), + ("'CUSTOM'=actual\n", vec![("CUSTOM", "actual")]), + ("export\tCUSTOM=actual\n", vec![("CUSTOM", "actual")]), + ( + "CUSTOM=actual\nAPP=\"first\nCUSTOM=decoy\nlast\"\n", + vec![("CUSTOM", "actual"), ("APP", "first\nCUSTOM=decoy\nlast")], + ), + ( + "CUSTOM=actual\nCUSTOM=\"decoy\" trailing\n", + vec![("CUSTOM", "actual")], + ), + ( + "CUSTOM=actual\nCUSTOM=\"decoy\n", + vec![("CUSTOM", "actual")], + ), + ("NAME\nCUSTOM=${NAME:-default}\n", vec![("CUSTOM", "")]), + ( + "NAME=first\nNAME\nCUSTOM=${NAME:-default}\n", + vec![("CUSTOM", "")], + ), + ( + "CUSTOM=actual\t# first # second\n", + vec![("CUSTOM", "actual")], + ), + ( + "CUSTOM=${NAME:unsupported}\n", + vec![("CUSTOM", "${NAME:unsupported}")], + ), + ( + "APP='first\r\nsecond'\rCUSTOM=actual\r", + vec![("APP", "first\nsecond"), ("CUSTOM", "actual")], + ), + ] { + let expected = expected + .into_iter() + .map(|(key, value)| (key.to_string(), value.to_string())) + .collect::>(); + assert_eq!(parse_dotenv(text, &var), expected, "dotenv input {text:?}"); + } + } + + #[tokio::test] + async fn pipenv_dotenv_syntax_keeps_the_native_project_environment() { + let (tmp, project, default, var) = pipenv_project_with_workon_venv(&[]); + let actual = fake_venv(&tmp.path().join("wh"), "actual"); + let decoy = fake_venv(&tmp.path().join("wh"), "decoy"); + for dotenv in [ + "NAME=actual\nPIPENV_CUSTOM_VENV_NAME='${NAME}'\n", + "'PIPENV_CUSTOM_VENV_NAME'=actual\n", + "export\tPIPENV_CUSTOM_VENV_NAME=actual\n", + "PIPENV_CUSTOM_VENV_NAME=actual\nAPP_SETTINGS=\"first\nPIPENV_CUSTOM_VENV_NAME=decoy\nlast\"\n", + ] { + std::fs::write(project.join(".env"), dotenv).unwrap(); + let found = find_local_venv_site_packages_with(&project, &var).await; + assert_eq!(found, vec![actual.clone(), default.clone()], "dotenv {dotenv:?}"); + assert!(!found.contains(&decoy), "text inside a quoted value is not a setting"); + } + } + + #[tokio::test] + async fn pipenv_dotenv_active_settings_are_resolved_per_view() { + let (tmp, project, default, base_var) = pipenv_project_with_workon_venv(&[]); + let actual_root = tmp.path().join("wh/actual"); + let actual = fake_venv(&tmp.path().join("wh"), "actual"); + let ambient_root = tmp.path().join("ambient"); + let ambient = fake_venv(tmp.path(), "ambient"); + // An empty active prefix is falsy, not the project directory. + let _project_lib = fake_venv(&project, ""); + let var = |key: &str| match key { + "VIRTUAL_ENV" => Some(ambient_root.to_string_lossy().into_owned()), + _ => base_var(key), + }; + for dotenv in [ + "PIPENV_IGNORE_VIRTUALENVS=1\nPIPENV_CUSTOM_VENV_NAME=actual\n".to_string(), + "PIPENV_ACTIVE=1\nPIPENV_CUSTOM_VENV_NAME=actual\n".to_string(), + format!( + "VIRTUAL_ENV='{}'\n", + actual_root.to_string_lossy().replace('\\', "/") + ), + "VIRTUAL_ENV=../wh/actual\n".to_string(), + "VIRTUAL_ENV=\nPIPENV_CUSTOM_VENV_NAME=actual\n".to_string(), + ] { + std::fs::write(project.join(".env"), &dotenv).unwrap(); + let found = find_local_venv_site_packages_with(&project, &var) + .await + .into_iter() + .map(|site| site.canonicalize().unwrap()) + .collect::>(); + assert_eq!( + found, + vec![ + actual.canonicalize().unwrap(), + ambient.canonicalize().unwrap() + ], + "dotenv {dotenv:?}" + ); + } + std::fs::write(project.join(".env"), "PIPENV_IGNORE_VIRTUALENVS=0\n").unwrap(); + let ignored = |key: &str| match key { + "PIPENV_IGNORE_VIRTUALENVS" => Some("1".to_string()), + _ => var(key), + }; + assert_eq!( + find_local_venv_site_packages_with(&project, &ignored).await, + vec![ambient.clone(), default], + "dotenv can clear a process-level opt-out" + ); + std::fs::write( + project.join(".env"), + "PIPENV_IGNORE_VIRTUALENVS=1\nPIPENV_CUSTOM_VENV_NAME=actual\n", + ) + .unwrap(); + let disabled = |key: &str| match key { + "PIPENV_DONT_LOAD_ENV" => Some("1".to_string()), + _ => var(key), + }; + assert_eq!( + find_local_venv_site_packages_with(&project, &disabled).await, + vec![ambient] + ); + } + #[test] fn dotenv_parsing_follows_python_dotenv() { let var = env_of(&[("OUTER", "out".to_string())]); @@ -3649,7 +3855,7 @@ G= assert_eq!(get("A"), Some("1")); assert_eq!(get("B"), Some("two words")); assert_eq!(get("C"), Some("quoted # not a comment")); - assert_eq!(get("D"), Some("single ${OUTER}")); + assert_eq!(get("D"), Some("single out")); assert_eq!(get("E"), Some("line\nbreak")); assert_eq!(get("F"), Some("1-out-dflt")); assert_eq!(get("NOVALUE"), None); diff --git a/docs/testing/pipenv-compatibility.md b/docs/testing/pipenv-compatibility.md index 6097807d4..50a28efa5 100644 --- a/docs/testing/pipenv-compatibility.md +++ b/docs/testing/pipenv-compatibility.md @@ -17,7 +17,7 @@ requirements.txt lanes of the same ecosystem. | Input | Hosted | Vendored | Agent | |-------|--------|----------|-------| -| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). Settings come from the project's `.env` (or `PIPENV_DOTENV_LOCATION`, unless `PIPENV_DONT_LOAD_ENV`) layered over the environment, as Pipenv loads it first, and from the environment alone. With a `.venv` directory and an existing WORKON_HOME venv, both are patched unless the project is explicitly in-project: Pipenv 2026.2+ prefers the WORKON_HOME venv when nothing is set, 2023.11.14+ uses it when the project is explicitly not in-project, and older releases use `.venv` either way. | +| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). Settings come from the project's `.env` (or `PIPENV_DOTENV_LOCATION`, unless `PIPENV_DONT_LOAD_ENV`) layered over the environment, as Pipenv loads it first, and from the environment alone. Each view applies its own active-venv decision before placement. The dotenv reader follows python-dotenv's record boundaries, quoted keys/values, multiline values, and interpolation (including single-quoted values); setting-like text inside a quoted value is not another setting. With a `.venv` directory and an existing WORKON_HOME venv, both are patched unless the project is explicitly in-project: Pipenv 2026.2+ prefers the WORKON_HOME venv when nothing is set, 2023.11.14+ uses it when the project is explicitly not in-project, and older releases use `.venv` either way. | | `Pipfile.lock`, `pipfile-spec` < 6 (Pipenv 0–6) | Refused (`redirect_pipenv_skipped`), lock untouched. | Refused (`pypi_pipenv_spec_unsupported`). | Works. | | Lock-only checkout (nothing installed) | Discovered from the lock and redirected. | Discovered from the lock; the patched wheel or source distribution is downloaded and verified from the service without a local install. | Nothing to patch (no installed distribution); the lock's pins are listed as lockfile-only packages. | From d8356ae2df83d1671c49efca6c22a5a893e6e79c Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Sat, 3 Oct 2026 03:09:53 -0400 Subject: [PATCH 6/8] fix(pipenv): include cached legacy shell environments --- .../tests/in_process_redirect_pipenv.rs | 102 +++++ .../src/crawlers/python_crawler.rs | 419 +++++++++++++++++- docs/testing/pipenv-compatibility.md | 4 +- 3 files changed, 502 insertions(+), 23 deletions(-) diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index 7b484a837..b187e86af 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -715,3 +715,105 @@ async fn dotenv_selected_pipenv_install_is_checked_before_vex() { assert!(read(&project.join("Pipfile.lock")).contains(HOSTED_URL)); } } + +/// Pipenv 2018 shell can select a third dotenv WORKON_HOME while current +/// Pipenv and pre-dotenv commands use a healthy cached environment. +#[tokio::test] +#[serial] +async fn legacy_dotenv_workon_install_is_checked_before_vex() { + for forward_reference in [true, false] { + let server = MockServer::start().await; + mock_api(&server).await; + let tmp = tempfile::tempdir().unwrap(); + let project = tmp.path().join("project"); + let legacy = tmp.path().join("legacy-workon"); + let cached = tmp.path().join("cached-workon"); + let make_install = |seed: &Path, root: &Path, bytes: &[u8]| { + std::fs::create_dir_all(seed).unwrap(); + std::fs::create_dir_all(root.parent().unwrap()).unwrap(); + write_project_with_upstream_install(seed); + let relative = site_packages(seed) + .strip_prefix(seed.join(".venv")) + .unwrap() + .to_path_buf(); + std::fs::rename(seed.join(".venv"), root).unwrap(); + let file = root.join(relative).join("urllib3/response.py"); + std::fs::write(&file, bytes).unwrap(); + file + }; + let stale_file = make_install(&project, &legacy.join("env"), UPSTREAM); + let healthy_file = make_install(&tmp.path().join("seed"), &cached.join("env"), PATCHED); + // The .venv file names a project-specific venv within WORKON_HOME + // in both native generations, avoiding unrelated directory scans. + std::fs::write(project.join(".venv"), "env\n").unwrap(); + let legacy_text = legacy.to_string_lossy().replace('\\', "/"); + let cached_text = cached.to_string_lossy().replace('\\', "/"); + let dotenv = if forward_reference { + format!("WORKON_HOME=${{BASE}}\nBASE={legacy_text}\n") + } else { + format!("BASE={cached_text}\nWORKON_HOME=${{BASE}}\n") + }; + std::fs::write(project.join(".env"), dotenv).unwrap(); + let vex = project.join("out.vex.json"); + let mut cmd = tokio::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + for (key, _) in std::env::vars_os() { + let text = key.to_string_lossy(); + if text.starts_with("SOCKET_") + || text.starts_with("PIPENV_") + || matches!(text.as_ref(), "VIRTUAL_ENV" | "WORKON_HOME" | "BASE") + { + cmd.env_remove(key); + } + } + cmd.env("SOCKET_TELEMETRY_DISABLED", "1") + .env(MAJOR_ENV, "2026") + .env("HOME", tmp.path().join("home")) + .env("USERPROFILE", tmp.path().join("home")) + .env("WORKON_HOME", &cached) + .args(["scan", "--mode", "hosted", "--yes", "--json", "--cwd"]) + .arg(&project) + .args([ + "--api-url", + &server.uri(), + "--org", + ORG, + "--api-token", + "fake", + "--vex", + ]) + .arg(&vex) + .args(["--vex-product", VEX_PRODUCT]); + if !forward_reference { + cmd.env("BASE", &legacy); + } + let out = cmd.output().await.unwrap(); + let json: serde_json::Value = serde_json::from_slice(&out.stdout).unwrap_or_else(|error| { + panic!( + "{error}: stdout={} stderr={}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) + }); + assert_eq!( + out.status.code(), + Some(1), + "forward={forward_reference}: {json}" + ); + assert!( + json["redirect"]["warnings"] + .as_array() + .unwrap() + .iter() + .any(|warning| warning["code"] == "redirect_pypi_stale_install"), + "{json}" + ); + assert!( + !vex.exists(), + "a healthy cached copy cannot attest the stale shell copy" + ); + assert_eq!(std::fs::read(stale_file).unwrap(), UPSTREAM); + assert_eq!(std::fs::read(healthy_file).unwrap(), PATCHED); + assert_eq!(read(&project.join(".venv")), "env\n"); + assert_eq!(read(&project.join("Pipfile")), PIPFILE); + } +} diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index 811c07bd7..4a141025b 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -692,19 +692,25 @@ fn pipenv_venv_in_project(cwd: &Path, var: &impl Fn(&str) -> Option) -> /// so both venvs are returned, WORKON_HOME first, and whichever one the /// installed Pipenv uses gets patched. /// -/// The settings come from the project's `.env` layered over the process -/// environment, as every Pipenv command loads it first (see -/// [`pipenv_dotenv`]), and then from the process environment alone (older -/// Pipenv read some settings before loading `.env`). Both views' venvs are -/// returned, the `.env` view first. Each view applies its own active-venv -/// decision before its in-project/WORKON_HOME placement. +/// Settings are resolved through concrete supported-generation views: +/// current dotenv settings, then process-only settings (older commands +/// cache placement before loading dotenv), then the 2018 and 2020 shell +/// profiles with cached Project settings. Only 2018 uses the older parser +/// and sets PIPENV_ACTIVE before its final placement lookup; 2020 uses +/// modern parsing but still caches IGNORE/IN_PROJECT. Only project-owned +/// environments from these views are +/// returned, with duplicates removed. /// /// Never `./venv`: no Pipenv release uses it. async fn pipenv_project_site_packages( cwd: &Path, var: &impl Fn(&str) -> Option, ) -> Vec { - let dotenv = pipenv_dotenv(cwd, var); + let text = pipenv_dotenv(cwd, var); + let dotenv = text + .as_deref() + .map(|text| parse_dotenv(text, var)) + .unwrap_or_default(); let mut results = Vec::new(); if !dotenv.is_empty() { let layered = |name: &str| { @@ -722,15 +728,41 @@ async fn pipenv_project_site_packages( results.push(site); } } + // Cached generations used Python bool(string), including "0" as true. + // The current profile above intentionally retains its newer flag parser. + if var("PIPENV_DONT_LOAD_ENV").is_some_and(|value| !value.is_empty()) { + return results; + } + if let Some(text) = text { + let legacy = parse_pipenv_2018_dotenv(&text, var); + if !legacy.is_empty() { + for site in + pipenv_cached_dotenv_site_packages(cwd, &legacy, var, PipenvCachedShell::V2018) + .await + { + if !results.contains(&site) { + results.push(site); + } + } + } + } + if !dotenv.is_empty() { + for site in + pipenv_cached_dotenv_site_packages(cwd, &dotenv, var, PipenvCachedShell::V2020).await + { + if !results.contains(&site) { + results.push(site); + } + } + } results } -/// The `.env` variables Pipenv loads before it resolves the venv -/// (`load_dot_env`, unchanged from 2018 through 2026): `PIPENV_DOTENV_LOCATION` -/// (relative to the project) or `/.env`, unless -/// `bool(PIPENV_DONT_LOAD_ENV)`. Pipenv loads it with `override=True`, so -/// these beat the process environment. Empty when there is nothing to load. -fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Vec<(String, String)> { +/// The dotenv file configured when Pipenv starts: `PIPENV_DOTENV_LOCATION` +/// (relative to the project), else `/.env`, unless +/// `bool(PIPENV_DONT_LOAD_ENV)`. Read it once for the supported parser and +/// settings-timing profiles; Pipenv commands load it at different points. +fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Option { let dont_load = match var("PIPENV_DONT_LOAD_ENV") { Some(value) => match value.to_ascii_lowercase().as_str() { "1" | "true" | "yes" | "on" => true, @@ -740,7 +772,7 @@ fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Vec<(Stri None => false, }; if dont_load { - return Vec::new(); + return None; } let path = match var("PIPENV_DOTENV_LOCATION").filter(|v| !v.is_empty()) { Some(location) => cwd.join(location), @@ -748,10 +780,7 @@ fn pipenv_dotenv(cwd: &Path, var: &impl Fn(&str) -> Option) -> Vec<(Stri }; // Regular files only, non-blocking: a FIFO `.env` must not wedge // discovery. - match read_regular_to_string_sync(&path) { - Ok(text) => parse_dotenv(&text, var), - Err(_) => Vec::new(), - } + read_regular_to_string_sync(&path).ok() } /// python-dotenv bindings are a stream, not independent lines: quoted @@ -948,6 +977,158 @@ fn dotenv_interpolate(value: &str, lookup: &impl Fn(&str) -> Option) -> out } +/// Pipenv 2018's shell loads dotenv before resolving its location, while +/// --venv/run may already have cached it. Its parser resolves the complete +/// final mapping in insertion order, with process variables taking priority. +/// This is one concrete legacy view, not arbitrary combinations of settings. +fn parse_pipenv_2018_dotenv( + text: &str, + var: &impl Fn(&str) -> Option, +) -> Vec<(String, String)> { + static VARIABLES: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"\$\{([^}]*)\}").unwrap()); + let normalized = text.replace("\r\n", "\n").replace('\r', "\n"); + let mut values: Vec<(String, String)> = Vec::new(); + for line in normalized.split('\n') { + let line = line.trim_matches(dotenv_whitespace); + if line.starts_with('#') { + continue; + } + let Some((key, value)) = line.split_once('=') else { + continue; + }; + let key = key + .strip_prefix("export ") + .unwrap_or(key) + .trim_matches(dotenv_whitespace); + if key.is_empty() { + continue; + } + let value = value.trim_matches(dotenv_whitespace); + let quote = value.chars().next(); + let value = if matches!(quote, Some('\'' | '"')) && quote == value.chars().next_back() { + // Legacy parse_line encodes unicode-escape first, then decodes + // matching quotes: the inner text (including escapes) survives. + value + .get(1..value.len().saturating_sub(1)) + .unwrap_or("") + .to_string() + } else { + pipenv_2018_unquoted_value(value) + }; + if let Some((_, previous)) = values.iter_mut().find(|(name, _)| name == key) { + *previous = value; + } else { + values.push((key.to_string(), value)); + } + } + for index in 0..values.len() { + let raw = values[index].1.clone(); + let mut resolved = String::new(); + let mut end = 0; + for captures in VARIABLES.captures_iter(&raw) { + let whole = captures.get(0).unwrap(); + let name = captures.get(1).unwrap().as_str(); + resolved.push_str(&raw[end..whole.start()]); + if let Some(value) = var(name).or_else(|| { + values + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + }) { + resolved.push_str(&value); + } + end = whole.end(); + } + resolved.push_str(&raw[end..]); + values[index].1 = resolved; + } + values +} + +/// The legacy parser leaves unicode-escape encoding on unquoted values. +/// Matching quoted values bypass this conversion above. +fn pipenv_2018_unquoted_value(value: &str) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + for c in value.chars() { + match c { + '\\' => out.push_str("\\\\"), + '\t' => out.push_str("\\t"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + ' '..='~' => out.push(c), + c if (c as u32) <= 0xff => { + let _ = write!(out, "\\x{:02x}", c as u32); + } + c if (c as u32) <= 0xffff => { + let _ = write!(out, "\\u{:04x}", c as u32); + } + c => { + let _ = write!(out, "\\U{:08x}", c as u32); + } + } + } + out +} + +/// Two native shell timing profiles: both cache Project settings before +/// dotenv; 2018 marks itself active before final placement, while 2020 +/// reads the active prefix first. Later releases reinitialize settings. +#[derive(Clone, Copy, PartialEq, Eq)] +enum PipenvCachedShell { + V2018, + V2020, +} + +async fn pipenv_cached_dotenv_site_packages( + cwd: &Path, + dotenv: &[(String, String)], + var: &impl Fn(&str) -> Option, + shell: PipenvCachedShell, +) -> Vec { + let layered = |name: &str| { + dotenv + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + .or_else(|| var(name)) + }; + // 2020's Project cached IGNORE before dotenv, but reads ACTIVE and + // VIRTUAL_ENV dynamically. In 2018 do_shell has already set ACTIVE, + // so an activated prefix cannot replace the final placement. + if shell == PipenvCachedShell::V2020 + && layered("PIPENV_ACTIVE").is_none() + && var("PIPENV_IGNORE_VIRTUALENVS").is_none_or(|value| value.is_empty()) + { + if let Some(prefix) = layered("VIRTUAL_ENV").filter(|prefix| !prefix.is_empty()) { + let found = find_site_packages_under(&cwd.join(prefix), "site-packages").await; + if !found.is_empty() { + return found; + } + } + } + let dot_venv = cwd.join(".venv"); + // An existing directory always wins in this generation; the cached + // flag only selects a missing in-project env. A .venv file still wins + // over that flag and is resolved by the shared placement helper. + if dot_venv.is_dir() + || (!dot_venv.exists() + && var("PIPENV_VENV_IN_PROJECT").is_some_and(|value| !value.is_empty())) + { + return find_site_packages_under(&dot_venv, "site-packages").await; + } + let workon_home = pipenv_workon_home(&layered).map(|home| cwd.join(home)); + let identity = |name: &str| match name { + // Project imported PIPFILE before dotenv; custom names did not + // exist yet. Path interpolation itself still sees the full overlay. + "PIPENV_PIPFILE" => var(name), + "PIPENV_CUSTOM_VENV_NAME" => None, + _ => layered(name), + }; + find_pipenv_virtualenv_site_packages_at(cwd, workon_home.as_deref(), &identity).await +} + /// [`pipenv_project_site_packages`] for one settings view. async fn pipenv_settings_site_packages( cwd: &Path, @@ -1458,6 +1639,18 @@ pub async fn find_pipenv_virtualenv_site_packages(cwd: &Path) -> Vec { async fn find_pipenv_virtualenv_site_packages_with( cwd: &Path, var: &impl Fn(&str) -> Option, +) -> Vec { + let workon_home = pipenv_workon_home(var); + find_pipenv_virtualenv_site_packages_at(cwd, workon_home.as_deref(), var).await +} + +/// Resolve a placement using an already-expanded WORKON_HOME. Legacy +/// Pipenv reads path variables after dotenv but caches project identity +/// inputs beforehand, so those two lookups must stay separate. +async fn find_pipenv_virtualenv_site_packages_at( + cwd: &Path, + workon_home: Option<&Path>, + var: &impl Fn(&str) -> Option, ) -> Vec { if !is_pipenv_project(cwd) { return Vec::new(); @@ -1473,15 +1666,15 @@ async fn find_pipenv_virtualenv_site_packages_with( if !name.is_empty() { if name.contains('/') || name.contains('\\') { venvs.push(cwd.join(name)); - } else if let Some(home) = pipenv_workon_home(var) { + } else if let Some(home) = workon_home { venvs.push(home.join(name)); } } } } if venvs.is_empty() { - if let Some(home) = pipenv_workon_home(var) { - venvs.extend(pipenv_workon_home_venvs(cwd, &home, var)); + if let Some(home) = workon_home { + venvs.extend(pipenv_workon_home_venvs(cwd, home, var)); } } let mut results = Vec::new(); @@ -3703,6 +3896,185 @@ mod tests { ); } + #[test] + fn pipenv_2018_dotenv_keeps_legacy_mapping_and_value_rules() { + let var = env_of(&[("ENV", "process".to_string())]); + for (text, expected) in [ + ( + "A=${B}\nB=forward\n", + vec![("A", "forward"), ("B", "forward")], + ), + ( + "ENV=file\nA=${ENV}\n", + vec![("ENV", "file"), ("A", "process")], + ), + ( + "A=first\nB=${A}\nA=last\n", + vec![("A", "last"), ("B", "last")], + ), + ("A=first\nA\nB=${A}\n", vec![("A", "first"), ("B", "first")]), + ("A=${MISSING:-default}\n", vec![("A", "")]), + ("A='literal\\ntext'\n", vec![("A", "literal\\ntext")]), + ("A=C:\\Temp\\x\n", vec![("A", "C:\\\\Temp\\\\x")]), + ("A=é\nB='é'\n", vec![("A", "\\xe9"), ("B", "é")]), + ("A=path # literal\n", vec![("A", "path # literal")]), + ] { + let expected = expected + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>(); + assert_eq!(parse_pipenv_2018_dotenv(text, &var), expected, "{text:?}"); + } + } + + #[tokio::test] + async fn pipenv_2020_dotenv_uses_cached_settings_and_dynamic_active_prefix() { + let (tmp, project, default, base_var) = pipenv_project_with_workon_venv(&[]); + let default_root = std::fs::read_dir(tmp.path().join("wh")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + let leaf = default_root.file_name().unwrap().to_str().unwrap(); + let modern_root = tmp.path().join("modern-workon"); + let legacy_root = tmp.path().join("legacy-workon"); + let modern = fake_venv(&modern_root, leaf); + let legacy = fake_venv(&legacy_root, leaf); + let var = |name: &str| match name { + "BASE" => Some(legacy_root.to_string_lossy().into_owned()), + _ => base_var(name), + }; + std::fs::write( + project.join(".env"), + format!( + "BASE={}\nWORKON_HOME=${{BASE}}\nPIPENV_VENV_IN_PROJECT=1\n", + modern_root.to_string_lossy().replace('\\', "/") + ), + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![default.clone(), legacy, modern.clone()], + "2020 cached IN_PROJECT before dotenv" + ); + let already_in_project = |name: &str| { + if name == "PIPENV_VENV_IN_PROJECT" { + Some("1".to_string()) + } else { + var(name) + } + }; + assert!( + find_local_venv_site_packages_with(&project, &already_in_project) + .await + .is_empty(), + "a real cached in-project setting must not fall back to WORKON_HOME" + ); + + let var = |name: &str| { + if name == "VIRTUAL_ENV" { + Some(default_root.to_string_lossy().into_owned()) + } else { + base_var(name) + } + }; + std::fs::write( + project.join(".env"), + format!( + "VIRTUAL_ENV={}\nPIPENV_IGNORE_VIRTUALENVS=1\n", + modern_root.join(leaf).to_string_lossy().replace('\\', "/") + ), + ) + .unwrap(); + assert_eq!( + find_local_venv_site_packages_with(&project, &var).await, + vec![default.clone(), modern], + "2020 cached IGNORE=false but reads the layered active prefix" + ); + let already_active = |name: &str| { + if name == "PIPENV_ACTIVE" { + Some("1".to_string()) + } else { + var(name) + } + }; + assert_eq!( + find_local_venv_site_packages_with(&project, &already_active).await, + vec![default], + "an existing ACTIVE marker still vetoes the active prefix" + ); + } + + #[tokio::test] + async fn pipenv_legacy_dotenv_shell_keeps_its_own_workon_home() { + let (tmp, project, default, base_var) = pipenv_project_with_workon_venv(&[]); + let leaf = std::fs::read_dir(tmp.path().join("wh")) + .unwrap() + .next() + .unwrap() + .unwrap() + .file_name(); + let legacy_root = tmp.path().join("legacy-workon"); + let modern_root = tmp.path().join("modern-workon"); + let legacy = fake_venv(&legacy_root, leaf.to_str().unwrap()); + let modern = fake_venv(&modern_root, leaf.to_str().unwrap()); + let unrelated = fake_venv(&legacy_root, "another-project-12345678"); + let legacy_text = legacy_root.to_string_lossy().replace('\\', "/"); + let modern_text = modern_root.to_string_lossy().replace('\\', "/"); + for (text, process_base, expected) in [ + ( + format!("WORKON_HOME=${{BASE}}\nBASE={legacy_text}\n"), + None, + vec![default.clone(), legacy.clone()], + ), + ( + format!("BASE={modern_text}\nWORKON_HOME=${{BASE}}\n"), + Some(legacy_text.clone()), + vec![modern.clone(), default.clone(), legacy.clone()], + ), + ] { + std::fs::write(project.join(".env"), &text).unwrap(); + let var = |name: &str| { + if name == "BASE" { + process_base.clone() + } else { + base_var(name) + } + }; + let found = find_local_venv_site_packages_with(&project, &var).await; + assert_eq!(found, expected, "dotenv {text:?}"); + assert!(!found.contains(&unrelated)); + let legacy_disabled = |name: &str| { + if name == "PIPENV_DONT_LOAD_ENV" { + Some("0".to_string()) + } else { + var(name) + } + }; + let without_legacy = expected + .into_iter() + .filter(|site| site != &legacy) + .collect::>(); + assert_eq!( + find_local_venv_site_packages_with(&project, &legacy_disabled).await, + without_legacy, + "cached generations treat nonempty 0 as true, while current dotenv still loads" + ); + let disabled = |name: &str| { + if name == "PIPENV_DONT_LOAD_ENV" { + Some("1".to_string()) + } else { + var(name) + } + }; + assert_eq!( + find_local_venv_site_packages_with(&project, &disabled).await, + vec![default.clone()] + ); + } + } + #[test] fn dotenv_bindings_match_native_quotes_and_record_boundaries() { let var = env_of(&[("NAME", "ambient".to_string())]); @@ -3800,9 +4172,12 @@ mod tests { .collect::>(); assert_eq!( found, + // Legacy shell ignores custom names and takes this + // project's own default even when an ambient env exists. vec![ actual.canonicalize().unwrap(), - ambient.canonicalize().unwrap() + ambient.canonicalize().unwrap(), + default.canonicalize().unwrap(), ], "dotenv {dotenv:?}" ); diff --git a/docs/testing/pipenv-compatibility.md b/docs/testing/pipenv-compatibility.md index 50a28efa5..faaca1b31 100644 --- a/docs/testing/pipenv-compatibility.md +++ b/docs/testing/pipenv-compatibility.md @@ -17,10 +17,12 @@ requirements.txt lanes of the same ecosystem. | Input | Hosted | Vendored | Agent | |-------|--------|----------|-------| -| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). Settings come from the project's `.env` (or `PIPENV_DOTENV_LOCATION`, unless `PIPENV_DONT_LOAD_ENV`) layered over the environment, as Pipenv loads it first, and from the environment alone. Each view applies its own active-venv decision before placement. The dotenv reader follows python-dotenv's record boundaries, quoted keys/values, multiline values, and interpolation (including single-quoted values); setting-like text inside a quoted value is not another setting. With a `.venv` directory and an existing WORKON_HOME venv, both are patched unless the project is explicitly in-project: Pipenv 2026.2+ prefers the WORKON_HOME venv when nothing is set, 2023.11.14+ uses it when the project is explicitly not in-project, and older releases use `.venv` either way. | +| `Pipfile.lock`, `pipfile-spec: 6` (Pipenv 7 and later) | Every category (`default`, `develop`, Pipenv 2022+ named categories) that pins the patched release becomes `{"file" \| "path": "#sha256=", "hashes": ["sha256:"]}` with `markers`/`extras`/`index` kept as Pipenv wrote them and `version` dropped. `path` for Pipenv 7–11, `file` from 2018. `_meta` (the Pipfile content hash) and the Pipfile are untouched. | Every matching category refers to the committed wheel under `.socket/vendor/pypi//`; wheels with extras use `path` (Pipenv 2022's file-URL bug). Requires Pipenv 2018 or later (`pypi_pipenv_installer_unsupported`). | Independent of the lock: patches the installed distribution in the venv Pipenv resolves for the project — `VIRTUAL_ENV` unless `PIPENV_ACTIVE` / `PIPENV_IGNORE_VIRTUALENVS` is set, in-project `.venv` subject to `PIPENV_VENV_IN_PROJECT` and the Pipfile's `[pipenv] venv_in_project`, or Pipenv's default `$WORKON_HOME/-[-]`; never `venv/` (discovered without running Pipenv). Settings come from the project's `.env` (or `PIPENV_DOTENV_LOCATION`, unless `PIPENV_DONT_LOAD_ENV`) and the process environment using the concrete current, process-cached, 2018 shell, and 2020 shell profiles described below. With a `.venv` directory and an existing WORKON_HOME venv, each profile retains its native in-project choice: Pipenv 2026.2+ prefers the WORKON_HOME venv when nothing is set, 2023.11.14+ uses it when the project is explicitly not in-project, and older releases use `.venv` either way. | | `Pipfile.lock`, `pipfile-spec` < 6 (Pipenv 0–6) | Refused (`redirect_pipenv_skipped`), lock untouched. | Refused (`pypi_pipenv_spec_unsupported`). | Works. | | Lock-only checkout (nothing installed) | Discovered from the lock and redirected. | Discovered from the lock; the patched wheel or source distribution is downloaded and verified from the service without a local install. | Nothing to patch (no installed distribution); the lock's pins are listed as lockfile-only packages. | +Pipenv command timing and dotenv behavior changed across supported releases. Current commands resolve complete python-dotenv records with preceding bindings ahead of process variables, then apply their active-environment settings. Older commands can cache placement before loading dotenv, so discovery also retains the process-only result. The 2018 shell instead resolves dotenv references from the complete final mapping with process variables first; its Project settings were cached before loading, and it marks itself active before final placement. The 2020 shell uses modern parsing but still caches Project settings such as `IGNORE_VIRTUALENVS` and `VENV_IN_PROJECT`; it reads the active prefix before setting `PIPENV_ACTIVE`. These two cached profiles do not support custom venv names. Discovery combines only project-owned environments from these observed profiles, without executing Pipenv or searching unrelated venvs. + Both hash fields are load-bearing: Pipenv 2023+ verifies the `#sha256=` URL fragment, 2018–2022 verify the `hashes` list, Pipenv 11 accepts either. A tampered hosted reference fails to install on every supported release. From 404b30e87c92bb63cef9384988e16b22c5d1c21f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 14:35:45 +0000 Subject: [PATCH 7/8] Spawn the pipenv redirect test CLI through hermetic::command Main's spawn_env_hygiene ratchet (#850) rejects new bare binary spawns; the two dotenv-view spawns in in_process_redirect_pipenv.rs now start from the shared hermetic builder and keep their own PIPENV_* and venv scrubs on top. Co-Authored-By: Claude --- .../tests/in_process_redirect_pipenv.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs index b187e86af..3c33af6d0 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect_pipenv.rs @@ -29,6 +29,8 @@ use std::path::Path; +#[path = "common/hermetic.rs"] +mod hermetic; #[path = "vex_e2e_common/mod.rs"] mod vex_e2e_common; #[path = "vex_pipenv_pip_steps/mod.rs"] @@ -646,7 +648,9 @@ async fn dotenv_selected_pipenv_install_is_checked_before_vex() { }; std::fs::write(project.join(".env"), &dotenv).unwrap(); let vex = project.join("out.vex.json"); - let mut cmd = tokio::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + let mut cmd = tokio::process::Command::from(hermetic::command(Path::new(env!( + "CARGO_BIN_EXE_socket-patch" + )))); for (key, _) in std::env::vars_os() { let key_text = key.to_string_lossy(); if key_text.starts_with("SOCKET_") @@ -755,7 +759,9 @@ async fn legacy_dotenv_workon_install_is_checked_before_vex() { }; std::fs::write(project.join(".env"), dotenv).unwrap(); let vex = project.join("out.vex.json"); - let mut cmd = tokio::process::Command::new(env!("CARGO_BIN_EXE_socket-patch")); + let mut cmd = tokio::process::Command::from(hermetic::command(Path::new(env!( + "CARGO_BIN_EXE_socket-patch" + )))); for (key, _) in std::env::vars_os() { let text = key.to_string_lossy(); if text.starts_with("SOCKET_") From 3ceb956083ab0b9a3b8d9871ca7de83d2befcfe7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 14:35:45 +0000 Subject: [PATCH 8/8] Resolve a relative WORKON_HOME from the Pipenv project The modern dotenv and process settings views passed WORKON_HOME through unjoined, so a relative value resolved against socket-patch's own cwd instead of the project Pipenv runs from (the legacy cached view already joined it). Under --cwd that missed the project's venv. Join it to the project directory in the shared helper, as the legacy path does. Co-Authored-By: Claude --- .../src/crawlers/python_crawler.rs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/crawlers/python_crawler.rs b/crates/socket-patch-core/src/crawlers/python_crawler.rs index a81392d91..7019e8945 100644 --- a/crates/socket-patch-core/src/crawlers/python_crawler.rs +++ b/crates/socket-patch-core/src/crawlers/python_crawler.rs @@ -1943,7 +1943,9 @@ async fn find_pipenv_virtualenv_site_packages_with( cwd: &Path, var: &impl Fn(&str) -> Option, ) -> Vec { - let workon_home = pipenv_workon_home(var); + // Pipenv runs from the project, so a relative WORKON_HOME (from `.env` + // or the process) is the project's, not this process's cwd (`--cwd`). + let workon_home = pipenv_workon_home(var).map(|home| cwd.join(home)); find_pipenv_virtualenv_site_packages_at(cwd, workon_home.as_deref(), var).await } @@ -4831,6 +4833,26 @@ mod tests { ); } + /// A relative WORKON_HOME names a directory under the project, not + /// under this process's cwd: `--cwd` scans must still find the venv. + #[tokio::test] + async fn pipenv_relative_workon_home_resolves_from_the_project() { + let (_tmp, project, site, base_var) = pipenv_project_with_workon_venv(&[]); + let var = |name: &str| { + if name == "WORKON_HOME" { + Some("../wh".to_string()) + } else { + base_var(name) + } + }; + let found = find_pipenv_virtualenv_site_packages_with(&project, &var).await; + let canon = |p: &PathBuf| std::fs::canonicalize(p).unwrap(); + assert_eq!( + found.iter().map(canon).collect::>(), + vec![canon(&site)] + ); + } + #[tokio::test] async fn pipenv_legacy_dotenv_shell_keeps_its_own_workon_home() { let (tmp, project, default, base_var) = pipenv_project_with_workon_venv(&[]);