From c98113a402e25d755ab9e67ed1983b797f377594 Mon Sep 17 00:00:00 2001 From: Makai Date: Fri, 20 Mar 2026 09:54:18 +0800 Subject: [PATCH 1/2] Implement symmetry filter --- josh-sync.example.toml | 8 +++++ src/bin/rustc_josh_sync.rs | 1 + src/config.rs | 2 ++ src/josh.rs | 46 ++++++++++++++++++++++++-- src/sync.rs | 67 +++++++++++++++++++++++++++----------- src/utils.rs | 52 ++++++++++++++++++++++++++++- 6 files changed, 154 insertions(+), 22 deletions(-) diff --git a/josh-sync.example.toml b/josh-sync.example.toml index 998005b..85fd3d4 100644 --- a/josh-sync.example.toml +++ b/josh-sync.example.toml @@ -15,3 +15,11 @@ path = "library/stdarch" #[[post-pull]] #cmd = ["cargo", "fmt"] #commit-message = "reformat" + +# Optionally, you can specify a symmetry filter. +# This will be applied to the local `HEAD` during the round-trip check. +# By default, the round-trip check expects the local repo to be a perfect 1:1 mirror +# of the upstream tree. Leave this empty if that is your intent. +# If you are not doing a 1:1 mirror, set this to a filter that transforms +# your local `HEAD` into a tree that exactly matches the upstream tree. +#sym-filter = ":/rustc_public:exclude[:[::build.rs,::tests/,::Cargo.toml]]:prefix=rustc_public" diff --git a/src/bin/rustc_josh_sync.rs b/src/bin/rustc_josh_sync.rs index 2c8a05a..da6535e 100644 --- a/src/bin/rustc_josh_sync.rs +++ b/src/bin/rustc_josh_sync.rs @@ -87,6 +87,7 @@ fn main() -> anyhow::Result<()> { path: Some("".to_string()), filter: None, post_pull: vec![], + sym_filter: None, }; config .write(Path::new(DEFAULT_CONFIG_PATH)) diff --git a/src/config.rs b/src/config.rs index ccbd9bd..273e067 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,6 +17,8 @@ pub struct JoshConfig { /// Can be used to post-process the state of the repository after a pull happens. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub post_pull: Vec, + /// Optional symmetry filter applied to the local `HEAD` during round-trip check. + pub sym_filter: Option, } /// Execute an operation after a pull, and if something changes in the local git state, diff --git a/src/josh.rs b/src/josh.rs index da7cf52..094afef 100644 --- a/src/josh.rs +++ b/src/josh.rs @@ -1,8 +1,8 @@ use crate::config::JoshConfig; -use crate::utils::run_command; +use crate::utils::{is_null_sha, run_command, run_command_by_path}; use anyhow::Context; use std::net::{SocketAddr, TcpStream}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::Duration; @@ -14,6 +14,10 @@ pub struct JoshProxy { path: PathBuf, } +pub struct JoshFilter { + path: PathBuf, +} + impl JoshProxy { /// Tries to figure out if `josh-proxy` is installed. pub fn lookup() -> Option { @@ -82,6 +86,44 @@ pub fn try_install_josh(verbose: bool) -> Option { JoshProxy::lookup() } +impl JoshFilter { + /// Tries to figure out if `josh-filter` is installed. + pub fn lookup() -> Option { + which::which("josh-filter").ok().map(|path| Self { path }) + } + + pub fn run<'a, Args: AsRef<[&'a str]>>( + &self, + args: Args, + workdir: &Path, + verbose: bool, + ) -> anyhow::Result<()> { + let args = args.as_ref(); + let output = run_command_by_path(&self.path, args, workdir, true, verbose)?; + if is_null_sha(&output) { + return Err(anyhow::anyhow!( + "josh-filter returned null SHA, filter may not match any content" + )); + } + Ok(()) + } +} + +pub fn try_install_josh_filter(verbose: bool) -> Option { + run_command( + &[ + "cargo", + "install", + "josh-cli", + "--git", + "https://github.com/josh-project/josh.git", + ], + verbose, + ) + .expect("cannot install josh-filter"); + JoshFilter::lookup() +} + /// Create a wrapper that represents a running instance of `josh-proxy` and stops it on drop. pub struct RunningJoshProxy { process: std::process::Child, diff --git a/src/sync.rs b/src/sync.rs index f9f0d26..79ecf43 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -1,6 +1,6 @@ use crate::SyncContext; -use crate::config::PostPullOperation; -use crate::josh::JoshProxy; +use crate::config::{JoshConfig, PostPullOperation}; +use crate::josh::{JoshFilter, JoshProxy, try_install_josh_filter}; use crate::utils::{ensure_clean_git_state, prompt}; use crate::utils::{get_current_head_sha, run_command_at}; use crate::utils::{run_command, stream_command}; @@ -329,23 +329,7 @@ After you fix the conflicts, `git add` the changes and run `git merge --continue println!(); // Do a round-trip check to make sure the push worked as expected. - run_command_at( - &["git", "fetch", &josh_url, &branch], - &std::env::current_dir().unwrap(), - self.verbose, - )?; - let head = get_current_head_sha(self.verbose)?; - let fetch_head = run_command(&["git", "rev-parse", "FETCH_HEAD"], self.verbose)?; - if head != fetch_head { - return Err(anyhow::anyhow!( - "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ - Expected {head}, got {fetch_head}." - )); - } - println!( - "Confirmed that the push round-trips back to {} properly. Please create a rustc PR.", - self.context.config.repo - ); + self.roundtrip_check(&self.context.config, &josh_url, &branch)?; Ok(()) } @@ -370,6 +354,51 @@ After you fix the conflicts, `git add` the changes and run `git merge --continue Ok(()) } + + fn roundtrip_check( + &self, + config: &JoshConfig, + josh_url: &str, + branch: &str, + ) -> anyhow::Result<()> { + run_command_at( + &["git", "fetch", josh_url, branch], + &std::env::current_dir().unwrap(), + self.verbose, + )?; + let head = if let Some(sym_filter) = &config.sym_filter { + let josh_filter = get_josh_filter(self.verbose)?; + josh_filter.run( + &[sym_filter, "HEAD"], + &std::env::current_dir().unwrap(), + self.verbose, + )?; + run_command(&["git", "rev-parse", "FILTERED_HEAD"], self.verbose) + .context("failed to get FILTERED_HEAD")? + } else { + get_current_head_sha(self.verbose)? + }; + let fetch_head = run_command(&["git", "rev-parse", "FETCH_HEAD"], self.verbose)?; + if head != fetch_head { + return Err(anyhow::anyhow!( + "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ + Expected {head}, got {fetch_head}." + )); + } + println!( + "Confirmed that the push round-trips back to {} properly. Please create a rustc PR.", + self.context.config.repo + ); + Ok(()) + } +} + +fn get_josh_filter(verbose: bool) -> anyhow::Result { + println!("Updating/installing josh-filter binary..."); + match try_install_josh_filter(verbose) { + Some(filter) => Ok(filter), + None => Err(anyhow::anyhow!("Could not install josh-filter")), + } } /// Find a rustc repo we can do our push preparation in. diff --git a/src/utils.rs b/src/utils.rs index 87204e8..19dffb7 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,5 +1,5 @@ use anyhow::Context; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; /// Run command and return its stdout. @@ -68,6 +68,51 @@ fn run_command_inner<'a, Args: AsRef<[&'a str]>>( } } +pub fn run_command_by_path<'a, Args: AsRef<[&'a str]>>( + cmd: &PathBuf, + args: Args, + workdir: &Path, + capture: bool, + verbose: bool, +) -> anyhow::Result { + let args = args.as_ref(); + + let mut cmd = Command::new(cmd); + cmd.current_dir(workdir); + cmd.args(args); + + if verbose { + eprintln!("+ {cmd:?}"); + } + if capture { + let out = cmd.output().expect("command failed"); + let stdout = String::from_utf8_lossy(out.stdout.trim_ascii()).to_string(); + let stderr = String::from_utf8_lossy(out.stderr.trim_ascii()).to_string(); + if !out.status.success() { + Err(anyhow::anyhow!( + "Command `{cmd:?}` failed with exit code {:?}. STDOUT:\n{stdout}\nSTDERR:\n{stderr}", + out.status.code() + )) + } else { + Ok(stdout) + } + } else { + let status = cmd + .spawn() + .expect("cannot spawn command") + .wait() + .expect("command failed"); + if !status.success() { + Err(anyhow::anyhow!( + "Command `{cmd:?}` failed with exit code {:?}", + status.code() + )) + } else { + Ok(String::new()) + } + } +} + /// Fail if there are files that need to be checked in. pub fn ensure_clean_git_state(verbose: bool) -> anyhow::Result<()> { let read = run_command( @@ -105,3 +150,8 @@ pub fn read_line() -> String { .expect("cannot read line from stdin"); line.trim().to_string() } + +pub fn is_null_sha(s: &str) -> bool { + let s = s.trim(); + !s.is_empty() && s.chars().all(|c| c == '0') +} From 57e0aaf4be9cc454ca799feb93f85cade0afcd68 Mon Sep 17 00:00:00 2001 From: Makai Date: Thu, 26 Mar 2026 23:25:58 +0800 Subject: [PATCH 2/2] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jakub Beránek --- josh-sync.example.toml | 13 +++++++++---- src/bin/rustc_josh_sync.rs | 2 +- src/config.rs | 5 +++-- src/josh.rs | 8 ++++++-- src/sync.rs | 4 ++-- src/utils.rs | 35 +++++------------------------------ 6 files changed, 26 insertions(+), 41 deletions(-) diff --git a/josh-sync.example.toml b/josh-sync.example.toml index 85fd3d4..ece01db 100644 --- a/josh-sync.example.toml +++ b/josh-sync.example.toml @@ -16,10 +16,15 @@ path = "library/stdarch" #cmd = ["cargo", "fmt"] #commit-message = "reformat" -# Optionally, you can specify a symmetry filter. +# Optionally, you can specify a subtree filter. # This will be applied to the local `HEAD` during the round-trip check. +# # By default, the round-trip check expects the local repo to be a perfect 1:1 mirror # of the upstream tree. Leave this empty if that is your intent. -# If you are not doing a 1:1 mirror, set this to a filter that transforms -# your local `HEAD` into a tree that exactly matches the upstream tree. -#sym-filter = ":/rustc_public:exclude[:[::build.rs,::tests/,::Cargo.toml]]:prefix=rustc_public" +# +# If you are not doing a 1:1 mirror, set this to a filter that transforms your local +# `HEAD` into a tree that exactly matches the upstream tree after the `filter` +# is applied. +# E.g., if the `filter` is ":/compiler/rustc_public:prefix=rustc_public", +# the `subtree-filter` should be: +#subtree-filter = ":/rustc_public:prefix=rustc_public" diff --git a/src/bin/rustc_josh_sync.rs b/src/bin/rustc_josh_sync.rs index da6535e..f149ba7 100644 --- a/src/bin/rustc_josh_sync.rs +++ b/src/bin/rustc_josh_sync.rs @@ -87,7 +87,7 @@ fn main() -> anyhow::Result<()> { path: Some("".to_string()), filter: None, post_pull: vec![], - sym_filter: None, + subtree_filter: None, }; config .write(Path::new(DEFAULT_CONFIG_PATH)) diff --git a/src/config.rs b/src/config.rs index 273e067..748e1fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,8 +17,9 @@ pub struct JoshConfig { /// Can be used to post-process the state of the repository after a pull happens. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub post_pull: Vec, - /// Optional symmetry filter applied to the local `HEAD` during round-trip check. - pub sym_filter: Option, + /// Optional subtree filter applied to the local `HEAD` during round-trip check. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subtree_filter: Option, } /// Execute an operation after a pull, and if something changes in the local git state, diff --git a/src/josh.rs b/src/josh.rs index 094afef..20e2d71 100644 --- a/src/josh.rs +++ b/src/josh.rs @@ -113,10 +113,14 @@ pub fn try_install_josh_filter(verbose: bool) -> Option { run_command( &[ "cargo", + "+stable", "install", - "josh-cli", + "--locked", "--git", - "https://github.com/josh-project/josh.git", + "https://github.com/josh-project/josh", + "--tag", + JOSH_VERSION, + "josh-filter", ], verbose, ) diff --git a/src/sync.rs b/src/sync.rs index 79ecf43..f555176 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -366,10 +366,10 @@ After you fix the conflicts, `git add` the changes and run `git merge --continue &std::env::current_dir().unwrap(), self.verbose, )?; - let head = if let Some(sym_filter) = &config.sym_filter { + let head = if let Some(subtree_filter) = &config.subtree_filter { let josh_filter = get_josh_filter(self.verbose)?; josh_filter.run( - &[sym_filter, "HEAD"], + &[subtree_filter, "HEAD"], &std::env::current_dir().unwrap(), self.verbose, )?; diff --git a/src/utils.rs b/src/utils.rs index 19dffb7..9c3a6ef 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -36,36 +36,7 @@ fn run_command_inner<'a, Args: AsRef<[&'a str]>>( cmd.current_dir(workdir); cmd.args(&args[1..]); - if verbose { - eprintln!("+ {cmd:?}"); - } - if capture { - let out = cmd.output().expect("command failed"); - let stdout = String::from_utf8_lossy(out.stdout.trim_ascii()).to_string(); - let stderr = String::from_utf8_lossy(out.stderr.trim_ascii()).to_string(); - if !out.status.success() { - Err(anyhow::anyhow!( - "Command `{cmd:?}` failed with exit code {:?}. STDOUT:\n{stdout}\nSTDERR:\n{stderr}", - out.status.code() - )) - } else { - Ok(stdout) - } - } else { - let status = cmd - .spawn() - .expect("cannot spawn command") - .wait() - .expect("command failed"); - if !status.success() { - Err(anyhow::anyhow!( - "Command `{cmd:?}` failed with exit code {:?}", - status.code() - )) - } else { - Ok(String::new()) - } - } + execute_command(cmd, capture, verbose) } pub fn run_command_by_path<'a, Args: AsRef<[&'a str]>>( @@ -81,6 +52,10 @@ pub fn run_command_by_path<'a, Args: AsRef<[&'a str]>>( cmd.current_dir(workdir); cmd.args(args); + execute_command(cmd, capture, verbose) +} + +fn execute_command(mut cmd: Command, capture: bool, verbose: bool) -> anyhow::Result { if verbose { eprintln!("+ {cmd:?}"); }