Add --base <run_id> to compare local run with another run

src/api_client.rs

@@ -81,39 +81,12 @@ nest! {
 
 #[derive(Serialize, Clone)]
 #[serde(rename_all = "camelCase")]
-pub struct FetchLocalRunReportVars {
+pub struct FetchLocalRunVars {
     pub owner: String,
     pub name: String,
     pub run_id: String,
 }
 
-#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
-pub enum ReportConclusion {
-    AcknowledgedFailure,
-    Failure,
-    MissingBaseRun,
-    NoBenchmarks,
-    Success,
-}
-
-impl Display for ReportConclusion {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            ReportConclusion::AcknowledgedFailure => {
-                write!(f, "{}", style("Acknowledged Failure").yellow().bold())
-            }
-            ReportConclusion::Failure => write!(f, "{}", style("Failure").red().bold()),
-            ReportConclusion::MissingBaseRun => {
-                write!(f, "{}", style("Missing Base Run").yellow().bold())
-            }
-            ReportConclusion::NoBenchmarks => {
-                write!(f, "{}", style("No Benchmarks").yellow().bold())
-            }
-            ReportConclusion::Success => write!(f, "{}", style("Success").green().bold()),
-        }
-    }
-}
-
 #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
 pub enum RunStatus {
@@ -123,18 +96,23 @@ pub enum RunStatus {
     Processing,
 }
 
+// Custom deserializer to convert string values to i64
+fn deserialize_i64_from_string<'de, D>(deserializer: D) -> Result<i64, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    use serde::de;
+    let s = String::deserialize(deserializer)?;
+    s.parse().map_err(de::Error::custom)
+}
+
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    pub struct FetchLocalRunReportRun {
+    pub struct FetchLocalRunRun {
         pub id: String,
         pub status: RunStatus,
         pub url: String,
-        pub head_reports: Vec<pub struct FetchLocalRunReportHeadReport {
-            pub id: String,
-            pub impact: Option<f64>,
-            pub conclusion: ReportConclusion,
-        }>,
         pub results: Vec<pub struct FetchLocalRunBenchmarkResult {
             pub value: f64,
             pub benchmark: pub struct FetchLocalRunBenchmark {
@@ -166,42 +144,113 @@ nest! {
     }
 }
 
-// Custom deserializer to convert string values to i64
-fn deserialize_i64_from_string<'de, D>(deserializer: D) -> Result<i64, D::Error>
-where
-    D: serde::Deserializer<'de>,
-{
-    use serde::de;
-    let s = String::deserialize(deserializer)?;
-    s.parse().map_err(de::Error::custom)
+nest! {
+    #[derive(Debug, Deserialize, Serialize)]*
+    #[serde(rename_all = "camelCase")]*
+    struct FetchLocalRunData {
+        repository: struct FetchLocalRunRepository {
+            run: FetchLocalRunRun,
+        }
+    }
+}
+
+pub struct FetchLocalRunResponse {
+    pub run: FetchLocalRunRun,
+}
+
+#[derive(Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct CompareRunsVars {
+    pub owner: String,
+    pub name: String,
+    pub base_run_id: String,
+    pub head_run_id: String,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "PascalCase")]
+pub enum ResultComparisonCategory {
+    Acknowledged,
+    Archived,
+    Ignored,
+    Improvement,
+    New,
+    Regression,
+    Skipped,
+    Untouched,
+}
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
+pub enum BenchmarkReportStatus {
+    Improvement,
+    Missing,
+    New,
+    NoChange,
+    Regression,
+}
+
+impl Display for BenchmarkReportStatus {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BenchmarkReportStatus::Improvement => {
+                write!(f, "{}", style("Improvement").green().bold())
+            }
+            BenchmarkReportStatus::Missing => write!(f, "{}", style("Missing").yellow().bold()),
+            BenchmarkReportStatus::New => write!(f, "{}", style("New").cyan().bold()),
+            BenchmarkReportStatus::NoChange => write!(f, "{}", style("No Change").dim()),
+            BenchmarkReportStatus::Regression => write!(f, "{}", style("Regression").red().bold()),
+        }
+    }
 }
 
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    struct FetchLocalRunReportData {
-        repository: pub struct FetchLocalRunReportRepository {
-            settings: struct FetchLocalRunReportSettings {
-                allowed_regression: f64,
-            },
-            run: FetchLocalRunReportRun,
-        }
+    pub struct CompareRunsBenchmarkResult {
+        pub value: Option<f64>,
+        pub base_value: Option<f64>,
+        pub change: Option<f64>,
+        pub category: ResultComparisonCategory,
+        pub status: BenchmarkReportStatus,
+        pub benchmark: pub struct CompareRunsBenchmark {
+            pub name: String,
+            pub executor: ExecutorName,
+        },
+    }
+}
+
+nest! {
+    #[derive(Debug, Deserialize, Serialize)]*
+    #[serde(rename_all = "camelCase")]*
+    pub struct CompareRunsHeadRun {
+        pub id: String,
+        pub status: RunStatus,
     }
 }
 
 nest! {
     #[derive(Debug, Deserialize, Serialize)]*
     #[serde(rename_all = "camelCase")]*
-    struct FetchLocalExecReportData {
-        project: pub struct FetchLocalExecReportProject {
-            run: FetchLocalRunReportRun,
+    struct CompareRunsData {
+        repository: struct CompareRunsRepository {
+            paginated_compare_runs: pub struct CompareRunsComparison {
+                pub impact: Option<f64>,
+                pub url: String,
+                pub head_run: CompareRunsHeadRun,
+                pub result_comparisons: Vec<CompareRunsBenchmarkResult>,
+            },
         }
     }
 }
 
-pub struct FetchLocalRunReportResponse {
-    pub allowed_regression: f64,
-    pub run: FetchLocalRunReportRun,
+pub struct CompareRunsResponse {
+    pub comparison: CompareRunsComparison,
+}
+
+pub enum CompareRunsOutcome {
+    Success(CompareRunsResponse),
+    BaseRunNotFound,
+    ExecutorMismatch,
 }
 
 #[derive(Serialize, Clone)]
@@ -274,26 +323,47 @@ impl CodSpeedAPIClient {
         }
     }
 
-    pub async fn fetch_local_run_report(
-        &self,
-        vars: FetchLocalRunReportVars,
-    ) -> Result<FetchLocalRunReportResponse> {
+    pub async fn compare_runs(&self, vars: CompareRunsVars) -> Result<CompareRunsOutcome> {
         let response = self
             .gql_client
-            .query_with_vars_unwrap::<FetchLocalRunReportData, FetchLocalRunReportVars>(
-                include_str!("queries/FetchLocalRunReport.gql"),
-                vars.clone(),
+            .query_with_vars_unwrap::<CompareRunsData, CompareRunsVars>(
+                include_str!("queries/CompareRuns.gql"),
+                vars,
+            )
+            .await;
+        match response {
+            Ok(response) => Ok(CompareRunsOutcome::Success(CompareRunsResponse {
+                comparison: response.repository.paginated_compare_runs,
+            })),
+            Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
+                bail!("Your session has expired, please login again using `codspeed auth login`")
+            }
+            Err(err) if err.contains_error_code("RUN_NOT_FOUND") => {
+                Ok(CompareRunsOutcome::BaseRunNotFound)
+            }
+            Err(err) if err.contains_error_code("NOT_FOUND") => {
+                Ok(CompareRunsOutcome::ExecutorMismatch)
+            }
+            Err(err) => bail!("Failed to compare runs: {err:?}"),
+        }
+    }
+
+    pub async fn fetch_local_run(&self, vars: FetchLocalRunVars) -> Result<FetchLocalRunResponse> {
+        let response = self
+            .gql_client
+            .query_with_vars_unwrap::<FetchLocalRunData, FetchLocalRunVars>(
+                include_str!("queries/FetchLocalRun.gql"),
+                vars,
             )
             .await;
         match response {
-            Ok(response) => Ok(FetchLocalRunReportResponse {
-                allowed_regression: response.repository.settings.allowed_regression,
+            Ok(response) => Ok(FetchLocalRunResponse {
                 run: response.repository.run,
             }),
             Err(err) if err.contains_error_code("UNAUTHENTICATED") => {
                 bail!("Your session has expired, please login again using `codspeed auth login`")
             }
-            Err(err) => bail!("Failed to fetch local run report: {err}"),
+            Err(err) => bail!("Failed to fetch local run: {err}"),
         }
     }
 

src/cli/exec/mod.rs

@@ -59,6 +59,7 @@ impl ExecArgs {
 fn build_orchestrator_config(
     args: ExecArgs,
     target: executor::BenchmarkTarget,
+    poll_results_options: PollResultsOptions,
 ) -> Result<OrchestratorConfig> {
     let modes = args.shared.resolve_modes()?;
     let raw_upload_url = args
@@ -90,7 +91,7 @@ fn build_orchestrator_config(
         allow_empty: args.shared.allow_empty,
         go_runner_version: args.shared.go_runner_version,
         show_full_output: args.shared.show_full_output,
-        poll_results_options: PollResultsOptions::for_exec(),
+        poll_results_options,
         extra_env: HashMap::new(),
     })
 }
@@ -103,12 +104,17 @@ pub async fn run(
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
     let merged_args = args.merge_with_project_config(project_config);
+    let base_run_id = merged_args.shared.base.clone();
     let target = executor::BenchmarkTarget::Exec {
         command: merged_args.command.clone(),
         name: merged_args.name.clone(),
         walltime_args: merged_args.walltime_args.clone(),
     };
-    let config = build_orchestrator_config(merged_args, target)?;
+    let config = build_orchestrator_config(
+        merged_args,
+        target,
+        PollResultsOptions::new(false, base_run_id),
+    )?;
 
     execute_config(config, api_client, codspeed_config, setup_cache_dir).await
 }

src/cli/run/helpers/mod.rs

@@ -1,4 +1,3 @@
-pub(crate) mod benchmark_display;
 mod download_file;
 mod find_repository_root;
 mod format_duration;

src/cli/run/mod.rs

@@ -81,6 +81,7 @@ impl RunArgs {
                 allow_empty: false,
                 go_runner_version: None,
                 show_full_output: false,
+                base: None,
                 perf_run_args: PerfRunArgs {
                     enable_perf: false,
                     perf_unwinding_mode: None,
@@ -156,6 +157,7 @@ pub async fn run(
     setup_cache_dir: Option<&Path>,
 ) -> Result<()> {
     let output_json = args.message_format == Some(MessageFormat::Json);
+    let base_run_id = args.shared.base.clone();
 
     let args = args.merge_with_project_config(project_config);
 
@@ -184,13 +186,14 @@ pub async fn run(
     match run_target {
         RunTarget::SingleCommand(args) => {
             let command = args.command.join(" ");
+            let poll_opts = PollResultsOptions::new(output_json, base_run_id);
             let config = build_orchestrator_config(
                 args,
                 vec![executor::BenchmarkTarget::Entrypoint {
                     command,
                     name: None,
                 }],
-                PollResultsOptions::for_run(output_json),
+                poll_opts,
             )?;
             let orchestrator =
                 executor::Orchestrator::new(config, codspeed_config, api_client).await?;
@@ -210,8 +213,8 @@ pub async fn run(
         } => {
             let benchmark_targets =
                 super::exec::multi_targets::build_benchmark_targets(targets, default_walltime)?;
-            let config =
-                build_orchestrator_config(args, benchmark_targets, PollResultsOptions::for_exec())?;
+            let poll_opts = PollResultsOptions::new(false, base_run_id);
+            let config = build_orchestrator_config(args, benchmark_targets, poll_opts)?;
             super::exec::execute_config(config, api_client, codspeed_config, setup_cache_dir)
                 .await?;
         }

src/cli/shared.rs

@@ -110,6 +110,10 @@ pub struct ExecAndRunSharedArgs {
     #[arg(long, default_value = "false")]
     pub show_full_output: bool,
 
+    /// Compare the results against this base run ID
+    #[arg(long)]
+    pub base: Option<String>,
+
     #[command(flatten)]
     pub perf_run_args: PerfRunArgs,
 }

src/executor/config.rs

@@ -221,7 +221,7 @@ impl OrchestratorConfig {
             allow_empty: false,
             go_runner_version: None,
             show_full_output: false,
-            poll_results_options: PollResultsOptions::for_exec(),
+            poll_results_options: PollResultsOptions::new(false, None),
             extra_env: HashMap::new(),
         }
     }

src/main.rs

@@ -1,9 +1,26 @@
 use codspeed_runner::{clean_logger, cli};
-use console::style;
+use console::{Term, style};
 use log::log_enabled;
 
+struct HiddenCursor(Term);
+
+impl HiddenCursor {
+    fn new() -> Self {
+        let term = Term::stderr();
+        let _ = term.hide_cursor();
+        Self(term)
+    }
+}
+
+impl Drop for HiddenCursor {
+    fn drop(&mut self) {
+        let _ = self.0.show_cursor();
+    }
+}
+
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
+    let _cursor = HiddenCursor::new();
     let res = cli::run().await;
     if let Err(err) = res {
         // Show the primary error