feat(cargo-codspeed): allow build with multiple modes

Author: not-matthias

crates/cargo-codspeed/src/app.rs

@@ -1,6 +1,10 @@
-use crate::{measurement_mode::MeasurementMode, prelude::*, run::run_benches};
+use crate::{
+    measurement_mode::{BuildMode, MeasurementMode},
+    prelude::*,
+    run::run_benches,
+};
 use cargo_metadata::MetadataCommand;
-use clap::{Args, Parser, Subcommand};
+use clap::{ArgAction, Args, Parser, Subcommand};
 use std::{ffi::OsString, process::exit};
 
 use crate::build::{build_benches, BuildConfig};
@@ -12,17 +16,12 @@ struct Cli {
     #[arg(short, long, global = true)]
     quiet: bool,
 
-    /// The measurement tool to use for measuring performance.
-    /// Automatically set to `walltime` on macro runners
-    #[arg(short, long, global = true, env = "CODSPEED_RUNNER_MODE")]
-    measurement_mode: Option<MeasurementMode>,
-
     #[command(subcommand)]
     command: Commands,
 }
 
 const PACKAGE_HELP: &str = "Package Selection";
-#[derive(Args)]
+#[derive(Args, Clone)]
 pub(crate) struct PackageFilters {
     /// Select all packages in the workspace
     #[arg(long, help_heading = PACKAGE_HELP)]
@@ -35,7 +34,7 @@ pub(crate) struct PackageFilters {
     pub(crate) package: Vec<String>,
 }
 
-#[derive(Args)]
+#[derive(Args, Clone)]
 pub(crate) struct BenchTargetFilters {
     /// Select only the specified benchmark target (all benchmark targets by default)
     #[arg(long, help_heading = TARGET_HELP)]
@@ -89,6 +88,18 @@ enum Commands {
 
         #[command(flatten)]
         bench_target_filters: BenchTargetFilters,
+
+        /// The measurement tool(s) to use for measuring performance.
+        /// Can be specified multiple times or comma-separated.
+        #[arg(
+            short = 'm',
+            long = "measurement-mode",
+            value_delimiter = ',',
+            action = ArgAction::Append,
+            help_heading = COMPILATION_HELP,
+            env = "CODSPEED_RUNNER_MODE"
+        )]
+        measurement_mode: Vec<MeasurementMode>,
     },
     /// Run the previously built benchmarks
     Run {
@@ -100,16 +111,18 @@ enum Commands {
 
         #[command(flatten)]
         bench_target_filters: BenchTargetFilters,
+
+        /// The measurement tool to use for measuring performance.
+        /// Automatically set to `walltime` on macro runners
+        #[arg(short = 'm', long = "measurement-mode", env = "CODSPEED_RUNNER_MODE")]
+        measurement_mode: Option<MeasurementMode>,
     },
 }
 
 pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
     let metadata = MetadataCommand::new().exec()?;
     let cli = Cli::try_parse_from(args)?;
 
-    let measurement_mode = cli.measurement_mode.unwrap_or_default();
-    eprintln!("[cargo-codspeed] Measurement mode: {measurement_mode:?}\n");
-
     let res = match cli.command {
         Commands::Build {
             package_filters,
@@ -122,6 +135,7 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
             locked,
             offline,
             frozen,
+            measurement_mode,
         } => {
             let passthrough_flags = {
                 let mut passthrough_flags = Vec::new();
@@ -147,30 +161,51 @@ pub fn run(args: impl Iterator<Item = OsString>) -> Result<()> {
             };
             let features =
                 features.map(|f| f.split([' ', ',']).map(|s| s.to_string()).collect_vec());
-            build_benches(
-                &metadata,
-                BuildConfig {
-                    package_filters,
-                    bench_target_filters,
-                    features,
-                    profile,
-                    quiet: cli.quiet,
-                    measurement_mode,
-                    passthrough_flags,
-                },
-            )
+
+            let build_modes: Vec<BuildMode> = measurement_mode
+                .into_iter()
+                .map(BuildMode::from)
+                .unique()
+                .collect();
+            let build_modes = if build_modes.is_empty() {
+                vec![BuildMode::default()]
+            } else {
+                build_modes
+            };
+
+            for build_mode in build_modes {
+                eprintln!("[cargo-codspeed] Building with build mode: {build_mode}\n");
+                build_benches(
+                    &metadata,
+                    BuildConfig {
+                        package_filters: package_filters.clone(),
+                        bench_target_filters: bench_target_filters.clone(),
+                        features: features.clone(),
+                        profile: profile.clone(),
+                        quiet: cli.quiet,
+                        build_mode,
+                        passthrough_flags: passthrough_flags.clone(),
+                    },
+                )?;
+            }
+            Ok(())
         }
         Commands::Run {
             benchname,
             package_filters,
             bench_target_filters,
-        } => run_benches(
-            &metadata,
-            benchname,
-            package_filters,
-            bench_target_filters,
             measurement_mode,
-        ),
+        } => {
+            let mode = measurement_mode.unwrap_or_default();
+            eprintln!("[cargo-codspeed] Measurement mode: {mode:?}\n");
+            run_benches(
+                &metadata,
+                benchname,
+                package_filters,
+                bench_target_filters,
+                mode,
+            )
+        }
     };
 
     if let Err(e) = res {

crates/cargo-codspeed/src/build.rs

@@ -1,7 +1,7 @@
 use crate::{
     app::{BenchTargetFilters, PackageFilters},
     helpers::{clear_dir, get_codspeed_target_dir},
-    measurement_mode::{BuildMode, MeasurementMode},
+    measurement_mode::BuildMode,
     prelude::*,
 };
 use anyhow::Context;
@@ -23,13 +23,14 @@ struct BuiltBench {
     executable_path: Utf8PathBuf,
 }
 
+#[derive(Clone)]
 pub struct BuildConfig {
     pub package_filters: PackageFilters,
     pub bench_target_filters: BenchTargetFilters,
     pub features: Option<Vec<String>>,
     pub profile: String,
     pub quiet: bool,
-    pub measurement_mode: MeasurementMode,
+    pub build_mode: BuildMode,
     pub passthrough_flags: Vec<String>,
 }
 
@@ -301,7 +302,7 @@ impl PackageFilters {
 }
 
 pub fn build_benches(metadata: &Metadata, config: BuildConfig) -> Result<()> {
-    let build_mode = config.measurement_mode.into();
+    let build_mode = config.build_mode;
     let built_benches = BuildOptions {
         bench_target_filters: config.bench_target_filters,
         package_filters: config.package_filters,

crates/cargo-codspeed/src/measurement_mode.rs

@@ -2,8 +2,9 @@ use clap::ValueEnum;
 use serde::Serialize;
 use std::fmt;
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
 pub enum BuildMode {
+    #[default]
     Analysis,
     Walltime,
 }

crates/cargo-codspeed/tests/measurement_modes.rs

@@ -0,0 +1,30 @@
+use predicates::str::contains;
+
+mod helpers;
+use helpers::*;
+
+const DIR: &str = "tests/simple-criterion.in";
+
+#[test]
+fn test_build_multiple_measurement_modes() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .args(["build", "-m", "simulation", "-m", "walltime"])
+        .assert()
+        .success()
+        .stderr(contains("build mode: analysis"))
+        .stderr(contains("build mode: walltime"));
+    teardown(dir);
+}
+
+#[test]
+fn test_build_multiple_measurement_modes_comma_separated() {
+    let dir = setup(DIR, Project::Simple);
+    cargo_codspeed(&dir)
+        .args(["build", "-m", "simulation,walltime"])
+        .assert()
+        .success()
+        .stderr(contains("build mode: analysis"))
+        .stderr(contains("build mode: walltime"));
+    teardown(dir);
+}