feat: support for rustdoc mergeable cross-crate info

This is an unstable feature that we designed to fix several
performance problems with the old system:

1. You couldn't easily build crate docs in hermetic environments.
   This doesn't matter for Cargo, but it was one of the original
   reasons to implement the feature.
2. We have to build all the doc resources in their final form at
   every step, instead of delaying slow parts (mostly the
   search index) until the end and only doing them once.
3. It requires rustdoc to take a lock at the end. This reduces
   available concurrency for generating docs.

A nightly feature `-Zrustdoc-mergeable-info` is added.

Co-authored-by: Michael Howell <[email protected]>
Co-authored-by: Weihang Lo <[email protected]>

Author: notriddle

crates/cargo-test-support/src/compare.rs

@@ -338,6 +338,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
     ("[BLOCKING]", "    Blocking"),
     ("[GENERATED]", "   Generated"),
     ("[OPENING]", "     Opening"),
+    ("[MERGING]", "     Merging"),
 ];
 
 /// Checks that the given string contains the given contiguous lines

src/cargo/core/compiler/build_context/target_info.rs

@@ -79,6 +79,8 @@ pub enum FileFlavor {
     DebugInfo,
     /// SBOM (Software Bill of Materials pre-cursor) file (e.g. cargo-sbon.json).
     Sbom,
+    /// Cross-crate info JSON files generated by rustdoc.
+    DocParts,
 }
 
 /// Type of each file generated by a Unit.

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -275,6 +275,15 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.layout(unit.kind).build_dir().deps(&dir)
     }
 
+    /// Returns the directories where Rust crate dependencies are found for the
+    /// specified unit. (new layout)
+    ///
+    /// New features should consider using this so we can avoid their migrations.
+    pub fn deps_dir_new_layout(&self, unit: &Unit) -> PathBuf {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).build_dir().deps_new_layout(&dir)
+    }
+
     /// Directory where the fingerprint for the given unit should go.
     pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {
         let dir = self.pkg_dir(unit);
@@ -495,12 +504,27 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                         .join("index.html")
                 };
 
-                vec![OutputFile {
+                let mut outputs = vec![OutputFile {
                     path,
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Normal,
-                }]
+                }];
+
+                if bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+                    // `-Zrustdoc-mergeable-info` always uses the new layout.
+                    outputs.push(OutputFile {
+                        path: self
+                            .deps_dir_new_layout(unit)
+                            .join(unit.target.crate_name())
+                            .with_extension("json"),
+                        hardlink: None,
+                        export_path: None,
+                        flavor: FileFlavor::DocParts,
+                    })
+                }
+
+                outputs
             }
             CompileMode::RunCustomBuild => {
                 // At this time, this code path does not handle build script

src/cargo/core/compiler/build_runner/mod.rs

@@ -224,6 +224,8 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         }
 
+        self.collect_doc_merge_info()?;
+
         // Collect the result of the build into `self.compilation`.
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
@@ -329,6 +331,58 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         Ok(())
     }
 
+    fn collect_doc_merge_info(&mut self) -> CargoResult<()> {
+        if !self.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            return Ok(());
+        }
+
+        if !self.bcx.build_config.intent.is_doc() {
+            return Ok(());
+        }
+
+        if self.bcx.build_config.intent.wants_doc_json_output() {
+            // rustdoc JSON output doesn't support merge (yet?)
+            return Ok(());
+        }
+
+        let mut doc_parts_map: HashMap<_, Vec<_>> = HashMap::new();
+
+        let unit_iter = if self.bcx.build_config.intent.wants_deps_docs() {
+            itertools::Either::Left(self.bcx.unit_graph.keys())
+        } else {
+            itertools::Either::Right(self.bcx.roots.iter())
+        };
+
+        for unit in unit_iter {
+            if !unit.mode.is_doc() {
+                continue;
+            }
+            // Assumption: one `rustdoc` call generates only one cross-crate info JSON.
+            let outputs = self.outputs(unit)?;
+
+            let Some(doc_parts) = outputs
+                .iter()
+                .find(|o| matches!(o.flavor, FileFlavor::DocParts))
+            else {
+                continue;
+            };
+
+            doc_parts_map
+                .entry(unit.kind)
+                .or_default()
+                .push(doc_parts.path.to_owned());
+        }
+
+        self.compilation.rustdoc_fingerprints = Some(
+            doc_parts_map
+                .into_iter()
+                .map(|(kind, doc_parts)| (kind, RustdocFingerprint::new(self, kind, doc_parts)))
+                .collect(),
+        );
+
+        Ok(())
+    }
+
     /// Returns the executable for the specified unit (if any).
     pub fn get_executable(&mut self, unit: &Unit) -> CargoResult<Option<PathBuf>> {
         let is_binary = unit.target.is_executable();

src/cargo/core/compiler/compilation.rs

@@ -9,6 +9,7 @@ use cargo_util::{ProcessBuilder, paths};
 
 use crate::core::Package;
 use crate::core::compiler::BuildContext;
+use crate::core::compiler::RustdocFingerprint;
 use crate::core::compiler::apply_env_config;
 use crate::core::compiler::{CompileKind, Unit, UnitHash};
 use crate::util::{CargoResult, GlobalContext, context};
@@ -106,6 +107,11 @@ pub struct Compilation<'gctx> {
     /// Libraries to test with rustdoc.
     pub to_doc_test: Vec<Doctest>,
 
+    /// Rustdoc fingerprint files to determine whether we need to run `rustdoc --merge=finalize`.
+    ///
+    /// See `-Zrustdoc-mergeable-info` for more.
+    pub rustdoc_fingerprints: Option<HashMap<CompileKind, RustdocFingerprint>>,
+
     /// The target host triple.
     pub host: String,
 
@@ -143,6 +149,7 @@ impl<'gctx> Compilation<'gctx> {
             root_crate_names: Vec::new(),
             extra_env: HashMap::new(),
             to_doc_test: Vec::new(),
+            rustdoc_fingerprints: None,
             gctx: bcx.gctx,
             host: bcx.host_triple().to_string(),
             rustc_process,

src/cargo/core/compiler/fingerprint/rustdoc.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use std::path::Path;
 use std::path::PathBuf;
 
@@ -15,6 +16,10 @@ use crate::core::compiler::CompileKind;
 struct RustdocFingerprintJson {
     /// `rustc -vV` verbose version output.
     pub rustc_vv: String,
+
+    /// Relative paths to cross crate info JSON files from previous `cargo doc` invocations.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub doc_parts: Vec<PathBuf>,
 }
 
 /// Structure used to deal with Rustdoc fingerprinting
@@ -29,7 +34,16 @@ struct RustdocFingerprintJson {
 /// they were compiled with the same Rustc version that we're currently using.
 /// Otherwise we must remove the `doc/` folder and compile again forcing a rebuild.
 #[derive(Debug)]
-pub struct RustdocFingerprint {}
+pub struct RustdocFingerprint {
+    /// Path to the fingerprint file.
+    path: PathBuf,
+    /// `rustc -vV` verbose version output for the current session.
+    rustc_vv: String,
+    /// Absolute paths to new cross crate info JSON files generated in the current session.
+    doc_parts: Vec<PathBuf>,
+    /// The fingerprint file on disk.
+    on_disk: Option<RustdocFingerprintJson>,
+}
 
 impl RustdocFingerprint {
     /// Checks whether the latest version of rustc used to compile this workspace's docs
@@ -58,6 +72,7 @@ impl RustdocFingerprint {
         }
         let new_fingerprint = RustdocFingerprintJson {
             rustc_vv: build_runner.bcx.rustc().verbose_version.clone(),
+            doc_parts: Vec::new(),
         };
 
         for kind in &build_runner.bcx.build_config.requested_kinds {
@@ -66,6 +81,111 @@ impl RustdocFingerprint {
 
         Ok(())
     }
+
+    /// Creates a new fingerprint with given doc parts paths.
+    pub fn new(
+        build_runner: &BuildRunner<'_, '_>,
+        kind: CompileKind,
+        doc_parts: Vec<PathBuf>,
+    ) -> Self {
+        let path = fingerprint_path(build_runner, kind);
+        let rustc_vv = build_runner.bcx.rustc().verbose_version.clone();
+        let on_disk = load_on_disk(&path);
+        Self {
+            path,
+            rustc_vv,
+            doc_parts,
+            on_disk,
+        }
+    }
+
+    /// Persists the fingerprint.
+    ///
+    /// The closure will run before persisting the fingerprint,
+    /// and will be given a list of doc parts directories for passing to
+    /// `rustdoc --include-parts-dir`.
+    pub fn persist<F>(&self, exec: F) -> CargoResult<()>
+    where
+        // 1. paths for `--include-parts-dir`
+        F: Fn(&[&Path]) -> CargoResult<()>,
+    {
+        // Dedupe crate with the same name by file stem (which is effectively crate name),
+        // since rustdoc doesn't distinguish different crate versions.
+        //
+        // Rules applied here:
+        //
+        // * If name collides, favor the one selected via CLI over cached ones
+        //   (done by the insertion order)
+        let base = self.path.parent().unwrap();
+        let on_disk_doc_parts: Vec<_> = self
+            .on_disk
+            .iter()
+            .flat_map(|on_disk| {
+                on_disk
+                    .doc_parts
+                    .iter()
+                    // Make absolute so that we can pass to rustdoc
+                    .map(|p| base.join(p))
+                    // Doc parts may be selectively cleaned by `cargo clean -p <doc>`.
+                    // We should stop caching those no-exist.
+                    .filter(|p| p.exists())
+            })
+            .collect();
+        let dedup_map = on_disk_doc_parts
+            .iter()
+            .chain(self.doc_parts.iter())
+            .map(|p| (p.file_stem(), p))
+            .collect::<HashMap<_, _>>();
+        let mut doc_parts: Vec<_> = dedup_map.into_values().collect();
+        doc_parts.sort_unstable();
+
+        // Prepare args for `rustdoc --include-parts-dir`
+        let doc_parts_dirs: Vec<_> = doc_parts.iter().map(|p| p.parent().unwrap()).collect();
+        exec(&doc_parts_dirs)?;
+
+        // Persist with relative paths to the directory where fingerprint file is at.
+        let json = RustdocFingerprintJson {
+            rustc_vv: self.rustc_vv.clone(),
+            doc_parts: doc_parts
+                .iter()
+                .map(|p| p.strip_prefix(base).unwrap_or(p).to_owned())
+                .collect(),
+        };
+        paths::write(&self.path, serde_json::to_string(&json)?)?;
+
+        Ok(())
+    }
+
+    /// Checks if the fingerprint is outdated comparing against given doc parts file paths.
+    pub fn is_dirty(&self) -> bool {
+        let Some(on_disk) = self.on_disk.as_ref() else {
+            return true;
+        };
+
+        let Some(fingerprint_mtime) = paths::mtime(&self.path).ok() else {
+            return true;
+        };
+
+        if self.rustc_vv != on_disk.rustc_vv {
+            return true;
+        }
+
+        for path in &self.doc_parts {
+            let parts_mtime = match paths::mtime(&path) {
+                Ok(mtime) => mtime,
+                Err(e) => {
+                    tracing::debug!("failed to read mtime of {}: {e}", path.display());
+                    return true;
+                }
+            };
+
+            if parts_mtime > fingerprint_mtime {
+                return true;
+            }
+        }
+
+        false
+    }
 }
 
 /// Returns the path to rustdoc fingerprint file for a given [`CompileKind`].
@@ -134,6 +254,25 @@ fn check_fingerprint(
     Ok(())
 }
 
+/// Loads an on-disk fingerprint JSON file.
+fn load_on_disk(path: &Path) -> Option<RustdocFingerprintJson> {
+    let on_disk = match paths::read(path) {
+        Ok(data) => data,
+        Err(e) => {
+            tracing::debug!("failed to read rustdoc fingerprint at {path:?}: {e}");
+            return None;
+        }
+    };
+
+    match serde_json::from_str::<RustdocFingerprintJson>(&on_disk) {
+        Ok(on_disk) => Some(on_disk),
+        Err(e) => {
+            tracing::debug!("could not deserialize {path:?}: {e}");
+            None
+        }
+    }
+}
+
 fn clean_doc(path: &Path) -> CargoResult<()> {
     let entries = path
         .read_dir()

src/cargo/core/compiler/layout.rs

@@ -309,11 +309,17 @@ impl BuildDirLayout {
     /// Fetch the deps path.
     pub fn deps(&self, pkg_dir: &str) -> PathBuf {
         if self.is_new_layout {
-            self.build_unit(pkg_dir).join("deps")
+            self.deps_new_layout(pkg_dir)
         } else {
             self.legacy_deps().to_path_buf()
         }
     }
+    /// Fetch the deps path. (new layout)
+    ///
+    /// New features should consider using this so we can avoid their migrations.
+    pub fn deps_new_layout(&self, pkg_dir: &str) -> PathBuf {
+        self.build_unit(pkg_dir).join("deps")
+    }
     /// Fetch the deps path. (old layout)
     pub fn legacy_deps(&self) -> &Path {
         &self.deps

src/cargo/core/compiler/mod.rs

@@ -833,8 +833,13 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     if build_runner.bcx.gctx.cli_unstable().rustdoc_depinfo {
         // toolchain-shared-resources is required for keeping the shared styling resources
         // invocation-specific is required for keeping the original rustdoc emission
-        let mut arg =
-            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=");
+        let mut arg = if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            // toolchain resources are written at the end, at the same time as merging
+            OsString::from("--emit=invocation-specific,dep-info=")
+        } else {
+            // if not using mergeable CCI, everything is written every time
+            OsString::from("--emit=toolchain-shared-resources,invocation-specific,dep-info=")
+        };
         arg.push(rustdoc_dep_info_loc(build_runner, unit));
         rustdoc.arg(arg);
 
@@ -843,6 +848,19 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
         }
 
         rustdoc.arg("-Zunstable-options");
+    } else if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // toolchain resources are written at the end, at the same time as merging
+        rustdoc.arg("--emit=invocation-specific");
+        rustdoc.arg("-Zunstable-options");
+    }
+
+    if build_runner.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+        // write out mergeable data to be imported
+        rustdoc.arg("--merge=none");
+        let mut arg = OsString::from("--parts-out-dir=");
+        // `-Zrustdoc-mergeable-info` always uses the new layout.
+        arg.push(build_runner.files().deps_dir_new_layout(unit));
+        rustdoc.arg(arg);
     }
 
     if let Some(trim_paths) = unit.profile.trim_paths.as_ref() {