feat: support for rustdoc mergeable cross-crate info

This is an unstable feature that we designed to fix several
performance problems with the old system:

1. You couldn't easily build crate docs in hermetic environments.
   This doesn't matter for Cargo, but it was one of the original
   reasons to implement the feature.
2. We have to build all the doc resources in their final form at
   every step, instead of delaying slow parts (mostly the
   search index) until the end and only doing them once.
3. It requires rustdoc to take a lock at the end. This reduces
   available concurrency for generating docs.

A nightly feature `-Zrustdoc-mergeable-info` is added.

Co-authored-by: Michael Howell <[email protected]>
Co-authored-by: Weihang Lo <[email protected]>

Author: notriddle

crates/cargo-test-support/src/compare.rs

@@ -338,6 +338,7 @@ static E2E_LITERAL_REDACTIONS: &[(&str, &str)] = &[
     ("[BLOCKING]", "    Blocking"),
     ("[GENERATED]", "   Generated"),
     ("[OPENING]", "     Opening"),
+    ("[MERGING]", "     Merging"),
 ];
 
 /// Checks that the given string contains the given contiguous lines

src/cargo/core/compiler/build_context/target_info.rs

@@ -79,6 +79,8 @@ pub enum FileFlavor {
     DebugInfo,
     /// SBOM (Software Bill of Materials pre-cursor) file (e.g. cargo-sbon.json).
     Sbom,
+    /// Cross-crate info JSON files generated by rustdoc.
+    DocParts,
 }
 
 /// Type of each file generated by a Unit.

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -275,6 +275,15 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.layout(unit.kind).build_dir().deps(&dir)
     }
 
+    /// Returns the directories where Rust crate dependencies are found for the
+    /// specified unit. (new layout)
+    ///
+    /// New features should consider using this so we can avoid their migrations.
+    pub fn deps_dir_new_layout(&self, unit: &Unit) -> PathBuf {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).build_dir().deps_new_layout(&dir)
+    }
+
     /// Directory where the fingerprint for the given unit should go.
     pub fn fingerprint_dir(&self, unit: &Unit) -> PathBuf {
         let dir = self.pkg_dir(unit);
@@ -495,12 +504,27 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                         .join("index.html")
                 };
 
-                vec![OutputFile {
+                let mut outputs = vec![OutputFile {
                     path,
                     hardlink: None,
                     export_path: None,
                     flavor: FileFlavor::Normal,
-                }]
+                }];
+
+                if bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+                    // `-Zrustdoc-mergeable-info` always uses the new layout.
+                    outputs.push(OutputFile {
+                        path: self
+                            .deps_dir_new_layout(unit)
+                            .join(unit.target.crate_name())
+                            .with_extension("json"),
+                        hardlink: None,
+                        export_path: None,
+                        flavor: FileFlavor::DocParts,
+                    })
+                }
+
+                outputs
             }
             CompileMode::RunCustomBuild => {
                 // At this time, this code path does not handle build script

src/cargo/core/compiler/build_runner/mod.rs

@@ -1,6 +1,7 @@
 //! [`BuildRunner`] is the mutable state used during the build process.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
+use std::collections::HashSet;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
@@ -224,6 +225,8 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         }
 
+        self.collect_doc_merge_info()?;
+
         // Collect the result of the build into `self.compilation`.
         for unit in &self.bcx.roots {
             self.collect_tests_and_executables(unit)?;
@@ -329,6 +332,77 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         Ok(())
     }
 
+    fn collect_doc_merge_info(&mut self) -> CargoResult<()> {
+        if !self.bcx.gctx.cli_unstable().rustdoc_mergeable_info {
+            return Ok(());
+        }
+
+        if !self.bcx.build_config.intent.is_doc() {
+            return Ok(());
+        }
+
+        if self.bcx.build_config.intent.wants_doc_json_output() {
+            // rustdoc JSON output doesn't support merge (yet?)
+            return Ok(());
+        }
+
+        let mut doc_parts_map: HashMap<_, Vec<_>> = HashMap::new();
+
+        let unit_iter = if self.bcx.build_config.intent.wants_deps_docs() {
+            itertools::Either::Left(self.bcx.unit_graph.keys())
+        } else {
+            itertools::Either::Right(self.bcx.roots.iter())
+        };
+
+        for unit in unit_iter {
+            if !unit.mode.is_doc() {
+                continue;
+            }
+            // Assumption: one `rustdoc` call generates only one cross-crate info JSON.
+            let outputs = self.outputs(unit)?;
+
+            let Some(doc_parts) = outputs
+                .iter()
+                .find(|o| matches!(o.flavor, FileFlavor::DocParts))
+            else {
+                continue;
+            };
+
+            doc_parts_map
+                .entry(unit.kind)
+                .or_default()
+                .push(doc_parts.path.to_owned());
+        }
+
+        self.compilation.doc_merge = Some(HashMap::from_iter(doc_parts_map.into_iter().map(
+            |(kind, doc_parts)| {
+                let out_dir = self
+                    .files()
+                    .layout(kind)
+                    .artifact_dir()
+                    .expect("artifact-dir was not locked")
+                    .doc()
+                    .to_path_buf();
+
+                let Some(fingerprint) = RustdocFingerprint::load(self, kind) else {
+                    let info = compiler::DocMergeInfo::new(doc_parts, out_dir, None);
+                    return (kind, compiler::DocMerge::Merge(info));
+                };
+
+                let doc_merge = if fingerprint.is_outdated(self, &doc_parts) {
+                    let info = compiler::DocMergeInfo::new(doc_parts, out_dir, Some(fingerprint));
+                    compiler::DocMerge::Merge(info)
+                } else {
+                    compiler::DocMerge::Fresh
+                };
+
+                (kind, doc_merge)
+            },
+        )));
+
+        Ok(())
+    }
+
     /// Returns the executable for the specified unit (if any).
     pub fn get_executable(&mut self, unit: &Unit) -> CargoResult<Option<PathBuf>> {
         let is_binary = unit.target.is_executable();

src/cargo/core/compiler/compilation.rs

@@ -2,13 +2,15 @@
 
 use std::collections::{BTreeSet, HashMap};
 use std::ffi::{OsStr, OsString};
+use std::path::Path;
 use std::path::PathBuf;
 
 use cargo_platform::CfgExpr;
 use cargo_util::{ProcessBuilder, paths};
 
 use crate::core::Package;
 use crate::core::compiler::BuildContext;
+use crate::core::compiler::RustdocFingerprint;
 use crate::core::compiler::apply_env_config;
 use crate::core::compiler::{CompileKind, Unit, UnitHash};
 use crate::util::{CargoResult, GlobalContext, context};
@@ -106,6 +108,11 @@ pub struct Compilation<'gctx> {
     /// Libraries to test with rustdoc.
     pub to_doc_test: Vec<Doctest>,
 
+    /// Compilation information for running `rustdoc --merge=finalize`.
+    ///
+    /// See `-Zrustdoc-mergeable-info` for more.
+    pub doc_merge: Option<HashMap<CompileKind, DocMerge>>,
+
     /// The target host triple.
     pub host: String,
 
@@ -143,6 +150,7 @@ impl<'gctx> Compilation<'gctx> {
             root_crate_names: Vec::new(),
             extra_env: HashMap::new(),
             to_doc_test: Vec::new(),
+            doc_merge: None,
             gctx: bcx.gctx,
             host: bcx.host_triple().to_string(),
             rustc_process,
@@ -383,6 +391,67 @@ impl<'gctx> Compilation<'gctx> {
     }
 }
 
+/// Whether `rustdoc --merge=finalize` output is stale or fresh.
+pub enum DocMerge {
+    /// Nothing is stale.
+    Fresh,
+    /// Doc merge is required.
+    Merge(DocMergeInfo),
+}
+
+/// Compilation information for running `rustdoc --merge=finalize`.
+pub struct DocMergeInfo {
+    /// Cross-crate info JSON files for each rustdoc invocation during this `cargo doc` call.
+    doc_parts: Vec<PathBuf>,
+    /// Output directory for rustdoc final artifacts.
+    out_dir: PathBuf,
+    /// Rustdoc fingerprint file information, if existing.
+    fingerprint: Option<RustdocFingerprint>,
+}
+
+impl DocMergeInfo {
+    pub fn new(
+        doc_parts: Vec<PathBuf>,
+        out_dir: PathBuf,
+        fingerprint: Option<RustdocFingerprint>,
+    ) -> Self {
+        Self {
+            doc_parts,
+            out_dir,
+            fingerprint,
+        }
+    }
+
+    /// Provides arguments for rustdoc cross-crate info finalization.
+    pub fn finalize<F>(&self, exec: F) -> CargoResult<()>
+    where
+        // 1. paths for `--include-parts-dir`
+        // 2. path for `--out-dir`
+        F: Fn(&[&Path], &Path) -> CargoResult<()>,
+    {
+        let mut doc_parts: Vec<_> = self
+            .doc_parts
+            .iter()
+            .chain(self.fingerprint.iter().flat_map(|f| f.doc_parts().iter()))
+            .cloned()
+            .collect();
+
+        doc_parts.sort_unstable();
+        doc_parts.dedup();
+
+        // rustdoc needs the directory holding doc parts files.
+        let parts_dirs: Vec<_> = doc_parts.iter().map(|p| p.parent().unwrap()).collect();
+
+        exec(&parts_dirs, &self.out_dir)?;
+
+        if let Some(fingerprint) = &self.fingerprint {
+            fingerprint.persist(doc_parts)?;
+        }
+
+        Ok(())
+    }
+}
+
 /// Prepares a `rustc_tool` process with additional environment variables
 /// that are only relevant in a context that has a unit
 fn fill_rustc_tool_env(mut cmd: ProcessBuilder, unit: &Unit) -> ProcessBuilder {

src/cargo/core/compiler/fingerprint/rustdoc.rs

@@ -3,6 +3,7 @@ use std::path::PathBuf;
 
 use anyhow::Context as _;
 use cargo_util::paths;
+use filetime::FileTime;
 use serde::Deserialize;
 use serde::Serialize;
 
@@ -15,6 +16,10 @@ use crate::core::compiler::CompileKind;
 struct RustdocFingerprintJson {
     /// `rustc -vV` verbose version output.
     pub rustc_vv: String,
+
+    /// Path to cross crate info JSON files from previous `cargo doc` invocations.
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub doc_parts: Vec<PathBuf>,
 }
 
 /// Structure used to deal with Rustdoc fingerprinting
@@ -29,7 +34,13 @@ struct RustdocFingerprintJson {
 /// they were compiled with the same Rustc version that we're currently using.
 /// Otherwise we must remove the `doc/` folder and compile again forcing a rebuild.
 #[derive(Debug)]
-pub struct RustdocFingerprint {}
+pub struct RustdocFingerprint {
+    /// File modified time when loading this fingerprint.
+    mtime: Option<FileTime>,
+    /// Path to this fingerprint file.
+    path: PathBuf,
+    fingerprint: RustdocFingerprintJson,
+}
 
 impl RustdocFingerprint {
     /// Checks whether the latest version of rustc used to compile this workspace's docs
@@ -58,6 +69,7 @@ impl RustdocFingerprint {
         }
         let new_fingerprint = RustdocFingerprintJson {
             rustc_vv: build_runner.bcx.rustc().verbose_version.clone(),
+            doc_parts: Vec::new(),
         };
 
         for kind in &build_runner.bcx.build_config.requested_kinds {
@@ -66,6 +78,74 @@ impl RustdocFingerprint {
 
         Ok(())
     }
+
+    /// Returns the path to rustdoc fingerprint file for a given [`CompileKind`].
+    pub fn load(build_runner: &BuildRunner<'_, '_>, kind: CompileKind) -> Option<Self> {
+        let path = fingerprint_path(build_runner, kind);
+        let fingerprint = match paths::read(&path) {
+            Ok(data) => data,
+            Err(e) => {
+                tracing::debug!("failed to read rustdoc fingerprint at {path:?}: {e}");
+                return None;
+            }
+        };
+
+        match serde_json::from_str::<RustdocFingerprintJson>(&fingerprint) {
+            Ok(mut fingerprint) => {
+                // Doc parts may be selectively cleaned via `cargo clean -p <doc>`.
+                // We should stop caching those.
+                fingerprint.doc_parts.retain(|p| p.exists());
+                Some(Self {
+                    mtime: paths::mtime(&path).ok(),
+                    path,
+                    fingerprint,
+                })
+            }
+            Err(e) => {
+                tracing::debug!("could not deserialize {:?}: {}", path, e);
+                None
+            }
+        }
+    }
+
+    /// Checks if the fingerprint is outdated comparing against given doc parts file paths.
+    pub fn is_outdated(&self, build_runner: &BuildRunner<'_, '_>, doc_parts: &[PathBuf]) -> bool {
+        let Some(fingerprint_mtime) = self.mtime.as_ref() else {
+            return true;
+        };
+
+        if self.fingerprint.rustc_vv != build_runner.bcx.rustc().verbose_version {
+            return true;
+        }
+
+        for path in doc_parts {
+            let parts_mtime = match paths::mtime(&path) {
+                Ok(mtime) => mtime,
+                Err(e) => {
+                    tracing::debug!("failed to read mtime of {}: {e}", path.display());
+                    return true;
+                }
+            };
+
+            if &parts_mtime > fingerprint_mtime {
+                return true;
+            }
+        }
+
+        false
+    }
+
+    pub fn persist(&self, doc_parts: Vec<PathBuf>) -> CargoResult<()> {
+        let new_fingerprint = RustdocFingerprintJson {
+            rustc_vv: self.fingerprint.rustc_vv.clone(),
+            doc_parts,
+        };
+        paths::write(&self.path, serde_json::to_string(&new_fingerprint)?)
+    }
+
+    pub fn doc_parts(&self) -> &[PathBuf] {
+        &self.fingerprint.doc_parts
+    }
 }
 
 /// Returns the path to rustdoc fingerprint file for a given [`CompileKind`].

src/cargo/core/compiler/layout.rs

@@ -309,11 +309,17 @@ impl BuildDirLayout {
     /// Fetch the deps path.
     pub fn deps(&self, pkg_dir: &str) -> PathBuf {
         if self.is_new_layout {
-            self.build_unit(pkg_dir).join("deps")
+            self.deps_new_layout(pkg_dir)
         } else {
             self.legacy_deps().to_path_buf()
         }
     }
+    /// Fetch the deps path. (new layout)
+    ///
+    /// New features should consider using this so we can avoid their migrations.
+    pub fn deps_new_layout(&self, pkg_dir: &str) -> PathBuf {
+        self.build_unit(pkg_dir).join("deps")
+    }
     /// Fetch the deps path. (old layout)
     pub fn legacy_deps(&self) -> &Path {
         &self.deps