1//! Documentation generation for bootstrap.2//!3//! This module implements generation for all bits and pieces of documentation4//! for the Rust project. This notably includes suites like the rust book, the5//! nomicon, rust by example, standalone documentation, etc.6//!7//! Everything here is basically just a shim around calling either `rustbook` or8//! `rustdoc`.910use std::io::{self, Write};11use std::path::{Path, PathBuf};12use std::{env, fs, mem};1314use crate::core::build_steps::compile;15use crate::core::build_steps::tool::{16    self, RustcPrivateCompilers, SourceType, Tool, prepare_tool_cargo,17};18use crate::core::builder::{19    self, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description,20};21use crate::core::config::{Config, TargetSelection};22use crate::helpers::{submodule_path_of, symlink_dir, t, up_to_date};23use crate::{FileType, Mode};2425macro_rules! book {26    ($($name:ident, $path:expr, $book_name:expr, $lang:expr ;)+) => {27        $(28        #[derive(Debug, Clone, Hash, PartialEq, Eq)]29        pub struct $name {30            target: TargetSelection,31        }3233        impl Step for $name {34            type Output = ();3536            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {37                run.path($path)38            }3940            fn is_default_step(builder: &Builder<'_>) -> bool {41                builder.config.docs42            }4344            fn make_run(run: RunConfig<'_>) {45                run.builder.ensure($name {46                    target: run.target,47                });48            }4950            fn run(self, builder: &Builder<'_>) {51                if let Some(submodule_path) = submodule_path_of(&builder, $path) {52                    builder.require_submodule(&submodule_path, None)53                }5455                builder.ensure(RustbookSrc {56                    target: self.target,57                    name: $book_name.to_owned(),58                    src: builder.src.join($path),59                    parent: Some(self),60                    languages: $lang.into(),61                    build_compiler: None,62                })63            }64        }65        )+66    }67}6869// NOTE: When adding a book here, make sure to ALSO build the book by70// adding a build step in `src/bootstrap/code/builder/mod.rs`!71// NOTE: Make sure to add the corresponding submodule when adding a new book.72book!(73    CargoBook, "src/tools/cargo/src/doc", "cargo", &[];74    ClippyBook, "src/tools/clippy/book", "clippy", &[];75    EditionGuide, "src/doc/edition-guide", "edition-guide", &[];76    EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[];77    Nomicon, "src/doc/nomicon", "nomicon", &[];78    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["es", "ja", "zh", "ko"];79    RustdocBook, "src/doc/rustdoc", "rustdoc", &[];80    StyleGuide, "src/doc/style-guide", "style-guide", &[];81);8283#[derive(Debug, Clone, Hash, PartialEq, Eq)]84pub struct UnstableBook {85    target: TargetSelection,86}8788impl Step for UnstableBook {89    type Output = ();9091    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {92        run.path("src/doc/unstable-book")93    }9495    fn is_default_step(builder: &Builder<'_>) -> bool {96        builder.config.docs97    }9899    fn make_run(run: RunConfig<'_>) {100        run.builder.ensure(UnstableBook { target: run.target });101    }102103    fn run(self, builder: &Builder<'_>) {104        builder.ensure(UnstableBookGen { target: self.target });105        builder.ensure(RustbookSrc {106            target: self.target,107            name: "unstable-book".to_owned(),108            src: builder.md_doc_out(self.target).join("unstable-book"),109            parent: Some(self),110            languages: vec![],111            build_compiler: None,112        })113    }114}115116#[derive(Debug, Clone, Hash, PartialEq, Eq)]117struct RustbookSrc<P: Step> {118    target: TargetSelection,119    name: String,120    src: PathBuf,121    parent: Option<P>,122    languages: Vec<&'static str>,123    /// Compiler whose rustdoc should be used to document things using `mdbook-spec`.124    build_compiler: Option<Compiler>,125}126127impl<P: Step> Step for RustbookSrc<P> {128    type Output = ();129130    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {131        run.never()132    }133134    /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.135    ///136    /// This will not actually generate any documentation if the documentation has137    /// already been generated.138    fn run(self, builder: &Builder<'_>) {139        let target = self.target;140        let name = self.name;141        let src = self.src;142        let out = builder.doc_out(target);143        t!(fs::create_dir_all(&out));144145        let out = out.join(&name);146        let index = out.join("index.html");147        let rustbook = builder.tool_exe(Tool::Rustbook);148149        if !builder.config.dry_run()150            && (!up_to_date(&src, &index) || !up_to_date(&rustbook, &index))151        {152            builder.info(&format!("Rustbook ({target}) - {name}"));153            let _ = fs::remove_dir_all(&out);154155            let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);156157            if let Some(compiler) = self.build_compiler {158                let mut rustdoc = builder.rustdoc_for_compiler(compiler);159                rustdoc.pop();160                let old_path = env::var_os("PATH").unwrap_or_default();161                let new_path =162                    env::join_paths(std::iter::once(rustdoc).chain(env::split_paths(&old_path)))163                        .expect("could not add rustdoc to PATH");164165                rustbook_cmd.env("PATH", new_path);166                builder.add_rustc_lib_path(compiler, &mut rustbook_cmd);167            }168169            rustbook_cmd170                .arg("build")171                .arg(&src)172                .arg("-d")173                .arg(&out)174                .arg("--rust-root")175                .arg(&builder.src)176                .run(builder);177178            for lang in &self.languages {179                let out = out.join(lang);180181                builder.info(&format!("Rustbook ({target}) - {name} - {lang}"));182                let _ = fs::remove_dir_all(&out);183184                builder185                    .tool_cmd(Tool::Rustbook)186                    .arg("build")187                    .arg(&src)188                    .arg("-d")189                    .arg(&out)190                    .arg("-l")191                    .arg(lang)192                    .run(builder);193            }194        }195196        if self.parent.is_some() {197            builder.maybe_open_in_browser::<P>(index)198        }199    }200201    fn metadata(&self) -> Option<StepMetadata> {202        let mut metadata = StepMetadata::doc(&format!("{} (book)", self.name), self.target);203        if let Some(compiler) = self.build_compiler {204            metadata = metadata.built_by(compiler);205        }206207        Some(metadata)208    }209}210211#[derive(Debug, Clone, Hash, PartialEq, Eq)]212pub struct TheBook {213    /// Compiler whose rustdoc will be used to generated documentation.214    build_compiler: Compiler,215    target: TargetSelection,216}217218impl Step for TheBook {219    type Output = ();220221    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {222        run.path("src/doc/book")223    }224225    fn is_default_step(builder: &Builder<'_>) -> bool {226        builder.config.docs227    }228229    fn make_run(run: RunConfig<'_>) {230        run.builder.ensure(TheBook {231            build_compiler: prepare_doc_compiler(run.builder, run.target, run.builder.top_stage),232            target: run.target,233        });234    }235236    /// Builds the book and associated stuff.237    ///238    /// We need to build:239    ///240    /// * Book241    /// * Older edition redirects242    /// * Version info and CSS243    /// * Index page244    /// * Redirect pages245    fn run(self, builder: &Builder<'_>) {246        builder.require_submodule("src/doc/book", None);247248        let build_compiler = self.build_compiler;249        let target = self.target;250251        let absolute_path = builder.src.join("src/doc/book");252        let redirect_path = absolute_path.join("redirects");253254        // build book255        builder.ensure(RustbookSrc {256            target,257            name: "book".to_owned(),258            src: absolute_path.clone(),259            parent: Some(self),260            languages: vec![],261            build_compiler: None,262        });263264        // building older edition redirects265        for edition in &["first-edition", "second-edition", "2018-edition"] {266            builder.ensure(RustbookSrc {267                target,268                name: format!("book/{edition}"),269                src: absolute_path.join(edition),270                // There should only be one book that is marked as the parent for each target, so271                // treat the other editions as not having a parent.272                parent: Option::<Self>::None,273                languages: vec![],274                build_compiler: None,275            });276        }277278        // build the version info page and CSS279        let shared_assets = builder.ensure(SharedAssets { target });280281        // build the redirect pages282        let _guard = builder.msg(Kind::Doc, "book redirect pages", None, build_compiler, target);283        if builder.config.dry_run() {284            return;285        }286287        for file in t!(fs::read_dir(redirect_path)) {288            let file = t!(file);289            let path = file.path();290            let path = path.to_str().unwrap();291292            invoke_rustdoc(builder, build_compiler, &shared_assets, target, path);293        }294    }295}296297fn invoke_rustdoc(298    builder: &Builder<'_>,299    build_compiler: Compiler,300    shared_assets: &SharedAssetsPaths,301    target: TargetSelection,302    markdown: &str,303) {304    let out = builder.doc_out(target);305306    let path = builder.src.join("src/doc").join(markdown);307308    let header = builder.src.join("src/doc/redirect.inc");309    let footer = builder.src.join("src/doc/footer.inc");310311    let mut cmd = builder.rustdoc_cmd(build_compiler);312313    let out = out.join("book");314315    cmd.arg("--html-after-content")316        .arg(&footer)317        .arg("--html-before-content")318        .arg(&shared_assets.version_info)319        .arg("--html-in-header")320        .arg(&header)321        .arg("--markdown-no-toc")322        .arg("--markdown-playground-url")323        .arg("https://play.rust-lang.org/")324        .arg("-o")325        .arg(&out)326        .arg(&path)327        .arg("--markdown-css")328        .arg("../rust.css")329        .arg("-Zunstable-options");330331    if !builder.config.docs_minification {332        cmd.arg("--disable-minification");333    }334335    cmd.run(builder);336}337338#[derive(Debug, Clone, Hash, PartialEq, Eq)]339pub struct Standalone {340    build_compiler: Compiler,341    target: TargetSelection,342}343344impl Step for Standalone {345    type Output = ();346347    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {348        run.path("src/doc").alias("standalone")349    }350351    fn is_default_step(builder: &Builder<'_>) -> bool {352        builder.config.docs353    }354355    fn make_run(run: RunConfig<'_>) {356        run.builder.ensure(Standalone {357            build_compiler: prepare_doc_compiler(358                run.builder,359                run.builder.host_target,360                run.builder.top_stage,361            ),362            target: run.target,363        });364    }365366    /// Generates all standalone documentation as compiled by the rustdoc in `stage`367    /// for the `target` into `out`.368    ///369    /// This will list all of `src/doc` looking for markdown files and appropriately370    /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and371    /// `STAMP` along with providing the various header/footer HTML we've customized.372    ///373    /// In the end, this is just a glorified wrapper around rustdoc!374    fn run(self, builder: &Builder<'_>) {375        let target = self.target;376        let build_compiler = self.build_compiler;377        let _guard = builder.msg(Kind::Doc, "standalone", None, build_compiler, target);378        let out = builder.doc_out(target);379        t!(fs::create_dir_all(&out));380381        let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;382383        let favicon = builder.src.join("src/doc/favicon.inc");384        let footer = builder.src.join("src/doc/footer.inc");385        let full_toc = builder.src.join("src/doc/full-toc.inc");386387        for file in t!(fs::read_dir(builder.src.join("src/doc"))) {388            let file = t!(file);389            let path = file.path();390            let filename = path.file_name().unwrap().to_str().unwrap();391            if !filename.ends_with(".md") || filename == "README.md" {392                continue;393            }394395            let html = out.join(filename).with_extension("html");396            let rustdoc = builder.rustdoc_for_compiler(build_compiler);397            if up_to_date(&path, &html)398                && up_to_date(&footer, &html)399                && up_to_date(&favicon, &html)400                && up_to_date(&full_toc, &html)401                && (builder.config.dry_run() || up_to_date(&version_info, &html))402                && (builder.config.dry_run() || up_to_date(&rustdoc, &html))403            {404                continue;405            }406407            let mut cmd = builder.rustdoc_cmd(build_compiler);408409            cmd.arg("--html-after-content")410                .arg(&footer)411                .arg("--html-before-content")412                .arg(&version_info)413                .arg("--html-in-header")414                .arg(&favicon)415                .arg("--markdown-no-toc")416                .arg("-Zunstable-options")417                .arg("--index-page")418                .arg(builder.src.join("src/doc/index.md"))419                .arg("--markdown-playground-url")420                .arg("https://play.rust-lang.org/")421                .arg("-o")422                .arg(&out)423                .arg(&path);424425            if !builder.config.docs_minification {426                cmd.arg("--disable-minification");427            }428429            if filename == "not_found.md" {430                cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css");431            } else {432                cmd.arg("--markdown-css").arg("rust.css");433            }434            cmd.run(builder);435        }436437        // We open doc/index.html as the default if invoked as `x.py doc --open`438        // with no particular explicit doc requested (e.g. library/core).439        if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) {440            let index = out.join("index.html");441            builder.open_in_browser(index);442        }443    }444445    fn metadata(&self) -> Option<StepMetadata> {446        Some(StepMetadata::doc("standalone", self.target).built_by(self.build_compiler))447    }448}449450#[derive(Debug, Clone, Hash, PartialEq, Eq)]451pub struct Releases {452    build_compiler: Compiler,453    target: TargetSelection,454}455456impl Step for Releases {457    type Output = ();458459    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {460        run.path("RELEASES.md").alias("releases")461    }462463    fn is_default_step(builder: &Builder<'_>) -> bool {464        builder.config.docs465    }466467    fn make_run(run: RunConfig<'_>) {468        run.builder.ensure(Releases {469            build_compiler: prepare_doc_compiler(470                run.builder,471                run.builder.host_target,472                run.builder.top_stage,473            ),474            target: run.target,475        });476    }477478    /// Generates HTML release notes to include in the final docs bundle.479    ///480    /// This uses the same stylesheet and other tools as Standalone, but the481    /// RELEASES.md file is included at the root of the repository and gets482    /// the headline added. In the end, the conversion is done by Rustdoc.483    fn run(self, builder: &Builder<'_>) {484        let target = self.target;485        let build_compiler = self.build_compiler;486        let _guard = builder.msg(Kind::Doc, "releases", None, build_compiler, target);487        let out = builder.doc_out(target);488        t!(fs::create_dir_all(&out));489490        builder.ensure(Standalone { build_compiler, target });491492        let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;493494        let favicon = builder.src.join("src/doc/favicon.inc");495        let footer = builder.src.join("src/doc/footer.inc");496        let full_toc = builder.src.join("src/doc/full-toc.inc");497498        let html = out.join("releases.html");499        let tmppath = out.join("releases.md");500        let inpath = builder.src.join("RELEASES.md");501        let rustdoc = builder.rustdoc_for_compiler(build_compiler);502        if !up_to_date(&inpath, &html)503            || !up_to_date(&footer, &html)504            || !up_to_date(&favicon, &html)505            || !up_to_date(&full_toc, &html)506            || !(builder.config.dry_run()507                || up_to_date(&version_info, &html)508                || up_to_date(&rustdoc, &html))509        {510            let mut tmpfile = t!(fs::File::create(&tmppath));511            t!(tmpfile.write_all(b"% Rust Release Notes\n\n"));512            t!(io::copy(&mut t!(fs::File::open(&inpath)), &mut tmpfile));513            mem::drop(tmpfile);514            let mut cmd = builder.rustdoc_cmd(build_compiler);515516            cmd.arg("--html-after-content")517                .arg(&footer)518                .arg("--html-before-content")519                .arg(&version_info)520                .arg("--html-in-header")521                .arg(&favicon)522                .arg("--markdown-no-toc")523                .arg("--markdown-css")524                .arg("rust.css")525                .arg("-Zunstable-options")526                .arg("--index-page")527                .arg(builder.src.join("src/doc/index.md"))528                .arg("--markdown-playground-url")529                .arg("https://play.rust-lang.org/")530                .arg("-o")531                .arg(&out)532                .arg(&tmppath);533534            if !builder.config.docs_minification {535                cmd.arg("--disable-minification");536            }537538            cmd.run(builder);539        }540541        // We open doc/RELEASES.html as the default if invoked as `x.py doc --open RELEASES.md`542        // with no particular explicit doc requested (e.g. library/core).543        if builder.was_invoked_explicitly::<Self>(Kind::Doc) {544            builder.open_in_browser(&html);545        }546    }547548    fn metadata(&self) -> Option<StepMetadata> {549        Some(StepMetadata::doc("releases", self.target).built_by(self.build_compiler))550    }551}552553#[derive(Debug, Clone)]554pub struct SharedAssetsPaths {555    pub version_info: PathBuf,556}557558#[derive(Debug, Clone, Hash, PartialEq, Eq)]559pub struct SharedAssets {560    target: TargetSelection,561}562563impl Step for SharedAssets {564    type Output = SharedAssetsPaths;565566    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {567        // Other tasks depend on this, no need to execute it on its own568        run.never()569    }570571    fn is_default_step(_builder: &Builder<'_>) -> bool {572        false573    }574575    /// Generate shared resources used by other pieces of documentation.576    fn run(self, builder: &Builder<'_>) -> Self::Output {577        let out = builder.doc_out(self.target);578579        let version_input = builder.src.join("src").join("doc").join("version_info.html.template");580        let version_info = out.join("version_info.html");581        if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) {582            let info = t!(fs::read_to_string(&version_input))583                .replace("VERSION", &builder.rust_release())584                .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or(""))585                .replace("STAMP", builder.rust_info().sha().unwrap_or(""));586            t!(fs::write(&version_info, info));587        }588589        builder.copy_link(590            &builder.src.join("src").join("doc").join("rust.css"),591            &out.join("rust.css"),592            FileType::Regular,593        );594595        builder.copy_link(596            &builder597                .src598                .join("src")599                .join("librustdoc")600                .join("html")601                .join("static")602                .join("images")603                .join("favicon.svg"),604            &out.join("favicon.svg"),605            FileType::Regular,606        );607        builder.copy_link(608            &builder609                .src610                .join("src")611                .join("librustdoc")612                .join("html")613                .join("static")614                .join("images")615                .join("favicon-32x32.png"),616            &out.join("favicon-32x32.png"),617            FileType::Regular,618        );619620        SharedAssetsPaths { version_info }621    }622}623624/// Document the standard library using `build_compiler`.625#[derive(Debug, Clone, Hash, PartialEq, Eq)]626pub struct Std {627    build_compiler: Compiler,628    target: TargetSelection,629    format: DocumentationFormat,630    crates: Vec<String>,631}632633impl Std {634    pub(crate) fn from_build_compiler(635        build_compiler: Compiler,636        target: TargetSelection,637        format: DocumentationFormat,638    ) -> Self {639        Std { build_compiler, target, format, crates: vec![] }640    }641}642643impl Step for Std {644    /// Path to a directory with the built documentation.645    type Output = PathBuf;646647    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {648        run.crate_or_deps("sysroot").path("library")649    }650651    fn is_default_step(builder: &Builder<'_>) -> bool {652        builder.config.docs653    }654655    fn make_run(run: RunConfig<'_>) {656        let crates = compile::std_crates_for_run_make(&run);657        let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false);658        if crates.is_empty() && target_is_no_std {659            return;660        }661        run.builder.ensure(Std {662            build_compiler: run.builder.compiler_for_std(run.builder.top_stage),663            target: run.target,664            format: if run.builder.config.cmd.json() {665                DocumentationFormat::Json666            } else {667                DocumentationFormat::Html668            },669            crates,670        });671    }672673    /// Compile all standard library documentation.674    ///675    /// This will generate all documentation for the standard library and its676    /// dependencies. This is largely just a wrapper around `cargo doc`.677    fn run(self, builder: &Builder<'_>) -> Self::Output {678        let target = self.target;679        let crates = if self.crates.is_empty() {680            builder681                .in_tree_crates("sysroot", Some(target))682                .iter()683                .map(|c| c.name.to_string())684                .collect()685        } else {686            self.crates687        };688689        let out = match self.format {690            DocumentationFormat::Html => builder.doc_out(target),691            DocumentationFormat::Json => builder.json_doc_out(target),692        };693694        t!(fs::create_dir_all(&out));695696        if self.format == DocumentationFormat::Html {697            builder.ensure(SharedAssets { target: self.target });698        }699700        let index_page = builder701            .src702            .join("src/doc/index.md")703            .into_os_string()704            .into_string()705            .expect("non-utf8 paths are unsupported");706        let mut extra_args = match self.format {707            DocumentationFormat::Html => {708                vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page]709            }710            DocumentationFormat::Json => vec!["--output-format", "json"],711        };712713        if !builder.config.docs_minification {714            extra_args.push("--disable-minification");715        }716        // For `--index-page` and `--output-format=json`.717        extra_args.push("-Zunstable-options");718719        doc_std(builder, self.format, self.build_compiler, target, &out, &extra_args, &crates);720721        // Open if the format is HTML722        if let DocumentationFormat::Html = self.format {723            if builder.paths.iter().any(|path| path.ends_with("library")) {724                // For `x.py doc library --open`, open `std` by default.725                let index = out.join("std").join("index.html");726                builder.maybe_open_in_browser::<Self>(index);727            } else {728                for requested_crate in crates {729                    if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) {730                        let index = out.join(requested_crate).join("index.html");731                        builder.maybe_open_in_browser::<Self>(index);732                        break;733                    }734                }735            }736        }737738        out739    }740741    fn metadata(&self) -> Option<StepMetadata> {742        Some(743            StepMetadata::doc("std", self.target)744                .built_by(self.build_compiler)745                .with_metadata(format!("crates=[{}]", self.crates.join(","))),746        )747    }748}749750/// Name of the crates that are visible to consumers of the standard library.751/// Documentation for internal crates is handled by the rustc step, so internal crates will show752/// up there.753///754/// Order here is important!755/// Crates need to be processed starting from the leaves, otherwise rustdoc will not756/// create correct links between crates because rustdoc depends on the757/// existence of the output directories to know if it should be a local758/// or remote link.759const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];760761#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]762pub enum DocumentationFormat {763    Html,764    Json,765}766767impl DocumentationFormat {768    fn as_str(&self) -> &str {769        match self {770            DocumentationFormat::Html => "HTML",771            DocumentationFormat::Json => "JSON",772        }773    }774}775776/// Build the documentation for public standard library crates.777fn doc_std(778    builder: &Builder<'_>,779    format: DocumentationFormat,780    build_compiler: Compiler,781    target: TargetSelection,782    out: &Path,783    extra_args: &[&str],784    requested_crates: &[String],785) {786    let target_doc_dir_name = if format == DocumentationFormat::Json { "json-doc" } else { "doc" };787    let target_dir =788        builder.stage_out(build_compiler, Mode::Std).join(target).join(target_doc_dir_name);789790    // This is directory where the compiler will place the output of the command.791    // We will then copy the files from this directory into the final `out` directory, the specified792    // as a function parameter.793    let out_dir = target_dir.join(target).join("doc");794795    let mut cargo = builder::Cargo::new(796        builder,797        build_compiler,798        Mode::Std,799        SourceType::InTree,800        target,801        Kind::Doc,802    );803804    compile::std_cargo(builder, target, &mut cargo, requested_crates);805    cargo806        .arg("--no-deps")807        .arg("--target-dir")808        .arg(&*target_dir.to_string_lossy())809        .arg("-Zskip-rustdoc-fingerprint")810        .arg("-Zrustdoc-map")811        .rustdocflag("--extern-html-root-url")812        .rustdocflag("std_detect=https://docs.rs/std_detect/latest/")813        .rustdocflag("--extern-html-root-takes-precedence")814        .rustdocflag("--resource-suffix")815        .rustdocflag(&builder.version);816    for arg in extra_args {817        cargo.rustdocflag(arg);818    }819820    if builder.config.library_docs_private_items {821        cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items");822    }823824    let description =825        format!("library{} in {} format", crate_description(requested_crates), format.as_str());826    let _guard = builder.msg(Kind::Doc, description, Mode::Std, build_compiler, target);827828    cargo.into_cmd().run(builder);829    builder.cp_link_r(&out_dir, out);830}831832/// Prepare a compiler that will be able to document something for `target` at `stage`.833pub fn prepare_doc_compiler(834    builder: &Builder<'_>,835    target: TargetSelection,836    stage: u32,837) -> Compiler {838    assert!(stage > 0, "Cannot document anything in stage 0");839    let build_compiler = builder.compiler(stage - 1, builder.host_target);840    builder.std(build_compiler, target);841    build_compiler842}843844/// Document the compiler for the given `target` using rustdoc from `build_compiler`.845#[derive(Debug, Clone, Hash, PartialEq, Eq)]846pub struct Rustc {847    build_compiler: Compiler,848    target: TargetSelection,849    crates: Vec<String>,850}851852impl Rustc {853    /// Document `stage` compiler for the given `target`.854    pub(crate) fn for_stage(builder: &Builder<'_>, stage: u32, target: TargetSelection) -> Self {855        let build_compiler = prepare_doc_compiler(builder, target, stage);856        Self::from_build_compiler(builder, build_compiler, target)857    }858859    fn from_build_compiler(860        builder: &Builder<'_>,861        build_compiler: Compiler,862        target: TargetSelection,863    ) -> Self {864        let crates = builder865            .in_tree_crates("rustc-main", Some(target))866            .into_iter()867            .map(|krate| krate.name.to_string())868            .collect();869        Self { build_compiler, target, crates }870    }871}872873impl Step for Rustc {874    type Output = ();875    const IS_HOST: bool = true;876877    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {878        run.crate_or_deps("rustc-main").path("compiler")879    }880881    fn is_default_step(builder: &Builder<'_>) -> bool {882        builder.config.compiler_docs883    }884885    fn make_run(run: RunConfig<'_>) {886        run.builder.ensure(Rustc::for_stage(run.builder, run.builder.top_stage, run.target));887    }888889    /// Generates compiler documentation.890    ///891    /// This will generate all documentation for compiler and dependencies.892    /// Compiler documentation is distributed separately, so we make sure893    /// we do not merge it with the other documentation from std, test and894    /// proc_macros. This is largely just a wrapper around `cargo doc`.895    fn run(self, builder: &Builder<'_>) {896        let target = self.target;897898        // This is the intended out directory for compiler documentation.899        let out = builder.compiler_doc_out(target);900        t!(fs::create_dir_all(&out));901902        // Build the standard library, so that proc-macros can use it.903        // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.)904        let build_compiler = self.build_compiler;905        builder.std(build_compiler, builder.config.host_target);906907        let _guard = builder.msg(908            Kind::Doc,909            format!("compiler{}", crate_description(&self.crates)),910            Mode::Rustc,911            build_compiler,912            target,913        );914915        // Build cargo command.916        let mut cargo = builder::Cargo::new(917            builder,918            build_compiler,919            Mode::Rustc,920            SourceType::InTree,921            target,922            Kind::Doc,923        );924925        cargo.rustdocflag("--document-private-items");926        // Since we always pass --document-private-items, there's no need to warn about linking to private items.927        cargo.rustdocflag("-Arustdoc::private-intra-doc-links");928        cargo.rustdocflag("--enable-index-page");929        cargo.rustdocflag("-Znormalize-docs");930        cargo.rustdocflag("--show-type-layout");931        // FIXME: `--generate-link-to-definition` tries to resolve cfged out code932        // see https://github.com/rust-lang/rust/pull/122066#issuecomment-1983049222933        // If there is any bug, please comment out the next line.934        cargo.rustdocflag("--generate-link-to-definition");935        cargo.rustdocflag("--generate-macro-expansion");936937        compile::rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates);938        cargo.arg("-Zskip-rustdoc-fingerprint");939940        // Only include compiler crates, no dependencies of those, such as `libc`.941        // Do link to dependencies on `docs.rs` however using `rustdoc-map`.942        cargo.arg("--no-deps");943        cargo.arg("-Zrustdoc-map");944945        // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies,946        // once this is no longer an issue the special case for `ena` can be removed.947        cargo.rustdocflag("--extern-html-root-url");948        cargo.rustdocflag("ena=https://docs.rs/ena/latest/");949950        let mut to_open = None;951952        let out_dir = builder.stage_out(build_compiler, Mode::Rustc).join(target).join("doc");953        for krate in &*self.crates {954            // Create all crate output directories first to make sure rustdoc uses955            // relative links.956            // FIXME: Cargo should probably do this itself.957            let dir_name = krate.replace('-', "_");958            t!(fs::create_dir_all(out_dir.join(&*dir_name)));959            cargo.arg("-p").arg(krate);960            if to_open.is_none() {961                to_open = Some(dir_name);962            }963        }964965        // This uses a shared directory so that librustdoc documentation gets966        // correctly built and merged with the rustc documentation.967        //968        // This is needed because rustdoc is built in a different directory from969        // rustc. rustdoc needs to be able to see everything, for example when970        // merging the search index, or generating local (relative) links.971        symlink_dir_force(&builder.config, &out, &out_dir);972        // Cargo puts proc macros in `target/doc` even if you pass `--target`973        // explicitly (https://github.com/rust-lang/cargo/issues/7677).974        let proc_macro_out_dir = builder.stage_out(build_compiler, Mode::Rustc).join("doc");975        symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);976977        cargo.into_cmd().run(builder);978979        if !builder.config.dry_run() {980            // Sanity check on linked compiler crates981            for krate in &*self.crates {982                let dir_name = krate.replace('-', "_");983                // Making sure the directory exists and is not empty.984                assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());985            }986        }987988        if builder.paths.iter().any(|path| path.ends_with("compiler")) {989            // For `x.py doc compiler --open`, open `rustc_middle` by default.990            let index = out.join("rustc_middle").join("index.html");991            builder.open_in_browser(index);992        } else if let Some(krate) = to_open {993            // Let's open the first crate documentation page:994            let index = out.join(krate).join("index.html");995            builder.open_in_browser(index);996        }997    }998999    fn metadata(&self) -> Option<StepMetadata> {1000        Some(StepMetadata::doc("rustc", self.target).built_by(self.build_compiler))1001    }1002}10031004macro_rules! tool_doc {1005    (1006        $tool: ident,1007        $path: literal,1008        mode = $mode:expr1009        $(, is_library = $is_library:expr )?1010        $(, crates = $crates:expr )?1011        // Subset of nightly features that are allowed to be used when documenting1012        $(, allow_features: $allow_features:expr )?1013       ) => {1014        #[derive(Debug, Clone, Hash, PartialEq, Eq)]1015        pub struct $tool {1016            build_compiler: Compiler,1017            mode: Mode,1018            target: TargetSelection,1019        }10201021        impl Step for $tool {1022            type Output = ();1023            const IS_HOST: bool = true;10241025            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1026                run.path($path)1027            }10281029            fn is_default_step(builder: &Builder<'_>) -> bool {1030                builder.config.compiler_docs1031            }10321033            fn make_run(run: RunConfig<'_>) {1034                let target = run.target;1035                let build_compiler = match $mode {1036                    Mode::ToolRustcPrivate => {1037                        // Rustdoc needs the rustc sysroot available to build.1038                        let compilers = RustcPrivateCompilers::new(run.builder, run.builder.top_stage, target);10391040                        // Build rustc docs so that we generate relative links.1041                        run.builder.ensure(Rustc::from_build_compiler(run.builder, compilers.build_compiler(), target));1042                        compilers.build_compiler()1043                    }1044                    Mode::ToolTarget => {1045                        // when shipping multiple docs together in one folder,1046                        // they all need to use the same rustdoc version1047                        prepare_doc_compiler(run.builder, run.builder.host_target, run.builder.top_stage)1048                    }1049                    _ => {1050                        panic!("Unexpected tool mode for documenting: {:?}", $mode);1051                    }1052                };10531054                run.builder.ensure($tool { build_compiler, mode: $mode, target });1055            }10561057            /// Generates documentation for a tool.1058            ///1059            /// This is largely just a wrapper around `cargo doc`.1060            fn run(self, builder: &Builder<'_>) {1061                let mut source_type = SourceType::InTree;10621063                if let Some(submodule_path) = submodule_path_of(&builder, $path) {1064                    source_type = SourceType::Submodule;1065                    builder.require_submodule(&submodule_path, None);1066                }10671068                let $tool { build_compiler, mode, target } = self;10691070                // This is the intended out directory for compiler documentation.1071                let out = builder.compiler_doc_out(target);1072                t!(fs::create_dir_all(&out));10731074                // Build cargo command.1075                let mut cargo = prepare_tool_cargo(1076                    builder,1077                    build_compiler,1078                    mode,1079                    target,1080                    Kind::Doc,1081                    $path,1082                    source_type,1083                    &[],1084                );1085                let allow_features = {1086                    let mut _value = "";1087                    $( _value = $allow_features; )?1088                    _value1089                };10901091                if !allow_features.is_empty() {1092                    cargo.allow_features(allow_features);1093                }10941095                cargo.arg("-Zskip-rustdoc-fingerprint");1096                // Only include compiler crates, no dependencies of those, such as `libc`.1097                cargo.arg("--no-deps");10981099                if false $(|| $is_library)? {1100                    cargo.arg("--lib");1101                }11021103                $(for krate in $crates {1104                    cargo.arg("-p").arg(krate);1105                })?11061107                cargo.rustdocflag("--document-private-items");1108                // Since we always pass --document-private-items, there's no need to warn about linking to private items.1109                cargo.rustdocflag("-Arustdoc::private-intra-doc-links");1110                cargo.rustdocflag("--enable-index-page");1111                cargo.rustdocflag("--show-type-layout");1112                cargo.rustdocflag("--generate-link-to-definition");11131114                let out_dir = builder.stage_out(build_compiler, mode).join(target).join("doc");1115                $(for krate in $crates {1116                    let dir_name = krate.replace("-", "_");1117                    t!(fs::create_dir_all(out_dir.join(&*dir_name)));1118                })?11191120                // Symlink compiler docs to the output directory of rustdoc documentation.1121                symlink_dir_force(&builder.config, &out, &out_dir);1122                let proc_macro_out_dir = builder.stage_out(build_compiler, mode).join("doc");1123                symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);11241125                let _guard = builder.msg(Kind::Doc, stringify!($tool).to_lowercase(), None, build_compiler, target);1126                cargo.into_cmd().run(builder);11271128                if !builder.config.dry_run() {1129                    // Sanity check on linked doc directories1130                    $(for krate in $crates {1131                        let dir_name = krate.replace("-", "_");1132                        // Making sure the directory exists and is not empty.1133                        assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());1134                    })?1135                }1136            }11371138            fn metadata(&self) -> Option<StepMetadata> {1139                Some(StepMetadata::doc(stringify!($tool), self.target).built_by(self.build_compiler))1140            }1141        }1142    }1143}11441145// NOTE: make sure to register these in `Builder::get_step_description`.1146tool_doc!(1147    BuildHelper,1148    "src/build_helper",1149    // ideally, this would use ToolBootstrap,1150    // but we distribute these docs together in the same folder1151    // as a bunch of stage1 tools, and you can't mix rustdoc versions1152    // because that breaks cross-crate data (particularly search)1153    mode = Mode::ToolTarget,1154    is_library = true,1155    crates = ["build_helper"]1156);1157tool_doc!(1158    Rustdoc,1159    "src/tools/rustdoc",1160    mode = Mode::ToolRustcPrivate,1161    crates = ["rustdoc", "rustdoc-json-types"]1162);1163tool_doc!(1164    Rustfmt,1165    "src/tools/rustfmt",1166    mode = Mode::ToolRustcPrivate,1167    crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"]1168);1169tool_doc!(1170    Clippy,1171    "src/tools/clippy",1172    mode = Mode::ToolRustcPrivate,1173    crates = ["clippy_config", "clippy_utils"]1174);1175tool_doc!(Miri, "src/tools/miri", mode = Mode::ToolRustcPrivate, crates = ["miri"]);1176tool_doc!(1177    Cargo,1178    "src/tools/cargo",1179    mode = Mode::ToolTarget,1180    crates = [1181        "cargo",1182        "cargo-credential",1183        "cargo-platform",1184        "cargo-test-macro",1185        "cargo-test-support",1186        "cargo-util",1187        "cargo-util-schemas",1188        "crates-io",1189        "mdman",1190        "rustfix",1191    ],1192    // Required because of the im-rc dependency of Cargo, which automatically opts into the1193    // "specialization" feature in its build script when it detects a nightly toolchain.1194    allow_features: "specialization"1195);1196tool_doc!(Tidy, "src/tools/tidy", mode = Mode::ToolTarget, crates = ["tidy"]);1197tool_doc!(1198    Bootstrap,1199    "src/bootstrap",1200    mode = Mode::ToolTarget,1201    is_library = true,1202    crates = ["bootstrap"]1203);1204tool_doc!(1205    RunMakeSupport,1206    "src/tools/run-make-support",1207    mode = Mode::ToolTarget,1208    is_library = true,1209    crates = ["run_make_support"]1210);1211tool_doc!(1212    Compiletest,1213    "src/tools/compiletest",1214    mode = Mode::ToolTarget,1215    is_library = true,1216    crates = ["compiletest"]1217);12181219#[derive(Debug, Clone, Hash, PartialEq, Eq)]1220pub struct ErrorIndex {1221    compilers: RustcPrivateCompilers,1222}12231224impl Step for ErrorIndex {1225    type Output = ();1226    const IS_HOST: bool = true;12271228    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1229        run.path("src/tools/error_index_generator")1230    }12311232    fn is_default_step(builder: &Builder<'_>) -> bool {1233        builder.config.docs1234    }12351236    fn make_run(run: RunConfig<'_>) {1237        run.builder.ensure(ErrorIndex {1238            compilers: RustcPrivateCompilers::new(run.builder, run.builder.top_stage, run.target),1239        });1240    }12411242    /// Generates the HTML rendered error-index by running the1243    /// `error_index_generator` tool.1244    fn run(self, builder: &Builder<'_>) {1245        builder.info(&format!("Documenting error index ({})", self.compilers.target()));1246        let out = builder.doc_out(self.compilers.target());1247        t!(fs::create_dir_all(&out));1248        tool::ErrorIndex::command(builder, self.compilers)1249            .arg("html")1250            .arg(&out)1251            .arg(&builder.version)1252            .run(builder);12531254        let index = out.join("error-index.html");1255        builder.maybe_open_in_browser::<Self>(index);1256    }12571258    fn metadata(&self) -> Option<StepMetadata> {1259        Some(1260            StepMetadata::doc("error-index", self.compilers.target())1261                .built_by(self.compilers.build_compiler()),1262        )1263    }1264}12651266#[derive(Debug, Clone, Hash, PartialEq, Eq)]1267pub struct UnstableBookGen {1268    target: TargetSelection,1269}12701271impl Step for UnstableBookGen {1272    type Output = ();1273    const IS_HOST: bool = true;12741275    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1276        run.path("src/tools/unstable-book-gen")1277    }12781279    fn is_default_step(builder: &Builder<'_>) -> bool {1280        builder.config.docs1281    }12821283    fn make_run(run: RunConfig<'_>) {1284        run.builder.ensure(UnstableBookGen { target: run.target });1285    }12861287    fn run(self, builder: &Builder<'_>) {1288        let target = self.target;12891290        builder.info(&format!("Generating unstable book md files ({target})"));1291        let out = builder.md_doc_out(target).join("unstable-book");1292        builder.create_dir(&out);1293        builder.remove_dir(&out);1294        let mut cmd = builder.tool_cmd(Tool::UnstableBookGen);1295        cmd.arg(builder.src.join("library"));1296        cmd.arg(builder.src.join("compiler"));1297        cmd.arg(builder.src.join("src"));1298        cmd.arg(out);12991300        cmd.run(builder);1301    }1302}13031304fn symlink_dir_force(config: &Config, original: &Path, link: &Path) {1305    if config.dry_run() {1306        return;1307    }1308    if let Ok(m) = fs::symlink_metadata(link) {1309        if m.file_type().is_dir() {1310            t!(fs::remove_dir_all(link));1311        } else {1312            // handle directory junctions on windows by falling back to1313            // `remove_dir`.1314            t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link)));1315        }1316    }13171318    t!(1319        symlink_dir(config, original, link),1320        format!("failed to create link from {} -> {}", link.display(), original.display())1321    );1322}13231324/// Builds the Rust compiler book.1325#[derive(Debug, Clone, Hash, PartialEq, Eq)]1326pub struct RustcBook {1327    build_compiler: Compiler,1328    target: TargetSelection,1329    /// Test that the examples of lints in the book produce the correct lints in the expected1330    /// format.1331    validate: bool,1332}13331334impl RustcBook {1335    pub fn validate(build_compiler: Compiler, target: TargetSelection) -> Self {1336        Self { build_compiler, target, validate: true }1337    }1338}13391340impl Step for RustcBook {1341    type Output = ();1342    const IS_HOST: bool = true;13431344    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1345        run.path("src/doc/rustc")1346    }13471348    fn is_default_step(builder: &Builder<'_>) -> bool {1349        builder.config.docs1350    }13511352    fn make_run(run: RunConfig<'_>) {1353        // Bump the stage to 2, because the rustc book requires an in-tree compiler.1354        // At the same time, since this step is enabled by default, we don't want `x doc` to fail1355        // in stage 1.1356        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {1357            run.builder.top_stage1358        } else {1359            21360        };13611362        run.builder.ensure(RustcBook {1363            build_compiler: prepare_doc_compiler(run.builder, run.target, stage),1364            target: run.target,1365            validate: false,1366        });1367    }13681369    /// Builds the rustc book.1370    ///1371    /// The lints are auto-generated by a tool, and then merged into the book1372    /// in the "md-doc" directory in the build output directory. Then1373    /// "rustbook" is used to convert it to HTML.1374    fn run(self, builder: &Builder<'_>) {1375        let out_base = builder.md_doc_out(self.target).join("rustc");1376        t!(fs::create_dir_all(&out_base));1377        let out_listing = out_base.join("src/lints");1378        builder.cp_link_r(&builder.src.join("src/doc/rustc"), &out_base);1379        builder.info(&format!("Generating lint docs ({})", self.target));13801381        let rustc = builder.rustc(self.build_compiler);1382        // The tool runs `rustc` for extracting output examples, so it needs a1383        // functional sysroot.1384        builder.std(self.build_compiler, self.target);1385        let mut cmd = builder.tool_cmd(Tool::LintDocs);1386        cmd.arg("--build-rustc-stage");1387        cmd.arg(self.build_compiler.stage.to_string());1388        cmd.arg("--src");1389        cmd.arg(builder.src.join("compiler"));1390        cmd.arg("--out");1391        cmd.arg(&out_listing);1392        cmd.arg("--rustc");1393        cmd.arg(&rustc);1394        cmd.arg("--rustc-target").arg(self.target.rustc_target_arg());1395        if let Some(target_linker) = builder.linker(self.target) {1396            cmd.arg("--rustc-linker").arg(target_linker);1397        }1398        if builder.is_verbose() {1399            cmd.arg("--verbose");1400        }1401        if self.validate {1402            cmd.arg("--validate");1403        }1404        // We need to validate nightly features, even on the stable channel.1405        // Set this unconditionally as the stage0 compiler may be being used to1406        // document.1407        cmd.env("RUSTC_BOOTSTRAP", "1");14081409        // If the lib directories are in an unusual location (changed in1410        // bootstrap.toml), then this needs to explicitly update the dylib search1411        // path.1412        builder.add_rustc_lib_path(self.build_compiler, &mut cmd);1413        let doc_generator_guard =1414            builder.msg(Kind::Run, "lint-docs", None, self.build_compiler, self.target);1415        cmd.run(builder);1416        drop(doc_generator_guard);14171418        // Run rustbook/mdbook to generate the HTML pages.1419        builder.ensure(RustbookSrc {1420            target: self.target,1421            name: "rustc".to_owned(),1422            src: out_base,1423            parent: Some(self),1424            languages: vec![],1425            build_compiler: None,1426        });1427    }1428}14291430/// Documents the reference.1431/// It has to always be done using a stage 1+ compiler, because it references in-tree1432/// compiler/stdlib concepts.1433#[derive(Debug, Clone, Hash, PartialEq, Eq)]1434pub struct Reference {1435    build_compiler: Compiler,1436    target: TargetSelection,1437}14381439impl Step for Reference {1440    type Output = ();14411442    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1443        run.path("src/doc/reference")1444    }14451446    fn is_default_step(builder: &Builder<'_>) -> bool {1447        builder.config.docs1448    }14491450    fn make_run(run: RunConfig<'_>) {1451        // Bump the stage to 2, because the reference requires an in-tree compiler.1452        // At the same time, since this step is enabled by default, we don't want `x doc` to fail1453        // in stage 1.1454        // FIXME: create a shared method on builder for auto-bumping, and print some warning when1455        // it happens.1456        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {1457            run.builder.top_stage1458        } else {1459            21460        };14611462        run.builder.ensure(Reference {1463            build_compiler: prepare_doc_compiler(run.builder, run.target, stage),1464            target: run.target,1465        });1466    }14671468    /// Builds the reference book.1469    fn run(self, builder: &Builder<'_>) {1470        builder.require_submodule("src/doc/reference", None);14711472        // This is needed for generating links to the standard library using1473        // the mdbook-spec plugin.1474        builder.std(self.build_compiler, builder.config.host_target);14751476        // Run rustbook/mdbook to generate the HTML pages.1477        builder.ensure(RustbookSrc {1478            target: self.target,1479            name: "reference".to_owned(),1480            src: builder.src.join("src/doc/reference"),1481            build_compiler: Some(self.build_compiler),1482            parent: Some(self),1483            languages: vec![],1484        });1485    }1486}