1//! Documentation generation for bootstrap.2//!3//! This module implements generation for all bits and pieces of documentation4//! for the Rust project. This notably includes suites like the rust book, the5//! nomicon, rust by example, standalone documentation, etc.6//!7//! Everything here is basically just a shim around calling either `rustbook` or8//! `rustdoc`.910use std::io::{self, Write};11use std::path::{Path, PathBuf};12use std::{env, fs, mem};1314use crate::core::build_steps::compile;15use crate::core::build_steps::tool::{16    self, RustcPrivateCompilers, SourceType, Tool, prepare_tool_cargo,17};18use crate::core::builder::{19    self, Builder, Compiler, Kind, RunConfig, ShouldRun, Step, StepMetadata, crate_description,20};21use crate::core::config::{Config, TargetSelection};22use crate::helpers::{submodule_path_of, symlink_dir, t, up_to_date};23use crate::{FileType, Mode};2425macro_rules! book {26    ($($name:ident, $path:expr, $book_name:expr, $lang:expr ;)+) => {27        $(28        #[derive(Debug, Clone, Hash, PartialEq, Eq)]29        pub struct $name {30            target: TargetSelection,31        }3233        impl Step for $name {34            type Output = ();3536            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {37                run.path($path)38            }3940            fn is_default_step(builder: &Builder<'_>) -> bool {41                builder.config.docs42            }4344            fn make_run(run: RunConfig<'_>) {45                run.builder.ensure($name {46                    target: run.target,47                });48            }4950            fn run(self, builder: &Builder<'_>) {51                if let Some(submodule_path) = submodule_path_of(&builder, $path) {52                    builder.require_submodule(&submodule_path, None)53                }5455                builder.ensure(RustbookSrc {56                    target: self.target,57                    name: $book_name.to_owned(),58                    src: builder.src.join($path),59                    parent: Some(self),60                    languages: $lang.into(),61                    build_compiler: None,62                })63            }64        }65        )+66    }67}6869// NOTE: When adding a book here, make sure to ALSO build the book by70// adding a build step in `src/bootstrap/code/builder/mod.rs`!71// NOTE: Make sure to add the corresponding submodule when adding a new book.72book!(73    CargoBook, "src/tools/cargo/src/doc", "cargo", &[];74    ClippyBook, "src/tools/clippy/book", "clippy", &[];75    EditionGuide, "src/doc/edition-guide", "edition-guide", &[];76    EmbeddedBook, "src/doc/embedded-book", "embedded-book", &[];77    Nomicon, "src/doc/nomicon", "nomicon", &[];78    RustByExample, "src/doc/rust-by-example", "rust-by-example", &["es", "ja", "zh", "ko"];79    RustdocBook, "src/doc/rustdoc", "rustdoc", &[];80    StyleGuide, "src/doc/style-guide", "style-guide", &[];81);8283#[derive(Debug, Clone, Hash, PartialEq, Eq)]84pub struct UnstableBook {85    build_compiler: Compiler,86    target: TargetSelection,87}8889impl Step for UnstableBook {90    type Output = ();9192    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {93        run.path("src/doc/unstable-book")94    }9596    fn is_default_step(builder: &Builder<'_>) -> bool {97        builder.config.docs98    }99100    fn make_run(run: RunConfig<'_>) {101        // Bump the stage to 2, because the unstable book requires an in-tree compiler.102        // At the same time, since this step is enabled by default, we don't want `x doc` to fail103        // in stage 1.104        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {105            run.builder.top_stage106        } else {107            2108        };109110        run.builder.ensure(UnstableBook {111            build_compiler: prepare_doc_compiler(run.builder, run.target, stage),112            target: run.target,113        });114    }115116    fn run(self, builder: &Builder<'_>) {117        builder118            .ensure(UnstableBookGen { build_compiler: self.build_compiler, target: self.target });119        builder.ensure(RustbookSrc {120            target: self.target,121            name: "unstable-book".to_owned(),122            src: builder.md_doc_out(self.target).join("unstable-book"),123            parent: Some(self),124            languages: vec![],125            build_compiler: None,126        })127    }128}129130#[derive(Debug, Clone, Hash, PartialEq, Eq)]131struct RustbookSrc<P: Step> {132    target: TargetSelection,133    name: String,134    src: PathBuf,135    parent: Option<P>,136    languages: Vec<&'static str>,137    /// Compiler whose rustdoc should be used to document things using `mdbook-spec`.138    build_compiler: Option<Compiler>,139}140141impl<P: Step> Step for RustbookSrc<P> {142    type Output = ();143144    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {145        run.never()146    }147148    /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.149    ///150    /// This will not actually generate any documentation if the documentation has151    /// already been generated.152    fn run(self, builder: &Builder<'_>) {153        let target = self.target;154        let name = self.name;155        let src = self.src;156        let out = builder.doc_out(target);157        t!(fs::create_dir_all(&out));158159        let out = out.join(&name);160        let index = out.join("index.html");161        let rustbook = builder.tool_exe(Tool::Rustbook);162163        if !builder.config.dry_run()164            && (!up_to_date(&src, &index) || !up_to_date(&rustbook, &index))165        {166            builder.info(&format!("Rustbook ({target}) - {name}"));167            let _ = fs::remove_dir_all(&out);168169            let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook);170171            if let Some(compiler) = self.build_compiler {172                let mut rustdoc = builder.rustdoc_for_compiler(compiler);173                rustdoc.pop();174                let old_path = env::var_os("PATH").unwrap_or_default();175                let new_path =176                    env::join_paths(std::iter::once(rustdoc).chain(env::split_paths(&old_path)))177                        .expect("could not add rustdoc to PATH");178179                rustbook_cmd.env("PATH", new_path);180                builder.add_rustc_lib_path(compiler, &mut rustbook_cmd);181            }182183            rustbook_cmd184                .arg("build")185                .arg(&src)186                .arg("-d")187                .arg(&out)188                .arg("--rust-root")189                .arg(&builder.src)190                .run(builder);191192            for lang in &self.languages {193                let out = out.join(lang);194195                builder.info(&format!("Rustbook ({target}) - {name} - {lang}"));196                let _ = fs::remove_dir_all(&out);197198                builder199                    .tool_cmd(Tool::Rustbook)200                    .arg("build")201                    .arg(&src)202                    .arg("-d")203                    .arg(&out)204                    .arg("-l")205                    .arg(lang)206                    .run(builder);207            }208        }209210        if self.parent.is_some() {211            builder.maybe_open_in_browser::<P>(index)212        }213    }214215    fn metadata(&self) -> Option<StepMetadata> {216        let mut metadata = StepMetadata::doc(&format!("{} (book)", self.name), self.target);217        if let Some(compiler) = self.build_compiler {218            metadata = metadata.built_by(compiler);219        }220221        Some(metadata)222    }223}224225#[derive(Debug, Clone, Hash, PartialEq, Eq)]226pub struct TheBook {227    /// Compiler whose rustdoc will be used to generated documentation.228    build_compiler: Compiler,229    target: TargetSelection,230}231232impl Step for TheBook {233    type Output = ();234235    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {236        run.path("src/doc/book")237    }238239    fn is_default_step(builder: &Builder<'_>) -> bool {240        builder.config.docs241    }242243    fn make_run(run: RunConfig<'_>) {244        run.builder.ensure(TheBook {245            build_compiler: prepare_doc_compiler(run.builder, run.target, run.builder.top_stage),246            target: run.target,247        });248    }249250    /// Builds the book and associated stuff.251    ///252    /// We need to build:253    ///254    /// * Book255    /// * Older edition redirects256    /// * Version info and CSS257    /// * Index page258    /// * Redirect pages259    fn run(self, builder: &Builder<'_>) {260        builder.require_submodule("src/doc/book", None);261262        let build_compiler = self.build_compiler;263        let target = self.target;264265        let absolute_path = builder.src.join("src/doc/book");266        let redirect_path = absolute_path.join("redirects");267268        // build book269        builder.ensure(RustbookSrc {270            target,271            name: "book".to_owned(),272            src: absolute_path.clone(),273            parent: Some(self),274            languages: vec![],275            build_compiler: None,276        });277278        // building older edition redirects279        for edition in &["first-edition", "second-edition", "2018-edition"] {280            builder.ensure(RustbookSrc {281                target,282                name: format!("book/{edition}"),283                src: absolute_path.join(edition),284                // There should only be one book that is marked as the parent for each target, so285                // treat the other editions as not having a parent.286                parent: Option::<Self>::None,287                languages: vec![],288                build_compiler: None,289            });290        }291292        // build the version info page and CSS293        let shared_assets = builder.ensure(SharedAssets { target });294295        // build the redirect pages296        let _guard = builder.msg(Kind::Doc, "book redirect pages", None, build_compiler, target);297        if builder.config.dry_run() {298            return;299        }300301        for file in t!(fs::read_dir(redirect_path)) {302            let file = t!(file);303            let path = file.path();304            let path = path.to_str().unwrap();305306            invoke_rustdoc(builder, build_compiler, &shared_assets, target, path);307        }308    }309}310311fn invoke_rustdoc(312    builder: &Builder<'_>,313    build_compiler: Compiler,314    shared_assets: &SharedAssetsPaths,315    target: TargetSelection,316    markdown: &str,317) {318    let out = builder.doc_out(target);319320    let path = builder.src.join("src/doc").join(markdown);321322    let header = builder.src.join("src/doc/redirect.inc");323    let footer = builder.src.join("src/doc/footer.inc");324325    let mut cmd = builder.rustdoc_cmd(build_compiler);326327    let out = out.join("book");328329    cmd.arg("--html-after-content")330        .arg(&footer)331        .arg("--html-before-content")332        .arg(&shared_assets.version_info)333        .arg("--html-in-header")334        .arg(&header)335        .arg("--markdown-no-toc")336        .arg("--markdown-playground-url")337        .arg("https://play.rust-lang.org/")338        .arg("-o")339        .arg(&out)340        .arg(&path)341        .arg("--markdown-css")342        .arg("../rust.css")343        .arg("-Zunstable-options");344345    if !builder.config.docs_minification {346        cmd.arg("--disable-minification");347    }348349    cmd.run(builder);350}351352#[derive(Debug, Clone, Hash, PartialEq, Eq)]353pub struct Standalone {354    build_compiler: Compiler,355    target: TargetSelection,356}357358impl Step for Standalone {359    type Output = ();360361    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {362        run.path("src/doc").alias("standalone")363    }364365    fn is_default_step(builder: &Builder<'_>) -> bool {366        builder.config.docs367    }368369    fn make_run(run: RunConfig<'_>) {370        run.builder.ensure(Standalone {371            build_compiler: prepare_doc_compiler(372                run.builder,373                run.builder.host_target,374                run.builder.top_stage,375            ),376            target: run.target,377        });378    }379380    /// Generates all standalone documentation as compiled by the rustdoc in `stage`381    /// for the `target` into `out`.382    ///383    /// This will list all of `src/doc` looking for markdown files and appropriately384    /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and385    /// `STAMP` along with providing the various header/footer HTML we've customized.386    ///387    /// In the end, this is just a glorified wrapper around rustdoc!388    fn run(self, builder: &Builder<'_>) {389        let target = self.target;390        let build_compiler = self.build_compiler;391        let _guard = builder.msg(Kind::Doc, "standalone", None, build_compiler, target);392        let out = builder.doc_out(target);393        t!(fs::create_dir_all(&out));394395        let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;396397        let favicon = builder.src.join("src/doc/favicon.inc");398        let footer = builder.src.join("src/doc/footer.inc");399        let full_toc = builder.src.join("src/doc/full-toc.inc");400401        for file in t!(fs::read_dir(builder.src.join("src/doc"))) {402            let file = t!(file);403            let path = file.path();404            let filename = path.file_name().unwrap().to_str().unwrap();405            if !filename.ends_with(".md") || filename == "README.md" {406                continue;407            }408409            let html = out.join(filename).with_extension("html");410            let rustdoc = builder.rustdoc_for_compiler(build_compiler);411            if up_to_date(&path, &html)412                && up_to_date(&footer, &html)413                && up_to_date(&favicon, &html)414                && up_to_date(&full_toc, &html)415                && (builder.config.dry_run() || up_to_date(&version_info, &html))416                && (builder.config.dry_run() || up_to_date(&rustdoc, &html))417            {418                continue;419            }420421            let mut cmd = builder.rustdoc_cmd(build_compiler);422423            cmd.arg("--html-after-content")424                .arg(&footer)425                .arg("--html-before-content")426                .arg(&version_info)427                .arg("--html-in-header")428                .arg(&favicon)429                .arg("--markdown-no-toc")430                .arg("-Zunstable-options")431                .arg("--index-page")432                .arg(builder.src.join("src/doc/index.md"))433                .arg("--markdown-playground-url")434                .arg("https://play.rust-lang.org/")435                .arg("-o")436                .arg(&out)437                .arg(&path);438439            if !builder.config.docs_minification {440                cmd.arg("--disable-minification");441            }442443            if filename == "not_found.md" {444                cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css");445            } else {446                cmd.arg("--markdown-css").arg("rust.css");447            }448            cmd.run(builder);449        }450451        // We open doc/index.html as the default if invoked as `x.py doc --open`452        // with no particular explicit doc requested (e.g. library/core).453        if builder.paths.is_empty() || builder.was_invoked_explicitly::<Self>(Kind::Doc) {454            let index = out.join("index.html");455            builder.open_in_browser(index);456        }457    }458459    fn metadata(&self) -> Option<StepMetadata> {460        Some(StepMetadata::doc("standalone", self.target).built_by(self.build_compiler))461    }462}463464#[derive(Debug, Clone, Hash, PartialEq, Eq)]465pub struct Releases {466    build_compiler: Compiler,467    target: TargetSelection,468}469470impl Step for Releases {471    type Output = ();472473    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {474        run.path("RELEASES.md").alias("releases")475    }476477    fn is_default_step(builder: &Builder<'_>) -> bool {478        builder.config.docs479    }480481    fn make_run(run: RunConfig<'_>) {482        run.builder.ensure(Releases {483            build_compiler: prepare_doc_compiler(484                run.builder,485                run.builder.host_target,486                run.builder.top_stage,487            ),488            target: run.target,489        });490    }491492    /// Generates HTML release notes to include in the final docs bundle.493    ///494    /// This uses the same stylesheet and other tools as Standalone, but the495    /// RELEASES.md file is included at the root of the repository and gets496    /// the headline added. In the end, the conversion is done by Rustdoc.497    fn run(self, builder: &Builder<'_>) {498        let target = self.target;499        let build_compiler = self.build_compiler;500        let _guard = builder.msg(Kind::Doc, "releases", None, build_compiler, target);501        let out = builder.doc_out(target);502        t!(fs::create_dir_all(&out));503504        builder.ensure(Standalone { build_compiler, target });505506        let version_info = builder.ensure(SharedAssets { target: self.target }).version_info;507508        let favicon = builder.src.join("src/doc/favicon.inc");509        let footer = builder.src.join("src/doc/footer.inc");510        let full_toc = builder.src.join("src/doc/full-toc.inc");511512        let html = out.join("releases.html");513        let tmppath = out.join("releases.md");514        let inpath = builder.src.join("RELEASES.md");515        let rustdoc = builder.rustdoc_for_compiler(build_compiler);516        if !up_to_date(&inpath, &html)517            || !up_to_date(&footer, &html)518            || !up_to_date(&favicon, &html)519            || !up_to_date(&full_toc, &html)520            || !(builder.config.dry_run()521                || up_to_date(&version_info, &html)522                || up_to_date(&rustdoc, &html))523        {524            let mut tmpfile = t!(fs::File::create(&tmppath));525            t!(tmpfile.write_all(b"% Rust Release Notes\n\n"));526            t!(io::copy(&mut t!(fs::File::open(&inpath)), &mut tmpfile));527            mem::drop(tmpfile);528            let mut cmd = builder.rustdoc_cmd(build_compiler);529530            cmd.arg("--html-after-content")531                .arg(&footer)532                .arg("--html-before-content")533                .arg(&version_info)534                .arg("--html-in-header")535                .arg(&favicon)536                .arg("--markdown-no-toc")537                .arg("--markdown-css")538                .arg("rust.css")539                .arg("-Zunstable-options")540                .arg("--index-page")541                .arg(builder.src.join("src/doc/index.md"))542                .arg("--markdown-playground-url")543                .arg("https://play.rust-lang.org/")544                .arg("-o")545                .arg(&out)546                .arg(&tmppath);547548            if !builder.config.docs_minification {549                cmd.arg("--disable-minification");550            }551552            cmd.run(builder);553        }554555        // We open doc/RELEASES.html as the default if invoked as `x.py doc --open RELEASES.md`556        // with no particular explicit doc requested (e.g. library/core).557        if builder.was_invoked_explicitly::<Self>(Kind::Doc) {558            builder.open_in_browser(&html);559        }560    }561562    fn metadata(&self) -> Option<StepMetadata> {563        Some(StepMetadata::doc("releases", self.target).built_by(self.build_compiler))564    }565}566567#[derive(Debug, Clone)]568pub struct SharedAssetsPaths {569    pub version_info: PathBuf,570}571572#[derive(Debug, Clone, Hash, PartialEq, Eq)]573pub struct SharedAssets {574    target: TargetSelection,575}576577impl Step for SharedAssets {578    type Output = SharedAssetsPaths;579580    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {581        // Other tasks depend on this, no need to execute it on its own582        run.never()583    }584585    fn is_default_step(_builder: &Builder<'_>) -> bool {586        false587    }588589    /// Generate shared resources used by other pieces of documentation.590    fn run(self, builder: &Builder<'_>) -> Self::Output {591        let out = builder.doc_out(self.target);592593        let version_input = builder.src.join("src").join("doc").join("version_info.html.template");594        let version_info = out.join("version_info.html");595        if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) {596            let info = t!(fs::read_to_string(&version_input))597                .replace("VERSION", &builder.rust_release())598                .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or(""))599                .replace("STAMP", builder.rust_info().sha().unwrap_or(""));600            t!(fs::write(&version_info, info));601        }602603        builder.copy_link(604            &builder.src.join("src").join("doc").join("rust.css"),605            &out.join("rust.css"),606            FileType::Regular,607        );608609        builder.copy_link(610            &builder611                .src612                .join("src")613                .join("librustdoc")614                .join("html")615                .join("static")616                .join("images")617                .join("favicon.svg"),618            &out.join("favicon.svg"),619            FileType::Regular,620        );621        builder.copy_link(622            &builder623                .src624                .join("src")625                .join("librustdoc")626                .join("html")627                .join("static")628                .join("images")629                .join("favicon-32x32.png"),630            &out.join("favicon-32x32.png"),631            FileType::Regular,632        );633634        SharedAssetsPaths { version_info }635    }636}637638/// Document the standard library using `build_compiler`.639#[derive(Debug, Clone, Hash, PartialEq, Eq)]640pub struct Std {641    build_compiler: Compiler,642    target: TargetSelection,643    format: DocumentationFormat,644    crates: Vec<String>,645}646647impl Std {648    pub(crate) fn from_build_compiler(649        build_compiler: Compiler,650        target: TargetSelection,651        format: DocumentationFormat,652    ) -> Self {653        Std { build_compiler, target, format, crates: vec![] }654    }655}656657impl Step for Std {658    /// Path to a directory with the built documentation.659    type Output = PathBuf;660661    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {662        run.crate_or_deps("sysroot").path("library")663    }664665    fn is_default_step(builder: &Builder<'_>) -> bool {666        builder.config.docs667    }668669    fn make_run(run: RunConfig<'_>) {670        let crates = compile::std_crates_for_run_make(&run);671        let target_is_no_std = run.builder.no_std(run.target).unwrap_or(false);672        if crates.is_empty() && target_is_no_std {673            return;674        }675        run.builder.ensure(Std {676            build_compiler: run.builder.compiler_for_std(run.builder.top_stage),677            target: run.target,678            format: if run.builder.config.cmd.json() {679                DocumentationFormat::Json680            } else {681                DocumentationFormat::Html682            },683            crates,684        });685    }686687    /// Compile all standard library documentation.688    ///689    /// This will generate all documentation for the standard library and its690    /// dependencies. This is largely just a wrapper around `cargo doc`.691    fn run(self, builder: &Builder<'_>) -> Self::Output {692        let target = self.target;693        let crates = if self.crates.is_empty() {694            builder695                .in_tree_crates("sysroot", Some(target))696                .iter()697                .map(|c| c.name.to_string())698                .collect()699        } else {700            self.crates701        };702703        let out = match self.format {704            DocumentationFormat::Html => builder.doc_out(target),705            DocumentationFormat::Json => builder.json_doc_out(target),706        };707708        t!(fs::create_dir_all(&out));709710        if self.format == DocumentationFormat::Html {711            builder.ensure(SharedAssets { target: self.target });712        }713714        let index_page = builder715            .src716            .join("src/doc/index.md")717            .into_os_string()718            .into_string()719            .expect("non-utf8 paths are unsupported");720        let mut extra_args = match self.format {721            DocumentationFormat::Html => {722                vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page]723            }724            DocumentationFormat::Json => vec!["--output-format", "json"],725        };726727        if !builder.config.docs_minification {728            extra_args.push("--disable-minification");729        }730        // For `--index-page` and `--output-format=json`.731        extra_args.push("-Zunstable-options");732733        doc_std(builder, self.format, self.build_compiler, target, &out, &extra_args, &crates);734735        // Open if the format is HTML736        if let DocumentationFormat::Html = self.format {737            if builder.paths.iter().any(|path| path.ends_with("library")) {738                // For `x.py doc library --open`, open `std` by default.739                let index = out.join("std").join("index.html");740                builder.maybe_open_in_browser::<Self>(index);741            } else {742                for requested_crate in crates {743                    if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) {744                        let index = out.join(requested_crate).join("index.html");745                        builder.maybe_open_in_browser::<Self>(index);746                        break;747                    }748                }749            }750        }751752        out753    }754755    fn metadata(&self) -> Option<StepMetadata> {756        Some(757            StepMetadata::doc("std", self.target)758                .built_by(self.build_compiler)759                .with_metadata(format!("crates=[{}]", self.crates.join(","))),760        )761    }762}763764/// Name of the crates that are visible to consumers of the standard library.765/// Documentation for internal crates is handled by the rustc step, so internal crates will show766/// up there.767///768/// Order here is important!769/// Crates need to be processed starting from the leaves, otherwise rustdoc will not770/// create correct links between crates because rustdoc depends on the771/// existence of the output directories to know if it should be a local772/// or remote link.773const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"];774775#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]776pub enum DocumentationFormat {777    Html,778    Json,779}780781impl DocumentationFormat {782    fn as_str(&self) -> &str {783        match self {784            DocumentationFormat::Html => "HTML",785            DocumentationFormat::Json => "JSON",786        }787    }788}789790/// Build the documentation for public standard library crates.791fn doc_std(792    builder: &Builder<'_>,793    format: DocumentationFormat,794    build_compiler: Compiler,795    target: TargetSelection,796    out: &Path,797    extra_args: &[&str],798    requested_crates: &[String],799) {800    let target_doc_dir_name = if format == DocumentationFormat::Json { "json-doc" } else { "doc" };801    let target_dir =802        builder.stage_out(build_compiler, Mode::Std).join(target).join(target_doc_dir_name);803804    // This is directory where the compiler will place the output of the command.805    // We will then copy the files from this directory into the final `out` directory, the specified806    // as a function parameter.807    let out_dir = target_dir.join(target).join("doc");808809    let mut cargo = builder::Cargo::new(810        builder,811        build_compiler,812        Mode::Std,813        SourceType::InTree,814        target,815        Kind::Doc,816    );817818    compile::std_cargo(builder, target, &mut cargo, requested_crates);819    cargo820        .arg("--no-deps")821        .arg("--target-dir")822        .arg(&*target_dir.to_string_lossy())823        .arg("-Zskip-rustdoc-fingerprint")824        .arg("-Zrustdoc-map")825        .rustdocflag("--extern-html-root-url")826        .rustdocflag("std_detect=https://docs.rs/std_detect/latest/")827        .rustdocflag("--extern-html-root-takes-precedence")828        .rustdocflag("--resource-suffix")829        .rustdocflag(&builder.version);830    for arg in extra_args {831        cargo.rustdocflag(arg);832    }833834    if builder.config.library_docs_private_items {835        cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items");836    }837838    let description =839        format!("library{} in {} format", crate_description(requested_crates), format.as_str());840    let _guard = builder.msg(Kind::Doc, description, Mode::Std, build_compiler, target);841842    cargo.into_cmd().run(builder);843    builder.cp_link_r(&out_dir, out);844}845846/// Prepare a compiler that will be able to document something for `target` at `stage`.847pub fn prepare_doc_compiler(848    builder: &Builder<'_>,849    target: TargetSelection,850    stage: u32,851) -> Compiler {852    assert!(stage > 0, "Cannot document anything in stage 0");853    let build_compiler = builder.compiler(stage - 1, builder.host_target);854    builder.std(build_compiler, target);855    build_compiler856}857858/// Document the compiler for the given `target` using rustdoc from `build_compiler`.859#[derive(Debug, Clone, Hash, PartialEq, Eq)]860pub struct Rustc {861    build_compiler: Compiler,862    target: TargetSelection,863    crates: Vec<String>,864}865866impl Rustc {867    /// Document `stage` compiler for the given `target`.868    pub(crate) fn for_stage(builder: &Builder<'_>, stage: u32, target: TargetSelection) -> Self {869        let build_compiler = prepare_doc_compiler(builder, target, stage);870        Self::from_build_compiler(builder, build_compiler, target)871    }872873    fn from_build_compiler(874        builder: &Builder<'_>,875        build_compiler: Compiler,876        target: TargetSelection,877    ) -> Self {878        let crates = builder879            .in_tree_crates("rustc-main", Some(target))880            .into_iter()881            .map(|krate| krate.name.to_string())882            .collect();883        Self { build_compiler, target, crates }884    }885}886887impl Step for Rustc {888    type Output = ();889    const IS_HOST: bool = true;890891    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {892        run.crate_or_deps("rustc-main").path("compiler")893    }894895    fn is_default_step(builder: &Builder<'_>) -> bool {896        builder.config.compiler_docs897    }898899    fn make_run(run: RunConfig<'_>) {900        run.builder.ensure(Rustc::for_stage(run.builder, run.builder.top_stage, run.target));901    }902903    /// Generates compiler documentation.904    ///905    /// This will generate all documentation for compiler and dependencies.906    /// Compiler documentation is distributed separately, so we make sure907    /// we do not merge it with the other documentation from std, test and908    /// proc_macros. This is largely just a wrapper around `cargo doc`.909    fn run(self, builder: &Builder<'_>) {910        let target = self.target;911912        // This is the intended out directory for compiler documentation.913        let out = builder.compiler_doc_out(target);914        t!(fs::create_dir_all(&out));915916        // Build the standard library, so that proc-macros can use it.917        // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.)918        let build_compiler = self.build_compiler;919        builder.std(build_compiler, builder.config.host_target);920921        let _guard = builder.msg(922            Kind::Doc,923            format!("compiler{}", crate_description(&self.crates)),924            Mode::Rustc,925            build_compiler,926            target,927        );928929        // Build cargo command.930        let mut cargo = builder::Cargo::new(931            builder,932            build_compiler,933            Mode::Rustc,934            SourceType::InTree,935            target,936            Kind::Doc,937        );938939        cargo.rustdocflag("--document-private-items");940        // Since we always pass --document-private-items, there's no need to warn about linking to private items.941        cargo.rustdocflag("-Arustdoc::private-intra-doc-links");942        cargo.rustdocflag("--enable-index-page");943        cargo.rustdocflag("-Znormalize-docs");944        cargo.rustdocflag("--show-type-layout");945        // FIXME: `--generate-link-to-definition` tries to resolve cfged out code946        // see https://github.com/rust-lang/rust/pull/122066#issuecomment-1983049222947        // If there is any bug, please comment out the next line.948        cargo.rustdocflag("--generate-link-to-definition");949        cargo.rustdocflag("--generate-macro-expansion");950951        compile::rustc_cargo(builder, &mut cargo, target, &build_compiler, &self.crates);952        cargo.arg("-Zskip-rustdoc-fingerprint");953954        // Only include compiler crates, no dependencies of those, such as `libc`.955        // Do link to dependencies on `docs.rs` however using `rustdoc-map`.956        cargo.arg("--no-deps");957        cargo.arg("-Zrustdoc-map");958959        // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies,960        // once this is no longer an issue the special case for `ena` can be removed.961        cargo.rustdocflag("--extern-html-root-url");962        cargo.rustdocflag("ena=https://docs.rs/ena/latest/");963964        let mut to_open = None;965966        let out_dir = builder.stage_out(build_compiler, Mode::Rustc).join(target).join("doc");967        for krate in &*self.crates {968            // Create all crate output directories first to make sure rustdoc uses969            // relative links.970            // FIXME: Cargo should probably do this itself.971            let dir_name = krate.replace('-', "_");972            t!(fs::create_dir_all(out_dir.join(&*dir_name)));973            cargo.arg("-p").arg(krate);974            if to_open.is_none() {975                to_open = Some(dir_name);976            }977        }978979        // This uses a shared directory so that librustdoc documentation gets980        // correctly built and merged with the rustc documentation.981        //982        // This is needed because rustdoc is built in a different directory from983        // rustc. rustdoc needs to be able to see everything, for example when984        // merging the search index, or generating local (relative) links.985        symlink_dir_force(&builder.config, &out, &out_dir);986        // Cargo puts proc macros in `target/doc` even if you pass `--target`987        // explicitly (https://github.com/rust-lang/cargo/issues/7677).988        let proc_macro_out_dir = builder.stage_out(build_compiler, Mode::Rustc).join("doc");989        symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);990991        cargo.into_cmd().run(builder);992993        if !builder.config.dry_run() {994            // Sanity check on linked compiler crates995            for krate in &*self.crates {996                let dir_name = krate.replace('-', "_");997                // Making sure the directory exists and is not empty.998                assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());999            }1000        }10011002        if builder.paths.iter().any(|path| path.ends_with("compiler")) {1003            // For `x.py doc compiler --open`, open `rustc_middle` by default.1004            let index = out.join("rustc_middle").join("index.html");1005            builder.open_in_browser(index);1006        } else if let Some(krate) = to_open {1007            // Let's open the first crate documentation page:1008            let index = out.join(krate).join("index.html");1009            builder.open_in_browser(index);1010        }1011    }10121013    fn metadata(&self) -> Option<StepMetadata> {1014        Some(StepMetadata::doc("rustc", self.target).built_by(self.build_compiler))1015    }1016}10171018macro_rules! tool_doc {1019    (1020        $tool: ident,1021        $path: literal,1022        mode = $mode:expr1023        $(, is_library = $is_library:expr )?1024        $(, crates = $crates:expr )?1025        // Subset of nightly features that are allowed to be used when documenting1026        $(, allow_features: $allow_features:expr )?1027       ) => {1028        #[derive(Debug, Clone, Hash, PartialEq, Eq)]1029        pub struct $tool {1030            build_compiler: Compiler,1031            mode: Mode,1032            target: TargetSelection,1033        }10341035        impl Step for $tool {1036            type Output = ();1037            const IS_HOST: bool = true;10381039            fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1040                run.path($path)1041            }10421043            fn is_default_step(builder: &Builder<'_>) -> bool {1044                builder.config.compiler_docs1045            }10461047            fn make_run(run: RunConfig<'_>) {1048                let target = run.target;1049                let build_compiler = match $mode {1050                    Mode::ToolRustcPrivate => {1051                        // Rustdoc needs the rustc sysroot available to build.1052                        let compilers = RustcPrivateCompilers::new(run.builder, run.builder.top_stage, target);10531054                        // Build rustc docs so that we generate relative links.1055                        run.builder.ensure(Rustc::from_build_compiler(run.builder, compilers.build_compiler(), target));1056                        compilers.build_compiler()1057                    }1058                    Mode::ToolTarget => {1059                        // when shipping multiple docs together in one folder,1060                        // they all need to use the same rustdoc version1061                        prepare_doc_compiler(run.builder, run.builder.host_target, run.builder.top_stage)1062                    }1063                    _ => {1064                        panic!("Unexpected tool mode for documenting: {:?}", $mode);1065                    }1066                };10671068                run.builder.ensure($tool { build_compiler, mode: $mode, target });1069            }10701071            /// Generates documentation for a tool.1072            ///1073            /// This is largely just a wrapper around `cargo doc`.1074            fn run(self, builder: &Builder<'_>) {1075                let mut source_type = SourceType::InTree;10761077                if let Some(submodule_path) = submodule_path_of(&builder, $path) {1078                    source_type = SourceType::Submodule;1079                    builder.require_submodule(&submodule_path, None);1080                }10811082                let $tool { build_compiler, mode, target } = self;10831084                // This is the intended out directory for compiler documentation.1085                let out = builder.compiler_doc_out(target);1086                t!(fs::create_dir_all(&out));10871088                // Build cargo command.1089                let mut cargo = prepare_tool_cargo(1090                    builder,1091                    build_compiler,1092                    mode,1093                    target,1094                    Kind::Doc,1095                    $path,1096                    source_type,1097                    &[],1098                );1099                let allow_features = {1100                    let mut _value = "";1101                    $( _value = $allow_features; )?1102                    _value1103                };11041105                if !allow_features.is_empty() {1106                    cargo.allow_features(allow_features);1107                }11081109                cargo.arg("-Zskip-rustdoc-fingerprint");1110                // Only include compiler crates, no dependencies of those, such as `libc`.1111                cargo.arg("--no-deps");11121113                if false $(|| $is_library)? {1114                    cargo.arg("--lib");1115                }11161117                $(for krate in $crates {1118                    cargo.arg("-p").arg(krate);1119                })?11201121                cargo.rustdocflag("--document-private-items");1122                // Since we always pass --document-private-items, there's no need to warn about linking to private items.1123                cargo.rustdocflag("-Arustdoc::private-intra-doc-links");1124                cargo.rustdocflag("--enable-index-page");1125                cargo.rustdocflag("--show-type-layout");1126                cargo.rustdocflag("--generate-link-to-definition");11271128                let out_dir = builder.stage_out(build_compiler, mode).join(target).join("doc");1129                $(for krate in $crates {1130                    let dir_name = krate.replace("-", "_");1131                    t!(fs::create_dir_all(out_dir.join(&*dir_name)));1132                })?11331134                // Symlink compiler docs to the output directory of rustdoc documentation.1135                symlink_dir_force(&builder.config, &out, &out_dir);1136                let proc_macro_out_dir = builder.stage_out(build_compiler, mode).join("doc");1137                symlink_dir_force(&builder.config, &out, &proc_macro_out_dir);11381139                let _guard = builder.msg(Kind::Doc, stringify!($tool).to_lowercase(), None, build_compiler, target);1140                cargo.into_cmd().run(builder);11411142                if !builder.config.dry_run() {1143                    // Sanity check on linked doc directories1144                    $(for krate in $crates {1145                        let dir_name = krate.replace("-", "_");1146                        // Making sure the directory exists and is not empty.1147                        assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some());1148                    })?1149                }1150            }11511152            fn metadata(&self) -> Option<StepMetadata> {1153                Some(StepMetadata::doc(stringify!($tool), self.target).built_by(self.build_compiler))1154            }1155        }1156    }1157}11581159// NOTE: make sure to register these in `Builder::get_step_description`.1160tool_doc!(1161    BuildHelper,1162    "src/build_helper",1163    // ideally, this would use ToolBootstrap,1164    // but we distribute these docs together in the same folder1165    // as a bunch of stage1 tools, and you can't mix rustdoc versions1166    // because that breaks cross-crate data (particularly search)1167    mode = Mode::ToolTarget,1168    is_library = true,1169    crates = ["build_helper"]1170);1171tool_doc!(1172    Rustdoc,1173    "src/tools/rustdoc",1174    mode = Mode::ToolRustcPrivate,1175    crates = ["rustdoc", "rustdoc-json-types"]1176);1177tool_doc!(1178    Rustfmt,1179    "src/tools/rustfmt",1180    mode = Mode::ToolRustcPrivate,1181    crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"]1182);1183tool_doc!(1184    Clippy,1185    "src/tools/clippy",1186    mode = Mode::ToolRustcPrivate,1187    crates = ["clippy_config", "clippy_utils"]1188);1189tool_doc!(Miri, "src/tools/miri", mode = Mode::ToolRustcPrivate, crates = ["miri"]);1190tool_doc!(1191    Cargo,1192    "src/tools/cargo",1193    mode = Mode::ToolTarget,1194    crates = [1195        "cargo",1196        "cargo-credential",1197        "cargo-platform",1198        "cargo-test-macro",1199        "cargo-test-support",1200        "cargo-util",1201        "cargo-util-schemas",1202        "crates-io",1203        "mdman",1204        "rustfix",1205    ],1206    // Required because of the im-rc dependency of Cargo, which automatically opts into the1207    // "specialization" feature in its build script when it detects a nightly toolchain.1208    allow_features: "specialization"1209);1210tool_doc!(Tidy, "src/tools/tidy", mode = Mode::ToolTarget, crates = ["tidy"]);1211tool_doc!(1212    Bootstrap,1213    "src/bootstrap",1214    mode = Mode::ToolTarget,1215    is_library = true,1216    crates = ["bootstrap"]1217);1218tool_doc!(1219    RunMakeSupport,1220    "src/tools/run-make-support",1221    mode = Mode::ToolTarget,1222    is_library = true,1223    crates = ["run_make_support"]1224);1225tool_doc!(1226    Compiletest,1227    "src/tools/compiletest",1228    mode = Mode::ToolTarget,1229    is_library = true,1230    crates = ["compiletest"]1231);12321233#[derive(Debug, Clone, Hash, PartialEq, Eq)]1234pub struct ErrorIndex {1235    compilers: RustcPrivateCompilers,1236}12371238impl Step for ErrorIndex {1239    type Output = ();1240    const IS_HOST: bool = true;12411242    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1243        run.path("src/tools/error_index_generator")1244    }12451246    fn is_default_step(builder: &Builder<'_>) -> bool {1247        builder.config.docs1248    }12491250    fn make_run(run: RunConfig<'_>) {1251        run.builder.ensure(ErrorIndex {1252            compilers: RustcPrivateCompilers::new(run.builder, run.builder.top_stage, run.target),1253        });1254    }12551256    /// Generates the HTML rendered error-index by running the1257    /// `error_index_generator` tool.1258    fn run(self, builder: &Builder<'_>) {1259        builder.info(&format!("Documenting error index ({})", self.compilers.target()));1260        let out = builder.doc_out(self.compilers.target());1261        t!(fs::create_dir_all(&out));1262        tool::ErrorIndex::command(builder, self.compilers)1263            .arg("html")1264            .arg(&out)1265            .arg(&builder.version)1266            .run(builder);12671268        let index = out.join("error-index.html");1269        builder.maybe_open_in_browser::<Self>(index);1270    }12711272    fn metadata(&self) -> Option<StepMetadata> {1273        Some(1274            StepMetadata::doc("error-index", self.compilers.target())1275                .built_by(self.compilers.build_compiler()),1276        )1277    }1278}12791280#[derive(Debug, Clone, Hash, PartialEq, Eq)]1281pub struct UnstableBookGen {1282    build_compiler: Compiler,1283    target: TargetSelection,1284}12851286impl Step for UnstableBookGen {1287    type Output = ();1288    const IS_HOST: bool = true;12891290    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1291        run.path("src/tools/unstable-book-gen")1292    }12931294    fn is_default_step(builder: &Builder<'_>) -> bool {1295        builder.config.docs1296    }12971298    fn make_run(run: RunConfig<'_>) {1299        run.builder.ensure(UnstableBookGen {1300            build_compiler: prepare_doc_compiler(run.builder, run.target, run.builder.top_stage),1301            target: run.target,1302        });1303    }13041305    fn run(self, builder: &Builder<'_>) {1306        let target = self.target;1307        let rustc_path = builder.rustc(self.build_compiler);13081309        builder.info(&format!("Generating unstable book md files ({target})"));1310        let out = builder.md_doc_out(target).join("unstable-book");1311        builder.create_dir(&out);1312        builder.remove_dir(&out);1313        let mut cmd = builder.tool_cmd(Tool::UnstableBookGen);1314        cmd.arg(builder.src.join("library"));1315        cmd.arg(builder.src.join("compiler"));1316        cmd.arg(builder.src.join("src"));1317        cmd.arg(rustc_path);1318        cmd.arg(out);13191320        cmd.run(builder);1321    }1322}13231324fn symlink_dir_force(config: &Config, original: &Path, link: &Path) {1325    if config.dry_run() {1326        return;1327    }1328    if let Ok(m) = fs::symlink_metadata(link) {1329        if m.file_type().is_dir() {1330            t!(fs::remove_dir_all(link));1331        } else {1332            // handle directory junctions on windows by falling back to1333            // `remove_dir`.1334            t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link)));1335        }1336    }13371338    t!(1339        symlink_dir(config, original, link),1340        format!("failed to create link from {} -> {}", link.display(), original.display())1341    );1342}13431344/// Builds the Rust compiler book.1345#[derive(Debug, Clone, Hash, PartialEq, Eq)]1346pub struct RustcBook {1347    build_compiler: Compiler,1348    target: TargetSelection,1349    /// Test that the examples of lints in the book produce the correct lints in the expected1350    /// format.1351    validate: bool,1352}13531354impl RustcBook {1355    pub fn validate(build_compiler: Compiler, target: TargetSelection) -> Self {1356        Self { build_compiler, target, validate: true }1357    }1358}13591360impl Step for RustcBook {1361    type Output = ();1362    const IS_HOST: bool = true;13631364    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1365        run.path("src/doc/rustc")1366    }13671368    fn is_default_step(builder: &Builder<'_>) -> bool {1369        builder.config.docs1370    }13711372    fn make_run(run: RunConfig<'_>) {1373        // Bump the stage to 2, because the rustc book requires an in-tree compiler.1374        // At the same time, since this step is enabled by default, we don't want `x doc` to fail1375        // in stage 1.1376        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {1377            run.builder.top_stage1378        } else {1379            21380        };13811382        run.builder.ensure(RustcBook {1383            build_compiler: prepare_doc_compiler(run.builder, run.target, stage),1384            target: run.target,1385            validate: false,1386        });1387    }13881389    /// Builds the rustc book.1390    ///1391    /// The lints are auto-generated by a tool, and then merged into the book1392    /// in the "md-doc" directory in the build output directory. Then1393    /// "rustbook" is used to convert it to HTML.1394    fn run(self, builder: &Builder<'_>) {1395        let out_base = builder.md_doc_out(self.target).join("rustc");1396        t!(fs::create_dir_all(&out_base));1397        let out_listing = out_base.join("src/lints");1398        builder.cp_link_r(&builder.src.join("src/doc/rustc"), &out_base);1399        builder.info(&format!("Generating lint docs ({})", self.target));14001401        let rustc = builder.rustc(self.build_compiler);1402        // The tool runs `rustc` for extracting output examples, so it needs a1403        // functional sysroot.1404        builder.std(self.build_compiler, self.target);1405        let mut cmd = builder.tool_cmd(Tool::LintDocs);1406        cmd.arg("--build-rustc-stage");1407        cmd.arg(self.build_compiler.stage.to_string());1408        cmd.arg("--src");1409        cmd.arg(builder.src.join("compiler"));1410        cmd.arg("--out");1411        cmd.arg(&out_listing);1412        cmd.arg("--rustc");1413        cmd.arg(&rustc);1414        cmd.arg("--rustc-target").arg(self.target.rustc_target_arg());1415        if let Some(target_linker) = builder.linker(self.target) {1416            cmd.arg("--rustc-linker").arg(target_linker);1417        }1418        if builder.is_verbose() {1419            cmd.arg("--verbose");1420        }1421        if self.validate {1422            cmd.arg("--validate");1423        }1424        // We need to validate nightly features, even on the stable channel.1425        // Set this unconditionally as the stage0 compiler may be being used to1426        // document.1427        cmd.env("RUSTC_BOOTSTRAP", "1");14281429        // If the lib directories are in an unusual location (changed in1430        // bootstrap.toml), then this needs to explicitly update the dylib search1431        // path.1432        builder.add_rustc_lib_path(self.build_compiler, &mut cmd);1433        let doc_generator_guard =1434            builder.msg(Kind::Run, "lint-docs", None, self.build_compiler, self.target);1435        cmd.run(builder);1436        drop(doc_generator_guard);14371438        // Run rustbook/mdbook to generate the HTML pages.1439        builder.ensure(RustbookSrc {1440            target: self.target,1441            name: "rustc".to_owned(),1442            src: out_base,1443            parent: Some(self),1444            languages: vec![],1445            build_compiler: None,1446        });1447    }1448}14491450/// Documents the reference.1451/// It has to always be done using a stage 1+ compiler, because it references in-tree1452/// compiler/stdlib concepts.1453#[derive(Debug, Clone, Hash, PartialEq, Eq)]1454pub struct Reference {1455    build_compiler: Compiler,1456    target: TargetSelection,1457}14581459impl Step for Reference {1460    type Output = ();14611462    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {1463        run.path("src/doc/reference")1464    }14651466    fn is_default_step(builder: &Builder<'_>) -> bool {1467        builder.config.docs1468    }14691470    fn make_run(run: RunConfig<'_>) {1471        // Bump the stage to 2, because the reference requires an in-tree compiler.1472        // At the same time, since this step is enabled by default, we don't want `x doc` to fail1473        // in stage 1.1474        // FIXME: create a shared method on builder for auto-bumping, and print some warning when1475        // it happens.1476        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {1477            run.builder.top_stage1478        } else {1479            21480        };14811482        run.builder.ensure(Reference {1483            build_compiler: prepare_doc_compiler(run.builder, run.target, stage),1484            target: run.target,1485        });1486    }14871488    /// Builds the reference book.1489    fn run(self, builder: &Builder<'_>) {1490        builder.require_submodule("src/doc/reference", None);14911492        // This is needed for generating links to the standard library using1493        // the mdbook-spec plugin.1494        builder.std(self.build_compiler, builder.config.host_target);14951496        // Run rustbook/mdbook to generate the HTML pages.1497        builder.ensure(RustbookSrc {1498            target: self.target,1499            name: "reference".to_owned(),1500            src: builder.src.join("src/doc/reference"),1501            build_compiler: Some(self.build_compiler),1502            parent: Some(self),1503            languages: vec![],1504        });1505    }1506}