1//! Optional checks for file types other than Rust source2//!3//! Handles python tool version management via a virtual environment in4//! `build/venv`.5//!6//! # Functional outline7//!8//! 1. Run tidy with an extra option: `--extra-checks=py,shell`,9//!    `--extra-checks=py:lint`, or similar. Optionally provide specific10//!    configuration after a double dash (`--extra-checks=py -- foo.py`)11//! 2. Build configuration based on args/environment:12//!    - Formatters by default are in check only mode13//!    - If `--bless` is provided, formatters may run14//!    - Pass any additional config after the `--`. If no files are specified,15//!      use a default.16//! 3. Print the output of the given command. If it fails, rerun the tool to print a suggestion17//!    diff.1819use std::ffi::{OsStr, OsString};20use std::path::{Path, PathBuf};21use std::process::Command;22use std::str::FromStr;23use std::{env, fmt, fs, io};2425use crate::diagnostics::TidyCtx;2627mod rustdoc_js;2829#[cfg(test)]30mod tests;3132const MIN_PY_REV: (u32, u32) = (3, 11);33const MIN_PY_REV_STR: &str = "≥3.11";3435/// Path to find the python executable within a virtual environment36#[cfg(target_os = "windows")]37const REL_PY_PATH: &[&str] = &["Scripts", "python.exe"];38#[cfg(not(target_os = "windows"))]39const REL_PY_PATH: &[&str] = &["bin", "python3"];4041const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"];42/// Location within build directory43const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"];44const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];4546const SPELLCHECK_DIRS: &[&str] = &["compiler", "library", "src/bootstrap", "src/librustdoc"];47const SPELLCHECK_VER: &str = "1.38.1";4849pub fn check(50    root_path: &Path,51    outdir: &Path,52    librustdoc_path: &Path,53    tools_path: &Path,54    npm: &Path,55    cargo: &Path,56    extra_checks: Option<Vec<String>>,57    pos_args: Vec<String>,58    tidy_ctx: TidyCtx,59) {60    // Split comma-separated args up61    let mut lint_args = match extra_checks {62        Some(s) => s63            .iter()64            .map(|s| {65                if s == "spellcheck:fix" {66                    eprintln!("warning: `spellcheck:fix` is no longer valid, use `--extra-checks=spellcheck --bless`");67                }68                (ExtraCheckArg::from_str(s), s)69            })70            .filter_map(|(res, src)| match res {71                Ok(arg) => {72                    Some(arg)73                }74                Err(err) => {75                    // only warn because before bad extra checks would be silently ignored.76                    eprintln!("warning: bad extra check argument {src:?}: {err:?}");77                    None78                }79            })80            .collect(),81        None => vec![],82    };83    lint_args.retain(|ck| ck.is_non_if_installed_or_matches(root_path, outdir));84    if lint_args.iter().any(|ck| ck.auto) {85        crate::files_modified_batch_filter(86            &tidy_ctx.base_commit,87            tidy_ctx.is_running_on_ci(),88            &mut lint_args,89            |ck, path| ck.is_non_auto_or_matches(path),90        );91    }9293    macro_rules! extra_check {94        ($lang:ident, $kind:ident) => {95            lint_args.iter().any(|arg| arg.matches(ExtraCheckLang::$lang, ExtraCheckKind::$kind))96        };97    }9899    let python_lint = extra_check!(Py, Lint);100    let python_fmt = extra_check!(Py, Fmt);101    let shell_lint = extra_check!(Shell, Lint);102    let cpp_fmt = extra_check!(Cpp, Fmt);103    let spellcheck = extra_check!(Spellcheck, None);104    let js_lint = extra_check!(Js, Lint);105    let js_typecheck = extra_check!(Js, Typecheck);106107    let mut py_path = None;108109    let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args110        .iter()111        .map(OsStr::new)112        .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with('-')));113114    if python_lint || python_fmt || cpp_fmt {115        // Since python lint, format and cpp format share python env, we need to ensure python env is installed before running those checks.116        let p = py_prepare(root_path, outdir, &tidy_ctx);117        if p.is_none() {118            return;119        }120        py_path = p;121    }122123    if python_lint {124        check_python_lint(125            root_path,126            outdir,127            &cfg_args,128            &file_args,129            py_path.as_ref().unwrap(),130            &tidy_ctx,131        );132    }133134    if python_fmt {135        check_python_fmt(136            root_path,137            outdir,138            &cfg_args,139            &file_args,140            py_path.as_ref().unwrap(),141            &tidy_ctx,142        );143    }144145    if cpp_fmt {146        check_cpp_fmt(root_path, &cfg_args, &file_args, py_path.as_ref().unwrap(), &tidy_ctx);147    }148149    if shell_lint {150        check_shell_lint(root_path, &cfg_args, &file_args, &tidy_ctx);151    }152153    if spellcheck {154        check_spellcheck(root_path, outdir, cargo, &tidy_ctx);155    }156157    if js_lint || js_typecheck {158        // Since js lint and format share node env, we need to ensure node env is installed before running those checks.159        if js_prepare(root_path, outdir, npm, &tidy_ctx).is_none() {160            return;161        }162    }163164    if js_lint {165        check_js_lint(outdir, librustdoc_path, tools_path, &tidy_ctx);166    }167168    if js_typecheck {169        check_js_typecheck(outdir, librustdoc_path, &tidy_ctx);170    }171}172173fn py_prepare(root_path: &Path, outdir: &Path, tidy_ctx: &TidyCtx) -> Option<PathBuf> {174    let mut check = tidy_ctx.start_check("extra_checks:py_prepare");175176    let venv_path = outdir.join("venv");177    let mut reqs_path = root_path.to_owned();178    reqs_path.extend(PIP_REQ_PATH);179180    match get_or_create_venv(&venv_path, &reqs_path) {181        Ok(p) => Some(p),182        Err(e) => {183            check.error(e);184            None185        }186    }187}188189fn js_prepare(root_path: &Path, outdir: &Path, npm: &Path, tidy_ctx: &TidyCtx) -> Option<()> {190    let mut check = tidy_ctx.start_check("extra_checks:js_prepare");191192    if let Err(e) = rustdoc_js::npm_install(root_path, outdir, npm) {193        check.error(e.to_string());194        return None;195    }196197    Some(())198}199200fn show_bless_help(mode: &str, action: &str, bless: bool) {201    if !bless {202        eprintln!(203            "rerun with `--bless` to {action}: `./x.py test tidy --extra-checks={mode} --bless`"204        );205    }206}207208fn check_spellcheck(root_path: &Path, outdir: &Path, cargo: &Path, tidy_ctx: &TidyCtx) {209    let mut check = tidy_ctx.start_check("extra_checks:spellcheck");210211    let bless = tidy_ctx.is_bless_enabled();212213    let config_path = root_path.join("typos.toml");214    let mut args = vec!["-c", config_path.as_os_str().to_str().unwrap()];215    args.extend_from_slice(SPELLCHECK_DIRS);216217    if bless {218        eprintln!("spellchecking files and fixing typos");219        args.push("--write-changes");220    } else {221        eprintln!("spellchecking files");222    }223224    if let Err(e) =225        spellcheck_runner(root_path, &outdir, &cargo, &args, tidy_ctx.is_running_on_ci())226    {227        show_bless_help("spellcheck", "fix typos", bless);228        check.error(e);229    }230}231232fn check_js_lint(outdir: &Path, librustdoc_path: &Path, tools_path: &Path, tidy_ctx: &TidyCtx) {233    let mut check = tidy_ctx.start_check("extra_checks:js_lint");234235    let bless = tidy_ctx.is_bless_enabled();236237    if bless {238        eprintln!("linting javascript files and applying suggestions");239    } else {240        eprintln!("linting javascript files");241    }242243    if let Err(e) = rustdoc_js::lint(outdir, librustdoc_path, tools_path, bless) {244        show_bless_help("js:lint", "apply esplint suggestion", bless);245        check.error(e);246        return;247    }248249    if let Err(e) = rustdoc_js::es_check(outdir, librustdoc_path) {250        check.error(e);251    }252}253254fn check_js_typecheck(outdir: &Path, librustdoc_path: &Path, tidy_ctx: &TidyCtx) {255    let mut check = tidy_ctx.start_check("extra_checks:js_typecheck");256257    eprintln!("typechecking javascript files");258    if let Err(e) = rustdoc_js::typecheck(outdir, librustdoc_path) {259        check.error(e);260    }261}262263fn check_shell_lint(264    root_path: &Path,265    cfg_args: &Vec<&OsStr>,266    file_args: &Vec<&OsStr>,267    tidy_ctx: &TidyCtx,268) {269    let mut check = tidy_ctx.start_check("extra_checks:shell_lint");270271    eprintln!("linting shell files");272273    let mut file_args_shc = file_args.clone();274    let files;275    if file_args.is_empty() {276        match find_with_extension(root_path, None, &[OsStr::new("sh")]) {277            Ok(f) => files = f,278            Err(e) => {279                check.error(e);280                return;281            }282        }283284        file_args_shc.extend(files.iter().map(|p| p.as_os_str()));285    }286287    if let Err(e) = shellcheck_runner(&merge_args(&cfg_args, &file_args_shc)) {288        check.error(e);289    }290}291292fn check_python_lint(293    root_path: &Path,294    outdir: &Path,295    cfg_args: &Vec<&OsStr>,296    file_args: &Vec<&OsStr>,297    py_path: &Path,298    tidy_ctx: &TidyCtx,299) {300    let mut check = tidy_ctx.start_check("extra_checks:python_lint");301302    let bless = tidy_ctx.is_bless_enabled();303304    let args: &[&OsStr] = if bless {305        eprintln!("linting python files and applying suggestions");306        &["check".as_ref(), "--fix".as_ref()]307    } else {308        eprintln!("linting python files");309        &["check".as_ref()]310    };311312    let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, args);313314    if res.is_err() && !bless {315        eprintln!("\npython linting failed! Printing diff suggestions:");316317        let diff_res = run_ruff(318            root_path,319            outdir,320            py_path,321            &cfg_args,322            &file_args,323            &["check".as_ref(), "--diff".as_ref()],324        );325        // `ruff check --diff` will return status 0 if there are no suggestions.326        if diff_res.is_err() {327            show_bless_help("py:lint", "apply ruff suggestions", bless);328        }329    }330    if let Err(e) = res {331        check.error(e);332    }333}334335fn check_python_fmt(336    root_path: &Path,337    outdir: &Path,338    cfg_args: &Vec<&OsStr>,339    file_args: &Vec<&OsStr>,340    py_path: &Path,341    tidy_ctx: &TidyCtx,342) {343    let mut check = tidy_ctx.start_check("extra_checks:python_fmt");344345    let bless = tidy_ctx.is_bless_enabled();346347    let mut args: Vec<&OsStr> = vec!["format".as_ref()];348    if bless {349        eprintln!("formatting python files");350    } else {351        eprintln!("checking python file formatting");352        args.push("--check".as_ref());353    }354355    let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &args);356357    if res.is_err() && !bless {358        eprintln!("\npython formatting does not match! Printing diff:");359360        let _ = run_ruff(361            root_path,362            outdir,363            py_path,364            &cfg_args,365            &file_args,366            &["format".as_ref(), "--diff".as_ref()],367        );368        show_bless_help("py:fmt", "reformat Python code", bless);369    }370371    if let Err(e) = res {372        check.error(e);373    }374}375376fn check_cpp_fmt(377    root_path: &Path,378    cfg_args: &Vec<&OsStr>,379    file_args: &Vec<&OsStr>,380    py_path: &Path,381    tidy_ctx: &TidyCtx,382) {383    let mut check = tidy_ctx.start_check("extra_checks:cpp_fmt");384385    let bless = tidy_ctx.is_bless_enabled();386387    let mut cfg_args_clang_format = cfg_args.clone();388    let mut file_args_clang_format = file_args.clone();389    let config_path = root_path.join(".clang-format");390    let mut config_file_arg = OsString::from("file:");391    config_file_arg.push(&config_path);392    cfg_args_clang_format.extend(&["--style".as_ref(), config_file_arg.as_ref()]);393    if bless {394        eprintln!("formatting C++ files");395        cfg_args_clang_format.push("-i".as_ref());396    } else {397        eprintln!("checking C++ file formatting");398        cfg_args_clang_format.extend(&["--dry-run".as_ref(), "--Werror".as_ref()]);399    }400    let files;401    if file_args_clang_format.is_empty() {402        let llvm_wrapper = root_path.join("compiler/rustc_llvm/llvm-wrapper");403        match find_with_extension(404            root_path,405            Some(llvm_wrapper.as_path()),406            &[OsStr::new("h"), OsStr::new("cpp")],407        ) {408            Ok(f) => files = f,409            Err(e) => {410                check.error(e);411                return;412            }413        }414        file_args_clang_format.extend(files.iter().map(|p| p.as_os_str()));415    }416    let args = merge_args(&cfg_args_clang_format, &file_args_clang_format);417    let res = py_runner(py_path, false, None, "clang-format", &args);418419    if res.is_err() && !bless {420        eprintln!("\nclang-format linting failed! Printing diff suggestions:");421422        let mut cfg_args_diff = cfg_args.clone();423        cfg_args_diff.extend(&["--style".as_ref(), config_file_arg.as_ref()]);424        for file in file_args {425            let mut formatted = String::new();426            let mut diff_args = cfg_args_diff.clone();427            diff_args.push(file);428            let _ = py_runner(py_path, false, Some(&mut formatted), "clang-format", &diff_args);429            if formatted.is_empty() {430                eprintln!(431                    "failed to obtain the formatted content for '{}'",432                    file.to_string_lossy()433                );434                continue;435            }436            let actual = std::fs::read_to_string(file).unwrap_or_else(|e| {437                panic!("failed to read the C++ file at '{}' due to '{e}'", file.to_string_lossy())438            });439            if formatted != actual {440                let diff = similar::TextDiff::from_lines(&actual, &formatted);441                eprintln!(442                    "{}",443                    diff.unified_diff().context_radius(4).header(444                        &format!("{} (actual)", file.to_string_lossy()),445                        &format!("{} (formatted)", file.to_string_lossy())446                    )447                );448            }449        }450        show_bless_help("cpp:fmt", "reformat C++ code", bless);451    }452453    if let Err(e) = res {454        check.error(e);455    }456}457458fn run_ruff(459    root_path: &Path,460    outdir: &Path,461    py_path: &Path,462    cfg_args: &[&OsStr],463    file_args: &[&OsStr],464    ruff_args: &[&OsStr],465) -> Result<(), Error> {466    let mut cfg_args_ruff = cfg_args.to_vec();467    let mut file_args_ruff = file_args.to_vec();468469    let mut cfg_path = root_path.to_owned();470    cfg_path.extend(RUFF_CONFIG_PATH);471    let mut cache_dir = outdir.to_owned();472    cache_dir.extend(RUFF_CACHE_PATH);473474    cfg_args_ruff.extend([475        "--config".as_ref(),476        cfg_path.as_os_str(),477        "--cache-dir".as_ref(),478        cache_dir.as_os_str(),479    ]);480481    if file_args_ruff.is_empty() {482        file_args_ruff.push(root_path.as_os_str());483    }484485    let mut args: Vec<&OsStr> = ruff_args.to_vec();486    args.extend(merge_args(&cfg_args_ruff, &file_args_ruff));487    py_runner(py_path, true, None, "ruff", &args)488}489490/// Helper to create `cfg1 cfg2 -- file1 file2` output491fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> {492    let mut args = cfg_args.to_owned();493    args.push("--".as_ref());494    args.extend(file_args);495    args496}497498/// Run a python command with given arguments. `py_path` should be a virtualenv.499///500/// Captures `stdout` to a string if provided, otherwise prints the output.501fn py_runner(502    py_path: &Path,503    as_module: bool,504    stdout: Option<&mut String>,505    bin: &'static str,506    args: &[&OsStr],507) -> Result<(), Error> {508    let mut cmd = Command::new(py_path);509    if as_module {510        cmd.arg("-m").arg(bin).args(args);511    } else {512        let bin_path = py_path.with_file_name(bin);513        cmd.arg(bin_path).args(args);514    }515    let status = if let Some(stdout) = stdout {516        let output = cmd.output()?;517        if let Ok(s) = std::str::from_utf8(&output.stdout) {518            stdout.push_str(s);519        }520        output.status521    } else {522        cmd.status()?523    };524    if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) }525}526527/// Create a virtuaenv at a given path if it doesn't already exist, or validate528/// the install if it does. Returns the path to that venv's python executable.529fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result<PathBuf, Error> {530    let mut py_path = venv_path.to_owned();531    py_path.extend(REL_PY_PATH);532533    if !has_py_tools(venv_path, src_reqs_path)? {534        let dst_reqs_path = venv_path.join("requirements.txt");535        eprintln!("removing old virtual environment");536        if venv_path.is_dir() {537            fs::remove_dir_all(venv_path).unwrap_or_else(|_| {538                panic!("failed to remove directory at {}", venv_path.display())539            });540        }541        create_venv_at_path(venv_path)?;542        install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?;543    }544545    verify_py_version(&py_path)?;546    Ok(py_path)547}548549fn has_py_tools(venv_path: &Path, src_reqs_path: &Path) -> Result<bool, Error> {550    let dst_reqs_path = venv_path.join("requirements.txt");551    if let Ok(req) = fs::read_to_string(&dst_reqs_path) {552        if req == fs::read_to_string(src_reqs_path)? {553            return Ok(true);554        }555        eprintln!("requirements.txt file mismatch");556    }557558    Ok(false)559}560561/// Attempt to create a virtualenv at this path. Cycles through all expected562/// valid python versions to find one that is installed.563fn create_venv_at_path(path: &Path) -> Result<(), Error> {564    /// Preferred python versions in order. Newest to oldest then current565    /// development versions566    const TRY_PY: &[&str] = &[567        "python3.14",568        "python3.13",569        "python3.12",570        "python3.11",571        "python3",572        "python",573        "python3.15",574    ];575576    let mut sys_py = None;577    let mut found = Vec::new();578579    for py in TRY_PY {580        match verify_py_version(Path::new(py)) {581            Ok(_) => {582                sys_py = Some(*py);583                break;584            }585            // Skip not found errors586            Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (),587            // Skip insufficient version errors588            Err(Error::Version { installed, .. }) => found.push(installed),589            // just log and skip unrecognized errors590            Err(e) => eprintln!("note: error running '{py}': {e}"),591        }592    }593594    let Some(sys_py) = sys_py else {595        let ret = if found.is_empty() {596            Error::MissingReq("python3", "python file checks", None)597        } else {598            found.sort();599            found.dedup();600            Error::Version {601                program: "python3",602                required: MIN_PY_REV_STR,603                installed: found.join(", "),604            }605        };606        return Err(ret);607    };608609    // First try venv, which should be packaged in the Python3 standard library.610    // If it is not available, try to create the virtual environment using the611    // virtualenv package.612    if try_create_venv(sys_py, path, "venv").is_ok() {613        return Ok(());614    }615    try_create_venv(sys_py, path, "virtualenv")616}617618fn try_create_venv(python: &str, path: &Path, module: &str) -> Result<(), Error> {619    eprintln!(620        "creating virtual environment at '{}' using '{python}' and '{module}'",621        path.display()622    );623    let out = Command::new(python).args(["-m", module]).arg(path).output().unwrap();624625    if out.status.success() {626        return Ok(());627    }628629    let stderr = String::from_utf8_lossy(&out.stderr);630    let err = if stderr.contains(&format!("No module named {module}")) {631        Error::Generic(format!(632            r#"{module} not found: you may need to install it:633`{python} -m pip install {module}`634If you see an error about "externally managed environment" when running the above command,635either install `{module}` using your system package manager636(e.g. `sudo apt-get install {python}-{module}`) or create a virtual environment manually, install637`{module}` in it and then activate it before running tidy.638"#639        ))640    } else {641        Error::Generic(format!(642            "failed to create venv at '{}' using {python} -m {module}: {stderr}",643            path.display()644        ))645    };646    Err(err)647}648649/// Parse python's version output (`Python x.y.z`) and ensure we have a650/// suitable version.651fn verify_py_version(py_path: &Path) -> Result<(), Error> {652    let out = Command::new(py_path).arg("--version").output()?;653    let outstr = String::from_utf8_lossy(&out.stdout);654    let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim();655    let mut vers_comps = vers.split('.');656    let major: u32 = vers_comps.next().unwrap().parse().unwrap();657    let minor: u32 = vers_comps.next().unwrap().parse().unwrap();658659    if (major, minor) < MIN_PY_REV {660        Err(Error::Version {661            program: "python",662            required: MIN_PY_REV_STR,663            installed: vers.to_owned(),664        })665    } else {666        Ok(())667    }668}669670fn install_requirements(671    py_path: &Path,672    src_reqs_path: &Path,673    dst_reqs_path: &Path,674) -> Result<(), Error> {675    let stat = Command::new(py_path)676        .args(["-m", "pip", "install", "--upgrade", "pip"])677        .status()678        .expect("failed to launch pip");679    if !stat.success() {680        return Err(Error::Generic(format!("pip install failed with status {stat}")));681    }682683    let stat = Command::new(py_path)684        .args(["-m", "pip", "install", "--quiet", "--require-hashes", "-r"])685        .arg(src_reqs_path)686        .status()?;687    if !stat.success() {688        return Err(Error::Generic(format!(689            "failed to install requirements at {}",690            src_reqs_path.display()691        )));692    }693    fs::copy(src_reqs_path, dst_reqs_path)?;694    assert_eq!(695        fs::read_to_string(src_reqs_path).unwrap(),696        fs::read_to_string(dst_reqs_path).unwrap()697    );698    Ok(())699}700701/// Returns `Ok` if shellcheck is installed, `Err` otherwise.702fn has_shellcheck() -> Result<(), Error> {703    match Command::new("shellcheck").arg("--version").status() {704        Ok(_) => Ok(()),705        Err(e) if e.kind() == io::ErrorKind::NotFound => Err(Error::MissingReq(706            "shellcheck",707            "shell file checks",708            Some(709                "see <https://github.com/koalaman/shellcheck#installing> \710                for installation instructions"711                    .to_owned(),712            ),713        )),714        Err(e) => Err(e.into()),715    }716}717718/// Check that shellcheck is installed then run it at the given path719fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {720    has_shellcheck()?;721722    let status = Command::new("shellcheck").args(args).status()?;723    if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) }724}725726/// Ensure that spellchecker is installed then run it at the given path727fn spellcheck_runner(728    src_root: &Path,729    outdir: &Path,730    cargo: &Path,731    args: &[&str],732    is_ci: bool,733) -> Result<(), Error> {734    let bin_path = ensure_version_or_cargo_install(735        outdir,736        cargo,737        "typos-cli",738        "typos",739        SPELLCHECK_VER,740        is_ci,741    )?;742    match Command::new(bin_path).current_dir(src_root).args(args).status() {743        Ok(status) => {744            if status.success() {745                Ok(())746            } else {747                Err(Error::FailedCheck("typos"))748            }749        }750        Err(err) => Err(Error::Generic(format!("failed to run typos tool: {err:?}"))),751    }752}753754/// Check git for tracked files matching an extension755fn find_with_extension(756    root_path: &Path,757    find_dir: Option<&Path>,758    extensions: &[&OsStr],759) -> Result<Vec<PathBuf>, Error> {760    // Untracked files show up for short status and are indicated with a leading `?`761    // -C changes git to be as if run from that directory762    let stat_output =763        Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout;764765    if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 {766        eprintln!("found untracked files, ignoring");767    }768769    let mut output = Vec::new();770    let binding = {771        let mut command = Command::new("git");772        command.arg("-C").arg(root_path).args(["ls-files"]);773        if let Some(find_dir) = find_dir {774            command.arg(find_dir);775        }776        command.output()?777    };778    let tracked = String::from_utf8_lossy(&binding.stdout);779780    for line in tracked.lines() {781        let line = line.trim();782        let path = Path::new(line);783784        let Some(ref extension) = path.extension() else {785            continue;786        };787        if extensions.contains(extension) {788            output.push(root_path.join(path));789        }790    }791792    Ok(output)793}794795/// Check if the given executable is installed and the version is expected.796fn ensure_version(build_dir: &Path, bin_name: &str, version: &str) -> Result<PathBuf, Error> {797    let bin_path = build_dir.join("misc-tools").join("bin").join(bin_name);798799    match Command::new(&bin_path).arg("--version").output() {800        Ok(output) => {801            let Some(v) = str::from_utf8(&output.stdout).unwrap().trim().split_whitespace().last()802            else {803                return Err(Error::Generic("version check failed".to_string()));804            };805806            if v != version {807                return Err(Error::Version { program: "", required: "", installed: v.to_string() });808            }809            Ok(bin_path)810        }811        Err(e) => Err(Error::Io(e)),812    }813}814815/// If the given executable is installed with the given version, use that,816/// otherwise install via cargo.817fn ensure_version_or_cargo_install(818    build_dir: &Path,819    cargo: &Path,820    pkg_name: &str,821    bin_name: &str,822    version: &str,823    is_ci: bool,824) -> Result<PathBuf, Error> {825    if let Ok(bin_path) = ensure_version(build_dir, bin_name, version) {826        return Ok(bin_path);827    }828829    eprintln!("building external tool {bin_name} from package {pkg_name}@{version}");830831    let tool_root_dir = build_dir.join("misc-tools");832    let tool_bin_dir = tool_root_dir.join("bin");833    let bin_path = tool_bin_dir.join(bin_name).with_extension(env::consts::EXE_EXTENSION);834835    // use --force to ensure that if the required version is bumped, we update it.836    // use --target-dir to ensure we have a build cache so repeated invocations aren't slow.837    // modify PATH so that cargo doesn't print a warning telling the user to modify the path.838    let mut cmd = Command::new(cargo);839    cmd.args(["install", "--locked", "--force", "--quiet"])840        .arg("--root")841        .arg(&tool_root_dir)842        .arg("--target-dir")843        .arg(tool_root_dir.join("target"))844        .arg(format!("{pkg_name}@{version}"))845        .env(846            "PATH",847            env::join_paths(848                env::split_paths(&env::var("PATH").unwrap())849                    .chain(std::iter::once(tool_bin_dir.clone())),850            )851            .expect("build dir contains invalid char"),852        );853854    // On CI, we set opt-level flag for quicker installation.855    // Since lower opt-level decreases the tool's performance,856    // we don't set this option on local.857    if is_ci {858        cmd.env("RUSTFLAGS", "-Copt-level=0");859    }860861    let cargo_exit_code = cmd.spawn()?.wait()?;862    if !cargo_exit_code.success() {863        return Err(Error::Generic("cargo install failed".to_string()));864    }865    assert!(866        matches!(bin_path.try_exists(), Ok(true)),867        "cargo install did not produce the expected binary"868    );869    eprintln!("finished building tool {bin_name}");870    Ok(bin_path)871}872873#[derive(Debug)]874enum Error {875    Io(io::Error),876    /// a is required to run b. c is extra info877    MissingReq(&'static str, &'static str, Option<String>),878    /// Tool x failed the check879    FailedCheck(&'static str),880    /// Any message, just print it881    Generic(String),882    /// Installed but wrong version883    Version {884        program: &'static str,885        required: &'static str,886        installed: String,887    },888}889890impl fmt::Display for Error {891    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {892        match self {893            Self::MissingReq(a, b, ex) => {894                write!(895                    f,896                    "{a} is required to run {b} but it could not be located. Is it installed?"897                )?;898                if let Some(s) = ex {899                    write!(f, "\n{s}")?;900                };901                Ok(())902            }903            Self::Version { program, required, installed } => write!(904                f,905                "insufficient version of '{program}' to run external tools: \906                {required} required but found {installed}",907            ),908            Self::Generic(s) => f.write_str(s),909            Self::Io(e) => write!(f, "IO error: {e}"),910            Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"),911        }912    }913}914915impl From<io::Error> for Error {916    fn from(value: io::Error) -> Self {917        Self::Io(value)918    }919}920921#[derive(Debug, PartialEq)]922enum ExtraCheckParseError {923    #[allow(dead_code, reason = "shown through Debug")]924    UnknownKind(String),925    #[allow(dead_code)]926    UnknownLang(String),927    UnsupportedKindForLang,928    /// Too many `:`929    TooManyParts,930    /// Tried to parse the empty string931    Empty,932    /// `auto` specified without lang part.933    AutoRequiresLang,934    /// `if-installed` specified without lang part.935    IfInstalledRequiresLang,936}937938#[derive(PartialEq, Debug)]939struct ExtraCheckArg {940    /// Only run the check if files to check have been modified.941    auto: bool,942    /// Only run the check if the requisite software is already installed.943    if_installed: bool,944    lang: ExtraCheckLang,945    /// None = run all extra checks for the given lang946    kind: Option<ExtraCheckKind>,947}948949impl ExtraCheckArg {950    fn matches(&self, lang: ExtraCheckLang, kind: ExtraCheckKind) -> bool {951        self.lang == lang && self.kind.map(|k| k == kind).unwrap_or(true)952    }953954    fn is_non_if_installed_or_matches(&self, root_path: &Path, build_dir: &Path) -> bool {955        if !self.if_installed {956            return true;957        }958959        match self.lang {960            ExtraCheckLang::Spellcheck => {961                match ensure_version(build_dir, "typos", SPELLCHECK_VER) {962                    Ok(_) => true,963                    Err(Error::Version { installed, .. }) => {964                        eprintln!(965                            "warning: the tool `typos` is detected, but version {installed} doesn't match with the expected version {SPELLCHECK_VER}"966                        );967                        false968                    }969                    _ => false,970                }971            }972            ExtraCheckLang::Shell => has_shellcheck().is_ok(),973            ExtraCheckLang::Js => {974                match self.kind {975                    Some(ExtraCheckKind::Lint) => {976                        // If Lint is enabled, check both eslint and es-check.977                        rustdoc_js::has_tool(build_dir, "eslint")978                            && rustdoc_js::has_tool(build_dir, "es-check")979                    }980                    Some(ExtraCheckKind::Typecheck) => {981                        // If Typecheck is enabled, check tsc.982                        rustdoc_js::has_tool(build_dir, "tsc")983                    }984                    None => {985                        // No kind means it will check both Lint and Typecheck.986                        rustdoc_js::has_tool(build_dir, "eslint")987                            && rustdoc_js::has_tool(build_dir, "es-check")988                            && rustdoc_js::has_tool(build_dir, "tsc")989                    }990                    Some(_) => unreachable!("js shouldn't have other type of ExtraCheckKind"),991                }992            }993            ExtraCheckLang::Py | ExtraCheckLang::Cpp => {994                let venv_path = build_dir.join("venv");995                let mut reqs_path = root_path.to_owned();996                reqs_path.extend(PIP_REQ_PATH);997                let Ok(v) = has_py_tools(&venv_path, &reqs_path) else {998                    return false;999                };10001001                v1002            }1003        }1004    }10051006    /// Returns `false` if this is an auto arg and the passed filename does not trigger the auto rule1007    fn is_non_auto_or_matches(&self, filepath: &str) -> bool {1008        if !self.auto {1009            return true;1010        }1011        let exts: &[&str] = match self.lang {1012            ExtraCheckLang::Py => &[".py"],1013            ExtraCheckLang::Cpp => &[".cpp"],1014            ExtraCheckLang::Shell => &[".sh"],1015            ExtraCheckLang::Js => &[".js", ".ts"],1016            ExtraCheckLang::Spellcheck => {1017                if SPELLCHECK_DIRS.iter().any(|dir| Path::new(filepath).starts_with(dir)) {1018                    return true;1019                }1020                &[]1021            }1022        };1023        exts.iter().any(|ext| filepath.ends_with(ext))1024    }10251026    fn has_supported_kind(&self) -> bool {1027        let Some(kind) = self.kind else {1028            // "run all extra checks" mode is supported for all languages.1029            return true;1030        };1031        use ExtraCheckKind::*;1032        let supported_kinds: &[_] = match self.lang {1033            ExtraCheckLang::Py => &[Fmt, Lint],1034            ExtraCheckLang::Cpp => &[Fmt],1035            ExtraCheckLang::Shell => &[Lint],1036            ExtraCheckLang::Spellcheck => &[],1037            ExtraCheckLang::Js => &[Lint, Typecheck],1038        };1039        supported_kinds.contains(&kind)1040    }1041}10421043impl FromStr for ExtraCheckArg {1044    type Err = ExtraCheckParseError;10451046    fn from_str(s: &str) -> Result<Self, Self::Err> {1047        let mut auto = false;1048        let mut if_installed = false;1049        let mut parts = s.split(':');1050        let mut first = match parts.next() {1051            Some("") | None => return Err(ExtraCheckParseError::Empty),1052            Some(part) => part,1053        };10541055        // The loop allows users to specify `auto` and `if-installed` in any order.1056        // Both auto:if-installed:<check> and if-installed:auto:<check> are valid.1057        loop {1058            match (first, auto, if_installed) {1059                ("auto", false, _) => {1060                    let Some(part) = parts.next() else {1061                        return Err(ExtraCheckParseError::AutoRequiresLang);1062                    };1063                    auto = true;1064                    first = part;1065                }1066                ("if-installed", _, false) => {1067                    let Some(part) = parts.next() else {1068                        return Err(ExtraCheckParseError::IfInstalledRequiresLang);1069                    };1070                    if_installed = true;1071                    first = part;1072                }1073                _ => break,1074            }1075        }1076        let second = parts.next();1077        if parts.next().is_some() {1078            return Err(ExtraCheckParseError::TooManyParts);1079        }1080        let arg = Self {1081            auto,1082            if_installed,1083            lang: first.parse()?,1084            kind: second.map(|s| s.parse()).transpose()?,1085        };1086        if !arg.has_supported_kind() {1087            return Err(ExtraCheckParseError::UnsupportedKindForLang);1088        }10891090        Ok(arg)1091    }1092}10931094#[derive(PartialEq, Copy, Clone, Debug)]1095enum ExtraCheckLang {1096    Py,1097    Shell,1098    Cpp,1099    Spellcheck,1100    Js,1101}11021103impl FromStr for ExtraCheckLang {1104    type Err = ExtraCheckParseError;11051106    fn from_str(s: &str) -> Result<Self, Self::Err> {1107        Ok(match s {1108            "py" => Self::Py,1109            "shell" => Self::Shell,1110            "cpp" => Self::Cpp,1111            "spellcheck" => Self::Spellcheck,1112            "js" => Self::Js,1113            _ => return Err(ExtraCheckParseError::UnknownLang(s.to_string())),1114        })1115    }1116}11171118#[derive(PartialEq, Copy, Clone, Debug)]1119enum ExtraCheckKind {1120    Lint,1121    Fmt,1122    Typecheck,1123    /// Never parsed, but used as a placeholder for1124    /// langs that never have a specific kind.1125    None,1126}11271128impl FromStr for ExtraCheckKind {1129    type Err = ExtraCheckParseError;11301131    fn from_str(s: &str) -> Result<Self, Self::Err> {1132        Ok(match s {1133            "lint" => Self::Lint,1134            "fmt" => Self::Fmt,1135            "typecheck" => Self::Typecheck,1136            _ => return Err(ExtraCheckParseError::UnknownKind(s.to_string())),1137        })1138    }1139}