1//! Tidy check to ensure that unstable features are all in order.2//!3//! This check will ensure properties like:4//!5//! * All stability attributes look reasonably well formed.6//! * The set of library features is disjoint from the set of language features.7//! * Library features have at most one stability level.8//! * Library features have at most one `since` value.9//! * All unstable lang features have tests to ensure they are actually unstable.10//! * Language features in a group are sorted by feature name.1112use std::collections::BTreeSet;13use std::collections::hash_map::{Entry, HashMap};14use std::ffi::OsStr;15use std::num::NonZeroU32;16use std::path::{Path, PathBuf};17use std::{fmt, fs};1819use crate::diagnostics::{RunningCheck, TidyCtx};20use crate::walk::{filter_dirs, filter_not_rust, walk, walk_many};2122#[cfg(test)]23mod tests;2425mod version;26use regex::Regex;27use version::Version;2829const FEATURE_GROUP_START_PREFIX: &str = "// feature-group-start";30const FEATURE_GROUP_END_PREFIX: &str = "// feature-group-end";3132#[derive(Debug, PartialEq, Clone)]33#[cfg_attr(feature = "build-metrics", derive(serde::Serialize))]34pub enum Status {35    Accepted,36    Removed,37    Unstable,38}3940impl fmt::Display for Status {41    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {42        let as_str = match *self {43            Status::Accepted => "accepted",44            Status::Unstable => "unstable",45            Status::Removed => "removed",46        };47        fmt::Display::fmt(as_str, f)48    }49}5051#[derive(Debug, Clone)]52#[cfg_attr(feature = "build-metrics", derive(serde::Serialize))]53pub struct Feature {54    pub level: Status,55    pub since: Option<Version>,56    pub has_gate_test: bool,57    pub tracking_issue: Option<NonZeroU32>,58    pub file: PathBuf,59    pub line: usize,60    pub description: Option<String>,61}62impl Feature {63    fn tracking_issue_display(&self) -> impl fmt::Display {64        match self.tracking_issue {65            None => "none".to_string(),66            Some(x) => x.to_string(),67        }68    }69}7071pub type Features = HashMap<String, Feature>;7273pub struct CollectedFeatures {74    pub lib: Features,75    pub lang: Features,76}7778// Currently only used for unstable book generation79pub fn collect_lib_features(base_src_path: &Path) -> Features {80    let mut lib_features = Features::new();8182    map_lib_features(base_src_path, &mut |res, _, _| {83        if let Ok((name, feature)) = res {84            lib_features.insert(name.to_owned(), feature);85        }86    });87    lib_features88}8990pub fn check(91    src_path: &Path,92    tests_path: &Path,93    compiler_path: &Path,94    lib_path: &Path,95    tidy_ctx: TidyCtx,96) -> CollectedFeatures {97    let mut check = tidy_ctx.start_check("features");9899    let mut features = collect_lang_features(compiler_path, &mut check);100    assert!(!features.is_empty());101102    let lib_features = get_and_check_lib_features(lib_path, &mut check, &features);103    assert!(!lib_features.is_empty());104105    walk_many(106        &[107            &tests_path.join("ui"),108            &tests_path.join("ui-fulldeps"),109            &tests_path.join("rustdoc-ui"),110            &tests_path.join("rustdoc-html"),111        ],112        |path, _is_dir| {113            filter_dirs(path)114                || filter_not_rust(path)115                || path.file_name() == Some(OsStr::new("features.rs"))116                || path.file_name() == Some(OsStr::new("diagnostic_list.rs"))117        },118        &mut |entry, contents| {119            let file = entry.path();120            let filename = file.file_name().unwrap().to_string_lossy();121            let filen_underscore = filename.replace('-', "_").replace(".rs", "");122            let filename_gate = test_filen_gate(&filen_underscore, &mut features);123124            for (i, line) in contents.lines().enumerate() {125                let mut err = |msg: &str| {126                    check.error(format!("{}:{}: {}", file.display(), i + 1, msg));127                };128129                let gate_test_str = "gate-test-";130131                let feature_name = match line.find(gate_test_str) {132                    // `split` always contains at least 1 element, even if the delimiter is not present.133                    Some(i) => line[i + gate_test_str.len()..].split(' ').next().unwrap(),134                    None => continue,135                };136                match features.get_mut(feature_name) {137                    Some(f) => {138                        if filename_gate == Some(feature_name) {139                            err(&format!(140                                "The file is already marked as gate test \141                                      through its name, no need for a \142                                      'gate-test-{feature_name}' comment"143                            ));144                        }145                        f.has_gate_test = true;146                    }147                    None => {148                        err(&format!(149                            "gate-test test found referencing a nonexistent feature '{feature_name}'"150                        ));151                    }152                }153            }154        },155    );156157    // Only check the number of lang features.158    // Obligatory testing for library features is dumb.159    let gate_untested = features160        .iter()161        .filter(|&(_, f)| f.level == Status::Unstable)162        .filter(|&(_, f)| !f.has_gate_test)163        .collect::<Vec<_>>();164165    for &(name, _) in gate_untested.iter() {166        println!("Expected a gate test for the feature '{name}'.");167        println!(168            "Hint: create a failing test file named 'tests/ui/feature-gates/feature-gate-{}.rs',\169                \n      with its failures due to missing usage of `#![feature({})]`.",170            name.replace("_", "-"),171            name172        );173        println!(174            "Hint: If you already have such a test and don't want to rename it,\175                \n      you can also add a // gate-test-{name} line to the test file."176        );177    }178179    if !gate_untested.is_empty() {180        check.error(format!("Found {} features without a gate test.", gate_untested.len()));181    }182183    let (version, channel) = get_version_and_channel(src_path);184185    let all_features_iter = features186        .iter()187        .map(|feat| (feat, "lang"))188        .chain(lib_features.iter().map(|feat| (feat, "lib")));189    for ((feature_name, feature), kind) in all_features_iter {190        let since = if let Some(since) = feature.since { since } else { continue };191        let file = feature.file.display();192        let line = feature.line;193        if since > version && since != Version::CurrentPlaceholder {194            check.error(format!(195                "{file}:{line}: The stabilization version {since} of {kind} feature `{feature_name}` is newer than the current {version}"196            ));197        }198        if channel == "nightly" && since == version {199            check.error(format!(200                "{file}:{line}: The stabilization version {since} of {kind} feature `{feature_name}` is written out but should be {}",201                version::VERSION_PLACEHOLDER202            ));203        }204        if channel != "nightly" && since == Version::CurrentPlaceholder {205            check.error(format!(206                "{file}:{line}: The placeholder use of {kind} feature `{feature_name}` is not allowed on the {channel} channel",207            ));208        }209    }210211    if !check.is_bad() && check.is_verbose_enabled() {212        let mut lines = Vec::new();213        lines.extend(format_features(&features, "lang"));214        lines.extend(format_features(&lib_features, "lib"));215        lines.sort();216217        check.verbose_msg(218            lines.into_iter().map(|l| format!("* {l}")).collect::<Vec<String>>().join("\n"),219        );220    }221222    CollectedFeatures { lib: lib_features, lang: features }223}224225fn get_version_and_channel(src_path: &Path) -> (Version, String) {226    let version_str = t!(std::fs::read_to_string(src_path.join("version")));227    let version_str = version_str.trim();228    let version = t!(std::str::FromStr::from_str(version_str).map_err(|e| format!("{e:?}")));229    let channel_str = t!(std::fs::read_to_string(src_path.join("ci").join("channel")));230    (version, channel_str.trim().to_owned())231}232233fn format_features<'a>(234    features: &'a Features,235    family: &'a str,236) -> impl Iterator<Item = String> + 'a {237    features.iter().map(move |(name, feature)| {238        format!(239            "{:<32} {:<8} {:<12} {:<8}",240            name,241            family,242            feature.level,243            feature.since.map_or("None".to_owned(), |since| since.to_string())244        )245    })246}247248fn find_attr_val<'a>(line: &'a str, attr: &str) -> Option<&'a str> {249    let r = match attr {250        "issue" => static_regex!(r#"issue\s*=\s*"([^"]*)""#),251        "feature" => static_regex!(r#"feature\s*=\s*"([^"]*)""#),252        "since" => static_regex!(r#"since\s*=\s*"([^"]*)""#),253        _ => unimplemented!("{attr} not handled"),254    };255256    r.captures(line).and_then(|c| c.get(1)).map(|m| m.as_str())257}258259fn test_filen_gate<'f>(filen_underscore: &'f str, features: &mut Features) -> Option<&'f str> {260    let prefix = "feature_gate_";261    if let Some(suffix) = filen_underscore.strip_prefix(prefix) {262        for (n, f) in features.iter_mut() {263            // Equivalent to filen_underscore == format!("feature_gate_{n}")264            if suffix == n {265                f.has_gate_test = true;266                return Some(suffix);267            }268        }269    }270    None271}272273pub fn collect_lang_features(base_compiler_path: &Path, check: &mut RunningCheck) -> Features {274    let mut features = Features::new();275    collect_lang_features_in(&mut features, base_compiler_path, "accepted.rs", check);276    collect_lang_features_in(&mut features, base_compiler_path, "removed.rs", check);277    collect_lang_features_in(&mut features, base_compiler_path, "unstable.rs", check);278    features279}280281fn collect_lang_features_in(282    features: &mut Features,283    base: &Path,284    file: &str,285    check: &mut RunningCheck,286) {287    let path = base.join("rustc_feature").join("src").join(file);288    let contents = t!(fs::read_to_string(&path));289290    // We allow rustc-internal features to omit a tracking issue.291    // To make tidy accept omitting a tracking issue, group the list of features292    // without one inside `// no-tracking-issue` and `// no-tracking-issue-end`.293    let mut next_feature_omits_tracking_issue = false;294295    let mut in_feature_group = false;296    let mut prev_names = vec![];297298    let lines = contents.lines().zip(1..);299    let mut doc_comments: Vec<String> = Vec::new();300    for (line, line_number) in lines {301        let line = line.trim();302303        // Within -start and -end, the tracking issue can be omitted.304        match line {305            "// no-tracking-issue-start" => {306                next_feature_omits_tracking_issue = true;307                continue;308            }309            "// no-tracking-issue-end" => {310                next_feature_omits_tracking_issue = false;311                continue;312            }313            _ => {}314        }315316        if line.starts_with(FEATURE_GROUP_START_PREFIX) {317            if in_feature_group {318                check.error(format!(319                    "{}:{line_number}: \320                        new feature group is started without ending the previous one",321                    path.display()322                ));323            }324325            in_feature_group = true;326            prev_names = vec![];327            continue;328        } else if line.starts_with(FEATURE_GROUP_END_PREFIX) {329            in_feature_group = false;330            prev_names = vec![];331            continue;332        }333334        if in_feature_group && let Some(doc_comment) = line.strip_prefix("///") {335            doc_comments.push(doc_comment.trim().to_string());336            continue;337        }338339        let mut parts = line.split(',');340        let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {341            Some("unstable") => Status::Unstable,342            Some("incomplete") => Status::Unstable,343            Some("internal") => Status::Unstable,344            Some("removed") => Status::Removed,345            Some("accepted") => Status::Accepted,346            _ => continue,347        };348        let name = parts.next().unwrap().trim();349350        let since_str = parts.next().unwrap().trim().trim_matches('"');351        let since = match since_str.parse() {352            Ok(since) => Some(since),353            Err(err) => {354                check.error(format!(355                    "{}:{line_number}: failed to parse since: {since_str} ({err:?})",356                    path.display()357                ));358                None359            }360        };361        if in_feature_group {362            if prev_names.last() > Some(&name) {363                // This assumes the user adds the feature name at the end of the list, as we're364                // not looking ahead.365                let correct_index = match prev_names.binary_search(&name) {366                    Ok(_) => {367                        // This only occurs when the feature name has already been declared.368                        check.error(format!(369                            "{}:{line_number}: duplicate feature {name}",370                            path.display()371                        ));372                        // skip any additional checks for this line373                        continue;374                    }375                    Err(index) => index,376                };377378                let correct_placement = if correct_index == 0 {379                    "at the beginning of the feature group".to_owned()380                } else if correct_index == prev_names.len() {381                    // I don't believe this is reachable given the above assumption, but it382                    // doesn't hurt to be safe.383                    "at the end of the feature group".to_owned()384                } else {385                    format!(386                        "between {} and {}",387                        prev_names[correct_index - 1],388                        prev_names[correct_index],389                    )390                };391392                check.error(format!(393                    "{}:{line_number}: feature {name} is not sorted by feature name (should be {correct_placement})",394                    path.display(),395                ));396            }397            prev_names.push(name);398        }399400        let issue_str = parts.next().unwrap().trim();401        let tracking_issue = if issue_str.starts_with("None") {402            if level == Status::Unstable && !next_feature_omits_tracking_issue {403                check.error(format!(404                    "{}:{line_number}: no tracking issue for feature {name}",405                    path.display(),406                ));407            }408            None409        } else {410            let s = issue_str.split('(').nth(1).unwrap().split(')').next().unwrap();411            Some(s.parse().unwrap())412        };413        match features.entry(name.to_owned()) {414            Entry::Occupied(e) => {415                check.error(format!(416                    "{}:{line_number} feature {name} already specified with status '{}'",417                    path.display(),418                    e.get().level,419                ));420            }421            Entry::Vacant(e) => {422                e.insert(Feature {423                    level,424                    since,425                    has_gate_test: false,426                    tracking_issue,427                    file: path.to_path_buf(),428                    line: line_number,429                    description: if doc_comments.is_empty() {430                        None431                    } else {432                        Some(doc_comments.join(" "))433                    },434                });435            }436        }437        doc_comments.clear();438    }439}440441fn get_and_check_lib_features(442    base_src_path: &Path,443    check: &mut RunningCheck,444    lang_features: &Features,445) -> Features {446    let mut lib_features = Features::new();447    map_lib_features(base_src_path, &mut |res, file, line| match res {448        Ok((name, f)) => {449            let mut check_features = |f: &Feature, list: &Features, display: &str| {450                if let Some(s) = list.get(name)451                    && f.tracking_issue != s.tracking_issue452                    && f.level != Status::Accepted453                {454                    check.error(format!(455                        "{}:{line}: feature gate {name} has inconsistent `issue`: \"{}\" mismatches the {display} `issue` of \"{}\"",456                        file.display(),457                        f.tracking_issue_display(),458                        s.tracking_issue_display(),459                    ));460                }461            };462            check_features(&f, lang_features, "corresponding lang feature");463            check_features(&f, &lib_features, "previous");464            lib_features.insert(name.to_owned(), f);465        }466        Err(msg) => {467            check.error(format!("{}:{line}: {msg}", file.display()));468        }469    });470    lib_features471}472473fn map_lib_features(474    base_src_path: &Path,475    mf: &mut (dyn Send + Sync + FnMut(Result<(&str, Feature), &str>, &Path, usize)),476) {477    walk(478        base_src_path,479        |path, _is_dir| filter_dirs(path) || path.ends_with("tests"),480        &mut |entry, contents| {481            let file = entry.path();482            let filename = file.file_name().unwrap().to_string_lossy();483            if !filename.ends_with(".rs")484                || filename == "features.rs"485                || filename == "diagnostic_list.rs"486                || filename == "error_codes.rs"487            {488                return;489            }490491            // This is an early exit -- all the attributes we're concerned with must contain this:492            // * rustc_const_unstable(493            // * unstable(494            // * stable(495            if !contents.contains("stable(") {496                return;497            }498499            let handle_issue_none = |s| match s {500                "none" => None,501                issue => {502                    let n = issue.parse().expect("issue number is not a valid integer");503                    assert_ne!(n, 0, "\"none\" should be used when there is no issue, not \"0\"");504                    NonZeroU32::new(n)505                }506            };507            let mut becoming_feature: Option<(&str, Feature)> = None;508            let mut iter_lines = contents.lines().enumerate().peekable();509            while let Some((i, line)) = iter_lines.next() {510                macro_rules! err {511                    ($msg:expr) => {{512                        mf(Err($msg), file, i + 1);513                        continue;514                    }};515                }516517                // exclude commented out lines518                if static_regex!(r"^\s*//").is_match(line) {519                    continue;520                }521522                if let Some((name, ref mut f)) = becoming_feature {523                    if f.tracking_issue.is_none() {524                        f.tracking_issue = find_attr_val(line, "issue").and_then(handle_issue_none);525                    }526                    if line.ends_with(']') {527                        mf(Ok((name, f.clone())), file, i + 1);528                    } else if !line.ends_with(',') && !line.ends_with('\\') && !line.ends_with('"')529                    {530                        // We need to bail here because we might have missed the531                        // end of a stability attribute above because the ']'532                        // might not have been at the end of the line.533                        // We could then get into the very unfortunate situation that534                        // we continue parsing the file assuming the current stability535                        // attribute has not ended, and ignoring possible feature536                        // attributes in the process.537                        err!("malformed stability attribute");538                    } else {539                        continue;540                    }541                }542                becoming_feature = None;543                if line.contains("rustc_const_unstable(") {544                    // `const fn` features are handled specially.545                    let feature_name = match find_attr_val(line, "feature").or_else(|| {546                        iter_lines.peek().and_then(|next| find_attr_val(next.1, "feature"))547                    }) {548                        Some(name) => name,549                        None => err!("malformed stability attribute: missing `feature` key"),550                    };551                    let feature = Feature {552                        level: Status::Unstable,553                        since: None,554                        has_gate_test: false,555                        tracking_issue: find_attr_val(line, "issue").and_then(handle_issue_none),556                        file: file.to_path_buf(),557                        line: i + 1,558                        description: None,559                    };560                    mf(Ok((feature_name, feature)), file, i + 1);561                    continue;562                }563                let level = if line.contains("[unstable(") {564                    Status::Unstable565                } else if line.contains("[stable(") {566                    Status::Accepted567                } else {568                    continue;569                };570                let feature_name = match find_attr_val(line, "feature")571                    .or_else(|| iter_lines.peek().and_then(|next| find_attr_val(next.1, "feature")))572                {573                    Some(name) => name,574                    None => err!("malformed stability attribute: missing `feature` key"),575                };576                let since = match find_attr_val(line, "since").map(|x| x.parse()) {577                    Some(Ok(since)) => Some(since),578                    Some(Err(_err)) => {579                        err!("malformed stability attribute: can't parse `since` key");580                    }581                    None if level == Status::Accepted => {582                        err!("malformed stability attribute: missing the `since` key");583                    }584                    None => None,585                };586                let tracking_issue = find_attr_val(line, "issue").and_then(handle_issue_none);587588                let feature = Feature {589                    level,590                    since,591                    has_gate_test: false,592                    tracking_issue,593                    file: file.to_path_buf(),594                    line: i + 1,595                    description: None,596                };597                if line.contains(']') {598                    mf(Ok((feature_name, feature)), file, i + 1);599                } else {600                    becoming_feature = Some((feature_name, feature));601                }602            }603        },604    );605}606607fn should_document(var: &str) -> bool {608    if var.starts_with("RUSTC_") || var.starts_with("RUST_") || var.starts_with("UNSTABLE_RUSTDOC_")609    {610        return true;611    }612    ["SDKROOT", "QNX_TARGET", "COLORTERM", "TERM"].contains(&var)613}614615pub fn collect_env_vars(compiler: &Path) -> BTreeSet<String> {616    let env_var_regex: Regex = Regex::new(r#"env::var(_os)?\("([^"]+)"#).unwrap();617618    let mut vars = BTreeSet::new();619    walk(620        compiler,621        // skip build scripts, tests, and non-rust files622        |path, _is_dir| {623            filter_dirs(path)624                || filter_not_rust(path)625                || path.ends_with("build.rs")626                || path.ends_with("tests.rs")627        },628        &mut |_entry, contents| {629            for env_var in env_var_regex.captures_iter(contents).map(|c| c.get(2).unwrap().as_str())630            {631                if should_document(env_var) {632                    vars.insert(env_var.to_owned());633                }634            }635        },636    );637    vars638}