1// SPDX-License-Identifier: MIT23package processor45import (6	"fmt"7	"io"8	"os"9	"path/filepath"10	"regexp"11	"runtime"12	"runtime/debug"13	"slices"14	"strconv"15	"strings"16	"sync"1718	"github.com/boyter/gocodewalker"19)2021// Version indicates the version of the application22var Version = "4.1.0"2324// Flags set via the CLI which control how the output is displayed2526// Files indicates if there should be file output or not when formatting27var Files = false2829// Languages indicates if the command line should print out the supported languages30var Languages = false3132// ListCounters indicates if the command line should print out the languages that33// have a scanner written for them rather than being counted by the generic loop.34// Sixteen names is unreadable in a flag help string, so the flag names the count35// and this prints the list.36var ListCounters = false3738// Verbose enables verbose logging output39var Verbose = false4041// Debug enables debug logging output42var Debug = false4344// Trace enables trace logging output which is extremely verbose45var Trace = false4647// Duplicates enables duplicate file detection48var Duplicates = false4950// MinifiedGenerated enables minified/generated file detection51var MinifiedGenerated = false5253// IgnoreMinifiedGenerate printing counts for minified/generated files54var IgnoreMinifiedGenerate = false5556// MinifiedGeneratedLineByteLength number of bytes per average line to determine file is minified/generated57var MinifiedGeneratedLineByteLength = 2555859// Minified enables minified file detection60var Minified = false6162// IgnoreMinified ignore printing counts for minified files63var IgnoreMinified = false6465// Generated enables generated file detection66var Generated = false6768// GeneratedMarkers defines head markers for generated file detection69var GeneratedMarkers []string7071// IgnoreGenerated ignore printing counts for generated files72var IgnoreGenerated = false7374// Complexity toggles complexity calculation75var Complexity = false7677// Cognitive toggles cognitive (nesting-weighted) complexity calculation78var Cognitive = false7980// More enables wider output with more information in formatter81var More = false8283// Cocomo toggles the COCOMO calculation84var Cocomo = false8586// SLOCCountFormat prints a more SLOCCount like COCOMO calculation87var SLOCCountFormat = false8889// CocomoProjectType allows the flipping between project types which impacts the calculation90var CocomoProjectType = "organic"9192// Size toggles the Size calculation93var Size = false9495// Draw horizontal borders between sections.96var HBorder = false9798// SizeUnit determines what size calculation is used for megabytes99var SizeUnit = "si"100101// Ci indicates if running inside a CI so to disable box drawing characters102var Ci = false103104// GitIgnore disables .gitignore checks105var GitIgnore = false106107// GitModuleIgnore disables .gitmodules checks108var GitModuleIgnore = false109110// Ignore disables ignore file checks111var Ignore = false112113// SccIgnore disables sccignore file checks114var SccIgnore = false115116// CountIgnore should we count ignore files?117var CountIgnore = false118119// CountUnsupported when set counts files scc does not recognise under an120// "Unknown" category, treating them as plain text. See issue #464.121var CountUnsupported = false122123// IgnoreFiles are paths to additional ignore files supplied via --ignore-file.124// They are applied as a low priority base layer in the order supplied so a later125// file can override an earlier one, and any in-tree .gitignore/.ignore/.sccignore126// discovered while walking overrides all of them.127var IgnoreFiles = []string{}128129// DisableCheckBinary toggles checking for binary files using NUL bytes130var DisableCheckBinary = false131132// UlocMode toggles checking for binary files using NUL bytes133var UlocMode = false134135// Percent toggles checking for binary files using NUL bytes136var Percent = false137138// MaxMean sets the calculation of the max and mean line length139var MaxMean = false140141// Dryness toggles checking for binary files using NUL bytes142var Dryness = false143144// SortBy sets which column output in formatter should be sorted by145var SortBy = ""146147// Exclude is a regular expression which is used to exclude files from being processed148var Exclude = []string{}149150// CountAs is a rule for mapping known or new extensions to other rules151var CountAs = ""152153// Format sets the output format of the formatter154var Format = ""155156// FormatMulti is a rule for defining multiple output formats157var FormatMulti = ""158159// SQLProject is used to store the name for the SQL insert formats but is optional160var SQLProject = ""161162// RemapUnknown allows remapping of unknown files with a string to search the content for163var RemapUnknown = ""164165// RemapAll allows remapping of all files with a string to search the content for166var RemapAll = ""167168type remapRule struct {169	pattern  []byte170	language string171}172173type remapConfig struct {174	all     []remapRule175	unknown []remapRule176}177178type processorContext struct {179	remap remapConfig180	// summary, when set, tells the counting workers to fold their own results181	// into it instead of sending every file down the summary channel. Only the182	// default and wide summaries can use it; see summariseInWorkers.183	summary *sharedSummaryTotals184}185186func parseRemapRules(value string) []remapRule {187	rules := []remapRule{}188189	for s := range strings.SplitSeq(value, ",") {190		t := strings.Split(s, ":")191		if len(t) == 2 {192			rules = append(rules, remapRule{193				pattern:  []byte(t[0]),194				language: t[1],195			})196		}197	}198199	return rules200}201202func newRemapConfig(remapAll string, remapUnknown string) remapConfig {203	c := remapConfig{204		all:     parseRemapRules(remapAll),205		unknown: parseRemapRules(remapUnknown),206	}207208	// Load the features for every language a rule can remap to. Remapping only209	// sets job.Language, and in lazy mode (which is every CLI run) nothing else210	// guarantees that language was ever loaded — CountStats would then find no211	// features and count the file as plain text, with no comments and no212	// complexity. Done once here at setup rather than in the remap functions213	// themselves, which run per file on the hot path.214	for _, rule := range c.all {215		LoadLanguageFeature(rule.language)216	}217	for _, rule := range c.unknown {218		LoadLanguageFeature(rule.language)219	}220221	return c222}223224// MatchEngine selects how a CountRule pattern is interpreted. Glob is the225// default; regex is opt-in via the re: prefix.226type MatchEngine int227228const (229	// MatchGlob is the default. The pattern is a glob ('*' and '?') translated230	// to an anchored regex and matched as a full match against the path.231	MatchGlob MatchEngine = iota232	// MatchRegex treats the pattern as a raw (unanchored) RE2 regex. Opt in233	// with the re: prefix.234	MatchRegex235)236237// CountRule is the typed, library-facing form of a --count-as-pattern rule.238// It matches files by their path and relabels them to a new named category239// whose counting rules are cloned from an existing base language.240type CountRule struct {241	Engine       MatchEngine // MatchGlob (the default) or MatchRegex242	Pattern      string      // glob or regex source243	Name         string      // new category display name244	BaseLanguage string      // existing language whose counting rules are cloned245}246247// CountRules is the typed input set either directly by library users or by the248// CLI after parsing CountAsPattern. Setup happens in setupCountRules.249var CountRules []CountRule250251// CountAsPattern holds the raw repeatable --count-as-pattern flag values. Each252// is parsed into a CountRule at setup. Library users may set CountRules directly.253var CountAsPattern []string254255// compiledCountRule is the runtime form scanned by newFileJob256type compiledCountRule struct {257	re   *regexp.Regexp258	name string259}260261var compiledCountRules []compiledCountRule262263// CurrencySymbol allows setting the currency symbol for cocomo project cost estimation264var CurrencySymbol = ""265266// FileOutput sets the file that output should be written to267var FileOutput = ""268269// PathDenyList sets the paths that should be skipped270var PathDenyList = []string{}271272// FileListQueueSize is the queue of files found and ready to be read into memory273var FileListQueueSize = runtime.NumCPU()274275// FileProcessJobWorkers is the number of workers that process the file collecting stats276var FileProcessJobWorkers = runtime.NumCPU() * 4277278// FileListJobWorkers is the number of workers that turn a path the walker found279// into a FileJob, which is a stat and a language lookup each280var FileListJobWorkers = runtime.NumCPU()281282// FileSummaryJobQueueSize is the queue used to hold processed file statistics before formatting283var FileSummaryJobQueueSize = runtime.NumCPU()284285// DirectoryWalkerJobWorkers is the number of workers which will walk the directory tree286var DirectoryWalkerJobWorkers = 8287288// AllowListExtensions is a list of extensions which are allowed to be processed289var AllowListExtensions = []string{}290291// ExcludeListExtensions is a list of extensions which should be ignored292var ExcludeListExtensions = []string{}293294// ExcludeFilename is a list of filenames which should be ignored295var ExcludeFilename = []string{}296297// AverageWage is the average wage in dollars used for the COCOMO cost estimate298var AverageWage int64 = 56286299300// Overhead is the overhead multiplier for corporate overhead (facilities, equipment, accounting, etc.)301var Overhead float64 = 2.4302303// EAF is the effort adjustment factor derived from the cost drivers, i.e. 1.0 if rated nominal304var EAF float64 = 1.0305306// Locomo toggles the LOCOMO (LLM Output COst MOdel) calculation307var Locomo = false308309// CostComparison enables both COCOMO and LOCOMO output for side-by-side comparison310var CostComparison = false311312// LocomoPresetName is the LLM model preset for pricing and throughput defaults313var LocomoPresetName = "medium"314315// LocomoInputPrice is the cost per 1M input tokens (overrides preset)316var LocomoInputPrice float64317var LocomoInputPriceSet = false318319// LocomoOutputPrice is the cost per 1M output tokens (overrides preset)320var LocomoOutputPrice float64321var LocomoOutputPriceSet = false322323// LocomoTPS is the output tokens per second (overrides preset)324var LocomoTPS float64325var LocomoTPSSet = false326327// LocomoReviewMinutesPerLine is the human review time per line of code in minutes328var LocomoReviewMinutesPerLine float64 = 0.01329330// LocomoConfig is the power-user config string "tokensPerLine,baseInputPerLine,complexityWeight,iterations,iterationWeight"331var LocomoConfig = ""332333// LocomoTokensPerLine is the average number of output tokens per line of code334var LocomoTokensPerLine float64 = 10335336// LocomoBaseInputPerLine is the base number of input tokens per output line337var LocomoBaseInputPerLine float64 = 20338339// LocomoComplexityWeight is the scaling weight applied to sqrt(complexity density) for input tokens340var LocomoComplexityWeight float64 = 5341342// LocomoIterations is the base number of iteration/retry attempts343var LocomoIterations float64 = 1.5344345// LocomoIterationWeight is the scaling weight for complexity-driven retries346var LocomoIterationWeight float64 = 2347348// LocomoCyclesOverride is the user-supplied iteration factor override (--locomo-cycles)349var LocomoCyclesOverride float64350351// LocomoCyclesSet indicates whether --locomo-cycles was explicitly set352var LocomoCyclesSet = false353354// GcFileCount is the number of files to process before turning the GC back on355var GcFileCount = 10000356var gcPercent = -1357var isLazy = false358359// NoLarge if set true will ignore files over a certain number of lines or bytes360var NoLarge = false361362// IncludeSymLinks if set true will count symlink files363var IncludeSymLinks = false364365// LargeLineCount number of lines before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44366var LargeLineCount int64 = 40000367368// LargeByteCount number of bytes before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44369var LargeByteCount int64 = 1000000370371// Hotspots toggles the hotspots git-history report372var Hotspots = false373374// Coupling toggles the change-coupling git-history report (file pairs that375// change together)376var Coupling = false377378// CouplingFor, when non-empty, switches the coupling report to the379// file-oriented "blast radius" view for the given path: what tends to change380// when that file changes.381var CouplingFor = ""382383// CouplingWeighted ranks coupling by degree × the pair's (smaller) file384// complexity instead of raw co-change, so pairs of genuinely complex files385// outrank generated/data-file churn. Implies Coupling. Honours the Cognitive386// global for its complexity source, matching --hotspots.387var CouplingWeighted = false388389// ByAuthor toggles the author-rollup git-history report390var ByAuthor = false391392// Timeline selects an over-time view. With ByAuthor, runs the author393// timeline report (plan 04); alone, runs the languages-over-time report394// (plan 05). With Hotspots set, the combination errors out.395var Timeline = false396397// HistoryBuckets is the time-bucket resolution for the timeline reports.398// Wired to --buckets in main.go; default 60.399var HistoryBuckets = 60400401// FoldAuthors enables the name+domain identity folding fallback applied402// after the mailmap. Toggled off via --no-fold-authors.403var FoldAuthors = true404405// DirFilePaths is not set via flags but by arguments following the flags for file or directory to process406var DirFilePaths = []string{}407408// ExtensionToLanguage is loaded from the JSON that is in constants.go409var ExtensionToLanguage = map[string][]string{}410411// ShebangLookup loaded from the JSON in constants.go contains shebang lookups412var ShebangLookup = map[string][]string{}413414// FilenameToLanguage similar to ExtensionToLanguage loaded from the JSON in constants.go415var FilenameToLanguage = map[string]string{}416417// LanguageFeatures contains the processed languages from processLanguageFeature418var LanguageFeatures = map[string]LanguageFeature{}419420// LanguageFeaturesMutex is the shared mutex used to control getting and setting of language features421// used rather than sync.Map because it turned out to be marginally faster422var LanguageFeaturesMutex = sync.Mutex{}423424// Start time in milli seconds in case we want the total time425var startTimeMilli = makeTimestampMilli()426427// ConfigureGc needs to be set outside of ProcessConstants because it should only be enabled in command line428// mode https://github.com/boyter/scc/issues/32429func ConfigureGc() {430	gcPercent = debug.SetGCPercent(gcPercent)431}432433// EnableGc restores the garbage collector to the percentage captured by ConfigureGc.434func EnableGc() {435	if gcPercent != -1 {436		debug.SetGCPercent(gcPercent)437	}438}439440// ConfigureLazy is a simple setter used to turn on lazy loading used only by command line441func ConfigureLazy(lazy bool) {442	isLazy = lazy443}444445// ProcessConstants is responsible for setting up the language features based on the JSON file that is stored in constants446// Needs to be called at least once in order for anything to actually happen447func ProcessConstants() {448	startTime := makeTimestampNano()449450	// Reset the reverse-lookup maps so ProcessConstants is idempotent. The451	// ExtensionToLanguage entries are built with append, so without clearing452	// first a repeated call (the long-lived MCP server invokes ProcessConstants453	// once per tool call) would accumulate duplicate languages for every454	// extension. scc was historically a one-shot CLI where this ran exactly455	// once, so it never surfaced until server mode.456	clear(ExtensionToLanguage)457	clear(FilenameToLanguage)458	clear(ShebangLookup)459460	for name, value := range languageDatabase {461		for _, ext := range value.Extensions {462			ExtensionToLanguage[ext] = append(ExtensionToLanguage[ext], name)463		}464465		for _, fname := range value.FileNames {466			FilenameToLanguage[fname] = name467		}468469		if len(value.SheBangs) != 0 {470			ShebangLookup[name] = value.SheBangs471		}472	}473474	// If we have anything in CountAs set it up now475	if len(CountAs) != 0 {476		setupCountAs()477	}478479	printTraceF("nanoseconds build extension to language: %d", makeTimestampNano()-startTime)480481	// Set up any path pattern count rules, minting new categories backed by a482	// base language. The function clones the base language and builds its483	// features so counting works in both lazy and non-lazy modes.484	if len(CountAsPattern) != 0 || len(CountRules) != 0 {485		setupCountRules()486	}487488	// Configure COCOMO setting. projectType is keyed by the canonical489	// lowercase name, so when the selection matches a built-in type we must490	// normalize CocomoProjectType to that lowercase key — otherwise the491	// membership check below passes for "Organic" while the real lookups in492	// EstimateEffort/EstimateScheduleMonths index projectType["Organic"], hit493	// a nil slice and panic. See `scc --cocomo-project-type Organic`.494	if _, ok := projectType[strings.ToLower(CocomoProjectType)]; ok {495		CocomoProjectType = strings.ToLower(CocomoProjectType)496	} else {497		// let's see if we can turn it into a custom one498		spl := strings.Split(CocomoProjectType, ",")499		val := []float64{}500		if len(spl) == 5 {501			// let's try to convert to float if we can502			for i := 1; i < 5; i++ {503				f, err := strconv.ParseFloat(spl[i], 64)504				if err == nil {505					val = append(val, f)506				}507			}508		}509510		if len(val) == 4 {511			projectType[CocomoProjectType] = val512		} else {513			// if nothing matches fall back to organic514			CocomoProjectType = "organic"515		}516	}517518	// If lazy is set then we want to load in the features as we find them not in one go519	// however otherwise being used as a library so just load them all in520	if !isLazy {521		startTime = makeTimestampMilli()522		for name, value := range languageDatabase {523			processLanguageFeature(name, value)524		}525526		printTraceF("milliseconds build language features: %d", makeTimestampMilli()-startTime)527	} else {528		printTrace("configured to lazy load language features")529	}530531	// Fix for https://github.com/boyter/scc/issues/250532	fixedPath := make([]string, 0, len(PathDenyList))533	for _, path := range PathDenyList {534		fixedPath = append(fixedPath, strings.TrimRight(path, "/"))535	}536	PathDenyList = fixedPath537}538539// Configure and setup any count-as params the use has supplied540func setupCountAs() {541	for s := range strings.SplitSeq(CountAs, ",") {542		t := strings.Split(s, ":")543		if len(t) != 2 {544			printError(fmt.Sprintf("ignoring malformed count-as rule %q: expected format <from>:<to>", s))545			continue546		}547548		// There are two cases here.549		// first is they provide the name e.g. "Cargo Lock"550		// second is that the user supplies the extension EG wsdl551		// we should support BOTH cases552		// always remember we only need to validate t[1] as that's the one553		// that tells us where we are trying to map554		target, ok := resolveBaseLanguage(t[1])555		if ok {556			ExtensionToLanguage[strings.ToLower(t[0])] = []string{target}557			printDebugF("set to count extension: %s as language %s", t[0], target)558			continue559		}560561		// The target t[1] matched neither a known language name nor a known562		// extension, so no mapping was registered. Warn rather than silently563		// ignoring the rule, since count-as cannot mint new categories yet.564		printError(fmt.Sprintf("ignoring count-as rule %q: target %q is not a known language or extension", s, t[1]))565	}566}567568// resolveBaseLanguage resolves a user supplied target to a canonical language569// name. It first tries to match a language name (most reliable as names are570// unique) and falls back to matching a known extension. Returns the canonical571// language name and whether it was resolved.572func resolveBaseLanguage(target string) (string, bool) {573	// Match by language name which is the most reliable as the name is unique574	for name := range languageDatabase {575		if strings.EqualFold(name, target) {576			return name, true577		}578	}579580	// Fall back to extension match, note this is less reliable as some581	// languages share extensions so we take the first registered language582	langs, ok := ExtensionToLanguage[strings.ToLower(target)]583	if ok && len(langs) != 0 {584		return langs[0], true585	}586587	return "", false588}589590// parseCountAsPattern parses a single --count-as-pattern rule of the form591// [engine:]pattern:name:baselang into a CountRule.592//593// The engine prefix is optional and the pattern is treated as a GLOB BY594// DEFAULT; prefix with re: to opt into a regex (or glob: to be explicit). We595// keep glob and regex as distinct modes rather than inferring, because the same596// string is valid in both engines with different meaning (e.g. "foo.rb" matches597// only foo.rb as a glob but also fooXrb as a regex), so guessing would silently598// match the wrong files.599//600// Because regex patterns and paths legitimately contain ':', name and baselang601// are peeled from the right and the pattern is whatever remains in between.602func parseCountAsPattern(s string) (CountRule, error) {603	engine := MatchGlob604	rest := s605606	switch {607	case strings.HasPrefix(rest, "re:"):608		engine = MatchRegex609		rest = rest[len("re:"):]610	case strings.HasPrefix(rest, "glob:"):611		engine = MatchGlob612		rest = rest[len("glob:"):]613	}614615	// baselang = after the last ':', name = between the 2nd-last and last ':'616	lastColon := strings.LastIndex(rest, ":")617	if lastColon == -1 {618		return CountRule{}, fmt.Errorf("expected format [engine:]pattern:name:baselang")619	}620	baseLanguage := rest[lastColon+1:]621622	nameColon := strings.LastIndex(rest[:lastColon], ":")623	if nameColon == -1 {624		return CountRule{}, fmt.Errorf("expected format [engine:]pattern:name:baselang")625	}626	name := rest[nameColon+1 : lastColon]627	pattern := rest[:nameColon]628629	if pattern == "" || name == "" || baseLanguage == "" {630		return CountRule{}, fmt.Errorf("pattern, name and baselang must all be non-empty")631	}632633	return CountRule{Engine: engine, Pattern: pattern, Name: name, BaseLanguage: baseLanguage}, nil634}635636// globToRegex converts a simple glob into an anchored regex. Glob is the637// default --count-as-pattern engine. Only '*' (any run of characters) and '?'638// (single character) are special, everything else is matched literally. The639// result is anchored as a full match.640func globToRegex(glob string) string {641	var b strings.Builder642	b.WriteByte('^')643	for _, r := range glob {644		switch r {645		case '*':646			b.WriteString(".*")647		case '?':648			b.WriteByte('.')649		default:650			b.WriteString(regexp.QuoteMeta(string(r)))651		}652	}653	b.WriteByte('$')654	return b.String()655}656657// setupCountRules parses CountAsPattern into CountRules, compiles each rule and658// registers a cloned language under its new name so counting works. Invalid659// rules are reported to stderr and skipped, consistent with --count-as.660func setupCountRules() {661	for _, s := range CountAsPattern {662		rule, err := parseCountAsPattern(s)663		if err != nil {664			printError(fmt.Sprintf("ignoring malformed count-as-pattern rule %q: %s", s, err))665			continue666		}667		CountRules = append(CountRules, rule)668	}669670	for _, rule := range CountRules {671		base, ok := resolveBaseLanguage(rule.BaseLanguage)672		if !ok {673			printError(fmt.Sprintf("ignoring count-as-pattern rule for %q: base language %q is not a known language or extension", rule.Name, rule.BaseLanguage))674			continue675		}676677		source := rule.Pattern678		if rule.Engine == MatchGlob {679			source = globToRegex(rule.Pattern)680		}681682		re, err := regexp.Compile(source)683		if err != nil {684			printError(fmt.Sprintf("ignoring count-as-pattern rule for %q: invalid pattern %q: %s", rule.Name, rule.Pattern, err))685			continue686		}687688		// Clone the base language under the new name so it has counting rules,689		// clearing the matchers so the minted category never participates in690		// normal extension/filename/shebang detection.691		cloned := languageDatabase[base]692		cloned.Extensions = nil693		cloned.FileNames = nil694		cloned.SheBangs = nil695		languageDatabase[rule.Name] = cloned696697		// Populate features now in non-lazy mode, otherwise LoadLanguageFeature698		// will build them on first use since the name is in languageDatabase.699		if !isLazy {700			processLanguageFeature(rule.Name, cloned)701		}702703		compiledCountRules = append(compiledCountRules, compiledCountRule{re: re, name: rule.Name})704		printDebugF("set to count path matching %q as new language %s based on %s", rule.Pattern, rule.Name, base)705	}706}707708// LoadLanguageFeature will load a single feature as requested given the name709func LoadLanguageFeature(loadName string) {710	if !isLazy {711		return712	}713714	// Check if already loaded and if so return because we don't need to do it again715	LanguageFeaturesMutex.Lock()716	_, ok := LanguageFeatures[loadName]717	LanguageFeaturesMutex.Unlock()718	if ok {719		return720	}721722	// A plain map lookup, not a range-with-break: ranging leaves value set to723	// whatever the iteration happened to land on last when loadName is absent,724	// which would register some arbitrary language's comment and complexity725	// rules under this name. An unknown name has no features to build, so a726	// miss must be a no-op — CountStats then treats the file as plain text.727	value, ok := languageDatabase[loadName]728	if !ok {729		return730	}731732	startTime := makeTimestampNano()733	processLanguageFeature(loadName, value)734	printTraceF("nanoseconds to build language %s features: %d", loadName, makeTimestampNano()-startTime)735}736737// caseSpellings returns the tokens as written, and for a language that reads738// its keywords in any case, every other spelling of them too. Batch and ASP739// open a comment with REM, and mean it however it is typed.740//741// Expanding here rather than folding case at match time keeps the counting loop742// exactly as it is: it walks a trie of bytes and knows nothing about letters. A743// token of n letters becomes 2^n entries, so the expansion is refused past a744// length that would be silly, which no comment token comes near.745func caseSpellings(tokens []string, caseInsensitive bool) []string {746	if !caseInsensitive {747		return tokens748	}749750	const maxLetters = 8751752	spellings := make([]string, 0, len(tokens))753	for _, token := range tokens {754		letters := 0755		for i := 0; i < len(token); i++ {756			if token[i] != asciiLower(token[i]) || token[i] != asciiUpper(token[i]) {757				letters++758			}759		}760761		if letters == 0 || letters > maxLetters {762			spellings = append(spellings, token)763			continue764		}765766		variants := []string{""}767		for i := 0; i < len(token); i++ {768			lower, upper := asciiLower(token[i]), asciiUpper(token[i])769			next := make([]string, 0, len(variants)*2)770			for _, prefix := range variants {771				next = append(next, prefix+string(lower))772				if upper != lower {773					next = append(next, prefix+string(upper))774				}775			}776			variants = next777		}778		spellings = append(spellings, variants...)779	}780781	return spellings782}783784func asciiLower(b byte) byte {785	if b >= 'A' && b <= 'Z' {786		return b + 'a' - 'A'787	}788789	return b790}791792func asciiUpper(b byte) byte {793	if b >= 'a' && b <= 'z' {794		return b - 'a' + 'A'795	}796797	return b798}799800func processLanguageFeature(name string, value Language) {801	complexityTrie := &Trie{}802	slCommentTrie := &Trie{}803	mlCommentTrie := &Trie{}804	stringTrie := &Trie{}805	tokenTrie := &Trie{}806	keywordBytes := make([][]byte, 0, len(value.Keywords))807	postfixExcludes := make([][]byte, 0, len(value.ComplexityChecksPostfixExcludes))808809	complexityMask := byte(0)810	singleLineCommentMask := byte(0)811	multiLineCommentMask := byte(0)812	stringMask := byte(0)813	processMask := byte(0)814815	for _, v := range value.ComplexityChecks {816		complexityMask |= v[0]817		complexityTrie.Insert(TComplexity, []byte(v))818		if !Complexity {819			tokenTrie.Insert(TComplexity, []byte(v))820		}821	}822	if !Complexity {823		processMask |= complexityMask824	}825826	for _, v := range value.ComplexityChecksPostfix {827		if !Complexity {828			tokenTrie.Insert(TComplexityPostfix, []byte(v))829			processMask |= v[0]830		}831	}832833	for _, v := range value.ComplexityChecksPostfixExcludes {834		postfixExcludes = append(postfixExcludes, []byte(v))835	}836837	for _, v := range caseSpellings(value.LineComment, value.CaseInsensitive) {838		singleLineCommentMask |= v[0]839		slCommentTrie.Insert(TSlcomment, []byte(v))840		tokenTrie.Insert(TSlcomment, []byte(v))841	}842	processMask |= singleLineCommentMask843844	for _, v := range value.MultiLine {845		multiLineCommentMask |= v[0][0]846		mlCommentTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))847		tokenTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))848	}849	processMask |= multiLineCommentMask850851	for _, v := range value.Quotes {852		stringMask |= v.Start[0]853		stringTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))854		tokenTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))855	}856	processMask |= stringMask857858	for _, v := range value.Keywords {859		keywordBytes = append(keywordBytes, []byte(v))860	}861862	// Compile any regex heuristics used to disambiguate shared extensions such863	// as .h between C / C++ / Objective-C. The patterns are validated at864	// generation time (scripts/include.go) so MustCompile is safe here, but we865	// guard with Compile anyway to honour the no-panics policy. Each pattern's866	// necessary literals are pre-converted to bytes so guessByHeuristics can867	// cheaply skip running the regex when none of them appear in the content.868	heuristics := make([]CompiledHeuristic, 0, len(value.Heuristics))869	for _, v := range value.Heuristics {870		re, err := regexp.Compile(v.Pattern)871		if err != nil {872			printWarnF("failed to compile heuristic %q for language %s: %v", v.Pattern, name, err)873			continue874		}875		literals := make([][]byte, 0, len(v.Literals))876		for _, l := range v.Literals {877			literals = append(literals, []byte(l))878		}879		heuristics = append(heuristics, CompiledHeuristic{Re: re, Literals: literals, Anchored: v.Anchored})880	}881882	// A line comment spelled as a word ends where the word does, which costs a883	// check on every token that matches. Almost no language has one, so work out884	// here whether this one does and let the hot loop skip the check entirely.885	escape := byte('\\')886	if len(value.Escape) != 0 {887		escape = value.Escape[0]888	}889890	// The exact set of bytes the counting loop has to stop on, which is every891	// byte that opens a token plus the newline and the nul. Built here once so892	// the loop pays a single load and branch per byte instead of a mask test893	// that nearly always passes followed by a failed trie walk.894	tokenFirst := newTokenFirst()895	for i := range tokenTrie.Table {896		if tokenTrie.Table[i] != nil {897			tokenFirst[i] = true898		}899	}900901	wordComments := false902	for _, token := range value.LineComment {903		if len(token) > 1 && isIdentifierContinue(token[len(token)-1]) {904			wordComments = true905			break906		}907	}908909	LanguageFeaturesMutex.Lock()910	LanguageFeatures[name] = LanguageFeature{911		Complexity:            complexityTrie,912		MultiLineComments:     mlCommentTrie,913		MultiLine:             value.MultiLine,914		SingleLineComments:    slCommentTrie,915		LineComment:           value.LineComment,916		Strings:               stringTrie,917		Tokens:                tokenTrie,918		Nested:                value.NestedMultiLine,919		LineSplice:            value.LineSplice,920		WordComments:          wordComments,921		CommentIsWord:         value.CommentIsWord,922		Escape:                escape,923		PostfixExcludes:       postfixExcludes,924		ComplexityCheckMask:   complexityMask,925		MultiLineCommentMask:  multiLineCommentMask,926		SingleLineCommentMask: singleLineCommentMask,927		StringCheckMask:       stringMask,928		ProcessMask:           processMask,929		TokenFirst:            tokenFirst,930		Keywords:              value.Keywords,931		KeywordBytes:          keywordBytes,932		Heuristics:            heuristics,933		Quotes:                value.Quotes,934	}935	LanguageFeaturesMutex.Unlock()936}937938func processFlags() {939	// If wide/more mode is enabled we want the complexity calculation940	// to happen regardless as that is the only purpose of the flag941	if More && Complexity {942		Complexity = false943	}944945	// If ignore minified/generated is on ensure we turn on the code to calculate that946	if IgnoreMinifiedGenerate {947		MinifiedGenerated = true948		IgnoreMinified = true949		IgnoreGenerated = true950	}951952	if MinifiedGenerated {953		Minified = true954		Generated = true955	}956957	if IgnoreMinified {958		Minified = true959	}960961	if IgnoreGenerated {962		Generated = true963	}964965	if Dryness {966		UlocMode = true967	}968969	printDebugF("Path Deny List: %v", PathDenyList)970	printDebugF("Sort By: %s", SortBy)971	printDebugF("White List: %v", AllowListExtensions)972	printDebugF("Files Output: %t", Files)973	printDebugF("Verbose: %t", Verbose)974	printDebugF("Duplicates Detection: %t", Duplicates)975	printDebugF("Complexity Calculation: %t", !Complexity)976	printDebugF("Wide: %t", More)977	// If cost-comparison is enabled, turn on both COCOMO and LOCOMO978	if CostComparison {979		Cocomo = false980		Locomo = true981	}982983	// LOCOMO needs complexity data to produce accurate estimates.984	// If complexity was disabled via --no-complexity, force it back on.985	if Locomo && Complexity {986		Complexity = false987	}988989	// Cognitive complexity is derived from the complexity tokens, so it needs990	// complexity counting enabled. Complexity is a disable switch, so force it991	// off (i.e. enable counting) even when --no-complexity was passed; cognitive992	// wins. Warn only when the user explicitly asked for both.993	if Cognitive && Complexity {994		Complexity = false995		printWarn("--no-complexity ignored because --cognitive requires complexity counting")996	}997998	printDebugF("Average Wage: %d", AverageWage)999	printDebugF("Cocomo: %t", !Cocomo)1000	printDebugF("Locomo: %t", Locomo)1001	printDebugF("Minified/Generated Detection: %t/%t", Minified, Generated)1002	printDebugF("Ignore Minified/Generated: %t/%t", IgnoreMinified, IgnoreGenerated)1003	printDebugF("IncludeSymLinks: %t", IncludeSymLinks)1004	printDebugF("Uloc: %t", UlocMode)1005	printDebugF("Dryness: %t", Dryness)1006}10071008// LanguageDatabase provides access to the internal language database1009// useful for consuming applications wanting to consume and use1010func LanguageDatabase() map[string]Language {1011	return languageDatabase1012}10131014func PrintLanguages(dst io.Writer) {1015	names := make([]string, 0, len(languageDatabase))1016	for key := range languageDatabase {1017		names = append(names, key)1018	}10191020	slices.SortFunc(names, func(a, b string) int {1021		return strings.Compare(strings.ToLower(a), strings.ToLower(b))1022	})10231024	for _, name := range names {1025		_, _ = fmt.Fprintf(dst, "%s (%s)\n", name, strings.Join(append(languageDatabase[name].Extensions, languageDatabase[name].FileNames...), ","))1026	}1027}10281029// global variables to deal with ULOC calculations1030var ulocMutex = sync.Mutex{}1031var ulocGlobalCount = map[string]struct{}{}1032var ulocLanguageCount = map[string]map[string]struct{}{}10331034// Process is the main entry point of the command line it sets everything up and starts running1035func Process() {1036	if Languages {1037		PrintLanguages(os.Stdout)1038		return1039	}10401041	if ListCounters {1042		PrintCounters(os.Stdout)1043		return1044	}10451046	ProcessConstants()1047	processFlags()1048	cleanVisitedPaths()1049	cleanDuplicates()10501051	// Clean up any invalid arguments before setting everything up1052	if len(DirFilePaths) == 0 {1053		DirFilePaths = append(DirFilePaths, ".")1054	}10551056	// --report mode short-circuits the normal format dispatch and writes a1057	// self-contained HTML report. Mutually exclusive with --format / -f: if1058	// the user passed both, warn on stderr and let --report win.1059	if ReportOut != "" {1060		if Format != "" && Format != "tabular" {1061			fmt.Fprintf(os.Stderr, "warning: --report overrides --format=%s\n", Format)1062		}1063		parseReportSkip(ReportSkip)1064		if len(DirFilePaths) > 1 {1065			fmt.Fprintf(os.Stderr, "warning: --report only analyses the first positional path (%s); other paths ignored\n", DirFilePaths[0])1066		}1067		if err := runReport(DirFilePaths); err != nil {1068			fmt.Fprintln(os.Stderr, err)1069			os.Exit(1)1070		}1071		return1072	}10731074	if Hotspots && (ByAuthor || Timeline) {1075		fmt.Fprintln(os.Stderr, "--hotspots is mutually exclusive with --by-author / --timeline; pick one report")1076		os.Exit(1)1077	}10781079	// --coupling-for implies the coupling report for a specific file, and1080	// --coupling-weighted is a modifier that implies the report too.1081	if CouplingFor != "" || CouplingWeighted {1082		Coupling = true1083	}10841085	// Coupling is a standalone report — it doesn't combine with any other.1086	if Coupling && (Hotspots || ByAuthor || Timeline) {1087		fmt.Fprintln(os.Stderr, "--coupling/--coupling-for is mutually exclusive with --hotspots / --by-author / --timeline; pick one report")1088		os.Exit(1)1089	}10901091	if Hotspots || Coupling || ByAuthor || Timeline {1092		if err := validateHistoryFlags(os.Stderr); err != nil {1093			fmt.Fprintln(os.Stderr, err)1094			os.Exit(1)1095		}1096	}10971098	if Hotspots {1099		if err := runHotspotsReport(DirFilePaths[0]); err != nil {1100			fmt.Fprintln(os.Stderr, err)1101			os.Exit(1)1102		}1103		return1104	}11051106	if Coupling {1107		if err := runCouplingReport(DirFilePaths[0]); err != nil {1108			fmt.Fprintln(os.Stderr, err)1109			os.Exit(1)1110		}1111		return1112	}11131114	if ByAuthor && Timeline {1115		if err := runAuthorTimelineReport(DirFilePaths[0]); err != nil {1116			fmt.Fprintln(os.Stderr, err)1117			os.Exit(1)1118		}1119		return1120	}11211122	if ByAuthor {1123		if err := runAuthorsReport(DirFilePaths[0]); err != nil {1124			fmt.Fprintln(os.Stderr, err)1125			os.Exit(1)1126		}1127		return1128	}11291130	if Timeline {1131		if err := runLanguagesTimelineReport(DirFilePaths[0]); err != nil {1132			fmt.Fprintln(os.Stderr, err)1133			os.Exit(1)1134		}1135		return1136	}11371138	filePaths := []string{}1139	dirPaths := []string{}11401141	// Check if the paths or files added exist and exit if not1142	for _, f := range DirFilePaths {1143		fpath := filepath.Clean(f)11441145		s, err := os.Stat(fpath)1146		if err != nil {1147			fmt.Println("file or directory could not be read: " + fpath)1148			os.Exit(1)1149		}11501151		if s.IsDir() {1152			dirPaths = append(dirPaths, fpath)1153		} else {1154			filePaths = append(filePaths, fpath)1155		}1156	}11571158	SortBy = strings.ToLower(SortBy)1159	ctx := processorContext{remap: newRemapConfig(RemapAll, RemapUnknown)}11601161	printDebugF("NumCPU: %d", runtime.NumCPU())1162	printDebugF("SortBy: %s", SortBy)1163	printDebugF("PathDenyList: %v", PathDenyList)11641165	potentialFilesQueue := make(chan []*gocodewalker.File, FileListQueueSize) // files that pass the .gitignore checks1166	fileListQueue := make(chan *FileJob, FileListQueueSize)                   // Files ready to be read from disk1167	fileSummaryJobQueue := make(chan *FileJob, FileSummaryJobQueueSize)       // Files ready to be summarised11681169	fileWalker := gocodewalker.NewParallelFileWalker(dirPaths, nil)1170	fileWalker.SetFileBatchQueue(potentialFilesQueue)1171	fileWalker.SetErrorHandler(func(e error) bool {1172		printError(e.Error())1173		return true1174	})1175	fileWalker.IgnoreGitIgnore = GitIgnore1176	fileWalker.IgnoreIgnoreFile = Ignore1177	fileWalker.IgnoreGitModules = GitModuleIgnore1178	fileWalker.IncludeHidden = true1179	fileWalker.ExcludeDirectory = PathDenyList1180	fileWalker.SetConcurrency(DirectoryWalkerJobWorkers)11811182	if !SccIgnore {1183		fileWalker.CustomIgnore = []string{".sccignore"}1184	}1185	fileWalker.CustomIgnoreFiles = IgnoreFiles11861187	var excludePathRegexes []*regexp.Regexp1188	for _, exclude := range Exclude {1189		regexpResult, err := regexp.Compile(exclude)1190		if err == nil {1191			fileWalker.ExcludeFilenameRegex = append(fileWalker.ExcludeFilenameRegex, regexpResult)1192			fileWalker.ExcludeDirectoryRegex = append(fileWalker.ExcludeDirectoryRegex, regexpResult)1193			excludePathRegexes = append(excludePathRegexes, regexpResult)1194		} else {1195			printError(err.Error())1196		}1197	}11981199	go func() {1200		err := fileWalker.Start()1201		if err != nil {1202			printError(err.Error())1203		}1204	}()12051206	startFileJobProducer(potentialFilesQueue, filePaths, excludePathRegexes, fileListQueue)12071208	// Where the output is one of the summaries that only ever prints per1209	// language totals, the workers add up their own results and merge once each1210	// at the end. That deletes a channel send and a channel receive per file1211	// from a stage where the receiver is a single goroutine, which is the1212	// difference between 86,000 handovers and one per worker.1213	if summariseInWorkers() {1214		ctx.summary = newSharedSummaryTotals()1215		workerSummary = ctx.summary1216	}12171218	go ctx.fileProcessorWorker(fileListQueue, fileSummaryJobQueue)12191220	result := fileSummarize(fileSummaryJobQueue)1221	if FileOutput == "" {1222		fmt.Print(result)1223	} else {1224		// A failed write must not report success: follow the --report path1225		// above and exit non-zero with the reason on stderr.1226		if err := os.WriteFile(FileOutput, []byte(result), 0644); err != nil {1227			fmt.Fprintln(os.Stderr, err)1228			os.Exit(1)1229		}1230		fmt.Println("results written to " + FileOutput)1231	}1232}