1// SPDX-License-Identifier: MIT23package processor45import (6	"fmt"7	"io"8	"os"9	"path/filepath"10	"regexp"11	"runtime"12	"runtime/debug"13	"slices"14	"strconv"15	"strings"16	"sync"1718	"github.com/boyter/gocodewalker"19)2021// Version indicates the version of the application22var Version = "4.0.0"2324// Flags set via the CLI which control how the output is displayed2526// Files indicates if there should be file output or not when formatting27var Files = false2829// Languages indicates if the command line should print out the supported languages30var Languages = false3132// Verbose enables verbose logging output33var Verbose = false3435// Debug enables debug logging output36var Debug = false3738// Trace enables trace logging output which is extremely verbose39var Trace = false4041// Duplicates enables duplicate file detection42var Duplicates = false4344// MinifiedGenerated enables minified/generated file detection45var MinifiedGenerated = false4647// IgnoreMinifiedGenerate printing counts for minified/generated files48var IgnoreMinifiedGenerate = false4950// MinifiedGeneratedLineByteLength number of bytes per average line to determine file is minified/generated51var MinifiedGeneratedLineByteLength = 2555253// Minified enables minified file detection54var Minified = false5556// IgnoreMinified ignore printing counts for minified files57var IgnoreMinified = false5859// Generated enables generated file detection60var Generated = false6162// GeneratedMarkers defines head markers for generated file detection63var GeneratedMarkers []string6465// IgnoreGenerated ignore printing counts for generated files66var IgnoreGenerated = false6768// Complexity toggles complexity calculation69var Complexity = false7071// Cognitive toggles cognitive (nesting-weighted) complexity calculation72var Cognitive = false7374// More enables wider output with more information in formatter75var More = false7677// Cocomo toggles the COCOMO calculation78var Cocomo = false7980// SLOCCountFormat prints a more SLOCCount like COCOMO calculation81var SLOCCountFormat = false8283// CocomoProjectType allows the flipping between project types which impacts the calculation84var CocomoProjectType = "organic"8586// Size toggles the Size calculation87var Size = false8889// Draw horizontal borders between sections.90var HBorder = false9192// SizeUnit determines what size calculation is used for megabytes93var SizeUnit = "si"9495// Ci indicates if running inside a CI so to disable box drawing characters96var Ci = false9798// GitIgnore disables .gitignore checks99var GitIgnore = false100101// GitModuleIgnore disables .gitmodules checks102var GitModuleIgnore = false103104// Ignore disables ignore file checks105var Ignore = false106107// SccIgnore disables sccignore file checks108var SccIgnore = false109110// CountIgnore should we count ignore files?111var CountIgnore = false112113// CountUnsupported when set counts files scc does not recognise under an114// "Unknown" category, treating them as plain text. See issue #464.115var CountUnsupported = false116117// IgnoreFiles are paths to additional ignore files supplied via --ignore-file.118// They are applied as a low priority base layer in the order supplied so a later119// file can override an earlier one, and any in-tree .gitignore/.ignore/.sccignore120// discovered while walking overrides all of them.121var IgnoreFiles = []string{}122123// DisableCheckBinary toggles checking for binary files using NUL bytes124var DisableCheckBinary = false125126// UlocMode toggles checking for binary files using NUL bytes127var UlocMode = false128129// Percent toggles checking for binary files using NUL bytes130var Percent = false131132// MaxMean sets the calculation of the max and mean line length133var MaxMean = false134135// Dryness toggles checking for binary files using NUL bytes136var Dryness = false137138// SortBy sets which column output in formatter should be sorted by139var SortBy = ""140141// Exclude is a regular expression which is used to exclude files from being processed142var Exclude = []string{}143144// CountAs is a rule for mapping known or new extensions to other rules145var CountAs = ""146147// Format sets the output format of the formatter148var Format = ""149150// FormatMulti is a rule for defining multiple output formats151var FormatMulti = ""152153// SQLProject is used to store the name for the SQL insert formats but is optional154var SQLProject = ""155156// RemapUnknown allows remapping of unknown files with a string to search the content for157var RemapUnknown = ""158159// RemapAll allows remapping of all files with a string to search the content for160var RemapAll = ""161162type remapRule struct {163	pattern  []byte164	language string165}166167type remapConfig struct {168	all     []remapRule169	unknown []remapRule170}171172type processorContext struct {173	remap remapConfig174}175176func parseRemapRules(value string) []remapRule {177	rules := []remapRule{}178179	for s := range strings.SplitSeq(value, ",") {180		t := strings.Split(s, ":")181		if len(t) == 2 {182			rules = append(rules, remapRule{183				pattern:  []byte(t[0]),184				language: t[1],185			})186		}187	}188189	return rules190}191192func newRemapConfig(remapAll string, remapUnknown string) remapConfig {193	c := remapConfig{194		all:     parseRemapRules(remapAll),195		unknown: parseRemapRules(remapUnknown),196	}197198	// Load the features for every language a rule can remap to. Remapping only199	// sets job.Language, and in lazy mode (which is every CLI run) nothing else200	// guarantees that language was ever loaded — CountStats would then find no201	// features and count the file as plain text, with no comments and no202	// complexity. Done once here at setup rather than in the remap functions203	// themselves, which run per file on the hot path.204	for _, rule := range c.all {205		LoadLanguageFeature(rule.language)206	}207	for _, rule := range c.unknown {208		LoadLanguageFeature(rule.language)209	}210211	return c212}213214// MatchEngine selects how a CountRule pattern is interpreted. Glob is the215// default; regex is opt-in via the re: prefix.216type MatchEngine int217218const (219	// MatchGlob is the default. The pattern is a glob ('*' and '?') translated220	// to an anchored regex and matched as a full match against the path.221	MatchGlob MatchEngine = iota222	// MatchRegex treats the pattern as a raw (unanchored) RE2 regex. Opt in223	// with the re: prefix.224	MatchRegex225)226227// CountRule is the typed, library-facing form of a --count-as-pattern rule.228// It matches files by their path and relabels them to a new named category229// whose counting rules are cloned from an existing base language.230type CountRule struct {231	Engine       MatchEngine // MatchGlob (the default) or MatchRegex232	Pattern      string      // glob or regex source233	Name         string      // new category display name234	BaseLanguage string      // existing language whose counting rules are cloned235}236237// CountRules is the typed input set either directly by library users or by the238// CLI after parsing CountAsPattern. Setup happens in setupCountRules.239var CountRules []CountRule240241// CountAsPattern holds the raw repeatable --count-as-pattern flag values. Each242// is parsed into a CountRule at setup. Library users may set CountRules directly.243var CountAsPattern []string244245// compiledCountRule is the runtime form scanned by newFileJob246type compiledCountRule struct {247	re   *regexp.Regexp248	name string249}250251var compiledCountRules []compiledCountRule252253// CurrencySymbol allows setting the currency symbol for cocomo project cost estimation254var CurrencySymbol = ""255256// FileOutput sets the file that output should be written to257var FileOutput = ""258259// PathDenyList sets the paths that should be skipped260var PathDenyList = []string{}261262// FileListQueueSize is the queue of files found and ready to be read into memory263var FileListQueueSize = runtime.NumCPU()264265// FileProcessJobWorkers is the number of workers that process the file collecting stats266var FileProcessJobWorkers = runtime.NumCPU() * 4267268// FileSummaryJobQueueSize is the queue used to hold processed file statistics before formatting269var FileSummaryJobQueueSize = runtime.NumCPU()270271// DirectoryWalkerJobWorkers is the number of workers which will walk the directory tree272var DirectoryWalkerJobWorkers = 8273274// AllowListExtensions is a list of extensions which are allowed to be processed275var AllowListExtensions = []string{}276277// ExcludeListExtensions is a list of extensions which should be ignored278var ExcludeListExtensions = []string{}279280// ExcludeFilename is a list of filenames which should be ignored281var ExcludeFilename = []string{}282283// AverageWage is the average wage in dollars used for the COCOMO cost estimate284var AverageWage int64 = 56286285286// Overhead is the overhead multiplier for corporate overhead (facilities, equipment, accounting, etc.)287var Overhead float64 = 2.4288289// EAF is the effort adjustment factor derived from the cost drivers, i.e. 1.0 if rated nominal290var EAF float64 = 1.0291292// Locomo toggles the LOCOMO (LLM Output COst MOdel) calculation293var Locomo = false294295// CostComparison enables both COCOMO and LOCOMO output for side-by-side comparison296var CostComparison = false297298// LocomoPresetName is the LLM model preset for pricing and throughput defaults299var LocomoPresetName = "medium"300301// LocomoInputPrice is the cost per 1M input tokens (overrides preset)302var LocomoInputPrice float64303var LocomoInputPriceSet = false304305// LocomoOutputPrice is the cost per 1M output tokens (overrides preset)306var LocomoOutputPrice float64307var LocomoOutputPriceSet = false308309// LocomoTPS is the output tokens per second (overrides preset)310var LocomoTPS float64311var LocomoTPSSet = false312313// LocomoReviewMinutesPerLine is the human review time per line of code in minutes314var LocomoReviewMinutesPerLine float64 = 0.01315316// LocomoConfig is the power-user config string "tokensPerLine,baseInputPerLine,complexityWeight,iterations,iterationWeight"317var LocomoConfig = ""318319// LocomoTokensPerLine is the average number of output tokens per line of code320var LocomoTokensPerLine float64 = 10321322// LocomoBaseInputPerLine is the base number of input tokens per output line323var LocomoBaseInputPerLine float64 = 20324325// LocomoComplexityWeight is the scaling weight applied to sqrt(complexity density) for input tokens326var LocomoComplexityWeight float64 = 5327328// LocomoIterations is the base number of iteration/retry attempts329var LocomoIterations float64 = 1.5330331// LocomoIterationWeight is the scaling weight for complexity-driven retries332var LocomoIterationWeight float64 = 2333334// LocomoCyclesOverride is the user-supplied iteration factor override (--locomo-cycles)335var LocomoCyclesOverride float64336337// LocomoCyclesSet indicates whether --locomo-cycles was explicitly set338var LocomoCyclesSet = false339340// GcFileCount is the number of files to process before turning the GC back on341var GcFileCount = 10000342var gcPercent = -1343var isLazy = false344345// NoLarge if set true will ignore files over a certain number of lines or bytes346var NoLarge = false347348// IncludeSymLinks if set true will count symlink files349var IncludeSymLinks = false350351// LargeLineCount number of lines before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44352var LargeLineCount int64 = 40000353354// LargeByteCount number of bytes before being counted as a large file based on https://github.com/pinpt/ripsrc/blob/master/ripsrc/fileinfo/fileinfo.go#L44355var LargeByteCount int64 = 1000000356357// Hotspots toggles the hotspots git-history report358var Hotspots = false359360// Coupling toggles the change-coupling git-history report (file pairs that361// change together)362var Coupling = false363364// CouplingFor, when non-empty, switches the coupling report to the365// file-oriented "blast radius" view for the given path: what tends to change366// when that file changes.367var CouplingFor = ""368369// CouplingWeighted ranks coupling by degree × the pair's (smaller) file370// complexity instead of raw co-change, so pairs of genuinely complex files371// outrank generated/data-file churn. Implies Coupling. Honours the Cognitive372// global for its complexity source, matching --hotspots.373var CouplingWeighted = false374375// ByAuthor toggles the author-rollup git-history report376var ByAuthor = false377378// Timeline selects an over-time view. With ByAuthor, runs the author379// timeline report (plan 04); alone, runs the languages-over-time report380// (plan 05). With Hotspots set, the combination errors out.381var Timeline = false382383// HistoryBuckets is the time-bucket resolution for the timeline reports.384// Wired to --buckets in main.go; default 60.385var HistoryBuckets = 60386387// FoldAuthors enables the name+domain identity folding fallback applied388// after the mailmap. Toggled off via --no-fold-authors.389var FoldAuthors = true390391// DirFilePaths is not set via flags but by arguments following the flags for file or directory to process392var DirFilePaths = []string{}393394// ExtensionToLanguage is loaded from the JSON that is in constants.go395var ExtensionToLanguage = map[string][]string{}396397// ShebangLookup loaded from the JSON in constants.go contains shebang lookups398var ShebangLookup = map[string][]string{}399400// FilenameToLanguage similar to ExtensionToLanguage loaded from the JSON in constants.go401var FilenameToLanguage = map[string]string{}402403// LanguageFeatures contains the processed languages from processLanguageFeature404var LanguageFeatures = map[string]LanguageFeature{}405406// LanguageFeaturesMutex is the shared mutex used to control getting and setting of language features407// used rather than sync.Map because it turned out to be marginally faster408var LanguageFeaturesMutex = sync.Mutex{}409410// Start time in milli seconds in case we want the total time411var startTimeMilli = makeTimestampMilli()412413// ConfigureGc needs to be set outside of ProcessConstants because it should only be enabled in command line414// mode https://github.com/boyter/scc/issues/32415func ConfigureGc() {416	gcPercent = debug.SetGCPercent(gcPercent)417}418419// EnableGc restores the garbage collector to the percentage captured by ConfigureGc.420func EnableGc() {421	if gcPercent != -1 {422		debug.SetGCPercent(gcPercent)423	}424}425426// ConfigureLazy is a simple setter used to turn on lazy loading used only by command line427func ConfigureLazy(lazy bool) {428	isLazy = lazy429}430431// ProcessConstants is responsible for setting up the language features based on the JSON file that is stored in constants432// Needs to be called at least once in order for anything to actually happen433func ProcessConstants() {434	startTime := makeTimestampNano()435436	// Reset the reverse-lookup maps so ProcessConstants is idempotent. The437	// ExtensionToLanguage entries are built with append, so without clearing438	// first a repeated call (the long-lived MCP server invokes ProcessConstants439	// once per tool call) would accumulate duplicate languages for every440	// extension. scc was historically a one-shot CLI where this ran exactly441	// once, so it never surfaced until server mode.442	clear(ExtensionToLanguage)443	clear(FilenameToLanguage)444	clear(ShebangLookup)445446	for name, value := range languageDatabase {447		for _, ext := range value.Extensions {448			ExtensionToLanguage[ext] = append(ExtensionToLanguage[ext], name)449		}450451		for _, fname := range value.FileNames {452			FilenameToLanguage[fname] = name453		}454455		if len(value.SheBangs) != 0 {456			ShebangLookup[name] = value.SheBangs457		}458	}459460	// If we have anything in CountAs set it up now461	if len(CountAs) != 0 {462		setupCountAs()463	}464465	printTraceF("nanoseconds build extension to language: %d", makeTimestampNano()-startTime)466467	// Set up any path pattern count rules, minting new categories backed by a468	// base language. The function clones the base language and builds its469	// features so counting works in both lazy and non-lazy modes.470	if len(CountAsPattern) != 0 || len(CountRules) != 0 {471		setupCountRules()472	}473474	// Configure COCOMO setting. projectType is keyed by the canonical475	// lowercase name, so when the selection matches a built-in type we must476	// normalize CocomoProjectType to that lowercase key — otherwise the477	// membership check below passes for "Organic" while the real lookups in478	// EstimateEffort/EstimateScheduleMonths index projectType["Organic"], hit479	// a nil slice and panic. See `scc --cocomo-project-type Organic`.480	if _, ok := projectType[strings.ToLower(CocomoProjectType)]; ok {481		CocomoProjectType = strings.ToLower(CocomoProjectType)482	} else {483		// let's see if we can turn it into a custom one484		spl := strings.Split(CocomoProjectType, ",")485		val := []float64{}486		if len(spl) == 5 {487			// let's try to convert to float if we can488			for i := 1; i < 5; i++ {489				f, err := strconv.ParseFloat(spl[i], 64)490				if err == nil {491					val = append(val, f)492				}493			}494		}495496		if len(val) == 4 {497			projectType[CocomoProjectType] = val498		} else {499			// if nothing matches fall back to organic500			CocomoProjectType = "organic"501		}502	}503504	// If lazy is set then we want to load in the features as we find them not in one go505	// however otherwise being used as a library so just load them all in506	if !isLazy {507		startTime = makeTimestampMilli()508		for name, value := range languageDatabase {509			processLanguageFeature(name, value)510		}511512		printTraceF("milliseconds build language features: %d", makeTimestampMilli()-startTime)513	} else {514		printTrace("configured to lazy load language features")515	}516517	// Fix for https://github.com/boyter/scc/issues/250518	fixedPath := make([]string, 0, len(PathDenyList))519	for _, path := range PathDenyList {520		fixedPath = append(fixedPath, strings.TrimRight(path, "/"))521	}522	PathDenyList = fixedPath523}524525// Configure and setup any count-as params the use has supplied526func setupCountAs() {527	for s := range strings.SplitSeq(CountAs, ",") {528		t := strings.Split(s, ":")529		if len(t) != 2 {530			printError(fmt.Sprintf("ignoring malformed count-as rule %q: expected format <from>:<to>", s))531			continue532		}533534		// There are two cases here.535		// first is they provide the name e.g. "Cargo Lock"536		// second is that the user supplies the extension EG wsdl537		// we should support BOTH cases538		// always remember we only need to validate t[1] as that's the one539		// that tells us where we are trying to map540		target, ok := resolveBaseLanguage(t[1])541		if ok {542			ExtensionToLanguage[strings.ToLower(t[0])] = []string{target}543			printDebugF("set to count extension: %s as language %s", t[0], target)544			continue545		}546547		// The target t[1] matched neither a known language name nor a known548		// extension, so no mapping was registered. Warn rather than silently549		// ignoring the rule, since count-as cannot mint new categories yet.550		printError(fmt.Sprintf("ignoring count-as rule %q: target %q is not a known language or extension", s, t[1]))551	}552}553554// resolveBaseLanguage resolves a user supplied target to a canonical language555// name. It first tries to match a language name (most reliable as names are556// unique) and falls back to matching a known extension. Returns the canonical557// language name and whether it was resolved.558func resolveBaseLanguage(target string) (string, bool) {559	// Match by language name which is the most reliable as the name is unique560	for name := range languageDatabase {561		if strings.EqualFold(name, target) {562			return name, true563		}564	}565566	// Fall back to extension match, note this is less reliable as some567	// languages share extensions so we take the first registered language568	langs, ok := ExtensionToLanguage[strings.ToLower(target)]569	if ok && len(langs) != 0 {570		return langs[0], true571	}572573	return "", false574}575576// parseCountAsPattern parses a single --count-as-pattern rule of the form577// [engine:]pattern:name:baselang into a CountRule.578//579// The engine prefix is optional and the pattern is treated as a GLOB BY580// DEFAULT; prefix with re: to opt into a regex (or glob: to be explicit). We581// keep glob and regex as distinct modes rather than inferring, because the same582// string is valid in both engines with different meaning (e.g. "foo.rb" matches583// only foo.rb as a glob but also fooXrb as a regex), so guessing would silently584// match the wrong files.585//586// Because regex patterns and paths legitimately contain ':', name and baselang587// are peeled from the right and the pattern is whatever remains in between.588func parseCountAsPattern(s string) (CountRule, error) {589	engine := MatchGlob590	rest := s591592	switch {593	case strings.HasPrefix(rest, "re:"):594		engine = MatchRegex595		rest = rest[len("re:"):]596	case strings.HasPrefix(rest, "glob:"):597		engine = MatchGlob598		rest = rest[len("glob:"):]599	}600601	// baselang = after the last ':', name = between the 2nd-last and last ':'602	lastColon := strings.LastIndex(rest, ":")603	if lastColon == -1 {604		return CountRule{}, fmt.Errorf("expected format [engine:]pattern:name:baselang")605	}606	baseLanguage := rest[lastColon+1:]607608	nameColon := strings.LastIndex(rest[:lastColon], ":")609	if nameColon == -1 {610		return CountRule{}, fmt.Errorf("expected format [engine:]pattern:name:baselang")611	}612	name := rest[nameColon+1 : lastColon]613	pattern := rest[:nameColon]614615	if pattern == "" || name == "" || baseLanguage == "" {616		return CountRule{}, fmt.Errorf("pattern, name and baselang must all be non-empty")617	}618619	return CountRule{Engine: engine, Pattern: pattern, Name: name, BaseLanguage: baseLanguage}, nil620}621622// globToRegex converts a simple glob into an anchored regex. Glob is the623// default --count-as-pattern engine. Only '*' (any run of characters) and '?'624// (single character) are special, everything else is matched literally. The625// result is anchored as a full match.626func globToRegex(glob string) string {627	var b strings.Builder628	b.WriteByte('^')629	for _, r := range glob {630		switch r {631		case '*':632			b.WriteString(".*")633		case '?':634			b.WriteByte('.')635		default:636			b.WriteString(regexp.QuoteMeta(string(r)))637		}638	}639	b.WriteByte('$')640	return b.String()641}642643// setupCountRules parses CountAsPattern into CountRules, compiles each rule and644// registers a cloned language under its new name so counting works. Invalid645// rules are reported to stderr and skipped, consistent with --count-as.646func setupCountRules() {647	for _, s := range CountAsPattern {648		rule, err := parseCountAsPattern(s)649		if err != nil {650			printError(fmt.Sprintf("ignoring malformed count-as-pattern rule %q: %s", s, err))651			continue652		}653		CountRules = append(CountRules, rule)654	}655656	for _, rule := range CountRules {657		base, ok := resolveBaseLanguage(rule.BaseLanguage)658		if !ok {659			printError(fmt.Sprintf("ignoring count-as-pattern rule for %q: base language %q is not a known language or extension", rule.Name, rule.BaseLanguage))660			continue661		}662663		source := rule.Pattern664		if rule.Engine == MatchGlob {665			source = globToRegex(rule.Pattern)666		}667668		re, err := regexp.Compile(source)669		if err != nil {670			printError(fmt.Sprintf("ignoring count-as-pattern rule for %q: invalid pattern %q: %s", rule.Name, rule.Pattern, err))671			continue672		}673674		// Clone the base language under the new name so it has counting rules,675		// clearing the matchers so the minted category never participates in676		// normal extension/filename/shebang detection.677		cloned := languageDatabase[base]678		cloned.Extensions = nil679		cloned.FileNames = nil680		cloned.SheBangs = nil681		languageDatabase[rule.Name] = cloned682683		// Populate features now in non-lazy mode, otherwise LoadLanguageFeature684		// will build them on first use since the name is in languageDatabase.685		if !isLazy {686			processLanguageFeature(rule.Name, cloned)687		}688689		compiledCountRules = append(compiledCountRules, compiledCountRule{re: re, name: rule.Name})690		printDebugF("set to count path matching %q as new language %s based on %s", rule.Pattern, rule.Name, base)691	}692}693694// LoadLanguageFeature will load a single feature as requested given the name695func LoadLanguageFeature(loadName string) {696	if !isLazy {697		return698	}699700	// Check if already loaded and if so return because we don't need to do it again701	LanguageFeaturesMutex.Lock()702	_, ok := LanguageFeatures[loadName]703	LanguageFeaturesMutex.Unlock()704	if ok {705		return706	}707708	// A plain map lookup, not a range-with-break: ranging leaves value set to709	// whatever the iteration happened to land on last when loadName is absent,710	// which would register some arbitrary language's comment and complexity711	// rules under this name. An unknown name has no features to build, so a712	// miss must be a no-op — CountStats then treats the file as plain text.713	value, ok := languageDatabase[loadName]714	if !ok {715		return716	}717718	startTime := makeTimestampNano()719	processLanguageFeature(loadName, value)720	printTraceF("nanoseconds to build language %s features: %d", loadName, makeTimestampNano()-startTime)721}722723func processLanguageFeature(name string, value Language) {724	complexityTrie := &Trie{}725	slCommentTrie := &Trie{}726	mlCommentTrie := &Trie{}727	stringTrie := &Trie{}728	tokenTrie := &Trie{}729	keywordBytes := make([][]byte, 0, len(value.Keywords))730	postfixExcludes := make([][]byte, 0, len(value.ComplexityChecksPostfixExcludes))731732	complexityMask := byte(0)733	singleLineCommentMask := byte(0)734	multiLineCommentMask := byte(0)735	stringMask := byte(0)736	processMask := byte(0)737738	for _, v := range value.ComplexityChecks {739		complexityMask |= v[0]740		complexityTrie.Insert(TComplexity, []byte(v))741		if !Complexity {742			tokenTrie.Insert(TComplexity, []byte(v))743		}744	}745	if !Complexity {746		processMask |= complexityMask747	}748749	for _, v := range value.ComplexityChecksPostfix {750		if !Complexity {751			tokenTrie.Insert(TComplexityPostfix, []byte(v))752			processMask |= v[0]753		}754	}755756	for _, v := range value.ComplexityChecksPostfixExcludes {757		postfixExcludes = append(postfixExcludes, []byte(v))758	}759760	for _, v := range value.LineComment {761		singleLineCommentMask |= v[0]762		slCommentTrie.Insert(TSlcomment, []byte(v))763		tokenTrie.Insert(TSlcomment, []byte(v))764	}765	processMask |= singleLineCommentMask766767	for _, v := range value.MultiLine {768		multiLineCommentMask |= v[0][0]769		mlCommentTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))770		tokenTrie.InsertClose(TMlcomment, []byte(v[0]), []byte(v[1]))771	}772	processMask |= multiLineCommentMask773774	for _, v := range value.Quotes {775		stringMask |= v.Start[0]776		stringTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))777		tokenTrie.InsertClose(TString, []byte(v.Start), []byte(v.End))778	}779	processMask |= stringMask780781	for _, v := range value.Keywords {782		keywordBytes = append(keywordBytes, []byte(v))783	}784785	// Compile any regex heuristics used to disambiguate shared extensions such786	// as .h between C / C++ / Objective-C. The patterns are validated at787	// generation time (scripts/include.go) so MustCompile is safe here, but we788	// guard with Compile anyway to honour the no-panics policy. Each pattern's789	// necessary literals are pre-converted to bytes so guessByHeuristics can790	// cheaply skip running the regex when none of them appear in the content.791	heuristics := make([]CompiledHeuristic, 0, len(value.Heuristics))792	for _, v := range value.Heuristics {793		re, err := regexp.Compile(v.Pattern)794		if err != nil {795			printWarnF("failed to compile heuristic %q for language %s: %v", v.Pattern, name, err)796			continue797		}798		literals := make([][]byte, 0, len(v.Literals))799		for _, l := range v.Literals {800			literals = append(literals, []byte(l))801		}802		heuristics = append(heuristics, CompiledHeuristic{Re: re, Literals: literals, Anchored: v.Anchored})803	}804805	LanguageFeaturesMutex.Lock()806	LanguageFeatures[name] = LanguageFeature{807		Complexity:            complexityTrie,808		MultiLineComments:     mlCommentTrie,809		MultiLine:             value.MultiLine,810		SingleLineComments:    slCommentTrie,811		LineComment:           value.LineComment,812		Strings:               stringTrie,813		Tokens:                tokenTrie,814		Nested:                value.NestedMultiLine,815		PostfixExcludes:       postfixExcludes,816		ComplexityCheckMask:   complexityMask,817		MultiLineCommentMask:  multiLineCommentMask,818		SingleLineCommentMask: singleLineCommentMask,819		StringCheckMask:       stringMask,820		ProcessMask:           processMask,821		Keywords:              value.Keywords,822		KeywordBytes:          keywordBytes,823		Heuristics:            heuristics,824		Quotes:                value.Quotes,825	}826	LanguageFeaturesMutex.Unlock()827}828829func processFlags() {830	// If wide/more mode is enabled we want the complexity calculation831	// to happen regardless as that is the only purpose of the flag832	if More && Complexity {833		Complexity = false834	}835836	// If ignore minified/generated is on ensure we turn on the code to calculate that837	if IgnoreMinifiedGenerate {838		MinifiedGenerated = true839		IgnoreMinified = true840		IgnoreGenerated = true841	}842843	if MinifiedGenerated {844		Minified = true845		Generated = true846	}847848	if IgnoreMinified {849		Minified = true850	}851852	if IgnoreGenerated {853		Generated = true854	}855856	if Dryness {857		UlocMode = true858	}859860	printDebugF("Path Deny List: %v", PathDenyList)861	printDebugF("Sort By: %s", SortBy)862	printDebugF("White List: %v", AllowListExtensions)863	printDebugF("Files Output: %t", Files)864	printDebugF("Verbose: %t", Verbose)865	printDebugF("Duplicates Detection: %t", Duplicates)866	printDebugF("Complexity Calculation: %t", !Complexity)867	printDebugF("Wide: %t", More)868	// If cost-comparison is enabled, turn on both COCOMO and LOCOMO869	if CostComparison {870		Cocomo = false871		Locomo = true872	}873874	// LOCOMO needs complexity data to produce accurate estimates.875	// If complexity was disabled via --no-complexity, force it back on.876	if Locomo && Complexity {877		Complexity = false878	}879880	// Cognitive complexity is derived from the complexity tokens, so it needs881	// complexity counting enabled. Complexity is a disable switch, so force it882	// off (i.e. enable counting) even when --no-complexity was passed; cognitive883	// wins. Warn only when the user explicitly asked for both.884	if Cognitive && Complexity {885		Complexity = false886		printWarn("--no-complexity ignored because --cognitive requires complexity counting")887	}888889	printDebugF("Average Wage: %d", AverageWage)890	printDebugF("Cocomo: %t", !Cocomo)891	printDebugF("Locomo: %t", Locomo)892	printDebugF("Minified/Generated Detection: %t/%t", Minified, Generated)893	printDebugF("Ignore Minified/Generated: %t/%t", IgnoreMinified, IgnoreGenerated)894	printDebugF("IncludeSymLinks: %t", IncludeSymLinks)895	printDebugF("Uloc: %t", UlocMode)896	printDebugF("Dryness: %t", Dryness)897}898899// LanguageDatabase provides access to the internal language database900// useful for consuming applications wanting to consume and use901func LanguageDatabase() map[string]Language {902	return languageDatabase903}904905func PrintLanguages(dst io.Writer) {906	names := make([]string, 0, len(languageDatabase))907	for key := range languageDatabase {908		names = append(names, key)909	}910911	slices.SortFunc(names, func(a, b string) int {912		return strings.Compare(strings.ToLower(a), strings.ToLower(b))913	})914915	for _, name := range names {916		_, _ = fmt.Fprintf(dst, "%s (%s)\n", name, strings.Join(append(languageDatabase[name].Extensions, languageDatabase[name].FileNames...), ","))917	}918}919920// global variables to deal with ULOC calculations921var ulocMutex = sync.Mutex{}922var ulocGlobalCount = map[string]struct{}{}923var ulocLanguageCount = map[string]map[string]struct{}{}924925// Process is the main entry point of the command line it sets everything up and starts running926func Process() {927	if Languages {928		PrintLanguages(os.Stdout)929		return930	}931932	ProcessConstants()933	processFlags()934	cleanVisitedPaths()935	cleanDuplicates()936937	// Clean up any invalid arguments before setting everything up938	if len(DirFilePaths) == 0 {939		DirFilePaths = append(DirFilePaths, ".")940	}941942	// --report mode short-circuits the normal format dispatch and writes a943	// self-contained HTML report. Mutually exclusive with --format / -f: if944	// the user passed both, warn on stderr and let --report win.945	if ReportOut != "" {946		if Format != "" && Format != "tabular" {947			fmt.Fprintf(os.Stderr, "warning: --report overrides --format=%s\n", Format)948		}949		parseReportSkip(ReportSkip)950		if len(DirFilePaths) > 1 {951			fmt.Fprintf(os.Stderr, "warning: --report only analyses the first positional path (%s); other paths ignored\n", DirFilePaths[0])952		}953		if err := runReport(DirFilePaths); err != nil {954			fmt.Fprintln(os.Stderr, err)955			os.Exit(1)956		}957		return958	}959960	if Hotspots && (ByAuthor || Timeline) {961		fmt.Fprintln(os.Stderr, "--hotspots is mutually exclusive with --by-author / --timeline; pick one report")962		os.Exit(1)963	}964965	// --coupling-for implies the coupling report for a specific file, and966	// --coupling-weighted is a modifier that implies the report too.967	if CouplingFor != "" || CouplingWeighted {968		Coupling = true969	}970971	// Coupling is a standalone report — it doesn't combine with any other.972	if Coupling && (Hotspots || ByAuthor || Timeline) {973		fmt.Fprintln(os.Stderr, "--coupling/--coupling-for is mutually exclusive with --hotspots / --by-author / --timeline; pick one report")974		os.Exit(1)975	}976977	if Hotspots || Coupling || ByAuthor || Timeline {978		if err := validateHistoryFlags(os.Stderr); err != nil {979			fmt.Fprintln(os.Stderr, err)980			os.Exit(1)981		}982	}983984	if Hotspots {985		if err := runHotspotsReport(DirFilePaths[0]); err != nil {986			fmt.Fprintln(os.Stderr, err)987			os.Exit(1)988		}989		return990	}991992	if Coupling {993		if err := runCouplingReport(DirFilePaths[0]); err != nil {994			fmt.Fprintln(os.Stderr, err)995			os.Exit(1)996		}997		return998	}9991000	if ByAuthor && Timeline {1001		if err := runAuthorTimelineReport(DirFilePaths[0]); err != nil {1002			fmt.Fprintln(os.Stderr, err)1003			os.Exit(1)1004		}1005		return1006	}10071008	if ByAuthor {1009		if err := runAuthorsReport(DirFilePaths[0]); err != nil {1010			fmt.Fprintln(os.Stderr, err)1011			os.Exit(1)1012		}1013		return1014	}10151016	if Timeline {1017		if err := runLanguagesTimelineReport(DirFilePaths[0]); err != nil {1018			fmt.Fprintln(os.Stderr, err)1019			os.Exit(1)1020		}1021		return1022	}10231024	filePaths := []string{}1025	dirPaths := []string{}10261027	// Check if the paths or files added exist and exit if not1028	for _, f := range DirFilePaths {1029		fpath := filepath.Clean(f)10301031		s, err := os.Stat(fpath)1032		if err != nil {1033			fmt.Println("file or directory could not be read: " + fpath)1034			os.Exit(1)1035		}10361037		if s.IsDir() {1038			dirPaths = append(dirPaths, fpath)1039		} else {1040			filePaths = append(filePaths, fpath)1041		}1042	}10431044	SortBy = strings.ToLower(SortBy)1045	ctx := processorContext{remap: newRemapConfig(RemapAll, RemapUnknown)}10461047	printDebugF("NumCPU: %d", runtime.NumCPU())1048	printDebugF("SortBy: %s", SortBy)1049	printDebugF("PathDenyList: %v", PathDenyList)10501051	potentialFilesQueue := make(chan *gocodewalker.File, FileListQueueSize) // files that pass the .gitignore checks1052	fileListQueue := make(chan *FileJob, FileListQueueSize)                 // Files ready to be read from disk1053	fileSummaryJobQueue := make(chan *FileJob, FileSummaryJobQueueSize)     // Files ready to be summarised10541055	fileWalker := gocodewalker.NewParallelFileWalker(dirPaths, potentialFilesQueue)1056	fileWalker.SetErrorHandler(func(e error) bool {1057		printError(e.Error())1058		return true1059	})1060	fileWalker.IgnoreGitIgnore = GitIgnore1061	fileWalker.IgnoreIgnoreFile = Ignore1062	fileWalker.IgnoreGitModules = GitModuleIgnore1063	fileWalker.IncludeHidden = true1064	fileWalker.ExcludeDirectory = PathDenyList1065	fileWalker.SetConcurrency(DirectoryWalkerJobWorkers)10661067	if !SccIgnore {1068		fileWalker.CustomIgnore = []string{".sccignore"}1069	}1070	fileWalker.CustomIgnoreFiles = IgnoreFiles10711072	var excludePathRegexes []*regexp.Regexp1073	for _, exclude := range Exclude {1074		regexpResult, err := regexp.Compile(exclude)1075		if err == nil {1076			fileWalker.ExcludeFilenameRegex = append(fileWalker.ExcludeFilenameRegex, regexpResult)1077			fileWalker.ExcludeDirectoryRegex = append(fileWalker.ExcludeDirectoryRegex, regexpResult)1078			excludePathRegexes = append(excludePathRegexes, regexpResult)1079		} else {1080			printError(err.Error())1081		}1082	}10831084	go func() {1085		err := fileWalker.Start()1086		if err != nil {1087			printError(err.Error())1088		}1089	}()10901091	go func() {1092		for _, f := range filePaths {1093			fileInfo, err := os.Lstat(f)1094			if err != nil {1095				continue1096			}10971098			fileJob := newFileJob(f, f, fileInfo)1099			if fileJob != nil {1100				fileListQueue <- fileJob1101			}1102		}11031104		for fi := range potentialFilesQueue {1105			shouldExclude := false1106			for _, re := range excludePathRegexes {1107				if re.MatchString(fi.Location) {1108					shouldExclude = true1109					break1110				}1111			}1112			if shouldExclude {1113				continue1114			}11151116			fileInfo, err := os.Lstat(fi.Location)1117			if err != nil {1118				continue1119			}11201121			if !fileInfo.IsDir() {1122				fileJob := newFileJob(fi.Location, fi.Filename, fileInfo)1123				if fileJob != nil {1124					fileListQueue <- fileJob1125				}1126			}1127		}1128		close(fileListQueue)1129	}()11301131	go ctx.fileProcessorWorker(fileListQueue, fileSummaryJobQueue)11321133	result := fileSummarize(fileSummaryJobQueue)1134	if FileOutput == "" {1135		fmt.Print(result)1136	} else {1137		// A failed write must not report success: follow the --report path1138		// above and exit non-zero with the reason on stderr.1139		if err := os.WriteFile(FileOutput, []byte(result), 0644); err != nil {1140			fmt.Fprintln(os.Stderr, err)1141			os.Exit(1)1142		}1143		fmt.Println("results written to " + FileOutput)1144	}1145}