1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"hash"8	"runtime/debug"9	"strings"10	"sync"11	"sync/atomic"1213	"golang.org/x/crypto/blake2b"14)1516// The below are used as identifiers for the code state machine17const (18	SBlank             int64 = 119	SCode              int64 = 220	SComment           int64 = 321	SCommentCode       int64 = 4 // Indicates comment after code22	SMulticomment      int64 = 523	SMulticommentCode  int64 = 6 // Indicates multi comment after code24	SMulticommentBlank int64 = 7 // Indicates multi comment ended with blank afterward25	SString            int64 = 826	SDocString         int64 = 927)2829// SheBang is a global constant for indicating a shebang file header30const SheBang string = "#!"3132// UnknownLanguage is the category files are counted under when --count-unsupported33// is set and scc does not recognise the file's language. It has no language34// features so such files are counted as plain text (no comments or complexity).35const UnknownLanguage string = "Unknown"3637// LineType what type of line are processing38type LineType int323940// These are not meant to be CAMEL_CASE but as it us used by an external project we cannot change it41const (42	LINE_BLANK LineType = iota43	LINE_CODE44	LINE_COMMENT45)4647// ByteOrderMarks are taken from https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding48// These indicate that we cannot count the file correctly so we can at least warn the user49var ByteOrderMarks = [][]byte{50	{254, 255},            // UTF-16 BE51	{255, 254},            // UTF-16 LE52	{0, 0, 254, 255},      // UTF-32 BE53	{255, 254, 0, 0},      // UTF-32 LE54	{43, 47, 118, 56},     // UTF-755	{43, 47, 118, 57},     // UTF-756	{43, 47, 118, 43},     // UTF-757	{43, 47, 118, 47},     // UTF-758	{43, 47, 118, 56, 45}, // UTF-759	{247, 100, 76},        // UTF-160	{221, 115, 102, 115},  // UTF-EBCDIC61	{14, 254, 255},        // SCSU62	{251, 238, 40},        // BOCU-163	{132, 49, 149, 51},    // GB-1803064}6566var duplicates = CheckDuplicates{67	hashes: make(map[int64][][]byte),68}6970// cleanDuplicates resets the duplicate-detection hashes between runs. scc was71// historically a one-shot CLI where this state died with the process, but the72// long-lived MCP server calls ProcessResult once per tool call: without the73// reset, every file in the second call matches a hash recorded by the first and74// the whole result comes back empty.75func cleanDuplicates() {76	duplicates.Clear()77}7879func checkForMatchSingle(currentByte byte, index int, endPoint int, matches []byte, fileJob *FileJob) bool {80	potentialMatch := true81	if currentByte == matches[0] {82		for j := range matches {83			if index+j >= endPoint || matches[j] != fileJob.Content[index+j] {84				potentialMatch = false85				break86			}87		}8889		if potentialMatch {90			return true91		}92	}9394	return false95}9697func isWhitespace(currentByte byte) bool {98	if currentByte != ' ' && currentByte != '\t' && currentByte != '\n' && currentByte != '\r' {99		return false100	}101102	return true103}104105func isIdentifierContinue(b byte) bool {106	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'107}108109func hasNonWhitespaceBefore(content []byte, index int) bool {110	for i := index - 1; i >= 0; i-- {111		if !isWhitespace(content[i]) {112			return true113		}114	}115116	return false117}118119func nextNonWhitespaceIndex(content []byte, index int) int {120	for index < len(content) && isWhitespace(content[index]) {121		index++122	}123124	return index125}126127func hasPostfixExclude(content []byte, index, offsetJump int, excludes [][]byte) bool {128	token := content[index : index+offsetJump]129	for _, exclude := range excludes {130		if len(exclude) < offsetJump || !bytes.Equal(token, exclude[:offsetJump]) {131			continue132		}133134		remaining := exclude[offsetJump:]135		if len(remaining) == 0 {136			return true137		}138139		next := nextNonWhitespaceIndex(content, index+offsetJump)140		if next+len(remaining) > len(content) || !bytes.Equal(content[next:next+len(remaining)], remaining) {141			continue142		}143144		afterExclude := next + len(remaining)145		if isIdentifierContinue(remaining[len(remaining)-1]) {146			return afterExclude == len(content) || !isIdentifierContinue(content[afterExclude])147		}148149		return true150	}151152	return false153}154155func countComplexityPostfix(fileJob *FileJob, index, offsetJump int, postfixExcludes [][]byte) {156	if index == 0 {157		return158	}159160	content := fileJob.Content161	if isWhitespace(content[index-1]) && !hasNonWhitespaceBefore(content, index-1) {162		return163	}164165	if len(postfixExcludes) > 0 && hasPostfixExclude(content, index, offsetJump, postfixExcludes) {166		return167	}168169	fileJob.Complexity++170	fileJob.bumpComplexityLine()171	fileJob.bumpCognitive()172}173174// bumpComplexityLine adds one complexity tick to the line currently being175// counted. No-op when TrackComplexityLines is off — ComplexityLine is left176// empty by CountStats in that case, so there is no slot to bump.177func (fileJob *FileJob) bumpComplexityLine() {178	if n := len(fileJob.ComplexityLine); n > 0 {179		fileJob.ComplexityLine[n-1]++180	}181}182183// bumpCognitive weights a single complexity token by the nesting level of the184// line it appears on. An approximation of nested complexity, but with almost185// no calculation overhead.186func (fileJob *FileJob) bumpCognitive() {187	if Cognitive {188		weight := 1 + int64(fileJob.cognitiveNesting)189		fileJob.Cognitive += weight190		if n := len(fileJob.CognitiveLine); n > 0 {191			fileJob.CognitiveLine[n-1] += weight192		}193	}194}195196// Check if this file is binary by checking for nul byte and if so bail out197// this is how GNU Grep, git and ripgrep check for binary files198func isBinary(index int, currentByte byte) bool {199	return index < 10000 && !DisableCheckBinary && currentByte == 0200}201202func shouldProcess(currentByte, processBytesMask byte) bool {203	return currentByte&processBytesMask == currentByte204}205206func stateToByteType(state int64) byte {207	switch state {208	case SCode:209		return ByteTypeCode210	case SString:211		return ByteTypeString212	case SComment, SCommentCode, SMulticomment, SMulticommentCode, SMulticommentBlank, SDocString:213		return ByteTypeComment214	default: // SBlank215		return ByteTypeBlank216	}217}218219func resetState(currentState int64) int64 {220	switch currentState {221	case SMulticomment, SMulticommentCode:222		currentState = SMulticomment223	case SString:224		currentState = SString225	default:226		currentState = SBlank227	}228229	return currentState230}231232func stringState(fileJob *FileJob, index int, endPoint int, endString []byte, currentState int64, ignoreEscape bool) (int, int64) {233	// It's not possible to enter this state without checking at least 1 byte so it is safe to check -1 here234	// without checking if it is out of bounds first235	for i := index; i < endPoint; i++ {236		index = i237238		if fileJob.ContentByteType != nil {239			fileJob.ContentByteType[i] = ByteTypeString240		}241242		// If we hit a newline, return because we want to count the stats but keep243		// the current state so we end up back in this loop when the outer244		// one calls again245		if fileJob.Content[i] == '\n' {246			return i, currentState247		}248249		is_escaped := false250		// if there is an escape symbol before us, investigate251		if fileJob.Content[i-1] == '\\' {252			num_escapes := 0253			for j := i - 1; j > 0; j-- {254				if fileJob.Content[j] != '\\' {255					break256				}257				num_escapes++258			}259260			// if number of escapes is even, all escapes are themselves escaped261			// otherwise the last escape does escape current string terminator262			if num_escapes%2 != 0 {263				is_escaped = true264			}265		}266267		// If we are in a literal string we want to ignore escapes OR we aren't checking for special ones268		if ignoreEscape || !is_escaped {269			if checkForMatchSingle(fileJob.Content[i], index, endPoint, endString, fileJob) {270				// Skip past the whole end delimiter. For multi byte terminators such271				// as the C++ raw string )" the trailing byte is itself a quote start,272				// so leaving the cursor on it would re-open a new string. See #175.273				return i + len(endString) - 1, SCode274			}275		}276	}277278	return index, currentState279}280281// This is a special state check pretty much only ever used by Python codebases282// but potentially it could be expanded to deal with other types283func docStringState(fileJob *FileJob, index int, endPoint int, endString []byte, currentState int64) (int, int64) {284	// It's not possible to enter this state without checking at least 1 byte so it is safe to check -1 here285	// without checking if it is out of bounds first286	for i := index; i < endPoint; i++ {287		index = i288289		if fileJob.ContentByteType != nil {290			fileJob.ContentByteType[i] = ByteTypeComment291		}292293		if fileJob.Content[i] == '\n' {294			return i, currentState295		}296297		if fileJob.Content[i-1] != '\\' {298			if checkForMatchSingle(fileJob.Content[i], index, endPoint, endString, fileJob) {299				// So we have hit end of docstring at this point in which case check if only whitespace characters till the next300				// newline and if so we change to a comment otherwise to code301				// need to start the loop after ending definition of docstring, therefore adding the length of the string to302				// the index303				for j := index + len(endString); j <= endPoint; j++ {304					if fileJob.Content[j] == '\n' {305						printDebug("Found newline so docstring is comment")306						return i, SComment307					}308309					if !isWhitespace(fileJob.Content[j]) {310						printDebugF("Found something not whitespace so is code: %s", string(fileJob.Content[j]))311						return i, SCode312					}313				}314315				return i, SCode316			}317		}318	}319320	return index, currentState321}322323func codeState(324	fileJob *FileJob,325	index int,326	endPoint int,327	currentState int64,328	endString []byte,329	endComments [][]byte,330	langFeatures LanguageFeature,331	digest *hash.Hash,332) (int, int64, []byte, [][]byte, bool) {333	// Hacky fix to https://github.com/boyter/scc/issues/181334	if endPoint > len(fileJob.Content) {335		endPoint--336	}337338	for i := index; i < endPoint; i++ {339		curByte := fileJob.Content[i]340		index = i341342		if fileJob.ContentByteType != nil {343			fileJob.ContentByteType[i] = ByteTypeCode344		}345346		if curByte == '\n' {347			return i, currentState, endString, endComments, false348		}349350		if isBinary(i, curByte) {351			fileJob.Binary = true352			return i, currentState, endString, endComments, false353		}354355		if shouldProcess(curByte, langFeatures.ProcessMask) {356			if Duplicates {357				// Technically this is wrong because we skip bytes, so this is not a true358				// hash of the file contents, but for duplicate files it shouldn't matter359				// as both will skip the same way360				digestible := []byte{fileJob.Content[index]}361				(*digest).Write(digestible)362			}363364			switch tokenType, offsetJump, endString := langFeatures.Tokens.Match(fileJob.Content[i:]); tokenType {365			case TString:366				// If we are in string state then check what sort of string so we know if docstring OR ignoreescape string367				i, ignoreEscape := verifyIgnoreEscape(langFeatures, fileJob, index)368369				// It is safe to -1 here as to enter the code state we need to have370				// transitioned from blank to here hence i should always be >= 1371				// This check is to ensure we aren't in a character declaration372				// TODO this should use language features373				if fileJob.Content[i-1] != '\\' {374					currentState = SString375				}376377				return i, currentState, endString, endComments, ignoreEscape378379			case TSlcomment:380				currentState = SCommentCode381				return i, currentState, endString, endComments, false382383			case TMlcomment:384				if langFeatures.Nested || len(endComments) == 0 {385					endComments = append(endComments, endString)386					currentState = SMulticommentCode387					i += offsetJump - 1388389					return i, currentState, endString, endComments, false390				}391392			case TComplexity:393				if index == 0 || !isIdentifierContinue(fileJob.Content[index-1]) {394					fileJob.Complexity++395					fileJob.bumpComplexityLine()396					fileJob.bumpCognitive()397				}398				// Skip past the matched token so a shorter token overlapping it399				// (e.g. 為是 inside 恆為是) is not also counted. See #466.400				i += offsetJump - 1401402			case TComplexityPostfix:403				countComplexityPostfix(fileJob, index, offsetJump, langFeatures.PostfixExcludes)404			}405		}406	}407408	return index, currentState, endString, endComments, false409}410411func commentState(fileJob *FileJob, index int, endPoint int, currentState int64, endComments [][]byte, endString []byte, langFeatures LanguageFeature) (int, int64, []byte, [][]byte) {412	for i := index; i < endPoint; i++ {413		curByte := fileJob.Content[i]414		index = i415416		if fileJob.ContentByteType != nil {417			fileJob.ContentByteType[i] = ByteTypeComment418		}419420		if curByte == '\n' {421			return i, currentState, endString, endComments422		}423424		if checkForMatchSingle(curByte, index, endPoint, endComments[len(endComments)-1], fileJob) {425			// set offset jump here426			offsetJump := len(endComments[len(endComments)-1])427			endComments = endComments[:len(endComments)-1]428429			if len(endComments) == 0 {430				// If we started as multiline code switch back to code so we count correctly431				// IE i := 1 /* for the lols */432				// TODO is that required? Might still be required to count correctly433				if currentState == SMulticommentCode {434					currentState = SCode // TODO pointless to change here, just set S_MULTICOMMENT_BLANK435				} else {436					currentState = SMulticommentBlank437				}438			}439440			i += offsetJump - 1441			return i, currentState, endString, endComments442		}443		// Check if we are entering another multiline comment444		// This should come below check for match single as it speeds up processing445		if langFeatures.Nested || len(endComments) == 0 {446			if ok, offsetJump, endString := langFeatures.MultiLineComments.Match(fileJob.Content[i:]); ok != 0 {447				endComments = append(endComments, endString)448				i += offsetJump - 1449450				return i, currentState, endString, endComments451			}452		}453	}454455	return index, currentState, endString, endComments456}457458func blankState(459	fileJob *FileJob,460	index int,461	currentState int64,462	endComments [][]byte,463	endString []byte,464	langFeatures LanguageFeature,465) (int, int64, []byte, [][]byte, bool) {466	switch tokenType, offsetJump, endString := langFeatures.Tokens.Match(fileJob.Content[index:]); tokenType {467	case TMlcomment:468		if langFeatures.Nested || len(endComments) == 0 {469			endComments = append(endComments, endString)470			currentState = SMulticomment471			index += offsetJump - 1472			if fileJob.ContentByteType != nil {473				fileJob.ContentByteType[index] = ByteTypeComment474			}475			return index, currentState, endString, endComments, false476		}477478	case TSlcomment:479		currentState = SComment480		if fileJob.ContentByteType != nil {481			fileJob.ContentByteType[index] = ByteTypeComment482		}483		return index, currentState, endString, endComments, false484485	case TString:486		index, ignoreEscape := verifyIgnoreEscape(langFeatures, fileJob, index)487488		for _, v := range langFeatures.Quotes {489			if v.End == string(endString) && v.DocString {490				currentState = SDocString491				if fileJob.ContentByteType != nil {492					fileJob.ContentByteType[index] = ByteTypeComment493				}494				return index, currentState, endString, endComments, ignoreEscape495			}496		}497		currentState = SString498		if fileJob.ContentByteType != nil {499			fileJob.ContentByteType[index] = ByteTypeString500		}501		return index, currentState, endString, endComments, ignoreEscape502503	case TComplexity:504		currentState = SCode505		if fileJob.ContentByteType != nil {506			fileJob.ContentByteType[index] = ByteTypeCode507		}508		if index == 0 || !isIdentifierContinue(fileJob.Content[index-1]) {509			fileJob.Complexity++510			fileJob.bumpComplexityLine()511			fileJob.bumpCognitive()512		}513		// Skip past the matched token so a shorter token overlapping it514		// (e.g. 為是 inside 恆為是) is not also counted. See #466.515		index += offsetJump - 1516517	case TComplexityPostfix:518		currentState = SCode519		if fileJob.ContentByteType != nil {520			fileJob.ContentByteType[index] = ByteTypeCode521		}522		countComplexityPostfix(fileJob, index, offsetJump, langFeatures.PostfixExcludes)523524	default:525		currentState = SCode526		if fileJob.ContentByteType != nil {527			fileJob.ContentByteType[index] = ByteTypeCode528		}529	}530531	return index, currentState, endString, endComments, false532}533534// Some languages such as C# have quoted strings like @"\" where no escape character is required535// this checks if there is one so we can cater for these cases536func verifyIgnoreEscape(langFeatures LanguageFeature, fileJob *FileJob, index int) (int, bool) {537	ignoreEscape := false538539	// loop over the string states and if we have the special flag match, and if so we need to ensure we can handle them540	for i := 0; i < len(langFeatures.Quotes); i++ {541		if langFeatures.Quotes[i].DocString || langFeatures.Quotes[i].IgnoreEscape {542			// If so we need to check if where we are falls into these conditions543			isMatch := true544			for j := 0; j < len(langFeatures.Quotes[i].Start); j++ {545				if len(fileJob.Content) <= index+j || fileJob.Content[index+j] != langFeatures.Quotes[i].Start[j] {546					isMatch = false547					break548				}549			}550551			// If we have a match then jump ahead enough so we don't pick it up again for cases like @"552			if isMatch {553				ignoreEscape = true554				index = index + len(langFeatures.Quotes[i].Start)555556				// Clamp to the last byte when the start token ends the file, such as a557				// Python file whose final bytes are """ with no trailing newline. Left558				// unbounded the caller lands on len(Content) and blankState writes past559				// the end of ContentByteType, and the final line is never counted.560				if index >= len(fileJob.Content) {561					index = len(fileJob.Content) - 1562				}563			}564		}565	}566567	return index, ignoreEscape568}569570// CountStats will process the fileJob571// If the file contains anything even just a newline its line count should be >= 1.572// If the file has a size of 0 its line count should be 0.573// Newlines belong to the line they started on so a file of \n means only 1 line574// This is the 'hot' path for the application and needs to be as fast as possible575func CountStats(fileJob *FileJob) {576	// For determining duplicates we need the below. The reason for creating577	// the byte array here is to avoid GC pressure. MD5 is in the standard library578	// and is fast enough to not warrant murmur3 hashing. No need to be579	// crypto secure here either so no need to eat the performance cost of a better580	// hash method581	if Duplicates {582		fileJob.Hash, _ = blake2b.New256(nil)583	}584585	// If the file has a length of 0 it is empty then we say it has no lines586	if fileJob.Bytes == 0 {587		fileJob.Lines = 0588		return589	}590591	LanguageFeaturesMutex.Lock()592	langFeatures := LanguageFeatures[fileJob.Language]593	LanguageFeaturesMutex.Unlock()594595	if langFeatures.Complexity == nil {596		langFeatures.Complexity = &Trie{}597	}598	if langFeatures.SingleLineComments == nil {599		langFeatures.SingleLineComments = &Trie{}600	}601	if langFeatures.MultiLineComments == nil {602		langFeatures.MultiLineComments = &Trie{}603	}604	if langFeatures.Strings == nil {605		langFeatures.Strings = &Trie{}606	}607	if langFeatures.Tokens == nil {608		langFeatures.Tokens = &Trie{}609	}610611	endPoint := int(fileJob.Bytes - 1)612	currentState := SBlank613	endComments := [][]byte{}614	endString := []byte{}615616	// TODO needs to be set via langFeatures.Quotes[0].IgnoreEscape for the matching feature617	ignoreEscape := false618	if fileJob.TrackComplexityLines {619		fileJob.ComplexityLine = append(fileJob.ComplexityLine, 0)620		if Cognitive {621			fileJob.CognitiveLine = append(fileJob.CognitiveLine, 0)622		}623	}624625	if fileJob.ClassifyContent {626		fileJob.ContentByteType = make([]byte, fileJob.Bytes)627	}628629	bomSkip := checkBomSkip(fileJob)630631	//We want to track cognitive complexity nesting. Cognitive complexity632	//means we assign higher complexity to nested branch conditions, so633	//634	//if something:635	//    if otherthing:636	//637	//would be assigned a higher complexity than638	//639	//if something:640	//if otherthing:641	//642	//because the nested if requires more mental overhead. To do this we need to track643	//how nested each condition is when we hit it. We do this by counting the number of644	//whitespace characters are in front of the condition.645	//This is an appoximation, true for languages like Python, and probably true for anything646	//else. However, the benefit of this approach is that it's almost free from a CPU point of view647	//and the increase in spotting complex code, is genuinely useful.648	var indentStack []int649	lineStart := bomSkip650	needIndent := true651652	for index := bomSkip; index < int(fileJob.Bytes); index++ {653		if fileJob.ContentByteType != nil {654			fileJob.ContentByteType[index] = stateToByteType(currentState)655		}656657		// Based on our current state determine if the state should change by checking658		// what the character is. The below is very CPU bound so need to be careful if659		// changing anything in here and profile/measure afterwards!660		// NB that the order of the if statements matters and has been set to what in benchmarks is most efficient661		if !isWhitespace(fileJob.Content[index]) {662663			// At the first non-whitespace byte of a code-bearing line, update the664			// indent stack so complexity tokens on this line are weighted by their665			// nesting depth. Lines that begin a comment must not move the stack;666			// lines inside a multiline comment/string never reach here in a667			// blank-derived state so they are excluded automatically.668			if Cognitive && needIndent && (currentState == SBlank || currentState == SMulticommentBlank) {669				if tokenType, _, _ := langFeatures.Tokens.Match(fileJob.Content[index:]); tokenType != TSlcomment && tokenType != TMlcomment {670					indent := index - lineStart671					for len(indentStack) > 0 && indent < indentStack[len(indentStack)-1] {672						indentStack = indentStack[:len(indentStack)-1]673					}674					if len(indentStack) == 0 || indent > indentStack[len(indentStack)-1] {675						indentStack = append(indentStack, indent)676					}677					nesting := len(indentStack) - 1678					if nesting < 0 {679						nesting = 0680					}681					fileJob.cognitiveNesting = nesting682					needIndent = false683				}684			}685686			switch currentState {687			case SCode:688				index, currentState, endString, endComments, ignoreEscape = codeState(689					fileJob,690					index,691					endPoint,692					currentState,693					endString,694					endComments,695					langFeatures,696					&fileJob.Hash,697				)698			case SString:699				index, currentState = stringState(fileJob, index, endPoint, endString, currentState, ignoreEscape)700			case SDocString:701				// For a docstring we can either move into blank in which case we count it as a docstring702				// or back into code in which case it should be counted as code703				index, currentState = docStringState(fileJob, index, endPoint, endString, currentState)704			case SMulticomment, SMulticommentCode:705				index, currentState, endString, endComments = commentState(706					fileJob,707					index,708					endPoint,709					currentState,710					endComments,711					endString,712					langFeatures,713				)714			case SBlank, SMulticommentBlank:715				// From blank we can move into comment, move into a multiline comment716				// or move into code but we can only do one.717				index, currentState, endString, endComments, ignoreEscape = blankState(718					fileJob,719					index,720					currentState,721					endComments,722					endString,723					langFeatures,724				)725			}726		}727728		// We shouldn't normally need this, but unclosed strings or comments729		// might leave the index past the end of the file when we reach this730		// point.731		if index >= len(fileJob.Content) {732			return733		}734735		// Only check the first 10000 characters for null bytes indicating a binary file736		// and if we find it then we return otherwise carry on and ignore binary markers737		if index < 10000 && fileJob.Binary {738			return739		}740741		// This means the end of processing the line so calculate the stats according to what state742		// we are currently in743		if fileJob.Content[index] == '\n' || index >= endPoint {744			fileJob.Lines++745			if Cognitive {746				lineStart = index + 1747				needIndent = true748			}749			if fileJob.TrackComplexityLines {750				fileJob.ComplexityLine = append(fileJob.ComplexityLine, 0)751				if Cognitive {752					fileJob.CognitiveLine = append(fileJob.CognitiveLine, 0)753				}754			}755756			if NoLarge && fileJob.Lines >= LargeLineCount {757				// Save memory by unsetting the content as we no longer require it758				fileJob.Content = nil759				return760			}761762			switch currentState {763			case SCode, SString, SCommentCode, SMulticommentCode:764				fileJob.Code++765				currentState = resetState(currentState)766				if fileJob.Callback != nil {767					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_CODE) {768						return769					}770				}771				if Trace {772					// Don't remove the outside if-statements, for performance773					printTraceF("%s line %d ended with state: %d: counted as code", fileJob.Location, fileJob.Lines, currentState)774				}775			case SComment, SMulticomment, SMulticommentBlank:776				fileJob.Comment++777				currentState = resetState(currentState)778				if fileJob.Callback != nil {779					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_COMMENT) {780						return781					}782				}783				if Trace {784					// Same as above785					printTraceF("%s line %d ended with state: %d: counted as comment", fileJob.Location, fileJob.Lines, currentState)786				}787			case SBlank:788				fileJob.Blank++789				if fileJob.Callback != nil {790					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_BLANK) {791						return792					}793				}794				if Trace {795					// Same as above796					printTraceF("%s line %d ended with state: %d: counted as blank", fileJob.Location, fileJob.Lines, currentState)797				}798			case SDocString:799				fileJob.Comment++800				if fileJob.Callback != nil {801					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_COMMENT) {802						return803					}804				}805				if Trace {806					// Same as above807					printTraceF("%s line %d ended with state: %d: counted as comment", fileJob.Location, fileJob.Lines, currentState)808				}809			}810		}811	}812813	if UlocMode {814		uloc := map[string]struct{}{}815		for l := range strings.SplitSeq(strings.TrimRight(string(fileJob.Content), "\n"), "\n") {816			uloc[l] = struct{}{}817		}818		fileJob.Uloc = len(uloc)819	}820821	if MaxMean {822		for l := range strings.SplitSeq(strings.TrimRight(string(fileJob.Content), "\n"), "\n") {823			fileJob.LineLength = append(fileJob.LineLength, len(l))824		}825	}826827	isGenerated := false828829	if Generated {830		headLen := min(1000, len(fileJob.Content))831		head := bytes.ToLower(fileJob.Content[0:headLen])832		for _, marker := range GeneratedMarkers {833			if bytes.Contains(head, bytes.ToLower([]byte(marker))) {834				fileJob.Generated = true835				fileJob.Language = fileJob.Language + " (gen)"836				isGenerated = true837				printWarnF("%s identified as isGenerated with heading comment", fileJob.Filename)838				break839			}840		}841	}842843	// check if 0 as well to avoid divide by zero https://github.com/boyter/scc/issues/223844	if !isGenerated && Minified && fileJob.Lines != 0 {845		avgLineByteCount := len(fileJob.Content) / int(fileJob.Lines)846		minifiedGeneratedCheck(avgLineByteCount, fileJob)847	}848849	if fileJob.TrackComplexityLines {850		fileJob.ComplexityLine = fileJob.ComplexityLine[:fileJob.Lines]851		if Cognitive {852			fileJob.CognitiveLine = fileJob.CognitiveLine[:fileJob.Lines]853		}854	}855}856857func minifiedGeneratedCheck(avgLineByteCount int, fileJob *FileJob) {858	if avgLineByteCount >= MinifiedGeneratedLineByteLength {859		fileJob.Minified = true860		fileJob.Language = fileJob.Language + " (min)"861		printWarnF("%s identified as minified/generated with average line byte length of %d >= %d", fileJob.Filename, avgLineByteCount, MinifiedGeneratedLineByteLength)862	} else {863		printDebugF("%s not identified as minified/generated with average line byte length of %d < %d", fileJob.Filename, avgLineByteCount, MinifiedGeneratedLineByteLength)864	}865}866867// Check if we have any Byte Order Marks (BOM) in front of the file868func checkBomSkip(fileJob *FileJob) int {869	// UTF-8 BOM which if detected we should skip the BOM as we can then count correctly870	// []byte is UTF-8 BOM taken from https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding871	if bytes.HasPrefix(fileJob.Content, []byte{239, 187, 191}) {872		printWarnF("UTF-8 BOM found for file %s skipping 3 bytes", fileJob.Filename)873		return 3874	}875876	// If we have one of the other BOM then we might not be able to count correctly so if verbose let the user know877	if Verbose {878		for _, v := range ByteOrderMarks {879			if bytes.HasPrefix(fileJob.Content, v) {880				printWarnF("BOM found for file %s indicating it is not ASCII/UTF-8 and may be counted incorrectly or ignored as a binary file", fileJob.Filename)881			}882		}883	}884885	return 0886}887888// Reads and processes files from input chan in parallel, and sends results to889// output chan890func (ctx processorContext) fileProcessorWorker(input chan *FileJob, output chan *FileJob) {891	var startTime int64892	var fileCount int64893	var gcEnabled int64894	var wg sync.WaitGroup895896	for i := 0; i < FileProcessJobWorkers; i++ {897		wg.Go(func() {898			reader := NewFileReader()899900			for job := range input {901				atomic.CompareAndSwapInt64(&startTime, 0, makeTimestampMilli())902903				loc := job.Location904				if job.Symlocation != "" {905					loc = job.Symlocation906				}907908				fileStartTime := makeTimestampNano()909				content, err := reader.ReadFile(loc, int(job.Bytes))910				atomic.AddInt64(&fileCount, 1)911912				if atomic.LoadInt64(&gcEnabled) == 0 && atomic.LoadInt64(&fileCount) >= int64(GcFileCount) {913					debug.SetGCPercent(gcPercent)914					atomic.AddInt64(&gcEnabled, 1)915					printWarn("read file limit exceeded GC re-enabled")916				}917918				printTraceF("nanoseconds read into memory: %s: %d", job.Location, makeTimestampNano()-fileStartTime)919920				if err == nil {921					job.Content = content922					if ctx.processFile(job) {923						output <- job924					}925				} else {926					printWarnF("error reading: %s %s", job.Location, err)927				}928			}929930		})931	}932933	go func() {934		wg.Wait()935		close(output)936937		printDebugF("milliseconds reading files into memory: %d", makeTimestampMilli()-startTime)938	}()939}940941// Process a single file942// File must have been read to job.Content already943func (ctx processorContext) processFile(job *FileJob) bool {944	fileStartTime := makeTimestampNano()945946	contents := job.Content947948	// Needs to always run to ensure the language is set949	job.Language = DetermineLanguage(job.Filename, job.Language, job.PossibleLanguages, job.Content)950951	remapped := false952	if len(ctx.remap.all) != 0 {953		ctx.hardRemapLanguage(job)954	}955956	// If the type is #! we should check to see if we can identify957	if job.Language == SheBang {958		if len(ctx.remap.unknown) != 0 {959			remapped = ctx.unknownRemapLanguage(job)960		}961962		// if we didn't remap we then want to see if it's a #! map963		if !remapped {964			cutoff := min(200, len(contents))965966			lang, err := DetectSheBang(contents[:cutoff])967			if err != nil {968				printWarnF("unable to determine #! language for %s", job.Location)969				return false970			}971972			printWarnF("detected #! %s for %s", lang, job.Location)973			job.Language = lang974			LoadLanguageFeature(lang)975		}976	}977978	CountStats(job)979980	if Duplicates {981		duplicates.mux.Lock()982		jobHash := job.Hash.Sum(nil)983		if duplicates.Check(job.Bytes, jobHash) {984			printWarnF("skipping duplicate file: %s", job.Location)985			duplicates.mux.Unlock()986			return false987		}988989		duplicates.Add(job.Bytes, jobHash)990		duplicates.mux.Unlock()991	}992993	if IgnoreMinified && job.Minified {994		printWarnF("skipping minified file: %s", job.Location)995		return false996	}997998	if IgnoreGenerated && job.Generated {999		printWarnF("skipping generated file: %s", job.Location)1000		return false1001	}10021003	if NoLarge && job.Lines >= LargeLineCount {1004		printWarnF("skipping large file due to line length: %s", job.Location)1005		return false1006	}10071008	printTraceF("nanoseconds process: %s: %d", job.Location, makeTimestampNano()-fileStartTime)10091010	if job.Binary {1011		printWarnF("skipping file identified as binary: %s", job.Location)1012		return false1013	}10141015	// This needs to be at the end so we can ensure duplicate detection et.al run first1016	// avoiding inflating the counts1017	if UlocMode {1018		ulocMutex.Lock()10191020		for l := range strings.SplitSeq(strings.TrimRight(string(job.Content), "\n"), "\n") {1021			ulocGlobalCount[l] = struct{}{}10221023			_, ok := ulocLanguageCount[job.Language]1024			if !ok {1025				ulocLanguageCount[job.Language] = map[string]struct{}{}1026			}1027			ulocLanguageCount[job.Language][l] = struct{}{}1028		}1029		ulocMutex.Unlock()1030	}10311032	return true1033}10341035func (ctx processorContext) hardRemapLanguage(job *FileJob) bool {1036	remapped := false1037	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look10381039	for _, rule := range ctx.remap.all {1040		if bytes.Contains(job.Content[:cutoff], rule.pattern) {1041			job.Language = rule.language1042			remapped = true1043			printWarnF("hard remapping: %s to %s", job.Location, job.Language)1044		}1045	}10461047	return remapped1048}10491050func (ctx processorContext) unknownRemapLanguage(job *FileJob) bool {1051	remapped := false1052	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look10531054	for _, rule := range ctx.remap.unknown {1055		if bytes.Contains(job.Content[:cutoff], rule.pattern) {1056			job.Language = rule.language1057			remapped = true1058			printWarnF("unknown remapping: %s to %s", job.Location, job.Language)1059		}1060	}10611062	return remapped1063}