1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"os"8	"regexp"9	"runtime/debug"10	"strings"11	"sync"12	"sync/atomic"1314	"github.com/boyter/gocodewalker"15	"golang.org/x/crypto/blake2b"16)1718// The below are used as identifiers for the code state machine19const (20	SBlank             int64 = 121	SCode              int64 = 222	SComment           int64 = 323	SCommentCode       int64 = 4 // Indicates comment after code24	SMulticomment      int64 = 525	SMulticommentCode  int64 = 6 // Indicates multi comment after code26	SMulticommentBlank int64 = 7 // Indicates multi comment ended with blank afterward27	SString            int64 = 828	SDocString         int64 = 929)3031// SheBang is a global constant for indicating a shebang file header32const SheBang string = "#!"3334// UnknownLanguage is the category files are counted under when --count-unsupported35// is set and scc does not recognise the file's language. It has no language36// features so such files are counted as plain text (no comments or complexity).37const UnknownLanguage string = "Unknown"3839// LineType what type of line are processing40type LineType int324142// These are not meant to be CAMEL_CASE but as it us used by an external project we cannot change it43const (44	LINE_BLANK LineType = iota45	LINE_CODE46	LINE_COMMENT47)4849// ByteOrderMarks are taken from https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding50// These indicate that we cannot count the file correctly so we can at least warn the user51var ByteOrderMarks = [][]byte{52	{254, 255},            // UTF-16 BE53	{255, 254},            // UTF-16 LE54	{0, 0, 254, 255},      // UTF-32 BE55	{255, 254, 0, 0},      // UTF-32 LE56	{43, 47, 118, 56},     // UTF-757	{43, 47, 118, 57},     // UTF-758	{43, 47, 118, 43},     // UTF-759	{43, 47, 118, 47},     // UTF-760	{43, 47, 118, 56, 45}, // UTF-761	{247, 100, 76},        // UTF-162	{221, 115, 102, 115},  // UTF-EBCDIC63	{14, 254, 255},        // SCSU64	{251, 238, 40},        // BOCU-165	{132, 49, 149, 51},    // GB-1803066}6768var duplicates = CheckDuplicates{69	hashes: make(map[int64][][]byte),70}7172// cleanDuplicates resets the duplicate-detection hashes between runs. scc was73// historically a one-shot CLI where this state died with the process, but the74// long-lived MCP server calls ProcessResult once per tool call: without the75// reset, every file in the second call matches a hash recorded by the first and76// the whole result comes back empty.77func cleanDuplicates() {78	duplicates.Clear()79}8081func checkForMatchSingle(currentByte byte, index int, endPoint int, matches []byte, fileJob *FileJob) bool {82	potentialMatch := true83	if currentByte == matches[0] {84		for j := range matches {85			if index+j >= endPoint || matches[j] != fileJob.Content[index+j] {86				potentialMatch = false87				break88			}89		}9091		if potentialMatch {92			return true93		}94	}9596	return false97}9899// isBlankRun marks the whitespace that carries no meaning at all to the100// counting loop. The newline is left out because a line ends on it, which is101// the one piece of whitespace the loop has to stop for.102var isBlankRun = [256]bool{' ': true, '\t': true, '\r': true}103104func isWhitespace(currentByte byte) bool {105	if currentByte != ' ' && currentByte != '\t' && currentByte != '\n' && currentByte != '\r' {106		return false107	}108109	return true110}111112func isIdentifierContinue(b byte) bool {113	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') || b == '_'114}115116// commentOpensHere reports whether a matched line comment token really opens a117// comment where it sits. A token spelled as a word, such as Batch's REM or118// Autoconf's dnl, ends where the word ends, so REMOVE is a command and not a119// comment. A token of a single byte is left alone because FORTRAN Legacy opens120// a comment with a C in the first column whatever follows it, which is how a121// rule of CCCCCCCC across the page is written.122// commentTokenOpens reports whether a line comment token found here actually123// opens a comment, for the languages that ask more of it than being present.124//125// Two rules, and a language may want either. A token ending in a letter needs126// an ordinary word boundary, so REM opens a comment and REMOVE does not; that127// is worked out from the token itself. A token spelled with something that is128// not a letter gets no boundary that way, and Forth needs one anyway, its129// comment being a backslash that is a word like any other: "\\ x" is a comment130// and "\\x" is not. That one is declared by the language.131func commentTokenOpens(langFeatures LanguageFeature, content []byte, index, offsetJump int) bool {132	if langFeatures.WordComments && !commentOpensHere(content, index, offsetJump) {133		return false134	}135136	if langFeatures.CommentIsWord && !commentWordStandsAlone(content, index, offsetJump) {137		return false138	}139140	return true141}142143// commentWordStandsAlone reports whether the token at index is a whole word,144// which for a language of whitespace delimited words means nothing but145// whitespace or an edge of the file on either side of it.146func commentWordStandsAlone(content []byte, index, offsetJump int) bool {147	if index > 0 && !isWhitespace(content[index-1]) {148		return false149	}150151	after := index + offsetJump152153	return after >= len(content) || isWhitespace(content[after])154}155156func commentOpensHere(content []byte, index, offsetJump int) bool {157	if offsetJump < 2 || !isIdentifierContinue(content[index+offsetJump-1]) {158		return true159	}160161	if index > 0 && isIdentifierContinue(content[index-1]) {162		return false163	}164165	after := index + offsetJump166167	return after >= len(content) || !isIdentifierContinue(content[after])168}169170func hasNonWhitespaceBefore(content []byte, index int) bool {171	for i := index - 1; i >= 0; i-- {172		if !isWhitespace(content[i]) {173			return true174		}175	}176177	return false178}179180func nextNonWhitespaceIndex(content []byte, index int) int {181	for index < len(content) && isWhitespace(content[index]) {182		index++183	}184185	return index186}187188func hasPostfixExclude(content []byte, index, offsetJump int, excludes [][]byte) bool {189	token := content[index : index+offsetJump]190	for _, exclude := range excludes {191		if len(exclude) < offsetJump || !bytes.Equal(token, exclude[:offsetJump]) {192			continue193		}194195		remaining := exclude[offsetJump:]196		if len(remaining) == 0 {197			return true198		}199200		next := nextNonWhitespaceIndex(content, index+offsetJump)201		if next+len(remaining) > len(content) || !bytes.Equal(content[next:next+len(remaining)], remaining) {202			continue203		}204205		afterExclude := next + len(remaining)206		if isIdentifierContinue(remaining[len(remaining)-1]) {207			return afterExclude == len(content) || !isIdentifierContinue(content[afterExclude])208		}209210		return true211	}212213	return false214}215216func countComplexityPostfix(fileJob *FileJob, index, offsetJump int, postfixExcludes [][]byte) {217	if index == 0 {218		return219	}220221	content := fileJob.Content222	if isWhitespace(content[index-1]) && !hasNonWhitespaceBefore(content, index-1) {223		return224	}225226	if len(postfixExcludes) > 0 && hasPostfixExclude(content, index, offsetJump, postfixExcludes) {227		return228	}229230	fileJob.Complexity++231	fileJob.bumpComplexityLine()232	fileJob.bumpCognitive()233}234235// bumpComplexityLine adds one complexity tick to the line currently being236// counted. No-op when TrackComplexityLines is off — ComplexityLine is left237// empty by CountStats in that case, so there is no slot to bump.238func (fileJob *FileJob) bumpComplexityLine() {239	if n := len(fileJob.ComplexityLine); n > 0 {240		fileJob.ComplexityLine[n-1]++241	}242}243244// bumpCognitive weights a single complexity token by the nesting level of the245// line it appears on. An approximation of nested complexity, but with almost246// no calculation overhead.247func (fileJob *FileJob) bumpCognitive() {248	if Cognitive {249		weight := 1 + int64(fileJob.cognitiveNesting)250		fileJob.Cognitive += weight251		if n := len(fileJob.CognitiveLine); n > 0 {252			fileJob.CognitiveLine[n-1] += weight253		}254	}255}256257// Check if this file is binary by checking for nul byte and if so bail out258// this is how GNU Grep, git and ripgrep check for binary files259func isBinary(index int, currentByte byte) bool {260	return index < 10000 && !DisableCheckBinary && currentByte == 0261}262263func shouldProcess(currentByte, processBytesMask byte) bool {264	return currentByte&processBytesMask == currentByte265}266267// newTokenFirst returns a stop table with nothing in it but the two bytes every268// language stops on whatever its tokens are: the newline that ends a line and269// the nul that marks the file binary.270func newTokenFirst() *[256]bool {271	t := &[256]bool{}272	t['\n'] = true273	t[0] = true274275	return t276}277278// emptyTokenFirst stands in for a language with no features at all, such as the279// Unknown bucket --count-unsupported puts files in, where there is no trie to280// build a table from.281var emptyTokenFirst = newTokenFirst()282283func stateToByteType(state int64) byte {284	switch state {285	case SCode:286		return ByteTypeCode287	case SString:288		return ByteTypeString289	case SComment, SCommentCode, SMulticomment, SMulticommentCode, SMulticommentBlank, SDocString:290		return ByteTypeComment291	default: // SBlank292		return ByteTypeBlank293	}294}295296func resetState(currentState int64) int64 {297	switch currentState {298	case SMulticomment, SMulticommentCode:299		currentState = SMulticomment300	case SString:301		currentState = SString302	default:303		currentState = SBlank304	}305306	return currentState307}308309// endsWithLineSplice reports whether the newline at index has a backslash310// immediately in front of it. C joins those two lines in translation phase 2,311// before it looks for a comment or a string, so the backslash splices whatever312// is in front of it and needs no escape counting of its own.313func endsWithLineSplice(content []byte, index int) bool {314	if index >= len(content) || content[index] != '\n' {315		return false316	}317318	i := index - 1319	if i >= 0 && content[i] == '\r' {320		i--321	}322323	return i >= 0 && content[i] == '\\'324}325326// resetLineState is resetState for a language that splices lines. It differs on327// two counts, and on both the C family wants the opposite of everything else: a328// line comment ending in a splice carries on into the next line, and a string329// that does not end in one is over at the newline whatever it holds.330//331// A raw string runs over a newline with no splice behind it, so ending an332// unspliced string would break every one of them. ignoreEscape marks exactly333// the delimiters that have no escape mechanism, which is to say the raw ones,334// and they are left to resetState.335func resetLineState(currentState int64, spliced bool, ignoreEscape bool) int64 {336	switch currentState {337	case SComment, SCommentCode:338		if spliced {339			return SComment340		}341	case SString:342		if !spliced && !ignoreEscape {343			return SBlank344		}345	}346347	return resetState(currentState)348}349350func stringState(fileJob *FileJob, index int, endPoint int, endString []byte, currentState int64, ignoreEscape bool, escape byte) (int, int64) {351	// It's not possible to enter this state without checking at least 1 byte so it is safe to check -1 here352	// without checking if it is out of bounds first353	for i := index; i < endPoint; i++ {354		index = i355356		if fileJob.ContentByteType != nil {357			fileJob.ContentByteType[i] = ByteTypeString358		}359360		// If we hit a newline, return because we want to count the stats but keep361		// the current state so we end up back in this loop when the outer362		// one calls again363		if fileJob.Content[i] == '\n' {364			return i, currentState365		}366367		is_escaped := false368		// if there is an escape symbol before us, investigate369		if fileJob.Content[i-1] == escape {370			num_escapes := 0371			for j := i - 1; j > 0; j-- {372				if fileJob.Content[j] != escape {373					break374				}375				num_escapes++376			}377378			// if number of escapes is even, all escapes are themselves escaped379			// otherwise the last escape does escape current string terminator380			if num_escapes%2 != 0 {381				is_escaped = true382			}383		}384385		// If we are in a literal string we want to ignore escapes OR we aren't checking for special ones386		if ignoreEscape || !is_escaped {387			if checkForMatchSingle(fileJob.Content[i], index, endPoint, endString, fileJob) {388				// Skip past the whole end delimiter. For multi byte terminators such389				// as the C++ raw string )" the trailing byte is itself a quote start,390				// so leaving the cursor on it would re-open a new string. See #175.391				return i + len(endString) - 1, SCode392			}393		}394	}395396	return index, currentState397}398399// This is a special state check pretty much only ever used by Python codebases400// but potentially it could be expanded to deal with other types401func docStringState(fileJob *FileJob, index int, endPoint int, endString []byte, currentState int64) (int, int64) {402	// It's not possible to enter this state without checking at least 1 byte so it is safe to check -1 here403	// without checking if it is out of bounds first404	for i := index; i < endPoint; i++ {405		index = i406407		if fileJob.ContentByteType != nil {408			fileJob.ContentByteType[i] = ByteTypeComment409		}410411		if fileJob.Content[i] == '\n' {412			return i, currentState413		}414415		if fileJob.Content[i-1] != '\\' {416			if checkForMatchSingle(fileJob.Content[i], index, endPoint, endString, fileJob) {417				// So we have hit end of docstring at this point in which case check if only whitespace characters till the next418				// newline and if so we change to a comment otherwise to code419				// need to start the loop after ending definition of docstring, therefore adding the length of the string to420				// the index421				for j := index + len(endString); j <= endPoint; j++ {422					if fileJob.Content[j] == '\n' {423						printDebug("Found newline so docstring is comment")424						return i, SComment425					}426427					if !isWhitespace(fileJob.Content[j]) {428						printDebugF("Found something not whitespace so is code: %s", string(fileJob.Content[j]))429						return i, SCode430					}431				}432433				return i, SCode434			}435		}436	}437438	return index, currentState439}440441func codeState(442	fileJob *FileJob,443	index int,444	endPoint int,445	currentState int64,446	endString []byte,447	endComments [][]byte,448	langFeatures LanguageFeature,449) (int, int64, []byte, [][]byte, bool) {450	// Hacky fix to https://github.com/boyter/scc/issues/181451	if endPoint > len(fileJob.Content) {452		endPoint--453	}454455	// Two rarely used features want a record of every byte the loop walked over456	// rather than only the ones it stopped on, so they keep the byte at a time457	// scan. Neither is on in a normal count.458	if Duplicates || fileJob.ContentByteType != nil {459		return codeStateSlow(fileJob, index, endPoint, currentState, endString, endComments, langFeatures)460	}461462	content := fileJob.Content463	stop := langFeatures.TokenFirst464	start := index465466	i := start467	for ; i < endPoint; i++ {468		curByte := content[i]469470		// One load and one branch answers for the newline, the nul and every471		// byte that could open a token. Everything else is ordinary code and472		// the loop has no business looking at it.473		if !stop[curByte] {474			continue475		}476477		index = i478479		if curByte == '\n' {480			return i, currentState, endString, endComments, false481		}482483		// The nul is the only byte that marks a file binary, so ask that of the484		// byte already in hand before reaching for the position and the global485		// that isBinary also wants. Reading them the other way round pays for486		// both on every token the scan stops on.487		if curByte == 0 && isBinary(i, curByte) {488			fileJob.Binary = true489			return i, currentState, endString, endComments, false490		}491492		switch tokenType, offsetJump, endString := langFeatures.Tokens.Match(content[i:]); tokenType {493		case TString:494			// If we are in string state then check what sort of string so we know if docstring OR ignoreescape string495			i, ignoreEscape, stringEnd := prepareString(langFeatures, fileJob, index, offsetJump, endString)496497			// It is safe to -1 here as to enter the code state we need to have498			// transitioned from blank to here hence i should always be >= 1499			// This check is to ensure we aren't in a character declaration500			// TODO this should use language features501			if content[i-1] != langFeatures.Escape {502				currentState = SString503			}504505			return i, currentState, stringEnd, endComments, ignoreEscape506507		case TSlcomment:508			// No word break check here, unlike blankState. A line that already509			// holds code counts as code whether the token opened a comment on510			// it or not, so the check buys nothing a line count can see, and511			// this is the hottest switch in the program.512			currentState = SCommentCode513			return i, currentState, endString, endComments, false514515		case TMlcomment:516			if langFeatures.Nested || len(endComments) == 0 {517				endComments = append(endComments, endString)518				currentState = SMulticommentCode519				// max(offsetJump, 1) because Trie.Match reports the depth its520				// walk reached, which is one short when the token runs to the521				// very end of the slice it was handed: there is no byte left to522				// stop on. For a token of a single byte that is zero, and the523				// step back by one drove the index to -1 and crashed the count524				// on a file ending in one. prepareString already clamps the same525				// way for the same reason.526				i += max(offsetJump, 1) - 1527528				return i, currentState, endString, endComments, false529			}530531		case TComplexity:532			if index == 0 || !isIdentifierContinue(content[index-1]) {533				fileJob.Complexity++534				fileJob.bumpComplexityLine()535				fileJob.bumpCognitive()536			}537			// Skip past the matched token so a shorter token overlapping it538			// (e.g. 為是 inside 恆為是) is not also counted. See #466.539			//540			// A jump that lands on or past the end leaves the byte at a time541			// scan sitting on the token it just matched, so say so rather than542			// falling out of the loop and reporting the last byte of the file.543			if i+offsetJump >= endPoint {544				return index, currentState, endString, endComments, false545			}546			i += max(offsetJump, 1) - 1547548		case TComplexityPostfix:549			countComplexityPostfix(fileJob, index, offsetJump, langFeatures.PostfixExcludes)550		}551	}552553	// The byte at a time scan reports the last byte it looked at, which after a554	// run to the end is the one before endPoint. Skipping over bytes does not555	// change which byte that is.556	if endPoint > start {557		index = endPoint - 1558	}559560	return index, currentState, endString, endComments, false561}562563// codeStateSlow is codeState for the two features that need every byte visited:564// --duplicates, which folds the bytes it walks past into the file hash, and565// content classification, which records a type for each of them.566func codeStateSlow(567	fileJob *FileJob,568	index int,569	endPoint int,570	currentState int64,571	endString []byte,572	endComments [][]byte,573	langFeatures LanguageFeature,574) (int, int64, []byte, [][]byte, bool) {575	for i := index; i < endPoint; i++ {576		curByte := fileJob.Content[i]577		index = i578579		if fileJob.ContentByteType != nil {580			fileJob.ContentByteType[i] = ByteTypeCode581		}582583		if curByte == '\n' {584			return i, currentState, endString, endComments, false585		}586587		if isBinary(i, curByte) {588			fileJob.Binary = true589			return i, currentState, endString, endComments, false590		}591592		if shouldProcess(curByte, langFeatures.ProcessMask) {593			switch tokenType, offsetJump, endString := langFeatures.Tokens.Match(fileJob.Content[i:]); tokenType {594			case TString:595				// If we are in string state then check what sort of string so we know if docstring OR ignoreescape string596				i, ignoreEscape, stringEnd := prepareString(langFeatures, fileJob, index, offsetJump, endString)597598				// It is safe to -1 here as to enter the code state we need to have599				// transitioned from blank to here hence i should always be >= 1600				// This check is to ensure we aren't in a character declaration601				// TODO this should use language features602				if fileJob.Content[i-1] != langFeatures.Escape {603					currentState = SString604				}605606				return i, currentState, stringEnd, endComments, ignoreEscape607608			case TSlcomment:609				// No word break check here, unlike blankState. A line that already610				// holds code counts as code whether the token opened a comment on611				// it or not, so the check buys nothing a line count can see, and612				// this is the hottest switch in the program.613				currentState = SCommentCode614				return i, currentState, endString, endComments, false615616			case TMlcomment:617				if langFeatures.Nested || len(endComments) == 0 {618					endComments = append(endComments, endString)619					currentState = SMulticommentCode620					i += max(offsetJump, 1) - 1621622					return i, currentState, endString, endComments, false623				}624625			case TComplexity:626				if index == 0 || !isIdentifierContinue(fileJob.Content[index-1]) {627					fileJob.Complexity++628					fileJob.bumpComplexityLine()629					fileJob.bumpCognitive()630				}631				// Skip past the matched token so a shorter token overlapping it632				// (e.g. 為是 inside 恆為是) is not also counted. See #466.633				i += max(offsetJump, 1) - 1634635			case TComplexityPostfix:636				countComplexityPostfix(fileJob, index, offsetJump, langFeatures.PostfixExcludes)637			}638		}639	}640641	return index, currentState, endString, endComments, false642}643644func commentState(fileJob *FileJob, index int, endPoint int, currentState int64, endComments [][]byte, endString []byte, langFeatures LanguageFeature) (int, int64, []byte, [][]byte) {645	for i := index; i < endPoint; i++ {646		curByte := fileJob.Content[i]647		index = i648649		if fileJob.ContentByteType != nil {650			fileJob.ContentByteType[i] = ByteTypeComment651		}652653		if curByte == '\n' {654			return i, currentState, endString, endComments655		}656657		if checkForMatchSingle(curByte, index, endPoint, endComments[len(endComments)-1], fileJob) {658			// set offset jump here659			offsetJump := len(endComments[len(endComments)-1])660			endComments = endComments[:len(endComments)-1]661662			if len(endComments) == 0 {663				// If we started as multiline code switch back to code so we count correctly664				// IE i := 1 /* for the lols */665				// TODO is that required? Might still be required to count correctly666				if currentState == SMulticommentCode {667					currentState = SCode // TODO pointless to change here, just set S_MULTICOMMENT_BLANK668				} else {669					currentState = SMulticommentBlank670				}671			}672673			i += max(offsetJump, 1) - 1674			return i, currentState, endString, endComments675		}676		// Check if we are entering another multiline comment677		// This should come below check for match single as it speeds up processing678		if langFeatures.Nested || len(endComments) == 0 {679			if ok, offsetJump, endString := langFeatures.MultiLineComments.Match(fileJob.Content[i:]); ok != 0 {680				endComments = append(endComments, endString)681				i += max(offsetJump, 1) - 1682683				return i, currentState, endString, endComments684			}685		}686	}687688	return index, currentState, endString, endComments689}690691func blankState(692	fileJob *FileJob,693	index int,694	currentState int64,695	endComments [][]byte,696	endString []byte,697	langFeatures LanguageFeature,698) (int, int64, []byte, [][]byte, bool) {699	// The first byte of a line that opens no token at all is the default case700	// below: ordinary code. The stop table answers that with one load, where701	// the trie walk that used to answer it started at the root of the tokens702	// and failed at its first step, and it is walked once for every line of703	// every file that starts with something the language has no token for.704	if !langFeatures.TokenFirst[fileJob.Content[index]] {705		currentState = SCode706		if fileJob.ContentByteType != nil {707			fileJob.ContentByteType[index] = ByteTypeCode708		}709710		return index, currentState, endString, endComments, false711	}712713	switch tokenType, offsetJump, endString := langFeatures.Tokens.Match(fileJob.Content[index:]); tokenType {714	case TMlcomment:715		if langFeatures.Nested || len(endComments) == 0 {716			endComments = append(endComments, endString)717			currentState = SMulticomment718			index += max(offsetJump, 1) - 1719			if fileJob.ContentByteType != nil {720				fileJob.ContentByteType[index] = ByteTypeComment721			}722			return index, currentState, endString, endComments, false723		}724725	case TSlcomment:726		if commentTokenOpens(langFeatures, fileJob.Content, index, offsetJump) {727			currentState = SComment728			if fileJob.ContentByteType != nil {729				fileJob.ContentByteType[index] = ByteTypeComment730			}731			return index, currentState, endString, endComments, false732		}733		currentState = SCode734		if fileJob.ContentByteType != nil {735			fileJob.ContentByteType[index] = ByteTypeCode736		}737738	case TString:739		index, ignoreEscape, endString := prepareString(langFeatures, fileJob, index, offsetJump, endString)740741		for _, v := range langFeatures.Quotes {742			if v.End == string(endString) && v.DocString {743				currentState = SDocString744				if fileJob.ContentByteType != nil {745					fileJob.ContentByteType[index] = ByteTypeComment746				}747				return index, currentState, endString, endComments, ignoreEscape748			}749		}750		currentState = SString751		if fileJob.ContentByteType != nil {752			fileJob.ContentByteType[index] = ByteTypeString753		}754		return index, currentState, endString, endComments, ignoreEscape755756	case TComplexity:757		currentState = SCode758		if fileJob.ContentByteType != nil {759			fileJob.ContentByteType[index] = ByteTypeCode760		}761		if index == 0 || !isIdentifierContinue(fileJob.Content[index-1]) {762			fileJob.Complexity++763			fileJob.bumpComplexityLine()764			fileJob.bumpCognitive()765		}766		// Skip past the matched token so a shorter token overlapping it767		// (e.g. 為是 inside 恆為是) is not also counted. See #466.768		index += max(offsetJump, 1) - 1769770	case TComplexityPostfix:771		currentState = SCode772		if fileJob.ContentByteType != nil {773			fileJob.ContentByteType[index] = ByteTypeCode774		}775		countComplexityPostfix(fileJob, index, offsetJump, langFeatures.PostfixExcludes)776777	default:778		currentState = SCode779		if fileJob.ContentByteType != nil {780			fileJob.ContentByteType[index] = ByteTypeCode781		}782	}783784	return index, currentState, endString, endComments, false785}786787// maxRawStringDelimiter is the longest delimiter a C++ raw string may name,788// which the standard puts at 16 characters.789const maxRawStringDelimiter = 16790791// rawStringEnd reads the delimiter a raw string names for itself and builds the792// closer from it. index points at the first byte after the start token, so for793// R"tag( it points at the t, and the closer that comes back is )tag". The794// second return is the index of the opening bracket, or -1 when there is no795// delimiter to be had, which is not valid C++ and is left to the plain string796// that the quote of the start token opens.797func rawStringEnd(content []byte, index int, terminator byte) ([]byte, int) {798	// The delimiter may be up to maxRawStringDelimiter bytes, so the bracket799	// that ends it sits one past that at the longest, and the bound has to800	// reach it. Stopping a byte short left a raw string with a full length801	// delimiter unrecognised, and the plain quote it fell back to opened a802	// string that ran to the end of the file.803	limit := index + maxRawStringDelimiter + 1804	if limit > len(content) {805		limit = len(content)806	}807808	for i := index; i < limit; i++ {809		switch content[i] {810		case '(':811			end := make([]byte, 0, (i-index)+2)812			end = append(end, ')')813			end = append(end, content[index:i]...)814815			return append(end, terminator), i816		case ')', '\\', ' ', '\t', '\n', '\r':817			// none of these may appear in a delimiter, so there is no raw string here818			return nil, -1819		}820	}821822	return nil, -1823}824825// Advance over a string prefix and report whether escapes are literal. The826// third return is a closer read out of the file, which is nil for every string827// whose closer the language already knows.828func prepareString(langFeatures LanguageFeature, fileJob *FileJob, index, tokenLength int, endString []byte) (int, bool, []byte) {829	ignoreEscape := false830831	// loop over the string states and if we have the special flag match, and if so we need to ensure we can handle them832	for i := 0; i < len(langFeatures.Quotes); i++ {833		if langFeatures.Quotes[i].DocString || langFeatures.Quotes[i].IgnoreEscape || langFeatures.Quotes[i].Delimited {834			// If so we need to check if where we are falls into these conditions835			isMatch := true836			for j := 0; j < len(langFeatures.Quotes[i].Start); j++ {837				if len(fileJob.Content) <= index+j || fileJob.Content[index+j] != langFeatures.Quotes[i].Start[j] {838					isMatch = false839					break840				}841			}842843			// If we have a match then jump ahead enough so we don't pick it up again for cases like @"844			if isMatch {845				ignoreEscape = true846				afterStart := index + len(langFeatures.Quotes[i].Start)847848				// A raw string naming its own closer has that closer read out of849				// the file here, the cursor landing on the opening bracket so the850				// delimiter is not walked again. A start token with nothing that851				// can follow it is not a raw string, and falls through to the852				// plain string its own quote opens.853				if langFeatures.Quotes[i].Delimited {854					end := langFeatures.Quotes[i].End855					if rawEnd, bracket := rawStringEnd(fileJob.Content, afterStart, end[len(end)-1]); bracket != -1 {856						return bracket, ignoreEscape, rawEnd857					}858				}859860				// Report the last byte of the opening token, the same as the plain861				// path below does with tokenLength. The caller steps on one byte862				// before it looks again, so returning the byte after the token863				// instead would step over the first byte of the string: for a one864				// byte quote that is the closer of an empty string, and '' would865				// swallow every line under it until something else closed it.866				index = afterStart - 1867868				// Clamp to the last byte when the start token ends the file, such as a869				// Python file whose final bytes are """ with no trailing newline. Left870				// unbounded the caller lands on len(Content) and blankState writes past871				// the end of ContentByteType, and the final line is never counted.872				if index >= len(fileJob.Content) {873					index = len(fileJob.Content) - 1874				}875				return index, ignoreEscape, endString876			}877		}878	}879880	return index + max(tokenLength, 1) - 1, ignoreEscape, endString881}882883// CountStats will process the fileJob884// If the file contains anything even just a newline its line count should be >= 1.885// If the file has a size of 0 its line count should be 0.886// Newlines belong to the line they started on so a file of \n means only 1 line887// This is the 'hot' path for the application and needs to be as fast as possible888func CountStats(fileJob *FileJob) {889	// For determining duplicates we need the below. The reason for creating890	// the byte array here is to avoid GC pressure. MD5 is in the standard library891	// and is fast enough to not warrant murmur3 hashing. No need to be892	// crypto secure here either so no need to eat the performance cost of a better893	// hash method894	if Duplicates {895	}896897	// If the file has a length of 0 it is empty then we say it has no lines898	if fileJob.Bytes == 0 {899		fileJob.Lines = 0900		return901	}902903	LanguageFeaturesMutex.Lock()904	langFeatures := LanguageFeatures[fileJob.Language]905	LanguageFeaturesMutex.Unlock()906907	if langFeatures.TokenFirst == nil {908		langFeatures.TokenFirst = emptyTokenFirst909	}910	if langFeatures.Complexity == nil {911		langFeatures.Complexity = &Trie{}912	}913	if langFeatures.SingleLineComments == nil {914		langFeatures.SingleLineComments = &Trie{}915	}916	if langFeatures.MultiLineComments == nil {917		langFeatures.MultiLineComments = &Trie{}918	}919	if langFeatures.Strings == nil {920		langFeatures.Strings = &Trie{}921	}922	if langFeatures.Tokens == nil {923		langFeatures.Tokens = &Trie{}924	}925926	endPoint := int(fileJob.Bytes - 1)927	currentState := SBlank928	endComments := [][]byte{}929	endString := []byte{}930931	// TODO needs to be set via langFeatures.Quotes[0].IgnoreEscape for the matching feature932	ignoreEscape := false933	if fileJob.TrackComplexityLines {934		fileJob.ComplexityLine = append(fileJob.ComplexityLine, 0)935		if Cognitive {936			fileJob.CognitiveLine = append(fileJob.CognitiveLine, 0)937		}938	}939940	if fileJob.ClassifyContent {941		fileJob.ContentByteType = make([]byte, fileJob.Bytes)942	}943944	bomSkip := checkBomSkip(fileJob)945946	// The main loop lives in its own function so a language with a counter of947	// its own can stand in for it while everything after it stays shared. It948	// reports whether it ran to the end of the file: a binary marker, a large949	// file cut short or a callback asking to stop all end the count there and950	// the work below is not wanted.951	if count := counterFor(fileJob); count != nil {952		if !count(fileJob, bomSkip, endPoint) {953			return954		}955	} else if noTokensAtAll(langFeatures) && specialisedCounterEligible(fileJob) {956		if !countLoopNoTokens(fileJob, bomSkip, endPoint) {957			return958		}959	} else if !countLoopGeneric(fileJob, langFeatures, bomSkip, endPoint, currentState, endString, endComments, ignoreEscape) {960		return961	}962963	if UlocMode {964		uloc := map[string]struct{}{}965		for l := range strings.SplitSeq(strings.TrimRight(string(fileJob.Content), "\n"), "\n") {966			uloc[l] = struct{}{}967		}968		fileJob.Uloc = len(uloc)969	}970971	if MaxMean {972		for l := range strings.SplitSeq(strings.TrimRight(string(fileJob.Content), "\n"), "\n") {973			fileJob.LineLength = append(fileJob.LineLength, len(l))974		}975	}976977	isGenerated := false978979	if Generated {980		headLen := min(1000, len(fileJob.Content))981		head := bytes.ToLower(fileJob.Content[0:headLen])982		for _, marker := range GeneratedMarkers {983			if bytes.Contains(head, bytes.ToLower([]byte(marker))) {984				fileJob.Generated = true985				fileJob.Language = fileJob.Language + " (gen)"986				isGenerated = true987				printWarnF("%s identified as isGenerated with heading comment", fileJob.Filename)988				break989			}990		}991	}992993	// check if 0 as well to avoid divide by zero https://github.com/boyter/scc/issues/223994	if !isGenerated && Minified && fileJob.Lines != 0 {995		avgLineByteCount := len(fileJob.Content) / int(fileJob.Lines)996		minifiedGeneratedCheck(avgLineByteCount, fileJob)997	}998999	if fileJob.TrackComplexityLines {1000		fileJob.ComplexityLine = fileJob.ComplexityLine[:fileJob.Lines]1001		if Cognitive {1002			fileJob.CognitiveLine = fileJob.CognitiveLine[:fileJob.Lines]1003		}1004	}1005}10061007func minifiedGeneratedCheck(avgLineByteCount int, fileJob *FileJob) {1008	if avgLineByteCount >= MinifiedGeneratedLineByteLength {1009		fileJob.Minified = true1010		fileJob.Language = fileJob.Language + " (min)"1011		printWarnF("%s identified as minified/generated with average line byte length of %d >= %d", fileJob.Filename, avgLineByteCount, MinifiedGeneratedLineByteLength)1012	} else {1013		printDebugF("%s not identified as minified/generated with average line byte length of %d < %d", fileJob.Filename, avgLineByteCount, MinifiedGeneratedLineByteLength)1014	}1015}10161017// Check if we have any Byte Order Marks (BOM) in front of the file1018func checkBomSkip(fileJob *FileJob) int {1019	// UTF-8 BOM which if detected we should skip the BOM as we can then count correctly1020	// []byte is UTF-8 BOM taken from https://en.wikipedia.org/wiki/Byte_order_mark#Byte_order_marks_by_encoding1021	if bytes.HasPrefix(fileJob.Content, []byte{239, 187, 191}) {1022		printWarnF("UTF-8 BOM found for file %s skipping 3 bytes", fileJob.Filename)1023		return 31024	}10251026	// If we have one of the other BOM then we might not be able to count correctly so if verbose let the user know1027	if Verbose {1028		for _, v := range ByteOrderMarks {1029			if bytes.HasPrefix(fileJob.Content, v) {1030				printWarnF("BOM found for file %s indicating it is not ASCII/UTF-8 and may be counted incorrectly or ignored as a binary file", fileJob.Filename)1031			}1032		}1033	}10341035	return 01036}10371038func pathExcluded(location string, excludePathRegexes []*regexp.Regexp) bool {1039	for _, re := range excludePathRegexes {1040		if re.MatchString(location) {1041			return true1042		}1043	}1044	return false1045}10461047// startFileJobProducer stats what the walker found and turns it into the1048// FileJobs the readers consume, closing fileListQueue once the walk is done and1049// every path has been handled.1050//1051// The stat is done by a pool rather than by one goroutine. A single goroutine1052// could not keep up with the walker in front of it or the readers behind it, so1053// it sat between a queue that was always full and one that was always empty:1054// the readers spent the run parked, and every file the stat goroutine produced1055// had to wake a sleeping thread to hand it over. On the Linux kernel that cost1056// 100,000 futex calls and roughly a third of the wall clock. A pool keeps the1057// read queue non-empty, so the readers stay on their cores and the wakes go1058// away — futex calls drop to about 6,000 and the count runs 1.5x faster.1059//1060// newFileJob is safe to call concurrently; its caches are sync.Maps.1061// atLeastOneWorker keeps a pool size sane. A count of zero, which the flags1062// accept, would start no goroutines at all: nothing would drain the queue, the1063// stage below would see it closed and the run would report a tree of no files1064// and exit successfully. A run that counts nothing and says so quietly is worse1065// than one that is slow, so the floor is one worker.1066func atLeastOneWorker(workers int) int {1067	if workers < 1 {1068		return 11069	}10701071	return workers1072}10731074func startFileJobProducer(potentialFilesQueue chan []*gocodewalker.File, filePaths []string, excludePathRegexes []*regexp.Regexp, fileListQueue chan *FileJob) {1075	var wg sync.WaitGroup10761077	if potentialFilesQueue != nil {1078		for i := 0; i < atLeastOneWorker(FileListJobWorkers); i++ {1079			wg.Go(func() {1080				for fis := range potentialFilesQueue {1081					for _, fi := range fis {1082						if pathExcluded(fi.Location, excludePathRegexes) {1083							continue1084						}10851086						// Ask the name first. Whether a file is one this run counts1087						// at all is settled by its extension and the include and1088						// exclude lists, none of which need a system call, so a run1089						// told to count one language does not stat the rest of the1090						// tree to find that out.1091						guess, ok := detectForJob(fi.Location, fi.Filename)1092						if !ok {1093							continue1094						}10951096						fileInfo, err := os.Lstat(fi.Location)1097						if err != nil || fileInfo.IsDir() {1098							continue1099						}11001101						if fileJob := newFileJobGuessed(fi.Location, fi.Filename, fileInfo, guess); fileJob != nil {1102							fileListQueue <- fileJob1103						}1104					}1105				}1106			})1107		}1108	}11091110	go func() {1111		for _, f := range filePaths {1112			fileInfo, err := os.Lstat(f)1113			if err != nil {1114				continue1115			}11161117			if fileJob := newFileJob(f, f, fileInfo); fileJob != nil {1118				fileListQueue <- fileJob1119			}1120		}11211122		wg.Wait()1123		close(fileListQueue)1124	}()1125}11261127// Reads and processes files from input chan in parallel, and sends results to1128// output chan1129func (ctx processorContext) fileProcessorWorker(input chan *FileJob, output chan *FileJob) {1130	var startTime int641131	var fileCount int641132	var gcEnabled int641133	var wg sync.WaitGroup11341135	for i := 0; i < atLeastOneWorker(FileProcessJobWorkers); i++ {1136		wg.Go(func() {1137			reader := NewFileReader()11381139			// Each worker folds into totals of its own, which nothing else1140			// touches for the length of the run, and merges them once when it1141			// is done. See summariseInWorkers.1142			var totals *summaryTotals1143			if ctx.summary != nil {1144				totals = newSummaryTotals(false, false, false)1145				defer func() { ctx.summary.mergeLocked(totals) }()1146			}11471148			for job := range input {1149				atomic.CompareAndSwapInt64(&startTime, 0, makeTimestampMilli())11501151				loc := job.Location1152				if job.Symlocation != "" {1153					loc = job.Symlocation1154				}11551156				fileStartTime := makeTimestampNano()1157				content, err := reader.ReadFile(loc, int(job.Bytes))1158				atomic.AddInt64(&fileCount, 1)11591160				if atomic.LoadInt64(&gcEnabled) == 0 && atomic.LoadInt64(&fileCount) >= int64(GcFileCount) {1161					debug.SetGCPercent(gcPercent)1162					atomic.AddInt64(&gcEnabled, 1)1163					printWarn("read file limit exceeded GC re-enabled")1164				}11651166				printTraceF("nanoseconds read into memory: %s: %d", job.Location, makeTimestampNano()-fileStartTime)11671168				if err == nil {1169					job.Content = content1170					if ctx.processFile(job) {1171						if totals != nil {1172							totals.add(job)1173						} else {1174							output <- job1175						}1176					}1177				} else {1178					printWarnF("error reading: %s %s", job.Location, err)1179				}1180			}11811182		})1183	}11841185	go func() {1186		wg.Wait()1187		close(output)11881189		printDebugF("milliseconds reading files into memory: %d", makeTimestampMilli()-startTime)1190	}()1191}11921193// Process a single file1194// File must have been read to job.Content already1195func (ctx processorContext) processFile(job *FileJob) bool {1196	fileStartTime := makeTimestampNano()11971198	contents := job.Content11991200	// Needs to always run to ensure the language is set1201	job.Language = DetermineLanguage(job.Filename, job.Language, job.PossibleLanguages, job.Content)12021203	remapped := false1204	if len(ctx.remap.all) != 0 {1205		ctx.hardRemapLanguage(job)1206	}12071208	// If the type is #! we should check to see if we can identify1209	if job.Language == SheBang {1210		if len(ctx.remap.unknown) != 0 {1211			remapped = ctx.unknownRemapLanguage(job)1212		}12131214		// if we didn't remap we then want to see if it's a #! map1215		if !remapped {1216			cutoff := min(200, len(contents))12171218			lang, err := DetectSheBang(contents[:cutoff])1219			if err != nil {1220				printWarnF("unable to determine #! language for %s", job.Location)1221				return false1222			}12231224			printWarnF("detected #! %s for %s", lang, job.Location)1225			job.Language = lang1226			LoadLanguageFeature(lang)1227		}1228	}12291230	CountStats(job)12311232	if Duplicates {1233		// Over the whole of the file, not the bytes the counter happened to1234		// look at. The digest used to be fed a byte at a time from codeState,1235		// which skipped comments and whitespace, so two files with the same1236		// code and different comments hashed alike without counting alike.1237		// Which of them survived was then whichever worker got here first, and1238		// the totals moved from run to run: the same tree answered ten1239		// different line counts in ten runs. Hashing the file itself makes a1240		// duplicate set a set of identical files, which count identically, so1241		// it no longer matters which one is kept.1242		sum := blake2b.Sum256(job.Content)1243		jobHash := sum[:]12441245		duplicates.mux.Lock()1246		if duplicates.Check(job.Bytes, jobHash) {1247			printWarnF("skipping duplicate file: %s", job.Location)1248			duplicates.mux.Unlock()1249			return false1250		}12511252		duplicates.Add(job.Bytes, jobHash)1253		duplicates.mux.Unlock()1254	}12551256	if IgnoreMinified && job.Minified {1257		printWarnF("skipping minified file: %s", job.Location)1258		return false1259	}12601261	if IgnoreGenerated && job.Generated {1262		printWarnF("skipping generated file: %s", job.Location)1263		return false1264	}12651266	if NoLarge && job.Lines >= LargeLineCount {1267		printWarnF("skipping large file due to line length: %s", job.Location)1268		return false1269	}12701271	printTraceF("nanoseconds process: %s: %d", job.Location, makeTimestampNano()-fileStartTime)12721273	if job.Binary {1274		printWarnF("skipping file identified as binary: %s", job.Location)1275		return false1276	}12771278	// This needs to be at the end so we can ensure duplicate detection et.al run first1279	// avoiding inflating the counts1280	if UlocMode {1281		ulocMutex.Lock()12821283		for l := range strings.SplitSeq(strings.TrimRight(string(job.Content), "\n"), "\n") {1284			ulocGlobalCount[l] = struct{}{}12851286			_, ok := ulocLanguageCount[job.Language]1287			if !ok {1288				ulocLanguageCount[job.Language] = map[string]struct{}{}1289			}1290			ulocLanguageCount[job.Language][l] = struct{}{}1291		}1292		ulocMutex.Unlock()1293	}12941295	return true1296}12971298func (ctx processorContext) hardRemapLanguage(job *FileJob) bool {1299	remapped := false1300	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look13011302	for _, rule := range ctx.remap.all {1303		if bytes.Contains(job.Content[:cutoff], rule.pattern) {1304			job.Language = rule.language1305			remapped = true1306			printWarnF("hard remapping: %s to %s", job.Location, job.Language)1307		}1308	}13091310	return remapped1311}13121313func (ctx processorContext) unknownRemapLanguage(job *FileJob) bool {1314	remapped := false1315	cutoff := min(1000, len(job.Content)) // at most 1000 bytes into the file to look13161317	for _, rule := range ctx.remap.unknown {1318		if bytes.Contains(job.Content[:cutoff], rule.pattern) {1319			job.Language = rule.language1320			remapped = true1321			printWarnF("unknown remapping: %s to %s", job.Location, job.Language)1322		}1323	}13241325	return remapped1326}13271328// countLoopGeneric walks a file byte by byte for any language, driven by the1329// tries built from its entry in languages.json. It returns false when it ended1330// the count early, which leaves the work after the loop undone.1331func countLoopGeneric(fileJob *FileJob, langFeatures LanguageFeature, bomSkip, endPoint int, currentState int64, endString []byte, endComments [][]byte, ignoreEscape bool) bool {1332	//We want to track cognitive complexity nesting. Cognitive complexity1333	//means we assign higher complexity to nested branch conditions, so1334	//1335	//if something:1336	//    if otherthing:1337	//1338	//would be assigned a higher complexity than1339	//1340	//if something:1341	//if otherthing:1342	//1343	//because the nested if requires more mental overhead. To do this we need to track1344	//how nested each condition is when we hit it. We do this by counting the number of1345	//whitespace characters are in front of the condition.1346	//This is an appoximation, true for languages like Python, and probably true for anything1347	//else. However, the benefit of this approach is that it's almost free from a CPU point of view1348	//and the increase in spotting complex code, is genuinely useful.1349	var indentStack []int1350	lineStart := bomSkip1351	needIndent := true13521353	content := fileJob.Content1354	// Hoisted because neither changes for the life of the loop and both are1355	// read on every byte of the file.1356	byteType := fileJob.ContentByteType1357	total := int(fileJob.Bytes)13581359	for index := bomSkip; index < total; index++ {1360		curByte := content[index]13611362		if byteType != nil {1363			byteType[index] = stateToByteType(currentState)1364		} else if index < endPoint && isBlankRun[curByte] {1365			// A run of spaces, tabs and carriage returns changes nothing: no1366			// state moves, no line ends, and the only byte of it the loop has1367			// anything to say about is the last. Walking it a byte at a time1368			// costs the whole of the loop body for each one, so find the end of1369			// the run and carry on from there. The newline is deliberately not1370			// in the table, because a line ends on it.1371			next := index + 11372			for next < endPoint && isBlankRun[content[next]] {1373				next++1374			}1375			index = next1376			curByte = content[index]1377		}13781379		// Based on our current state determine if the state should change by checking1380		// what the character is. The below is very CPU bound so need to be careful if1381		// changing anything in here and profile/measure afterwards!1382		// NB that the order of the if statements matters and has been set to what in benchmarks is most efficient1383		if !isWhitespace(curByte) {13841385			// At the first non-whitespace byte of a code-bearing line, update the1386			// indent stack so complexity tokens on this line are weighted by their1387			// nesting depth. Lines that begin a comment must not move the stack;1388			// lines inside a multiline comment/string never reach here in a1389			// blank-derived state so they are excluded automatically.1390			if Cognitive && needIndent && (currentState == SBlank || currentState == SMulticommentBlank) {1391				if tokenType, _, _ := langFeatures.Tokens.Match(fileJob.Content[index:]); tokenType != TSlcomment && tokenType != TMlcomment {1392					indent := index - lineStart1393					for len(indentStack) > 0 && indent < indentStack[len(indentStack)-1] {1394						indentStack = indentStack[:len(indentStack)-1]1395					}1396					if len(indentStack) == 0 || indent > indentStack[len(indentStack)-1] {1397						indentStack = append(indentStack, indent)1398					}1399					nesting := len(indentStack) - 11400					if nesting < 0 {1401						nesting = 01402					}1403					fileJob.cognitiveNesting = nesting1404					needIndent = false1405				}1406			}14071408			switch currentState {1409			case SCode:1410				index, currentState, endString, endComments, ignoreEscape = codeState(1411					fileJob,1412					index,1413					endPoint,1414					currentState,1415					endString,1416					endComments,1417					langFeatures,1418				)1419			case SString:1420				index, currentState = stringState(fileJob, index, endPoint, endString, currentState, ignoreEscape, langFeatures.Escape)1421			case SDocString:1422				// For a docstring we can either move into blank in which case we count it as a docstring1423				// or back into code in which case it should be counted as code1424				index, currentState = docStringState(fileJob, index, endPoint, endString, currentState)1425			case SMulticomment, SMulticommentCode:1426				index, currentState, endString, endComments = commentState(1427					fileJob,1428					index,1429					endPoint,1430					currentState,1431					endComments,1432					endString,1433					langFeatures,1434				)1435			case SComment, SCommentCode:1436				// A line comment runs to the end of the line and nothing in it1437				// can move the state, so the only byte of it the loop has1438				// anything to say about is the newline that ends it. Before1439				// this case existed the switch had none for either comment1440				// state and every byte of a line comment walked the whole loop1441				// body to do nothing at all, which on a tree of LLVM IR was1442				// nine tenths of every iteration the loop ran.1443				// byteType wants a classification written for every byte1444				// of the file, so the skip is only for a plain count.1445				if byteType == nil {1446					if j := bytes.IndexByte(content[index:endPoint], '\n'); j >= 0 {1447						index += j1448					} else {1449						// endPoint is Bytes-1, so landing on it lands on the1450						// last byte of the file and the line-end test below1451						// takes it. This is only here to save the iterations1452						// between here and there - dropping it answers the1453						// same, a byte at a time - but it is the one line that1454						// puts index somewhere the walk did not. Were endPoint1455						// ever to become len(content), it would jump the loop1456						// past its own bound and the count would stop early1457						// with no error, so the two have to move together.1458						index = endPoint1459					}1460				}1461			case SBlank, SMulticommentBlank:1462				// From blank we can move into comment, move into a multiline comment1463				// or move into code but we can only do one.1464				index, currentState, endString, endComments, ignoreEscape = blankState(1465					fileJob,1466					index,1467					currentState,1468					endComments,1469					endString,1470					langFeatures,1471				)1472			}14731474			// Only a state above moves the index or marks the file binary, so1475			// both of the checks that follow belong here rather than on the1476			// whitespace the loop walked over to get to one.14771478			// We shouldn't normally need this, but unclosed strings or comments1479			// might leave the index past the end of the file when we reach this1480			// point.1481			if index >= len(content) {1482				return false1483			}14841485			// Only check the first 10000 characters for null bytes indicating a binary file1486			// and if we find it then we return otherwise carry on and ignore binary markers1487			if index < 10000 && fileJob.Binary {1488				return false1489			}14901491			curByte = content[index]1492		}14931494		// This means the end of processing the line so calculate the stats according to what state1495		// we are currently in1496		if curByte == '\n' || index >= endPoint {1497			fileJob.Lines++1498			if Cognitive {1499				lineStart = index + 11500				needIndent = true1501			}1502			if fileJob.TrackComplexityLines {1503				fileJob.ComplexityLine = append(fileJob.ComplexityLine, 0)1504				if Cognitive {1505					fileJob.CognitiveLine = append(fileJob.CognitiveLine, 0)1506				}1507			}15081509			// Whether this line hands its state to the next one changes for a1510			// language that splices, so work out once whether it ends in a splice1511			// and let resetLineState answer for both of the branches below.1512			spliced := false1513			if langFeatures.LineSplice {1514				spliced = endsWithLineSplice(fileJob.Content, index)1515			}15161517			if NoLarge && fileJob.Lines >= LargeLineCount {1518				// Save memory by unsetting the content as we no longer require it1519				fileJob.Content = nil1520				return false1521			}15221523			switch currentState {1524			case SCode, SString, SCommentCode, SMulticommentCode:1525				fileJob.Code++1526				if langFeatures.LineSplice {1527					currentState = resetLineState(currentState, spliced, ignoreEscape)1528				} else {1529					currentState = resetState(currentState)1530				}1531				// A string that the newline closes, such as a Zig \\ line, ends1532				// with its line. stringState stops on every newline before it1533				// looks for a closer, so it never closes one itself.1534				if currentState == SString && len(endString) == 1 && endString[0] == '\n' {1535					currentState = SBlank1536				}1537				if fileJob.Callback != nil {1538					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_CODE) {1539						return false1540					}1541				}1542				if Trace {1543					// Don't remove the outside if-statements, for performance1544					printTraceF("%s line %d ended with state: %d: counted as code", fileJob.Location, fileJob.Lines, currentState)1545				}1546			case SComment, SMulticomment, SMulticommentBlank:1547				fileJob.Comment++1548				if langFeatures.LineSplice {1549					currentState = resetLineState(currentState, spliced, ignoreEscape)1550				} else {1551					currentState = resetState(currentState)1552				}1553				if fileJob.Callback != nil {1554					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_COMMENT) {1555						return false1556					}1557				}1558				if Trace {1559					// Same as above1560					printTraceF("%s line %d ended with state: %d: counted as comment", fileJob.Location, fileJob.Lines, currentState)1561				}1562			case SBlank:1563				fileJob.Blank++1564				if fileJob.Callback != nil {1565					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_BLANK) {1566						return false1567					}1568				}1569				if Trace {1570					// Same as above1571					printTraceF("%s line %d ended with state: %d: counted as blank", fileJob.Location, fileJob.Lines, currentState)1572				}1573			case SDocString:1574				fileJob.Comment++1575				if fileJob.Callback != nil {1576					if !fileJob.Callback.ProcessLine(fileJob, fileJob.Lines, LINE_COMMENT) {1577						return false1578					}1579				}1580				if Trace {1581					// Same as above1582					printTraceF("%s line %d ended with state: %d: counted as comment", fileJob.Location, fileJob.Lines, currentState)1583				}1584			}1585		}1586	}15871588	return true1589}