1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"hash"8	"regexp"9	"slices"10	"sync"1112	jsoniter "github.com/json-iterator/go"13)1415// cognitiveJSON is the marshaler used by the FileJob / LanguageSummary16// MarshalJSON hooks. Matched to the one formatters_json.go uses so nested17// marshaling produces byte-identical output.18var cognitiveJSON = jsoniter.ConfigCompatibleWithStandardLibrary1920// Used by trie structure to store the types21const (22	TString int = iota + 123	TSlcomment24	TMlcomment25	TComplexity26	TComplexityPostfix27)2829// ByteType constants for per-byte content classification.30// When FileJob.ClassifyContent is true, CountStats populates31// FileJob.ContentByteType with one of these values per byte.32const (33	ByteTypeBlank   byte = 034	ByteTypeCode    byte = 135	ByteTypeComment byte = 236	ByteTypeString  byte = 337)3839// Quote is a struct which holds rules and start/end values for string quotes40type Quote struct {41	Start        string `json:"start"`42	End          string `json:"end"`43	IgnoreEscape bool   `json:"ignoreEscape"` // To enable turning off the \ check for C# @"\" string examples https://github.com/boyter/scc/issues/7144	DocString    bool   `json:"docString"`    // To enable docstring check for Python where "If the triple quote string starts following a newline with only white-space characters in front and ends followed by only a newline or white-space characters it is a comment" https://github.com/boyter/scc/issues/6245}4647// Heuristic is a regex pattern used to disambiguate shared file extensions (for48// example .h between C / C++ / Objective-C) along with a cheap set of necessary49// string literals. The expensive regex is only run when one of Literals is50// present in the content, which is a fast reject for the overwhelmingly common51// case where the file is not the language being guessed. See guessByHeuristics.52type Heuristic struct {53	// Pattern is the regex evaluated against the file content.54	Pattern string `json:"pattern"`55	// Literals is the set of substrings of which at least one must be present56	// (case sensitive) for Pattern to have any chance of matching. When empty57	// the regex is always run, so a pattern is never silently disabled.58	Literals []string `json:"literals"`59	// Anchored, when true, requires each literal to sit at the start of a line60	// preceded only by spaces or tabs. This mirrors the (?m)^[ \t]* prefix used61	// by the keyword patterns and avoids false positives such as the substring62	// "entry" satisfying a check for the "try" keyword.63	Anchored bool `json:"anchored"`64}6566// Language is a struct which contains the values for each language stored in languages.json67type Language struct {68	LineComment                     []string    `json:"line_comment"`69	ComplexityChecks                []string    `json:"complexitychecks"`70	ComplexityChecksPostfix         []string    `json:"complexitychecks_postfix"`71	ComplexityChecksPostfixExcludes []string    `json:"complexitychecks_postfix_excludes"`72	Extensions                      []string    `json:"extensions"`73	MultiLine                       [][]string  `json:"multi_line"`74	Quotes                          []Quote     `json:"quotes"`75	Keywords                        []string    `json:"keywords"`76	Heuristics                      []Heuristic `json:"heuristics"`77	FileNames                       []string    `json:"filenames"`78	SheBangs                        []string    `json:"shebangs"`79	ExtensionFile                   bool        `json:"extensionFile"`80	NestedMultiLine                 bool        `json:"nestedmultiline"`81}8283// LanguageFeature is a struct which represents the conversion from Language into what is used for matching84type LanguageFeature struct {85	Complexity            *Trie86	MultiLineComments     *Trie87	MultiLine             [][]string // in case someone needs the actual value88	SingleLineComments    *Trie89	LineComment           []string // in case someone needs the actual value90	Strings               *Trie91	Tokens                *Trie92	Nested                bool93	PostfixExcludes       [][]byte94	ComplexityCheckMask   byte95	SingleLineCommentMask byte96	MultiLineCommentMask  byte97	StringCheckMask       byte98	ProcessMask           byte99	Keywords              []string100	KeywordBytes          [][]byte101	Heuristics            []CompiledHeuristic102	Quotes                []Quote103}104105// CompiledHeuristic is the runtime form of a Heuristic with its regex compiled106// and its literals pre-converted to bytes for matching.107type CompiledHeuristic struct {108	Re       *regexp.Regexp109	Literals [][]byte110	Anchored bool111}112113// FileJobCallback is an interface that FileJobs can implement to get a per line callback with the line type114type FileJobCallback interface {115	// ProcessLine should return true to continue processing or false to stop further processing and return116	ProcessLine(job *FileJob, currentLine int64, lineType LineType) bool117}118119// FileJob is a struct used to hold all of the results of processing internally before sent to the formatter120type FileJob struct {121	Language             string122	PossibleLanguages    []string // Used to hold potentially more than one language which populates language when determined123	Filename             string124	Extension            string125	Location             string126	Symlocation          string127	Content              []byte `json:"-"`128	Bytes                int64129	Lines                int64130	Code                 int64131	Comment              int64132	Blank                int64133	Complexity           int64134	Cognitive            int64   // nesting-weighted complexity; JSON emission is gated on the Cognitive global via MarshalJSON135	ComplexityLine       []int64 `json:"-"`136	CognitiveLine        []int64 `json:"-"` // per-line cognitive weight; populated only when TrackComplexityLines and Cognitive are both enabled137	WeightedComplexity   float64138	Hash                 hash.Hash139	Callback             FileJobCallback `json:"-"`140	Binary               bool141	Minified             bool142	Generated            bool143	EndPoint             int144	Uloc                 int145	LineLength           []int  `json:"-"`146	ClassifyContent      bool   `json:"-"` // When true, CountStats populates ContentByteType147	ContentByteType      []byte `json:"-"` // Per-byte classification, allocated by CountStats when ClassifyContent is true148	TrackComplexityLines bool   `json:"-"` // When true, CountStats populates ComplexityLine149	cognitiveNesting     int    // transient per-line nesting level used during CountStats when Cognitive is enabled150}151152// MarshalJSON emits FileJob with the Cognitive field present (even when 0) while153// the Cognitive global is on, and omitted entirely when it is off. A static154// `omitempty` tag cannot express this because it would also drop a legitimately155// zero cognitive value on a branch-free file while the metric is active. The156// unexported alias type carries the same field tags but none of the methods, so157// marshaling it does not recurse.158func (fileJob *FileJob) MarshalJSON() ([]byte, error) {159	type alias FileJob160	if Cognitive {161		return cognitiveJSON.Marshal((*alias)(fileJob))162	}163	// Shadow the promoted Cognitive with a zero-valued omitempty field of the164	// same JSON name: the shallower field wins the name, then omitempty drops165	// it, so the key is absent. (A json:"-" shadow would be removed before166	// conflict resolution, letting the embedded field resurface.)167	return cognitiveJSON.Marshal(struct {168		*alias169		Cognitive int64 `json:"Cognitive,omitempty"`170	}{alias: (*alias)(fileJob)})171}172173// FilterContentByType returns a copy of Content with bytes not matching any of174// the given types replaced by spaces. Newlines are always preserved regardless175// of type. Returns nil if ContentByteType is nil.176func (fj *FileJob) FilterContentByType(keepTypes ...byte) []byte {177	if fj.ContentByteType == nil {178		return nil179	}180181	keep := make(map[byte]bool, len(keepTypes))182	for _, t := range keepTypes {183		keep[t] = true184	}185186	result := make([]byte, len(fj.Content))187	for i, b := range fj.Content {188		if b == '\n' || keep[fj.ContentByteType[i]] {189			result[i] = b190		} else {191			result[i] = ' '192		}193	}194	return result195}196197// LanguageSummary is used to hold summarized results for a single language198type LanguageSummary struct {199	Name               string200	Bytes              int64201	CodeBytes          int64202	Lines              int64203	Code               int64204	Comment            int64205	Blank              int64206	Complexity         int64207	Cognitive          int64 // nesting-weighted complexity; JSON emission is gated on the Cognitive global via MarshalJSON208	Count              int64209	WeightedComplexity float64210	Files              []*FileJob211	LineLength         []int212	ULOC               int213	CodePercent        *float64 `json:",omitempty"`214	CommentPercent     *float64 `json:",omitempty"`215	BlankPercent       *float64 `json:",omitempty"`216	LinePercent        *float64 `json:",omitempty"`217	ComplexityPercent  *float64 `json:",omitempty"`218	BytePercent        *float64 `json:",omitempty"`219	FilePercent        *float64 `json:",omitempty"`220}221222// MarshalJSON gates the language-level Cognitive field on the Cognitive global,223// mirroring FileJob.MarshalJSON: present (even at 0) when the metric is on,224// omitted when off. See FileJob.MarshalJSON for the rationale.225func (l LanguageSummary) MarshalJSON() ([]byte, error) {226	type alias LanguageSummary227	if Cognitive {228		return cognitiveJSON.Marshal(alias(l))229	}230	return cognitiveJSON.Marshal(struct {231		alias232		Cognitive int64 `json:"Cognitive,omitempty"`233	}{alias: alias(l)})234}235236// OpenClose is used to hold an open/close pair for matching such as multi line comments237type OpenClose struct {238	Open  []byte239	Close []byte240}241242// CheckDuplicates is used to hold hashes if duplicate detection is enabled it comes with a mutex243// that should be locked while a check is being performed then added244type CheckDuplicates struct {245	hashes map[int64][][]byte246	mux    sync.Mutex247}248249// Add is a non thread safe add a key into the duplicates check need to use mutex inside struct before calling this250func (c *CheckDuplicates) Add(key int64, hash []byte) {251	hashes, ok := c.hashes[key]252	if ok {253		c.hashes[key] = append(hashes, hash)254	} else {255		c.hashes[key] = [][]byte{hash}256	}257}258259// Check is a non thread safe check to see if the key exists already need to use mutex inside struct before calling this260func (c *CheckDuplicates) Check(key int64, hash []byte) bool {261	hashes, ok := c.hashes[key]262263	return ok && slices.ContainsFunc(hashes, func(h []byte) bool {264		return bytes.Equal(h, hash)265	})266}267268// Trie is a structure used to store matches efficiently269type Trie struct {270	Type  int271	Close []byte272	Table [256]*Trie273}274275// Insert inserts a string into the trie for matching276func (root *Trie) Insert(tokenType int, token []byte) {277	var node *Trie278279	node = root280	for _, c := range token {281		if node.Table[c] == nil {282			node.Table[c] = &Trie{}283		}284		node = node.Table[c]285	}286	node.Type = tokenType287}288289// InsertClose closes off a string in the trie290func (root *Trie) InsertClose(tokenType int, openToken, closeToken []byte) {291	var node *Trie292293	node = root294	for _, c := range openToken {295		if node.Table[c] == nil {296			node.Table[c] = &Trie{}297		}298		node = node.Table[c]299	}300	node.Type = tokenType301	node.Close = closeToken302}303304// Match checks the created trie structure for a match305func (root *Trie) Match(token []byte) (int, int, []byte) {306	var node *Trie307	var depth int308	var c byte309310	node = root311	var prevClosedNode *Trie312	var prevClosedDepth int313	for depth, c = range token {314		if node.Table[c] == nil {315			break316		}317		node = node.Table[c]318		if len(node.Close) > 0 {319			prevClosedNode = node320			prevClosedDepth = depth321		}322	}323	if len(node.Close) == 0 && prevClosedNode != nil {324		return prevClosedNode.Type, prevClosedDepth, prevClosedNode.Close325	}326	return node.Type, depth, node.Close327}