1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"reflect"8	"strings"9	"testing"10)1112func (job *FileJob) SetContent(content string) {13	job.Content = []byte(content)14	job.Bytes = int64(len(job.Content))15}1617func TestIsWhitespace(t *testing.T) {18	if !isWhitespace(' ') {19		t.Errorf("Expected to be true")20	}21}2223func TestIsBinaryTrue(t *testing.T) {24	DisableCheckBinary = false2526	if !isBinary(0, 0) {27		t.Errorf("Expected to be true")28	}29}3031func TestIsBinaryDisableCheck(t *testing.T) {32	DisableCheckBinary = true3334	if isBinary(0, 0) {35		t.Errorf("Expected to be false")36	}37}3839func TestCountStatsLines(t *testing.T) {40	Trace = false41	Debug = false42	Verbose = false4344	fileJob := FileJob{45		Content: []byte(""),46		Lines:   0,47	}4849	// Both tokei and sloccount count this as 0 so lets follow suit50	// cloc ignores the file itself because it is empty51	CountStats(&fileJob)52	if fileJob.Lines != 0 {53		t.Errorf("Zero lines expected got %d", fileJob.Lines)54	}5556	// Interestingly this file would be 0 lines in "wc -l" because it only counts newlines57	// all others count this as 158	fileJob.Lines = 059	fileJob.SetContent("a")60	CountStats(&fileJob)61	if fileJob.Lines != 1 {62		t.Errorf("One line expected got %d", fileJob.Lines)63	}6465	fileJob.Lines = 066	fileJob.SetContent("a\n")67	CountStats(&fileJob)68	if fileJob.Lines != 1 {69		t.Errorf("One line expected got %d", fileJob.Lines)70	}7172	// tokei counts this as 1 because it's still on a single line unless something follows73	// the newline it's still 1 line74	fileJob.Lines = 075	fileJob.SetContent("1\n")76	CountStats(&fileJob)77	if fileJob.Lines != 1 {78		t.Errorf("One line expected got %d", fileJob.Lines)79	}8081	fileJob.Lines = 082	fileJob.SetContent("1\n2\n")83	CountStats(&fileJob)84	if fileJob.Lines != 2 {85		t.Errorf("Two lines expected got %d", fileJob.Lines)86	}8788	fileJob.Lines = 089	fileJob.SetContent("1\n2\n3")90	CountStats(&fileJob)91	if fileJob.Lines != 3 {92		t.Errorf("Three lines expected got %d", fileJob.Lines)93	}9495	content := ""96	for i := range 5000 {97		content += "a\n"98		fileJob.Lines = 099		fileJob.SetContent(content)100		CountStats(&fileJob)101		if fileJob.Lines != int64(i+1) {102			t.Errorf("Expected %d got %d", i+1, fileJob.Lines)103		}104	}105}106107func TestCountStatsCode(t *testing.T) {108	fileJob := FileJob{109		Content: []byte(""),110		Code:    0,111	}112113	// Both tokei and sloccount count this as 0 so lets follow suit114	// cloc ignores the file itself because it is empty115	CountStats(&fileJob)116	if fileJob.Code != 0 {117		t.Errorf("Zero lines expected got %d", fileJob.Code)118	}119120	// Interestingly this file would be 0 lines in "wc -l" because it only counts newlines121	// all others count this as 1122	fileJob.Code = 0123	fileJob.SetContent("a")124	CountStats(&fileJob)125	if fileJob.Code != 1 {126		t.Errorf("One line expected got %d", fileJob.Code)127	}128129	fileJob.Code = 0130	fileJob.SetContent("i++ # comment")131	CountStats(&fileJob)132	if fileJob.Code != 1 {133		t.Errorf("One line expected got %d", fileJob.Code)134	}135136	fileJob.Code = 0137	fileJob.SetContent("i++ // comment")138	CountStats(&fileJob)139	if fileJob.Code != 1 {140		t.Errorf("One line expected got %d", fileJob.Code)141	}142143	fileJob.Code = 0144	fileJob.SetContent("a\n")145	CountStats(&fileJob)146	if fileJob.Code != 1 {147		t.Errorf("One line expected got %d", fileJob.Code)148	}149150	// tokei counts this as 1 because it's still on a single line unless something follows151	// the newline it's still 1 line152	fileJob.Code = 0153	fileJob.SetContent("1\n")154	CountStats(&fileJob)155	if fileJob.Code != 1 {156		t.Errorf("One line expected got %d", fileJob.Code)157	}158159	fileJob.Code = 0160	fileJob.SetContent("1\n2\n")161	CountStats(&fileJob)162	if fileJob.Code != 2 {163		t.Errorf("Two lines expected got %d", fileJob.Code)164	}165166	fileJob.Code = 0167	fileJob.SetContent("1\n2\n3")168	CountStats(&fileJob)169	if fileJob.Code != 3 {170		t.Errorf("Three lines expected got %d", fileJob.Code)171	}172173	content := ""174	for i := range 100 {175		content += "a\n"176		fileJob.Code = 0177		fileJob.SetContent(content)178		CountStats(&fileJob)179		if fileJob.Code != int64(i+1) {180			t.Errorf("Expected %d got %d", i+1, fileJob.Code)181		}182	}183}184185func TestCountStatsWithQuotes(t *testing.T) {186	fileJob := FileJob{}187188	fileJob.Code = 0189	fileJob.Comment = 0190	fileJob.Complexity = 0191	fileJob.SetContent(`var test = "/*";`)192	CountStats(&fileJob)193	if fileJob.Code != 1 {194		t.Errorf("One line expected got %d", fileJob.Code)195	}196	if fileJob.Comment != 0 {197		t.Errorf("No line expected got %d", fileJob.Comment)198	}199200	fileJob.Code = 0201	fileJob.Comment = 0202	fileJob.Complexity = 0203	fileJob.SetContent(`t = " if ";`)204	CountStats(&fileJob)205	if fileJob.Code != 1 {206		t.Errorf("One line expected got %d", fileJob.Code)207	}208	if fileJob.Comment != 0 {209		t.Errorf("No line expected got %d", fileJob.Comment)210	}211	if fileJob.Complexity != 0 {212		t.Errorf("No line expected got %d", fileJob.Complexity)213	}214215	fileJob.Code = 0216	fileJob.Comment = 0217	fileJob.Complexity = 0218	fileJob.SetContent(`t = " if switch for while do loop != == && || ";`)219	CountStats(&fileJob)220	if fileJob.Code != 1 {221		t.Errorf("One line expected got %d", fileJob.Code)222	}223	if fileJob.Comment != 0 {224		t.Errorf("No line expected got %d", fileJob.Comment)225	}226	if fileJob.Complexity != 0 {227		t.Errorf("No line expected got %d", fileJob.Complexity)228	}229}230231func TestCountStatsBlankLines(t *testing.T) {232	fileJob := FileJob{233		Content: []byte(""),234		Blank:   0,235	}236237	CountStats(&fileJob)238	if fileJob.Blank != 0 {239		t.Errorf("Zero lines expected got %d", fileJob.Blank)240	}241242	fileJob.Blank = 0243	fileJob.SetContent(" ")244	CountStats(&fileJob)245	if fileJob.Blank != 1 {246		t.Errorf("One line expected got %d", fileJob.Blank)247	}248249	fileJob.Blank = 0250	fileJob.SetContent("\n")251	CountStats(&fileJob)252	if fileJob.Blank != 1 {253		t.Errorf("One line expected got %d", fileJob.Blank)254	}255256	fileJob.Blank = 0257	fileJob.SetContent("\n ")258	CountStats(&fileJob)259	if fileJob.Blank != 2 {260		t.Errorf("Two line expected got %d", fileJob.Blank)261	}262263	fileJob.Blank = 0264	fileJob.SetContent("            ")265	CountStats(&fileJob)266	if fileJob.Blank != 1 {267		t.Errorf("One line expected got %d", fileJob.Blank)268	}269270	fileJob.Blank = 0271	fileJob.SetContent("            \n             ")272	CountStats(&fileJob)273	if fileJob.Blank != 2 {274		t.Errorf("Two lines expected got %d", fileJob.Blank)275	}276277	fileJob.Blank = 0278	fileJob.SetContent("\r\n\r\n")279	CountStats(&fileJob)280	if fileJob.Blank != 2 {281		t.Errorf("Two lines expected got %d", fileJob.Blank)282	}283284	fileJob.Blank = 0285	fileJob.SetContent("\r\n")286	CountStats(&fileJob)287	if fileJob.Blank != 1 {288		t.Errorf("One line expected got %d", fileJob.Blank)289	}290}291292func TestCountStatsComplexityCount(t *testing.T) {293	ProcessConstants()294	fileJob := FileJob{}295296	checks := []string{297		"if ",298		"	if ",299		"if a.equals(b) {",300		"if(",301		" if(i.equals(0))",302		"    if(",303		"    if( ",304	}305306	for _, check := range checks {307		fileJob.Complexity = 0308		fileJob.SetContent(check)309		fileJob.Language = "Java"310		CountStats(&fileJob)311		if fileJob.Complexity != 1 {312			t.Errorf("Expected complexity of 1 got %d for %s", fileJob.Complexity, check)313		}314	}315}316317func TestCountStatsComplexityCountFalse(t *testing.T) {318	ProcessConstants()319	fileJob := FileJob{}320321	checks := []string{322		"if",323		"aif ",324		"aif(",325	}326327	for _, check := range checks {328		fileJob.Complexity = 0329		fileJob.SetContent(check)330		fileJob.Language = "Java"331		CountStats(&fileJob)332		if fileJob.Complexity != 0 {333			t.Errorf("Expected complexity of 0 got %d for %s", fileJob.Complexity, check)334		}335	}336337}338339func TestCountStatsComplexityRustQuestionOperator(t *testing.T) {340	ProcessConstants()341342	checks := []struct {343		content string344		want    int64345	}{346		{"foo()?;", 1},347		{"foo() ?;", 1},348		{"foo()?.bar();", 1},349		{"foo()? as u16;", 1},350		{"foo()??;", 2},351		{"let y = x?;", 1},352	}353354	for _, c := range checks {355		fileJob := FileJob{Language: "Rust"}356		fileJob.SetContent(c.content)357		CountStats(&fileJob)358		if fileJob.Complexity != c.want {359			t.Errorf("Expected complexity of %d got %d for %q", c.want, fileJob.Complexity, c.content)360		}361	}362}363364// ?Sized is a trait-bound prefix, not the try operator. The postfix matcher365// must skip it whether or not whitespace separates it from the preceding366// token (`T: ?Sized`, `T:?Sized`, `Debug+?Sized` are all valid Rust).367func TestCountStatsComplexityRustQuestionSizedNotCounted(t *testing.T) {368	ProcessConstants()369370	checks := []string{371		"fn foo<T: ?Sized>() {}",372		"fn foo<T:?Sized>() {}",373		"fn foo<T:? Sized>() {}",374		"struct Bar<T: ?Sized>(T);",375		"where T: Debug + ?Sized,",376		"where T: Debug+?Sized,",377	}378379	for _, content := range checks {380		fileJob := FileJob{Language: "Rust"}381		fileJob.SetContent(content)382		CountStats(&fileJob)383		if fileJob.Complexity != 0 {384			t.Errorf("Expected complexity of 0 got %d for %q (?Sized must not count)", fileJob.Complexity, content)385		}386	}387}388389func TestCountStatsComplexityTypeScriptPostfixOperators(t *testing.T) {390	ProcessConstants()391392	checks := []struct {393		content string394		want    int64395	}{396		{"obj.attr?.method();", 1},397		{"path ?? url;", 1},398		{"path??url;", 1},399		{"value ??= fallback;", 1},400		{"value??=fallback;", 1},401		{"function get(path?: string) { return path ?? url; }", 1},402		{"interface User { age?: number; method?(): void; }", 0},403	}404405	for _, c := range checks {406		fileJob := FileJob{Language: "TypeScript"}407		fileJob.SetContent(c.content)408		CountStats(&fileJob)409		if fileJob.Complexity != c.want {410			t.Errorf("Expected complexity of %d got %d for %q", c.want, fileJob.Complexity, c.content)411		}412	}413}414415type linecounter struct {416	blanks   int417	comments int418	code     int419	loc      int420	stop     bool421}422423func (l *linecounter) ProcessLine(job *FileJob, currentLine int64, lineType LineType) bool {424	l.loc++425	switch lineType {426	case LINE_BLANK:427		l.blanks++428	case LINE_COMMENT:429		l.comments++430	case LINE_CODE:431		l.code++432	}433	return !l.stop434}435436func TestCountStatsCallback(t *testing.T) {437	ProcessConstants()438	fileJob := FileJob{}439440	fileJob.SetContent(`package foo441442import com.foo.bar;443444// this is a comment445class A {446}`)447	var lc linecounter448	fileJob.Language = "Java"449	fileJob.Callback = &lc450	CountStats(&fileJob)451	if lc.loc != 7 {452		t.Errorf("Expected loc of 7 got %d", lc.loc)453	}454	if lc.blanks != 2 {455		t.Errorf("Expected loc of 2 got %d", lc.blanks)456	}457	if lc.comments != 1 {458		t.Errorf("Expected loc of 1 got %d", lc.comments)459	}460	if lc.code != 4 {461		t.Errorf("Expected loc of 4 got %d", lc.code)462	}463}464465func TestCountStatsCallbackInterrupt(t *testing.T) {466	ProcessConstants()467	fileJob := FileJob{}468469	fileJob.SetContent(`package foo470471import com.foo.bar;472473// this is a comment474class A {475}`)476	var lc linecounter477	lc.stop = true478	fileJob.Language = "Java"479	fileJob.Callback = &lc480	CountStats(&fileJob)481	if lc.loc != 1 {482		t.Errorf("Expected loc of 1 got %d", lc.loc)483	}484}485486// Edge case condition where if ending with comment it would be counted487// as code due to how internal state work.488func TestCountStatsEdgeCase1(t *testing.T) {489	ProcessConstants()490	fileJob := FileJob{491		Language: "Java",492	}493494	fileJob.SetContent(`/**/495`)496497	CountStats(&fileJob)498499	if fileJob.Lines != 1 {500		t.Errorf("Expected 1 lines got %d", fileJob.Lines)501	}502503	if fileJob.Comment != 1 {504		t.Errorf("Expected 1 lines got %d", fileJob.Comment)505	}506}507508// Turns out that some languages such as Rust support509// nested comments. Check that it works here510func TestCountStatsNestedComments(t *testing.T) {511	ProcessConstants()512	fileJob := FileJob{513		Language: "Rust",514	}515516	fileJob.SetContent(`/*/**/*/`)517518	CountStats(&fileJob)519520	if fileJob.Lines != 1 {521		t.Errorf("Expected 1 lines got %d", fileJob.Lines)522	}523524	if fileJob.Code != 0 {525		t.Errorf("Expected 0 lines got %d", fileJob.Code)526	}527528	if fileJob.Comment != 1 {529		t.Errorf("Expected 1 lines got %d", fileJob.Comment)530	}531532	if fileJob.Blank != 0 {533		t.Errorf("Expected 0 lines got %d", fileJob.Blank)534	}535}536537// Java does not support nested multiline comments538func TestCountStatsNestedCommentsJava(t *testing.T) {539	ProcessConstants()540	fileJob := FileJob{541		Language: "Java",542	}543544	fileJob.SetContent(`/*/**/*/`)545546	CountStats(&fileJob)547548	if fileJob.Lines != 1 {549		t.Errorf("Expected 1 lines got %d", fileJob.Lines)550	}551552	if fileJob.Code != 1 {553		t.Errorf("Expected 1 lines got %d", fileJob.Code)554	}555556	if fileJob.Comment != 0 {557		t.Errorf("Expected 0 lines got %d", fileJob.Comment)558	}559560	if fileJob.Blank != 0 {561		t.Errorf("Expected 0 lines got %d", fileJob.Blank)562	}563}564565func TestCountStatsNestedCommentsRegression(t *testing.T) {566	ProcessConstants()567	fileJob := FileJob{568		Language: "Rust",569	}570571	fileJob.SetContent(`t/*/**/*/`)572573	CountStats(&fileJob)574575	if fileJob.Lines != 1 {576		t.Errorf("Expected 1 lines got %d", fileJob.Lines)577	}578579	if fileJob.Code != 1 {580		t.Errorf("Expected 1 lines got %d", fileJob.Code)581	}582583	if fileJob.Comment != 0 {584		t.Errorf("Expected 0 lines got %d", fileJob.Comment)585	}586587	if fileJob.Blank != 0 {588		t.Errorf("Expected 0 lines got %d", fileJob.Blank)589	}590}591592func TestCountStatsSingleCommentRegression(t *testing.T) {593	ProcessConstants()594	fileJob := FileJob{595		Language: "Rust",596	}597598	fileJob.SetContent(`t = "599/*600";`)601602	CountStats(&fileJob)603604	if fileJob.Lines != 3 {605		t.Errorf("Expected 3 lines got %d", fileJob.Lines)606	}607608	if fileJob.Code != 3 {609		t.Errorf("Expected 3 lines got %d", fileJob.Code)610	}611612	if fileJob.Comment != 0 {613		t.Errorf("Expected 0 lines got %d", fileJob.Comment)614	}615616	if fileJob.Blank != 0 {617		t.Errorf("Expected 0 lines got %d", fileJob.Blank)618	}619}620621func TestCountStatsStringCheck(t *testing.T) {622	ProcessConstants()623	fileJob := FileJob{624		Language: "Rust",625	}626627	fileJob.SetContent(`let does_not_start = // "628"until here,629test/*630test"; // a quote: "`)631632	CountStats(&fileJob)633634	if fileJob.Lines != 4 {635		t.Errorf("Expected 4 lines got %d", fileJob.Lines)636	}637638	if fileJob.Code != 4 {639		t.Errorf("Expected 4 code lines got %d", fileJob.Code)640	}641642	if fileJob.Comment != 0 {643		t.Errorf("Expected 0 comment lines got %d", fileJob.Comment)644	}645646	if fileJob.Blank != 0 {647		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)648	}649}650651func TestCountStatsBosque(t *testing.T) {652	ProcessConstants()653	fileJob := FileJob{654		Language: "Bosque",655	}656657	fileJob.SetContent(`//This is a bosque test658method offsetMomentum(px: Float, py: Float, pz: Float): Body {659      return this<~(vx=Float::div(px->negate(), Body::solar_mass), vy=Float::div(py->negate(), Body::solar_mass), vz=Float::div(pz->negate(), Body::solar_mass));660}`)661662	CountStats(&fileJob)663664	if fileJob.Lines != 4 {665		t.Errorf("Expected 4 lines got %d", fileJob.Lines)666	}667668	if fileJob.Code != 3 {669		t.Errorf("Expected 4 code lines got %d", fileJob.Code)670	}671672	if fileJob.Comment != 1 {673		t.Errorf("Expected 0 comment lines got %d", fileJob.Comment)674	}675676	if fileJob.Blank != 0 {677		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)678	}679}680681func TestCheckForMatchNoMatch(t *testing.T) {682	ProcessConstants()683684	fileJob := FileJob{685		Language: "Rust",686		Content:  []byte("one does not simply walk into mordor"),687	}688689	matches := &Trie{}690	matches.Insert(TSlcomment, []byte("//"))691	matches.Insert(TSlcomment, []byte("--"))692693	match, _, _ := matches.Match(fileJob.Content)694695	if match != 0 {696		t.Errorf("Expected no match")697	}698}699700func TestCheckForMatchHasMatch(t *testing.T) {701	ProcessConstants()702703	fileJob := FileJob{704		Language: "Rust",705		Content:  []byte("// one does not simply walk into mordor"),706	}707708	matches := &Trie{}709	matches.Insert(TSlcomment, []byte("//"))710	matches.Insert(TSlcomment, []byte("--"))711712	match, _, _ := matches.Match(fileJob.Content)713714	if match != TSlcomment {715		t.Errorf("Expected match")716	}717}718719func TestCheckForMatchSingleNoMatch(t *testing.T) {720	ProcessConstants()721722	fileJob := FileJob{723		Language: "Rust",724		Content:  []byte("// one does not simply walk into mordor"),725	}726727	matches := []byte("*/")728729	match := checkForMatchSingle('/', 0, 100, matches, &fileJob)730731	if match != false {732		t.Errorf("Expected no match")733	}734}735736func TestCheckForMatchSingleMatch(t *testing.T) {737	ProcessConstants()738739	fileJob := FileJob{740		Language: "Rust",741		Content:  []byte("*/ one does not simply walk into mordor"),742	}743744	matches := []byte("*/")745746	match := checkForMatchSingle('*', 0, 100, matches, &fileJob)747748	if match != true {749		t.Errorf("Expected match")750	}751}752753func TestCheckComplexityMatch(t *testing.T) {754	ProcessConstants()755756	fileJob := FileJob{757		Language: "Java",758		Content:  []byte("for (int i=0; i<100; i++) {"),759	}760761	matches := &Trie{}762	matches.Insert(TComplexity, []byte("for "))763	matches.Insert(TComplexity, []byte("for("))764765	match, n, _ := matches.Match(fileJob.Content)766767	if match != TComplexity || n != 4 {768		t.Errorf("Expected match")769	}770}771772func TestCheckComplexityNoMatch(t *testing.T) {773	ProcessConstants()774775	fileJob := FileJob{776		Language: "Java",777		Content:  []byte("far (int i=0; i<100; i++) {"),778	}779780	matches := &Trie{}781	matches.Insert(TComplexity, []byte("for "))782	matches.Insert(TComplexity, []byte("for("))783784	match, _, _ := matches.Match(fileJob.Content)785786	if match != 0 {787		t.Errorf("Expected no match")788	}789}790791func TestCountStatsRubyRegression(t *testing.T) {792	ProcessConstants()793	fileJob := FileJob{794		Language: "Ruby",795	}796797	fileJob.SetContent(`=begin798=end799t`)800801	CountStats(&fileJob)802803	if fileJob.Lines != 3 {804		t.Errorf("Expected 3 lines got %d", fileJob.Lines)805	}806807	if fileJob.Code != 1 {808		t.Errorf("Expected 1 code lines got %d", fileJob.Code)809	}810811	if fileJob.Comment != 2 {812		t.Errorf("Expected 2 comment lines got %d", fileJob.Comment)813	}814815	if fileJob.Blank != 0 {816		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)817	}818}819820func TestFileProcessorWorker(t *testing.T) {821	inputChan := make(chan *FileJob, 10000)822823	inputChan <- &FileJob{824		Filename:  "testing.go",825		Location:  "./",826		Extension: "go",827		Content:   []byte("this is some content"),828	}829830	close(inputChan)831	outputChan := make(chan *FileJob, 10000)832833	Duplicates = true834835	ctx := processorContext{remap: newRemapConfig("", "")}836	ctx.fileProcessorWorker(inputChan, outputChan)837838	for res := range outputChan {839		if res.Bytes == 0 {840			t.Error("Expect bytes to have something")841		}842	}843}844845func TestParseRemapRulesIgnoresInvalidEntries(t *testing.T) {846	rules := parseRemapRules("match:Go,invalid,too:many:parts,:Rust")847848	if len(rules) != 2 {849		t.Fatalf("expected 2 rules, got %d", len(rules))850	}851852	if string(rules[0].pattern) != "match" || rules[0].language != "Go" {853		t.Fatalf("unexpected first rule: %#v", rules[0])854	}855856	if string(rules[1].pattern) != "" || rules[1].language != "Rust" {857		t.Fatalf("unexpected second rule: %#v", rules[1])858	}859}860861func TestHardRemapLanguageUsesParsedRules(t *testing.T) {862	job := &FileJob{863		Language: "Plain Text",864		Location: "./test.txt",865		Content:  []byte("prefix -*- C++ -*- suffix"),866	}867868	ctx := processorContext{remap: newRemapConfig("-*- C++ -*-:C Header", "")}869	remapped := ctx.hardRemapLanguage(job)870871	if !remapped {872		t.Fatal("expected file to be remapped")873	}874875	if job.Language != "C Header" {876		t.Fatalf("expected remapped language to be C Header, got %s", job.Language)877	}878}879880func TestEdgeCase(t *testing.T) {881	ProcessConstants()882	fileJob := FileJob{883		Language: "C#",884	}885886	// For C# we can enter a string using @" or " but if we do the former,887	// and we don't skip over the full length we exit the string in this case888	// which means we pick up the /* and the count is incorrect889	fileJob.SetContent(`@"\ /*"890a`)891892	CountStats(&fileJob)893894	if fileJob.Lines != 2 {895		t.Errorf("Expected 2 lines")896	}897898	if fileJob.Code != 2 {899		t.Errorf("Expected 2 lines got %d", fileJob.Code)900	}901902	if fileJob.Comment != 0 {903		t.Errorf("Expected 0 lines got %d", fileJob.Comment)904	}905}906907func TestEdgeCaseOther(t *testing.T) {908	ProcessConstants()909	fileJob := FileJob{910		Language: "C#",911	}912913	// For C# we can enter a string using @" or " but if we do the former,914	// and we don't skip over the full length we exit the string in this case915	// which means we pick up the /* and the count is incorrect916	fileJob.SetContent(`@"C:\" /*917a */`)918919	CountStats(&fileJob)920921	if fileJob.Lines != 2 {922		t.Errorf("Expected 2 lines")923	}924925	if fileJob.Code != 1 {926		t.Errorf("Expected 1 lines got %d", fileJob.Code)927	}928929	if fileJob.Comment != 1 {930		t.Errorf("Expected 1 lines got %d", fileJob.Comment)931	}932}933934func TestCountStatsCSharpIgnoreEscape(t *testing.T) {935	ProcessConstants()936	fileJob := FileJob{937		Language: "C#",938	}939940	fileJob.SetContent(`namespace Ns941{942   public class Cls943   {944       private const string BasePath = @"a:\";945946       [Fact]947       public void MyTest()948       {949           // Arrange.950           Foo();951952           // Act.953           Bar();954955           // Assert.956           Baz();957       }958   }959}`)960961	CountStats(&fileJob)962963	if fileJob.Lines != 20 {964		t.Errorf("Expected 20 lines")965	}966967	if fileJob.Code != 14 {968		t.Errorf("Expected 14 lines got %d", fileJob.Code)969	}970971	if fileJob.Comment != 3 {972		t.Errorf("Expected 3 lines got %d", fileJob.Comment)973	}974975	if fileJob.Blank != 3 {976		t.Errorf("Expected 3 lines got %d", fileJob.Blank)977	}978}979980func TestCheckBomSkipUTF8(t *testing.T) {981	fileJob := &FileJob{982		Content: []byte{239, 187, 191}, // UTF-8 BOM983	}984985	skip := checkBomSkip(fileJob)986	if skip != 3 {987		t.Errorf("Expected skip length to match 3 got %d", skip)988	}989}990991func TestCheckBomSkip(t *testing.T) {992	Verbose = true993	for _, v := range ByteOrderMarks {994		fileJob := &FileJob{995			Content: v,996		}997998		skip := checkBomSkip(fileJob)999		if skip != 0 {1000			t.Errorf("Expected skip length to match %d got %d", len(v), skip)1001		}1002	}1003}10041005// Captures checking if a quote is prefixed by \ such as in1006// a char which should otherwise trigger the string state which is incorrect1007func TestCountStatsIssue73(t *testing.T) {1008	ProcessConstants()1009	fileJob := FileJob{1010		Language: "Java",1011	}10121013	fileJob.SetContent(`'\"'{1014code10151016`)1017	fileJob.Bytes = int64(len(fileJob.Content))10181019	CountStats(&fileJob)10201021	if fileJob.Lines != 3 {1022		t.Errorf("Expected 3 lines")1023	}10241025	if fileJob.Code != 2 {1026		t.Errorf("Expected 2 lines got %d", fileJob.Code)1027	}10281029	if fileJob.Comment != 0 {1030		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1031	}10321033	if fileJob.Blank != 1 {1034		t.Errorf("Expected 1 lines got %d", fileJob.Blank)1035	}1036}10371038func TestCountStatsIssue106(t *testing.T) {1039	ProcessConstants()1040	fileJob := FileJob{1041		Language: "Go",1042	}10431044	fileJob.SetContent("foo = `\nabc\"\ndef\n`")10451046	CountStats(&fileJob)1047}10481049func TestMinifiedGeneratedCheck(t *testing.T) {1050	fileJob := FileJob{1051		Language: "Go",1052	}10531054	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF")1055	Minified = true1056	CountStats(&fileJob)1057	Minified = false10581059	if fileJob.Minified != true {1060		t.Error("Expected minified to come back true")1061	}1062}10631064func TestMinifiedGeneratedCheckTwoLines(t *testing.T) {1065	fileJob := FileJob{1066		Language: "Go",1067	}10681069	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF\n1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF")1070	Minified = true1071	CountStats(&fileJob)1072	Minified = false10731074	if fileJob.Minified != true {1075		t.Error("Expected minified to come back true")1076	}1077}10781079func TestMinifiedGeneratedCheckEdge(t *testing.T) {1080	fileJob := FileJob{1081		Language: "Go",1082	}10831084	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCD")1085	Minified = true1086	CountStats(&fileJob)1087	Minified = false10881089	if fileJob.Minified != false {1090		t.Error("Expected minified to come back false")1091	}1092}10931094func TestGenerated(t *testing.T) {1095	fileJob := FileJob{1096		Language: "Go",1097	}10981099	fileJob.SetContent(`1100// Code generated by some tool, DO NOT EDIT.11011102// Package some contains something.1103package some1104`)1105	Generated = true1106	GeneratedMarkers = []string{"do not edit", "generated"}1107	CountStats(&fileJob)1108	Generated = false11091110	if fileJob.Generated != true {1111		t.Error("Expected generated to come back true")1112	}11131114	if fileJob.Language != "Go (gen)" {1115		t.Errorf("Expected Language \"Go (gen)\", received %q", fileJob.Language)1116	}1117}11181119func TestCountStatsIssue182(t *testing.T) {1120	ProcessConstants()1121	fileJob := FileJob{1122		Language: "Pascal",1123	}11241125	fileJob.SetContent(`uses1126    someunit;11271128{This is a comment}1129procedure Something1130var1131    avar: String;1132begin1133    Println('Oho');1134end;1135{This is a comment}1136procedure Nothing1137begin1138end.1139`)11401141	CountStats(&fileJob)11421143	if fileJob.Code != 11 {1144		t.Errorf("Expected 11 lines got %d", fileJob.Code)1145	}11461147	if fileJob.Comment != 2 {1148		t.Errorf("Expected 2 lines got %d", fileJob.Comment)1149	}11501151	if fileJob.Blank != 1 {1152		t.Errorf("Expected 1 lines got %d", fileJob.Blank)1153	}1154}11551156func TestCountStatsIssue182Delphi(t *testing.T) {1157	ProcessConstants()1158	fileJob := FileJob{1159		Language: "Pascal",1160	}11611162	fileJob.SetContent(`// this isnt a comment in pascal but is in delphi1163`)11641165	CountStats(&fileJob)11661167	if fileJob.Code != 0 {1168		t.Errorf("Expected 0 lines got %d", fileJob.Code)1169	}11701171	if fileJob.Comment != 1 {1172		t.Errorf("Expected 1 lines got %d", fileJob.Comment)1173	}11741175	if fileJob.Blank != 0 {1176		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1177	}1178}11791180//////////////////////////////////////////////////1181// Content Classification Tests1182//////////////////////////////////////////////////11831184// Classification should not change any line counts1185func TestClassifyContentCountInvariance(t *testing.T) {1186	ProcessConstants()11871188	content := `package main11891190import "fmt"11911192// main function1193func main() {1194	fmt.Println("hello") // inline comment1195	/* block1196	   comment */1197}1198`1199	// Run without classification1200	fj1 := FileJob{Language: "Go"}1201	fj1.SetContent(content)1202	CountStats(&fj1)12031204	// Run with classification1205	fj2 := FileJob{Language: "Go", ClassifyContent: true}1206	fj2.SetContent(content)1207	CountStats(&fj2)12081209	if fj1.Lines != fj2.Lines {1210		t.Errorf("Lines mismatch: %d vs %d", fj1.Lines, fj2.Lines)1211	}1212	if fj1.Code != fj2.Code {1213		t.Errorf("Code mismatch: %d vs %d", fj1.Code, fj2.Code)1214	}1215	if fj1.Comment != fj2.Comment {1216		t.Errorf("Comment mismatch: %d vs %d", fj1.Comment, fj2.Comment)1217	}1218	if fj1.Blank != fj2.Blank {1219		t.Errorf("Blank mismatch: %d vs %d", fj1.Blank, fj2.Blank)1220	}1221	if fj1.Complexity != fj2.Complexity {1222		t.Errorf("Complexity mismatch: %d vs %d", fj1.Complexity, fj2.Complexity)1223	}1224}12251226// ContentByteType should stay nil when ClassifyContent is false (default)1227func TestClassifyContentNilGuard(t *testing.T) {1228	ProcessConstants()1229	fileJob := FileJob{Language: "Go"}1230	fileJob.SetContent("x := 1\n")1231	CountStats(&fileJob)12321233	if fileJob.ContentByteType != nil {1234		t.Error("Expected ContentByteType to be nil when ClassifyContent is false")1235	}1236}12371238// Code-only file: all non-whitespace bytes should be ByteTypeCode1239func TestClassifyContentCodeOnly(t *testing.T) {1240	ProcessConstants()1241	fileJob := FileJob{Language: "Go", ClassifyContent: true}1242	fileJob.SetContent("x := 1")1243	CountStats(&fileJob)12441245	if fileJob.ContentByteType == nil {1246		t.Fatal("Expected ContentByteType to be non-nil")1247	}12481249	for i, b := range fileJob.Content {1250		bt := fileJob.ContentByteType[i]1251		if b == ' ' || b == '\t' || b == '\n' || b == '\r' {1252			continue // whitespace can be blank or code depending on state1253		}1254		if bt != ByteTypeCode {1255			t.Errorf("byte %d (%q): expected ByteTypeCode(%d), got %d", i, string(b), ByteTypeCode, bt)1256		}1257	}1258}12591260// Comment-only file: "// comment" bytes after // should be ByteTypeComment1261func TestClassifyContentCommentOnly(t *testing.T) {1262	ProcessConstants()1263	fileJob := FileJob{Language: "Go", ClassifyContent: true}1264	fileJob.SetContent("// this is a comment")1265	CountStats(&fileJob)12661267	if fileJob.ContentByteType == nil {1268		t.Fatal("Expected ContentByteType to be non-nil")1269	}12701271	// After the first byte that triggers comment state, subsequent bytes should be comment1272	hasComment := false1273	for i := range fileJob.Content {1274		if fileJob.ContentByteType[i] == ByteTypeComment {1275			hasComment = true1276		}1277	}1278	if !hasComment {1279		t.Error("Expected at least some ByteTypeComment bytes for a comment-only line")1280	}12811282	if fileJob.Comment != 1 {1283		t.Errorf("Expected 1 comment line, got %d", fileJob.Comment)1284	}1285}12861287// Multi-line comment: inner bytes should be ByteTypeComment1288func TestClassifyContentMultiLineComment(t *testing.T) {1289	ProcessConstants()1290	fileJob := FileJob{Language: "Go", ClassifyContent: true}1291	fileJob.SetContent("/* hello\nworld */")1292	CountStats(&fileJob)12931294	if fileJob.ContentByteType == nil {1295		t.Fatal("Expected ContentByteType to be non-nil")1296	}12971298	// Check that "hello" inside the comment is classified as comment1299	helloStart := bytes.Index(fileJob.Content, []byte("hello"))1300	for i := helloStart; i < helloStart+5; i++ {1301		if fileJob.ContentByteType[i] != ByteTypeComment {1302			t.Errorf("byte %d (%q): expected ByteTypeComment(%d), got %d", i, string(fileJob.Content[i]), ByteTypeComment, fileJob.ContentByteType[i])1303		}1304	}1305}13061307// String literal: bytes inside "..." should be ByteTypeString1308func TestClassifyContentString(t *testing.T) {1309	ProcessConstants()1310	fileJob := FileJob{Language: "Go", ClassifyContent: true}1311	fileJob.SetContent(`x := "hello"`)1312	CountStats(&fileJob)13131314	if fileJob.ContentByteType == nil {1315		t.Fatal("Expected ContentByteType to be non-nil")1316	}13171318	// Find "hello" inside the string1319	helloStart := bytes.Index(fileJob.Content, []byte("hello"))1320	for i := helloStart; i < helloStart+5; i++ {1321		if fileJob.ContentByteType[i] != ByteTypeString {1322			t.Errorf("byte %d (%q): expected ByteTypeString(%d), got %d", i, string(fileJob.Content[i]), ByteTypeString, fileJob.ContentByteType[i])1323		}1324	}13251326	// "x" at the start should be code1327	if fileJob.ContentByteType[0] != ByteTypeCode {1328		t.Errorf("byte 0 (%q): expected ByteTypeCode(%d), got %d", string(fileJob.Content[0]), ByteTypeCode, fileJob.ContentByteType[0])1329	}1330}13311332func TestMojoRawTStringsDoNotCountComplexity(t *testing.T) {1333	ProcessConstants()13341335	for _, prefix := range []string{"rt\"", "tr\"", "rT\"", "tR\""} {1336		content := "value = " + prefix + "for item in items\""1337		fileJob := FileJob{Language: "Mojo"}1338		fileJob.SetContent(content)1339		CountStats(&fileJob)13401341		if fileJob.Complexity != 0 {1342			t.Errorf("prefix %q counted string contents as complexity: %d", prefix, fileJob.Complexity)1343		}1344	}1345}13461347// Mixed line: "x := 1 // comment" has code then comment1348func TestClassifyContentMixedLine(t *testing.T) {1349	ProcessConstants()1350	fileJob := FileJob{Language: "Go", ClassifyContent: true}1351	fileJob.SetContent("x := 1 // comment")1352	CountStats(&fileJob)13531354	if fileJob.ContentByteType == nil {1355		t.Fatal("Expected ContentByteType to be non-nil")1356	}13571358	// 'x' should be code1359	if fileJob.ContentByteType[0] != ByteTypeCode {1360		t.Errorf("byte 0 (%q): expected ByteTypeCode, got %d", string(fileJob.Content[0]), fileJob.ContentByteType[0])1361	}13621363	// "comment" text after // should be ByteTypeComment1364	commentStart := bytes.Index(fileJob.Content, []byte("comment"))1365	for i := commentStart; i < commentStart+7; i++ {1366		if fileJob.ContentByteType[i] != ByteTypeComment {1367			t.Errorf("byte %d (%q): expected ByteTypeComment, got %d", i, string(fileJob.Content[i]), fileJob.ContentByteType[i])1368		}1369	}13701371	// Line should be counted as code (code + comment = code line)1372	if fileJob.Code != 1 {1373		t.Errorf("Expected 1 code line, got %d", fileJob.Code)1374	}1375}13761377// Python docstring: content classified as ByteTypeComment1378func TestClassifyContentPythonDocstring(t *testing.T) {1379	ProcessConstants()1380	fileJob := FileJob{Language: "Python", ClassifyContent: true}1381	fileJob.SetContent(`"""1382docstring content1383"""`)1384	CountStats(&fileJob)13851386	if fileJob.ContentByteType == nil {1387		t.Fatal("Expected ContentByteType to be non-nil")1388	}13891390	// "docstring content" should be classified as comment1391	docStart := bytes.Index(fileJob.Content, []byte("docstring"))1392	if docStart >= 0 {1393		for i := docStart; i < docStart+9; i++ {1394			if fileJob.ContentByteType[i] != ByteTypeComment {1395				t.Errorf("byte %d (%q): expected ByteTypeComment(%d), got %d",1396					i, string(fileJob.Content[i]), ByteTypeComment, fileJob.ContentByteType[i])1397			}1398		}1399	}1400}14011402// Python raw-string docstrings (r"""/r”') should be counted as comments just1403// like plain """/”' docstrings. https://github.com/boyter/scc raw docstrings1404func TestCountStatsPythonRawDocstring(t *testing.T) {1405	ProcessConstants()14061407	content := `r'''This is a module docstring'''14081409class C:1410  r'''1411  This is a class docstring1412  '''14131414  def f():1415    r"""This is a function docstring.1416    simple quotes and double quotes are equivalent1417    """1418    pass1419`14201421	fileJob := FileJob{Language: "Python"}1422	fileJob.SetContent(content)1423	CountStats(&fileJob)14241425	if fileJob.Comment != 7 {1426		t.Errorf("Expected 7 comment lines (docstrings) got %d", fileJob.Comment)1427	}1428	if fileJob.Code != 3 {1429		t.Errorf("Expected 3 code lines got %d", fileJob.Code)1430	}1431	if fileJob.Blank != 2 {1432		t.Errorf("Expected 2 blank lines got %d", fileJob.Blank)1433	}14341435	plain := strings.ReplaceAll(content, "r'''", "'''")1436	plain = strings.ReplaceAll(plain, "r\"\"\"", "\"\"\"")14371438	plainJob := FileJob{Language: "Python"}1439	plainJob.SetContent(plain)1440	CountStats(&plainJob)14411442	if plainJob.Comment != fileJob.Comment || plainJob.Code != fileJob.Code || plainJob.Blank != fileJob.Blank {1443		t.Errorf("Raw docstring counts (c=%d code=%d b=%d) differ from plain (c=%d code=%d b=%d)",1444			fileJob.Comment, fileJob.Code, fileJob.Blank,1445			plainJob.Comment, plainJob.Code, plainJob.Blank)1446	}1447}14481449// FilterContentByType: returns filtered content correctly1450func TestFilterContentByType(t *testing.T) {1451	ProcessConstants()1452	fileJob := FileJob{Language: "Go", ClassifyContent: true}1453	fileJob.SetContent("x := 1 // comment\n")1454	CountStats(&fileJob)14551456	// Filter to only code1457	codeOnly := fileJob.FilterContentByType(ByteTypeCode)1458	if codeOnly == nil {1459		t.Fatal("Expected non-nil result from FilterContentByType")1460	}14611462	// Should preserve newlines1463	if codeOnly[len(codeOnly)-1] != '\n' {1464		t.Error("Expected newline to be preserved")1465	}14661467	// 'x' should be preserved1468	if codeOnly[0] != 'x' {1469		t.Errorf("Expected 'x' to be preserved, got %q", string(codeOnly[0]))1470	}14711472	// "comment" should be replaced with spaces1473	commentStart := bytes.Index(fileJob.Content, []byte("comment"))1474	for i := commentStart; i < commentStart+7; i++ {1475		if codeOnly[i] != ' ' {1476			t.Errorf("byte %d: expected space, got %q", i, string(codeOnly[i]))1477		}1478	}1479}14801481// FilterContentByType returns nil when ContentByteType is nil1482func TestFilterContentByTypeNil(t *testing.T) {1483	fileJob := FileJob{}1484	fileJob.SetContent("hello")14851486	result := fileJob.FilterContentByType(ByteTypeCode)1487	if result != nil {1488		t.Error("Expected nil result when ContentByteType is nil")1489	}1490}14911492// FilterContentByType with multiple keep types1493func TestFilterContentByTypeMultiple(t *testing.T) {1494	ProcessConstants()1495	fileJob := FileJob{Language: "Go", ClassifyContent: true}1496	fileJob.SetContent(`x := "hello" // comment`)1497	CountStats(&fileJob)14981499	// Keep both code and string, filter out comments1500	result := fileJob.FilterContentByType(ByteTypeCode, ByteTypeString)1501	if result == nil {1502		t.Fatal("Expected non-nil result")1503	}15041505	// 'x' (code) should be preserved1506	if result[0] != 'x' {1507		t.Errorf("Expected 'x' preserved, got %q", string(result[0]))1508	}15091510	// "hello" (string) content should be preserved1511	helloStart := bytes.Index(fileJob.Content, []byte("hello"))1512	for i := helloStart; i < helloStart+5; i++ {1513		if result[i] != fileJob.Content[i] {1514			t.Errorf("byte %d: expected %q preserved, got %q", i, string(fileJob.Content[i]), string(result[i]))1515		}1516	}1517}15181519//////////////////////////////////////////////////1520// Benchmarks Below1521//////////////////////////////////////////////////15221523func BenchmarkCountStatsLinesEmpty(b *testing.B) {1524	fileJob := FileJob{1525		Content: []byte(""),1526	}15271528	for i := 0; i < b.N; i++ {1529		CountStats(&fileJob)1530	}1531}15321533func BenchmarkCountStatsLinesSingleChar(b *testing.B) {1534	fileJob := FileJob{1535		Content: []byte("a"),1536	}15371538	for i := 0; i < b.N; i++ {1539		CountStats(&fileJob)1540	}1541}15421543func BenchmarkCountStatsLinesTwoLines(b *testing.B) {1544	fileJob := FileJob{1545		Content: []byte("a\na"),1546	}15471548	for i := 0; i < b.N; i++ {1549		CountStats(&fileJob)1550	}1551}15521553func BenchmarkCountStatsLinesThreeLines(b *testing.B) {1554	fileJob := FileJob{1555		Content: []byte("a\na\na"),1556	}15571558	for i := 0; i < b.N; i++ {1559		CountStats(&fileJob)1560	}1561}15621563func BenchmarkCountStatsLinesShortLine(b *testing.B) {1564	fileJob := FileJob{1565		Content: []byte("1234567890"),1566	}15671568	for i := 0; i < b.N; i++ {1569		CountStats(&fileJob)1570	}1571}15721573func BenchmarkCountStatsLinesShortEmptyLine(b *testing.B) {1574	fileJob := FileJob{1575		Content: []byte("          "),1576	}15771578	for i := 0; i < b.N; i++ {1579		CountStats(&fileJob)1580	}1581}15821583func BenchmarkCountStatsLinesThreeShortLines(b *testing.B) {1584	fileJob := FileJob{1585		Content: []byte("1234567890\n1234567890\n1234567890"),1586	}15871588	for i := 0; i < b.N; i++ {1589		CountStats(&fileJob)1590	}1591}15921593func BenchmarkCountStatsLinesThreeShortEmptyLines(b *testing.B) {1594	fileJob := FileJob{1595		Content: []byte("          \n          \n          "),1596	}15971598	for i := 0; i < b.N; i++ {1599		CountStats(&fileJob)1600	}1601}16021603func BenchmarkCountStatsLinesLongLine(b *testing.B) {1604	fileJob := FileJob{1605		Content: []byte("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),1606	}16071608	for i := 0; i < b.N; i++ {1609		CountStats(&fileJob)1610	}1611}16121613func BenchmarkCountStatsLinesLongMixedLine(b *testing.B) {1614	fileJob := FileJob{1615		Content: []byte("1234567890          1234567890          1234567890          1234567890          1234567890          "),1616	}16171618	for i := 0; i < b.N; i++ {1619		CountStats(&fileJob)1620	}1621}16221623func BenchmarkCountStatsLinesLongAlternateLine(b *testing.B) {1624	fileJob := FileJob{1625		Content: []byte("a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a "),1626	}16271628	for i := 0; i < b.N; i++ {1629		CountStats(&fileJob)1630	}1631}16321633func BenchmarkCountStatsLinesFiveHundredLongLines(b *testing.B) {1634	b.StopTimer()1635	content := ""1636	for range 500 {1637		content += "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\n"1638	}16391640	fileJob := FileJob{1641		Content: []byte(content),1642	}1643	b.StartTimer()1644	for i := 0; i < b.N; i++ {1645		CountStats(&fileJob)1646	}1647}16481649func BenchmarkCountStatsLinesFiveHundredLongLinesTriggerComplexityIf(b *testing.B) {1650	b.StopTimer()1651	content := ""1652	for range 500 {1653		content += "iiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiiii\n"1654	}16551656	fileJob := FileJob{1657		Content: []byte(content),1658	}1659	b.StartTimer()1660	for i := 0; i < b.N; i++ {1661		CountStats(&fileJob)1662	}1663}16641665func BenchmarkCountStatsLinesFiveHundredLongLinesTriggerComplexityFor(b *testing.B) {1666	b.StopTimer()1667	content := ""1668	for range 500 {1669		content += "fofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofofo\n"1670	}16711672	fileJob := FileJob{1673		Content: []byte(content),1674	}1675	b.StartTimer()1676	for i := 0; i < b.N; i++ {1677		CountStats(&fileJob)1678	}1679}16801681func BenchmarkCountStatsLinesFourHundredLongLinesMixed(b *testing.B) {1682	b.StopTimer()1683	content := ""1684	for range 100 {1685		content += "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890\n"1686		content += "1234567890          1234567890          1234567890          1234567890          1234567890          \n"1687		content += "                                                                                                    \n"1688		content += "a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a \n"1689	}16901691	fileJob := FileJob{1692		Content: []byte(content),1693	}1694	b.StartTimer()1695	for i := 0; i < b.N; i++ {1696		CountStats(&fileJob)1697	}1698}16991700func BenchmarkCheckByteEqualityReflect(b *testing.B) {1701	b.StopTimer()1702	one := []byte("for")1703	two := []byte("for")17041705	count := 017061707	b.StartTimer()1708	for i := 0; i < b.N; i++ {1709		equal := reflect.DeepEqual(one[1:], two[1:])17101711		if equal {1712			count++1713		}1714	}17151716	b.Log(count)1717}17181719func BenchmarkCheckByteEqualityBytes(b *testing.B) {1720	b.StopTimer()1721	one := []byte("for")1722	two := []byte("for")17231724	count := 017251726	b.StartTimer()1727	for i := 0; i < b.N; i++ {1728		equal := bytes.Equal(one[1:], two[1:])17291730		if equal {1731			count++1732		}1733	}17341735	b.Log(count)1736}17371738// This appears to be faster than bytes.Equal because it does not need1739// to do length comparison checks at the start1740func BenchmarkCheckByteEqualityLoop(b *testing.B) {1741	b.StopTimer()1742	one := []byte("for")1743	two := []byte("for")17441745	count := 017461747	b.StartTimer()1748	for i := 0; i < b.N; i++ {1749		equal := true17501751		for j := 1; j < len(one); j++ {1752			if one[j] != two[j] {1753				equal = false1754				break1755			}1756		}17571758		if equal {1759			count++1760		}1761	}17621763	b.Log(count)1764}17651766// Check if the 1 offset makes a difference, which it does by ~1 ns1767func BenchmarkCheckByteEqualityLoopWithAdditional(b *testing.B) {1768	b.StopTimer()1769	one := []byte("for")1770	two := []byte("for")17711772	count := 017731774	b.StartTimer()1775	for i := 0; i < b.N; i++ {1776		equal := true17771778		// Don't start at 1 like the above but 0 to do a full scan1779		for j := range one {1780			if one[j] != two[j] {1781				equal = false1782				break1783			}1784		}17851786		if equal {1787			count++1788		}1789	}17901791	b.Log(count)1792}17931794func BenchmarkCheckArrayCheck(b *testing.B) {1795	array := []byte{1796		'a',1797		'b',1798		'c',1799		'd',1800		'e',1801		'f',1802		'g',1803		'h',1804		'i',1805		'j',1806	}18071808	var searchFor byte = 'j'1809	found := 018101811	for i := 0; i < b.N; i++ {1812		for index := 0; index < len(array); index++ {1813			if array[index] == searchFor {1814				found++1815				break1816			}1817		}1818	}18191820	b.Log(found)1821}18221823func BenchmarkCheckMapCheck(b *testing.B) {1824	array := map[byte]bool{1825		'a': true,1826		'b': true,1827		'c': true,1828		'd': true,1829		'e': true,1830		'f': true,1831		'g': true,1832		'h': true,1833		'i': true,1834		'j': true,1835	}18361837	var searchFor byte = 'j'1838	found := 018391840	for i := 0; i < b.N; i++ {18411842		_, ok := array[searchFor]18431844		if ok {1845			found++1846		}1847	}18481849	b.Log(found)1850}18511852func BenchmarkStringLoop(b *testing.B) {1853	b.StopTimer()18541855	var str strings.Builder1856	for range 10000 {1857		str.WriteString("1")1858	}1859	search := str.String()1860	count := 01861	b.StartTimer()18621863	for i := 0; i < b.N; i++ {1864		for j := 0; j < len(search); j++ {1865			if search[j] != '\n' {1866				count++1867			}18681869		}1870	}1871	b.Log(count)1872}18731874func BenchmarkByteLoop(b *testing.B) {1875	b.StopTimer()18761877	var str strings.Builder1878	for range 10000 {1879		str.WriteString("1")1880	}1881	search := []byte(str.String())1882	count := 01883	b.StartTimer()18841885	for i := 0; i < b.N; i++ {1886		for j := range search {1887			if search[j] != '\n' {1888				count++1889			}18901891		}1892	}1893	b.Log(count)1894}18951896func BenchmarkLoopInLoop(b *testing.B) {1897	search := []byte("this is a long from for string which we will search")1898	matches := [][]byte{1899		[]byte("if"),1900		[]byte("if("),1901		[]byte("else"),1902		[]byte("while"),1903		[]byte("while("),1904		[]byte("for"),1905		[]byte("foreach"),1906	}1907	endPoint := len(search)1908	b.ResetTimer()19091910	potentialMatch := true1911	for i := 0; i < b.N; i++ {19121913		potentialMatch = true1914		for index := range search {19151916			for k := range matches {19171918				for j := 0; j < len(matches[k]); j++ {1919					if index+j >= endPoint || matches[k][j] != search[index+j] {1920						potentialMatch = false1921					}1922				}1923			}19241925		}19261927	}1928	b.Log(potentialMatch)1929}19301931func BenchmarkFlattenedLoop(b *testing.B) {1932	index := 01933	search := []byte("this is a long from for string which we will search")1934	matches := []byte("if if( else while while( for foreach")19351936	b.ResetTimer()19371938	potentialMatch := true1939	count := 01940	for i := 0; i < b.N; i++ {19411942		potentialMatch = true1943		for j := range matches {1944			if matches[j] == ' ' {1945				count = 01946			} else {1947				if matches[j] != search[index+count] {1948					potentialMatch = false1949				}19501951			}1952		}19531954	}19551956	b.Log(potentialMatch)1957}19581959func BenchmarkCheckComplexity(b *testing.B) {1960	ProcessConstants()19611962	fileJob := FileJob{1963		Language: "Java",1964		Content:  []byte("A little while ago, I passed my first year mark of working for Google. This also marked the "),1965	}19661967	matches := &Trie{}1968	matches.Insert(TComplexity, []byte("for "))1969	matches.Insert(TComplexity, []byte("for("))1970	matches.Insert(TComplexity, []byte("if "))1971	matches.Insert(TComplexity, []byte("if("))1972	matches.Insert(TComplexity, []byte("switch "))1973	matches.Insert(TComplexity, []byte("while "))1974	matches.Insert(TComplexity, []byte("else "))1975	matches.Insert(TComplexity, []byte("|| "))1976	matches.Insert(TComplexity, []byte("&& "))1977	matches.Insert(TComplexity, []byte("!= "))1978	matches.Insert(TComplexity, []byte("== "))19791980	b.ResetTimer()1981	for i := 0; i < b.N; i++ {1982		for j := 0; j < len(fileJob.Content); j++ {1983			matches.Match(fileJob.Content)1984		}1985	}1986}19871988func BenchmarkCheckLen(b *testing.B) {1989	matches := [][]byte{1990		[]byte("for "),1991		[]byte("for("),1992		[]byte("if "),1993		[]byte("if("),1994		[]byte("switch "),1995		[]byte("while "),1996		[]byte("else "),1997		[]byte("|| "),1998		[]byte("&& "),1999		[]byte("!= "),2000		[]byte("== "),