1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"reflect"8	"strings"9	"testing"10)1112func (job *FileJob) SetContent(content string) {13	job.Content = []byte(content)14	job.Bytes = int64(len(job.Content))15}1617func TestIsWhitespace(t *testing.T) {18	if !isWhitespace(' ') {19		t.Errorf("Expected to be true")20	}21}2223func TestIsBinaryTrue(t *testing.T) {24	DisableCheckBinary = false2526	if !isBinary(0, 0) {27		t.Errorf("Expected to be true")28	}29}3031func TestIsBinaryDisableCheck(t *testing.T) {32	DisableCheckBinary = true3334	if isBinary(0, 0) {35		t.Errorf("Expected to be false")36	}37}3839func TestCountStatsLines(t *testing.T) {40	Trace = false41	Debug = false42	Verbose = false4344	fileJob := FileJob{45		Content: []byte(""),46		Lines:   0,47	}4849	// Both tokei and sloccount count this as 0 so lets follow suit50	// cloc ignores the file itself because it is empty51	CountStats(&fileJob)52	if fileJob.Lines != 0 {53		t.Errorf("Zero lines expected got %d", fileJob.Lines)54	}5556	// Interestingly this file would be 0 lines in "wc -l" because it only counts newlines57	// all others count this as 158	fileJob.Lines = 059	fileJob.SetContent("a")60	CountStats(&fileJob)61	if fileJob.Lines != 1 {62		t.Errorf("One line expected got %d", fileJob.Lines)63	}6465	fileJob.Lines = 066	fileJob.SetContent("a\n")67	CountStats(&fileJob)68	if fileJob.Lines != 1 {69		t.Errorf("One line expected got %d", fileJob.Lines)70	}7172	// tokei counts this as 1 because it's still on a single line unless something follows73	// the newline it's still 1 line74	fileJob.Lines = 075	fileJob.SetContent("1\n")76	CountStats(&fileJob)77	if fileJob.Lines != 1 {78		t.Errorf("One line expected got %d", fileJob.Lines)79	}8081	fileJob.Lines = 082	fileJob.SetContent("1\n2\n")83	CountStats(&fileJob)84	if fileJob.Lines != 2 {85		t.Errorf("Two lines expected got %d", fileJob.Lines)86	}8788	fileJob.Lines = 089	fileJob.SetContent("1\n2\n3")90	CountStats(&fileJob)91	if fileJob.Lines != 3 {92		t.Errorf("Three lines expected got %d", fileJob.Lines)93	}9495	content := ""96	for i := range 5000 {97		content += "a\n"98		fileJob.Lines = 099		fileJob.SetContent(content)100		CountStats(&fileJob)101		if fileJob.Lines != int64(i+1) {102			t.Errorf("Expected %d got %d", i+1, fileJob.Lines)103		}104	}105}106107func TestCountStatsCode(t *testing.T) {108	fileJob := FileJob{109		Content: []byte(""),110		Code:    0,111	}112113	// Both tokei and sloccount count this as 0 so lets follow suit114	// cloc ignores the file itself because it is empty115	CountStats(&fileJob)116	if fileJob.Code != 0 {117		t.Errorf("Zero lines expected got %d", fileJob.Code)118	}119120	// Interestingly this file would be 0 lines in "wc -l" because it only counts newlines121	// all others count this as 1122	fileJob.Code = 0123	fileJob.SetContent("a")124	CountStats(&fileJob)125	if fileJob.Code != 1 {126		t.Errorf("One line expected got %d", fileJob.Code)127	}128129	fileJob.Code = 0130	fileJob.SetContent("i++ # comment")131	CountStats(&fileJob)132	if fileJob.Code != 1 {133		t.Errorf("One line expected got %d", fileJob.Code)134	}135136	fileJob.Code = 0137	fileJob.SetContent("i++ // comment")138	CountStats(&fileJob)139	if fileJob.Code != 1 {140		t.Errorf("One line expected got %d", fileJob.Code)141	}142143	fileJob.Code = 0144	fileJob.SetContent("a\n")145	CountStats(&fileJob)146	if fileJob.Code != 1 {147		t.Errorf("One line expected got %d", fileJob.Code)148	}149150	// tokei counts this as 1 because it's still on a single line unless something follows151	// the newline it's still 1 line152	fileJob.Code = 0153	fileJob.SetContent("1\n")154	CountStats(&fileJob)155	if fileJob.Code != 1 {156		t.Errorf("One line expected got %d", fileJob.Code)157	}158159	fileJob.Code = 0160	fileJob.SetContent("1\n2\n")161	CountStats(&fileJob)162	if fileJob.Code != 2 {163		t.Errorf("Two lines expected got %d", fileJob.Code)164	}165166	fileJob.Code = 0167	fileJob.SetContent("1\n2\n3")168	CountStats(&fileJob)169	if fileJob.Code != 3 {170		t.Errorf("Three lines expected got %d", fileJob.Code)171	}172173	content := ""174	for i := range 100 {175		content += "a\n"176		fileJob.Code = 0177		fileJob.SetContent(content)178		CountStats(&fileJob)179		if fileJob.Code != int64(i+1) {180			t.Errorf("Expected %d got %d", i+1, fileJob.Code)181		}182	}183}184185// A quote delimiter is written in languages.json as "\"" and JSON already186// escapes it there, so a delimiter carrying a literal backslash in front of187// its quote is one that was escaped twice and matches nothing a real file188// holds. Zig's "\\" is the one true backslash delimiter, being the opener of189// its multiline string literal.190func TestLanguageQuotesAreNotDoubleEscaped(t *testing.T) {191	ProcessConstants()192193	for name, language := range languageDatabase {194		for _, quote := range language.Quotes {195			for _, delimiter := range []string{quote.Start, quote.End} {196				if strings.Contains(delimiter, `\"`) {197					t.Errorf("%s: quote delimiter %q holds a literal backslash, so it never matches", name, delimiter)198				}199			}200		}201	}202}203204// The double escaped delimiters above left these languages with no working205// double quoted string at all. It only shows on a string running over more206// than one line, because a single line one sits on a line that is code207// whether the string was seen or not. Here the middle line opens with the208// language's own comment token while inside the string, so a language that209// missed the string reads it as a comment.210func TestCountStatsMultiLineStringHoldingACommentToken(t *testing.T) {211	ProcessConstants()212213	for _, test := range []struct {214		language    string215		lineComment string216	}{217		{"Shell", "#"},218		{"Zsh", "#"},219		{"BASH", "#"},220		{"Korn Shell", "#"},221		{"Fish", "#"},222		{"Ruby", "#"},223		{"Rakefile", "#"},224		{"Gemfile", "#"},225		{"Crystal", "#"},226		{"TOML", "#"},227		{"Julia", "#"},228		{"Nim", "#"},229		{"Dockerfile", "#"},230		{"TCL", "#"},231		{"Puppet", "#"},232		{"CoffeeScript", "#"},233		{"Cython", "#"},234		{"Just", "#"},235		{"Scons", "#"},236		{"Expect", "#"},237		{"Luna", "#"},238		{"Raku", "#"},239		{"Zig", "//"},240		{"Typst", "//"},241		{"Pony", "//"},242		{"Idris", "--"},243		{"Assembly", ";"},244		{"Arturo", ";"},245		{"Basic", "'"},246		{"Visual Basic", "'"},247		{"Visual Basic for Applications", "'"},248		{"Softbridge Basic", "'"},249		{"Fortran Modern", "!"},250	} {251		fileJob := FileJob{Language: test.language}252		fileJob.SetContent("x = \"line one\n" + test.lineComment + " still inside the string\nend\"\ny = 2\n")253254		CountStats(&fileJob)255256		if fileJob.Lines != 4 {257			t.Errorf("%s: expected 4 lines got %d", test.language, fileJob.Lines)258		}259260		if fileJob.Code != 4 {261			t.Errorf("%s: expected 4 code got %d", test.language, fileJob.Code)262		}263264		if fileJob.Comment != 0 {265			t.Errorf("%s: expected 0 comment got %d", test.language, fileJob.Comment)266		}267268		if fileJob.Blank != 0 {269			t.Errorf("%s: expected 0 blank got %d", test.language, fileJob.Blank)270		}271	}272}273274// Vim Script is the one language in the sweep above that must not be given a275// working double quoted string, because the double quote is what opens its276// comments. A string delimiter is matched ahead of a comment one, so handing277// it the quote turns every comment in a vimrc into code.278func TestCountStatsVimScriptDoubleQuoteOpensAComment(t *testing.T) {279	ProcessConstants()280281	fileJob := FileJob{Language: "Vim Script"}282	fileJob.SetContent("\" a vim comment\nlet x = 1\n\" another comment\nlet y = 2\n")283284	CountStats(&fileJob)285286	if fileJob.Lines != 4 {287		t.Errorf("Expected 4 lines got %d", fileJob.Lines)288	}289290	if fileJob.Code != 2 {291		t.Errorf("Expected 2 code got %d", fileJob.Code)292	}293294	if fileJob.Comment != 2 {295		t.Errorf("Expected 2 comment got %d", fileJob.Comment)296	}297298	if fileJob.Blank != 0 {299		t.Errorf("Expected 0 blank got %d", fileJob.Blank)300	}301}302303// Standard SQL escapes a quote by doubling it and the backslash is an ordinary304// character, so a path ending in one closes the string it sits in. Reading the305// backslash as an escape leaves the string open and takes the comment under it306// along with everything below. Oracle's PL/SQL reads the backslash the same307// way. MySQL is the dialect that does escape with it, and it loses here.308func TestCountStatsSQLBackslashDoesNotEscapeTheClosingQuote(t *testing.T) {309	ProcessConstants()310311	for _, language := range []string{"SQL", "PL/SQL"} {312		fileJob := FileJob{Language: language}313		fileJob.SetContent("UPDATE t SET path = 'C:\\';\n-- a real comment\nSELECT 2;\n")314315		CountStats(&fileJob)316317		if fileJob.Lines != 3 {318			t.Errorf("%s: expected 3 lines got %d", language, fileJob.Lines)319		}320321		if fileJob.Code != 2 {322			t.Errorf("%s: expected 2 code got %d", language, fileJob.Code)323		}324325		if fileJob.Comment != 1 {326			t.Errorf("%s: expected 1 comment got %d", language, fileJob.Comment)327		}328329		if fileJob.Blank != 0 {330			t.Errorf("%s: expected 0 blank got %d", language, fileJob.Blank)331		}332	}333}334335// Groovy writes a string four ways and scc knew one of them, so a comment336// opener inside any of the other three opened a comment. The single quoted337// string is the idiomatic one and the triple quoted ones run over lines, which338// is what lets an unclosed block comment take the rest of the file.339func TestCountStatsGroovyStringForms(t *testing.T) {340	ProcessConstants()341342	for _, language := range []string{"Groovy", "Gradle"} {343		for _, quote := range []string{"'", `"`} {344			fileJob := FileJob{Language: language}345			fileJob.SetContent("def a = " + quote + "/* not a comment" + quote + "\ndef b = 1\ndef c = 2\n")346347			CountStats(&fileJob)348349			if fileJob.Lines != 3 {350				t.Errorf("%s %s: expected 3 lines got %d", language, quote, fileJob.Lines)351			}352353			if fileJob.Code != 3 {354				t.Errorf("%s %s: expected 3 code got %d", language, quote, fileJob.Code)355			}356357			if fileJob.Comment != 0 {358				t.Errorf("%s %s: expected 0 comment got %d", language, quote, fileJob.Comment)359			}360		}361362		for _, quote := range []string{"'''", `"""`} {363			fileJob := FileJob{Language: language}364			fileJob.SetContent("def a = " + quote + "\n// this is text inside the string\n/* and so is this\n" + quote + "\ndef b = 1\n")365366			CountStats(&fileJob)367368			if fileJob.Lines != 5 {369				t.Errorf("%s %s: expected 5 lines got %d", language, quote, fileJob.Lines)370			}371372			if fileJob.Code != 5 {373				t.Errorf("%s %s: expected 5 code got %d", language, quote, fileJob.Code)374			}375376			if fileJob.Comment != 0 {377				t.Errorf("%s %s: expected 0 comment got %d", language, quote, fileJob.Comment)378			}379		}380	}381}382383// CMake opens a bracket comment with any number of equals signs between its384// brackets and only a closer carrying the same number ends it. There is no385// dynamic delimiter here so the levels are enumerated, the way Rust's raw386// strings are, and this walks the ones that are.387func TestCountStatsCMakeBracketComment(t *testing.T) {388	ProcessConstants()389390	for level := 0; level <= 8; level++ {391		equals := strings.Repeat("=", level)392		fileJob := FileJob{Language: "CMake"}393		fileJob.SetContent("#[" + equals + "[ a bracket comment\n a second comment line\n]" + equals + "]\nset(X 1)\n")394395		CountStats(&fileJob)396397		if fileJob.Lines != 4 {398			t.Errorf("level %d: expected 4 lines got %d", level, fileJob.Lines)399		}400401		if fileJob.Code != 1 {402			t.Errorf("level %d: expected 1 code got %d", level, fileJob.Code)403		}404405		if fileJob.Comment != 3 {406			t.Errorf("level %d: expected 3 comment got %d", level, fileJob.Comment)407		}408	}409}410411// A closer carrying fewer equals signs than the opener is text, so the comment412// runs past it to the one that matches.413func TestCountStatsCMakeBracketCommentShorterCloserIsText(t *testing.T) {414	ProcessConstants()415416	fileJob := FileJob{Language: "CMake"}417	fileJob.SetContent("#[==[ a bracket comment\n]] not the end\n]==]\nset(X 1)\n")418419	CountStats(&fileJob)420421	if fileJob.Lines != 4 {422		t.Errorf("Expected 4 lines got %d", fileJob.Lines)423	}424425	if fileJob.Code != 1 {426		t.Errorf("Expected 1 code got %d", fileJob.Code)427	}428429	if fileJob.Comment != 3 {430		t.Errorf("Expected 3 comment got %d", fileJob.Comment)431	}432}433434// The same brackets without the leading # are a bracket argument, which is how435// CMake writes a string running over lines. A # inside one is content.436func TestCountStatsCMakeBracketArgument(t *testing.T) {437	ProcessConstants()438439	for level := 0; level <= 8; level++ {440		equals := strings.Repeat("=", level)441		fileJob := FileJob{Language: "CMake"}442		fileJob.SetContent("set(X [" + equals + "[\n# not a comment\n]" + equals + "])\nset(Y 1)\n")443444		CountStats(&fileJob)445446		if fileJob.Lines != 4 {447			t.Errorf("level %d: expected 4 lines got %d", level, fileJob.Lines)448		}449450		if fileJob.Code != 4 {451			t.Errorf("level %d: expected 4 code got %d", level, fileJob.Code)452		}453454		if fileJob.Comment != 0 {455			t.Errorf("level %d: expected 0 comment got %d", level, fileJob.Comment)456		}457	}458}459460// C removes a backslash and the newline behind it before it looks for comments461// or strings, so a backslash ending a line comment carries the comment onto462// the next line, and a string that does not end in one is over at the newline463// whatever it holds. The cases below are LineJudge 5020 to 5070.464func TestCountStatsLineSplice(t *testing.T) {465	ProcessConstants()466467	for _, test := range []struct {468		name    string469		content string470		lines   int64471		code    int64472		comment int64473		blank   int64474	}{475		{476			name:    "a splice inside a line comment carries it on",477			content: "// this comment continues \\\n   onto the next line in C\nint x = 1;\n",478			lines:   3, code: 1, comment: 2, blank: 0,479		},480		{481			name:    "a spliced comment holding no words still carries on",482			content: "int a = 1;\n// ---- \\\nint b = 2;\nint c = 3;\n",483			lines:   4, code: 2, comment: 2, blank: 0,484		},485		{486			name:    "a splice after a closed block comment carries the line comment on",487			content: "int a = 1;\n/* block */ // trailing \\\nint x = 1;\nint y = 2;\n",488			lines:   4, code: 2, comment: 2, blank: 0,489		},490		{491			name:    "code then a spliced line comment carries the comment on",492			content: "int a = 1; // comment \\\n   joined to the comment\nint x = 1;\n",493			lines:   3, code: 2, comment: 1, blank: 0,494		},495		{496			name:    "a blank line inside a spliced comment is comment and not blank",497			content: "// the blank line below is still this comment \\\n\nint x = 1;\n",498			lines:   3, code: 1, comment: 2, blank: 0,499		},500		{501			name:    "a string is carried only by a splice, so a blank line ends it",502			content: "char *s = \"abc \\\n\n// a comment once the string has ended\";\nint x = 1;\n",503			lines:   4, code: 3, comment: 1, blank: 0,504		},505	} {506		fileJob := FileJob{Language: "C"}507		fileJob.SetContent(test.content)508509		CountStats(&fileJob)510511		if fileJob.Lines != test.lines {512			t.Errorf("%s: expected %d lines got %d", test.name, test.lines, fileJob.Lines)513		}514515		if fileJob.Code != test.code {516			t.Errorf("%s: expected %d code got %d", test.name, test.code, fileJob.Code)517		}518519		if fileJob.Comment != test.comment {520			t.Errorf("%s: expected %d comment got %d", test.name, test.comment, fileJob.Comment)521		}522523		if fileJob.Blank != test.blank {524			t.Errorf("%s: expected %d blank got %d", test.name, test.blank, fileJob.Blank)525		}526	}527}528529// A raw string carries over a newline with no splice behind it, so the rule530// that ends an unspliced string has to leave it alone. The ignoreEscape flag531// is what tells the two apart, being set on exactly the delimiters that have532// no escape mechanism to splice with.533func TestCountStatsLineSpliceLeavesRawStringsAlone(t *testing.T) {534	ProcessConstants()535536	fileJob := FileJob{Language: "C++"}537	fileJob.SetContent("const char *s = R\"(abc\n// not a comment, it is inside the raw string\n)\";\nint x = 1;\n")538539	CountStats(&fileJob)540541	if fileJob.Lines != 4 {542		t.Errorf("Expected 4 lines got %d", fileJob.Lines)543	}544545	if fileJob.Code != 4 {546		t.Errorf("Expected 4 code got %d", fileJob.Code)547	}548549	if fileJob.Comment != 0 {550		t.Errorf("Expected 0 comment got %d", fileJob.Comment)551	}552}553554// Only the C family splices. A backslash ending a line comment in a language555// that does not do it must leave the next line alone.556func TestCountStatsLineSpliceIsNotUniversal(t *testing.T) {557	ProcessConstants()558559	for _, language := range []string{"Java", "C#", "Go", "JavaScript"} {560		fileJob := FileJob{Language: language}561		fileJob.SetContent("// a comment ending in a backslash \\\nint x = 1;\nint y = 2;\n")562563		CountStats(&fileJob)564565		if fileJob.Code != 2 {566			t.Errorf("%s: expected 2 code got %d", language, fileJob.Code)567		}568569		if fileJob.Comment != 1 {570			t.Errorf("%s: expected 1 comment got %d", language, fileJob.Comment)571		}572	}573}574575// A markup language holds prose, and prose holds apostrophes. Listing the576// single quote as a string delimiter turns every contraction into the start of577// a string that nothing closes, which then swallows the comments under it and578// runs to the end of the file. HTML, XML, XHTML, Svelte and JSX are all579// written with the double quote alone for that reason, and Svelte is the same580// single file component shape as Vue.581func TestCountStatsMarkupApostropheIsNotAString(t *testing.T) {582	ProcessConstants()583584	for _, test := range []struct {585		language string586		comment  string587	}{588		{"Vue", "<!-- a comment -->"},589		{"Astro", "<!-- a comment -->"},590		{"Handlebars", "<!-- a comment -->"},591		{"Mustache", "{{! a comment }}"},592		// already written with the double quote alone, here to keep it that way593		{"HTML", "<!-- a comment -->"},594		{"Svelte", "<!-- a comment -->"},595		{"XML", "<!-- a comment -->"},596	} {597		fileJob := FileJob{Language: test.language}598		fileJob.SetContent("<p>it's fine</p>\n" + test.comment + "\n<p>x</p>\n")599600		CountStats(&fileJob)601602		if fileJob.Lines != 3 {603			t.Errorf("%s: expected 3 lines got %d", test.language, fileJob.Lines)604		}605606		if fileJob.Code != 2 {607			t.Errorf("%s: expected 2 code got %d", test.language, fileJob.Code)608		}609610		if fileJob.Comment != 1 {611			t.Errorf("%s: expected 1 comment got %d", test.language, fileJob.Comment)612		}613	}614}615616// A line comment opened by a word ends where the word does. REM opens a Batch617// comment and REMOVE is a different word, so matching those three letters as a618// prefix reads a command as a comment. The same holds for Autoconf's dnl and619// LOLCODE's BTW.620func TestCountStatsWordCommentNeedsAWordBreak(t *testing.T) {621	ProcessConstants()622623	for _, test := range []struct {624		language string625		comment  string626		longer   string627	}{628		{"Batch", "REM", "REMOVE /Q file.txt"},629		{"Batch", "rem", "remove /Q file.txt"},630		{"ASP", "REM", "REMOVE x"},631		{"Autoconf", "dnl", "dnlfoo bar"},632		{"LOLCODE", "BTW", "BTWISE x"},633	} {634		fileJob := FileJob{Language: test.language}635		fileJob.SetContent(test.comment + " a comment\n" + test.longer + "\nx\n")636637		CountStats(&fileJob)638639		if fileJob.Lines != 3 {640			t.Errorf("%s %s: expected 3 lines got %d", test.language, test.comment, fileJob.Lines)641		}642643		if fileJob.Code != 2 {644			t.Errorf("%s %s: expected 2 code got %d", test.language, test.comment, fileJob.Code)645		}646647		if fileJob.Comment != 1 {648			t.Errorf("%s %s: expected 1 comment got %d", test.language, test.comment, fileJob.Comment)649		}650	}651}652653// A word break is wanted after a word, not after a symbol, so :: and // and #654// keep opening a comment with whatever behind them. FORTRAN Legacy is the one655// that would break: a C in the first column opens a comment there whatever656// follows it, which is how CCCCCCCC is written as a rule across the page, so657// the check is only for tokens longer than a single byte.658func TestCountStatsWordBreakLeavesSymbolCommentsAlone(t *testing.T) {659	ProcessConstants()660661	for _, test := range []struct {662		language string663		content  string664	}{665		{"Batch", "::comment\nx\n"},666		{"C", "//comment\nint x = 1;\n"},667		{"Shell", "#comment\nx=1\n"},668		{"Haskell", "--comment\nx = 1\n"},669		{"FORTRAN Legacy", "CCCCCCCCCCCCCCCCCCCC\n      x = 1\n"},670	} {671		fileJob := FileJob{Language: test.language}672		fileJob.SetContent(test.content)673674		CountStats(&fileJob)675676		if fileJob.Code != 1 {677			t.Errorf("%s: expected 1 code got %d", test.language, fileJob.Code)678		}679680		if fileJob.Comment != 1 {681			t.Errorf("%s: expected 1 comment got %d", test.language, fileJob.Comment)682		}683	}684}685686// Forth opens a comment with a backslash standing as its own word, so it wants687// the break after it the way a word does. The delimiter was written \\ before688// this, which matched nothing, so Forth had no line comment at all.689func TestCountStatsForthBackslashComment(t *testing.T) {690	ProcessConstants()691692	fileJob := FileJob{Language: "Forth"}693	fileJob.SetContent("\\ a forth comment\n: double dup + ;\n")694695	CountStats(&fileJob)696697	if fileJob.Code != 1 {698		t.Errorf("Expected 1 code got %d", fileJob.Code)699	}700701	if fileJob.Comment != 1 {702		t.Errorf("Expected 1 comment got %d", fileJob.Comment)703	}704}705706// PowerShell escapes with a backtick and not a backslash, so the quote in the707// middle of the first line below closes nothing, and the backslash ending a708// Windows path closes the string it sits in rather than escaping its way past709// the closing quote. Reading the backslash as an escape left the string open710// and swallowed the comment under it.711func TestCountStatsPowershellEscape(t *testing.T) {712	ProcessConstants()713714	for _, test := range []struct {715		name    string716		content string717	}{718		{719			name:    "a backtick escapes the quote in the middle",720			content: "Write-Host \"a `\" b\"\n# a real comment\n$x = 1\n",721		},722		{723			name:    "a backslash ending a path does not escape the closer",724			content: "$p = \"C:\\temp\\\"\n# a real comment\n$x = 1\n",725		},726	} {727		fileJob := FileJob{Language: "Powershell"}728		fileJob.SetContent(test.content)729730		CountStats(&fileJob)731732		if fileJob.Lines != 3 {733			t.Errorf("%s: expected 3 lines got %d", test.name, fileJob.Lines)734		}735736		if fileJob.Code != 2 {737			t.Errorf("%s: expected 2 code got %d", test.name, fileJob.Code)738		}739740		if fileJob.Comment != 1 {741			t.Errorf("%s: expected 1 comment got %d", test.name, fileJob.Comment)742		}743	}744}745746// Every other language keeps the backslash, so a quote escaped with one still747// does not close the string it is in.748func TestCountStatsBackslashStillEscapesElsewhere(t *testing.T) {749	ProcessConstants()750751	for _, language := range []string{"C", "Go", "Java", "Python", "JavaScript"} {752		fileJob := FileJob{Language: language}753		fileJob.SetContent("x = \"a \\\" b\"\ny = 1\nz = 2\n")754755		CountStats(&fileJob)756757		if fileJob.Code != 3 {758			t.Errorf("%s: expected 3 code got %d", language, fileJob.Code)759		}760761		if fileJob.Comment != 0 {762			t.Errorf("%s: expected 0 comment got %d", language, fileJob.Comment)763		}764	}765}766767// A C++ raw string may put a word of its own between the R" and the bracket,768// and then ends only at a closer carrying that same word. The closer is chosen769// where the string is written, so no list of delimiters can hold it and it has770// to be read out of the file.771func TestCountStatsCppDelimitedRawString(t *testing.T) {772	ProcessConstants()773774	for _, test := range []struct {775		name    string776		content string777		lines   int64778		code    int64779	}{780		{781			name:    "a named delimiter holds a quote and a comment opener",782			content: "const char *raw = R\"xy(a lone \" quote and\n/* this opens nothing, it is inside the raw string\n)xy\";\nprintf(\"done\");\n",783			lines:   4, code: 4,784		},785		{786			name:    "the delimiter may still be empty",787			content: "const char *raw = R\"(a \" quote\n// not a comment\n)\";\nint x = 1;\n",788			lines:   4, code: 4,789		},790		{791			name:    "a closer naming a different word does not end it",792			content: "const char *raw = R\"outer(\n)inner\"\nstill inside\n)outer\";\nint x = 1;\n",793			lines:   5, code: 5,794		},795		{796			name:    "the encoding prefixes carry a delimiter too",797			content: "auto a = u8R\"tag(\n// inside\n)tag\";\nauto b = LR\"tag(\n/* inside\n)tag\";\nint x = 1;\n",798			lines:   7, code: 7,799		},800	} {801		for _, language := range []string{"C++", "C++ Header"} {802			fileJob := FileJob{Language: language}803			fileJob.SetContent(test.content)804805			CountStats(&fileJob)806807			if fileJob.Lines != test.lines {808				t.Errorf("%s %s: expected %d lines got %d", language, test.name, test.lines, fileJob.Lines)809			}810811			if fileJob.Code != test.code {812				t.Errorf("%s %s: expected %d code got %d", language, test.name, test.code, fileJob.Code)813			}814815			if fileJob.Comment != 0 {816				t.Errorf("%s %s: expected 0 comment got %d", language, test.name, fileJob.Comment)817			}818		}819	}820}821822// An R that does not open a raw string is an ordinary identifier, and the823// quote after one is an ordinary string.824func TestCountStatsCppRWithoutARawString(t *testing.T) {825	ProcessConstants()826827	fileJob := FileJob{Language: "C++"}828	fileJob.SetContent("int R = 1;\nconst char *s = \"plain\";\n// a real comment\n")829830	CountStats(&fileJob)831832	if fileJob.Code != 2 {833		t.Errorf("Expected 2 code got %d", fileJob.Code)834	}835836	if fileJob.Comment != 1 {837		t.Errorf("Expected 1 comment got %d", fileJob.Comment)838	}839}840841func TestCountStatsWithQuotes(t *testing.T) {842	fileJob := FileJob{}843844	fileJob.Code = 0845	fileJob.Comment = 0846	fileJob.Complexity = 0847	fileJob.SetContent(`var test = "/*";`)848	CountStats(&fileJob)849	if fileJob.Code != 1 {850		t.Errorf("One line expected got %d", fileJob.Code)851	}852	if fileJob.Comment != 0 {853		t.Errorf("No line expected got %d", fileJob.Comment)854	}855856	fileJob.Code = 0857	fileJob.Comment = 0858	fileJob.Complexity = 0859	fileJob.SetContent(`t = " if ";`)860	CountStats(&fileJob)861	if fileJob.Code != 1 {862		t.Errorf("One line expected got %d", fileJob.Code)863	}864	if fileJob.Comment != 0 {865		t.Errorf("No line expected got %d", fileJob.Comment)866	}867	if fileJob.Complexity != 0 {868		t.Errorf("No line expected got %d", fileJob.Complexity)869	}870871	fileJob.Code = 0872	fileJob.Comment = 0873	fileJob.Complexity = 0874	fileJob.SetContent(`t = " if switch for while do loop != == && || ";`)875	CountStats(&fileJob)876	if fileJob.Code != 1 {877		t.Errorf("One line expected got %d", fileJob.Code)878	}879	if fileJob.Comment != 0 {880		t.Errorf("No line expected got %d", fileJob.Comment)881	}882	if fileJob.Complexity != 0 {883		t.Errorf("No line expected got %d", fileJob.Complexity)884	}885}886887func TestCountStatsBlankLines(t *testing.T) {888	fileJob := FileJob{889		Content: []byte(""),890		Blank:   0,891	}892893	CountStats(&fileJob)894	if fileJob.Blank != 0 {895		t.Errorf("Zero lines expected got %d", fileJob.Blank)896	}897898	fileJob.Blank = 0899	fileJob.SetContent(" ")900	CountStats(&fileJob)901	if fileJob.Blank != 1 {902		t.Errorf("One line expected got %d", fileJob.Blank)903	}904905	fileJob.Blank = 0906	fileJob.SetContent("\n")907	CountStats(&fileJob)908	if fileJob.Blank != 1 {909		t.Errorf("One line expected got %d", fileJob.Blank)910	}911912	fileJob.Blank = 0913	fileJob.SetContent("\n ")914	CountStats(&fileJob)915	if fileJob.Blank != 2 {916		t.Errorf("Two line expected got %d", fileJob.Blank)917	}918919	fileJob.Blank = 0920	fileJob.SetContent("            ")921	CountStats(&fileJob)922	if fileJob.Blank != 1 {923		t.Errorf("One line expected got %d", fileJob.Blank)924	}925926	fileJob.Blank = 0927	fileJob.SetContent("            \n             ")928	CountStats(&fileJob)929	if fileJob.Blank != 2 {930		t.Errorf("Two lines expected got %d", fileJob.Blank)931	}932933	fileJob.Blank = 0934	fileJob.SetContent("\r\n\r\n")935	CountStats(&fileJob)936	if fileJob.Blank != 2 {937		t.Errorf("Two lines expected got %d", fileJob.Blank)938	}939940	fileJob.Blank = 0941	fileJob.SetContent("\r\n")942	CountStats(&fileJob)943	if fileJob.Blank != 1 {944		t.Errorf("One line expected got %d", fileJob.Blank)945	}946}947948func TestCountStatsComplexityCount(t *testing.T) {949	ProcessConstants()950	fileJob := FileJob{}951952	checks := []string{953		"if ",954		"	if ",955		"if a.equals(b) {",956		"if(",957		" if(i.equals(0))",958		"    if(",959		"    if( ",960	}961962	for _, check := range checks {963		fileJob.Complexity = 0964		fileJob.SetContent(check)965		fileJob.Language = "Java"966		CountStats(&fileJob)967		if fileJob.Complexity != 1 {968			t.Errorf("Expected complexity of 1 got %d for %s", fileJob.Complexity, check)969		}970	}971}972973func TestCountStatsComplexityCountFalse(t *testing.T) {974	ProcessConstants()975	fileJob := FileJob{}976977	checks := []string{978		"if",979		"aif ",980		"aif(",981	}982983	for _, check := range checks {984		fileJob.Complexity = 0985		fileJob.SetContent(check)986		fileJob.Language = "Java"987		CountStats(&fileJob)988		if fileJob.Complexity != 0 {989			t.Errorf("Expected complexity of 0 got %d for %s", fileJob.Complexity, check)990		}991	}992993}994995func TestCountStatsComplexityRustQuestionOperator(t *testing.T) {996	ProcessConstants()997998	checks := []struct {999		content string1000		want    int641001	}{1002		{"foo()?;", 1},1003		{"foo() ?;", 1},1004		{"foo()?.bar();", 1},1005		{"foo()? as u16;", 1},1006		{"foo()??;", 2},1007		{"let y = x?;", 1},1008	}10091010	for _, c := range checks {1011		fileJob := FileJob{Language: "Rust"}1012		fileJob.SetContent(c.content)1013		CountStats(&fileJob)1014		if fileJob.Complexity != c.want {1015			t.Errorf("Expected complexity of %d got %d for %q", c.want, fileJob.Complexity, c.content)1016		}1017	}1018}10191020// ?Sized is a trait-bound prefix, not the try operator. The postfix matcher1021// must skip it whether or not whitespace separates it from the preceding1022// token (`T: ?Sized`, `T:?Sized`, `Debug+?Sized` are all valid Rust).1023func TestCountStatsComplexityRustQuestionSizedNotCounted(t *testing.T) {1024	ProcessConstants()10251026	checks := []string{1027		"fn foo<T: ?Sized>() {}",1028		"fn foo<T:?Sized>() {}",1029		"fn foo<T:? Sized>() {}",1030		"struct Bar<T: ?Sized>(T);",1031		"where T: Debug + ?Sized,",1032		"where T: Debug+?Sized,",1033	}10341035	for _, content := range checks {1036		fileJob := FileJob{Language: "Rust"}1037		fileJob.SetContent(content)1038		CountStats(&fileJob)1039		if fileJob.Complexity != 0 {1040			t.Errorf("Expected complexity of 0 got %d for %q (?Sized must not count)", fileJob.Complexity, content)1041		}1042	}1043}10441045func TestCountStatsComplexityTypeScriptPostfixOperators(t *testing.T) {1046	ProcessConstants()10471048	checks := []struct {1049		content string1050		want    int641051	}{1052		{"obj.attr?.method();", 1},1053		{"path ?? url;", 1},1054		{"path??url;", 1},1055		{"value ??= fallback;", 1},1056		{"value??=fallback;", 1},1057		{"function get(path?: string) { return path ?? url; }", 1},1058		{"interface User { age?: number; method?(): void; }", 0},1059	}10601061	for _, c := range checks {1062		fileJob := FileJob{Language: "TypeScript"}1063		fileJob.SetContent(c.content)1064		CountStats(&fileJob)1065		if fileJob.Complexity != c.want {1066			t.Errorf("Expected complexity of %d got %d for %q", c.want, fileJob.Complexity, c.content)1067		}1068	}1069}10701071type linecounter struct {1072	blanks   int1073	comments int1074	code     int1075	loc      int1076	stop     bool1077}10781079func (l *linecounter) ProcessLine(job *FileJob, currentLine int64, lineType LineType) bool {1080	l.loc++1081	switch lineType {1082	case LINE_BLANK:1083		l.blanks++1084	case LINE_COMMENT:1085		l.comments++1086	case LINE_CODE:1087		l.code++1088	}1089	return !l.stop1090}10911092func TestCountStatsCallback(t *testing.T) {1093	ProcessConstants()1094	fileJob := FileJob{}10951096	fileJob.SetContent(`package foo10971098import com.foo.bar;10991100// this is a comment1101class A {1102}`)1103	var lc linecounter1104	fileJob.Language = "Java"1105	fileJob.Callback = &lc1106	CountStats(&fileJob)1107	if lc.loc != 7 {1108		t.Errorf("Expected loc of 7 got %d", lc.loc)1109	}1110	if lc.blanks != 2 {1111		t.Errorf("Expected loc of 2 got %d", lc.blanks)1112	}1113	if lc.comments != 1 {1114		t.Errorf("Expected loc of 1 got %d", lc.comments)1115	}1116	if lc.code != 4 {1117		t.Errorf("Expected loc of 4 got %d", lc.code)1118	}1119}11201121func TestCountStatsCallbackInterrupt(t *testing.T) {1122	ProcessConstants()1123	fileJob := FileJob{}11241125	fileJob.SetContent(`package foo11261127import com.foo.bar;11281129// this is a comment1130class A {1131}`)1132	var lc linecounter1133	lc.stop = true1134	fileJob.Language = "Java"1135	fileJob.Callback = &lc1136	CountStats(&fileJob)1137	if lc.loc != 1 {1138		t.Errorf("Expected loc of 1 got %d", lc.loc)1139	}1140}11411142// Edge case condition where if ending with comment it would be counted1143// as code due to how internal state work.1144func TestCountStatsEdgeCase1(t *testing.T) {1145	ProcessConstants()1146	fileJob := FileJob{1147		Language: "Java",1148	}11491150	fileJob.SetContent(`/**/1151`)11521153	CountStats(&fileJob)11541155	if fileJob.Lines != 1 {1156		t.Errorf("Expected 1 lines got %d", fileJob.Lines)1157	}11581159	if fileJob.Comment != 1 {1160		t.Errorf("Expected 1 lines got %d", fileJob.Comment)1161	}1162}11631164// Turns out that some languages such as Rust support1165// nested comments. Check that it works here1166func TestCountStatsNestedComments(t *testing.T) {1167	ProcessConstants()1168	fileJob := FileJob{1169		Language: "Rust",1170	}11711172	fileJob.SetContent(`/*/**/*/`)11731174	CountStats(&fileJob)11751176	if fileJob.Lines != 1 {1177		t.Errorf("Expected 1 lines got %d", fileJob.Lines)1178	}11791180	if fileJob.Code != 0 {1181		t.Errorf("Expected 0 lines got %d", fileJob.Code)1182	}11831184	if fileJob.Comment != 1 {1185		t.Errorf("Expected 1 lines got %d", fileJob.Comment)1186	}11871188	if fileJob.Blank != 0 {1189		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1190	}1191}11921193// Java does not support nested multiline comments1194func TestCountStatsNestedCommentsJava(t *testing.T) {1195	ProcessConstants()1196	fileJob := FileJob{1197		Language: "Java",1198	}11991200	fileJob.SetContent(`/*/**/*/`)12011202	CountStats(&fileJob)12031204	if fileJob.Lines != 1 {1205		t.Errorf("Expected 1 lines got %d", fileJob.Lines)1206	}12071208	if fileJob.Code != 1 {1209		t.Errorf("Expected 1 lines got %d", fileJob.Code)1210	}12111212	if fileJob.Comment != 0 {1213		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1214	}12151216	if fileJob.Blank != 0 {1217		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1218	}1219}12201221func TestCountStatsNestedCommentsRegression(t *testing.T) {1222	ProcessConstants()1223	fileJob := FileJob{1224		Language: "Rust",1225	}12261227	fileJob.SetContent(`t/*/**/*/`)12281229	CountStats(&fileJob)12301231	if fileJob.Lines != 1 {1232		t.Errorf("Expected 1 lines got %d", fileJob.Lines)1233	}12341235	if fileJob.Code != 1 {1236		t.Errorf("Expected 1 lines got %d", fileJob.Code)1237	}12381239	if fileJob.Comment != 0 {1240		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1241	}12421243	if fileJob.Blank != 0 {1244		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1245	}1246}12471248// Block comments nest in a good many more languages than the ones that had the1249// flag set. Each of these opens an inner comment inside an outer one, and the1250// text between the inner closer and the outer one is still comment.1251func TestCountStatsNestedCommentsByLanguage(t *testing.T) {1252	ProcessConstants()12531254	for _, test := range []struct {1255		language string1256		open     string1257		close    string1258	}{1259		{"Haskell", "{-", "-}"},1260		{"Odin", "/*", "*/"},1261		{"Scala", "/*", "*/"},1262		{"F#", "(*", "*)"},1263		{"OCaml", "(*", "*)"},1264		{"Coq", "(*", "*)"},1265		{"Dart", "/*", "*/"},1266		{"Elm", "{-", "-}"},1267		{"PureScript", "{-", "-}"},1268		{"Agda", "{-", "-}"},1269	} {1270		fileJob := FileJob{Language: test.language}1271		fileJob.SetContent(test.open + " outer " + test.open + " inner " + test.close + " still the outer comment " + test.close + "\nx = 1\n")12721273		CountStats(&fileJob)12741275		if fileJob.Lines != 2 {1276			t.Errorf("%s: expected 2 lines got %d", test.language, fileJob.Lines)1277		}12781279		if fileJob.Code != 1 {1280			t.Errorf("%s: expected 1 code got %d", test.language, fileJob.Code)1281		}12821283		if fileJob.Comment != 1 {1284			t.Errorf("%s: expected 1 comment got %d", test.language, fileJob.Comment)1285		}12861287		if fileJob.Blank != 0 {1288			t.Errorf("%s: expected 0 blank got %d", test.language, fileJob.Blank)1289		}1290	}1291}12921293func TestCountStatsSingleCommentRegression(t *testing.T) {1294	ProcessConstants()1295	fileJob := FileJob{1296		Language: "Rust",1297	}12981299	fileJob.SetContent(`t = "1300/*1301";`)13021303	CountStats(&fileJob)13041305	if fileJob.Lines != 3 {1306		t.Errorf("Expected 3 lines got %d", fileJob.Lines)1307	}13081309	if fileJob.Code != 3 {1310		t.Errorf("Expected 3 lines got %d", fileJob.Code)1311	}13121313	if fileJob.Comment != 0 {1314		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1315	}13161317	if fileJob.Blank != 0 {1318		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1319	}1320}13211322func TestCountStatsStringCheck(t *testing.T) {1323	ProcessConstants()1324	fileJob := FileJob{1325		Language: "Rust",1326	}13271328	fileJob.SetContent(`let does_not_start = // "1329"until here,1330test/*1331test"; // a quote: "`)13321333	CountStats(&fileJob)13341335	if fileJob.Lines != 4 {1336		t.Errorf("Expected 4 lines got %d", fileJob.Lines)1337	}13381339	if fileJob.Code != 4 {1340		t.Errorf("Expected 4 code lines got %d", fileJob.Code)1341	}13421343	if fileJob.Comment != 0 {1344		t.Errorf("Expected 0 comment lines got %d", fileJob.Comment)1345	}13461347	if fileJob.Blank != 0 {1348		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)1349	}1350}13511352func TestCountStatsBosque(t *testing.T) {1353	ProcessConstants()1354	fileJob := FileJob{1355		Language: "Bosque",1356	}13571358	fileJob.SetContent(`//This is a bosque test1359method offsetMomentum(px: Float, py: Float, pz: Float): Body {1360      return this<~(vx=Float::div(px->negate(), Body::solar_mass), vy=Float::div(py->negate(), Body::solar_mass), vz=Float::div(pz->negate(), Body::solar_mass));1361}`)13621363	CountStats(&fileJob)13641365	if fileJob.Lines != 4 {1366		t.Errorf("Expected 4 lines got %d", fileJob.Lines)1367	}13681369	if fileJob.Code != 3 {1370		t.Errorf("Expected 4 code lines got %d", fileJob.Code)1371	}13721373	if fileJob.Comment != 1 {1374		t.Errorf("Expected 0 comment lines got %d", fileJob.Comment)1375	}13761377	if fileJob.Blank != 0 {1378		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)1379	}1380}13811382func TestCheckForMatchNoMatch(t *testing.T) {1383	ProcessConstants()13841385	fileJob := FileJob{1386		Language: "Rust",1387		Content:  []byte("one does not simply walk into mordor"),1388	}13891390	matches := &Trie{}1391	matches.Insert(TSlcomment, []byte("//"))1392	matches.Insert(TSlcomment, []byte("--"))13931394	match, _, _ := matches.Match(fileJob.Content)13951396	if match != 0 {1397		t.Errorf("Expected no match")1398	}1399}14001401func TestCheckForMatchHasMatch(t *testing.T) {1402	ProcessConstants()14031404	fileJob := FileJob{1405		Language: "Rust",1406		Content:  []byte("// one does not simply walk into mordor"),1407	}14081409	matches := &Trie{}1410	matches.Insert(TSlcomment, []byte("//"))1411	matches.Insert(TSlcomment, []byte("--"))14121413	match, _, _ := matches.Match(fileJob.Content)14141415	if match != TSlcomment {1416		t.Errorf("Expected match")1417	}1418}14191420func TestCheckForMatchSingleNoMatch(t *testing.T) {1421	ProcessConstants()14221423	fileJob := FileJob{1424		Language: "Rust",1425		Content:  []byte("// one does not simply walk into mordor"),1426	}14271428	matches := []byte("*/")14291430	match := checkForMatchSingle('/', 0, 100, matches, &fileJob)14311432	if match != false {1433		t.Errorf("Expected no match")1434	}1435}14361437func TestCheckForMatchSingleMatch(t *testing.T) {1438	ProcessConstants()14391440	fileJob := FileJob{1441		Language: "Rust",1442		Content:  []byte("*/ one does not simply walk into mordor"),1443	}14441445	matches := []byte("*/")14461447	match := checkForMatchSingle('*', 0, 100, matches, &fileJob)14481449	if match != true {1450		t.Errorf("Expected match")1451	}1452}14531454func TestCheckComplexityMatch(t *testing.T) {1455	ProcessConstants()14561457	fileJob := FileJob{1458		Language: "Java",1459		Content:  []byte("for (int i=0; i<100; i++) {"),1460	}14611462	matches := &Trie{}1463	matches.Insert(TComplexity, []byte("for "))1464	matches.Insert(TComplexity, []byte("for("))14651466	match, n, _ := matches.Match(fileJob.Content)14671468	if match != TComplexity || n != 4 {1469		t.Errorf("Expected match")1470	}1471}14721473func TestCheckComplexityNoMatch(t *testing.T) {1474	ProcessConstants()14751476	fileJob := FileJob{1477		Language: "Java",1478		Content:  []byte("far (int i=0; i<100; i++) {"),1479	}14801481	matches := &Trie{}1482	matches.Insert(TComplexity, []byte("for "))1483	matches.Insert(TComplexity, []byte("for("))14841485	match, _, _ := matches.Match(fileJob.Content)14861487	if match != 0 {1488		t.Errorf("Expected no match")1489	}1490}14911492func TestCountStatsRubyRegression(t *testing.T) {1493	ProcessConstants()1494	fileJob := FileJob{1495		Language: "Ruby",1496	}14971498	fileJob.SetContent(`=begin1499=end1500t`)15011502	CountStats(&fileJob)15031504	if fileJob.Lines != 3 {1505		t.Errorf("Expected 3 lines got %d", fileJob.Lines)1506	}15071508	if fileJob.Code != 1 {1509		t.Errorf("Expected 1 code lines got %d", fileJob.Code)1510	}15111512	if fileJob.Comment != 2 {1513		t.Errorf("Expected 2 comment lines got %d", fileJob.Comment)1514	}15151516	if fileJob.Blank != 0 {1517		t.Errorf("Expected 0 blank lines got %d", fileJob.Blank)1518	}1519}15201521func TestFileProcessorWorker(t *testing.T) {1522	inputChan := make(chan *FileJob, 10000)15231524	inputChan <- &FileJob{1525		Filename:  "testing.go",1526		Location:  "./",1527		Extension: "go",1528		Content:   []byte("this is some content"),1529	}15301531	close(inputChan)1532	outputChan := make(chan *FileJob, 10000)15331534	Duplicates = true15351536	ctx := processorContext{remap: newRemapConfig("", "")}1537	ctx.fileProcessorWorker(inputChan, outputChan)15381539	for res := range outputChan {1540		if res.Bytes == 0 {1541			t.Error("Expect bytes to have something")1542		}1543	}1544}15451546func TestParseRemapRulesIgnoresInvalidEntries(t *testing.T) {1547	rules := parseRemapRules("match:Go,invalid,too:many:parts,:Rust")15481549	if len(rules) != 2 {1550		t.Fatalf("expected 2 rules, got %d", len(rules))1551	}15521553	if string(rules[0].pattern) != "match" || rules[0].language != "Go" {1554		t.Fatalf("unexpected first rule: %#v", rules[0])1555	}15561557	if string(rules[1].pattern) != "" || rules[1].language != "Rust" {1558		t.Fatalf("unexpected second rule: %#v", rules[1])1559	}1560}15611562func TestHardRemapLanguageUsesParsedRules(t *testing.T) {1563	job := &FileJob{1564		Language: "Plain Text",1565		Location: "./test.txt",1566		Content:  []byte("prefix -*- C++ -*- suffix"),1567	}15681569	ctx := processorContext{remap: newRemapConfig("-*- C++ -*-:C Header", "")}1570	remapped := ctx.hardRemapLanguage(job)15711572	if !remapped {1573		t.Fatal("expected file to be remapped")1574	}15751576	if job.Language != "C Header" {1577		t.Fatalf("expected remapped language to be C Header, got %s", job.Language)1578	}1579}15801581func TestEdgeCase(t *testing.T) {1582	ProcessConstants()1583	fileJob := FileJob{1584		Language: "C#",1585	}15861587	// For C# we can enter a string using @" or " but if we do the former,1588	// and we don't skip over the full length we exit the string in this case1589	// which means we pick up the /* and the count is incorrect1590	fileJob.SetContent(`@"\ /*"1591a`)15921593	CountStats(&fileJob)15941595	if fileJob.Lines != 2 {1596		t.Errorf("Expected 2 lines")1597	}15981599	if fileJob.Code != 2 {1600		t.Errorf("Expected 2 lines got %d", fileJob.Code)1601	}16021603	if fileJob.Comment != 0 {1604		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1605	}1606}16071608func TestEdgeCaseOther(t *testing.T) {1609	ProcessConstants()1610	fileJob := FileJob{1611		Language: "C#",1612	}16131614	// For C# we can enter a string using @" or " but if we do the former,1615	// and we don't skip over the full length we exit the string in this case1616	// which means we pick up the /* and the count is incorrect1617	fileJob.SetContent(`@"C:\" /*1618a */`)16191620	CountStats(&fileJob)16211622	if fileJob.Lines != 2 {1623		t.Errorf("Expected 2 lines")1624	}16251626	if fileJob.Code != 1 {1627		t.Errorf("Expected 1 lines got %d", fileJob.Code)1628	}16291630	if fileJob.Comment != 1 {1631		t.Errorf("Expected 1 lines got %d", fileJob.Comment)1632	}1633}16341635func TestCountStatsCSharpIgnoreEscape(t *testing.T) {1636	ProcessConstants()1637	fileJob := FileJob{1638		Language: "C#",1639	}16401641	fileJob.SetContent(`namespace Ns1642{1643   public class Cls1644   {1645       private const string BasePath = @"a:\";16461647       [Fact]1648       public void MyTest()1649       {1650           // Arrange.1651           Foo();16521653           // Act.1654           Bar();16551656           // Assert.1657           Baz();1658       }1659   }1660}`)16611662	CountStats(&fileJob)16631664	if fileJob.Lines != 20 {1665		t.Errorf("Expected 20 lines")1666	}16671668	if fileJob.Code != 14 {1669		t.Errorf("Expected 14 lines got %d", fileJob.Code)1670	}16711672	if fileJob.Comment != 3 {1673		t.Errorf("Expected 3 lines got %d", fileJob.Comment)1674	}16751676	if fileJob.Blank != 3 {1677		t.Errorf("Expected 3 lines got %d", fileJob.Blank)1678	}1679}16801681func TestCheckBomSkipUTF8(t *testing.T) {1682	fileJob := &FileJob{1683		Content: []byte{239, 187, 191}, // UTF-8 BOM1684	}16851686	skip := checkBomSkip(fileJob)1687	if skip != 3 {1688		t.Errorf("Expected skip length to match 3 got %d", skip)1689	}1690}16911692func TestCheckBomSkip(t *testing.T) {1693	Verbose = true1694	for _, v := range ByteOrderMarks {1695		fileJob := &FileJob{1696			Content: v,1697		}16981699		skip := checkBomSkip(fileJob)1700		if skip != 0 {1701			t.Errorf("Expected skip length to match %d got %d", len(v), skip)1702		}1703	}1704}17051706// Captures checking if a quote is prefixed by \ such as in1707// a char which should otherwise trigger the string state which is incorrect1708func TestCountStatsIssue73(t *testing.T) {1709	ProcessConstants()1710	fileJob := FileJob{1711		Language: "Java",1712	}17131714	fileJob.SetContent(`'\"'{1715code17161717`)1718	fileJob.Bytes = int64(len(fileJob.Content))17191720	CountStats(&fileJob)17211722	if fileJob.Lines != 3 {1723		t.Errorf("Expected 3 lines")1724	}17251726	if fileJob.Code != 2 {1727		t.Errorf("Expected 2 lines got %d", fileJob.Code)1728	}17291730	if fileJob.Comment != 0 {1731		t.Errorf("Expected 0 lines got %d", fileJob.Comment)1732	}17331734	if fileJob.Blank != 1 {1735		t.Errorf("Expected 1 lines got %d", fileJob.Blank)1736	}1737}17381739func TestCountStatsIssue106(t *testing.T) {1740	ProcessConstants()1741	fileJob := FileJob{1742		Language: "Go",1743	}17441745	fileJob.SetContent("foo = `\nabc\"\ndef\n`")17461747	CountStats(&fileJob)1748}17491750func TestMinifiedGeneratedCheck(t *testing.T) {1751	fileJob := FileJob{1752		Language: "Go",1753	}17541755	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF")1756	Minified = true1757	CountStats(&fileJob)1758	Minified = false17591760	if fileJob.Minified != true {1761		t.Error("Expected minified to come back true")1762	}1763}17641765func TestMinifiedGeneratedCheckTwoLines(t *testing.T) {1766	fileJob := FileJob{1767		Language: "Go",1768	}17691770	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF\n1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCDEF")1771	Minified = true1772	CountStats(&fileJob)1773	Minified = false17741775	if fileJob.Minified != true {1776		t.Error("Expected minified to come back true")1777	}1778}17791780func TestMinifiedGeneratedCheckEdge(t *testing.T) {1781	fileJob := FileJob{1782		Language: "Go",1783	}17841785	fileJob.SetContent("1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890ABCD")1786	Minified = true1787	CountStats(&fileJob)1788	Minified = false17891790	if fileJob.Minified != false {1791		t.Error("Expected minified to come back false")1792	}1793}17941795func TestGenerated(t *testing.T) {1796	fileJob := FileJob{1797		Language: "Go",1798	}17991800	fileJob.SetContent(`1801// Code generated by some tool, DO NOT EDIT.18021803// Package some contains something.1804package some1805`)1806	Generated = true1807	GeneratedMarkers = []string{"do not edit", "generated"}1808	CountStats(&fileJob)1809	Generated = false18101811	if fileJob.Generated != true {1812		t.Error("Expected generated to come back true")1813	}18141815	if fileJob.Language != "Go (gen)" {1816		t.Errorf("Expected Language \"Go (gen)\", received %q", fileJob.Language)1817	}1818}18191820func TestCountStatsIssue182(t *testing.T) {1821	ProcessConstants()1822	fileJob := FileJob{1823		Language: "Pascal",1824	}18251826	fileJob.SetContent(`uses1827    someunit;18281829{This is a comment}1830procedure Something1831var1832    avar: String;1833begin1834    Println('Oho');1835end;1836{This is a comment}1837procedure Nothing1838begin1839end.1840`)18411842	CountStats(&fileJob)18431844	if fileJob.Code != 11 {1845		t.Errorf("Expected 11 lines got %d", fileJob.Code)1846	}18471848	if fileJob.Comment != 2 {1849		t.Errorf("Expected 2 lines got %d", fileJob.Comment)1850	}18511852	if fileJob.Blank != 1 {1853		t.Errorf("Expected 1 lines got %d", fileJob.Blank)1854	}1855}18561857func TestCountStatsIssue182Delphi(t *testing.T) {1858	ProcessConstants()1859	fileJob := FileJob{1860		Language: "Pascal",1861	}18621863	fileJob.SetContent(`// this isnt a comment in pascal but is in delphi1864`)18651866	CountStats(&fileJob)18671868	if fileJob.Code != 0 {1869		t.Errorf("Expected 0 lines got %d", fileJob.Code)1870	}18711872	if fileJob.Comment != 1 {1873		t.Errorf("Expected 1 lines got %d", fileJob.Comment)1874	}18751876	if fileJob.Blank != 0 {1877		t.Errorf("Expected 0 lines got %d", fileJob.Blank)1878	}1879}18801881//////////////////////////////////////////////////1882// Content Classification Tests1883//////////////////////////////////////////////////18841885// Classification should not change any line counts1886func TestClassifyContentCountInvariance(t *testing.T) {1887	ProcessConstants()18881889	content := `package main18901891import "fmt"18921893// main function1894func main() {1895	fmt.Println("hello") // inline comment1896	/* block1897	   comment */1898}1899`1900	// Run without classification1901	fj1 := FileJob{Language: "Go"}1902	fj1.SetContent(content)1903	CountStats(&fj1)19041905	// Run with classification1906	fj2 := FileJob{Language: "Go", ClassifyContent: true}1907	fj2.SetContent(content)1908	CountStats(&fj2)19091910	if fj1.Lines != fj2.Lines {1911		t.Errorf("Lines mismatch: %d vs %d", fj1.Lines, fj2.Lines)1912	}1913	if fj1.Code != fj2.Code {1914		t.Errorf("Code mismatch: %d vs %d", fj1.Code, fj2.Code)1915	}1916	if fj1.Comment != fj2.Comment {1917		t.Errorf("Comment mismatch: %d vs %d", fj1.Comment, fj2.Comment)1918	}1919	if fj1.Blank != fj2.Blank {1920		t.Errorf("Blank mismatch: %d vs %d", fj1.Blank, fj2.Blank)1921	}1922	if fj1.Complexity != fj2.Complexity {1923		t.Errorf("Complexity mismatch: %d vs %d", fj1.Complexity, fj2.Complexity)1924	}1925}19261927// ContentByteType should stay nil when ClassifyContent is false (default)1928func TestClassifyContentNilGuard(t *testing.T) {1929	ProcessConstants()1930	fileJob := FileJob{Language: "Go"}1931	fileJob.SetContent("x := 1\n")1932	CountStats(&fileJob)19331934	if fileJob.ContentByteType != nil {1935		t.Error("Expected ContentByteType to be nil when ClassifyContent is false")1936	}1937}19381939// Code-only file: all non-whitespace bytes should be ByteTypeCode1940func TestClassifyContentCodeOnly(t *testing.T) {1941	ProcessConstants()1942	fileJob := FileJob{Language: "Go", ClassifyContent: true}1943	fileJob.SetContent("x := 1")1944	CountStats(&fileJob)19451946	if fileJob.ContentByteType == nil {1947		t.Fatal("Expected ContentByteType to be non-nil")1948	}19491950	for i, b := range fileJob.Content {1951		bt := fileJob.ContentByteType[i]1952		if b == ' ' || b == '\t' || b == '\n' || b == '\r' {1953			continue // whitespace can be blank or code depending on state1954		}1955		if bt != ByteTypeCode {1956			t.Errorf("byte %d (%q): expected ByteTypeCode(%d), got %d", i, string(b), ByteTypeCode, bt)1957		}1958	}1959}19601961// Comment-only file: "// comment" bytes after // should be ByteTypeComment1962func TestClassifyContentCommentOnly(t *testing.T) {1963	ProcessConstants()1964	fileJob := FileJob{Language: "Go", ClassifyContent: true}1965	fileJob.SetContent("// this is a comment")1966	CountStats(&fileJob)19671968	if fileJob.ContentByteType == nil {1969		t.Fatal("Expected ContentByteType to be non-nil")1970	}19711972	// Every byte of the line, not merely one of them. The two bytes of the //1973	// are classified before the loop reaches the comment state at all, so "at1974	// least one" passed even with the whole body left unclassified, which is1975	// exactly what the SComment skip does when its byteType guard is removed.1976	for i := range fileJob.Content {1977		if fileJob.ContentByteType[i] != ByteTypeComment {1978			t.Errorf("byte %d (%q) is %v, want ByteTypeComment: the whole of a comment-only line is comment",1979				i, fileJob.Content[i], fileJob.ContentByteType[i])1980		}1981	}19821983	if fileJob.Comment != 1 {1984		t.Errorf("Expected 1 comment line, got %d", fileJob.Comment)1985	}1986}19871988// Multi-line comment: inner bytes should be ByteTypeComment1989func TestClassifyContentMultiLineComment(t *testing.T) {1990	ProcessConstants()1991	fileJob := FileJob{Language: "Go", ClassifyContent: true}1992	fileJob.SetContent("/* hello\nworld */")1993	CountStats(&fileJob)19941995	if fileJob.ContentByteType == nil {1996		t.Fatal("Expected ContentByteType to be non-nil")1997	}19981999	// Check that "hello" inside the comment is classified as comment2000	helloStart := bytes.Index(fileJob.Content, []byte("hello"))