1// SPDX-License-Identifier: MIT23package processor45import (6	"bytes"7	"cmp"8	"errors"9	"slices"10	"strings"11)1213var (14	errMissingShebang              = errors.New("missing shebang")15	errUnknownShebang              = errors.New("unknown shebang")16	errUnableToDetermineShebangCmd = errors.New("unable to determine shebang command")17)1819// DetectLanguage detects a language based on the filename returns the language extension and error20func DetectLanguage(name string) ([]string, string) {21	extension := ""2223	if len(AllowListExtensions) == 0 {24		// Check the full name for special languages such as xmake.lua, meson.build, ...25		lang, ok := FilenameToLanguage[strings.ToLower(name)]26		if ok {27			return []string{lang}, name28		}2930		t := strings.Count(name, ".")31		// If there is no . in the filename or it starts with one then check if #!32		if t == 0 || (name[0] == '.' && t == 1) {33			printWarnF("possible #! file: %s", name)3435			// No extension indicates possible #! so mark as such for processing36			return []string{SheBang}, name37		}38	}3940	// Lookup in case the full name matches41	language, ok := ExtensionToLanguage[strings.ToLower(name)]4243	// If no match check if we have a matching extension44	if !ok {45		extension = getExtension(name)46		language, ok = ExtensionToLanguage[extension]47	}4849	// Convert from d.ts to ts and check that in case of multiple extensions50	if !ok {51		extension = getExtension(extension)52		language = ExtensionToLanguage[extension]53	}5455	return language, extension56}5758// DetectSheBang given some content attempt to determine if it has a #! that maps to a known language and return the language59func DetectSheBang(content []byte) (string, error) {60	if !bytes.HasPrefix(content, []byte("#!")) {61		return "", errMissingShebang62	}6364	content, _, _ = bytes.Cut(content, []byte{'\n'})6566	cmd, err := scanForSheBang(content)67	if err != nil {68		return "", err69	}7071	for k, v := range ShebangLookup {72		if slices.Contains(v, cmd) {73			// detects both full path and env usage74			return k, nil75		}76	}7778	return "", errUnknownShebang79}8081func scanForSheBang(content []byte) (string, error) {82	state := 083	lastSlash := 08485	candidate1 := ""86	candidate2 := ""8788loop:89	for i := range content {90		switch state {91		case 0: // Deals with whitespace after #! and before first /92			if content[i] == '/' {93				lastSlash = i94				state = 195			}96		case 1: // Once we found the first / keep going till we hit whitespace97			if content[i] == '/' {98				lastSlash = i99			}100101			// when at the end pull out the candidate102			if i == len(content)-1 {103				candidate1 = string(content[lastSlash+1 : i+1])104			}105106			// between last slash and here is the first candidate which is either env or Perl/PHP/Python etc..107			if isWhitespace(content[i]) {108				// mark from lastSlash to here as first argument109				candidate1 = string(content[lastSlash+1 : i])110				state = 2111			}112		case 2: // We have the first candidate, see if there is another113			// go till end of whitespace, mark that spot as new start114			if !isWhitespace(content[i]) {115				lastSlash = i116				state = 3117			}118		case 3:119			if i == len(content)-1 {120				candidate2 = string(content[lastSlash : i+1])121			}122123			if isWhitespace(content[i]) {124				candidate2 = string(content[lastSlash:i])125				state = 4126			}127		case 4:128			break loop129		}130	}131132	switch {133	case candidate1 == "env":134		return candidate2, nil135	case candidate1 != "":136		return candidate1, nil137	}138139	return "", errUnableToDetermineShebangCmd140}141142type languageGuess struct {143	Name  string144	Count int145}146147// heuristicCanMatch reports whether content contains at least one of the148// heuristic's necessary literals, meaning the regex is worth running. Each149// literal is a substring that must be present for the pattern to match, so a150// negative result here means the regex cannot match and can be safely skipped.151// A heuristic with no literals is always run so a pattern is never silently152// disabled.153func heuristicCanMatch(h CompiledHeuristic, content []byte) bool {154	if len(h.Literals) == 0 {155		return true156	}157	for _, lit := range h.Literals {158		if h.Anchored {159			if anchoredContains(content, lit) {160				return true161			}162		} else if bytes.Contains(content, lit) {163			return true164		}165	}166	return false167}168169// anchoredContains reports whether lit appears in content at the start of a line170// preceded only by spaces or tabs, mirroring a (?m)^[ \t]* regex prefix. This is171// far cheaper than running the regex and avoids substring false positives such172// as "entry" satisfying a check for the "try" keyword.173func anchoredContains(content, lit []byte) bool {174	offset := 0175	for {176		i := bytes.Index(content[offset:], lit)177		if i < 0 {178			return false179		}180		pos := offset + i181		j := pos182		for j > 0 && (content[j-1] == ' ' || content[j-1] == '\t') {183			j--184		}185		if j == 0 || content[j-1] == '\n' {186			return true187		}188		offset = pos + 1189	}190}191192func guessByHeuristics(filename string, possibleLanguages []string, toCheck []byte) (string, bool) {193	guesses := make([]languageGuess, 0, len(possibleLanguages))194	hasHeuristics := false195196	for _, lan := range possibleLanguages {197		LanguageFeaturesMutex.Lock()198		langFeatures := LanguageFeatures[lan]199		LanguageFeaturesMutex.Unlock()200201		if len(langFeatures.Heuristics) == 0 {202			continue203		}204		hasHeuristics = true205206		count := 0207		for _, h := range langFeatures.Heuristics {208			// Cheap necessary-literal pre-check so the expensive regex only runs209			// on the rare files that could actually match it.210			if !heuristicCanMatch(h, toCheck) {211				continue212			}213			if h.Re.Match(toCheck) {214				count++215			}216		}217218		guesses = append(guesses, languageGuess{Name: lan, Count: count})219	}220221	if !hasHeuristics {222		return "", false223	}224225	slices.SortFunc(guesses, func(a, b languageGuess) int {226		if order := cmp.Compare(b.Count, a.Count); order != 0 {227			return order228		}229		return strings.Compare(a.Name, b.Name)230	})231232	if len(guesses) != 0 && guesses[0].Count > 0 {233		printWarnF("guessing language %s for file %s via heuristics", guesses[0].Name, filename)234		return guesses[0].Name, true235	}236237	return "", false238}239240// DetermineLanguage given a filename, fallback language, possible languages and content make a guess to the type.241// If multiple possible it will guess based on keywords similar to how https://github.com/vmchale/polyglot does242func DetermineLanguage(filename string, fallbackLanguage string, possibleLanguages []string, content []byte) string {243	// If being called through an API it's possible nothing is set here and as244	// such should just return as the Language value should have already been set245	if len(possibleLanguages) == 0 {246		return fallbackLanguage247	}248249	// There should only be two possibilities now, either we have a single fallbackLanguage250	// in which case we set it and return251	// or we have multiple in which case we try to determine it heuristically252	if len(possibleLanguages) == 1 {253		return possibleLanguages[0]254	}255256	startTime := makeTimestampNano()257258	toCheck := content259	if len(content) > 20_000 {260		toCheck = content[:20_000]261	}262263	// First attempt regex heuristic disambiguation, used for shared extensions264	// such as .h between C / C++ / Objective-C.265	// Based on how linguist does it https://github.com/github-linguist/linguist/266	// which should be fine as its under MIT license267	if lang, ok := guessByHeuristics(filename, possibleLanguages, toCheck); ok {268		printTraceF("nanoseconds to guess language: %s: %d", filename, makeTimestampNano()-startTime)269		return lang270	}271272	primary := ""273274	toSort := make([]languageGuess, 0, len(possibleLanguages))275	for _, lan := range possibleLanguages {276		LanguageFeaturesMutex.Lock()277		langFeatures := LanguageFeatures[lan]278		LanguageFeaturesMutex.Unlock()279280		// We only do language checks if no heuristics exist281		if len(langFeatures.Heuristics) != 0 {282			continue283		}284285		count := 0286		for _, key := range langFeatures.KeywordBytes {287			if bytes.Contains(toCheck, key) {288				count++289			}290		}291292		// if no features are found that means that this one is considered the primary293		// and as such the default fallback if we don't find a suitable number of matching294		// keywords295		// consider YAML files for example, where cloudformation files can also be YAML296		// YAML can have any form so it's not possible to say "this is a yaml file"297		// so we can only say "this is likely to be a cloudformation file", and as such298		// we need to handle a fallback case, which in this case is nothing299		// When several candidates qualify as the fallback we pick the300		// alphabetically first so the choice is deterministic.301		if len(langFeatures.Keywords) == 0 && len(langFeatures.Heuristics) == 0 {302			if primary == "" || lan < primary {303				primary = lan304			}305		}306307		toSort = append(toSort, languageGuess{Name: lan, Count: count})308	}309310	slices.SortFunc(toSort, func(a, b languageGuess) int {311		if order := cmp.Compare(b.Count, a.Count); order != 0 {312			return order313		}314		return strings.Compare(a.Name, b.Name)315	})316317	if primary != "" && len(toSort) != 0 {318		// OK at this point we have a primary, which means we want 3 or more matches to count as something else319		if toSort[0].Count < 3 {320			// we didn't find enough results, so lets return the primary in this case321			return primary322		}323	}324325	printWarnF("guessing language %s for file %s", toSort[0].Name, filename)326	printTraceF("nanoseconds to guess language: %s: %d", filename, makeTimestampNano()-startTime)327328	if len(toSort) != 0 {329		return toSort[0].Name330	}331332	return fallbackLanguage333}