1// SPDX-License-Identifier: MIT23package processor45import (6	"os"7	"path/filepath"8	"strings"9	"sync"10)1112// Used as quick lookup for files with the same name to avoid some processing13// needs to be sync.Map as it potentially could be called by many GoRoutines14var extensionCache sync.Map1516// Added as a way to track files per run.17var visitedPaths sync.Map1819// A custom version of extracting extensions for a file20// which also has a case-insensitive cache in order to save21// some needless processing22func getExtension(name string) string {23	name = strings.ToLower(name)24	extension, ok := extensionCache.Load(name)2526	if ok {27		return extension.(string)28	}2930	ext := filepath.Ext(name)3132	if ext == "" || strings.LastIndex(name, ".") == 0 {33		extension = name34	} else {35		// Handling multiple dots or multiple extensions only needs to delete the last extension36		// and then call filepath.Ext.37		// If there are multiple extensions, it is the value of subExt,38		// otherwise subExt is an empty string.39		subExt := filepath.Ext(strings.TrimSuffix(name, ext))40		extension = strings.TrimPrefix(subExt+ext, ".")41	}4243	extensionCache.Store(name, extension)44	return extension.(string)45}4647func cleanVisitedPaths() {48	visitedPaths.Clear()49}5051func newFileJob(path, name string, fileInfo os.FileInfo) *FileJob {52	if NoLarge {53		if fileInfo.Size() >= LargeByteCount {54			printWarnF("skipping large file due to byte size: %s", path)55			return nil56		}57	}5859	var symPath = ""60	// Check if the file is a symlink and if we want to count those then work out its path and rejig61	// everything so we can count the real file to ensure the counts are correct62	if fileInfo.Mode()&os.ModeSymlink == os.ModeSymlink {63		if !IncludeSymLinks {64			printWarnF("skipping symlink file: %s", name)65			return nil66		}6768		var err error69		symPath, err = filepath.EvalSymlinks(path)70		if err != nil {71			printError(err.Error())72			return nil73		}74		fileInfo, err = os.Lstat(symPath)75		if err != nil {76			printError(err.Error())77			return nil78		}79	}8081	// Skip non-regular files. They are unlikely to be code and may hang if we82	// try and read them.83	if !fileInfo.Mode().IsRegular() {84		printWarnF("skipping non-regular file: %s", path)85		return nil86	}8788	// This determines the real path89	realPath := path90	if symPath != "" {91		realPath = symPath92	}9394	// Prevent duplicate processing and loops95	if _, exists := visitedPaths.Load(realPath); exists {96		printWarnF("skipping already processed file: %s", realPath)97		return nil98	}99	visitedPaths.Store(realPath, true)100101	language, extension := DetectLanguage(name)102103	if len(language) != 0 {104		// check if extensions in the allow list, which should limit to just those extensions105		if len(AllowListExtensions) != 0 {106			ok := false107			for _, x := range AllowListExtensions {108				if x == extension {109					ok = true110				}111			}112113			if !ok {114				printWarnF("skipping file as not in allow list: %s", name)115				return nil116			}117		}118119		// check if we should exclude this type120		if len(ExcludeListExtensions) != 0 {121			ok := true122			for _, x := range ExcludeListExtensions {123				if x == extension {124					ok = false125				}126			}127128			if !ok {129				printWarnF("skipping file as in exclude list: %s", name)130				return nil131			}132		}133134		if len(ExcludeFilename) != 0 {135			ok := true136			for _, x := range ExcludeFilename {137				if strings.Contains(name, x) {138					ok = false139				}140			}141142			if !ok {143				printWarnF("skipping file as in exclude file list: %s", name)144				return nil145			}146		}147148		for _, l := range language {149			LoadLanguageFeature(l)150		}151152		if !CountIgnore {153			for _, l := range language {154				if l == "ignore" || l == "gitignore" {155					return nil156				}157			}158		}159160		return &FileJob{161			Location:          path,162			Symlocation:       symPath,163			Filename:          name,164			Extension:         extension,165			PossibleLanguages: language,166			Bytes:             fileInfo.Size(),167		}168	} else {169		printWarnF("skipping file unknown extension: %s", name)170	}171172	return nil173}