Ensure overridden ignores are applied

Fixes a regression in 51a7be2.

Author: Nixinova

src/cli.ts

@@ -5,6 +5,7 @@ import path from 'path';
 import { program } from 'commander';
 
 import linguist from './index';
+import { normPath } from './helpers/norm-path';
 
 const colouredMsg = ([r, g, b]: number[], msg: string): string => `\u001B[${38};2;${r};${g};${b}m${msg}${'\u001b[0m'}`;
 const hexToRgb = (hex: string): number[] => [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)];
@@ -91,7 +92,7 @@ if (args.analyze) (async () => {
 			if (args.listFiles) {
 				console.log(); // padding
 				for (const file of filesPerLanguage[lang]) {
-					let relFile = path.relative(path.resolve('.'), file).replace(/\\/g, '/');
+					let relFile = normPath(path.relative(path.resolve('.'), file));
 					if (!relFile.startsWith('../')) relFile = './' + relFile;
 					const bytes = (await fs.promises.stat(file)).size;
 					const fmtd2 = {

src/helpers/norm-path.ts

@@ -0,0 +1,10 @@
+import paths from 'path';
+
+export const normPath = function normalisedPath(...inputPaths: string[]) {
+	return paths.join(...inputPaths).replace(/\\/g, '/');
+}
+
+export const normAbsPath = function normalisedAbsolutePath(...inputPaths: string[]) {
+	return paths.resolve(...inputPaths).replace(/\\/g, '/');
+}
+

src/helpers/parse-gitattributes.ts

@@ -1,6 +1,5 @@
-import path from 'path';
-
 import * as T from '../types';
+import { normPath } from './norm-path';
 
 export type FlagAttributes = {
 	'vendored': boolean | null,
@@ -27,7 +26,7 @@ export default function parseAttributes(content: string, folderRoot: string = '.
 
 		const parts = line.split(/\s+/g);
 		const fileGlob = parts[0];
-		const relFileGlob = path.join(folderRoot, fileGlob).replace(/\\/g, '/');
+		const relFileGlob = normPath(folderRoot, fileGlob);
 		const attrParts = parts.slice(1);
 		const isTrue = (str: string) => !str.startsWith('-') && !str.endsWith('=false');
 		const isFalse = (str: string) => str.startsWith('-') || str.endsWith('=false');

src/helpers/walk-tree.ts

@@ -2,6 +2,7 @@ import fs from 'fs';
 import paths from 'path';
 import ignore, { Ignore } from 'ignore';
 import parseGitignore from './parse-gitignore';
+import { normPath, normAbsPath } from './norm-path';
 
 let allFiles: Set<string>;
 let allFolders: Set<string>;
@@ -17,8 +18,6 @@ interface WalkInput {
 	folders: string[],
 	/** An instantiated Ignore object listing ignored files */
 	ignored: Ignore,
-	/** A list of regexes to ignore */
-	regexIgnores: RegExp[],
 };
 
 interface WalkOutput {
@@ -28,7 +27,7 @@ interface WalkOutput {
 
 /** Generate list of files in a directory. */
 export default function walk(data: WalkInput): WalkOutput {
-	const { init, commonRoot, folderRoots, folders, ignored, regexIgnores } = data;
+	const { init, commonRoot, folderRoots, folders, ignored } = data;
 
 	// Initialise files and folders lists
 	if (init) {
@@ -44,47 +43,46 @@ export default function walk(data: WalkInput): WalkOutput {
 		// Get list of files and folders inside this folder
 		const files = fs.readdirSync(folder).map(file => {
 			// Create path relative to root
-			const base = paths.resolve(folder, file).replace(/\\/g, '/').replace(commonRoot, '.');
+			const base = normAbsPath(folder, file).replace(commonRoot, '.');
 			// Add trailing slash to mark directories
 			const isDir = fs.lstatSync(paths.resolve(commonRoot, base)).isDirectory();
 			return isDir ? `${base}/` : base;
 		});
 
 		// Read and apply gitignores
-		const gitignoreFilename = paths.join(folder, '.gitignore');
+		const gitignoreFilename = normPath(folder, '.gitignore');
 		if (fs.existsSync(gitignoreFilename)) {
 			const gitignoreContents = fs.readFileSync(gitignoreFilename, 'utf-8');
 			const ignoredPaths = parseGitignore(gitignoreContents);
 			ignored.add(ignoredPaths);
 		}
 
 		// Add gitattributes if present
-		const gitattributesPath = paths.join(folder, '.gitattributes');
+		const gitattributesPath = normPath(folder, '.gitattributes');
 		if (fs.existsSync(gitattributesPath)) {
 			allFiles.add(gitattributesPath);
 		}
 
 		// Loop through files and folders
 		for (const file of files) {
 			// Create absolute path for disc operations
-			const path = paths.resolve(commonRoot, file).replace(/\\/g, '/');
+			const path = normAbsPath(commonRoot, file);
 			const localPath = localRoot ? file.replace(`./${localRoot}/`, '') : file.replace('./', '');
 
 			// Skip if nonexistant
 			const nonExistant = !fs.existsSync(path);
 			if (nonExistant) continue;
-			// Skip if marked as ignored
+			// Skip if marked in gitignore
 			const isIgnored = ignored.test(localPath).ignored;
-			const isRegexIgnored = regexIgnores.some(pattern => pattern.test(localPath));
-			if (isIgnored || isRegexIgnored) continue;
+			if (isIgnored) continue;
 
 			// Add absolute folder path to list
-			allFolders.add(paths.resolve(folder).replace(/\\/g, '/'));
+			allFolders.add(normAbsPath(folder));
 			// Check if this is a folder or file
 			if (file.endsWith('/')) {
 				// Recurse into subfolders
 				allFolders.add(path);
-				walk({ init: false, commonRoot, folderRoots, folders: [path], ignored, regexIgnores });
+				walk({ init: false, commonRoot, folderRoots, folders: [path], ignored });
 			}
 			else {
 				// Add file path to list
@@ -95,7 +93,7 @@ export default function walk(data: WalkInput): WalkOutput {
 	// Recurse into all folders
 	else {
 		for (const i in folders) {
-			walk({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored, regexIgnores });
+			walk({ init: false, commonRoot, folderRoots: [folderRoots[i]], folders: [folders[i]], ignored });
 		}
 	}
 	// Return absolute files and folders lists

src/index.ts

@@ -11,6 +11,7 @@ import loadFile, { parseGeneratedDataFile } from './helpers/load-data';
 import readFile from './helpers/read-file';
 import parseAttributes, { FlagAttributes } from './helpers/parse-gitattributes';
 import pcre from './helpers/convert-pcre';
+import { normPath } from './helpers/norm-path';
 import * as T from './types';
 import * as S from './schema';
 
@@ -50,10 +51,8 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom
 	};
 
 	// Set a common root path so that vendor paths do not incorrectly match parent folders
-	const normPath = (file: string) => file.replace(/\\/g, '/');
 	const resolvedInput = input.map(path => normPath(paths.resolve(path)));
 	const commonRoot = (input.length > 1 ? commonPrefix(resolvedInput) : resolvedInput[0]).replace(/\/?$/, '');
-	const localRoot = (folder: T.AbsFile): T.RelFile => folder.replace(commonRoot, '').replace(/^\//, '');
 	const relPath = (file: T.AbsFile): T.RelFile => useRawContent ? file : normPath(paths.relative(commonRoot, file));
 	const unRelPath = (file: T.RelFile): T.AbsFile => useRawContent ? file : normPath(paths.resolve(commonRoot, file));
 
@@ -77,7 +76,7 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom
 	}
 	else {
 		// Uses directory on disc
-		const data = walk({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored, regexIgnores });
+		const data = walk({ init: true, commonRoot, folderRoots: resolvedInput, folders: resolvedInput, ignored });
 		files = data.files;
 	}
 
@@ -86,6 +85,22 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom
 	const getFlaggedGlobs = (attr: keyof FlagAttributes, val: boolean) => {
 		return Object.entries(manualAttributes).filter(([, attrs]) => attrs[attr] === val).map(([glob,]) => glob)
 	};
+	const findAttrsForPath = (filePath: string): FlagAttributes | null => {
+		const resultAttrs: Record<string, string | boolean | null> = {};
+		for (const glob in manualAttributes) {
+			if (ignore().add(glob).ignores(relPath(filePath))) {
+				const matchingAttrs = manualAttributes[glob];
+				for (const [attr, val] of Object.entries(matchingAttrs)) {
+					if (val !== null) resultAttrs[attr] = val;
+				}
+			}
+		}
+
+		if (!JSON.stringify(resultAttrs)) {
+			return null;
+		}
+		return resultAttrs as FlagAttributes;
+	}
 	if (!useRawContent && opts.checkAttributes) {
 		const nestedAttrFiles = files.filter(file => file.endsWith('.gitattributes'));
 		for (const attrFile of nestedAttrFiles) {
@@ -99,6 +114,27 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom
 		}
 	}
 
+	// Remove files that are linguist-ignored via regex by default unless explicitly unignored in gitattributes
+	const filesToIgnore: T.AbsFile[] = [];
+	for (const file of files) {
+		const relFile = relPath(file);
+
+		const isRegexIgnored = regexIgnores.some(pattern => pattern.test(relFile));
+		if (!isRegexIgnored) {
+			// Checking overrides is moot if file is not even marked as ignored by default
+			continue;
+		}
+
+		const fileAttrs = findAttrsForPath(file);
+		if (fileAttrs?.generated === false || fileAttrs?.vendored === false) {
+			// File is explicitly marked as *not* to be ignored
+			// do nothing
+		} else {
+			filesToIgnore.push(file);
+		}
+	}
+	files = files.filter(file => !filesToIgnore.includes(file));
+
 	// Apply vendor file path matches and filter out vendored files
 	if (!opts.keepVendored) {
 		// Get data of files that have been manually marked with metadata
@@ -361,7 +397,7 @@ async function analyse(rawPaths?: string | string[], opts: T.Options = {}): Prom
 	if (!useRawContent && opts.relativePaths) {
 		const newMap: Record<T.RelFile, T.LanguageResult> = {};
 		for (const [file, lang] of Object.entries(results.files.results)) {
-			let relPath = paths.relative(process.cwd(), file).replace(/\\/g, '/');
+			let relPath = normPath(paths.relative(process.cwd(), file));
 			if (!relPath.startsWith('../')) {
 				relPath = './' + relPath;
 			}