Enhance file indexing and exclusion logic for improved performance

- Updated the `indexWatch` module to refine the criteria for skipping non-source directories and files, including a comprehensive list of ignored paths. - Enhanced the `semanticIndex` module with additional file extensions and directory segments to ensure only relevant source files are indexed. - Introduced new utility functions to determine indexable paths, improving the accuracy of the indexing process and reducing unnecessary overhead. - Expanded the `IGNORE` set in the `shared` module to include more build and cache directories, streamlining the file walking process.
2026-07-18 08:05:57 +02:00 · 2026-07-15 15:28:56 +03:00
parent 205c6cd1b7
commit a3d917d1b5
3 changed files with 100 additions and 10 deletions
@@ -65,10 +65,16 @@ function onFs(uri: vscode.Uri, action: "up" | "del"): void {
  if (!root) return;
  const abs = uri.fsPath;
  if (!abs.startsWith(root) && !abs.toLowerCase().startsWith(root.toLowerCase())) return;
-  // Skip heavy/vendor trees (walk already ignores some; watcher still sees them).
+  // Skip vendor/build/non-source (upsertFile also filters; early-out saves work).
  const rel = path.relative(root, abs).split(path.sep).join("/");
  if (!rel || rel.startsWith("..")) return;
-  if (/(^|\/)(node_modules|\.git|dist|out|build|\.next|coverage)(\/|$)/.test(rel)) return;
+  if (
+    /(^|\/)(node_modules|\.git|dist|out|build|\.next|\.nuxt|\.output|\.turbo|\.cache|coverage|\.venv|venv|__pycache__|target|vendor|Pods|\.gradle|\.idea|\.vscode|bower_components|jspm_packages|\.pnpm-store|\.yarn|site-packages|\.terraform|\.svelte-kit|\.angular|storybook-static)(\/|$)/.test(
+      rel,
+    )
+  ) {
+    return;
+  }
  pending.set(abs, action);
  scheduleFlush();
 }
@@ -79,10 +79,41 @@ export function setRemoteEmbedModel(cfg: RemoteEmbedConfig): void {

 const CHUNK_LINES = 40;
 const CHUNK_OVERLAP = 10;
+/** Source / doc extensions worth embedding. No binaries, lockfiles, or assets. */
 const EMBED_EXTS = new Set([
-  ".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".rs", ".go", ".java",
-  ".c", ".h", ".cpp", ".cc", ".hpp", ".cs", ".rb", ".php", ".swift", ".kt",
-  ".scala", ".md", ".json", ".html", ".css", ".scss", ".vue", ".svelte", ".sql",
+  ".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs",
+  ".py", ".pyi", ".rs", ".go", ".java", ".kt", ".kts", ".scala",
+  ".c", ".h", ".cpp", ".cc", ".cxx", ".hpp", ".hh", ".cs",
+  ".rb", ".php", ".swift", ".m", ".mm",
+  ".vue", ".svelte", ".astro",
+  ".css", ".scss", ".less", ".sass",
+  ".html", ".htm", ".sql", ".graphql", ".gql",
+  ".md", ".mdx", ".rst", ".txt",
+  ".sh", ".bash", ".zsh", ".ps1", ".bat", ".cmd",
+  ".toml", ".yaml", ".yml", ".ini", ".cfg", ".conf",
+  ".json", ".jsonc",
+  ".proto", ".thrift", ".r", ".lua", ".ex", ".exs", ".erl", ".hs", ".clj", ".cljs",
+  ".zig", ".nim", ".dart", ".tf", ".hcl",
+]);
+/** Path segment names to never index (modules, build, caches, VCS). */
+const SKIP_DIR_SEGMENTS = new Set([
+  "node_modules", ".git", "dist", "out", "build", ".next", ".nuxt", ".output",
+  ".turbo", ".cache", "coverage", ".venv", "venv", "__pycache__", ".tox",
+  ".mypy_cache", ".pytest_cache", ".ruff_cache", "target", "vendor", "Pods",
+  ".gradle", ".idea", ".vscode", "bower_components", "jspm_packages",
+  ".pnpm-store", ".yarn", "site-packages", ".svn", ".hg", ".hgcheck",
+  "DerivedData", "xcuserdata", ".terraform", ".serverless", ".parcel-cache",
+  ".svelte-kit", ".angular", "storybook-static", "cypress", "playwright-report",
+  "test-results", ".nyc_output", "htmlcov",
+]);
+/** Exact basenames that are never source to embed. */
+const SKIP_BASENAMES = new Set([
+  "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb", "bun.lock",
+  "composer.lock", "Cargo.lock", "Gemfile.lock", "poetry.lock", "Pipfile.lock",
+  "go.sum", "flake.lock", "uv.lock",
+  ".DS_Store", "Thumbs.db", "desktop.ini",
+  "LICENSE", "LICENSE.txt", "LICENSE.md", "COPYING", "CHANGELOG.md", "CHANGELOG",
+  "package-lock.json",
 ]);
 const MAX_FILE_BYTES = 512 * 1024;

@@ -284,8 +315,31 @@ export async function deleteIndex(root: string): Promise<void> {
  emitStatus(root);
 }

+/** True when a workspace-relative path is real source (not vendor/build/junk). */
 function isIndexableRel(rel: string): boolean {
-  return EMBED_EXTS.has(path.extname(rel).toLowerCase());
+  if (!rel || rel.startsWith("..")) return false;
+  const parts = rel.split("/");
+  const base = parts[parts.length - 1] || "";
+  // Hidden files at any depth except common source dots (.env.example etc. skipped too).
+  if (base.startsWith(".") && base !== ".gitignore" && base !== ".editorconfig") return false;
+  for (const seg of parts) {
+    if (!seg) continue;
+    if (SKIP_DIR_SEGMENTS.has(seg)) return false;
+    // Nested deps / generated trees often use these prefixes.
+    if (seg.startsWith(".") && (seg === ".git" || seg.endsWith("_cache") || seg.endsWith("-cache"))) return false;
+  }
+  if (SKIP_BASENAMES.has(base)) return false;
+  // Minified / bundled / source maps (not author source).
+  if (/\.(min|bundle|chunk)\.(js|css|mjs|cjs)$/i.test(base)) return false;
+  if (/\.map$/i.test(base)) return false;
+  if (/\.(png|jpe?g|gif|webp|ico|svg|woff2?|ttf|eot|mp[34]|wav|zip|gz|tgz|7z|rar|pdf|wasm|exe|dll|so|dylib|bin|o|a|class|jar|war|ear|pyc|pyo|whl|lock)$/i.test(base)) {
+    return false;
+  }
+  const ext = path.extname(base).toLowerCase();
+  if (!EMBED_EXTS.has(ext)) return false;
+  // package.json etc. OK; skip huge generated JSON dumps by name pattern.
+  if (ext === ".json" && /(^|[-_.])(lock|bundle|manifest|sourcemap)([-_.]|$)/i.test(base)) return false;
+  return true;
 }

 async function embedFileInto(idx: IndexFile, root: string, rel: string): Promise<boolean> {
@@ -420,8 +474,8 @@ export async function buildIndex(root: string, onProgress?: (done: number, total
    const targets: string[] = [];
    const seen = new Set<string>();
    for (const f of all) {
-      if (!EMBED_EXTS.has(path.extname(f).toLowerCase())) continue;
      const rel = path.relative(root, f).split(path.sep).join("/");
+      if (!isIndexableRel(rel)) continue;
      let st;
      try { st = await fs.stat(f); } catch { continue; }
      if (st.size > MAX_FILE_BYTES) continue;
@@ -13,8 +13,38 @@ import { spawn } from "child_process";
 import type { ChildProcess } from "child_process";
 import type { SubagentRunner, QuestionAsker } from "./types";

-// Directories never walked/listed.
-export const IGNORE = new Set([".git", "node_modules", "dist", "out"]);
+// Directories never walked/listed (tools + indexing).
+export const IGNORE = new Set([
+  ".git",
+  "node_modules",
+  "dist",
+  "out",
+  "build",
+  ".next",
+  ".nuxt",
+  ".output",
+  ".turbo",
+  ".cache",
+  "coverage",
+  ".venv",
+  "venv",
+  "__pycache__",
+  ".tox",
+  ".mypy_cache",
+  ".pytest_cache",
+  ".ruff_cache",
+  "target",
+  "vendor",
+  "Pods",
+  ".gradle",
+  ".idea",
+  ".vscode",
+  "bower_components",
+  "jspm_packages",
+  ".pnpm-store",
+  ".yarn",
+  "site-packages",
+]);

 // ---------------------------------------------------------------------------
 // Per-tool hard timeouts (ms). Prevents a hung Grep/Glob/Shell/etc. from
@@ -195,7 +225,7 @@ export function firstDiffLine(before: string, after: string): number {

 /**
 * Recursively collect file paths under `dir` (depth-capped). IGNORE dirs
- * (.git/node_modules/dist/out) are skipped unless `includeIgnored` is true.
+ * (node_modules, .git, build caches, …) are skipped unless `includeIgnored` is true.
 */
 export async function walk(
  dir: string,