Filter Chokidar file watching by dataset exts array to reduce noise

apps/service/src/watcher.service.ts

@@ -1,8 +1,8 @@
 import { Inject, Injectable, Logger } from '@nestjs/common';
-import { Worker } from 'worker_threads';
 import chokidar, { FSWatcher } from 'chokidar';
 import fs from 'fs';
 import path from 'path';
+import { Worker } from 'worker_threads';
 import { DatasetsService } from './datasets.service';
 import { DbService } from './db.service';
 import { EventsGateway } from './events.gateway';
@@ -31,7 +31,9 @@ export class WatcherService {
     @Inject(EventsGateway) private readonly eventsGateway: EventsGateway,
     @Inject(TaskQueueService) private readonly taskQueue: TaskQueueService,
   ) {
-    this.validationWorker = new Worker(path.join(__dirname, 'file-validation-worker.js'));
+    this.validationWorker = new Worker(
+      path.join(__dirname, 'file-validation-worker.js'),
+    );
     this.validationWorker.on('message', (message) => {
       if (message.type === 'validation_result') {
         const callback = this.validationCallbacks.get(message.file);
@@ -57,12 +59,50 @@ export class WatcherService {
         ? watches
         : this.datasetsService.getEnabledDatasetPaths();
 
+    // Get dataset configuration to determine extensions to watch
+    const datasetConfig = this.datasetsService.getDatasetConfig();
+
+    // Create a function to determine if a file should be watched based on dataset extensions
+    const shouldWatchFile = (filePath: string): boolean => {
+      // Get the dataset for this file path
+      const dataset = this.getDatasetFromPath(filePath);
+      if (!dataset) {
+        return false; // Don't watch files that don't belong to any dataset
+      }
+
+      // Get the dataset configuration
+      const datasetSettings = datasetConfig[dataset];
+      if (!datasetSettings || !datasetSettings.enabled) {
+        return false;
+      }
+
+      // Get the exts array for this dataset
+      const exts = datasetSettings.exts;
+      if (!exts || !Array.isArray(exts) || exts.length === 0) {
+        // If no exts specified, watch all files (backward compatibility)
+        return true;
+      }
+
+      // Check if file extension matches any of the allowed extensions
+      const fileExt = path.extname(filePath).toLowerCase();
+      return exts.some((ext: string) => {
+        const normalizedExt = ext.startsWith('.')
+          ? ext.toLowerCase()
+          : `.${ext.toLowerCase()}`;
+        return fileExt === normalizedExt;
+      });
+    };
+
     // Override options to be more conservative for file descriptor limits
     const conservativeOptions = {
       ...options,
       interval: Math.max(options.interval || 10000, 30000), // Minimum 30 seconds
       depth: options.depth !== undefined ? options.depth : 1,
       ignorePermissionErrors: true,
+      ignored: (filePath: string) => {
+        // Use the shouldWatchFile function to filter files
+        return !shouldWatchFile(filePath);
+      },
     };
 
     this.watcher = chokidar.watch(enabledWatches, conservativeOptions);
@@ -101,6 +141,34 @@ export class WatcherService {
       return;
     }
 
+    // Get dataset configuration to check extensions
+    const datasetConfig = this.datasetsService.getDatasetConfig();
+    const datasetSettings = datasetConfig[dataset];
+
+    if (!datasetSettings || !datasetSettings.enabled) {
+      return;
+    }
+
+    // Check if file extension matches the dataset's exts array
+    const exts = datasetSettings.exts;
+    if (exts && Array.isArray(exts) && exts.length > 0) {
+      const fileExt = path.extname(file).toLowerCase();
+      const extensionMatches = exts.some((ext: string) => {
+        const normalizedExt = ext.startsWith('.')
+          ? ext.toLowerCase()
+          : `.${ext.toLowerCase()}`;
+        return fileExt === normalizedExt;
+      });
+
+      if (!extensionMatches) {
+        // File extension doesn't match, skip processing
+        this.logger.debug(
+          `Skipping file ${file} - extension not in dataset exts array`,
+        );
+        return;
+      }
+    }
+
     // Offload validation to worker
     this.validationCallbacks.set(file, (result) => {
       if (!result.isValid) {
@@ -362,8 +430,6 @@ export class WatcherService {
     return null;
   }
 
-
-
   async stop() {
     if (this.watcher && this.isWatching) {
       await this.watcher.close();