refactor(files): remove status column from files table

- Remove status field from files table schema
- Update db.service methods to exclude status handling
- Replace getFilesByStatus with getAllFiles method
- Update watcher to not set status when discovering files
- Remove status field from FileCrud form
- Fix FileList to fetch all files instead of filtering by status

BREAKING CHANGE: Files table no longer tracks processing status. The files table now only records file observations (dataset, input, output, date). Task processing status is tracked separately in the tasks table.

apps/service/src/app.controller.ts

@@ -5,6 +5,7 @@ import {
   Get,
   Param,
   Post,
+  Put,
   Query,
 } from '@nestjs/common';
 import * as path from 'path';
@@ -15,7 +16,6 @@ interface FileRecord {
   dataset: string;
   input: string;
   output: string;
-  status: string;
   date: string;
 }
 
@@ -124,6 +124,19 @@ export class AppController {
     return result;
   }
 
+  // Clear all file records from database
+  @Delete('files')
+  clearAllFiles() {
+    const result = this.appService.clearAllFiles();
+    // Emit a general file update to notify clients
+    this.eventsGateway.emitFileUpdate({
+      type: 'cleared',
+      dataset: '*',
+      file: '*',
+    });
+    return { message: 'All file records cleared' };
+  }
+
   // Requeue a file for processing (creates a task)
   @Post('files/:dataset/:file/requeue')
   requeueFile(
@@ -139,53 +152,74 @@ export class AppController {
     // Try to find existing file record first
     const fileRecord = this.appService.findFile(dataset, file) as FileRecord;
 
-    if (fileRecord) {
-      // Use existing file record
-      output = fileRecord.output;
-    } else {
-      // No existing file record - determine output path from dataset config
-      let destination: string | undefined;
-      let ext = '.mkv'; // Default extension
-
-      if (datasetConfig[dataset]) {
-        const datasetObj = datasetConfig[dataset];
-
-        // Find the path configuration that matches this file
-        for (const pathKey of Object.keys(datasetObj)) {
-          if (pathKey !== 'enabled' && file.startsWith(pathKey)) {
-            const pathConfig = datasetObj[pathKey];
-            if (pathConfig) {
-              if (pathConfig.preset && !processingPreset) {
-                processingPreset = pathConfig.preset;
-              }
-              if (pathConfig.destination) {
-                destination = pathConfig.destination;
-              }
-              if (pathConfig.ext) {
-                ext = pathConfig.ext.startsWith('.')
-                  ? pathConfig.ext
-                  : '.' + pathConfig.ext;
-              }
-            }
+    // Determine preset from dataset config (same logic as watcher)
+    if (!processingPreset) {
+      processingPreset = 'Fast 1080p30'; // Default fallback
+    }
+
+    if (datasetConfig[dataset]) {
+      const datasetObj = datasetConfig[dataset];
+
+      // Find the path configuration that matches this file
+      for (const pathKey of Object.keys(datasetObj)) {
+        if (pathKey !== 'enabled' && file.startsWith(pathKey)) {
+          const pathConfig = datasetObj[pathKey];
+          if (pathConfig && pathConfig.preset) {
+            processingPreset = pathConfig.preset;
             break;
           }
         }
+      }
 
-        // Fallback to old format
-        if (!processingPreset && datasetObj.preset) {
-          processingPreset = datasetObj.preset;
-        }
-        if (!destination && datasetObj.destination) {
-          destination = datasetObj.destination;
-        }
-        if (ext === '.mkv' && datasetObj.ext) {
-          ext = datasetObj.ext.startsWith('.')
-            ? datasetObj.ext
-            : '.' + datasetObj.ext;
+      // Fallback to dataset-level preset if no path-specific preset found
+      if (processingPreset === 'Fast 1080p30' && datasetObj.preset) {
+        processingPreset = datasetObj.preset;
+      }
+    }
+
+    console.log(`Final processingPreset: ${processingPreset}`);
+
+    // Determine destination and ext from dataset config
+    let destination: string | undefined;
+    let ext = '.mkv'; // Default extension
+
+    if (datasetConfig[dataset]) {
+      const datasetObj = datasetConfig[dataset];
+
+      // Find the path configuration that matches this file
+      for (const pathKey of Object.keys(datasetObj)) {
+        if (pathKey !== 'enabled' && file.startsWith(pathKey)) {
+          const pathConfig = datasetObj[pathKey];
+          if (pathConfig) {
+            if (pathConfig.destination) {
+              destination = pathConfig.destination;
+            }
+            if (pathConfig.ext) {
+              ext = pathConfig.ext.startsWith('.')
+                ? pathConfig.ext
+                : '.' + pathConfig.ext;
+            }
+          }
+          break;
         }
       }
 
-      // Determine output path
+      // Fallback to dataset-level config
+      if (!destination && datasetObj.destination) {
+        destination = datasetObj.destination;
+      }
+      if (ext === '.mkv' && datasetObj.ext) {
+        ext = datasetObj.ext.startsWith('.')
+          ? datasetObj.ext
+          : '.' + datasetObj.ext;
+      }
+    }
+
+    if (fileRecord) {
+      // Use existing file record
+      output = fileRecord.output;
+    } else {
+      // No existing file record - determine output path from dataset config
       if (destination) {
         const fileName = path.basename(file, path.extname(file));
         output = path.join(destination, fileName + ext);
@@ -210,12 +244,14 @@ export class AppController {
       output: output,
       preset: processingPreset,
       priority: 1, // Higher priority for manual requeues
-      status: 'requeued', // Mark as manually requeued
+      status: 'pending', // Mark as pending for processing
     });
 
-    // Update file status to pending (this will create the file record if it doesn't exist)
+    // Check if file already exists
+    const existingFile = this.appService.findFile(dataset, file);
+
+    // Update file record (create if doesn't exist, or update output if requeueing)
     this.appService.setFile(dataset, file, {
-      status: 'pending',
       date: new Date().toISOString(),
       output: output,
     });
@@ -241,12 +277,9 @@ export class AppController {
     return { taskId: task.id, message: 'Task created for processing' };
   }
 
-  @Get('files/:dataset/status/:status')
-  getFilesByStatus(
-    @Param('dataset') dataset: string,
-    @Param('status') status: string,
-  ) {
-    return this.appService.getFilesByStatus(dataset, status);
+  @Get('files/:dataset')
+  getAllFiles(@Param('dataset') dataset: string) {
+    return this.appService.getAllFiles(dataset);
   }
 
   @Get('files/:dataset/deleted-older-than/:isoDate')
@@ -502,4 +535,25 @@ export class AppController {
   stopTaskProcessing() {
     return this.appService.stopTaskProcessing();
   }
+
+  // Task management endpoints
+  @Get('tasks')
+  getTasks() {
+    return this.appService.getAllTasks();
+  }
+
+  @Get('tasks/:id')
+  getTask(@Param('id') id: string) {
+    return this.appService.getTaskById(parseInt(id));
+  }
+
+  @Put('tasks/:id')
+  updateTask(@Param('id') id: string, @Body() updates: any) {
+    return this.appService.updateTask(parseInt(id), updates);
+  }
+
+  @Post('tasks')
+  createTask(@Body() taskData: any) {
+    return this.appService.createTask(taskData);
+  }
 }

apps/service/src/app.service.ts

@@ -32,12 +32,11 @@ export class AppService {
   }
 
   getTotalSuccessfulFiles() {
-    const datasetPaths = this.datasets.getEnabledDatasetPaths();
+    const datasetNames = this.datasets.getAllDatasetNames();
     let total = 0;
-    for (const path of datasetPaths) {
-      const datasetName = path.split('/').pop();
+    for (const datasetName of datasetNames) {
       if (datasetName) {
-        const files = this.db.getFilesByStatus(datasetName, 'success');
+        const files = this.db.getAllFiles(datasetName);
         total += files.length;
       }
     }
@@ -45,14 +44,12 @@ export class AppService {
   }
 
   getTotalProcessedFiles() {
-    const datasetPaths = this.datasets.getEnabledDatasetPaths();
+    const datasetNames = this.datasets.getAllDatasetNames();
     let total = 0;
-    for (const path of datasetPaths) {
-      const datasetName = path.split('/').pop();
+    for (const datasetName of datasetNames) {
       if (datasetName) {
-        const successful = this.db.getFilesByStatus(datasetName, 'success');
-        const deleted = this.db.getFilesByStatus(datasetName, 'deleted');
-        total += successful.length + deleted.length;
+        const files = this.db.getAllFiles(datasetName);
+        total += files.length;
       }
     }
     return total;
@@ -102,6 +99,10 @@ export class AppService {
     return this.db.getTaskById(id);
   }
 
+  updateTask(id: number, updates: any) {
+    return this.db.updateTask(id, updates);
+  }
+
   deleteTask(id: number) {
     return this.db.deleteTask(id);
   }
@@ -209,8 +210,12 @@ export class AppService {
     return this.db.removeFile(dataset, file, soft);
   }
 
-  getFilesByStatus(dataset: string, status: string) {
-    return this.db.getFilesByStatus(dataset, status);
+  clearAllFiles() {
+    return this.db.clearAllFiles();
+  }
+
+  getAllFiles(dataset: string) {
+    return this.db.getAllFiles(dataset);
   }
 
   getDeletedOlderThan(dataset: string, isoDate: string) {

apps/service/src/db.service.ts

@@ -20,8 +20,27 @@ export class DbService {
 
   constructor() {
     // Use unified database for all settings/configuration
-    const rootDataPath = path.resolve(process.cwd(), 'data/database.db');
+    // Find project root by traversing up from current directory until we find the root package.json
+    let projectRoot = process.cwd();
+    while (projectRoot !== path.dirname(projectRoot)) {
+      if (fs.existsSync(path.join(projectRoot, 'package.json'))) {
+        try {
+          const pkg = JSON.parse(
+            fs.readFileSync(path.join(projectRoot, 'package.json'), 'utf-8'),
+          );
+          if (pkg.name === 'watch-finished-turbo') {
+            break;
+          }
+        } catch (e) {
+          // ignore
+        }
+      }
+      projectRoot = path.dirname(projectRoot);
+    }
+
+    const rootDataPath = path.resolve(projectRoot, 'data/database.db');
     console.log('Database path:', rootDataPath);
+    console.log('Project root:', projectRoot);
     console.log('Current working directory:', process.cwd());
 
     // Ensure the directory exists
@@ -88,14 +107,13 @@ export class DbService {
         dataset TEXT,
         input TEXT,
         output TEXT,
-        status TEXT,
         date TEXT,
         PRIMARY KEY (dataset, input)
       );
     `);
 
     const insert = dbInstance.prepare(
-      'INSERT INTO files (dataset, input, output, status, date) VALUES (?, ?, ?, ?, ?)',
+      'INSERT INTO files (dataset, input, output, date) VALUES (?, ?, ?, ?)',
     );
     for (const dataset of datasets) {
       const filePath = path.join(dataDir, `${dataset}.json`);
@@ -107,7 +125,6 @@ export class DbService {
           dataset,
           rec.input || null,
           rec.output || null,
-          rec.status || null,
           rec.date || null,
         );
       }
@@ -123,7 +140,6 @@ export class DbService {
         dataset TEXT,
         input TEXT,
         output TEXT,
-        status TEXT,
         date TEXT,
         PRIMARY KEY (dataset, input)
       );
@@ -157,6 +173,11 @@ export class DbService {
     const existingColumns = tableInfo.map((col) => col.name);
 
     const columnsToAdd = [
+      { name: 'dataset', type: 'TEXT' },
+      { name: 'input', type: 'TEXT' },
+      { name: 'output', type: 'TEXT' },
+      { name: 'preset', type: 'TEXT' },
+      { name: 'error_message', type: 'TEXT' },
       { name: 'priority', type: 'INTEGER DEFAULT 0' },
       { name: 'retry_count', type: 'INTEGER DEFAULT 0' },
       { name: 'max_retries', type: 'INTEGER' },
@@ -185,13 +206,12 @@ export class DbService {
       const rec = file;
       this.db
         .prepare(
-          'INSERT INTO files (dataset, input, output, status, date) VALUES (?, ?, ?, ?, ?)',
+          'INSERT INTO files (dataset, input, output, date) VALUES (?, ?, ?, ?)',
         )
         .run(
           dataset,
           rec.input,
           rec.output,
-          rec.status,
           rec.date ? new Date(rec.date).toISOString() : null,
         );
       return;
@@ -200,11 +220,10 @@ export class DbService {
     if (found) {
       this.db
         .prepare(
-          'UPDATE files SET output = COALESCE(?, output), status = COALESCE(?, status), date = COALESCE(?, date) WHERE dataset = ? AND input = ?',
+          'UPDATE files SET output = COALESCE(?, output), date = COALESCE(?, date) WHERE dataset = ? AND input = ?',
         )
         .run(
           payload.output,
-          payload.status,
           payload.date ? new Date(payload.date).toISOString() : null,
           dataset,
           file,
@@ -212,13 +231,12 @@ export class DbService {
     } else {
       this.db
         .prepare(
-          'INSERT INTO files (dataset, input, output, status, date) VALUES (?, ?, ?, ?, ?)',
+          'INSERT INTO files (dataset, input, output, date) VALUES (?, ?, ?, ?)',
         )
         .run(
           dataset,
           file,
           payload.output,
-          payload.status,
           payload.date ? new Date(payload.date).toISOString() : null,
         );
     }
@@ -238,18 +256,20 @@ export class DbService {
     }
   }
 
-  getFilesByStatus(dataset: string, status: string) {
+  clearAllFiles() {
+    this.db.prepare('DELETE FROM files').run();
+  }
+
+  getAllFiles(dataset: string) {
     return this.db
-      .prepare('SELECT * FROM files WHERE dataset = ? AND status = ?')
-      .all(dataset, status);
+      .prepare('SELECT * FROM files WHERE dataset = ?')
+      .all(dataset);
   }
 
   getDeletedOlderThan(dataset: string, isoDate: string) {
     return this.db
-      .prepare(
-        'SELECT * FROM files WHERE dataset = ? AND status = ? AND date < ?',
-      )
-      .all(dataset, 'deleted', isoDate);
+      .prepare('SELECT * FROM files WHERE dataset = ? AND date < ?')
+      .all(dataset, isoDate);
   }
 
   // Task CRUD methods
@@ -271,6 +291,10 @@ export class DbService {
     return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id);
   }
 
+  getTaskByInput(input: string) {
+    return this.db.prepare('SELECT * FROM tasks WHERE input = ?').get(input);
+  }
+
   createTask(task: {
     type: string;
     status?: string;
@@ -388,4 +412,107 @@ export class DbService {
 
     return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id);
   }
+
+  // Task maintenance methods
+  deleteTasksByStatus(status: string, olderThanDays?: number) {
+    let query = 'DELETE FROM tasks WHERE status = ?';
+    const params = [status];
+
+    if (olderThanDays !== undefined) {
+      query +=
+        ' AND created_at < datetime("now", "-' + olderThanDays + ' days")';
+    }
+
+    return this.db.prepare(query).run(...params);
+  }
+
+  deleteTasksOlderThan(days: number) {
+    return this.db
+      .prepare(
+        'DELETE FROM tasks WHERE created_at < datetime("now", "-' +
+          days +
+          ' days")',
+      )
+      .run();
+  }
+
+  getTaskStats() {
+    const stats = this.db
+      .prepare(
+        `
+        SELECT
+          status,
+          COUNT(*) as count,
+          MIN(created_at) as oldest,
+          MAX(created_at) as newest
+        FROM tasks
+        GROUP BY status
+      `,
+      )
+      .all();
+
+    const total = this.db
+      .prepare('SELECT COUNT(*) as total FROM tasks')
+      .get() as { total: number };
+
+    return {
+      total: total.total,
+      byStatus: stats,
+    };
+  }
+
+  archiveOldTasks(daysOld: number = 30): { changes?: number } {
+    // Create archive table if it doesn't exist
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS tasks_archive (
+        id INTEGER PRIMARY KEY,
+        type TEXT NOT NULL,
+        status TEXT NOT NULL,
+        progress REAL DEFAULT 0,
+        dataset TEXT,
+        input TEXT,
+        output TEXT,
+        preset TEXT,
+        priority INTEGER DEFAULT 0,
+        retry_count INTEGER DEFAULT 0,
+        max_retries INTEGER,
+        error_message TEXT,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        archived_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      )
+    `);
+
+    // Move old completed/failed tasks to archive
+    const insertResult = this.db
+      .prepare(
+        `
+      INSERT INTO tasks_archive
+      SELECT *, CURRENT_TIMESTAMP as archived_at
+      FROM tasks
+      WHERE status IN ('completed', 'failed', 'skipped')
+      AND created_at < datetime('now', '-${daysOld} days')
+    `,
+      )
+      .run();
+
+    // Delete archived tasks from main table
+    const deleteResult = this.db
+      .prepare(
+        `
+      DELETE FROM tasks
+      WHERE status IN ('completed', 'failed', 'skipped')
+      AND created_at < datetime('now', '-${daysOld} days')
+    `,
+      )
+      .run();
+
+    return { changes: insertResult.changes };
+  }
+
+  // Purge all tasks from the database
+  purgeAllTasks() {
+    const result = this.db.prepare('DELETE FROM tasks').run();
+    return result;
+  }
 }

apps/service/src/watcher.service.ts

@@ -1,10 +1,19 @@
 import { Inject, Injectable, Logger } from '@nestjs/common';
 import chokidar, { FSWatcher } from 'chokidar';
+import fs from 'fs';
 import path from 'path';
 import { DatasetsService } from './datasets.service';
+import { DbService } from './db.service';
 import { EventsGateway } from './events.gateway';
 import { TaskQueueService } from './task-queue.service';
 
+interface FileRecord {
+  dataset: string;
+  input: string;
+  output: string;
+  date: string;
+}
+
 @Injectable()
 export class WatcherService {
   private watcher: FSWatcher | null = null;
@@ -15,6 +24,7 @@ export class WatcherService {
 
   constructor(
     @Inject(DatasetsService) private readonly datasetsService: DatasetsService,
+    @Inject(DbService) private readonly db: DbService,
     @Inject(EventsGateway) private readonly eventsGateway: EventsGateway,
     @Inject(TaskQueueService) private readonly taskQueue: TaskQueueService,
   ) {}
@@ -29,10 +39,19 @@ export class WatcherService {
       watches && watches.length > 0
         ? watches
         : this.datasetsService.getEnabledDatasetPaths();
-    this.watcher = chokidar.watch(enabledWatches, options);
+
+    // Override options to be more conservative for file descriptor limits
+    const conservativeOptions = {
+      ...options,
+      interval: Math.max(options.interval || 10000, 30000), // Minimum 30 seconds
+      depth: options.depth !== undefined ? options.depth : 1,
+      ignorePermissionErrors: true,
+    };
+
+    this.watcher = chokidar.watch(enabledWatches, conservativeOptions);
     this.isWatching = true;
     this.lastWatches = enabledWatches;
-    this.lastOptions = options;
+    this.lastOptions = conservativeOptions;
     this.watcher
       .on('add', (file: string) => {
         this.logger.log(`File added: ${file}`);
@@ -75,6 +94,12 @@ export class WatcherService {
       return;
     }
 
+    // Validate that the file has proper video headers
+    if (!this.isValidVideoFile(file)) {
+      this.logger.warn(`File appears to be corrupted or incomplete: ${file}`);
+      return;
+    }
+
     // Get dataset configuration
     const datasetConfig = this.datasetsService.getDatasetConfig();
     const datasetSettings = datasetConfig[dataset];
@@ -86,14 +111,211 @@ export class WatcherService {
       return;
     }
 
-    // Determine preset
-    let preset = datasetSettings.preset || 'Fast 1080p30';
+    // Determine preset and output configuration - find the specific path configuration
+    let preset = 'Fast 1080p30'; // Default fallback
+    let destination: string | undefined;
+    let ext = '.mkv'; // Default extension
+    let clean: any;
+    let folder = false; // Default: don't create subfolders
+
+    if (datasetConfig[dataset]) {
+      const datasetObj = datasetConfig[dataset];
+
+      // Find the path configuration that matches this file
+      for (const pathKey of Object.keys(datasetObj)) {
+        if (pathKey !== 'enabled' && file.startsWith(pathKey)) {
+          const pathConfig = datasetObj[pathKey];
+          if (pathConfig) {
+            if (pathConfig.preset) {
+              preset = pathConfig.preset;
+            }
+            if (pathConfig.destination) {
+              destination = pathConfig.destination;
+            }
+            if (pathConfig.ext) {
+              // Ensure extension starts with a dot
+              ext = pathConfig.ext.startsWith('.')
+                ? pathConfig.ext
+                : '.' + pathConfig.ext;
+            }
+            if (pathConfig.clean) {
+              clean = pathConfig.clean;
+            }
+            if (typeof pathConfig.folder === 'boolean') {
+              folder = pathConfig.folder;
+            }
+          }
+          break;
+        }
+      }
+
+      // If no path-specific config found, try the old format (for backward compatibility)
+      if (preset === 'Fast 1080p30' && datasetObj.preset) {
+        preset = datasetObj.preset;
+      }
+      if (!destination && datasetObj.destination) {
+        destination = datasetObj.destination;
+      }
+      if (ext === '.mkv' && datasetObj.ext) {
+        ext = datasetObj.ext.startsWith('.')
+          ? datasetObj.ext
+          : '.' + datasetObj.ext;
+      }
+      if (!clean && datasetObj.clean) {
+        clean = datasetObj.clean;
+      }
+      if (!folder && typeof datasetObj.folder === 'boolean') {
+        folder = datasetObj.folder;
+      }
+    }
+
+    // Create output path based on configuration
+    let output: string;
+    if (destination) {
+      // If destination is specified, use it as the base path
+      const fileName = path.basename(file, path.extname(file));
+      let cleanFileName = fileName;
+
+      // Apply cleaning rules if specified
+      if (clean && typeof clean === 'object') {
+        for (const [pattern, replacement] of Object.entries(clean)) {
+          try {
+            const regex = new RegExp(pattern, 'g');
+            cleanFileName = cleanFileName.replace(regex, replacement as string);
+          } catch (error) {
+            this.logger.warn(
+              `Invalid regex pattern in clean config: ${pattern}`,
+            );
+          }
+        }
+      }
+
+      // If folder is enabled, create a subdirectory based on the cleaned filename
+      if (folder) {
+        // Try to extract series/site name from filename pattern
+        // Look for common date/episode patterns and take everything before the first separator
+        const patterns = [
+          /\d{2}\.\d{2}\.\d{2}/, // 24.12.17
+          /[A-Za-z]\d{3,4}/, // E651, S123, etc.
+          /\d{4}/, // 2024, 1234, etc.
+          /\.\d+/, // .123, .2024, etc.
+        ];
+
+        let folderName = cleanFileName.charAt(0).toUpperCase(); // fallback
+        let foundMatch = false;
+
+        for (const pattern of patterns) {
+          const match = cleanFileName.match(pattern);
+          if (match && match.index !== undefined && match.index > 0) {
+            // Take everything before the pattern as the potential folder name
+            let potentialFolderName = cleanFileName
+              .substring(0, match.index)
+              .trim();
+            // Remove trailing dots if any
+            potentialFolderName = potentialFolderName.replace(/\.$/, '');
+
+            // If the potential folder name contains dots, take only the first part (site name)
+            // This handles patterns like "Site.Series.Date..." where we want just "Site"
+            if (potentialFolderName.includes('.')) {
+              folderName = potentialFolderName.split('.')[0];
+            } else {
+              folderName = potentialFolderName;
+            }
+
+            foundMatch = true;
+            break;
+          }
+        }
+
+        // If no pattern matched but filename contains dots, try to extract site name
+        if (!foundMatch && cleanFileName.includes('.')) {
+          const parts = cleanFileName.split('.');
+          if (parts.length > 1 && parts[0].length > 0) {
+            folderName = parts[0];
+          }
+        } else if (cleanFileName.toLowerCase().startsWith('the ')) {
+          // For titles starting with "The", use the next word
+          const words = cleanFileName.split(' ');
+          if (words.length > 1) {
+            folderName = words[1].charAt(0).toUpperCase();
+          }
+        }
+
+        output = path.join(destination, folderName, cleanFileName + ext);
+      } else {
+        output = path.join(destination, cleanFileName + ext);
+      }
+
+      // Ensure destination directory exists
+      const outputDir = path.dirname(output);
+      if (!fs.existsSync(outputDir)) {
+        try {
+          fs.mkdirSync(outputDir, { recursive: true });
+          this.logger.log(`Created output directory: ${outputDir}`);
+        } catch (error) {
+          this.logger.error(
+            `Failed to create output directory ${outputDir}: ${error.message}`,
+          );
+          return;
+        }
+      }
+    } else {
+      // Default behavior: same directory with new extension
+      output = path.join(
+        path.dirname(file),
+        path.basename(file, path.extname(file)) + ext,
+      );
+    }
+
+    // Always create/update file record for discovered files
+    const existingFileRecord = this.db.findFile(dataset, file) as
+      | FileRecord
+      | undefined;
+    if (!existingFileRecord) {
+      // Create file record for newly discovered file
+      this.db.setFile(dataset, file, {
+        date: new Date().toISOString(),
+        output: output,
+      });
+      this.logger.log(`Discovered new file: ${file}`);
+    } else {
+      // Update existing file record with current output path (in case config changed)
+      this.db.setFile(dataset, file, {
+        output: output,
+      });
+    }
+
+    // Automatic task creation: only when output doesn't exist
+    const outputExists = fs.existsSync(output);
 
-    // Create output path (same directory, .mkv extension)
-    const output = path.join(
-      path.dirname(file),
-      path.basename(file, path.extname(file)) + '.mkv',
-    );
+    if (outputExists) {
+      this.logger.log(
+        `Output file already exists, skipping automatic task creation: ${output}`,
+      );
+      return;
+    }
+
+    // Check if task already exists for this input file
+    const existingTask = this.taskQueue.getTaskByInput(file);
+    if (existingTask) {
+      // If task exists and is currently processing, reset to pending for retry
+      if (existingTask.status === 'processing') {
+        this.logger.log(
+          `Resetting stuck processing task ${existingTask.id} to pending for file: ${file}`,
+        );
+        this.taskQueue.updateTaskStatus(existingTask.id, 'pending');
+        this.eventsGateway.emitTaskUpdate({
+          type: 'reset',
+          taskId: existingTask.id,
+          file,
+        });
+      } else {
+        this.logger.log(
+          `Task already exists for file: ${file} (status: ${existingTask.status})`,
+        );
+      }
+      return;
+    }
 
     // Create task for processing
     try {
@@ -104,6 +326,12 @@ export class WatcherService {
         preset,
       });
 
+      // Update file record to indicate processing has started
+      this.db.setFile(dataset, file, {
+        status: 'pending',
+        date: new Date().toISOString(),
+      });
+
       this.logger.log(`Created task ${task.id} for file: ${file}`);
 
       // Emit file update event
@@ -121,12 +349,19 @@ export class WatcherService {
   }
 
   private getDatasetFromPath(file: string): string | null {
-    const enabledPaths = this.datasetsService.getEnabledDatasetPaths();
+    const datasetConfig = this.datasetsService.getDatasetConfig();
 
-    for (const datasetPath of enabledPaths) {
-      if (file.startsWith(datasetPath)) {
-        // Extract dataset name from path (last directory name)
-        return path.basename(datasetPath);
+    // Iterate through each dataset and its paths
+    for (const datasetName of Object.keys(datasetConfig)) {
+      const datasetObj = datasetConfig[datasetName];
+      if (typeof datasetObj === 'object' && datasetObj !== null) {
+        // Check each path in the dataset configuration
+        for (const pathKey of Object.keys(datasetObj)) {
+          if (pathKey !== 'enabled' && file.startsWith(pathKey)) {
+            // Return the actual dataset name (e.g., "tvshows", "pr0n")
+            return datasetName;
+          }
+        }
       }
     }
 
@@ -148,9 +383,65 @@ export class WatcherService {
     return videoExtensions.includes(ext);
   }
 
-  stop() {
+  private isValidVideoFile(file: string): boolean {
+    try {
+      // Check if file exists and is readable
+      if (!fs.existsSync(file)) {
+        return false;
+      }
+
+      const stats = fs.statSync(file);
+      if (stats.size === 0) {
+        return false;
+      }
+
+      // Read first few bytes to check for video file signatures
+      const buffer = Buffer.alloc(12);
+      const fd = fs.openSync(file, 'r');
+      try {
+        fs.readSync(fd, buffer, 0, 12, 0);
+      } finally {
+        fs.closeSync(fd);
+      }
+
+      // Check for common video file signatures
+      const signature = buffer.toString('hex');
+
+      // MP4 signature (ftyp box)
+      if (signature.includes('66747970')) {
+        return true;
+      }
+
+      // MKV/WebM signature (EBML)
+      if (signature.startsWith('1a45dfa3')) {
+        return true;
+      }
+
+      // AVI signature (RIFF)
+      if (
+        signature.startsWith('52494646') &&
+        buffer.toString('ascii', 8, 12) === 'AVI '
+      ) {
+        return true;
+      }
+
+      // MOV signature (ftyp)
+      if (signature.includes('66747971') || signature.includes('66747970')) {
+        return true;
+      }
+
+      // For other formats, just check if file is large enough to be a video (> 1MB)
+      // This is a basic heuristic since not all video formats have easily detectable headers
+      return stats.size > 1024 * 1024;
+    } catch (error) {
+      this.logger.warn(`Error validating video file ${file}: ${error.message}`);
+      return false;
+    }
+  }
+
+  async stop() {
     if (this.watcher && this.isWatching) {
-      this.watcher.close();
+      await this.watcher.close();
       this.isWatching = false;
       this.logger.log('Watcher stopped.');
       this.eventsGateway.emitWatcherUpdate({ type: 'stopped' });

apps/web/src/app/components/FileCrud.tsx

@@ -12,13 +12,6 @@ interface FileCrudProps {
   onEditClose?: () => void;
 }
 
-const STATUS_OPTIONS = [
-  { value: "pending", label: "Pending" },
-  { value: "success", label: "Success" },
-  { value: "failed", label: "Failed" },
-  { value: "processing", label: "Processing" }
-];
-
 export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
   const queryClient = useQueryClient();
   const { addNotification } = useNotifications();
@@ -26,7 +19,6 @@ export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
   const [dataset, setDataset] = useState("pr0n");
   const [input, setInput] = useState("");
   const [output, setOutput] = useState("");
-  const [fileStatus, setFileStatus] = useState("");
   const [date, setDate] = useState("");
 
   const isEditing = !!editFile;
@@ -42,21 +34,18 @@ export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
       setDataset(editFile.dataset || "pr0n");
       setInput(editFile.input || "");
       setOutput(editFile.output || "");
-      setFileStatus(editFile.status || "");
       setDate(editFile.date || "");
       setIsOpen(true);
     }
   }, [editFile, isEditing]);
 
   const createMutation = useMutation({
-    mutationFn: () =>
-      post(`/files/${dataset}/${input}`, { output, status: fileStatus, date }),
+    mutationFn: () => post(`/files/${dataset}/${input}`, { output, date }),
     onSuccess: () => {
       queryClient.invalidateQueries({ queryKey: ["files"] });
       setDataset("pr0n");
       setInput("");
       setOutput("");
-      setFileStatus("");
       setDate("");
       setIsOpen(false);
       if (onEditClose) onEditClose();
@@ -79,7 +68,6 @@ export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
     setDataset("pr0n");
     setInput("");
     setOutput("");
-    setFileStatus("");
     setDate("");
     if (onEditClose) onEditClose();
   };
@@ -151,23 +139,6 @@ export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
               onChange={(e) => setOutput(e.target.value)}
             />
           </div>
-          <div>
-            <label className="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
-              Status
-            </label>
-            <select
-              className="block w-full rounded-md border-gray-300 dark:border-gray-700 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 sm:text-sm bg-white dark:bg-gray-900 text-gray-900 dark:text-gray-100"
-              value={fileStatus}
-              onChange={(e) => setFileStatus(e.target.value)}
-            >
-              <option value="">Select status</option>
-              {STATUS_OPTIONS.map((option) => (
-                <option key={option.value} value={option.value}>
-                  {option.label}
-                </option>
-              ))}
-            </select>
-          </div>
           <div>
             <label className="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
               Date
@@ -262,23 +233,6 @@ export default function FileCrud({ editFile, onEditClose }: FileCrudProps) {
               onChange={(e) => setOutput(e.target.value)}
             />
           </div>
-          <div>
-            <label className="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
-              Status
-            </label>
-            <select
-              className="block w-full rounded-md border-gray-300 dark:border-gray-700 shadow-sm focus:border-indigo-500 focus:ring-indigo-500 sm:text-sm bg-white dark:bg-gray-900 text-gray-900 dark:text-gray-100"
-              value={fileStatus}
-              onChange={(e) => setFileStatus(e.target.value)}
-            >
-              <option value="">Select status</option>
-              {STATUS_OPTIONS.map((option) => (
-                <option key={option.value} value={option.value}>
-                  {option.label}
-                </option>
-              ))}
-            </select>
-          </div>
           <div>
             <label className="block text-sm font-medium text-gray-700 dark:text-gray-200 mb-1">
               Date

apps/web/src/app/files/FileList.tsx

@@ -33,15 +33,12 @@ export default function FileList() {
     queryKey: ["all-files"],
     queryFn: async () => {
       if (!datasets) return [];
-      const allFilesPromises = datasets.map((datasetPath: string) =>
-        get(`/files/${datasetPath.split("/").pop()}/status/success`).catch(
-          () => []
-        )
+      const allFilesPromises = datasets.map((datasetName: string) =>
+        get(`/files/${datasetName}`).catch(() => [])
       );
       const results = await Promise.all(allFilesPromises);
-      return results.flat().map((file: any, index: number) => ({
-        ...file,
-        dataset: file.dataset || datasets[index].split("/").pop()
+      return results.flat().map((file: any) => ({
+        ...file
       }));
     },
     enabled: !!datasets
@@ -73,15 +70,15 @@ export default function FileList() {
   useEffect(() => {
     const handleFileUpdate = (event: CustomEvent) => {
       const fileData = event.detail;
-      // Invalidate and refetch files when file updates occur
-      queryClient.invalidateQueries({ queryKey: ["all-files"] });
+      // Refetch files when file updates occur
+      queryClient.refetchQueries({ queryKey: ["all-files"] });
     };
 
     const handleTaskUpdate = (event: CustomEvent) => {
       const taskData = event.detail;
-      // Invalidate and refetch files when task updates occur (e.g., requeue processing)
+      // Refetch files when task updates occur (e.g., requeue processing)
       if (taskData.task === "handbrake") {
-        queryClient.invalidateQueries({ queryKey: ["all-files"] });
+        queryClient.refetchQueries({ queryKey: ["all-files"] });
       }
     };
 
@@ -139,7 +136,7 @@ export default function FileList() {
       }
     },
     onSuccess: (result, params) => {
-      queryClient.invalidateQueries({ queryKey: ["all-files"] });
+      queryClient.refetchQueries({ queryKey: ["all-files"] });
       setSelectedFiles(new Set()); // Clear selections after delete
 
       if (params.files && params.files.length > 0) {
@@ -173,8 +170,8 @@ export default function FileList() {
     mutationFn: ({ dataset, file }: { dataset: string; file: string }) =>
       post(`/files/${dataset}/${encodeURIComponent(file)}/requeue`),
     onSuccess: (_, { file }) => {
-      queryClient.invalidateQueries({ queryKey: ["all-files"] });
-      queryClient.invalidateQueries({ queryKey: ["tasks"] });
+      queryClient.refetchQueries({ queryKey: ["all-files"] });
+      queryClient.refetchQueries({ queryKey: ["tasks"] });
       toast.success("File requeued for processing");
       addNotification({
         type: "success",
@@ -415,7 +412,7 @@ export default function FileList() {
         </p>
         <button
           onClick={() =>
-            queryClient.invalidateQueries({ queryKey: ["all-files"] })
+            queryClient.refetchQueries({ queryKey: ["all-files"] })
           }
           className="inline-flex items-center px-6 py-3 border border-transparent text-base font-medium rounded-md text-white bg-red-600 hover:bg-red-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-red-500"
         >