feat: implement database migration system

- Add MigrationRunner class for managing database schema changes
- Create initial migration with current schema
- Add migration CLI scripts
- Update DbService to use migrations instead of ad-hoc schema changes
- Update documentation with migration best practices
- Database now includes migrations tracking table

apps/service/src/migration-runner.ts

@@ -0,0 +1,116 @@
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+
+export class MigrationRunner {
+  private db: Database.Database;
+  private migrationsDir: string;
+
+  constructor(db: Database.Database, migrationsDir: string) {
+    this.db = db;
+    this.migrationsDir = migrationsDir;
+  }
+
+  /**
+   * Initialize the migrations table if it doesn't exist
+   */
+  init() {
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS migrations (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        name TEXT NOT NULL UNIQUE,
+        applied_at TEXT DEFAULT CURRENT_TIMESTAMP
+      );
+    `);
+  }
+
+  /**
+   * Get list of applied migrations
+   */
+  getAppliedMigrations(): string[] {
+    const rows = this.db.prepare('SELECT name FROM migrations ORDER BY id').all() as { name: string }[];
+    return rows.map(row => row.name);
+  }
+
+  /**
+   * Get list of available migration files
+   */
+  getAvailableMigrations(): string[] {
+    if (!fs.existsSync(this.migrationsDir)) {
+      return [];
+    }
+
+    return fs.readdirSync(this.migrationsDir)
+      .filter(file => file.endsWith('.sql'))
+      .sort();
+  }
+
+  /**
+   * Apply a single migration
+   */
+  applyMigration(migrationName: string): void {
+    const migrationPath = path.join(this.migrationsDir, migrationName);
+
+    if (!fs.existsSync(migrationPath)) {
+      throw new Error(`Migration file not found: ${migrationPath}`);
+    }
+
+    const sql = fs.readFileSync(migrationPath, 'utf8');
+
+    console.log(`Applying migration: ${migrationName}`);
+
+    // Execute the migration in a transaction
+    const transaction = this.db.transaction(() => {
+      this.db.exec(sql);
+      this.db.prepare('INSERT INTO migrations (name) VALUES (?)').run(migrationName);
+    });
+
+    transaction();
+    console.log(`Migration applied: ${migrationName}`);
+  }
+
+  /**
+   * Apply all pending migrations
+   */
+  applyPendingMigrations(): void {
+    this.init();
+
+    const applied = this.getAppliedMigrations();
+    const available = this.getAvailableMigrations();
+
+    const pending = available.filter(migration => !applied.includes(migration));
+
+    if (pending.length === 0) {
+      console.log('No pending migrations');
+      return;
+    }
+
+    console.log(`Applying ${pending.length} pending migrations...`);
+
+    for (const migration of pending) {
+      this.applyMigration(migration);
+    }
+
+    console.log('All migrations applied');
+  }
+
+  /**
+   * Create a new migration file
+   */
+  createMigration(name: string): string {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, -5);
+    const filename = `${timestamp}_${name}.sql`;
+    const filepath = path.join(this.migrationsDir, filename);
+
+    const template = `-- Migration: ${name}
+-- Created at: ${new Date().toISOString()}
+
+-- Add your SQL migration here
+
+`;
+
+    fs.writeFileSync(filepath, template);
+    console.log(`Created migration: ${filepath}`);
+    return filepath;
+  }
+}

data/README.md

@@ -117,6 +117,51 @@ Each dataset configuration includes:
 - The system creates `database.db.bak` during migrations
 - To restore from backup: `cp database.db.bak database.db`
 
-## Migration
+## Migration System
 
-The system includes migration tools to convert from the legacy JSON-based storage to SQLite. See the service README for migration commands.
+The application uses a proper database migration system to manage schema changes. This ensures that database changes can be versioned and applied consistently across different environments.
+
+### Migration Files
+
+Migration files are stored in `data/migrations/` and are named with timestamps: `YYYY-MM-DDTHH-MM-SS_migration_name.sql`.
+
+### Running Migrations
+
+Migrations are automatically applied when the service starts. You can also run them manually:
+
+```bash
+# Check migration status
+pnpm run migrate:status
+
+# Apply pending migrations
+pnpm run migrate:up
+
+# Create a new migration
+pnpm run migrate:create <migration_name>
+```
+
+### Creating Migrations
+
+When you need to make schema changes:
+
+1. Create a new migration file:
+   ```bash
+   pnpm run migrate:create add_new_table
+   ```
+
+2. Edit the generated SQL file in `data/migrations/` with your schema changes.
+
+3. Test the migration by running it:
+   ```bash
+   pnpm run migrate:up
+   ```
+
+4. Commit both the migration file and any code changes that depend on the new schema.
+
+### Migration Best Practices
+
+- Never modify existing migration files after they've been committed.
+- If you need to change a migration, create a new one that undoes and redoes the changes.
+- Test migrations on a copy of production data before applying to production.
+- Keep migrations small and focused on a single change.
+- Use descriptive names for migration files.

data/migrations/2026-01-06T00-12-00_initial_schema.sql

@@ -0,0 +1,56 @@
+-- Migration: initial_schema
+-- Created at: 2026-01-06T00:12:00.129Z
+
+-- Initial database schema for Watch Finished Turbo
+
+-- Files table for tracking processed video files
+CREATE TABLE IF NOT EXISTS files (
+  dataset TEXT,
+  input TEXT,
+  output TEXT,
+  date TEXT,
+  status TEXT DEFAULT 'pending',
+  PRIMARY KEY (dataset, input)
+);
+
+-- Tasks table for video processing queue
+CREATE TABLE IF NOT EXISTS tasks (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  type TEXT NOT NULL,
+  status TEXT DEFAULT 'pending',
+  progress INTEGER DEFAULT 0,
+  dataset TEXT,
+  input TEXT,
+  output TEXT,
+  preset TEXT,
+  priority INTEGER DEFAULT 0,
+  retry_count INTEGER DEFAULT 0,
+  max_retries INTEGER,
+  error_message TEXT,
+  created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+  updated_at TEXT DEFAULT CURRENT_TIMESTAMP
+);
+
+-- Duplicate files table for duplicate detection
+CREATE TABLE IF NOT EXISTS duplicate_files (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  dataset TEXT,
+  destination TEXT,
+  hash TEXT,
+  size INTEGER,
+  files TEXT,
+  status TEXT DEFAULT 'pending',
+  note TEXT,
+  created_at TEXT DEFAULT CURRENT_TIMESTAMP,
+  reviewed_at TEXT
+);
+
+-- Settings table for application configuration
+CREATE TABLE IF NOT EXISTS settings (
+  key TEXT PRIMARY KEY,
+  value TEXT
+);
+
+-- Indexes for performance
+CREATE UNIQUE INDEX IF NOT EXISTS idx_duplicate_files_key
+  ON duplicate_files(dataset, destination, hash, size);

package.json

@@ -14,7 +14,11 @@
     "web": "pnpm --filter ./apps/web run dev --port 3000",
     "web:prod": "pnpm --filter ./apps/web run start",
     "test:e2e": "pnpm --filter ./apps/web run test:e2e",
-    "test:e2e:full": "concurrently \"pnpm run service\" \"pnpm run web\" --names \"service,web\" --prefix name --success first"
+    "test:e2e:full": "concurrently \"pnpm run service\" \"pnpm run web\" --names \"service,web\" --prefix name --success first",
+    "migrate": "tsx scripts/migrate.ts",
+    "migrate:up": "tsx scripts/migrate.ts up",
+    "migrate:create": "tsx scripts/migrate.ts create",
+    "migrate:status": "tsx scripts/migrate.ts status"
   },
   "devDependencies": {
     "@tailwindcss/postcss": "^4.1.18",

scripts/migrate.ts

@@ -0,0 +1,90 @@
+#!/usr/bin/env tsx
+
+import Database from 'better-sqlite3';
+import fs from 'fs';
+import path from 'path';
+import { MigrationRunner } from '../apps/service/src/migration-runner';
+
+const args = process.argv.slice(2);
+const command = args[0];
+
+if (!command) {
+  console.log('Usage: migrate <command>');
+  console.log('Commands:');
+  console.log('  up          - Apply all pending migrations');
+  console.log('  create <name> - Create a new migration file');
+  console.log('  status      - Show migration status');
+  process.exit(1);
+}
+
+// Find project root
+let projectRoot = process.cwd();
+while (projectRoot !== path.dirname(projectRoot)) {
+  if (fs.existsSync(path.join(projectRoot, 'package.json'))) {
+    try {
+      const pkg = JSON.parse(
+        fs.readFileSync(path.join(projectRoot, 'package.json'), 'utf-8'),
+      );
+      if (pkg.name === 'watch-finished-turbo') {
+        break;
+      }
+    } catch (e) {
+      // ignore
+    }
+  }
+  projectRoot = path.dirname(projectRoot);
+}
+
+const dbPath = path.resolve(projectRoot, 'data/database.db');
+const migrationsDir = path.resolve(projectRoot, 'data/migrations');
+
+// Ensure directories exist
+if (!fs.existsSync(path.dirname(dbPath))) {
+  fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+}
+if (!fs.existsSync(migrationsDir)) {
+  fs.mkdirSync(migrationsDir, { recursive: true });
+}
+
+const db = new Database(dbPath);
+const runner = new MigrationRunner(db, migrationsDir);
+
+try {
+  switch (command) {
+    case 'up':
+      runner.applyPendingMigrations();
+      break;
+
+    case 'create':
+      const name = args[1];
+      if (!name) {
+        console.error('Migration name is required');
+        process.exit(1);
+      }
+      runner.createMigration(name);
+      break;
+
+    case 'status':
+      runner.init();
+      const applied = runner.getAppliedMigrations();
+      const available = runner.getAvailableMigrations();
+      const pending = available.filter(m => !applied.includes(m));
+
+      console.log('Migration Status:');
+      console.log(`Applied: ${applied.length}`);
+      console.log(`Available: ${available.length}`);
+      console.log(`Pending: ${pending.length}`);
+
+      if (pending.length > 0) {
+        console.log('\nPending migrations:');
+        pending.forEach(m => console.log(`  - ${m}`));
+      }
+      break;
+
+    default:
+      console.error(`Unknown command: ${command}`);
+      process.exit(1);
+  }
+} finally {
+  db.close();
+}