feat: add directory indexing with autocomplete, fix indexing worker database insertion

- Add /directories API endpoint to list subdirectories for autocomplete
- Create PathAutocomplete component with typeahead support for directory paths
- Update indexing page with improved destination path selection
  - Defaults to dataset configured destination
  - Allows editing to add deeper paths
  - Provides autocomplete suggestions as user types
- Fix indexing worker database insertion logic to properly handle destination files
  - Added INSERT OR IGNORE fallback for new destination files
  - Improved error handling for database operations
  - Better logging for batch processing progress

apps/service/src/app.controller.ts

@@ -8,6 +8,7 @@ import {
   Put,
   Query,
 } from '@nestjs/common';
+import * as fs from 'fs';
 import * as path from 'path';
 import { AppService } from './app.service';
 import { EventsGateway } from './events.gateway';
@@ -626,4 +627,35 @@ export class AppController {
   createTask(@Body() taskData: any) {
     return this.appService.createTask(taskData);
   }
+
+  // List directories at a given path (for autocomplete)
+  @Get('directories')
+  listDirectories(@Query('path') dirPath: string) {
+    try {
+      // Validate and normalize the path
+      if (!dirPath) {
+        return { directories: [] };
+      }
+
+      // Check if path exists and is a directory
+      if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) {
+        return { directories: [] };
+      }
+
+      // Read directory contents and filter for directories only
+      const entries = fs.readdirSync(dirPath, { withFileTypes: true });
+      const directories = entries
+        .filter((entry) => entry.isDirectory())
+        .map((entry) => ({
+          name: entry.name,
+          path: path.join(dirPath, entry.name),
+        }))
+        .sort((a, b) => a.name.localeCompare(b.name));
+
+      return { directories };
+    } catch (error) {
+      // Return empty array on error (permissions, etc.)
+      return { directories: [] };
+    }
+  }
 }

apps/service/src/indexing-worker.ts

@@ -106,6 +106,12 @@ async function indexDestination(
         date = excluded.date
     `);
 
+    // Also prepare an insert-only statement for files not yet in the database
+    const insertStmt = db.prepare(`
+      INSERT OR IGNORE INTO files (dataset, input, destination_path, hash, file_size, date)
+      VALUES (?, ?, ?, ?, ?, datetime('now'))
+    `);
+
     // Process files in batches
     for (let i = 0; i < files.length; i += batchSize) {
       const batch = files.slice(i, i + batchSize);
@@ -142,8 +148,18 @@ async function indexDestination(
               return;
             }
 
-            // Store in database
-            upsertStmt.run(dataset, filePath, filePath, hash, stat.size);
+            // Store in database - try upsert first for existing files, then insert-only for new files
+            try {
+              const result = upsertStmt.run(dataset, filePath, filePath, hash, stat.size);
+              if (result.changes === 0) {
+                // File wasn't in database, try insert
+                insertStmt.run(dataset, filePath, filePath, hash, stat.size);
+              }
+            } catch (dbError) {
+              console.error(`Worker: Database error for ${filePath}: ${dbError}`);
+              errors++;
+              return;
+            }
             indexed++;
 
             // Log every 50 files

apps/web/src/app/components/PathAutocomplete.tsx

@@ -0,0 +1,179 @@
+"use client";
+
+import { useQuery } from "@tanstack/react-query";
+import { useEffect, useRef, useState } from "react";
+import { get } from "../../lib/api";
+
+interface Directory {
+  name: string;
+  path: string;
+}
+
+interface PathAutocompleteProps {
+  value: string;
+  onChange: (value: string) => void;
+  placeholder?: string;
+  className?: string;
+  defaultPath?: string;
+}
+
+export default function PathAutocomplete({
+  value,
+  onChange,
+  placeholder = "/path/to/destination",
+  className = "",
+  defaultPath = "",
+}: PathAutocompleteProps) {
+  const [showSuggestions, setShowSuggestions] = useState(false);
+  const [selectedIndex, setSelectedIndex] = useState(-1);
+  const inputRef = useRef<HTMLInputElement>(null);
+  const suggestionsRef = useRef<HTMLDivElement>(null);
+
+  // Get the parent directory to search for subdirectories
+  const searchPath = value || defaultPath;
+  const parentPath = searchPath.endsWith("/")
+    ? searchPath.slice(0, -1)
+    : searchPath;
+
+  // Fetch directories for autocomplete
+  const { data: dirData } = useQuery<{ directories: Directory[] }>({
+    queryKey: ["directories", parentPath],
+    queryFn: async () => {
+      if (!parentPath) return { directories: [] };
+      return get("/directories", { path: parentPath });
+    },
+    enabled: !!parentPath && showSuggestions,
+    staleTime: 30000, // Cache for 30 seconds
+  });
+
+  const directories = dirData?.directories || [];
+
+  // Close suggestions when clicking outside
+  useEffect(() => {
+    const handleClickOutside = (event: MouseEvent) => {
+      if (
+        suggestionsRef.current &&
+        !suggestionsRef.current.contains(event.target as Node) &&
+        !inputRef.current?.contains(event.target as Node)
+      ) {
+        setShowSuggestions(false);
+      }
+    };
+
+    document.addEventListener("mousedown", handleClickOutside);
+    return () => document.removeEventListener("mousedown", handleClickOutside);
+  }, []);
+
+  const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const newValue = e.target.value;
+    onChange(newValue);
+    setShowSuggestions(true);
+    setSelectedIndex(-1);
+  };
+
+  const handleSuggestionClick = (dirPath: string) => {
+    onChange(dirPath);
+    setShowSuggestions(false);
+    setSelectedIndex(-1);
+  };
+
+  const handleKeyDown = (e: React.KeyboardEvent<HTMLInputElement>) => {
+    if (!showSuggestions || directories.length === 0) {
+      if (e.key === "ArrowDown") {
+        e.preventDefault();
+        setShowSuggestions(true);
+      }
+      return;
+    }
+
+    switch (e.key) {
+      case "ArrowDown":
+        e.preventDefault();
+        setSelectedIndex((prev) =>
+          prev < directories.length - 1 ? prev + 1 : prev
+        );
+        break;
+      case "ArrowUp":
+        e.preventDefault();
+        setSelectedIndex((prev) => (prev > 0 ? prev - 1 : -1));
+        break;
+      case "Enter":
+        e.preventDefault();
+        if (selectedIndex >= 0 && selectedIndex < directories.length) {
+          handleSuggestionClick(directories[selectedIndex].path);
+        }
+        break;
+      case "Escape":
+        setShowSuggestions(false);
+        setSelectedIndex(-1);
+        break;
+    }
+  };
+
+  const handleFocus = () => {
+    if (directories.length > 0) {
+      setShowSuggestions(true);
+    }
+  };
+
+  return (
+    <div className="relative">
+      <input
+        ref={inputRef}
+        type="text"
+        value={value}
+        onChange={handleInputChange}
+        onKeyDown={handleKeyDown}
+        onFocus={handleFocus}
+        placeholder={placeholder}
+        className={className}
+      />
+
+      {showSuggestions && directories.length > 0 && (
+        <div
+          ref={suggestionsRef}
+          className="absolute z-10 w-full mt-1 bg-white dark:bg-gray-800 border border-gray-300 dark:border-gray-600 rounded-md shadow-lg max-h-60 overflow-y-auto"
+        >
+          {directories.map((dir, index) => (
+            <button
+              key={dir.path}
+              type="button"
+              onClick={() => handleSuggestionClick(dir.path)}
+              className={`w-full text-left px-3 py-2 text-sm hover:bg-gray-100 dark:hover:bg-gray-700 focus:outline-none focus:bg-gray-100 dark:focus:bg-gray-700 ${
+                index === selectedIndex ? "bg-blue-50 dark:bg-blue-900/20" : ""
+              }`}
+            >
+              <div className="flex items-center">
+                <svg
+                  className="h-4 w-4 mr-2 text-gray-400"
+                  fill="none"
+                  viewBox="0 0 24 24"
+                  stroke="currentColor"
+                >
+                  <path
+                    strokeLinecap="round"
+                    strokeLinejoin="round"
+                    strokeWidth={2}
+                    d="M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z"
+                  />
+                </svg>
+                <span className="text-gray-900 dark:text-gray-100">
+                  {dir.name}
+                </span>
+              </div>
+              <div className="text-xs text-gray-500 dark:text-gray-400 ml-6 truncate">
+                {dir.path}
+              </div>
+            </button>
+          ))}
+        </div>
+      )}
+
+      {defaultPath && value !== defaultPath && (
+        <p className="mt-1 text-xs text-gray-500 dark:text-gray-400">
+          Default: {defaultPath}
+        </p>
+      )}
+    </div>
+  );
+}

apps/web/src/app/indexing/page.tsx

@@ -11,6 +11,7 @@ import { useEffect, useState } from "react";
 import toast from "react-hot-toast";
 import { del, get, post } from "../../lib/api";
 import LoadingCard from "../components/Loading";
+import PathAutocomplete from "../components/PathAutocomplete";
 import { useAppContext } from "../providers/AppContext";
 
 interface IndexStats {
@@ -33,6 +34,7 @@ export default function IndexManagementPage() {
   const { datasets, datasetsConfig } = useAppContext();
   const [selectedDataset, setSelectedDataset] = useState<string>("");
   const [destinationPath, setDestinationPath] = useState<string>("");
+  const [defaultDestination, setDefaultDestination] = useState<string>("");
   const [batchSize, setBatchSize] = useState<number>(100);
 
   const datasetNames = datasets
@@ -65,10 +67,14 @@ export default function IndexManagementPage() {
     };
 
     const destination = tryFindDestination(cfg);
-    if (destination && destination !== destinationPath) {
-      setDestinationPath(destination);
+    if (destination) {
+      setDefaultDestination(destination);
+      // Only set destinationPath if it's empty or different from the new default
+      if (!destinationPath || destinationPath === defaultDestination) {
+        setDestinationPath(destination);
+      }
     }
-  }, [selectedDataset, datasetsConfig, destinationPath]);
+  }, [selectedDataset, datasetsConfig]);
 
   // Get index count for selected dataset
   const {
@@ -217,11 +223,11 @@ export default function IndexManagementPage() {
               <label className="block text-sm font-medium text-gray-700 dark:text-gray-300 mb-2">
                 Destination Path
               </label>
-              <input
-                type="text"
+              <PathAutocomplete
                 value={destinationPath}
-                onChange={(e) => setDestinationPath(e.target.value)}
+                onChange={setDestinationPath}
                 placeholder="/path/to/destination"
+                defaultPath={defaultDestination}
                 className="w-full px-3 py-2 border border-gray-300 dark:border-gray-600 rounded-md shadow-sm focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:text-gray-100"
               />
             </div>