Improve duplicate scanning: add loading indicator, cancel option, notifications, and detailed logging

apps/service/src/maintenance.service.ts

@@ -115,8 +115,11 @@ export class MaintenanceService {
   findDuplicateFiles(options: { resetExisting?: boolean } = {}) {
     const { resetExisting = false } = options;
 
+    this.logger.log('Starting duplicate file scan');
+
     if (resetExisting) {
       this.db.clearDuplicateGroups();
+      this.logger.log('Cleared existing duplicate groups');
     }
 
     const existing = this.db.listDuplicateGroups();
@@ -149,9 +152,11 @@ export class MaintenanceService {
         datasetObj.enabled === false ||
         datasetObj.enabled === 'false'
       ) {
+        this.logger.log(`Skipping disabled dataset: ${datasetName}`);
         continue;
       }
 
+      this.logger.log(`Scanning dataset: ${datasetName}`);
       const destinations = this.collectDestinations(datasetObj);
       for (const destination of destinations) {
         if (!destination || !fs.existsSync(destination)) {
@@ -161,7 +166,9 @@ export class MaintenanceService {
           continue;
         }
 
+        this.logger.log(`Scanning destination: ${destination}`);
         const groups = this.scanDestinationForDuplicates(destination);
+        this.scanForSimilarNames(destination);
         for (const group of groups) {
           const entry = {
             dataset: datasetName,
@@ -200,6 +207,10 @@ export class MaintenanceService {
       }
     }
 
+    this.logger.log(
+      `Duplicate scan completed. Processed ${duplicates.length} groups.`,
+    );
+
     return duplicates;
   }
 
@@ -222,7 +233,9 @@ export class MaintenanceService {
 
   private scanDestinationForDuplicates(destination: string) {
     const files = this.walkFiles(destination);
+    this.logger.log(`Found ${files.length} files to scan in ${destination}`);
     const groups = new Map<string, { size: number; files: string[] }>();
+    let processed = 0;
 
     for (const filePath of files) {
       try {
@@ -236,6 +249,12 @@ export class MaintenanceService {
           group.files.push(filePath);
           groups.set(key, group);
         }
+        processed++;
+        if (processed % 100 === 0) {
+          this.logger.log(
+            `Processed ${processed}/${files.length} files in ${destination}`,
+          );
+        }
       } catch (error) {
         this.logger.warn(
           `Failed to process file for duplicate scan: ${filePath} (${error})`,
@@ -243,6 +262,8 @@ export class MaintenanceService {
       }
     }
 
+    this.logger.log(`Completed scanning ${processed} files in ${destination}`);
+
     return Array.from(groups.entries())
       .filter(([, group]) => group.files.length > 1)
       .map(([key, group]) => ({
@@ -252,6 +273,60 @@ export class MaintenanceService {
       }));
   }
 
+  private scanForSimilarNames(destination: string) {
+    const files = this.walkFiles(destination);
+    this.logger.log(
+      `Checking ${files.length} files for similar names in ${destination}`,
+    );
+    const nameGroups = new Map<string, string[]>();
+    let processed = 0;
+
+    for (const filePath of files) {
+      try {
+        const stat = fs.statSync(filePath);
+        if (!stat.isFile()) continue;
+
+        const baseName = path
+          .basename(filePath, path.extname(filePath))
+          .toLowerCase();
+        const group = nameGroups.get(baseName) || [];
+        group.push(filePath);
+        nameGroups.set(baseName, group);
+        processed++;
+        if (processed % 100 === 0) {
+          this.logger.log(
+            `Processed ${processed}/${files.length} files for similar names in ${destination}`,
+          );
+        }
+      } catch (error) {
+        this.logger.warn(
+          `Failed to process file for similar name scan: ${filePath} (${error})`,
+        );
+      }
+    }
+
+    this.logger.log(
+      `Completed similar name check for ${processed} files in ${destination}`,
+    );
+
+    const similarGroups = Array.from(nameGroups.entries())
+      .filter(([, files]) => files.length > 1)
+      .map(([baseName, files]) => ({ baseName, files }));
+
+    if (similarGroups.length) {
+      this.logger.log(
+        `Found ${similarGroups.length} groups of files with similar names in ${destination}`,
+      );
+      for (const group of similarGroups) {
+        this.logger.log(
+          `Similar: ${group.baseName} - ${group.files.join(', ')}`,
+        );
+      }
+    }
+
+    return similarGroups;
+  }
+
   private walkFiles(root: string) {
     const pending = [root];
     const files: string[] = [];

apps/web/src/app/duplicates/DuplicateList.tsx

@@ -5,7 +5,7 @@ import {
   CheckCircleIcon,
   EyeSlashIcon,
   Squares2X2Icon,
-  TrashIcon
+  TrashIcon,
 } from "@heroicons/react/24/outline";
 import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { useEffect, useMemo, useState } from "react";
@@ -61,14 +61,14 @@ export default function DuplicateList() {
     data: duplicateGroups,
     isLoading,
     error,
-    refetch
+    refetch,
   } = useQuery<DuplicateGroup[]>({
     queryKey: ["duplicate-files"],
-    queryFn: async () => get("/maintenance/duplicates?status=pending")
+    queryFn: async () => get("/maintenance/duplicates?status=pending"),
   });
 
   const [enabledDatasets, setEnabledDatasets] = useState<Set<string>>(
-    new Set()
+    new Set(),
   );
   const [searchTerm, setSearchTerm] = useState("");
   const [sortField, setSortField] = useState<SortField>("count");
@@ -78,7 +78,7 @@ export default function DuplicateList() {
   const [selectedFiles, setSelectedFiles] = useState<Set<string>>(new Set());
   const [deleteSelection, setDeleteSelection] = useState<DeleteSelection>({
     isOpen: false,
-    groups: []
+    groups: [],
   });
 
   // Initialize enabled datasets on first load
@@ -91,22 +91,28 @@ export default function DuplicateList() {
     }
   }, [datasets]);
 
+  const [scanController, setScanController] = useState<AbortController | null>(
+    null,
+  );
+
   const scanMutation = useMutation({
-    mutationFn: () =>
-      post("/maintenance/duplicates/scan", { resetExisting: false }),
+    mutationFn: ({ signal }: { signal?: AbortSignal } = {}) =>
+      post("/maintenance/duplicates/scan", { resetExisting: false }, signal),
     onSuccess: () => {
       toast.success("Duplicate scan completed");
       addNotification({
         type: "info",
         title: "Scan Finished",
-        message: "Duplicate scan completed. Refreshing results."
+        message: "Duplicate scan completed. Refreshing results.",
       });
       refetch();
     },
     onError: (err: any) => {
+      if (err.message === "Request aborted") return;
       console.error(err);
       toast.error("Failed to start duplicate scan");
-    }
+    },
+    onSettled: () => setScanController(null),
   });
 
   const markNotDuplicateMutation = useMutation({
@@ -115,9 +121,9 @@ export default function DuplicateList() {
         ids.map((id) =>
           post(`/maintenance/duplicates/${id}/mark`, {
             status: "reviewed",
-            note: "not_duplicate"
-          })
-        )
+            note: "not_duplicate",
+          }),
+        ),
       );
     },
     onSuccess: () => {
@@ -128,7 +134,7 @@ export default function DuplicateList() {
     onError: (err: any) => {
       console.error(err);
       toast.error("Failed to update duplicates");
-    }
+    },
   });
 
   const deleteMutation = useMutation({
@@ -137,9 +143,9 @@ export default function DuplicateList() {
       await Promise.all(
         groups.map((group) =>
           post(`/maintenance/duplicates/${group.id}/purge`, {
-            files: group.files
-          })
-        )
+            files: group.files,
+          }),
+        ),
       );
     },
     onSuccess: () => {
@@ -152,7 +158,7 @@ export default function DuplicateList() {
     onError: (err: any) => {
       console.error(err);
       toast.error("Failed to delete selected files");
-    }
+    },
   });
 
   const filteredData = useMemo(() => {
@@ -250,7 +256,7 @@ export default function DuplicateList() {
 
     const groups = Array.from(grouped.entries()).map(([id, files]) => ({
       id,
-      files
+      files,
     }));
 
     setDeleteSelection({ isOpen: true, groups });
@@ -304,12 +310,35 @@ export default function DuplicateList() {
         </div>
         <div className="flex flex-wrap gap-2">
           <button
-            onClick={() => scanMutation.mutate()}
-            className="inline-flex items-center gap-2 rounded-md bg-blue-600 px-3 py-2 text-sm font-medium text-white shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2"
+            onClick={() => {
+              addNotification({
+                type: "info",
+                title: "Scan Started",
+                message: "Scanning for duplicates in /Volumes/Shares/.Private",
+              });
+              const controller = new AbortController();
+              setScanController(controller);
+              scanMutation.mutate({ signal: controller.signal });
+            }}
+            disabled={scanMutation.isPending}
+            className="inline-flex items-center gap-2 rounded-md bg-blue-600 px-3 py-2 text-sm font-medium text-white shadow-sm hover:bg-blue-700 focus:outline-none focus:ring-2 focus:ring-blue-500 focus:ring-offset-2 disabled:opacity-50 disabled:cursor-not-allowed"
           >
-            <ArrowPathIcon className="h-4 w-4" />
-            Rescan
+            <ArrowPathIcon
+              className={`h-4 w-4 ${scanMutation.isPending ? "animate-spin" : ""}`}
+            />
+            {scanMutation.isPending ? "Scanning..." : "Rescan"}
           </button>
+          {scanController && (
+            <button
+              onClick={() => {
+                scanController.abort();
+                setScanController(null);
+              }}
+              className="inline-flex items-center gap-2 rounded-md bg-gray-600 px-3 py-2 text-sm font-medium text-white shadow-sm hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-500 focus:ring-offset-2"
+            >
+              Cancel Scan
+            </button>
+          )}
           <button
             onClick={markSelectedNotDuplicate}
             disabled={selectedGroups.size === 0}
@@ -438,7 +467,7 @@ export default function DuplicateList() {
               {filteredData.map((group) => {
                 const isExpanded = expandedRows.has(group.id);
                 const allSelected = group.files.every((f) =>
-                  selectedFiles.has(makeFileKey(group.id, f))
+                  selectedFiles.has(makeFileKey(group.id, f)),
                 );
                 return (
                   <>

apps/web/src/lib/api.ts

@@ -22,7 +22,7 @@ function buildUrl(path: string, params?: any) {
     ? new URL(fullPath, base)
     : new URL(
         fullPath,
-        isBrowser ? window.location.origin : "http://localhost:3000"
+        isBrowser ? window.location.origin : "http://localhost:3000",
       );
   if (params && typeof params === "object") {
     Object.entries(params).forEach(([k, v]) => {
@@ -41,7 +41,7 @@ export async function get<T = any>(path: string, params?: any): Promise<T> {
       method: "GET",
       credentials: "same-origin",
       headers: { Accept: "application/json" },
-      signal: controller.signal
+      signal: controller.signal,
     });
     clearTimeout(timeoutId);
     if (!res.ok) throw new Error(await res.text());
@@ -55,20 +55,29 @@ export async function get<T = any>(path: string, params?: any): Promise<T> {
   }
 }
 
-export async function post<T = any>(path: string, data?: any): Promise<T> {
+export async function post<T = any>(
+  path: string,
+  data?: any,
+  signal?: AbortSignal,
+): Promise<T> {
   const controller = new AbortController();
   const timeoutId = setTimeout(() => controller.abort(), 90000); // 90 second timeout
 
+  // Combine provided signal with our timeout
+  const combinedSignal = signal
+    ? AbortSignal.any([controller.signal, signal])
+    : controller.signal;
+
   try {
     const res = await fetch(buildUrl(path), {
       method: "POST",
       credentials: "same-origin",
       headers: {
         "Content-Type": "application/json",
-        Accept: "application/json"
+        Accept: "application/json",
       },
       body: data ? JSON.stringify(data) : undefined,
-      signal: controller.signal
+      signal: combinedSignal,
     });
     clearTimeout(timeoutId);
     if (!res.ok) throw new Error(await res.text());
@@ -76,7 +85,7 @@ export async function post<T = any>(path: string, data?: any): Promise<T> {
   } catch (error) {
     clearTimeout(timeoutId);
     if (error instanceof Error && error.name === "AbortError") {
-      throw new Error("Request timeout");
+      throw new Error("Request aborted");
     }
     throw error;
   }
@@ -92,10 +101,10 @@ export async function put<T = any>(path: string, data?: any): Promise<T> {
       credentials: "same-origin",
       headers: {
         "Content-Type": "application/json",
-        Accept: "application/json"
+        Accept: "application/json",
       },
       body: data ? JSON.stringify(data) : undefined,
-      signal: controller.signal
+      signal: controller.signal,
     });
     clearTimeout(timeoutId);
     if (!res.ok) throw new Error(await res.text());
@@ -118,7 +127,7 @@ export async function del<T = any>(path: string, params?: any): Promise<T> {
       method: "DELETE",
       credentials: "same-origin",
       headers: { Accept: "application/json" },
-      signal: controller.signal
+      signal: controller.signal,
     });
     clearTimeout(timeoutId);
     if (!res.ok) throw new Error(await res.text());