Add restarting state to prevent network errors during API restart

- Cancel all in-flight queries when restart is initiated
- Pause health checks during restart period
- Implement smart reconnection with exponential backoff (up to 10 attempts)
- Show 'Restarting...' status during reconnection phase
- Disable restart button while restart is in progress
- No longer throws NetworkError in console during expected downtime
- Automatically invalidates and refreshes all queries once reconnected

apps/web/src/app/components/ApiHealth.tsx

@@ -8,6 +8,7 @@ import { get, post } from "../../lib/api";
 export default function ApiHealth() {
   const [responseTime, setResponseTime] = useState<number | null>(null);
   const [showRestartConfirm, setShowRestartConfirm] = useState(false);
+  const [isRestarting, setIsRestarting] = useState(false);
   const queryClient = useQueryClient();
 
   const { data, isLoading, error } = useQuery({
@@ -22,45 +23,57 @@ export default function ApiHealth() {
     refetchInterval: 30000, // Check every 30 seconds
     retry: 3, // Retry failed requests 3 times
     retryDelay: (attemptIndex) => Math.min(1000 * 2 ** attemptIndex, 30000), // Exponential backoff
+    enabled: !isRestarting, // Pause health checks during restart
   });
 
   const restartMutation = useMutation({
     mutationFn: () => post("/restart", {}),
+    onMutate: () => {
+      // Set restarting state and pause all queries
+      setIsRestarting(true);
+      queryClient.cancelQueries(); // Cancel all in-flight queries
+    },
     onSuccess: () => {
       toast.success("API service restarting...");
       setShowRestartConfirm(false);
-      // Invalidate all queries to prevent stale data
-      queryClient.invalidateQueries();
-      // Retry health check after delays to see if service is back
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 3000);
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 5000);
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 8000);
+      // Wait for server to come back, then resume
+      attemptReconnect();
     },
     onError: (error: any) => {
       // Expected to fail since server is restarting
       toast.success("API service restart initiated...");
       setShowRestartConfirm(false);
-      // Invalidate all queries to prevent stale data
-      queryClient.invalidateQueries();
       // Still try to reconnect
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 3000);
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 5000);
-      setTimeout(() => {
-        queryClient.refetchQueries({ queryKey: ["api", "health"] });
-      }, 8000);
+      attemptReconnect();
     },
   });
 
+  const attemptReconnect = () => {
+    let attempts = 0;
+    const maxAttempts = 10;
+    const checkHealth = async () => {
+      attempts++;
+      try {
+        await get("/health");
+        // Success! Server is back
+        setIsRestarting(false);
+        toast.success("API service reconnected");
+        queryClient.invalidateQueries(); // Refresh all data
+      } catch (error) {
+        if (attempts < maxAttempts) {
+          // Try again with exponential backoff
+          setTimeout(checkHealth, Math.min(1000 * Math.pow(1.5, attempts), 10000));
+        } else {
+          setIsRestarting(false);
+          toast.error("Failed to reconnect to API service");
+          queryClient.invalidateQueries();
+        }
+      }
+    };
+    // Start checking after 2 seconds
+    setTimeout(checkHealth, 2000);
+  };
+
   const isHealthy = data?.status === "healthy";
 
   return (
@@ -72,7 +85,7 @@ export default function ApiHealth() {
           </div>
           <div>
             <div className="text-2xl font-bold text-white">
-              {isLoading ? (
+              {isLoading || isRestarting ? (
                 <div className="flex justify-center">
                   <div className="animate-spin rounded-full h-5 w-5 border-b-2 border-white"></div>
                 </div>
@@ -86,8 +99,10 @@ export default function ApiHealth() {
                 </span>
               )}
             </div>
-            <div className="text-sm font-medium text-gray-400">API Health</div>
-            {responseTime !== null && (
+            <div className="text-sm font-medium text-gray-400">
+              {isRestarting ? "Restarting..." : "API Health"}
+            </div>
+            {responseTime !== null && !isRestarting && (
               <div className="text-xs text-gray-500 mt-1">
                 {responseTime}ms response
               </div>
@@ -96,12 +111,12 @@ export default function ApiHealth() {
         </div>
         <button
           onClick={() => setShowRestartConfirm(true)}
-          disabled={restartMutation.isPending}
+          disabled={restartMutation.isPending || isRestarting}
           className="px-3 py-1 rounded text-xs font-medium transition-colors bg-blue-600 hover:bg-blue-700 text-white disabled:opacity-50 disabled:cursor-not-allowed flex items-center gap-1"
           title="Restart API service"
         >
           <ArrowPathIcon
-            className={`h-3 w-3 ${restartMutation.isPending ? "animate-spin" : ""}`}
+            className={`h-3 w-3 ${restartMutation.isPending || isRestarting ? "animate-spin" : ""}`}
           />
           Restart
         </button>