Automation revamp part 1

frontend/src/pages/Automation/AutomationPage.tsx

@@ -5,7 +5,7 @@
 import React, { useState, useEffect } from 'react';
 import { useToast } from '../../components/ui/toast/ToastContainer';
 import { useSiteStore } from '../../store/siteStore';
-import { automationService, AutomationRun, AutomationConfig, PipelineStage } from '../../services/automationService';
+import { automationService, AutomationRun, AutomationConfig, PipelineStage, RunProgressResponse, GlobalProgress, StageProgress, InitialSnapshot } from '../../services/automationService';
 import {
   fetchKeywords,
   fetchClusters,
@@ -18,6 +18,7 @@ import ActivityLog from '../../components/Automation/ActivityLog';
 import ConfigModal from '../../components/Automation/ConfigModal';
 import RunHistory from '../../components/Automation/RunHistory';
 import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard';
+import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar';
 import PageMeta from '../../components/common/PageMeta';
 import PageHeader from '../../components/common/PageHeader';
 import ComponentCard from '../../components/common/ComponentCard';
@@ -58,6 +59,11 @@ const AutomationPage: React.FC = () => {
   const [showProcessingCard, setShowProcessingCard] = useState<boolean>(true);
   const [loading, setLoading] = useState(true);
   const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null);
+  
+  // New state for unified progress data
+  const [globalProgress, setGlobalProgress] = useState<GlobalProgress | null>(null);
+  const [stageProgress, setStageProgress] = useState<StageProgress[]>([]);
+  const [initialSnapshot, setInitialSnapshot] = useState<InitialSnapshot | null>(null);
 
   useEffect(() => {
     if (!activeSite) return;
@@ -167,12 +173,34 @@ const AutomationPage: React.FC = () => {
       const data = await automationService.getCurrentRun(activeSite.id);
       setCurrentRun(data.run);
       // ensure processing card is visible when a run exists
-      if (data.run) setShowProcessingCard(true);
+      if (data.run) {
+        setShowProcessingCard(true);
+        // Also load unified progress data for GlobalProgressBar
+        await loadRunProgress(data.run.run_id);
+      }
     } catch (error) {
       console.error('Failed to poll current run', error);
     }
   };
 
+  const loadRunProgress = async (runId?: string) => {
+    if (!activeSite) return;
+    try {
+      const progressData = await automationService.getRunProgress(activeSite.id, runId);
+      if (progressData.global_progress) {
+        setGlobalProgress(progressData.global_progress);
+      }
+      if (progressData.stages) {
+        setStageProgress(progressData.stages);
+      }
+      if (progressData.initial_snapshot) {
+        setInitialSnapshot(progressData.initial_snapshot);
+      }
+    } catch (error) {
+      console.error('Failed to load run progress', error);
+    }
+  };
+
   const loadPipelineOverview = async () => {
     if (!activeSite) return;
     try {
@@ -667,6 +695,16 @@ const AutomationPage: React.FC = () => {
           </div>
         </div>
 
+        {/* Global Progress Bar - Shows full pipeline progress during automation run */}
+        {currentRun && (currentRun.status === 'running' || currentRun.status === 'paused') && (
+          <GlobalProgressBar
+            currentRun={currentRun}
+            globalProgress={globalProgress}
+            stages={stageProgress}
+            initialSnapshot={initialSnapshot}
+          />
+        )}
+
         {/* Current Processing Card - Shows real-time automation progress */}
         {currentRun && showProcessingCard && activeSite && (
           <CurrentProcessingCard
@@ -697,8 +735,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
-              const total = (stage.pending ?? 0) + processed;
+              // FIXED: Use stage-specific key for processed count instead of summing all numeric values
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available, otherwise fallback to pending + processed
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -738,8 +780,8 @@ const AutomationPage: React.FC = () => {
                   {/* Queue Metrics */}
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -748,7 +790,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor} dark:${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}
@@ -796,8 +838,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
-              const total = (stage.pending ?? 0) + processed;
+              // FIXED: Use stage-specific key for processed count
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -835,8 +881,8 @@ const AutomationPage: React.FC = () => {
                   
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -845,7 +891,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}