Automation revamp part 1

frontend/src/components/Automation/CurrentProcessingCard.tsx

@@ -259,9 +259,25 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
   const stageOverview = pipelineOverview && pipelineOverview[currentStageIndex] ? pipelineOverview[currentStageIndex] : null;
   const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`];
 
+  // FIXED: Helper to get processed count using correct key
+  const getProcessedFromResult = (result: any, stageNumber: number): number => {
+    if (!result) return 0;
+    const keyMap: Record<number, string> = {
+      1: 'keywords_processed',
+      2: 'clusters_processed',
+      3: 'ideas_processed',
+      4: 'tasks_processed',
+      5: 'content_processed',
+      6: 'images_processed',
+      7: 'ready_for_review'
+    };
+    return result[keyMap[stageNumber]] ?? 0;
+  };
+
   const fallbackState: ProcessingState | null = ((): ProcessingState | null => {
     if (!processingState && (stageOverview || stageResult)) {
-      const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => typeof v === 'number' ? s + v : s, 0) : 0;
+      // FIXED: Use stage-specific key instead of summing all numeric values
+      const processed = getProcessedFromResult(stageResult, currentRun.current_stage);
       const total = (stageOverview?.pending || 0) + processed;
       const percentage = total > 0 ? Math.round((processed / total) * 100) : 0;
 
@@ -288,8 +304,8 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
   const computedProcessed = ((): number => {
     if (displayState && typeof displayState.processed_items === 'number') return displayState.processed_items;
     if (stageResult) {
-      // Sum numeric values in stageResult as a heuristic for processed count
-      return Object.values(stageResult).reduce((s: number, v: any) => (typeof v === 'number' ? s + v : s), 0);
+      // FIXED: Use stage-specific key for processed count
+      return getProcessedFromResult(stageResult, currentRun.current_stage);
     }
     return 0;
   })();

frontend/src/components/Automation/GlobalProgressBar.tsx

@@ -0,0 +1,232 @@
+/**
+ * Global Progress Bar Component
+ * Shows full pipeline progress across all 7 automation stages.
+ * Persists until 100% complete or run is finished.
+ */
+import React from 'react';
+import { AutomationRun, InitialSnapshot, StageProgress, GlobalProgress } from '../../services/automationService';
+import { BoltIcon, CheckCircleIcon, PauseIcon } from '../../icons';
+
+// Stage colors matching AutomationPage STAGE_CONFIG
+const STAGE_COLORS = [
+  'from-blue-500 to-blue-600',     // Stage 1: Keywords → Clusters
+  'from-purple-500 to-purple-600', // Stage 2: Clusters → Ideas
+  'from-indigo-500 to-indigo-600', // Stage 3: Ideas → Tasks
+  'from-green-500 to-green-600',   // Stage 4: Tasks → Content
+  'from-amber-500 to-amber-600',   // Stage 5: Content → Image Prompts
+  'from-pink-500 to-pink-600',     // Stage 6: Image Prompts → Images
+  'from-teal-500 to-teal-600',     // Stage 7: Manual Review Gate
+];
+
+const STAGE_NAMES = [
+  'Keywords',
+  'Clusters',
+  'Ideas',
+  'Tasks',
+  'Content',
+  'Prompts',
+  'Review',
+];
+
+// Helper to get processed count from stage result using correct key
+export const getProcessedFromResult = (result: any, stageNumber: number): number => {
+  if (!result) return 0;
+  
+  const keyMap: Record<number, string> = {
+    1: 'keywords_processed',
+    2: 'clusters_processed',
+    3: 'ideas_processed',
+    4: 'tasks_processed',
+    5: 'content_processed',
+    6: 'images_processed',
+    7: 'ready_for_review'
+  };
+  
+  return result[keyMap[stageNumber]] ?? 0;
+};
+
+interface GlobalProgressBarProps {
+  currentRun: AutomationRun | null;
+  globalProgress?: GlobalProgress | null;
+  stages?: StageProgress[];
+  initialSnapshot?: InitialSnapshot | null;
+}
+
+const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({
+  currentRun,
+  globalProgress,
+  stages,
+  initialSnapshot,
+}) => {
+  // Don't render if no run or run is completed with 100%
+  if (!currentRun) {
+    return null;
+  }
+  
+  // Calculate global progress if not provided from API
+  const calculateGlobalProgress = (): { percentage: number; completed: number; total: number } => {
+    // If we have API-provided global progress, use it
+    if (globalProgress) {
+      return {
+        percentage: globalProgress.percentage,
+        completed: globalProgress.completed_items,
+        total: globalProgress.total_items,
+      };
+    }
+    
+    // Fallback: Calculate from currentRun and initialSnapshot
+    const snapshot = initialSnapshot || (currentRun as any)?.initial_snapshot;
+    if (!snapshot) {
+      return { percentage: 0, completed: 0, total: 0 };
+    }
+    
+    const totalInitial = snapshot.total_initial_items || 0;
+    let totalCompleted = 0;
+    
+    for (let i = 1; i <= 7; i++) {
+      const result = (currentRun as any)[`stage_${i}_result`];
+      if (result) {
+        totalCompleted += getProcessedFromResult(result, i);
+      }
+    }
+    
+    const percentage = totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0;
+    
+    return {
+      percentage: Math.min(percentage, 100),
+      completed: totalCompleted,
+      total: totalInitial,
+    };
+  };
+  
+  const { percentage, completed, total } = calculateGlobalProgress();
+  
+  // Hide if completed and at 100%
+  if (currentRun.status === 'completed' && percentage >= 100) {
+    return null;
+  }
+  
+  const isPaused = currentRun.status === 'paused';
+  const currentStage = currentRun.current_stage;
+  
+  // Get stage status for segmented bar
+  const getStageStatus = (stageNum: number): 'completed' | 'active' | 'pending' => {
+    if (stages && stages[stageNum - 1]) {
+      return stages[stageNum - 1].status as 'completed' | 'active' | 'pending';
+    }
+    if (currentStage > stageNum) return 'completed';
+    if (currentStage === stageNum) return 'active';
+    return 'pending';
+  };
+  
+  const formatDuration = (): string => {
+    if (!currentRun.started_at) return '';
+    const start = new Date(currentRun.started_at).getTime();
+    const now = Date.now();
+    const diffMs = now - start;
+    const diffMins = Math.floor(diffMs / 60000);
+    const diffHours = Math.floor(diffMins / 60);
+    
+    if (diffHours > 0) {
+      return `${diffHours}h ${diffMins % 60}m`;
+    }
+    return `${diffMins}m`;
+  };
+  
+  return (
+    <div className={`
+      rounded-xl p-4 mb-6 border-2 transition-all
+      ${isPaused 
+        ? 'bg-gradient-to-r from-amber-50 to-amber-100 dark:from-amber-900/20 dark:to-amber-800/20 border-amber-300 dark:border-amber-700'
+        : 'bg-gradient-to-r from-brand-50 to-brand-100 dark:from-brand-900/20 dark:to-brand-800/20 border-brand-300 dark:border-brand-700'
+      }
+    `}>
+      {/* Header Row */}
+      <div className="flex justify-between items-center mb-3">
+        <div className="flex items-center gap-3">
+          <div className={`
+            size-10 rounded-lg flex items-center justify-center shadow-md
+            ${isPaused 
+              ? 'bg-gradient-to-br from-amber-500 to-amber-600'
+              : 'bg-gradient-to-br from-brand-500 to-brand-600'
+            }
+          `}>
+            {isPaused ? (
+              <PauseIcon className="w-5 h-5 text-white" />
+            ) : percentage >= 100 ? (
+              <CheckCircleIcon className="w-5 h-5 text-white" />
+            ) : (
+              <BoltIcon className="w-5 h-5 text-white animate-pulse" />
+            )}
+          </div>
+          <div>
+            <div className={`font-bold ${isPaused ? 'text-amber-800 dark:text-amber-200' : 'text-brand-800 dark:text-brand-200'}`}>
+              {isPaused ? 'Pipeline Paused' : 'Full Pipeline Progress'}
+            </div>
+            <div className="text-xs text-gray-600 dark:text-gray-400">
+              Stage {currentStage} of 7 • {formatDuration()}
+            </div>
+          </div>
+        </div>
+        <div className={`text-3xl font-bold ${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'}`}>
+          {percentage}%
+        </div>
+      </div>
+      
+      {/* Segmented Progress Bar */}
+      <div className="flex h-4 rounded-full overflow-hidden bg-gray-200 dark:bg-gray-700 gap-0.5 mb-2">
+        {[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
+          const status = getStageStatus(stageNum);
+          const stageColor = STAGE_COLORS[stageNum - 1];
+          
+          return (
+            <div
+              key={stageNum}
+              className={`flex-1 transition-all duration-500 relative group ${
+                status === 'completed' 
+                  ? `bg-gradient-to-r ${stageColor}` 
+                  : status === 'active'
+                  ? `bg-gradient-to-r ${stageColor} opacity-60 ${!isPaused ? 'animate-pulse' : ''}`
+                  : 'bg-gray-300 dark:bg-gray-600'
+              }`}
+              title={`Stage ${stageNum}: ${STAGE_NAMES[stageNum - 1]}`}
+            >
+              {/* Tooltip on hover */}
+              <div className="absolute bottom-full left-1/2 -translate-x-1/2 mb-2 px-2 py-1 bg-gray-900 text-white text-xs rounded opacity-0 group-hover:opacity-100 transition-opacity whitespace-nowrap pointer-events-none z-10">
+                S{stageNum}: {STAGE_NAMES[stageNum - 1]}
+                {status === 'completed' && ' ✓'}
+                {status === 'active' && ' ●'}
+              </div>
+            </div>
+          );
+        })}
+      </div>
+      
+      {/* Footer Row */}
+      <div className="flex justify-between text-xs text-gray-600 dark:text-gray-400">
+        <span>{completed} / {total} items processed</span>
+        <div className="flex gap-4">
+          {[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
+            const status = getStageStatus(stageNum);
+            return (
+              <span 
+                key={stageNum}
+                className={`
+                  ${status === 'completed' ? 'text-green-600 dark:text-green-400 font-medium' : ''}
+                  ${status === 'active' ? `${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'} font-bold` : ''}
+                  ${status === 'pending' ? 'text-gray-400 dark:text-gray-500' : ''}
+                `}
+              >
+                {stageNum}
+                {status === 'completed' && '✓'}
+                {status === 'active' && '●'}
+              </span>
+            );
+          })}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default GlobalProgressBar;

frontend/src/pages/Automation/AutomationPage.tsx

@@ -5,7 +5,7 @@
 import React, { useState, useEffect } from 'react';
 import { useToast } from '../../components/ui/toast/ToastContainer';
 import { useSiteStore } from '../../store/siteStore';
-import { automationService, AutomationRun, AutomationConfig, PipelineStage } from '../../services/automationService';
+import { automationService, AutomationRun, AutomationConfig, PipelineStage, RunProgressResponse, GlobalProgress, StageProgress, InitialSnapshot } from '../../services/automationService';
 import {
   fetchKeywords,
   fetchClusters,
@@ -18,6 +18,7 @@ import ActivityLog from '../../components/Automation/ActivityLog';
 import ConfigModal from '../../components/Automation/ConfigModal';
 import RunHistory from '../../components/Automation/RunHistory';
 import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard';
+import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar';
 import PageMeta from '../../components/common/PageMeta';
 import PageHeader from '../../components/common/PageHeader';
 import ComponentCard from '../../components/common/ComponentCard';
@@ -58,6 +59,11 @@ const AutomationPage: React.FC = () => {
   const [showProcessingCard, setShowProcessingCard] = useState<boolean>(true);
   const [loading, setLoading] = useState(true);
   const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null);
+  
+  // New state for unified progress data
+  const [globalProgress, setGlobalProgress] = useState<GlobalProgress | null>(null);
+  const [stageProgress, setStageProgress] = useState<StageProgress[]>([]);
+  const [initialSnapshot, setInitialSnapshot] = useState<InitialSnapshot | null>(null);
 
   useEffect(() => {
     if (!activeSite) return;
@@ -167,12 +173,34 @@ const AutomationPage: React.FC = () => {
       const data = await automationService.getCurrentRun(activeSite.id);
       setCurrentRun(data.run);
       // ensure processing card is visible when a run exists
-      if (data.run) setShowProcessingCard(true);
+      if (data.run) {
+        setShowProcessingCard(true);
+        // Also load unified progress data for GlobalProgressBar
+        await loadRunProgress(data.run.run_id);
+      }
     } catch (error) {
       console.error('Failed to poll current run', error);
     }
   };
 
+  const loadRunProgress = async (runId?: string) => {
+    if (!activeSite) return;
+    try {
+      const progressData = await automationService.getRunProgress(activeSite.id, runId);
+      if (progressData.global_progress) {
+        setGlobalProgress(progressData.global_progress);
+      }
+      if (progressData.stages) {
+        setStageProgress(progressData.stages);
+      }
+      if (progressData.initial_snapshot) {
+        setInitialSnapshot(progressData.initial_snapshot);
+      }
+    } catch (error) {
+      console.error('Failed to load run progress', error);
+    }
+  };
+
   const loadPipelineOverview = async () => {
     if (!activeSite) return;
     try {
@@ -667,6 +695,16 @@ const AutomationPage: React.FC = () => {
           </div>
         </div>
 
+        {/* Global Progress Bar - Shows full pipeline progress during automation run */}
+        {currentRun && (currentRun.status === 'running' || currentRun.status === 'paused') && (
+          <GlobalProgressBar
+            currentRun={currentRun}
+            globalProgress={globalProgress}
+            stages={stageProgress}
+            initialSnapshot={initialSnapshot}
+          />
+        )}
+
         {/* Current Processing Card - Shows real-time automation progress */}
         {currentRun && showProcessingCard && activeSite && (
           <CurrentProcessingCard
@@ -697,8 +735,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
-              const total = (stage.pending ?? 0) + processed;
+              // FIXED: Use stage-specific key for processed count instead of summing all numeric values
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available, otherwise fallback to pending + processed
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -738,8 +780,8 @@ const AutomationPage: React.FC = () => {
                   {/* Queue Metrics */}
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -748,7 +790,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor} dark:${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}
@@ -796,8 +838,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
-              const total = (stage.pending ?? 0) + processed;
+              // FIXED: Use stage-specific key for processed count
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -835,8 +881,8 @@ const AutomationPage: React.FC = () => {
                   
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -845,7 +891,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}

frontend/src/services/automationService.ts

@@ -78,6 +78,62 @@ export interface ProcessingState {
   remaining_count: number;
 }
 
+// NEW: Types for unified run_progress endpoint
+export interface StageProgress {
+  number: number;
+  name: string;
+  type: 'AI' | 'Local' | 'Manual';
+  status: 'pending' | 'active' | 'completed' | 'skipped';
+  input_count: number;
+  output_count: number;
+  processed_count: number;
+  progress_percentage: number;
+  credits_used: number;
+  time_elapsed: string;
+  currently_processing?: ProcessingItem[];
+  up_next?: ProcessingItem[];
+  remaining_count?: number;
+}
+
+export interface GlobalProgress {
+  total_items: number;
+  completed_items: number;
+  percentage: number;
+  current_stage: number;
+  total_stages: number;
+}
+
+export interface InitialSnapshot {
+  stage_1_initial: number;
+  stage_2_initial: number;
+  stage_3_initial: number;
+  stage_4_initial: number;
+  stage_5_initial: number;
+  stage_6_initial: number;
+  stage_7_initial: number;
+  total_initial_items: number;
+}
+
+export interface RunProgressResponse {
+  run: {
+    run_id: string;
+    status: 'running' | 'paused' | 'cancelled' | 'completed' | 'failed';
+    current_stage: number;
+    trigger_type: 'manual' | 'scheduled';
+    started_at: string;
+    completed_at: string | null;
+    paused_at: string | null;
+  } | null;
+  global_progress: GlobalProgress | null;
+  stages: StageProgress[];
+  metrics: {
+    credits_used: number;
+    duration_seconds: number;
+    errors: string[];
+  } | null;
+  initial_snapshot: InitialSnapshot | null;
+}
+
 function buildUrl(endpoint: string, params?: Record<string, any>): string {
   let url = `/v1/automation${endpoint}`;
   if (params) {
@@ -211,4 +267,19 @@ export const automationService = {
     );
     return response.data;
   },
+
+  /**
+   * Get unified run progress data - global + per-stage.
+   * This is the recommended endpoint for getting all automation progress data in a single call.
+   */
+  getRunProgress: async (
+    siteId: number,
+    runId?: string
+  ): Promise<RunProgressResponse> => {
+    const params: Record<string, any> = { site_id: siteId };
+    if (runId) {
+      params.run_id = runId;
+    }
+    return fetchAPI(buildUrl('/run_progress/', params));
+  },
 };