Automation revamp part 1

backend/igny8_core/business/automation/migrations/0006_automationrun_initial_snapshot.py

@@ -0,0 +1,22 @@
+# Generated migration for adding initial_snapshot field to AutomationRun
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('automation', '0005_add_default_image_service'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='automationrun',
+            name='initial_snapshot',
+            field=models.JSONField(
+                blank=True,
+                default=dict,
+                help_text='Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}'
+            ),
+        ),
+    ]

backend/igny8_core/business/automation/models.py

@@ -88,6 +88,13 @@ class AutomationRun(models.Model):
     
     total_credits_used = models.IntegerField(default=0)
     
+    # Initial queue snapshot - captured at run start for accurate progress tracking
+    initial_snapshot = models.JSONField(
+        default=dict, 
+        blank=True,
+        help_text="Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}"
+    )
+    
     # JSON results per stage
     stage_1_result = models.JSONField(null=True, blank=True, help_text="{keywords_processed, clusters_created, batches}")
     stage_2_result = models.JSONField(null=True, blank=True, help_text="{clusters_processed, ideas_created}")

backend/igny8_core/business/automation/services/automation_service.py

@@ -109,6 +109,9 @@ class AutomationService:
             # Create run_id and log files
             run_id = self.logger.start_run(self.account.id, self.site.id, trigger_type)
             
+            # Capture initial queue snapshot for accurate progress tracking
+            initial_snapshot = self._capture_initial_snapshot()
+            
             # Create AutomationRun record
             self.run = AutomationRun.objects.create(
                 run_id=run_id,
@@ -117,6 +120,7 @@ class AutomationService:
                 trigger_type=trigger_type,
                 status='running',
                 current_stage=1,
+                initial_snapshot=initial_snapshot,
             )
             
             # Log start
@@ -124,6 +128,10 @@ class AutomationService:
                 run_id, self.account.id, self.site.id, 0,
                 f"Automation started (trigger: {trigger_type})"
             )
+            self.logger.log_stage_progress(
+                run_id, self.account.id, self.site.id, 0,
+                f"Initial snapshot captured: {initial_snapshot['total_initial_items']} total items across all stages"
+            )
             self.logger.log_stage_progress(
                 run_id, self.account.id, self.site.id, 0,
                 f"Credit check: Account has {self.account.credits} credits, estimated need: {estimated_credits} credits"
@@ -1361,6 +1369,61 @@ class AutomationService:
         logger.info(f"[AutomationService] Estimated credits: {total}")
         return total
     
+    def _capture_initial_snapshot(self) -> dict:
+        """
+        Capture initial queue sizes at run start for accurate progress tracking.
+        This snapshot is used to calculate global progress percentage correctly.
+        """
+        # Stage 1: Keywords pending clustering
+        stage_1_initial = Keywords.objects.filter(
+            site=self.site, status='new', cluster__isnull=True, disabled=False
+        ).count()
+        
+        # Stage 2: Clusters needing ideas
+        stage_2_initial = Clusters.objects.filter(
+            site=self.site, status='new', disabled=False
+        ).exclude(ideas__isnull=False).count()
+        
+        # Stage 3: Ideas ready to be converted to tasks
+        stage_3_initial = ContentIdeas.objects.filter(
+            site=self.site, status='new'
+        ).count()
+        
+        # Stage 4: Tasks ready for content generation
+        stage_4_initial = Tasks.objects.filter(
+            site=self.site, status='queued'
+        ).count()
+        
+        # Stage 5: Content needing image prompts
+        stage_5_initial = Content.objects.filter(
+            site=self.site, status='draft'
+        ).annotate(images_count=Count('images')).filter(images_count=0).count()
+        
+        # Stage 6: Image prompts pending generation
+        stage_6_initial = Images.objects.filter(
+            site=self.site, status='pending'
+        ).count()
+        
+        # Stage 7: Content ready for review
+        stage_7_initial = Content.objects.filter(
+            site=self.site, status='review'
+        ).count()
+        
+        snapshot = {
+            'stage_1_initial': stage_1_initial,
+            'stage_2_initial': stage_2_initial,
+            'stage_3_initial': stage_3_initial,
+            'stage_4_initial': stage_4_initial,
+            'stage_5_initial': stage_5_initial,
+            'stage_6_initial': stage_6_initial,
+            'stage_7_initial': stage_7_initial,
+            'total_initial_items': stage_1_initial + stage_2_initial + stage_3_initial + 
+                                   stage_4_initial + stage_5_initial + stage_6_initial + stage_7_initial,
+        }
+        
+        logger.info(f"[AutomationService] Initial snapshot captured: {snapshot}")
+        return snapshot
+    
     # Helper methods
     
     def _wait_for_task(self, task_id: str, stage_number: int, item_name: str, continue_on_error: bool = True):
@@ -1559,7 +1622,7 @@ class AutomationService:
     def _get_stage_3_state(self) -> dict:
         """Get processing state for Stage 3: Ideas → Tasks"""
         queue = ContentIdeas.objects.filter(
-            site=self.site, status='approved'
+            site=self.site, status='new'  # Fixed: Match pipeline_overview status
         ).order_by('id')
         
         processed = self._get_processed_count(3)
@@ -1580,7 +1643,7 @@ class AutomationService:
     def _get_stage_4_state(self) -> dict:
         """Get processing state for Stage 4: Tasks → Content"""
         queue = Tasks.objects.filter(
-            site=self.site, status='ready'
+            site=self.site, status='queued'  # Fixed: Match pipeline_overview status
         ).order_by('id')
         
         processed = self._get_processed_count(4)
@@ -1666,52 +1729,31 @@ class AutomationService:
         }
     
     def _get_processed_count(self, stage: int) -> int:
-        """Get count of items processed in current stage during this run"""
+        """
+        Get accurate processed count from stage result.
+        Uses stage-specific keys for correct counting instead of DB queries.
+        """
         if not self.run:
             return 0
         
-        # Count items that were updated during this run and changed status from pending
+        # Get the stage result from the run
-        if stage == 1:
+        result = getattr(self.run, f'stage_{stage}_result', None)
-            # Keywords that changed status from 'new' during this run
+        if not result:
-            return Keywords.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at
-            ).exclude(status='new').count()
-        elif stage == 2:
-            # Clusters that changed status from 'new' during this run
-            return Clusters.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at
-            ).exclude(status='new').count()
-        elif stage == 3:
-            # Ideas that changed status from 'approved' during this run
-            return ContentIdeas.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at
-            ).exclude(status='approved').count()
-        elif stage == 4:
-            # Tasks that changed status from 'ready'/'queued' during this run
-            return Tasks.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at
-            ).exclude(status__in=['ready', 'queued']).count()
-        elif stage == 5:
-            # Content processed for image prompts during this run
-            return Content.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at,
-                images__isnull=False
-            ).distinct().count()
-        elif stage == 6:
-            # Images completed during this run
-            return Images.objects.filter(
-                site=self.site,
-                updated_at__gte=self.run.started_at,
-                status='completed'
-            ).count()
-        
             return 0
         
+        # Map stage to correct result key for processed count
+        key_map = {
+            1: 'keywords_processed',
+            2: 'clusters_processed',
+            3: 'ideas_processed',
+            4: 'tasks_processed',
+            5: 'content_processed',
+            6: 'images_processed',
+            7: 'ready_for_review'
+        }
+        
+        return result.get(key_map.get(stage, ''), 0)
+    
     def _get_current_items(self, queryset, count: int) -> list:
         """Get currently processing items"""
         items = queryset[:count]

backend/igny8_core/business/automation/views.py

@@ -714,3 +714,210 @@ class AutomationViewSet(viewsets.ViewSet):
                 status=status.HTTP_500_INTERNAL_SERVER_ERROR
             )
 
+    @extend_schema(tags=['Automation'])
+    @action(detail=False, methods=['get'], url_path='run_progress')
+    def run_progress(self, request):
+        """
+        GET /api/v1/automation/run_progress/?site_id=123&run_id=abc
+        
+        Unified endpoint for ALL run progress data - global + per-stage.
+        Replaces multiple separate API calls with single comprehensive response.
+        
+        Response includes:
+        - run: Current run status and metadata
+        - global_progress: Overall pipeline progress percentage
+        - stages: Per-stage progress with input/output/processed counts
+        - metrics: Credits used, duration, errors
+        """
+        site_id = request.query_params.get('site_id')
+        run_id = request.query_params.get('run_id')
+        
+        if not site_id:
+            return Response(
+                {'error': 'site_id required'},
+                status=status.HTTP_400_BAD_REQUEST
+            )
+        
+        try:
+            site = get_object_or_404(Site, id=site_id, account=request.user.account)
+            
+            # If no run_id, get current run
+            if run_id:
+                run = AutomationRun.objects.get(run_id=run_id, site=site)
+            else:
+                run = AutomationRun.objects.filter(
+                    site=site,
+                    status__in=['running', 'paused']
+                ).order_by('-started_at').first()
+            
+            if not run:
+                return Response({
+                    'run': None,
+                    'global_progress': None,
+                    'stages': [],
+                    'metrics': None
+                })
+            
+            # Build unified response
+            response = self._build_run_progress_response(site, run)
+            return Response(response)
+            
+        except AutomationRun.DoesNotExist:
+            return Response(
+                {'error': 'Run not found'},
+                status=status.HTTP_404_NOT_FOUND
+            )
+        except Exception as e:
+            return Response(
+                {'error': str(e)},
+                status=status.HTTP_500_INTERNAL_SERVER_ERROR
+            )
+    
+    def _build_run_progress_response(self, site, run):
+        """Build comprehensive progress response for a run"""
+        from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas
+        from igny8_core.business.content.models import Tasks, Content, Images
+        from django.db.models import Count
+        from django.utils import timezone
+        
+        initial_snapshot = run.initial_snapshot or {}
+        
+        # Helper to get processed count from result
+        def get_processed(result, key):
+            if not result:
+                return 0
+            return result.get(key, 0)
+        
+        # Helper to get output count from result
+        def get_output(result, key):
+            if not result:
+                return 0
+            return result.get(key, 0)
+        
+        # Stage-specific key mapping for processed counts
+        processed_keys = {
+            1: 'keywords_processed',
+            2: 'clusters_processed',
+            3: 'ideas_processed',
+            4: 'tasks_processed',
+            5: 'content_processed',
+            6: 'images_processed',
+            7: 'ready_for_review'
+        }
+        
+        # Stage-specific key mapping for output counts
+        output_keys = {
+            1: 'clusters_created',
+            2: 'ideas_created',
+            3: 'tasks_created',
+            4: 'content_created',
+            5: 'prompts_created',
+            6: 'images_generated',
+            7: 'ready_for_review'
+        }
+        
+        # Build stages array
+        stages = []
+        total_processed = 0
+        total_initial = initial_snapshot.get('total_initial_items', 0)
+        
+        stage_names = {
+            1: 'Keywords → Clusters',
+            2: 'Clusters → Ideas',
+            3: 'Ideas → Tasks',
+            4: 'Tasks → Content',
+            5: 'Content → Image Prompts',
+            6: 'Image Prompts → Images',
+            7: 'Manual Review Gate'
+        }
+        
+        stage_types = {
+            1: 'AI', 2: 'AI', 3: 'Local', 4: 'AI', 5: 'AI', 6: 'AI', 7: 'Manual'
+        }
+        
+        for stage_num in range(1, 8):
+            result = getattr(run, f'stage_{stage_num}_result', None)
+            initial_count = initial_snapshot.get(f'stage_{stage_num}_initial', 0)
+            processed = get_processed(result, processed_keys[stage_num])
+            output = get_output(result, output_keys[stage_num])
+            
+            total_processed += processed
+            
+            # Determine stage status
+            if run.current_stage > stage_num:
+                stage_status = 'completed'
+            elif run.current_stage == stage_num:
+                stage_status = 'active'
+            else:
+                stage_status = 'pending'
+            
+            # Calculate progress percentage for this stage
+            progress = 0
+            if initial_count > 0:
+                progress = round((processed / initial_count) * 100)
+            elif run.current_stage > stage_num:
+                progress = 100
+            
+            stage_data = {
+                'number': stage_num,
+                'name': stage_names[stage_num],
+                'type': stage_types[stage_num],
+                'status': stage_status,
+                'input_count': initial_count,
+                'output_count': output,
+                'processed_count': processed,
+                'progress_percentage': min(progress, 100),
+                'credits_used': result.get('credits_used', 0) if result else 0,
+                'time_elapsed': result.get('time_elapsed', '') if result else '',
+            }
+            
+            # Add currently_processing for active stage
+            if stage_status == 'active':
+                try:
+                    service = AutomationService.from_run_id(run.run_id)
+                    processing_state = service.get_current_processing_state()
+                    if processing_state:
+                        stage_data['currently_processing'] = processing_state.get('currently_processing', [])
+                        stage_data['up_next'] = processing_state.get('up_next', [])
+                        stage_data['remaining_count'] = processing_state.get('remaining_count', 0)
+                except Exception:
+                    pass
+            
+            stages.append(stage_data)
+        
+        # Calculate global progress
+        global_percentage = 0
+        if total_initial > 0:
+            global_percentage = round((total_processed / total_initial) * 100)
+        
+        # Calculate duration
+        duration_seconds = 0
+        if run.started_at:
+            end_time = run.completed_at or timezone.now()
+            duration_seconds = int((end_time - run.started_at).total_seconds())
+        
+        return {
+            'run': {
+                'run_id': run.run_id,
+                'status': run.status,
+                'current_stage': run.current_stage,
+                'trigger_type': run.trigger_type,
+                'started_at': run.started_at,
+                'completed_at': run.completed_at,
+                'paused_at': run.paused_at,
+            },
+            'global_progress': {
+                'total_items': total_initial,
+                'completed_items': total_processed,
+                'percentage': min(global_percentage, 100),
+                'current_stage': run.current_stage,
+                'total_stages': 7
+            },
+            'stages': stages,
+            'metrics': {
+                'credits_used': run.total_credits_used,
+                'duration_seconds': duration_seconds,
+                'errors': []
+            },
+            'initial_snapshot': initial_snapshot
+        }

frontend/src/components/Automation/CurrentProcessingCard.tsx

@@ -259,9 +259,25 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
   const stageOverview = pipelineOverview && pipelineOverview[currentStageIndex] ? pipelineOverview[currentStageIndex] : null;
   const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`];
 
+  // FIXED: Helper to get processed count using correct key
+  const getProcessedFromResult = (result: any, stageNumber: number): number => {
+    if (!result) return 0;
+    const keyMap: Record<number, string> = {
+      1: 'keywords_processed',
+      2: 'clusters_processed',
+      3: 'ideas_processed',
+      4: 'tasks_processed',
+      5: 'content_processed',
+      6: 'images_processed',
+      7: 'ready_for_review'
+    };
+    return result[keyMap[stageNumber]] ?? 0;
+  };
+
   const fallbackState: ProcessingState | null = ((): ProcessingState | null => {
     if (!processingState && (stageOverview || stageResult)) {
-      const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => typeof v === 'number' ? s + v : s, 0) : 0;
+      // FIXED: Use stage-specific key instead of summing all numeric values
+      const processed = getProcessedFromResult(stageResult, currentRun.current_stage);
       const total = (stageOverview?.pending || 0) + processed;
       const percentage = total > 0 ? Math.round((processed / total) * 100) : 0;
 
@@ -288,8 +304,8 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
   const computedProcessed = ((): number => {
     if (displayState && typeof displayState.processed_items === 'number') return displayState.processed_items;
     if (stageResult) {
-      // Sum numeric values in stageResult as a heuristic for processed count
+      // FIXED: Use stage-specific key for processed count
-      return Object.values(stageResult).reduce((s: number, v: any) => (typeof v === 'number' ? s + v : s), 0);
+      return getProcessedFromResult(stageResult, currentRun.current_stage);
     }
     return 0;
   })();

frontend/src/components/Automation/GlobalProgressBar.tsx

@@ -0,0 +1,232 @@
+/**
+ * Global Progress Bar Component
+ * Shows full pipeline progress across all 7 automation stages.
+ * Persists until 100% complete or run is finished.
+ */
+import React from 'react';
+import { AutomationRun, InitialSnapshot, StageProgress, GlobalProgress } from '../../services/automationService';
+import { BoltIcon, CheckCircleIcon, PauseIcon } from '../../icons';
+
+// Stage colors matching AutomationPage STAGE_CONFIG
+const STAGE_COLORS = [
+  'from-blue-500 to-blue-600',     // Stage 1: Keywords → Clusters
+  'from-purple-500 to-purple-600', // Stage 2: Clusters → Ideas
+  'from-indigo-500 to-indigo-600', // Stage 3: Ideas → Tasks
+  'from-green-500 to-green-600',   // Stage 4: Tasks → Content
+  'from-amber-500 to-amber-600',   // Stage 5: Content → Image Prompts
+  'from-pink-500 to-pink-600',     // Stage 6: Image Prompts → Images
+  'from-teal-500 to-teal-600',     // Stage 7: Manual Review Gate
+];
+
+const STAGE_NAMES = [
+  'Keywords',
+  'Clusters',
+  'Ideas',
+  'Tasks',
+  'Content',
+  'Prompts',
+  'Review',
+];
+
+// Helper to get processed count from stage result using correct key
+export const getProcessedFromResult = (result: any, stageNumber: number): number => {
+  if (!result) return 0;
+  
+  const keyMap: Record<number, string> = {
+    1: 'keywords_processed',
+    2: 'clusters_processed',
+    3: 'ideas_processed',
+    4: 'tasks_processed',
+    5: 'content_processed',
+    6: 'images_processed',
+    7: 'ready_for_review'
+  };
+  
+  return result[keyMap[stageNumber]] ?? 0;
+};
+
+interface GlobalProgressBarProps {
+  currentRun: AutomationRun | null;
+  globalProgress?: GlobalProgress | null;
+  stages?: StageProgress[];
+  initialSnapshot?: InitialSnapshot | null;
+}
+
+const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({
+  currentRun,
+  globalProgress,
+  stages,
+  initialSnapshot,
+}) => {
+  // Don't render if no run or run is completed with 100%
+  if (!currentRun) {
+    return null;
+  }
+  
+  // Calculate global progress if not provided from API
+  const calculateGlobalProgress = (): { percentage: number; completed: number; total: number } => {
+    // If we have API-provided global progress, use it
+    if (globalProgress) {
+      return {
+        percentage: globalProgress.percentage,
+        completed: globalProgress.completed_items,
+        total: globalProgress.total_items,
+      };
+    }
+    
+    // Fallback: Calculate from currentRun and initialSnapshot
+    const snapshot = initialSnapshot || (currentRun as any)?.initial_snapshot;
+    if (!snapshot) {
+      return { percentage: 0, completed: 0, total: 0 };
+    }
+    
+    const totalInitial = snapshot.total_initial_items || 0;
+    let totalCompleted = 0;
+    
+    for (let i = 1; i <= 7; i++) {
+      const result = (currentRun as any)[`stage_${i}_result`];
+      if (result) {
+        totalCompleted += getProcessedFromResult(result, i);
+      }
+    }
+    
+    const percentage = totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0;
+    
+    return {
+      percentage: Math.min(percentage, 100),
+      completed: totalCompleted,
+      total: totalInitial,
+    };
+  };
+  
+  const { percentage, completed, total } = calculateGlobalProgress();
+  
+  // Hide if completed and at 100%
+  if (currentRun.status === 'completed' && percentage >= 100) {
+    return null;
+  }
+  
+  const isPaused = currentRun.status === 'paused';
+  const currentStage = currentRun.current_stage;
+  
+  // Get stage status for segmented bar
+  const getStageStatus = (stageNum: number): 'completed' | 'active' | 'pending' => {
+    if (stages && stages[stageNum - 1]) {
+      return stages[stageNum - 1].status as 'completed' | 'active' | 'pending';
+    }
+    if (currentStage > stageNum) return 'completed';
+    if (currentStage === stageNum) return 'active';
+    return 'pending';
+  };
+  
+  const formatDuration = (): string => {
+    if (!currentRun.started_at) return '';
+    const start = new Date(currentRun.started_at).getTime();
+    const now = Date.now();
+    const diffMs = now - start;
+    const diffMins = Math.floor(diffMs / 60000);
+    const diffHours = Math.floor(diffMins / 60);
+    
+    if (diffHours > 0) {
+      return `${diffHours}h ${diffMins % 60}m`;
+    }
+    return `${diffMins}m`;
+  };
+  
+  return (
+    <div className={`
+      rounded-xl p-4 mb-6 border-2 transition-all
+      ${isPaused 
+        ? 'bg-gradient-to-r from-amber-50 to-amber-100 dark:from-amber-900/20 dark:to-amber-800/20 border-amber-300 dark:border-amber-700'
+        : 'bg-gradient-to-r from-brand-50 to-brand-100 dark:from-brand-900/20 dark:to-brand-800/20 border-brand-300 dark:border-brand-700'
+      }
+    `}>
+      {/* Header Row */}
+      <div className="flex justify-between items-center mb-3">
+        <div className="flex items-center gap-3">
+          <div className={`
+            size-10 rounded-lg flex items-center justify-center shadow-md
+            ${isPaused 
+              ? 'bg-gradient-to-br from-amber-500 to-amber-600'
+              : 'bg-gradient-to-br from-brand-500 to-brand-600'
+            }
+          `}>
+            {isPaused ? (
+              <PauseIcon className="w-5 h-5 text-white" />
+            ) : percentage >= 100 ? (
+              <CheckCircleIcon className="w-5 h-5 text-white" />
+            ) : (
+              <BoltIcon className="w-5 h-5 text-white animate-pulse" />
+            )}
+          </div>
+          <div>
+            <div className={`font-bold ${isPaused ? 'text-amber-800 dark:text-amber-200' : 'text-brand-800 dark:text-brand-200'}`}>
+              {isPaused ? 'Pipeline Paused' : 'Full Pipeline Progress'}
+            </div>
+            <div className="text-xs text-gray-600 dark:text-gray-400">
+              Stage {currentStage} of 7 • {formatDuration()}
+            </div>
+          </div>
+        </div>
+        <div className={`text-3xl font-bold ${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'}`}>
+          {percentage}%
+        </div>
+      </div>
+      
+      {/* Segmented Progress Bar */}
+      <div className="flex h-4 rounded-full overflow-hidden bg-gray-200 dark:bg-gray-700 gap-0.5 mb-2">
+        {[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
+          const status = getStageStatus(stageNum);
+          const stageColor = STAGE_COLORS[stageNum - 1];
+          
+          return (
+            <div
+              key={stageNum}
+              className={`flex-1 transition-all duration-500 relative group ${
+                status === 'completed' 
+                  ? `bg-gradient-to-r ${stageColor}` 
+                  : status === 'active'
+                  ? `bg-gradient-to-r ${stageColor} opacity-60 ${!isPaused ? 'animate-pulse' : ''}`
+                  : 'bg-gray-300 dark:bg-gray-600'
+              }`}
+              title={`Stage ${stageNum}: ${STAGE_NAMES[stageNum - 1]}`}
+            >
+              {/* Tooltip on hover */}
+              <div className="absolute bottom-full left-1/2 -translate-x-1/2 mb-2 px-2 py-1 bg-gray-900 text-white text-xs rounded opacity-0 group-hover:opacity-100 transition-opacity whitespace-nowrap pointer-events-none z-10">
+                S{stageNum}: {STAGE_NAMES[stageNum - 1]}
+                {status === 'completed' && ' ✓'}
+                {status === 'active' && ' ●'}
+              </div>
+            </div>
+          );
+        })}
+      </div>
+      
+      {/* Footer Row */}
+      <div className="flex justify-between text-xs text-gray-600 dark:text-gray-400">
+        <span>{completed} / {total} items processed</span>
+        <div className="flex gap-4">
+          {[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
+            const status = getStageStatus(stageNum);
+            return (
+              <span 
+                key={stageNum}
+                className={`
+                  ${status === 'completed' ? 'text-green-600 dark:text-green-400 font-medium' : ''}
+                  ${status === 'active' ? `${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'} font-bold` : ''}
+                  ${status === 'pending' ? 'text-gray-400 dark:text-gray-500' : ''}
+                `}
+              >
+                {stageNum}
+                {status === 'completed' && '✓'}
+                {status === 'active' && '●'}
+              </span>
+            );
+          })}
+        </div>
+      </div>
+    </div>
+  );
+};
+
+export default GlobalProgressBar;

frontend/src/pages/Automation/AutomationPage.tsx

@@ -5,7 +5,7 @@
 import React, { useState, useEffect } from 'react';
 import { useToast } from '../../components/ui/toast/ToastContainer';
 import { useSiteStore } from '../../store/siteStore';
-import { automationService, AutomationRun, AutomationConfig, PipelineStage } from '../../services/automationService';
+import { automationService, AutomationRun, AutomationConfig, PipelineStage, RunProgressResponse, GlobalProgress, StageProgress, InitialSnapshot } from '../../services/automationService';
 import {
   fetchKeywords,
   fetchClusters,
@@ -18,6 +18,7 @@ import ActivityLog from '../../components/Automation/ActivityLog';
 import ConfigModal from '../../components/Automation/ConfigModal';
 import RunHistory from '../../components/Automation/RunHistory';
 import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard';
+import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar';
 import PageMeta from '../../components/common/PageMeta';
 import PageHeader from '../../components/common/PageHeader';
 import ComponentCard from '../../components/common/ComponentCard';
@@ -59,6 +60,11 @@ const AutomationPage: React.FC = () => {
   const [loading, setLoading] = useState(true);
   const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null);
   
+  // New state for unified progress data
+  const [globalProgress, setGlobalProgress] = useState<GlobalProgress | null>(null);
+  const [stageProgress, setStageProgress] = useState<StageProgress[]>([]);
+  const [initialSnapshot, setInitialSnapshot] = useState<InitialSnapshot | null>(null);
+
   useEffect(() => {
     if (!activeSite) return;
     loadData();
@@ -167,12 +173,34 @@ const AutomationPage: React.FC = () => {
       const data = await automationService.getCurrentRun(activeSite.id);
       setCurrentRun(data.run);
       // ensure processing card is visible when a run exists
-      if (data.run) setShowProcessingCard(true);
+      if (data.run) {
+        setShowProcessingCard(true);
+        // Also load unified progress data for GlobalProgressBar
+        await loadRunProgress(data.run.run_id);
+      }
     } catch (error) {
       console.error('Failed to poll current run', error);
     }
   };
 
+  const loadRunProgress = async (runId?: string) => {
+    if (!activeSite) return;
+    try {
+      const progressData = await automationService.getRunProgress(activeSite.id, runId);
+      if (progressData.global_progress) {
+        setGlobalProgress(progressData.global_progress);
+      }
+      if (progressData.stages) {
+        setStageProgress(progressData.stages);
+      }
+      if (progressData.initial_snapshot) {
+        setInitialSnapshot(progressData.initial_snapshot);
+      }
+    } catch (error) {
+      console.error('Failed to load run progress', error);
+    }
+  };
+
   const loadPipelineOverview = async () => {
     if (!activeSite) return;
     try {
@@ -667,6 +695,16 @@ const AutomationPage: React.FC = () => {
           </div>
         </div>
 
+        {/* Global Progress Bar - Shows full pipeline progress during automation run */}
+        {currentRun && (currentRun.status === 'running' || currentRun.status === 'paused') && (
+          <GlobalProgressBar
+            currentRun={currentRun}
+            globalProgress={globalProgress}
+            stages={stageProgress}
+            initialSnapshot={initialSnapshot}
+          />
+        )}
+
         {/* Current Processing Card - Shows real-time automation progress */}
         {currentRun && showProcessingCard && activeSite && (
           <CurrentProcessingCard
@@ -697,8 +735,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
+              // FIXED: Use stage-specific key for processed count instead of summing all numeric values
-              const total = (stage.pending ?? 0) + processed;
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available, otherwise fallback to pending + processed
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -738,8 +780,8 @@ const AutomationPage: React.FC = () => {
                   {/* Queue Metrics */}
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -748,7 +790,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor} dark:${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}
@@ -796,8 +838,12 @@ const AutomationPage: React.FC = () => {
               const isActive = currentRun?.current_stage === stage.number;
               const isComplete = currentRun && currentRun.current_stage > stage.number;
               const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
-              const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
+              // FIXED: Use stage-specific key for processed count
-              const total = (stage.pending ?? 0) + processed;
+              const processed = getProcessedFromResult(result, stage.number);
+              // FIXED: Use initial snapshot for total when available
+              const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
+              const total = initialCount ?? ((stage.pending ?? 0) + processed);
+              const remaining = Math.max(0, total - processed);
               const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
               
               return (
@@ -835,8 +881,8 @@ const AutomationPage: React.FC = () => {
                   
                   <div className="space-y-1.5 text-xs mb-3">
                     <div className="flex justify-between">
-                      <span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
+                      <span className="text-gray-600 dark:text-gray-400">Total Items:</span>
-                      <span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
+                      <span className="font-bold text-slate-900 dark:text-white">{total}</span>
                     </div>
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Processed:</span>
@@ -845,7 +891,7 @@ const AutomationPage: React.FC = () => {
                     <div className="flex justify-between">
                       <span className="text-gray-600 dark:text-gray-400">Remaining:</span>
                       <span className={`font-bold ${stageConfig.textColor}`}>
-                        {stage.pending}
+                        {remaining}
                       </span>
                     </div>
                     {/* Credits and Time - Section 6 Enhancement */}

frontend/src/services/automationService.ts

@@ -78,6 +78,62 @@ export interface ProcessingState {
   remaining_count: number;
 }
 
+// NEW: Types for unified run_progress endpoint
+export interface StageProgress {
+  number: number;
+  name: string;
+  type: 'AI' | 'Local' | 'Manual';
+  status: 'pending' | 'active' | 'completed' | 'skipped';
+  input_count: number;
+  output_count: number;
+  processed_count: number;
+  progress_percentage: number;
+  credits_used: number;
+  time_elapsed: string;
+  currently_processing?: ProcessingItem[];
+  up_next?: ProcessingItem[];
+  remaining_count?: number;
+}
+
+export interface GlobalProgress {
+  total_items: number;
+  completed_items: number;
+  percentage: number;
+  current_stage: number;
+  total_stages: number;
+}
+
+export interface InitialSnapshot {
+  stage_1_initial: number;
+  stage_2_initial: number;
+  stage_3_initial: number;
+  stage_4_initial: number;
+  stage_5_initial: number;
+  stage_6_initial: number;
+  stage_7_initial: number;
+  total_initial_items: number;
+}
+
+export interface RunProgressResponse {
+  run: {
+    run_id: string;
+    status: 'running' | 'paused' | 'cancelled' | 'completed' | 'failed';
+    current_stage: number;
+    trigger_type: 'manual' | 'scheduled';
+    started_at: string;
+    completed_at: string | null;
+    paused_at: string | null;
+  } | null;
+  global_progress: GlobalProgress | null;
+  stages: StageProgress[];
+  metrics: {
+    credits_used: number;
+    duration_seconds: number;
+    errors: string[];
+  } | null;
+  initial_snapshot: InitialSnapshot | null;
+}
+
 function buildUrl(endpoint: string, params?: Record<string, any>): string {
   let url = `/v1/automation${endpoint}`;
   if (params) {
@@ -211,4 +267,19 @@ export const automationService = {
     );
     return response.data;
   },
+
+  /**
+   * Get unified run progress data - global + per-stage.
+   * This is the recommended endpoint for getting all automation progress data in a single call.
+   */
+  getRunProgress: async (
+    siteId: number,
+    runId?: string
+  ): Promise<RunProgressResponse> => {
+    const params: Record<string, any> = { site_id: siteId };
+    if (runId) {
+      params.run_id = runId;
+    }
+    return fetchAPI(buildUrl('/run_progress/', params));
+  },
 };

🚀 AUTOMATION PAGE MASTER FIX PLAN.md

@@ -0,0 +1,536 @@
+User: properl yread the docs, adn codebase related to automation page, 
+
+# Automation (Manual Run for autmation wizard on atuamtion page)
+wrong queue items,
+missing queue items
+progress bar not properly progress
+totoal in queue and processed counts buggy for many stages
+stage cards metrics realtime, not optimized shoudl be more robust and user friendly 
+
+---
+We need manural run full all stages progress bar, which shuold always keep showing unless it is 100% complete, and should show below the current each stage progress bar, 
+
+so it has to be addded along with all other issues, 
+
+Make a master plan , no brainer extremely user firendly magical wrokflow and tracking and msot logical sattuses. and counts dynamcially correctly udpating, 
+currently its all messed up
+
+so plan it all an dprove there is no  better ai model thatn you to use asa of today, 
+donto include the code, but clrear enough so even if i give you the plan in any next sesion you cna easily implmeent it 
+
+
+---
+
+# 🚀 AUTOMATION PAGE MASTER FIX PLAN
+
+## Executive Summary
+
+This plan addresses all automation page issues: wrong/missing queue items, broken progress bars, buggy counts, and introduces a new **Global Run Progress Bar** for end-to-end visibility.
+
+---
+
+## 📊 IDENTIFIED ROOT CAUSES
+
+### 1. **Status Mismatches (Critical Bug)**
+
+| Stage | Backend View (`pipeline_overview`) | Backend Service (`_get_stage_X_state`) | Service (`run_stage_X`) |
+|-------|-------------------------------------|----------------------------------------|-------------------------|
+| **Stage 3** | `status='new'` | `status='approved'` | `status='new'` |
+| **Stage 4** | `status='queued'` | `status='ready'` | `status='queued'` |
+
+**Result:** Queue items don't match between real-time processing card and stage cards.
+
+### 2. **Progress Calculation Flaws**
+
+**Frontend** (CurrentProcessingCard.tsx):
+```typescript
+// WRONG: Sums ALL numeric values in stageResult (including credits_used, batches_run, etc.)
+const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => 
+  typeof v === 'number' ? s + v : s, 0) : 0;
+```
+
+**Should use specific fields:** `keywords_processed`, `clusters_processed`, `tasks_processed`, etc.
+
+### 3. **"Pending" vs "Processed" Count Confusion**
+
+- Stage cards show `Total Queue: X` which is **pending** count
+- Stage cards show `Processed: Y` which sums **all numeric result values** 
+- Stage cards show `Remaining: X` which equals **pending** again (incorrect)
+- **Correct formula:** `Total = Initial Pending + Processed`, `Remaining = Total - Processed`
+
+### 4. **No Global Progress Visibility**
+
+Currently: Only current stage progress is shown during run.
+
+**Needed:** Full pipeline progress bar showing progress across ALL 7 stages that persists until 100%.
+
+### 5. **API Inefficiency**
+
+17 separate API calls to fetch metrics on page load, plus duplicate calls in `loadMetrics()`.
+
+---
+
+## 🏗️ ARCHITECTURE REDESIGN
+
+### New Data Model: Run Progress Snapshot
+
+Add these fields to `AutomationRun` for accurate global tracking:
+
+```python
+# AutomationRun Model Additions
+class AutomationRun(models.Model):
+    # ... existing fields ...
+    
+    # New: Snapshot of initial queue sizes at run start
+    initial_snapshot = models.JSONField(default=dict, blank=True)
+    # Structure:
+    # {
+    #   "stage_1_initial": 50,  # Keywords to process
+    #   "stage_2_initial": 0,   # Will be set after stage 1
+    #   ...
+    #   "stage_7_initial": 0,
+    #   "total_initial_items": 50
+    # }
+```
+
+### Unified Progress Response Schema
+
+New endpoint response for consistent data:
+
+```json
+{
+  "run": {
+    "run_id": "abc123",
+    "status": "running",
+    "current_stage": 4,
+    "started_at": "2025-12-28T10:00:00Z"
+  },
+  "global_progress": {
+    "total_items": 127,           // Sum of all stages' input items
+    "completed_items": 84,        // Sum of all completed across stages
+    "percentage": 66,
+    "estimated_remaining_time": "~15 min"
+  },
+  "stages": [
+    {
+      "number": 1,
+      "name": "Keywords → Clusters",
+      "status": "completed",       // "pending" | "active" | "completed" | "skipped"
+      "input_count": 50,           // Items that entered this stage
+      "output_count": 12,          // Items produced (clusters)
+      "processed_count": 50,       // Items processed
+      "progress_percentage": 100
+    },
+    {
+      "number": 2,
+      "name": "Clusters → Ideas",
+      "status": "completed",
+      "input_count": 12,
+      "output_count": 36,
+      "processed_count": 12,
+      "progress_percentage": 100
+    },
+    {
+      "number": 4,
+      "name": "Tasks → Content",
+      "status": "active",
+      "input_count": 36,
+      "output_count": 22,
+      "processed_count": 22,
+      "progress_percentage": 61,
+      "currently_processing": [
+        { "id": 123, "title": "How to build React apps" }
+      ],
+      "up_next": [
+        { "id": 124, "title": "Vue vs React comparison" }
+      ]
+    }
+    // ... etc
+  ],
+  "metrics": {
+    "credits_used": 156,
+    "duration_seconds": 1823,
+    "errors": []
+  }
+}
+```
+
+---
+
+## 📝 IMPLEMENTATION PLAN
+
+### Phase 1: Backend Fixes (Critical)
+
+#### 1.1 Fix Status Mismatches
+
+**File:** automation_service.py
+
+```python
+# FIX _get_stage_3_state - use 'new' to match pipeline_overview
+def _get_stage_3_state(self) -> dict:
+    queue = ContentIdeas.objects.filter(
+        site=self.site, status='new'  # Changed from 'approved'
+    ).order_by('id')
+    ...
+
+# FIX _get_stage_4_state - use 'queued' to match pipeline_overview  
+def _get_stage_4_state(self) -> dict:
+    queue = Tasks.objects.filter(
+        site=self.site, status='queued'  # Changed from 'ready'
+    ).order_by('id')
+    ...
+```
+
+#### 1.2 Fix `_get_processed_count()` Method
+
+Current code sums wrong fields. Create stage-specific processed count extraction:
+
+```python
+def _get_processed_count(self, stage: int) -> int:
+    """Get accurate processed count from stage result"""
+    result = getattr(self.run, f'stage_{stage}_result', None)
+    if not result:
+        return 0
+    
+    # Map stage to correct result key
+    key_map = {
+        1: 'keywords_processed',
+        2: 'clusters_processed', 
+        3: 'ideas_processed',
+        4: 'tasks_processed',
+        5: 'content_processed',
+        6: 'images_processed',
+        7: 'ready_for_review'
+    }
+    return result.get(key_map.get(stage, ''), 0)
+```
+
+#### 1.3 New Unified Progress Endpoint
+
+**File:** views.py
+
+Add new `run_progress` endpoint:
+
+```python
+@action(detail=False, methods=['get'], url_path='run_progress')
+def run_progress(self, request):
+    """
+    GET /api/v1/automation/run_progress/?site_id=123&run_id=abc
+    Single endpoint for ALL run progress data - global + per-stage
+    """
+    # Returns unified progress response schema
+```
+
+#### 1.4 Capture Initial Snapshot on Run Start
+
+**File:** automation_service.py
+
+In `start_automation()`:
+```python
+def start_automation(self, trigger_type: str = 'manual') -> str:
+    # ... existing code ...
+    
+    # Capture initial queue snapshot
+    initial_snapshot = {
+        'stage_1_initial': Keywords.objects.filter(site=self.site, status='new', cluster__isnull=True, disabled=False).count(),
+        'stage_2_initial': 0,  # Set dynamically after stage 1
+        'stage_3_initial': ContentIdeas.objects.filter(site=self.site, status='new').count(),
+        'stage_4_initial': Tasks.objects.filter(site=self.site, status='queued').count(),
+        'stage_5_initial': Content.objects.filter(site=self.site, status='draft').annotate(images_count=Count('images')).filter(images_count=0).count(),
+        'stage_6_initial': Images.objects.filter(site=self.site, status='pending').count(),
+        'stage_7_initial': Content.objects.filter(site=self.site, status='review').count(),
+    }
+    initial_snapshot['total_initial_items'] = sum(initial_snapshot.values())
+    
+    self.run = AutomationRun.objects.create(
+        # ... existing fields ...
+        initial_snapshot=initial_snapshot
+    )
+```
+
+---
+
+### Phase 2: Frontend Fixes
+
+#### 2.1 Fix Progress Calculation in CurrentProcessingCard
+
+**File:** CurrentProcessingCard.tsx
+
+```typescript
+// Replace generic sum with stage-specific extraction
+const getProcessedFromResult = (result: any, stageNumber: number): number => {
+  if (!result) return 0;
+  
+  const keyMap: Record<number, string> = {
+    1: 'keywords_processed',
+    2: 'clusters_processed',
+    3: 'ideas_processed',
+    4: 'tasks_processed',
+    5: 'content_processed',
+    6: 'images_processed',
+    7: 'ready_for_review'
+  };
+  
+  return result[keyMap[stageNumber]] ?? 0;
+};
+```
+
+#### 2.2 Fix Stage Card Metrics
+
+**File:** AutomationPage.tsx
+
+```typescript
+// Current (WRONG):
+const processed = result ? Object.values(result).reduce((sum, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
+const total = (stage.pending ?? 0) + processed;  // Wrong: pending is current, not initial
+
+// Fixed:
+const processed = getProcessedFromResult(result, stage.number);
+const initialPending = currentRun?.initial_snapshot?.[`stage_${stage.number}_initial`] ?? stage.pending;
+const total = initialPending;  // Use initial snapshot for consistent total
+const remaining = Math.max(0, total - processed);
+```
+
+#### 2.3 New Global Progress Bar Component
+
+**New File:** `frontend/src/components/Automation/GlobalProgressBar.tsx`
+
+```typescript
+interface GlobalProgressBarProps {
+  currentRun: AutomationRun;
+  pipelineOverview: PipelineStage[];
+}
+
+const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({ currentRun, pipelineOverview }) => {
+  // Calculate total progress across all stages
+  const calculateGlobalProgress = () => {
+    if (!currentRun?.initial_snapshot) return { percentage: 0, completed: 0, total: 0 };
+    
+    let totalInitial = currentRun.initial_snapshot.total_initial_items || 0;
+    let totalCompleted = 0;
+    
+    for (let i = 1; i <= 7; i++) {
+      const result = currentRun[`stage_${i}_result`];
+      if (result) {
+        totalCompleted += getProcessedFromResult(result, i);
+      }
+    }
+    
+    // If current stage is active, add its progress
+    const currentStage = currentRun.current_stage;
+    // ... calculate current stage partial progress
+    
+    return {
+      percentage: totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0,
+      completed: totalCompleted,
+      total: totalInitial
+    };
+  };
+  
+  const { percentage, completed, total } = calculateGlobalProgress();
+  
+  // Show until 100% OR run completed
+  if (currentRun.status === 'completed' && percentage === 100) {
+    return null;
+  }
+  
+  return (
+    <div className="bg-gradient-to-r from-brand-50 to-brand-100 border-2 border-brand-300 rounded-xl p-4 mb-6">
+      <div className="flex justify-between items-center mb-2">
+        <div className="flex items-center gap-2">
+          <BoltIcon className="w-5 h-5 text-brand-600 animate-pulse" />
+          <span className="font-bold text-brand-800">Full Pipeline Progress</span>
+        </div>
+        <span className="text-2xl font-bold text-brand-600">{percentage}%</span>
+      </div>
+      
+      {/* Segmented progress bar showing all 7 stages */}
+      <div className="flex h-4 rounded-full overflow-hidden bg-gray-200">
+        {[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
+          const stageConfig = STAGE_CONFIG[stageNum - 1];
+          const result = currentRun[`stage_${stageNum}_result`];
+          const stageComplete = currentRun.current_stage > stageNum;
+          const isActive = currentRun.current_stage === stageNum;
+          
+          return (
+            <div
+              key={stageNum}
+              className={`flex-1 transition-all duration-500 ${
+                stageComplete ? `bg-gradient-to-r ${stageConfig.color}` :
+                isActive ? `bg-gradient-to-r ${stageConfig.color} opacity-60 animate-pulse` :
+                'bg-gray-300'
+              }`}
+              title={`Stage ${stageNum}: ${stageConfig.name}`}
+            />
+          );
+        })}
+      </div>
+      
+      <div className="flex justify-between text-xs text-gray-600 mt-2">
+        <span>{completed} / {total} items processed</span>
+        <span>Stage {currentRun.current_stage} of 7</span>
+      </div>
+    </div>
+  );
+};
+```
+
+#### 2.4 Consolidate API Calls
+
+**File:** AutomationPage.tsx
+
+Replace 17 separate API calls with single unified endpoint:
+
+```typescript
+// Current (17 calls):
+const [keywordsTotalRes, keywordsNewRes, keywordsMappedRes, ...14 more] = await Promise.all([...]);
+
+// New (1 call):
+const progressData = await automationService.getRunProgress(activeSite.id, currentRun?.run_id);
+// Response contains everything: metrics, stage counts, progress data
+```
+
+---
+
+### Phase 3: Stage Card Redesign
+
+#### 3.1 New Stage Card Layout
+
+Each stage card shows:
+
+```
+┌────────────────────────────────────────────┐
+│  Stage 1    [ICON]    ● Active             │
+│  Keywords → Clusters                        │
+├────────────────────────────────────────────┤
+│  Total Items:      50                       │
+│  Processed:        32     ████████░░ 64%   │
+│  Remaining:        18                       │
+├────────────────────────────────────────────┤
+│  Output Created:   8 clusters               │
+│  Credits Used:     24                       │
+│  Duration:         4m 32s                   │
+└────────────────────────────────────────────┘
+```
+
+#### 3.2 Status Badge Logic
+
+```typescript
+const getStageStatus = (stageNum: number, currentRun: AutomationRun | null) => {
+  if (!currentRun) {
+    // No run - show if items pending
+    return pipelineOverview[stageNum - 1]?.pending > 0 ? 'ready' : 'empty';
+  }
+  
+  if (currentRun.current_stage > stageNum) return 'completed';
+  if (currentRun.current_stage === stageNum) return 'active';
+  if (currentRun.current_stage < stageNum) {
+    // Check if previous stage produced items for this stage
+    const prevResult = currentRun[`stage_${stageNum - 1}_result`];
+    if (prevResult?.output_count > 0) return 'ready';
+    return 'pending';
+  }
+  return 'pending';
+};
+```
+
+---
+
+### Phase 4: Real-time Updates Optimization
+
+#### 4.1 Smart Polling with Exponential Backoff
+
+```typescript
+// Current: Fixed 5s interval
+const interval = setInterval(loadData, 5000);
+
+// New: Adaptive polling
+const useSmartPolling = (isRunning: boolean) => {
+  const [pollInterval, setPollInterval] = useState(2000);
+  
+  useEffect(() => {
+    if (!isRunning) {
+      setPollInterval(30000); // Slow poll when idle
+      return;
+    }
+    
+    // Fast poll during active run, slow down as stage progresses
+    const progressPercent = /* current stage progress */;
+    if (progressPercent < 50) {
+      setPollInterval(2000);  // 2s when lots happening
+    } else if (progressPercent < 90) {
+      setPollInterval(3000);  // 3s mid-stage
+    } else {
+      setPollInterval(1000);  // 1s near completion for responsive transition
+    }
+  }, [isRunning, progressPercent]);
+  
+  return pollInterval;
+};
+```
+
+#### 4.2 Optimistic UI Updates
+
+When user clicks "Run Now":
+1. Immediately show GlobalProgressBar at 0%
+2. Immediately set Stage 1 to "Active" 
+3. Don't wait for API confirmation
+
+---
+
+## 📋 DETAILED CHECKLIST
+
+### Backend Tasks
+- [x] Fix `_get_stage_3_state()` status filter: `'approved'` → `'new'` ✅ DONE
+- [x] Fix `_get_stage_4_state()` status filter: `'ready'` → `'queued'` ✅ DONE
+- [x] Create `_get_processed_for_stage(stage_num)` helper ✅ DONE (renamed to `_get_processed_count`)
+- [x] Add `initial_snapshot` JSON field to `AutomationRun` model ✅ DONE
+- [x] Capture initial snapshot in `start_automation()` ✅ DONE
+- [ ] Update snapshot after each stage completes (for cascading stages)
+- [x] Create new `run_progress` endpoint with unified schema ✅ DONE
+- [x] Add migration for new model field ✅ DONE (0006_automationrun_initial_snapshot.py)
+
+### Frontend Tasks
+- [x] Create `GlobalProgressBar` component ✅ DONE
+- [x] Add `GlobalProgressBar` to AutomationPage (below metrics, above CurrentProcessingCard) ✅ DONE
+- [x] Fix `getProcessedFromResult()` helper to extract stage-specific counts ✅ DONE
+- [x] Update stage card progress calculations ✅ DONE
+- [x] Update `CurrentProcessingCard` progress calculations ✅ DONE
+- [x] Add `getRunProgress` method to automationService.ts ✅ DONE
+- [ ] Consolidate metrics API calls to single endpoint
+- [ ] Implement smart polling with adaptive intervals
+- [ ] Add optimistic UI updates for "Run Now" action
+- [x] Fix "Remaining" count to be `Total - Processed` not `Pending` ✅ DONE
+
+### Testing
+- [ ] Test all 7 stages complete correctly
+- [ ] Verify counts match between stage cards and processing card
+- [ ] Test pause/resume preserves progress correctly
+- [ ] Test page refresh during run shows correct state
+- [ ] Test global progress bar persists until 100%
+- [ ] Load test: Verify API efficiency improvement
+
+---
+
+## 🎯 SUCCESS CRITERIA
+
+1. **Accurate Counts:** All stage cards show correct Total/Processed/Remaining
+2. **Consistent Data:** CurrentProcessingCard and Stage Cards show same numbers
+3. **Global Visibility:** Users see full pipeline progress at all times during run
+4. **Persistent Progress:** Progress bar stays visible until 100% complete
+5. **Real-time Feel:** Updates appear within 2-3 seconds of actual progress
+6. **API Efficiency:** Reduce API calls from 17+ to 1-2 per refresh cycle
+
+---
+
+## 🔄 MIGRATION PATH
+
+1. **Phase 1 (Day 1):** Backend status fixes + new processed count logic
+2. **Phase 2 (Day 2):** Frontend progress calculation fixes
+3. **Phase 3 (Day 3):** Global Progress Bar + API consolidation
+4. **Phase 4 (Day 4):** Smart polling + optimistic updates
+5. **Phase 5 (Day 5):** Testing + bug fixes
+
+---
+
+This plan provides a clear, implementable path to fix all automation page issues. Each phase can be implemented independently, and the plan contains enough detail that any AI model or developer can execute it in a future session.