From ea9125b805e2524b39ec9f5f730ac27a7893ad84 Mon Sep 17 00:00:00 2001 From: "IGNY8 VPS (Salman)" Date: Sun, 28 Dec 2025 01:46:27 +0000 Subject: [PATCH] Automation revamp part 1 --- .../0006_automationrun_initial_snapshot.py | 22 + .../igny8_core/business/automation/models.py | 7 + .../automation/services/automation_service.py | 128 +++-- .../igny8_core/business/automation/views.py | 207 +++++++ .../Automation/CurrentProcessingCard.tsx | 22 +- .../Automation/GlobalProgressBar.tsx | 232 ++++++++ .../src/pages/Automation/AutomationPage.tsx | 70 ++- frontend/src/services/automationService.ts | 71 +++ 🚀 AUTOMATION PAGE MASTER FIX PLAN.md | 536 ++++++++++++++++++ 9 files changed, 1237 insertions(+), 58 deletions(-) create mode 100644 backend/igny8_core/business/automation/migrations/0006_automationrun_initial_snapshot.py create mode 100644 frontend/src/components/Automation/GlobalProgressBar.tsx create mode 100644 🚀 AUTOMATION PAGE MASTER FIX PLAN.md diff --git a/backend/igny8_core/business/automation/migrations/0006_automationrun_initial_snapshot.py b/backend/igny8_core/business/automation/migrations/0006_automationrun_initial_snapshot.py new file mode 100644 index 00000000..a4f9f20e --- /dev/null +++ b/backend/igny8_core/business/automation/migrations/0006_automationrun_initial_snapshot.py @@ -0,0 +1,22 @@ +# Generated migration for adding initial_snapshot field to AutomationRun + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('automation', '0005_add_default_image_service'), + ] + + operations = [ + migrations.AddField( + model_name='automationrun', + name='initial_snapshot', + field=models.JSONField( + blank=True, + default=dict, + help_text='Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}' + ), + ), + ] diff --git a/backend/igny8_core/business/automation/models.py b/backend/igny8_core/business/automation/models.py index 8f4836c5..9246532e 100644 --- a/backend/igny8_core/business/automation/models.py +++ b/backend/igny8_core/business/automation/models.py @@ -88,6 +88,13 @@ class AutomationRun(models.Model): total_credits_used = models.IntegerField(default=0) + # Initial queue snapshot - captured at run start for accurate progress tracking + initial_snapshot = models.JSONField( + default=dict, + blank=True, + help_text="Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}" + ) + # JSON results per stage stage_1_result = models.JSONField(null=True, blank=True, help_text="{keywords_processed, clusters_created, batches}") stage_2_result = models.JSONField(null=True, blank=True, help_text="{clusters_processed, ideas_created}") diff --git a/backend/igny8_core/business/automation/services/automation_service.py b/backend/igny8_core/business/automation/services/automation_service.py index 0b73177e..1213510f 100644 --- a/backend/igny8_core/business/automation/services/automation_service.py +++ b/backend/igny8_core/business/automation/services/automation_service.py @@ -109,6 +109,9 @@ class AutomationService: # Create run_id and log files run_id = self.logger.start_run(self.account.id, self.site.id, trigger_type) + # Capture initial queue snapshot for accurate progress tracking + initial_snapshot = self._capture_initial_snapshot() + # Create AutomationRun record self.run = AutomationRun.objects.create( run_id=run_id, @@ -117,6 +120,7 @@ class AutomationService: trigger_type=trigger_type, status='running', current_stage=1, + initial_snapshot=initial_snapshot, ) # Log start @@ -124,6 +128,10 @@ class AutomationService: run_id, self.account.id, self.site.id, 0, f"Automation started (trigger: {trigger_type})" ) + self.logger.log_stage_progress( + run_id, self.account.id, self.site.id, 0, + f"Initial snapshot captured: {initial_snapshot['total_initial_items']} total items across all stages" + ) self.logger.log_stage_progress( run_id, self.account.id, self.site.id, 0, f"Credit check: Account has {self.account.credits} credits, estimated need: {estimated_credits} credits" @@ -1361,6 +1369,61 @@ class AutomationService: logger.info(f"[AutomationService] Estimated credits: {total}") return total + def _capture_initial_snapshot(self) -> dict: + """ + Capture initial queue sizes at run start for accurate progress tracking. + This snapshot is used to calculate global progress percentage correctly. + """ + # Stage 1: Keywords pending clustering + stage_1_initial = Keywords.objects.filter( + site=self.site, status='new', cluster__isnull=True, disabled=False + ).count() + + # Stage 2: Clusters needing ideas + stage_2_initial = Clusters.objects.filter( + site=self.site, status='new', disabled=False + ).exclude(ideas__isnull=False).count() + + # Stage 3: Ideas ready to be converted to tasks + stage_3_initial = ContentIdeas.objects.filter( + site=self.site, status='new' + ).count() + + # Stage 4: Tasks ready for content generation + stage_4_initial = Tasks.objects.filter( + site=self.site, status='queued' + ).count() + + # Stage 5: Content needing image prompts + stage_5_initial = Content.objects.filter( + site=self.site, status='draft' + ).annotate(images_count=Count('images')).filter(images_count=0).count() + + # Stage 6: Image prompts pending generation + stage_6_initial = Images.objects.filter( + site=self.site, status='pending' + ).count() + + # Stage 7: Content ready for review + stage_7_initial = Content.objects.filter( + site=self.site, status='review' + ).count() + + snapshot = { + 'stage_1_initial': stage_1_initial, + 'stage_2_initial': stage_2_initial, + 'stage_3_initial': stage_3_initial, + 'stage_4_initial': stage_4_initial, + 'stage_5_initial': stage_5_initial, + 'stage_6_initial': stage_6_initial, + 'stage_7_initial': stage_7_initial, + 'total_initial_items': stage_1_initial + stage_2_initial + stage_3_initial + + stage_4_initial + stage_5_initial + stage_6_initial + stage_7_initial, + } + + logger.info(f"[AutomationService] Initial snapshot captured: {snapshot}") + return snapshot + # Helper methods def _wait_for_task(self, task_id: str, stage_number: int, item_name: str, continue_on_error: bool = True): @@ -1559,7 +1622,7 @@ class AutomationService: def _get_stage_3_state(self) -> dict: """Get processing state for Stage 3: Ideas → Tasks""" queue = ContentIdeas.objects.filter( - site=self.site, status='approved' + site=self.site, status='new' # Fixed: Match pipeline_overview status ).order_by('id') processed = self._get_processed_count(3) @@ -1580,7 +1643,7 @@ class AutomationService: def _get_stage_4_state(self) -> dict: """Get processing state for Stage 4: Tasks → Content""" queue = Tasks.objects.filter( - site=self.site, status='ready' + site=self.site, status='queued' # Fixed: Match pipeline_overview status ).order_by('id') processed = self._get_processed_count(4) @@ -1666,51 +1729,30 @@ class AutomationService: } def _get_processed_count(self, stage: int) -> int: - """Get count of items processed in current stage during this run""" + """ + Get accurate processed count from stage result. + Uses stage-specific keys for correct counting instead of DB queries. + """ if not self.run: return 0 - # Count items that were updated during this run and changed status from pending - if stage == 1: - # Keywords that changed status from 'new' during this run - return Keywords.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at - ).exclude(status='new').count() - elif stage == 2: - # Clusters that changed status from 'new' during this run - return Clusters.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at - ).exclude(status='new').count() - elif stage == 3: - # Ideas that changed status from 'approved' during this run - return ContentIdeas.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at - ).exclude(status='approved').count() - elif stage == 4: - # Tasks that changed status from 'ready'/'queued' during this run - return Tasks.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at - ).exclude(status__in=['ready', 'queued']).count() - elif stage == 5: - # Content processed for image prompts during this run - return Content.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at, - images__isnull=False - ).distinct().count() - elif stage == 6: - # Images completed during this run - return Images.objects.filter( - site=self.site, - updated_at__gte=self.run.started_at, - status='completed' - ).count() + # Get the stage result from the run + result = getattr(self.run, f'stage_{stage}_result', None) + if not result: + return 0 - return 0 + # Map stage to correct result key for processed count + key_map = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + } + + return result.get(key_map.get(stage, ''), 0) def _get_current_items(self, queryset, count: int) -> list: """Get currently processing items""" diff --git a/backend/igny8_core/business/automation/views.py b/backend/igny8_core/business/automation/views.py index 6eb67c75..00d09c7e 100644 --- a/backend/igny8_core/business/automation/views.py +++ b/backend/igny8_core/business/automation/views.py @@ -714,3 +714,210 @@ class AutomationViewSet(viewsets.ViewSet): status=status.HTTP_500_INTERNAL_SERVER_ERROR ) + @extend_schema(tags=['Automation']) + @action(detail=False, methods=['get'], url_path='run_progress') + def run_progress(self, request): + """ + GET /api/v1/automation/run_progress/?site_id=123&run_id=abc + + Unified endpoint for ALL run progress data - global + per-stage. + Replaces multiple separate API calls with single comprehensive response. + + Response includes: + - run: Current run status and metadata + - global_progress: Overall pipeline progress percentage + - stages: Per-stage progress with input/output/processed counts + - metrics: Credits used, duration, errors + """ + site_id = request.query_params.get('site_id') + run_id = request.query_params.get('run_id') + + if not site_id: + return Response( + {'error': 'site_id required'}, + status=status.HTTP_400_BAD_REQUEST + ) + + try: + site = get_object_or_404(Site, id=site_id, account=request.user.account) + + # If no run_id, get current run + if run_id: + run = AutomationRun.objects.get(run_id=run_id, site=site) + else: + run = AutomationRun.objects.filter( + site=site, + status__in=['running', 'paused'] + ).order_by('-started_at').first() + + if not run: + return Response({ + 'run': None, + 'global_progress': None, + 'stages': [], + 'metrics': None + }) + + # Build unified response + response = self._build_run_progress_response(site, run) + return Response(response) + + except AutomationRun.DoesNotExist: + return Response( + {'error': 'Run not found'}, + status=status.HTTP_404_NOT_FOUND + ) + except Exception as e: + return Response( + {'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR + ) + + def _build_run_progress_response(self, site, run): + """Build comprehensive progress response for a run""" + from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas + from igny8_core.business.content.models import Tasks, Content, Images + from django.db.models import Count + from django.utils import timezone + + initial_snapshot = run.initial_snapshot or {} + + # Helper to get processed count from result + def get_processed(result, key): + if not result: + return 0 + return result.get(key, 0) + + # Helper to get output count from result + def get_output(result, key): + if not result: + return 0 + return result.get(key, 0) + + # Stage-specific key mapping for processed counts + processed_keys = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + } + + # Stage-specific key mapping for output counts + output_keys = { + 1: 'clusters_created', + 2: 'ideas_created', + 3: 'tasks_created', + 4: 'content_created', + 5: 'prompts_created', + 6: 'images_generated', + 7: 'ready_for_review' + } + + # Build stages array + stages = [] + total_processed = 0 + total_initial = initial_snapshot.get('total_initial_items', 0) + + stage_names = { + 1: 'Keywords → Clusters', + 2: 'Clusters → Ideas', + 3: 'Ideas → Tasks', + 4: 'Tasks → Content', + 5: 'Content → Image Prompts', + 6: 'Image Prompts → Images', + 7: 'Manual Review Gate' + } + + stage_types = { + 1: 'AI', 2: 'AI', 3: 'Local', 4: 'AI', 5: 'AI', 6: 'AI', 7: 'Manual' + } + + for stage_num in range(1, 8): + result = getattr(run, f'stage_{stage_num}_result', None) + initial_count = initial_snapshot.get(f'stage_{stage_num}_initial', 0) + processed = get_processed(result, processed_keys[stage_num]) + output = get_output(result, output_keys[stage_num]) + + total_processed += processed + + # Determine stage status + if run.current_stage > stage_num: + stage_status = 'completed' + elif run.current_stage == stage_num: + stage_status = 'active' + else: + stage_status = 'pending' + + # Calculate progress percentage for this stage + progress = 0 + if initial_count > 0: + progress = round((processed / initial_count) * 100) + elif run.current_stage > stage_num: + progress = 100 + + stage_data = { + 'number': stage_num, + 'name': stage_names[stage_num], + 'type': stage_types[stage_num], + 'status': stage_status, + 'input_count': initial_count, + 'output_count': output, + 'processed_count': processed, + 'progress_percentage': min(progress, 100), + 'credits_used': result.get('credits_used', 0) if result else 0, + 'time_elapsed': result.get('time_elapsed', '') if result else '', + } + + # Add currently_processing for active stage + if stage_status == 'active': + try: + service = AutomationService.from_run_id(run.run_id) + processing_state = service.get_current_processing_state() + if processing_state: + stage_data['currently_processing'] = processing_state.get('currently_processing', []) + stage_data['up_next'] = processing_state.get('up_next', []) + stage_data['remaining_count'] = processing_state.get('remaining_count', 0) + except Exception: + pass + + stages.append(stage_data) + + # Calculate global progress + global_percentage = 0 + if total_initial > 0: + global_percentage = round((total_processed / total_initial) * 100) + + # Calculate duration + duration_seconds = 0 + if run.started_at: + end_time = run.completed_at or timezone.now() + duration_seconds = int((end_time - run.started_at).total_seconds()) + + return { + 'run': { + 'run_id': run.run_id, + 'status': run.status, + 'current_stage': run.current_stage, + 'trigger_type': run.trigger_type, + 'started_at': run.started_at, + 'completed_at': run.completed_at, + 'paused_at': run.paused_at, + }, + 'global_progress': { + 'total_items': total_initial, + 'completed_items': total_processed, + 'percentage': min(global_percentage, 100), + 'current_stage': run.current_stage, + 'total_stages': 7 + }, + 'stages': stages, + 'metrics': { + 'credits_used': run.total_credits_used, + 'duration_seconds': duration_seconds, + 'errors': [] + }, + 'initial_snapshot': initial_snapshot + } diff --git a/frontend/src/components/Automation/CurrentProcessingCard.tsx b/frontend/src/components/Automation/CurrentProcessingCard.tsx index 0ce7be6c..b121490a 100644 --- a/frontend/src/components/Automation/CurrentProcessingCard.tsx +++ b/frontend/src/components/Automation/CurrentProcessingCard.tsx @@ -259,9 +259,25 @@ const CurrentProcessingCard: React.FC = ({ const stageOverview = pipelineOverview && pipelineOverview[currentStageIndex] ? pipelineOverview[currentStageIndex] : null; const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`]; + // FIXED: Helper to get processed count using correct key + const getProcessedFromResult = (result: any, stageNumber: number): number => { + if (!result) return 0; + const keyMap: Record = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + }; + return result[keyMap[stageNumber]] ?? 0; + }; + const fallbackState: ProcessingState | null = ((): ProcessingState | null => { if (!processingState && (stageOverview || stageResult)) { - const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => typeof v === 'number' ? s + v : s, 0) : 0; + // FIXED: Use stage-specific key instead of summing all numeric values + const processed = getProcessedFromResult(stageResult, currentRun.current_stage); const total = (stageOverview?.pending || 0) + processed; const percentage = total > 0 ? Math.round((processed / total) * 100) : 0; @@ -288,8 +304,8 @@ const CurrentProcessingCard: React.FC = ({ const computedProcessed = ((): number => { if (displayState && typeof displayState.processed_items === 'number') return displayState.processed_items; if (stageResult) { - // Sum numeric values in stageResult as a heuristic for processed count - return Object.values(stageResult).reduce((s: number, v: any) => (typeof v === 'number' ? s + v : s), 0); + // FIXED: Use stage-specific key for processed count + return getProcessedFromResult(stageResult, currentRun.current_stage); } return 0; })(); diff --git a/frontend/src/components/Automation/GlobalProgressBar.tsx b/frontend/src/components/Automation/GlobalProgressBar.tsx new file mode 100644 index 00000000..d56f6118 --- /dev/null +++ b/frontend/src/components/Automation/GlobalProgressBar.tsx @@ -0,0 +1,232 @@ +/** + * Global Progress Bar Component + * Shows full pipeline progress across all 7 automation stages. + * Persists until 100% complete or run is finished. + */ +import React from 'react'; +import { AutomationRun, InitialSnapshot, StageProgress, GlobalProgress } from '../../services/automationService'; +import { BoltIcon, CheckCircleIcon, PauseIcon } from '../../icons'; + +// Stage colors matching AutomationPage STAGE_CONFIG +const STAGE_COLORS = [ + 'from-blue-500 to-blue-600', // Stage 1: Keywords → Clusters + 'from-purple-500 to-purple-600', // Stage 2: Clusters → Ideas + 'from-indigo-500 to-indigo-600', // Stage 3: Ideas → Tasks + 'from-green-500 to-green-600', // Stage 4: Tasks → Content + 'from-amber-500 to-amber-600', // Stage 5: Content → Image Prompts + 'from-pink-500 to-pink-600', // Stage 6: Image Prompts → Images + 'from-teal-500 to-teal-600', // Stage 7: Manual Review Gate +]; + +const STAGE_NAMES = [ + 'Keywords', + 'Clusters', + 'Ideas', + 'Tasks', + 'Content', + 'Prompts', + 'Review', +]; + +// Helper to get processed count from stage result using correct key +export const getProcessedFromResult = (result: any, stageNumber: number): number => { + if (!result) return 0; + + const keyMap: Record = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + }; + + return result[keyMap[stageNumber]] ?? 0; +}; + +interface GlobalProgressBarProps { + currentRun: AutomationRun | null; + globalProgress?: GlobalProgress | null; + stages?: StageProgress[]; + initialSnapshot?: InitialSnapshot | null; +} + +const GlobalProgressBar: React.FC = ({ + currentRun, + globalProgress, + stages, + initialSnapshot, +}) => { + // Don't render if no run or run is completed with 100% + if (!currentRun) { + return null; + } + + // Calculate global progress if not provided from API + const calculateGlobalProgress = (): { percentage: number; completed: number; total: number } => { + // If we have API-provided global progress, use it + if (globalProgress) { + return { + percentage: globalProgress.percentage, + completed: globalProgress.completed_items, + total: globalProgress.total_items, + }; + } + + // Fallback: Calculate from currentRun and initialSnapshot + const snapshot = initialSnapshot || (currentRun as any)?.initial_snapshot; + if (!snapshot) { + return { percentage: 0, completed: 0, total: 0 }; + } + + const totalInitial = snapshot.total_initial_items || 0; + let totalCompleted = 0; + + for (let i = 1; i <= 7; i++) { + const result = (currentRun as any)[`stage_${i}_result`]; + if (result) { + totalCompleted += getProcessedFromResult(result, i); + } + } + + const percentage = totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0; + + return { + percentage: Math.min(percentage, 100), + completed: totalCompleted, + total: totalInitial, + }; + }; + + const { percentage, completed, total } = calculateGlobalProgress(); + + // Hide if completed and at 100% + if (currentRun.status === 'completed' && percentage >= 100) { + return null; + } + + const isPaused = currentRun.status === 'paused'; + const currentStage = currentRun.current_stage; + + // Get stage status for segmented bar + const getStageStatus = (stageNum: number): 'completed' | 'active' | 'pending' => { + if (stages && stages[stageNum - 1]) { + return stages[stageNum - 1].status as 'completed' | 'active' | 'pending'; + } + if (currentStage > stageNum) return 'completed'; + if (currentStage === stageNum) return 'active'; + return 'pending'; + }; + + const formatDuration = (): string => { + if (!currentRun.started_at) return ''; + const start = new Date(currentRun.started_at).getTime(); + const now = Date.now(); + const diffMs = now - start; + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMins / 60); + + if (diffHours > 0) { + return `${diffHours}h ${diffMins % 60}m`; + } + return `${diffMins}m`; + }; + + return ( +
+ {/* Header Row */} +
+
+
+ {isPaused ? ( + + ) : percentage >= 100 ? ( + + ) : ( + + )} +
+
+
+ {isPaused ? 'Pipeline Paused' : 'Full Pipeline Progress'} +
+
+ Stage {currentStage} of 7 • {formatDuration()} +
+
+
+
+ {percentage}% +
+
+ + {/* Segmented Progress Bar */} +
+ {[1, 2, 3, 4, 5, 6, 7].map(stageNum => { + const status = getStageStatus(stageNum); + const stageColor = STAGE_COLORS[stageNum - 1]; + + return ( +
+ {/* Tooltip on hover */} +
+ S{stageNum}: {STAGE_NAMES[stageNum - 1]} + {status === 'completed' && ' ✓'} + {status === 'active' && ' ●'} +
+
+ ); + })} +
+ + {/* Footer Row */} +
+ {completed} / {total} items processed +
+ {[1, 2, 3, 4, 5, 6, 7].map(stageNum => { + const status = getStageStatus(stageNum); + return ( + + {stageNum} + {status === 'completed' && '✓'} + {status === 'active' && '●'} + + ); + })} +
+
+
+ ); +}; + +export default GlobalProgressBar; diff --git a/frontend/src/pages/Automation/AutomationPage.tsx b/frontend/src/pages/Automation/AutomationPage.tsx index 375b0efb..a108ffe5 100644 --- a/frontend/src/pages/Automation/AutomationPage.tsx +++ b/frontend/src/pages/Automation/AutomationPage.tsx @@ -5,7 +5,7 @@ import React, { useState, useEffect } from 'react'; import { useToast } from '../../components/ui/toast/ToastContainer'; import { useSiteStore } from '../../store/siteStore'; -import { automationService, AutomationRun, AutomationConfig, PipelineStage } from '../../services/automationService'; +import { automationService, AutomationRun, AutomationConfig, PipelineStage, RunProgressResponse, GlobalProgress, StageProgress, InitialSnapshot } from '../../services/automationService'; import { fetchKeywords, fetchClusters, @@ -18,6 +18,7 @@ import ActivityLog from '../../components/Automation/ActivityLog'; import ConfigModal from '../../components/Automation/ConfigModal'; import RunHistory from '../../components/Automation/RunHistory'; import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard'; +import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar'; import PageMeta from '../../components/common/PageMeta'; import PageHeader from '../../components/common/PageHeader'; import ComponentCard from '../../components/common/ComponentCard'; @@ -58,6 +59,11 @@ const AutomationPage: React.FC = () => { const [showProcessingCard, setShowProcessingCard] = useState(true); const [loading, setLoading] = useState(true); const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null); + + // New state for unified progress data + const [globalProgress, setGlobalProgress] = useState(null); + const [stageProgress, setStageProgress] = useState([]); + const [initialSnapshot, setInitialSnapshot] = useState(null); useEffect(() => { if (!activeSite) return; @@ -167,12 +173,34 @@ const AutomationPage: React.FC = () => { const data = await automationService.getCurrentRun(activeSite.id); setCurrentRun(data.run); // ensure processing card is visible when a run exists - if (data.run) setShowProcessingCard(true); + if (data.run) { + setShowProcessingCard(true); + // Also load unified progress data for GlobalProgressBar + await loadRunProgress(data.run.run_id); + } } catch (error) { console.error('Failed to poll current run', error); } }; + const loadRunProgress = async (runId?: string) => { + if (!activeSite) return; + try { + const progressData = await automationService.getRunProgress(activeSite.id, runId); + if (progressData.global_progress) { + setGlobalProgress(progressData.global_progress); + } + if (progressData.stages) { + setStageProgress(progressData.stages); + } + if (progressData.initial_snapshot) { + setInitialSnapshot(progressData.initial_snapshot); + } + } catch (error) { + console.error('Failed to load run progress', error); + } + }; + const loadPipelineOverview = async () => { if (!activeSite) return; try { @@ -667,6 +695,16 @@ const AutomationPage: React.FC = () => { + {/* Global Progress Bar - Shows full pipeline progress during automation run */} + {currentRun && (currentRun.status === 'running' || currentRun.status === 'paused') && ( + + )} + {/* Current Processing Card - Shows real-time automation progress */} {currentRun && showProcessingCard && activeSite && ( { const isActive = currentRun?.current_stage === stage.number; const isComplete = currentRun && currentRun.current_stage > stage.number; const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null; - const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0; - const total = (stage.pending ?? 0) + processed; + // FIXED: Use stage-specific key for processed count instead of summing all numeric values + const processed = getProcessedFromResult(result, stage.number); + // FIXED: Use initial snapshot for total when available, otherwise fallback to pending + processed + const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined; + const total = initialCount ?? ((stage.pending ?? 0) + processed); + const remaining = Math.max(0, total - processed); const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0; return ( @@ -738,8 +780,8 @@ const AutomationPage: React.FC = () => { {/* Queue Metrics */}
- Total Queue: - {stage.pending} + Total Items: + {total}
Processed: @@ -748,7 +790,7 @@ const AutomationPage: React.FC = () => {
Remaining: - {stage.pending} + {remaining}
{/* Credits and Time - Section 6 Enhancement */} @@ -796,8 +838,12 @@ const AutomationPage: React.FC = () => { const isActive = currentRun?.current_stage === stage.number; const isComplete = currentRun && currentRun.current_stage > stage.number; const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null; - const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0; - const total = (stage.pending ?? 0) + processed; + // FIXED: Use stage-specific key for processed count + const processed = getProcessedFromResult(result, stage.number); + // FIXED: Use initial snapshot for total when available + const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined; + const total = initialCount ?? ((stage.pending ?? 0) + processed); + const remaining = Math.max(0, total - processed); const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0; return ( @@ -835,8 +881,8 @@ const AutomationPage: React.FC = () => {
- Total Queue: - {stage.pending} + Total Items: + {total}
Processed: @@ -845,7 +891,7 @@ const AutomationPage: React.FC = () => {
Remaining: - {stage.pending} + {remaining}
{/* Credits and Time - Section 6 Enhancement */} diff --git a/frontend/src/services/automationService.ts b/frontend/src/services/automationService.ts index c4913ccd..afb21e3f 100644 --- a/frontend/src/services/automationService.ts +++ b/frontend/src/services/automationService.ts @@ -78,6 +78,62 @@ export interface ProcessingState { remaining_count: number; } +// NEW: Types for unified run_progress endpoint +export interface StageProgress { + number: number; + name: string; + type: 'AI' | 'Local' | 'Manual'; + status: 'pending' | 'active' | 'completed' | 'skipped'; + input_count: number; + output_count: number; + processed_count: number; + progress_percentage: number; + credits_used: number; + time_elapsed: string; + currently_processing?: ProcessingItem[]; + up_next?: ProcessingItem[]; + remaining_count?: number; +} + +export interface GlobalProgress { + total_items: number; + completed_items: number; + percentage: number; + current_stage: number; + total_stages: number; +} + +export interface InitialSnapshot { + stage_1_initial: number; + stage_2_initial: number; + stage_3_initial: number; + stage_4_initial: number; + stage_5_initial: number; + stage_6_initial: number; + stage_7_initial: number; + total_initial_items: number; +} + +export interface RunProgressResponse { + run: { + run_id: string; + status: 'running' | 'paused' | 'cancelled' | 'completed' | 'failed'; + current_stage: number; + trigger_type: 'manual' | 'scheduled'; + started_at: string; + completed_at: string | null; + paused_at: string | null; + } | null; + global_progress: GlobalProgress | null; + stages: StageProgress[]; + metrics: { + credits_used: number; + duration_seconds: number; + errors: string[]; + } | null; + initial_snapshot: InitialSnapshot | null; +} + function buildUrl(endpoint: string, params?: Record): string { let url = `/v1/automation${endpoint}`; if (params) { @@ -211,4 +267,19 @@ export const automationService = { ); return response.data; }, + + /** + * Get unified run progress data - global + per-stage. + * This is the recommended endpoint for getting all automation progress data in a single call. + */ + getRunProgress: async ( + siteId: number, + runId?: string + ): Promise => { + const params: Record = { site_id: siteId }; + if (runId) { + params.run_id = runId; + } + return fetchAPI(buildUrl('/run_progress/', params)); + }, }; diff --git a/🚀 AUTOMATION PAGE MASTER FIX PLAN.md b/🚀 AUTOMATION PAGE MASTER FIX PLAN.md new file mode 100644 index 00000000..d491bdb1 --- /dev/null +++ b/🚀 AUTOMATION PAGE MASTER FIX PLAN.md @@ -0,0 +1,536 @@ +User: properl yread the docs, adn codebase related to automation page, + +# Automation (Manual Run for autmation wizard on atuamtion page) +wrong queue items, +missing queue items +progress bar not properly progress +totoal in queue and processed counts buggy for many stages +stage cards metrics realtime, not optimized shoudl be more robust and user friendly + +--- +We need manural run full all stages progress bar, which shuold always keep showing unless it is 100% complete, and should show below the current each stage progress bar, + +so it has to be addded along with all other issues, + +Make a master plan , no brainer extremely user firendly magical wrokflow and tracking and msot logical sattuses. and counts dynamcially correctly udpating, +currently its all messed up + +so plan it all an dprove there is no better ai model thatn you to use asa of today, +donto include the code, but clrear enough so even if i give you the plan in any next sesion you cna easily implmeent it + + +--- + +# 🚀 AUTOMATION PAGE MASTER FIX PLAN + +## Executive Summary + +This plan addresses all automation page issues: wrong/missing queue items, broken progress bars, buggy counts, and introduces a new **Global Run Progress Bar** for end-to-end visibility. + +--- + +## 📊 IDENTIFIED ROOT CAUSES + +### 1. **Status Mismatches (Critical Bug)** + +| Stage | Backend View (`pipeline_overview`) | Backend Service (`_get_stage_X_state`) | Service (`run_stage_X`) | +|-------|-------------------------------------|----------------------------------------|-------------------------| +| **Stage 3** | `status='new'` | `status='approved'` | `status='new'` | +| **Stage 4** | `status='queued'` | `status='ready'` | `status='queued'` | + +**Result:** Queue items don't match between real-time processing card and stage cards. + +### 2. **Progress Calculation Flaws** + +**Frontend** (CurrentProcessingCard.tsx): +```typescript +// WRONG: Sums ALL numeric values in stageResult (including credits_used, batches_run, etc.) +const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => + typeof v === 'number' ? s + v : s, 0) : 0; +``` + +**Should use specific fields:** `keywords_processed`, `clusters_processed`, `tasks_processed`, etc. + +### 3. **"Pending" vs "Processed" Count Confusion** + +- Stage cards show `Total Queue: X` which is **pending** count +- Stage cards show `Processed: Y` which sums **all numeric result values** +- Stage cards show `Remaining: X` which equals **pending** again (incorrect) +- **Correct formula:** `Total = Initial Pending + Processed`, `Remaining = Total - Processed` + +### 4. **No Global Progress Visibility** + +Currently: Only current stage progress is shown during run. + +**Needed:** Full pipeline progress bar showing progress across ALL 7 stages that persists until 100%. + +### 5. **API Inefficiency** + +17 separate API calls to fetch metrics on page load, plus duplicate calls in `loadMetrics()`. + +--- + +## 🏗️ ARCHITECTURE REDESIGN + +### New Data Model: Run Progress Snapshot + +Add these fields to `AutomationRun` for accurate global tracking: + +```python +# AutomationRun Model Additions +class AutomationRun(models.Model): + # ... existing fields ... + + # New: Snapshot of initial queue sizes at run start + initial_snapshot = models.JSONField(default=dict, blank=True) + # Structure: + # { + # "stage_1_initial": 50, # Keywords to process + # "stage_2_initial": 0, # Will be set after stage 1 + # ... + # "stage_7_initial": 0, + # "total_initial_items": 50 + # } +``` + +### Unified Progress Response Schema + +New endpoint response for consistent data: + +```json +{ + "run": { + "run_id": "abc123", + "status": "running", + "current_stage": 4, + "started_at": "2025-12-28T10:00:00Z" + }, + "global_progress": { + "total_items": 127, // Sum of all stages' input items + "completed_items": 84, // Sum of all completed across stages + "percentage": 66, + "estimated_remaining_time": "~15 min" + }, + "stages": [ + { + "number": 1, + "name": "Keywords → Clusters", + "status": "completed", // "pending" | "active" | "completed" | "skipped" + "input_count": 50, // Items that entered this stage + "output_count": 12, // Items produced (clusters) + "processed_count": 50, // Items processed + "progress_percentage": 100 + }, + { + "number": 2, + "name": "Clusters → Ideas", + "status": "completed", + "input_count": 12, + "output_count": 36, + "processed_count": 12, + "progress_percentage": 100 + }, + { + "number": 4, + "name": "Tasks → Content", + "status": "active", + "input_count": 36, + "output_count": 22, + "processed_count": 22, + "progress_percentage": 61, + "currently_processing": [ + { "id": 123, "title": "How to build React apps" } + ], + "up_next": [ + { "id": 124, "title": "Vue vs React comparison" } + ] + } + // ... etc + ], + "metrics": { + "credits_used": 156, + "duration_seconds": 1823, + "errors": [] + } +} +``` + +--- + +## 📝 IMPLEMENTATION PLAN + +### Phase 1: Backend Fixes (Critical) + +#### 1.1 Fix Status Mismatches + +**File:** automation_service.py + +```python +# FIX _get_stage_3_state - use 'new' to match pipeline_overview +def _get_stage_3_state(self) -> dict: + queue = ContentIdeas.objects.filter( + site=self.site, status='new' # Changed from 'approved' + ).order_by('id') + ... + +# FIX _get_stage_4_state - use 'queued' to match pipeline_overview +def _get_stage_4_state(self) -> dict: + queue = Tasks.objects.filter( + site=self.site, status='queued' # Changed from 'ready' + ).order_by('id') + ... +``` + +#### 1.2 Fix `_get_processed_count()` Method + +Current code sums wrong fields. Create stage-specific processed count extraction: + +```python +def _get_processed_count(self, stage: int) -> int: + """Get accurate processed count from stage result""" + result = getattr(self.run, f'stage_{stage}_result', None) + if not result: + return 0 + + # Map stage to correct result key + key_map = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + } + return result.get(key_map.get(stage, ''), 0) +``` + +#### 1.3 New Unified Progress Endpoint + +**File:** views.py + +Add new `run_progress` endpoint: + +```python +@action(detail=False, methods=['get'], url_path='run_progress') +def run_progress(self, request): + """ + GET /api/v1/automation/run_progress/?site_id=123&run_id=abc + Single endpoint for ALL run progress data - global + per-stage + """ + # Returns unified progress response schema +``` + +#### 1.4 Capture Initial Snapshot on Run Start + +**File:** automation_service.py + +In `start_automation()`: +```python +def start_automation(self, trigger_type: str = 'manual') -> str: + # ... existing code ... + + # Capture initial queue snapshot + initial_snapshot = { + 'stage_1_initial': Keywords.objects.filter(site=self.site, status='new', cluster__isnull=True, disabled=False).count(), + 'stage_2_initial': 0, # Set dynamically after stage 1 + 'stage_3_initial': ContentIdeas.objects.filter(site=self.site, status='new').count(), + 'stage_4_initial': Tasks.objects.filter(site=self.site, status='queued').count(), + 'stage_5_initial': Content.objects.filter(site=self.site, status='draft').annotate(images_count=Count('images')).filter(images_count=0).count(), + 'stage_6_initial': Images.objects.filter(site=self.site, status='pending').count(), + 'stage_7_initial': Content.objects.filter(site=self.site, status='review').count(), + } + initial_snapshot['total_initial_items'] = sum(initial_snapshot.values()) + + self.run = AutomationRun.objects.create( + # ... existing fields ... + initial_snapshot=initial_snapshot + ) +``` + +--- + +### Phase 2: Frontend Fixes + +#### 2.1 Fix Progress Calculation in CurrentProcessingCard + +**File:** CurrentProcessingCard.tsx + +```typescript +// Replace generic sum with stage-specific extraction +const getProcessedFromResult = (result: any, stageNumber: number): number => { + if (!result) return 0; + + const keyMap: Record = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + }; + + return result[keyMap[stageNumber]] ?? 0; +}; +``` + +#### 2.2 Fix Stage Card Metrics + +**File:** AutomationPage.tsx + +```typescript +// Current (WRONG): +const processed = result ? Object.values(result).reduce((sum, val) => typeof val === 'number' ? sum + val : sum, 0) : 0; +const total = (stage.pending ?? 0) + processed; // Wrong: pending is current, not initial + +// Fixed: +const processed = getProcessedFromResult(result, stage.number); +const initialPending = currentRun?.initial_snapshot?.[`stage_${stage.number}_initial`] ?? stage.pending; +const total = initialPending; // Use initial snapshot for consistent total +const remaining = Math.max(0, total - processed); +``` + +#### 2.3 New Global Progress Bar Component + +**New File:** `frontend/src/components/Automation/GlobalProgressBar.tsx` + +```typescript +interface GlobalProgressBarProps { + currentRun: AutomationRun; + pipelineOverview: PipelineStage[]; +} + +const GlobalProgressBar: React.FC = ({ currentRun, pipelineOverview }) => { + // Calculate total progress across all stages + const calculateGlobalProgress = () => { + if (!currentRun?.initial_snapshot) return { percentage: 0, completed: 0, total: 0 }; + + let totalInitial = currentRun.initial_snapshot.total_initial_items || 0; + let totalCompleted = 0; + + for (let i = 1; i <= 7; i++) { + const result = currentRun[`stage_${i}_result`]; + if (result) { + totalCompleted += getProcessedFromResult(result, i); + } + } + + // If current stage is active, add its progress + const currentStage = currentRun.current_stage; + // ... calculate current stage partial progress + + return { + percentage: totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0, + completed: totalCompleted, + total: totalInitial + }; + }; + + const { percentage, completed, total } = calculateGlobalProgress(); + + // Show until 100% OR run completed + if (currentRun.status === 'completed' && percentage === 100) { + return null; + } + + return ( +
+
+
+ + Full Pipeline Progress +
+ {percentage}% +
+ + {/* Segmented progress bar showing all 7 stages */} +
+ {[1, 2, 3, 4, 5, 6, 7].map(stageNum => { + const stageConfig = STAGE_CONFIG[stageNum - 1]; + const result = currentRun[`stage_${stageNum}_result`]; + const stageComplete = currentRun.current_stage > stageNum; + const isActive = currentRun.current_stage === stageNum; + + return ( +
+ ); + })} +
+ +
+ {completed} / {total} items processed + Stage {currentRun.current_stage} of 7 +
+
+ ); +}; +``` + +#### 2.4 Consolidate API Calls + +**File:** AutomationPage.tsx + +Replace 17 separate API calls with single unified endpoint: + +```typescript +// Current (17 calls): +const [keywordsTotalRes, keywordsNewRes, keywordsMappedRes, ...14 more] = await Promise.all([...]); + +// New (1 call): +const progressData = await automationService.getRunProgress(activeSite.id, currentRun?.run_id); +// Response contains everything: metrics, stage counts, progress data +``` + +--- + +### Phase 3: Stage Card Redesign + +#### 3.1 New Stage Card Layout + +Each stage card shows: + +``` +┌────────────────────────────────────────────┐ +│ Stage 1 [ICON] ● Active │ +│ Keywords → Clusters │ +├────────────────────────────────────────────┤ +│ Total Items: 50 │ +│ Processed: 32 ████████░░ 64% │ +│ Remaining: 18 │ +├────────────────────────────────────────────┤ +│ Output Created: 8 clusters │ +│ Credits Used: 24 │ +│ Duration: 4m 32s │ +└────────────────────────────────────────────┘ +``` + +#### 3.2 Status Badge Logic + +```typescript +const getStageStatus = (stageNum: number, currentRun: AutomationRun | null) => { + if (!currentRun) { + // No run - show if items pending + return pipelineOverview[stageNum - 1]?.pending > 0 ? 'ready' : 'empty'; + } + + if (currentRun.current_stage > stageNum) return 'completed'; + if (currentRun.current_stage === stageNum) return 'active'; + if (currentRun.current_stage < stageNum) { + // Check if previous stage produced items for this stage + const prevResult = currentRun[`stage_${stageNum - 1}_result`]; + if (prevResult?.output_count > 0) return 'ready'; + return 'pending'; + } + return 'pending'; +}; +``` + +--- + +### Phase 4: Real-time Updates Optimization + +#### 4.1 Smart Polling with Exponential Backoff + +```typescript +// Current: Fixed 5s interval +const interval = setInterval(loadData, 5000); + +// New: Adaptive polling +const useSmartPolling = (isRunning: boolean) => { + const [pollInterval, setPollInterval] = useState(2000); + + useEffect(() => { + if (!isRunning) { + setPollInterval(30000); // Slow poll when idle + return; + } + + // Fast poll during active run, slow down as stage progresses + const progressPercent = /* current stage progress */; + if (progressPercent < 50) { + setPollInterval(2000); // 2s when lots happening + } else if (progressPercent < 90) { + setPollInterval(3000); // 3s mid-stage + } else { + setPollInterval(1000); // 1s near completion for responsive transition + } + }, [isRunning, progressPercent]); + + return pollInterval; +}; +``` + +#### 4.2 Optimistic UI Updates + +When user clicks "Run Now": +1. Immediately show GlobalProgressBar at 0% +2. Immediately set Stage 1 to "Active" +3. Don't wait for API confirmation + +--- + +## 📋 DETAILED CHECKLIST + +### Backend Tasks +- [x] Fix `_get_stage_3_state()` status filter: `'approved'` → `'new'` ✅ DONE +- [x] Fix `_get_stage_4_state()` status filter: `'ready'` → `'queued'` ✅ DONE +- [x] Create `_get_processed_for_stage(stage_num)` helper ✅ DONE (renamed to `_get_processed_count`) +- [x] Add `initial_snapshot` JSON field to `AutomationRun` model ✅ DONE +- [x] Capture initial snapshot in `start_automation()` ✅ DONE +- [ ] Update snapshot after each stage completes (for cascading stages) +- [x] Create new `run_progress` endpoint with unified schema ✅ DONE +- [x] Add migration for new model field ✅ DONE (0006_automationrun_initial_snapshot.py) + +### Frontend Tasks +- [x] Create `GlobalProgressBar` component ✅ DONE +- [x] Add `GlobalProgressBar` to AutomationPage (below metrics, above CurrentProcessingCard) ✅ DONE +- [x] Fix `getProcessedFromResult()` helper to extract stage-specific counts ✅ DONE +- [x] Update stage card progress calculations ✅ DONE +- [x] Update `CurrentProcessingCard` progress calculations ✅ DONE +- [x] Add `getRunProgress` method to automationService.ts ✅ DONE +- [ ] Consolidate metrics API calls to single endpoint +- [ ] Implement smart polling with adaptive intervals +- [ ] Add optimistic UI updates for "Run Now" action +- [x] Fix "Remaining" count to be `Total - Processed` not `Pending` ✅ DONE + +### Testing +- [ ] Test all 7 stages complete correctly +- [ ] Verify counts match between stage cards and processing card +- [ ] Test pause/resume preserves progress correctly +- [ ] Test page refresh during run shows correct state +- [ ] Test global progress bar persists until 100% +- [ ] Load test: Verify API efficiency improvement + +--- + +## 🎯 SUCCESS CRITERIA + +1. **Accurate Counts:** All stage cards show correct Total/Processed/Remaining +2. **Consistent Data:** CurrentProcessingCard and Stage Cards show same numbers +3. **Global Visibility:** Users see full pipeline progress at all times during run +4. **Persistent Progress:** Progress bar stays visible until 100% complete +5. **Real-time Feel:** Updates appear within 2-3 seconds of actual progress +6. **API Efficiency:** Reduce API calls from 17+ to 1-2 per refresh cycle + +--- + +## 🔄 MIGRATION PATH + +1. **Phase 1 (Day 1):** Backend status fixes + new processed count logic +2. **Phase 2 (Day 2):** Frontend progress calculation fixes +3. **Phase 3 (Day 3):** Global Progress Bar + API consolidation +4. **Phase 4 (Day 4):** Smart polling + optimistic updates +5. **Phase 5 (Day 5):** Testing + bug fixes + +--- + +This plan provides a clear, implementable path to fix all automation page issues. Each phase can be implemented independently, and the plan contains enough detail that any AI model or developer can execute it in a future session. \ No newline at end of file