Automation revamp part 1
This commit is contained in:
IGNY8 VPS (Salman)
@@ -0,0 +1,22 @@
|
||||
# Generated migration for adding initial_snapshot field to AutomationRun
|
||||
|
||||
from django.db import migrations, models
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('automation', '0005_add_default_image_service'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AddField(
|
||||
model_name='automationrun',
|
||||
name='initial_snapshot',
|
||||
field=models.JSONField(
|
||||
blank=True,
|
||||
default=dict,
|
||||
help_text='Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}'
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -88,6 +88,13 @@ class AutomationRun(models.Model):
|
||||
|
||||
total_credits_used = models.IntegerField(default=0)
|
||||
|
||||
# Initial queue snapshot - captured at run start for accurate progress tracking
|
||||
initial_snapshot = models.JSONField(
|
||||
default=dict,
|
||||
blank=True,
|
||||
help_text="Snapshot of initial queue sizes: {stage_1_initial, stage_2_initial, ..., total_initial_items}"
|
||||
)
|
||||
|
||||
# JSON results per stage
|
||||
stage_1_result = models.JSONField(null=True, blank=True, help_text="{keywords_processed, clusters_created, batches}")
|
||||
stage_2_result = models.JSONField(null=True, blank=True, help_text="{clusters_processed, ideas_created}")
|
||||
|
||||
@@ -109,6 +109,9 @@ class AutomationService:
|
||||
# Create run_id and log files
|
||||
run_id = self.logger.start_run(self.account.id, self.site.id, trigger_type)
|
||||
|
||||
# Capture initial queue snapshot for accurate progress tracking
|
||||
initial_snapshot = self._capture_initial_snapshot()
|
||||
|
||||
# Create AutomationRun record
|
||||
self.run = AutomationRun.objects.create(
|
||||
run_id=run_id,
|
||||
@@ -117,6 +120,7 @@ class AutomationService:
|
||||
trigger_type=trigger_type,
|
||||
status='running',
|
||||
current_stage=1,
|
||||
initial_snapshot=initial_snapshot,
|
||||
)
|
||||
|
||||
# Log start
|
||||
@@ -124,6 +128,10 @@ class AutomationService:
|
||||
run_id, self.account.id, self.site.id, 0,
|
||||
f"Automation started (trigger: {trigger_type})"
|
||||
)
|
||||
self.logger.log_stage_progress(
|
||||
run_id, self.account.id, self.site.id, 0,
|
||||
f"Initial snapshot captured: {initial_snapshot['total_initial_items']} total items across all stages"
|
||||
)
|
||||
self.logger.log_stage_progress(
|
||||
run_id, self.account.id, self.site.id, 0,
|
||||
f"Credit check: Account has {self.account.credits} credits, estimated need: {estimated_credits} credits"
|
||||
@@ -1361,6 +1369,61 @@ class AutomationService:
|
||||
logger.info(f"[AutomationService] Estimated credits: {total}")
|
||||
return total
|
||||
|
||||
def _capture_initial_snapshot(self) -> dict:
|
||||
"""
|
||||
Capture initial queue sizes at run start for accurate progress tracking.
|
||||
This snapshot is used to calculate global progress percentage correctly.
|
||||
"""
|
||||
# Stage 1: Keywords pending clustering
|
||||
stage_1_initial = Keywords.objects.filter(
|
||||
site=self.site, status='new', cluster__isnull=True, disabled=False
|
||||
).count()
|
||||
|
||||
# Stage 2: Clusters needing ideas
|
||||
stage_2_initial = Clusters.objects.filter(
|
||||
site=self.site, status='new', disabled=False
|
||||
).exclude(ideas__isnull=False).count()
|
||||
|
||||
# Stage 3: Ideas ready to be converted to tasks
|
||||
stage_3_initial = ContentIdeas.objects.filter(
|
||||
site=self.site, status='new'
|
||||
).count()
|
||||
|
||||
# Stage 4: Tasks ready for content generation
|
||||
stage_4_initial = Tasks.objects.filter(
|
||||
site=self.site, status='queued'
|
||||
).count()
|
||||
|
||||
# Stage 5: Content needing image prompts
|
||||
stage_5_initial = Content.objects.filter(
|
||||
site=self.site, status='draft'
|
||||
).annotate(images_count=Count('images')).filter(images_count=0).count()
|
||||
|
||||
# Stage 6: Image prompts pending generation
|
||||
stage_6_initial = Images.objects.filter(
|
||||
site=self.site, status='pending'
|
||||
).count()
|
||||
|
||||
# Stage 7: Content ready for review
|
||||
stage_7_initial = Content.objects.filter(
|
||||
site=self.site, status='review'
|
||||
).count()
|
||||
|
||||
snapshot = {
|
||||
'stage_1_initial': stage_1_initial,
|
||||
'stage_2_initial': stage_2_initial,
|
||||
'stage_3_initial': stage_3_initial,
|
||||
'stage_4_initial': stage_4_initial,
|
||||
'stage_5_initial': stage_5_initial,
|
||||
'stage_6_initial': stage_6_initial,
|
||||
'stage_7_initial': stage_7_initial,
|
||||
'total_initial_items': stage_1_initial + stage_2_initial + stage_3_initial +
|
||||
stage_4_initial + stage_5_initial + stage_6_initial + stage_7_initial,
|
||||
}
|
||||
|
||||
logger.info(f"[AutomationService] Initial snapshot captured: {snapshot}")
|
||||
return snapshot
|
||||
|
||||
# Helper methods
|
||||
|
||||
def _wait_for_task(self, task_id: str, stage_number: int, item_name: str, continue_on_error: bool = True):
|
||||
@@ -1559,7 +1622,7 @@ class AutomationService:
|
||||
def _get_stage_3_state(self) -> dict:
|
||||
"""Get processing state for Stage 3: Ideas → Tasks"""
|
||||
queue = ContentIdeas.objects.filter(
|
||||
site=self.site, status='approved'
|
||||
site=self.site, status='new' # Fixed: Match pipeline_overview status
|
||||
).order_by('id')
|
||||
|
||||
processed = self._get_processed_count(3)
|
||||
@@ -1580,7 +1643,7 @@ class AutomationService:
|
||||
def _get_stage_4_state(self) -> dict:
|
||||
"""Get processing state for Stage 4: Tasks → Content"""
|
||||
queue = Tasks.objects.filter(
|
||||
site=self.site, status='ready'
|
||||
site=self.site, status='queued' # Fixed: Match pipeline_overview status
|
||||
).order_by('id')
|
||||
|
||||
processed = self._get_processed_count(4)
|
||||
@@ -1666,51 +1729,30 @@ class AutomationService:
|
||||
}
|
||||
|
||||
def _get_processed_count(self, stage: int) -> int:
|
||||
"""Get count of items processed in current stage during this run"""
|
||||
"""
|
||||
Get accurate processed count from stage result.
|
||||
Uses stage-specific keys for correct counting instead of DB queries.
|
||||
"""
|
||||
if not self.run:
|
||||
return 0
|
||||
|
||||
# Count items that were updated during this run and changed status from pending
|
||||
if stage == 1:
|
||||
# Keywords that changed status from 'new' during this run
|
||||
return Keywords.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at
|
||||
).exclude(status='new').count()
|
||||
elif stage == 2:
|
||||
# Clusters that changed status from 'new' during this run
|
||||
return Clusters.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at
|
||||
).exclude(status='new').count()
|
||||
elif stage == 3:
|
||||
# Ideas that changed status from 'approved' during this run
|
||||
return ContentIdeas.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at
|
||||
).exclude(status='approved').count()
|
||||
elif stage == 4:
|
||||
# Tasks that changed status from 'ready'/'queued' during this run
|
||||
return Tasks.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at
|
||||
).exclude(status__in=['ready', 'queued']).count()
|
||||
elif stage == 5:
|
||||
# Content processed for image prompts during this run
|
||||
return Content.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at,
|
||||
images__isnull=False
|
||||
).distinct().count()
|
||||
elif stage == 6:
|
||||
# Images completed during this run
|
||||
return Images.objects.filter(
|
||||
site=self.site,
|
||||
updated_at__gte=self.run.started_at,
|
||||
status='completed'
|
||||
).count()
|
||||
# Get the stage result from the run
|
||||
result = getattr(self.run, f'stage_{stage}_result', None)
|
||||
if not result:
|
||||
return 0
|
||||
|
||||
return 0
|
||||
# Map stage to correct result key for processed count
|
||||
key_map = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
}
|
||||
|
||||
return result.get(key_map.get(stage, ''), 0)
|
||||
|
||||
def _get_current_items(self, queryset, count: int) -> list:
|
||||
"""Get currently processing items"""
|
||||
|
||||
@@ -714,3 +714,210 @@ class AutomationViewSet(viewsets.ViewSet):
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
@extend_schema(tags=['Automation'])
|
||||
@action(detail=False, methods=['get'], url_path='run_progress')
|
||||
def run_progress(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/run_progress/?site_id=123&run_id=abc
|
||||
|
||||
Unified endpoint for ALL run progress data - global + per-stage.
|
||||
Replaces multiple separate API calls with single comprehensive response.
|
||||
|
||||
Response includes:
|
||||
- run: Current run status and metadata
|
||||
- global_progress: Overall pipeline progress percentage
|
||||
- stages: Per-stage progress with input/output/processed counts
|
||||
- metrics: Credits used, duration, errors
|
||||
"""
|
||||
site_id = request.query_params.get('site_id')
|
||||
run_id = request.query_params.get('run_id')
|
||||
|
||||
if not site_id:
|
||||
return Response(
|
||||
{'error': 'site_id required'},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
try:
|
||||
site = get_object_or_404(Site, id=site_id, account=request.user.account)
|
||||
|
||||
# If no run_id, get current run
|
||||
if run_id:
|
||||
run = AutomationRun.objects.get(run_id=run_id, site=site)
|
||||
else:
|
||||
run = AutomationRun.objects.filter(
|
||||
site=site,
|
||||
status__in=['running', 'paused']
|
||||
).order_by('-started_at').first()
|
||||
|
||||
if not run:
|
||||
return Response({
|
||||
'run': None,
|
||||
'global_progress': None,
|
||||
'stages': [],
|
||||
'metrics': None
|
||||
})
|
||||
|
||||
# Build unified response
|
||||
response = self._build_run_progress_response(site, run)
|
||||
return Response(response)
|
||||
|
||||
except AutomationRun.DoesNotExist:
|
||||
return Response(
|
||||
{'error': 'Run not found'},
|
||||
status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
except Exception as e:
|
||||
return Response(
|
||||
{'error': str(e)},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
def _build_run_progress_response(self, site, run):
|
||||
"""Build comprehensive progress response for a run"""
|
||||
from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas
|
||||
from igny8_core.business.content.models import Tasks, Content, Images
|
||||
from django.db.models import Count
|
||||
from django.utils import timezone
|
||||
|
||||
initial_snapshot = run.initial_snapshot or {}
|
||||
|
||||
# Helper to get processed count from result
|
||||
def get_processed(result, key):
|
||||
if not result:
|
||||
return 0
|
||||
return result.get(key, 0)
|
||||
|
||||
# Helper to get output count from result
|
||||
def get_output(result, key):
|
||||
if not result:
|
||||
return 0
|
||||
return result.get(key, 0)
|
||||
|
||||
# Stage-specific key mapping for processed counts
|
||||
processed_keys = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
}
|
||||
|
||||
# Stage-specific key mapping for output counts
|
||||
output_keys = {
|
||||
1: 'clusters_created',
|
||||
2: 'ideas_created',
|
||||
3: 'tasks_created',
|
||||
4: 'content_created',
|
||||
5: 'prompts_created',
|
||||
6: 'images_generated',
|
||||
7: 'ready_for_review'
|
||||
}
|
||||
|
||||
# Build stages array
|
||||
stages = []
|
||||
total_processed = 0
|
||||
total_initial = initial_snapshot.get('total_initial_items', 0)
|
||||
|
||||
stage_names = {
|
||||
1: 'Keywords → Clusters',
|
||||
2: 'Clusters → Ideas',
|
||||
3: 'Ideas → Tasks',
|
||||
4: 'Tasks → Content',
|
||||
5: 'Content → Image Prompts',
|
||||
6: 'Image Prompts → Images',
|
||||
7: 'Manual Review Gate'
|
||||
}
|
||||
|
||||
stage_types = {
|
||||
1: 'AI', 2: 'AI', 3: 'Local', 4: 'AI', 5: 'AI', 6: 'AI', 7: 'Manual'
|
||||
}
|
||||
|
||||
for stage_num in range(1, 8):
|
||||
result = getattr(run, f'stage_{stage_num}_result', None)
|
||||
initial_count = initial_snapshot.get(f'stage_{stage_num}_initial', 0)
|
||||
processed = get_processed(result, processed_keys[stage_num])
|
||||
output = get_output(result, output_keys[stage_num])
|
||||
|
||||
total_processed += processed
|
||||
|
||||
# Determine stage status
|
||||
if run.current_stage > stage_num:
|
||||
stage_status = 'completed'
|
||||
elif run.current_stage == stage_num:
|
||||
stage_status = 'active'
|
||||
else:
|
||||
stage_status = 'pending'
|
||||
|
||||
# Calculate progress percentage for this stage
|
||||
progress = 0
|
||||
if initial_count > 0:
|
||||
progress = round((processed / initial_count) * 100)
|
||||
elif run.current_stage > stage_num:
|
||||
progress = 100
|
||||
|
||||
stage_data = {
|
||||
'number': stage_num,
|
||||
'name': stage_names[stage_num],
|
||||
'type': stage_types[stage_num],
|
||||
'status': stage_status,
|
||||
'input_count': initial_count,
|
||||
'output_count': output,
|
||||
'processed_count': processed,
|
||||
'progress_percentage': min(progress, 100),
|
||||
'credits_used': result.get('credits_used', 0) if result else 0,
|
||||
'time_elapsed': result.get('time_elapsed', '') if result else '',
|
||||
}
|
||||
|
||||
# Add currently_processing for active stage
|
||||
if stage_status == 'active':
|
||||
try:
|
||||
service = AutomationService.from_run_id(run.run_id)
|
||||
processing_state = service.get_current_processing_state()
|
||||
if processing_state:
|
||||
stage_data['currently_processing'] = processing_state.get('currently_processing', [])
|
||||
stage_data['up_next'] = processing_state.get('up_next', [])
|
||||
stage_data['remaining_count'] = processing_state.get('remaining_count', 0)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
stages.append(stage_data)
|
||||
|
||||
# Calculate global progress
|
||||
global_percentage = 0
|
||||
if total_initial > 0:
|
||||
global_percentage = round((total_processed / total_initial) * 100)
|
||||
|
||||
# Calculate duration
|
||||
duration_seconds = 0
|
||||
if run.started_at:
|
||||
end_time = run.completed_at or timezone.now()
|
||||
duration_seconds = int((end_time - run.started_at).total_seconds())
|
||||
|
||||
return {
|
||||
'run': {
|
||||
'run_id': run.run_id,
|
||||
'status': run.status,
|
||||
'current_stage': run.current_stage,
|
||||
'trigger_type': run.trigger_type,
|
||||
'started_at': run.started_at,
|
||||
'completed_at': run.completed_at,
|
||||
'paused_at': run.paused_at,
|
||||
},
|
||||
'global_progress': {
|
||||
'total_items': total_initial,
|
||||
'completed_items': total_processed,
|
||||
'percentage': min(global_percentage, 100),
|
||||
'current_stage': run.current_stage,
|
||||
'total_stages': 7
|
||||
},
|
||||
'stages': stages,
|
||||
'metrics': {
|
||||
'credits_used': run.total_credits_used,
|
||||
'duration_seconds': duration_seconds,
|
||||
'errors': []
|
||||
},
|
||||
'initial_snapshot': initial_snapshot
|
||||
}
|
||||
|
||||
@@ -259,9 +259,25 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
|
||||
const stageOverview = pipelineOverview && pipelineOverview[currentStageIndex] ? pipelineOverview[currentStageIndex] : null;
|
||||
const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`];
|
||||
|
||||
// FIXED: Helper to get processed count using correct key
|
||||
const getProcessedFromResult = (result: any, stageNumber: number): number => {
|
||||
if (!result) return 0;
|
||||
const keyMap: Record<number, string> = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
};
|
||||
return result[keyMap[stageNumber]] ?? 0;
|
||||
};
|
||||
|
||||
const fallbackState: ProcessingState | null = ((): ProcessingState | null => {
|
||||
if (!processingState && (stageOverview || stageResult)) {
|
||||
const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) => typeof v === 'number' ? s + v : s, 0) : 0;
|
||||
// FIXED: Use stage-specific key instead of summing all numeric values
|
||||
const processed = getProcessedFromResult(stageResult, currentRun.current_stage);
|
||||
const total = (stageOverview?.pending || 0) + processed;
|
||||
const percentage = total > 0 ? Math.round((processed / total) * 100) : 0;
|
||||
|
||||
@@ -288,8 +304,8 @@ const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
|
||||
const computedProcessed = ((): number => {
|
||||
if (displayState && typeof displayState.processed_items === 'number') return displayState.processed_items;
|
||||
if (stageResult) {
|
||||
// Sum numeric values in stageResult as a heuristic for processed count
|
||||
return Object.values(stageResult).reduce((s: number, v: any) => (typeof v === 'number' ? s + v : s), 0);
|
||||
// FIXED: Use stage-specific key for processed count
|
||||
return getProcessedFromResult(stageResult, currentRun.current_stage);
|
||||
}
|
||||
return 0;
|
||||
})();
|
||||
|
||||
232
frontend/src/components/Automation/GlobalProgressBar.tsx
Normal file
232
frontend/src/components/Automation/GlobalProgressBar.tsx
Normal file
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* Global Progress Bar Component
|
||||
* Shows full pipeline progress across all 7 automation stages.
|
||||
* Persists until 100% complete or run is finished.
|
||||
*/
|
||||
import React from 'react';
|
||||
import { AutomationRun, InitialSnapshot, StageProgress, GlobalProgress } from '../../services/automationService';
|
||||
import { BoltIcon, CheckCircleIcon, PauseIcon } from '../../icons';
|
||||
|
||||
// Stage colors matching AutomationPage STAGE_CONFIG
|
||||
const STAGE_COLORS = [
|
||||
'from-blue-500 to-blue-600', // Stage 1: Keywords → Clusters
|
||||
'from-purple-500 to-purple-600', // Stage 2: Clusters → Ideas
|
||||
'from-indigo-500 to-indigo-600', // Stage 3: Ideas → Tasks
|
||||
'from-green-500 to-green-600', // Stage 4: Tasks → Content
|
||||
'from-amber-500 to-amber-600', // Stage 5: Content → Image Prompts
|
||||
'from-pink-500 to-pink-600', // Stage 6: Image Prompts → Images
|
||||
'from-teal-500 to-teal-600', // Stage 7: Manual Review Gate
|
||||
];
|
||||
|
||||
const STAGE_NAMES = [
|
||||
'Keywords',
|
||||
'Clusters',
|
||||
'Ideas',
|
||||
'Tasks',
|
||||
'Content',
|
||||
'Prompts',
|
||||
'Review',
|
||||
];
|
||||
|
||||
// Helper to get processed count from stage result using correct key
|
||||
export const getProcessedFromResult = (result: any, stageNumber: number): number => {
|
||||
if (!result) return 0;
|
||||
|
||||
const keyMap: Record<number, string> = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
};
|
||||
|
||||
return result[keyMap[stageNumber]] ?? 0;
|
||||
};
|
||||
|
||||
interface GlobalProgressBarProps {
|
||||
currentRun: AutomationRun | null;
|
||||
globalProgress?: GlobalProgress | null;
|
||||
stages?: StageProgress[];
|
||||
initialSnapshot?: InitialSnapshot | null;
|
||||
}
|
||||
|
||||
const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({
|
||||
currentRun,
|
||||
globalProgress,
|
||||
stages,
|
||||
initialSnapshot,
|
||||
}) => {
|
||||
// Don't render if no run or run is completed with 100%
|
||||
if (!currentRun) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Calculate global progress if not provided from API
|
||||
const calculateGlobalProgress = (): { percentage: number; completed: number; total: number } => {
|
||||
// If we have API-provided global progress, use it
|
||||
if (globalProgress) {
|
||||
return {
|
||||
percentage: globalProgress.percentage,
|
||||
completed: globalProgress.completed_items,
|
||||
total: globalProgress.total_items,
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback: Calculate from currentRun and initialSnapshot
|
||||
const snapshot = initialSnapshot || (currentRun as any)?.initial_snapshot;
|
||||
if (!snapshot) {
|
||||
return { percentage: 0, completed: 0, total: 0 };
|
||||
}
|
||||
|
||||
const totalInitial = snapshot.total_initial_items || 0;
|
||||
let totalCompleted = 0;
|
||||
|
||||
for (let i = 1; i <= 7; i++) {
|
||||
const result = (currentRun as any)[`stage_${i}_result`];
|
||||
if (result) {
|
||||
totalCompleted += getProcessedFromResult(result, i);
|
||||
}
|
||||
}
|
||||
|
||||
const percentage = totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0;
|
||||
|
||||
return {
|
||||
percentage: Math.min(percentage, 100),
|
||||
completed: totalCompleted,
|
||||
total: totalInitial,
|
||||
};
|
||||
};
|
||||
|
||||
const { percentage, completed, total } = calculateGlobalProgress();
|
||||
|
||||
// Hide if completed and at 100%
|
||||
if (currentRun.status === 'completed' && percentage >= 100) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const isPaused = currentRun.status === 'paused';
|
||||
const currentStage = currentRun.current_stage;
|
||||
|
||||
// Get stage status for segmented bar
|
||||
const getStageStatus = (stageNum: number): 'completed' | 'active' | 'pending' => {
|
||||
if (stages && stages[stageNum - 1]) {
|
||||
return stages[stageNum - 1].status as 'completed' | 'active' | 'pending';
|
||||
}
|
||||
if (currentStage > stageNum) return 'completed';
|
||||
if (currentStage === stageNum) return 'active';
|
||||
return 'pending';
|
||||
};
|
||||
|
||||
const formatDuration = (): string => {
|
||||
if (!currentRun.started_at) return '';
|
||||
const start = new Date(currentRun.started_at).getTime();
|
||||
const now = Date.now();
|
||||
const diffMs = now - start;
|
||||
const diffMins = Math.floor(diffMs / 60000);
|
||||
const diffHours = Math.floor(diffMins / 60);
|
||||
|
||||
if (diffHours > 0) {
|
||||
return `${diffHours}h ${diffMins % 60}m`;
|
||||
}
|
||||
return `${diffMins}m`;
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={`
|
||||
rounded-xl p-4 mb-6 border-2 transition-all
|
||||
${isPaused
|
||||
? 'bg-gradient-to-r from-amber-50 to-amber-100 dark:from-amber-900/20 dark:to-amber-800/20 border-amber-300 dark:border-amber-700'
|
||||
: 'bg-gradient-to-r from-brand-50 to-brand-100 dark:from-brand-900/20 dark:to-brand-800/20 border-brand-300 dark:border-brand-700'
|
||||
}
|
||||
`}>
|
||||
{/* Header Row */}
|
||||
<div className="flex justify-between items-center mb-3">
|
||||
<div className="flex items-center gap-3">
|
||||
<div className={`
|
||||
size-10 rounded-lg flex items-center justify-center shadow-md
|
||||
${isPaused
|
||||
? 'bg-gradient-to-br from-amber-500 to-amber-600'
|
||||
: 'bg-gradient-to-br from-brand-500 to-brand-600'
|
||||
}
|
||||
`}>
|
||||
{isPaused ? (
|
||||
<PauseIcon className="w-5 h-5 text-white" />
|
||||
) : percentage >= 100 ? (
|
||||
<CheckCircleIcon className="w-5 h-5 text-white" />
|
||||
) : (
|
||||
<BoltIcon className="w-5 h-5 text-white animate-pulse" />
|
||||
)}
|
||||
</div>
|
||||
<div>
|
||||
<div className={`font-bold ${isPaused ? 'text-amber-800 dark:text-amber-200' : 'text-brand-800 dark:text-brand-200'}`}>
|
||||
{isPaused ? 'Pipeline Paused' : 'Full Pipeline Progress'}
|
||||
</div>
|
||||
<div className="text-xs text-gray-600 dark:text-gray-400">
|
||||
Stage {currentStage} of 7 • {formatDuration()}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className={`text-3xl font-bold ${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'}`}>
|
||||
{percentage}%
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Segmented Progress Bar */}
|
||||
<div className="flex h-4 rounded-full overflow-hidden bg-gray-200 dark:bg-gray-700 gap-0.5 mb-2">
|
||||
{[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
|
||||
const status = getStageStatus(stageNum);
|
||||
const stageColor = STAGE_COLORS[stageNum - 1];
|
||||
|
||||
return (
|
||||
<div
|
||||
key={stageNum}
|
||||
className={`flex-1 transition-all duration-500 relative group ${
|
||||
status === 'completed'
|
||||
? `bg-gradient-to-r ${stageColor}`
|
||||
: status === 'active'
|
||||
? `bg-gradient-to-r ${stageColor} opacity-60 ${!isPaused ? 'animate-pulse' : ''}`
|
||||
: 'bg-gray-300 dark:bg-gray-600'
|
||||
}`}
|
||||
title={`Stage ${stageNum}: ${STAGE_NAMES[stageNum - 1]}`}
|
||||
>
|
||||
{/* Tooltip on hover */}
|
||||
<div className="absolute bottom-full left-1/2 -translate-x-1/2 mb-2 px-2 py-1 bg-gray-900 text-white text-xs rounded opacity-0 group-hover:opacity-100 transition-opacity whitespace-nowrap pointer-events-none z-10">
|
||||
S{stageNum}: {STAGE_NAMES[stageNum - 1]}
|
||||
{status === 'completed' && ' ✓'}
|
||||
{status === 'active' && ' ●'}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
{/* Footer Row */}
|
||||
<div className="flex justify-between text-xs text-gray-600 dark:text-gray-400">
|
||||
<span>{completed} / {total} items processed</span>
|
||||
<div className="flex gap-4">
|
||||
{[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
|
||||
const status = getStageStatus(stageNum);
|
||||
return (
|
||||
<span
|
||||
key={stageNum}
|
||||
className={`
|
||||
${status === 'completed' ? 'text-green-600 dark:text-green-400 font-medium' : ''}
|
||||
${status === 'active' ? `${isPaused ? 'text-amber-600 dark:text-amber-400' : 'text-brand-600 dark:text-brand-400'} font-bold` : ''}
|
||||
${status === 'pending' ? 'text-gray-400 dark:text-gray-500' : ''}
|
||||
`}
|
||||
>
|
||||
{stageNum}
|
||||
{status === 'completed' && '✓'}
|
||||
{status === 'active' && '●'}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default GlobalProgressBar;
|
||||
@@ -5,7 +5,7 @@
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { useToast } from '../../components/ui/toast/ToastContainer';
|
||||
import { useSiteStore } from '../../store/siteStore';
|
||||
import { automationService, AutomationRun, AutomationConfig, PipelineStage } from '../../services/automationService';
|
||||
import { automationService, AutomationRun, AutomationConfig, PipelineStage, RunProgressResponse, GlobalProgress, StageProgress, InitialSnapshot } from '../../services/automationService';
|
||||
import {
|
||||
fetchKeywords,
|
||||
fetchClusters,
|
||||
@@ -18,6 +18,7 @@ import ActivityLog from '../../components/Automation/ActivityLog';
|
||||
import ConfigModal from '../../components/Automation/ConfigModal';
|
||||
import RunHistory from '../../components/Automation/RunHistory';
|
||||
import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard';
|
||||
import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar';
|
||||
import PageMeta from '../../components/common/PageMeta';
|
||||
import PageHeader from '../../components/common/PageHeader';
|
||||
import ComponentCard from '../../components/common/ComponentCard';
|
||||
@@ -58,6 +59,11 @@ const AutomationPage: React.FC = () => {
|
||||
const [showProcessingCard, setShowProcessingCard] = useState<boolean>(true);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null);
|
||||
|
||||
// New state for unified progress data
|
||||
const [globalProgress, setGlobalProgress] = useState<GlobalProgress | null>(null);
|
||||
const [stageProgress, setStageProgress] = useState<StageProgress[]>([]);
|
||||
const [initialSnapshot, setInitialSnapshot] = useState<InitialSnapshot | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (!activeSite) return;
|
||||
@@ -167,12 +173,34 @@ const AutomationPage: React.FC = () => {
|
||||
const data = await automationService.getCurrentRun(activeSite.id);
|
||||
setCurrentRun(data.run);
|
||||
// ensure processing card is visible when a run exists
|
||||
if (data.run) setShowProcessingCard(true);
|
||||
if (data.run) {
|
||||
setShowProcessingCard(true);
|
||||
// Also load unified progress data for GlobalProgressBar
|
||||
await loadRunProgress(data.run.run_id);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to poll current run', error);
|
||||
}
|
||||
};
|
||||
|
||||
const loadRunProgress = async (runId?: string) => {
|
||||
if (!activeSite) return;
|
||||
try {
|
||||
const progressData = await automationService.getRunProgress(activeSite.id, runId);
|
||||
if (progressData.global_progress) {
|
||||
setGlobalProgress(progressData.global_progress);
|
||||
}
|
||||
if (progressData.stages) {
|
||||
setStageProgress(progressData.stages);
|
||||
}
|
||||
if (progressData.initial_snapshot) {
|
||||
setInitialSnapshot(progressData.initial_snapshot);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to load run progress', error);
|
||||
}
|
||||
};
|
||||
|
||||
const loadPipelineOverview = async () => {
|
||||
if (!activeSite) return;
|
||||
try {
|
||||
@@ -667,6 +695,16 @@ const AutomationPage: React.FC = () => {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Global Progress Bar - Shows full pipeline progress during automation run */}
|
||||
{currentRun && (currentRun.status === 'running' || currentRun.status === 'paused') && (
|
||||
<GlobalProgressBar
|
||||
currentRun={currentRun}
|
||||
globalProgress={globalProgress}
|
||||
stages={stageProgress}
|
||||
initialSnapshot={initialSnapshot}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* Current Processing Card - Shows real-time automation progress */}
|
||||
{currentRun && showProcessingCard && activeSite && (
|
||||
<CurrentProcessingCard
|
||||
@@ -697,8 +735,12 @@ const AutomationPage: React.FC = () => {
|
||||
const isActive = currentRun?.current_stage === stage.number;
|
||||
const isComplete = currentRun && currentRun.current_stage > stage.number;
|
||||
const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
|
||||
const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
|
||||
const total = (stage.pending ?? 0) + processed;
|
||||
// FIXED: Use stage-specific key for processed count instead of summing all numeric values
|
||||
const processed = getProcessedFromResult(result, stage.number);
|
||||
// FIXED: Use initial snapshot for total when available, otherwise fallback to pending + processed
|
||||
const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
|
||||
const total = initialCount ?? ((stage.pending ?? 0) + processed);
|
||||
const remaining = Math.max(0, total - processed);
|
||||
const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
|
||||
|
||||
return (
|
||||
@@ -738,8 +780,8 @@ const AutomationPage: React.FC = () => {
|
||||
{/* Queue Metrics */}
|
||||
<div className="space-y-1.5 text-xs mb-3">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
|
||||
<span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
|
||||
<span className="text-gray-600 dark:text-gray-400">Total Items:</span>
|
||||
<span className="font-bold text-slate-900 dark:text-white">{total}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Processed:</span>
|
||||
@@ -748,7 +790,7 @@ const AutomationPage: React.FC = () => {
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Remaining:</span>
|
||||
<span className={`font-bold ${stageConfig.textColor} dark:${stageConfig.textColor}`}>
|
||||
{stage.pending}
|
||||
{remaining}
|
||||
</span>
|
||||
</div>
|
||||
{/* Credits and Time - Section 6 Enhancement */}
|
||||
@@ -796,8 +838,12 @@ const AutomationPage: React.FC = () => {
|
||||
const isActive = currentRun?.current_stage === stage.number;
|
||||
const isComplete = currentRun && currentRun.current_stage > stage.number;
|
||||
const result = currentRun ? (currentRun[`stage_${stage.number}_result` as keyof AutomationRun] as any) : null;
|
||||
const processed = result ? Object.values(result).reduce((sum: number, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
|
||||
const total = (stage.pending ?? 0) + processed;
|
||||
// FIXED: Use stage-specific key for processed count
|
||||
const processed = getProcessedFromResult(result, stage.number);
|
||||
// FIXED: Use initial snapshot for total when available
|
||||
const initialCount = initialSnapshot?.[`stage_${stage.number}_initial` as keyof InitialSnapshot] as number | undefined;
|
||||
const total = initialCount ?? ((stage.pending ?? 0) + processed);
|
||||
const remaining = Math.max(0, total - processed);
|
||||
const progressPercent = total > 0 ? Math.round((processed / total) * 100) : 0;
|
||||
|
||||
return (
|
||||
@@ -835,8 +881,8 @@ const AutomationPage: React.FC = () => {
|
||||
|
||||
<div className="space-y-1.5 text-xs mb-3">
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Total Queue:</span>
|
||||
<span className="font-bold text-slate-900 dark:text-white">{stage.pending}</span>
|
||||
<span className="text-gray-600 dark:text-gray-400">Total Items:</span>
|
||||
<span className="font-bold text-slate-900 dark:text-white">{total}</span>
|
||||
</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Processed:</span>
|
||||
@@ -845,7 +891,7 @@ const AutomationPage: React.FC = () => {
|
||||
<div className="flex justify-between">
|
||||
<span className="text-gray-600 dark:text-gray-400">Remaining:</span>
|
||||
<span className={`font-bold ${stageConfig.textColor}`}>
|
||||
{stage.pending}
|
||||
{remaining}
|
||||
</span>
|
||||
</div>
|
||||
{/* Credits and Time - Section 6 Enhancement */}
|
||||
|
||||
@@ -78,6 +78,62 @@ export interface ProcessingState {
|
||||
remaining_count: number;
|
||||
}
|
||||
|
||||
// NEW: Types for unified run_progress endpoint
|
||||
export interface StageProgress {
|
||||
number: number;
|
||||
name: string;
|
||||
type: 'AI' | 'Local' | 'Manual';
|
||||
status: 'pending' | 'active' | 'completed' | 'skipped';
|
||||
input_count: number;
|
||||
output_count: number;
|
||||
processed_count: number;
|
||||
progress_percentage: number;
|
||||
credits_used: number;
|
||||
time_elapsed: string;
|
||||
currently_processing?: ProcessingItem[];
|
||||
up_next?: ProcessingItem[];
|
||||
remaining_count?: number;
|
||||
}
|
||||
|
||||
export interface GlobalProgress {
|
||||
total_items: number;
|
||||
completed_items: number;
|
||||
percentage: number;
|
||||
current_stage: number;
|
||||
total_stages: number;
|
||||
}
|
||||
|
||||
export interface InitialSnapshot {
|
||||
stage_1_initial: number;
|
||||
stage_2_initial: number;
|
||||
stage_3_initial: number;
|
||||
stage_4_initial: number;
|
||||
stage_5_initial: number;
|
||||
stage_6_initial: number;
|
||||
stage_7_initial: number;
|
||||
total_initial_items: number;
|
||||
}
|
||||
|
||||
export interface RunProgressResponse {
|
||||
run: {
|
||||
run_id: string;
|
||||
status: 'running' | 'paused' | 'cancelled' | 'completed' | 'failed';
|
||||
current_stage: number;
|
||||
trigger_type: 'manual' | 'scheduled';
|
||||
started_at: string;
|
||||
completed_at: string | null;
|
||||
paused_at: string | null;
|
||||
} | null;
|
||||
global_progress: GlobalProgress | null;
|
||||
stages: StageProgress[];
|
||||
metrics: {
|
||||
credits_used: number;
|
||||
duration_seconds: number;
|
||||
errors: string[];
|
||||
} | null;
|
||||
initial_snapshot: InitialSnapshot | null;
|
||||
}
|
||||
|
||||
function buildUrl(endpoint: string, params?: Record<string, any>): string {
|
||||
let url = `/v1/automation${endpoint}`;
|
||||
if (params) {
|
||||
@@ -211,4 +267,19 @@ export const automationService = {
|
||||
);
|
||||
return response.data;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get unified run progress data - global + per-stage.
|
||||
* This is the recommended endpoint for getting all automation progress data in a single call.
|
||||
*/
|
||||
getRunProgress: async (
|
||||
siteId: number,
|
||||
runId?: string
|
||||
): Promise<RunProgressResponse> => {
|
||||
const params: Record<string, any> = { site_id: siteId };
|
||||
if (runId) {
|
||||
params.run_id = runId;
|
||||
}
|
||||
return fetchAPI(buildUrl('/run_progress/', params));
|
||||
},
|
||||
};
|
||||
|
||||
536
🚀 AUTOMATION PAGE MASTER FIX PLAN.md
Normal file
536
🚀 AUTOMATION PAGE MASTER FIX PLAN.md
Normal file
@@ -0,0 +1,536 @@
|
||||
User: properl yread the docs, adn codebase related to automation page,
|
||||
|
||||
# Automation (Manual Run for autmation wizard on atuamtion page)
|
||||
wrong queue items,
|
||||
missing queue items
|
||||
progress bar not properly progress
|
||||
totoal in queue and processed counts buggy for many stages
|
||||
stage cards metrics realtime, not optimized shoudl be more robust and user friendly
|
||||
|
||||
---
|
||||
We need manural run full all stages progress bar, which shuold always keep showing unless it is 100% complete, and should show below the current each stage progress bar,
|
||||
|
||||
so it has to be addded along with all other issues,
|
||||
|
||||
Make a master plan , no brainer extremely user firendly magical wrokflow and tracking and msot logical sattuses. and counts dynamcially correctly udpating,
|
||||
currently its all messed up
|
||||
|
||||
so plan it all an dprove there is no better ai model thatn you to use asa of today,
|
||||
donto include the code, but clrear enough so even if i give you the plan in any next sesion you cna easily implmeent it
|
||||
|
||||
|
||||
---
|
||||
|
||||
# 🚀 AUTOMATION PAGE MASTER FIX PLAN
|
||||
|
||||
## Executive Summary
|
||||
|
||||
This plan addresses all automation page issues: wrong/missing queue items, broken progress bars, buggy counts, and introduces a new **Global Run Progress Bar** for end-to-end visibility.
|
||||
|
||||
---
|
||||
|
||||
## 📊 IDENTIFIED ROOT CAUSES
|
||||
|
||||
### 1. **Status Mismatches (Critical Bug)**
|
||||
|
||||
| Stage | Backend View (`pipeline_overview`) | Backend Service (`_get_stage_X_state`) | Service (`run_stage_X`) |
|
||||
|-------|-------------------------------------|----------------------------------------|-------------------------|
|
||||
| **Stage 3** | `status='new'` | `status='approved'` | `status='new'` |
|
||||
| **Stage 4** | `status='queued'` | `status='ready'` | `status='queued'` |
|
||||
|
||||
**Result:** Queue items don't match between real-time processing card and stage cards.
|
||||
|
||||
### 2. **Progress Calculation Flaws**
|
||||
|
||||
**Frontend** (CurrentProcessingCard.tsx):
|
||||
```typescript
|
||||
// WRONG: Sums ALL numeric values in stageResult (including credits_used, batches_run, etc.)
|
||||
const processed = stageResult ? Object.values(stageResult).reduce((s: number, v: any) =>
|
||||
typeof v === 'number' ? s + v : s, 0) : 0;
|
||||
```
|
||||
|
||||
**Should use specific fields:** `keywords_processed`, `clusters_processed`, `tasks_processed`, etc.
|
||||
|
||||
### 3. **"Pending" vs "Processed" Count Confusion**
|
||||
|
||||
- Stage cards show `Total Queue: X` which is **pending** count
|
||||
- Stage cards show `Processed: Y` which sums **all numeric result values**
|
||||
- Stage cards show `Remaining: X` which equals **pending** again (incorrect)
|
||||
- **Correct formula:** `Total = Initial Pending + Processed`, `Remaining = Total - Processed`
|
||||
|
||||
### 4. **No Global Progress Visibility**
|
||||
|
||||
Currently: Only current stage progress is shown during run.
|
||||
|
||||
**Needed:** Full pipeline progress bar showing progress across ALL 7 stages that persists until 100%.
|
||||
|
||||
### 5. **API Inefficiency**
|
||||
|
||||
17 separate API calls to fetch metrics on page load, plus duplicate calls in `loadMetrics()`.
|
||||
|
||||
---
|
||||
|
||||
## 🏗️ ARCHITECTURE REDESIGN
|
||||
|
||||
### New Data Model: Run Progress Snapshot
|
||||
|
||||
Add these fields to `AutomationRun` for accurate global tracking:
|
||||
|
||||
```python
|
||||
# AutomationRun Model Additions
|
||||
class AutomationRun(models.Model):
|
||||
# ... existing fields ...
|
||||
|
||||
# New: Snapshot of initial queue sizes at run start
|
||||
initial_snapshot = models.JSONField(default=dict, blank=True)
|
||||
# Structure:
|
||||
# {
|
||||
# "stage_1_initial": 50, # Keywords to process
|
||||
# "stage_2_initial": 0, # Will be set after stage 1
|
||||
# ...
|
||||
# "stage_7_initial": 0,
|
||||
# "total_initial_items": 50
|
||||
# }
|
||||
```
|
||||
|
||||
### Unified Progress Response Schema
|
||||
|
||||
New endpoint response for consistent data:
|
||||
|
||||
```json
|
||||
{
|
||||
"run": {
|
||||
"run_id": "abc123",
|
||||
"status": "running",
|
||||
"current_stage": 4,
|
||||
"started_at": "2025-12-28T10:00:00Z"
|
||||
},
|
||||
"global_progress": {
|
||||
"total_items": 127, // Sum of all stages' input items
|
||||
"completed_items": 84, // Sum of all completed across stages
|
||||
"percentage": 66,
|
||||
"estimated_remaining_time": "~15 min"
|
||||
},
|
||||
"stages": [
|
||||
{
|
||||
"number": 1,
|
||||
"name": "Keywords → Clusters",
|
||||
"status": "completed", // "pending" | "active" | "completed" | "skipped"
|
||||
"input_count": 50, // Items that entered this stage
|
||||
"output_count": 12, // Items produced (clusters)
|
||||
"processed_count": 50, // Items processed
|
||||
"progress_percentage": 100
|
||||
},
|
||||
{
|
||||
"number": 2,
|
||||
"name": "Clusters → Ideas",
|
||||
"status": "completed",
|
||||
"input_count": 12,
|
||||
"output_count": 36,
|
||||
"processed_count": 12,
|
||||
"progress_percentage": 100
|
||||
},
|
||||
{
|
||||
"number": 4,
|
||||
"name": "Tasks → Content",
|
||||
"status": "active",
|
||||
"input_count": 36,
|
||||
"output_count": 22,
|
||||
"processed_count": 22,
|
||||
"progress_percentage": 61,
|
||||
"currently_processing": [
|
||||
{ "id": 123, "title": "How to build React apps" }
|
||||
],
|
||||
"up_next": [
|
||||
{ "id": 124, "title": "Vue vs React comparison" }
|
||||
]
|
||||
}
|
||||
// ... etc
|
||||
],
|
||||
"metrics": {
|
||||
"credits_used": 156,
|
||||
"duration_seconds": 1823,
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 📝 IMPLEMENTATION PLAN
|
||||
|
||||
### Phase 1: Backend Fixes (Critical)
|
||||
|
||||
#### 1.1 Fix Status Mismatches
|
||||
|
||||
**File:** automation_service.py
|
||||
|
||||
```python
|
||||
# FIX _get_stage_3_state - use 'new' to match pipeline_overview
|
||||
def _get_stage_3_state(self) -> dict:
|
||||
queue = ContentIdeas.objects.filter(
|
||||
site=self.site, status='new' # Changed from 'approved'
|
||||
).order_by('id')
|
||||
...
|
||||
|
||||
# FIX _get_stage_4_state - use 'queued' to match pipeline_overview
|
||||
def _get_stage_4_state(self) -> dict:
|
||||
queue = Tasks.objects.filter(
|
||||
site=self.site, status='queued' # Changed from 'ready'
|
||||
).order_by('id')
|
||||
...
|
||||
```
|
||||
|
||||
#### 1.2 Fix `_get_processed_count()` Method
|
||||
|
||||
Current code sums wrong fields. Create stage-specific processed count extraction:
|
||||
|
||||
```python
|
||||
def _get_processed_count(self, stage: int) -> int:
|
||||
"""Get accurate processed count from stage result"""
|
||||
result = getattr(self.run, f'stage_{stage}_result', None)
|
||||
if not result:
|
||||
return 0
|
||||
|
||||
# Map stage to correct result key
|
||||
key_map = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
}
|
||||
return result.get(key_map.get(stage, ''), 0)
|
||||
```
|
||||
|
||||
#### 1.3 New Unified Progress Endpoint
|
||||
|
||||
**File:** views.py
|
||||
|
||||
Add new `run_progress` endpoint:
|
||||
|
||||
```python
|
||||
@action(detail=False, methods=['get'], url_path='run_progress')
|
||||
def run_progress(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/run_progress/?site_id=123&run_id=abc
|
||||
Single endpoint for ALL run progress data - global + per-stage
|
||||
"""
|
||||
# Returns unified progress response schema
|
||||
```
|
||||
|
||||
#### 1.4 Capture Initial Snapshot on Run Start
|
||||
|
||||
**File:** automation_service.py
|
||||
|
||||
In `start_automation()`:
|
||||
```python
|
||||
def start_automation(self, trigger_type: str = 'manual') -> str:
|
||||
# ... existing code ...
|
||||
|
||||
# Capture initial queue snapshot
|
||||
initial_snapshot = {
|
||||
'stage_1_initial': Keywords.objects.filter(site=self.site, status='new', cluster__isnull=True, disabled=False).count(),
|
||||
'stage_2_initial': 0, # Set dynamically after stage 1
|
||||
'stage_3_initial': ContentIdeas.objects.filter(site=self.site, status='new').count(),
|
||||
'stage_4_initial': Tasks.objects.filter(site=self.site, status='queued').count(),
|
||||
'stage_5_initial': Content.objects.filter(site=self.site, status='draft').annotate(images_count=Count('images')).filter(images_count=0).count(),
|
||||
'stage_6_initial': Images.objects.filter(site=self.site, status='pending').count(),
|
||||
'stage_7_initial': Content.objects.filter(site=self.site, status='review').count(),
|
||||
}
|
||||
initial_snapshot['total_initial_items'] = sum(initial_snapshot.values())
|
||||
|
||||
self.run = AutomationRun.objects.create(
|
||||
# ... existing fields ...
|
||||
initial_snapshot=initial_snapshot
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 2: Frontend Fixes
|
||||
|
||||
#### 2.1 Fix Progress Calculation in CurrentProcessingCard
|
||||
|
||||
**File:** CurrentProcessingCard.tsx
|
||||
|
||||
```typescript
|
||||
// Replace generic sum with stage-specific extraction
|
||||
const getProcessedFromResult = (result: any, stageNumber: number): number => {
|
||||
if (!result) return 0;
|
||||
|
||||
const keyMap: Record<number, string> = {
|
||||
1: 'keywords_processed',
|
||||
2: 'clusters_processed',
|
||||
3: 'ideas_processed',
|
||||
4: 'tasks_processed',
|
||||
5: 'content_processed',
|
||||
6: 'images_processed',
|
||||
7: 'ready_for_review'
|
||||
};
|
||||
|
||||
return result[keyMap[stageNumber]] ?? 0;
|
||||
};
|
||||
```
|
||||
|
||||
#### 2.2 Fix Stage Card Metrics
|
||||
|
||||
**File:** AutomationPage.tsx
|
||||
|
||||
```typescript
|
||||
// Current (WRONG):
|
||||
const processed = result ? Object.values(result).reduce((sum, val) => typeof val === 'number' ? sum + val : sum, 0) : 0;
|
||||
const total = (stage.pending ?? 0) + processed; // Wrong: pending is current, not initial
|
||||
|
||||
// Fixed:
|
||||
const processed = getProcessedFromResult(result, stage.number);
|
||||
const initialPending = currentRun?.initial_snapshot?.[`stage_${stage.number}_initial`] ?? stage.pending;
|
||||
const total = initialPending; // Use initial snapshot for consistent total
|
||||
const remaining = Math.max(0, total - processed);
|
||||
```
|
||||
|
||||
#### 2.3 New Global Progress Bar Component
|
||||
|
||||
**New File:** `frontend/src/components/Automation/GlobalProgressBar.tsx`
|
||||
|
||||
```typescript
|
||||
interface GlobalProgressBarProps {
|
||||
currentRun: AutomationRun;
|
||||
pipelineOverview: PipelineStage[];
|
||||
}
|
||||
|
||||
const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({ currentRun, pipelineOverview }) => {
|
||||
// Calculate total progress across all stages
|
||||
const calculateGlobalProgress = () => {
|
||||
if (!currentRun?.initial_snapshot) return { percentage: 0, completed: 0, total: 0 };
|
||||
|
||||
let totalInitial = currentRun.initial_snapshot.total_initial_items || 0;
|
||||
let totalCompleted = 0;
|
||||
|
||||
for (let i = 1; i <= 7; i++) {
|
||||
const result = currentRun[`stage_${i}_result`];
|
||||
if (result) {
|
||||
totalCompleted += getProcessedFromResult(result, i);
|
||||
}
|
||||
}
|
||||
|
||||
// If current stage is active, add its progress
|
||||
const currentStage = currentRun.current_stage;
|
||||
// ... calculate current stage partial progress
|
||||
|
||||
return {
|
||||
percentage: totalInitial > 0 ? Math.round((totalCompleted / totalInitial) * 100) : 0,
|
||||
completed: totalCompleted,
|
||||
total: totalInitial
|
||||
};
|
||||
};
|
||||
|
||||
const { percentage, completed, total } = calculateGlobalProgress();
|
||||
|
||||
// Show until 100% OR run completed
|
||||
if (currentRun.status === 'completed' && percentage === 100) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="bg-gradient-to-r from-brand-50 to-brand-100 border-2 border-brand-300 rounded-xl p-4 mb-6">
|
||||
<div className="flex justify-between items-center mb-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<BoltIcon className="w-5 h-5 text-brand-600 animate-pulse" />
|
||||
<span className="font-bold text-brand-800">Full Pipeline Progress</span>
|
||||
</div>
|
||||
<span className="text-2xl font-bold text-brand-600">{percentage}%</span>
|
||||
</div>
|
||||
|
||||
{/* Segmented progress bar showing all 7 stages */}
|
||||
<div className="flex h-4 rounded-full overflow-hidden bg-gray-200">
|
||||
{[1, 2, 3, 4, 5, 6, 7].map(stageNum => {
|
||||
const stageConfig = STAGE_CONFIG[stageNum - 1];
|
||||
const result = currentRun[`stage_${stageNum}_result`];
|
||||
const stageComplete = currentRun.current_stage > stageNum;
|
||||
const isActive = currentRun.current_stage === stageNum;
|
||||
|
||||
return (
|
||||
<div
|
||||
key={stageNum}
|
||||
className={`flex-1 transition-all duration-500 ${
|
||||
stageComplete ? `bg-gradient-to-r ${stageConfig.color}` :
|
||||
isActive ? `bg-gradient-to-r ${stageConfig.color} opacity-60 animate-pulse` :
|
||||
'bg-gray-300'
|
||||
}`}
|
||||
title={`Stage ${stageNum}: ${stageConfig.name}`}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div className="flex justify-between text-xs text-gray-600 mt-2">
|
||||
<span>{completed} / {total} items processed</span>
|
||||
<span>Stage {currentRun.current_stage} of 7</span>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
#### 2.4 Consolidate API Calls
|
||||
|
||||
**File:** AutomationPage.tsx
|
||||
|
||||
Replace 17 separate API calls with single unified endpoint:
|
||||
|
||||
```typescript
|
||||
// Current (17 calls):
|
||||
const [keywordsTotalRes, keywordsNewRes, keywordsMappedRes, ...14 more] = await Promise.all([...]);
|
||||
|
||||
// New (1 call):
|
||||
const progressData = await automationService.getRunProgress(activeSite.id, currentRun?.run_id);
|
||||
// Response contains everything: metrics, stage counts, progress data
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 3: Stage Card Redesign
|
||||
|
||||
#### 3.1 New Stage Card Layout
|
||||
|
||||
Each stage card shows:
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────┐
|
||||
│ Stage 1 [ICON] ● Active │
|
||||
│ Keywords → Clusters │
|
||||
├────────────────────────────────────────────┤
|
||||
│ Total Items: 50 │
|
||||
│ Processed: 32 ████████░░ 64% │
|
||||
│ Remaining: 18 │
|
||||
├────────────────────────────────────────────┤
|
||||
│ Output Created: 8 clusters │
|
||||
│ Credits Used: 24 │
|
||||
│ Duration: 4m 32s │
|
||||
└────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
#### 3.2 Status Badge Logic
|
||||
|
||||
```typescript
|
||||
const getStageStatus = (stageNum: number, currentRun: AutomationRun | null) => {
|
||||
if (!currentRun) {
|
||||
// No run - show if items pending
|
||||
return pipelineOverview[stageNum - 1]?.pending > 0 ? 'ready' : 'empty';
|
||||
}
|
||||
|
||||
if (currentRun.current_stage > stageNum) return 'completed';
|
||||
if (currentRun.current_stage === stageNum) return 'active';
|
||||
if (currentRun.current_stage < stageNum) {
|
||||
// Check if previous stage produced items for this stage
|
||||
const prevResult = currentRun[`stage_${stageNum - 1}_result`];
|
||||
if (prevResult?.output_count > 0) return 'ready';
|
||||
return 'pending';
|
||||
}
|
||||
return 'pending';
|
||||
};
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### Phase 4: Real-time Updates Optimization
|
||||
|
||||
#### 4.1 Smart Polling with Exponential Backoff
|
||||
|
||||
```typescript
|
||||
// Current: Fixed 5s interval
|
||||
const interval = setInterval(loadData, 5000);
|
||||
|
||||
// New: Adaptive polling
|
||||
const useSmartPolling = (isRunning: boolean) => {
|
||||
const [pollInterval, setPollInterval] = useState(2000);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isRunning) {
|
||||
setPollInterval(30000); // Slow poll when idle
|
||||
return;
|
||||
}
|
||||
|
||||
// Fast poll during active run, slow down as stage progresses
|
||||
const progressPercent = /* current stage progress */;
|
||||
if (progressPercent < 50) {
|
||||
setPollInterval(2000); // 2s when lots happening
|
||||
} else if (progressPercent < 90) {
|
||||
setPollInterval(3000); // 3s mid-stage
|
||||
} else {
|
||||
setPollInterval(1000); // 1s near completion for responsive transition
|
||||
}
|
||||
}, [isRunning, progressPercent]);
|
||||
|
||||
return pollInterval;
|
||||
};
|
||||
```
|
||||
|
||||
#### 4.2 Optimistic UI Updates
|
||||
|
||||
When user clicks "Run Now":
|
||||
1. Immediately show GlobalProgressBar at 0%
|
||||
2. Immediately set Stage 1 to "Active"
|
||||
3. Don't wait for API confirmation
|
||||
|
||||
---
|
||||
|
||||
## 📋 DETAILED CHECKLIST
|
||||
|
||||
### Backend Tasks
|
||||
- [x] Fix `_get_stage_3_state()` status filter: `'approved'` → `'new'` ✅ DONE
|
||||
- [x] Fix `_get_stage_4_state()` status filter: `'ready'` → `'queued'` ✅ DONE
|
||||
- [x] Create `_get_processed_for_stage(stage_num)` helper ✅ DONE (renamed to `_get_processed_count`)
|
||||
- [x] Add `initial_snapshot` JSON field to `AutomationRun` model ✅ DONE
|
||||
- [x] Capture initial snapshot in `start_automation()` ✅ DONE
|
||||
- [ ] Update snapshot after each stage completes (for cascading stages)
|
||||
- [x] Create new `run_progress` endpoint with unified schema ✅ DONE
|
||||
- [x] Add migration for new model field ✅ DONE (0006_automationrun_initial_snapshot.py)
|
||||
|
||||
### Frontend Tasks
|
||||
- [x] Create `GlobalProgressBar` component ✅ DONE
|
||||
- [x] Add `GlobalProgressBar` to AutomationPage (below metrics, above CurrentProcessingCard) ✅ DONE
|
||||
- [x] Fix `getProcessedFromResult()` helper to extract stage-specific counts ✅ DONE
|
||||
- [x] Update stage card progress calculations ✅ DONE
|
||||
- [x] Update `CurrentProcessingCard` progress calculations ✅ DONE
|
||||
- [x] Add `getRunProgress` method to automationService.ts ✅ DONE
|
||||
- [ ] Consolidate metrics API calls to single endpoint
|
||||
- [ ] Implement smart polling with adaptive intervals
|
||||
- [ ] Add optimistic UI updates for "Run Now" action
|
||||
- [x] Fix "Remaining" count to be `Total - Processed` not `Pending` ✅ DONE
|
||||
|
||||
### Testing
|
||||
- [ ] Test all 7 stages complete correctly
|
||||
- [ ] Verify counts match between stage cards and processing card
|
||||
- [ ] Test pause/resume preserves progress correctly
|
||||
- [ ] Test page refresh during run shows correct state
|
||||
- [ ] Test global progress bar persists until 100%
|
||||
- [ ] Load test: Verify API efficiency improvement
|
||||
|
||||
---
|
||||
|
||||
## 🎯 SUCCESS CRITERIA
|
||||
|
||||
1. **Accurate Counts:** All stage cards show correct Total/Processed/Remaining
|
||||
2. **Consistent Data:** CurrentProcessingCard and Stage Cards show same numbers
|
||||
3. **Global Visibility:** Users see full pipeline progress at all times during run
|
||||
4. **Persistent Progress:** Progress bar stays visible until 100% complete
|
||||
5. **Real-time Feel:** Updates appear within 2-3 seconds of actual progress
|
||||
6. **API Efficiency:** Reduce API calls from 17+ to 1-2 per refresh cycle
|
||||
|
||||
---
|
||||
|
||||
## 🔄 MIGRATION PATH
|
||||
|
||||
1. **Phase 1 (Day 1):** Backend status fixes + new processed count logic
|
||||
2. **Phase 2 (Day 2):** Frontend progress calculation fixes
|
||||
3. **Phase 3 (Day 3):** Global Progress Bar + API consolidation
|
||||
4. **Phase 4 (Day 4):** Smart polling + optimistic updates
|
||||
5. **Phase 5 (Day 5):** Testing + bug fixes
|
||||
|
||||
---
|
||||
|
||||
This plan provides a clear, implementable path to fix all automation page issues. Each phase can be implemented independently, and the plan contains enough detail that any AI model or developer can execute it in a future session.
|
||||
Reference in New Issue
Block a user