From f92b3fba6e344a0351ecb505090d72d0a409bef4 Mon Sep 17 00:00:00 2001 From: "IGNY8 VPS (Salman)" Date: Sun, 28 Dec 2025 03:15:39 +0000 Subject: [PATCH] automation fixes (part2) --- .../automation/services/automation_service.py | 124 +++++- .../Automation/CurrentProcessingCardV2.tsx | 378 ++++++++++++++++++ .../Automation/GlobalProgressBar.tsx | 43 +- .../src/pages/Automation/AutomationPage.tsx | 35 +- 4 files changed, 543 insertions(+), 37 deletions(-) create mode 100644 frontend/src/components/Automation/CurrentProcessingCardV2.tsx diff --git a/backend/igny8_core/business/automation/services/automation_service.py b/backend/igny8_core/business/automation/services/automation_service.py index 1213510f..9ffaf5f3 100644 --- a/backend/igny8_core/business/automation/services/automation_service.py +++ b/backend/igny8_core/business/automation/services/automation_service.py @@ -284,6 +284,22 @@ class AutomationService: stage_number, f"Batch {batch_num} complete" ) + # INCREMENTAL SAVE: Update stage result after each batch for real-time UI progress + clusters_so_far = Clusters.objects.filter( + site=self.site, + created_at__gte=self.run.started_at + ).count() + self.run.stage_1_result = { + 'keywords_processed': keywords_processed, + 'keywords_total': len(keyword_ids), + 'clusters_created': clusters_so_far, + 'batches_run': batches_run, + 'credits_used': self._get_credits_used() - credits_before, + 'time_elapsed': self._format_time_elapsed(start_time), + 'in_progress': True + } + self.run.save(update_fields=['stage_1_result']) + # Emit per-item trace event for UI progress tracking try: self.logger.append_trace(self.account.id, self.site.id, self.run.run_id, { @@ -358,6 +374,10 @@ class AutomationService: } self.run.current_stage = 2 self.run.total_credits_used += credits_used + + # UPDATE SNAPSHOT: Record new items created for Stage 2 + self._update_snapshot_after_stage(1, {'stage_2_initial': clusters_created}) + self.run.save() logger.info(f"[AutomationService] Stage 1 complete: {keywords_processed} keywords → {clusters_created} clusters") @@ -484,6 +504,21 @@ class AutomationService: self.run.run_id, self.account.id, self.site.id, stage_number, f"Cluster '{cluster.name}' complete" ) + + # INCREMENTAL SAVE: Update stage result after each cluster for real-time UI progress + ideas_so_far = ContentIdeas.objects.filter( + site=self.site, + created_at__gte=self.run.started_at + ).count() + self.run.stage_2_result = { + 'clusters_processed': clusters_processed, + 'clusters_total': total_count, + 'ideas_created': ideas_so_far, + 'credits_used': self._get_credits_used() - credits_before, + 'time_elapsed': self._format_time_elapsed(start_time), + 'in_progress': True + } + self.run.save(update_fields=['stage_2_result']) except Exception as e: # FIXED: Log error but continue processing remaining clusters error_msg = f"Failed to generate ideas for cluster '{cluster.name}': {str(e)}" @@ -525,6 +560,10 @@ class AutomationService: } self.run.current_stage = 3 self.run.total_credits_used += credits_used + + # UPDATE SNAPSHOT: Record new items created for Stage 3 + self._update_snapshot_after_stage(2, {'stage_3_initial': ideas_created}) + self.run.save() logger.info(f"[AutomationService] Stage 2 complete: {clusters_processed} clusters → {ideas_created} ideas") @@ -687,6 +726,10 @@ class AutomationService: 'time_elapsed': time_elapsed } self.run.current_stage = 4 + + # UPDATE SNAPSHOT: Record new items created for Stage 4 + self._update_snapshot_after_stage(3, {'stage_4_initial': tasks_created}) + self.run.save() logger.info(f"[AutomationService] Stage 3 complete: {ideas_processed} ideas → {tasks_created} tasks") @@ -809,6 +852,21 @@ class AutomationService: stage_number, f"Task '{task.title}' complete ({tasks_processed}/{total_tasks})" ) + # INCREMENTAL SAVE: Update stage result after each item for real-time UI progress + content_created_so_far = Content.objects.filter( + site=self.site, + created_at__gte=self.run.started_at + ).count() + self.run.stage_4_result = { + 'tasks_processed': tasks_processed, + 'tasks_total': total_tasks, + 'content_created': content_created_so_far, + 'credits_used': self._get_credits_used() - credits_before, + 'time_elapsed': self._format_time_elapsed(start_time), + 'in_progress': True + } + self.run.save(update_fields=['stage_4_result']) + # Emit per-item trace event for UI progress tracking try: self.logger.append_trace(self.account.id, self.site.id, self.run.run_id, { @@ -1424,6 +1482,32 @@ class AutomationService: logger.info(f"[AutomationService] Initial snapshot captured: {snapshot}") return snapshot + def _update_snapshot_after_stage(self, completed_stage: int, updates: dict): + """ + Update snapshot after a stage completes with new items created. + This ensures accurate counts for cascading stages. + + Args: + completed_stage: The stage number that just completed + updates: Dict of snapshot keys to update, e.g., {'stage_4_initial': 12} + """ + if not self.run or not self.run.initial_snapshot: + return + + snapshot = self.run.initial_snapshot.copy() + old_total = snapshot.get('total_initial_items', 0) + + for key, value in updates.items(): + old_value = snapshot.get(key, 0) + snapshot[key] = value + # Adjust total + old_total = old_total - old_value + value + + snapshot['total_initial_items'] = old_total + self.run.initial_snapshot = snapshot + + logger.info(f"[AutomationService] Snapshot updated after Stage {completed_stage}: {updates}, new total: {old_total}") + # Helper methods def _wait_for_task(self, task_id: str, stage_number: int, item_name: str, continue_on_error: bool = True): @@ -1584,7 +1668,16 @@ class AutomationService: ).order_by('id') processed = self._get_processed_count(1) - total = queue.count() + processed + remaining = queue.count() + + # Use keywords_total from incremental result if available + result = getattr(self.run, 'stage_1_result', None) + if result and result.get('keywords_total'): + total = result.get('keywords_total') + elif self.run and self.run.initial_snapshot: + total = self.run.initial_snapshot.get('stage_1_initial', remaining + processed) + else: + total = remaining + processed return { 'stage_number': 1, @@ -1595,7 +1688,7 @@ class AutomationService: 'percentage': round((processed / total * 100) if total > 0 else 0), 'currently_processing': self._get_current_items(queue, 3), 'up_next': self._get_next_items(queue, 2, skip=3), - 'remaining_count': queue.count() + 'remaining_count': remaining } def _get_stage_2_state(self) -> dict: @@ -1605,7 +1698,16 @@ class AutomationService: ).order_by('id') processed = self._get_processed_count(2) - total = queue.count() + processed + remaining = queue.count() + + # Use clusters_total from incremental result if available + result = getattr(self.run, 'stage_2_result', None) + if result and result.get('clusters_total'): + total = result.get('clusters_total') + elif self.run and self.run.initial_snapshot: + total = self.run.initial_snapshot.get('stage_2_initial', remaining + processed) + else: + total = remaining + processed return { 'stage_number': 2, @@ -1616,7 +1718,7 @@ class AutomationService: 'percentage': round((processed / total * 100) if total > 0 else 0), 'currently_processing': self._get_current_items(queue, 1), 'up_next': self._get_next_items(queue, 2, skip=1), - 'remaining_count': queue.count() + 'remaining_count': remaining } def _get_stage_3_state(self) -> dict: @@ -1647,7 +1749,17 @@ class AutomationService: ).order_by('id') processed = self._get_processed_count(4) - total = queue.count() + processed + remaining = queue.count() + + # Use tasks_total from incremental result if available (during active processing) + result = getattr(self.run, 'stage_4_result', None) + if result and result.get('tasks_total'): + total = result.get('tasks_total') + elif self.run and self.run.initial_snapshot: + # Fall back to snapshot (may be updated after Stage 3) + total = self.run.initial_snapshot.get('stage_4_initial', remaining + processed) + else: + total = remaining + processed return { 'stage_number': 4, @@ -1658,7 +1770,7 @@ class AutomationService: 'percentage': round((processed / total * 100) if total > 0 else 0), 'currently_processing': self._get_current_items(queue, 1), 'up_next': self._get_next_items(queue, 2, skip=1), - 'remaining_count': queue.count() + 'remaining_count': remaining } def _get_stage_5_state(self) -> dict: diff --git a/frontend/src/components/Automation/CurrentProcessingCardV2.tsx b/frontend/src/components/Automation/CurrentProcessingCardV2.tsx new file mode 100644 index 00000000..c68c8f77 --- /dev/null +++ b/frontend/src/components/Automation/CurrentProcessingCardV2.tsx @@ -0,0 +1,378 @@ +/** + * Current Processing Card V2 - Simplified + * Shows real-time automation progress with animated progress bar + * Clean UI without cluttered "Currently Processing" and "Up Next" sections + */ +import React, { useEffect, useState, useRef } from 'react'; +import { automationService, ProcessingState, AutomationRun, PipelineStage } from '../../services/automationService'; +import { useToast } from '../ui/toast/ToastContainer'; +import Button from '../ui/button/Button'; +import { + PlayIcon, + PauseIcon, + XMarkIcon, + ClockIcon, + BoltIcon +} from '../../icons'; + +interface CurrentProcessingCardProps { + runId: string; + siteId: number; + currentRun: AutomationRun; + onUpdate: () => void; + onClose: () => void; + pipelineOverview?: PipelineStage[]; +} + +// Stage config matching AutomationPage +const STAGE_NAMES: Record = { + 1: 'Keywords → Clusters', + 2: 'Clusters → Ideas', + 3: 'Ideas → Tasks', + 4: 'Tasks → Content', + 5: 'Content → Image Prompts', + 6: 'Image Prompts → Images', + 7: 'Review Gate', +}; + +const STAGE_OUTPUT_LABELS: Record = { + 1: 'Clusters', + 2: 'Ideas', + 3: 'Tasks', + 4: 'Content', + 5: 'Image Prompts', + 6: 'Images', + 7: 'Items', +}; + +// Helper to get processed count from stage result using correct key +const getProcessedFromResult = (result: any, stageNumber: number): number => { + if (!result) return 0; + const keyMap: Record = { + 1: 'keywords_processed', + 2: 'clusters_processed', + 3: 'ideas_processed', + 4: 'tasks_processed', + 5: 'content_processed', + 6: 'images_processed', + 7: 'ready_for_review' + }; + return result[keyMap[stageNumber]] ?? 0; +}; + +// Helper to get total from stage result +const getTotalFromResult = (result: any, stageNumber: number): number => { + if (!result) return 0; + const keyMap: Record = { + 1: 'keywords_total', + 2: 'clusters_total', + 3: 'ideas_total', + 4: 'tasks_total', + 5: 'content_total', + 6: 'images_total', + 7: 'review_total' + }; + return result[keyMap[stageNumber]] ?? 0; +}; + +const formatDuration = (startedAt: string): string => { + if (!startedAt) return '0m'; + const start = new Date(startedAt).getTime(); + const now = Date.now(); + const diffMs = now - start; + const diffMins = Math.floor(diffMs / 60000); + const diffHours = Math.floor(diffMins / 60); + if (diffHours > 0) { + return `${diffHours}h ${diffMins % 60}m`; + } + return `${diffMins}m`; +}; + +const CurrentProcessingCard: React.FC = ({ + runId, + siteId, + currentRun, + onUpdate, + onClose, + pipelineOverview, +}) => { + const [processingState, setProcessingState] = useState(null); + const [isPausing, setIsPausing] = useState(false); + const [isResuming, setIsResuming] = useState(false); + const [isCancelling, setIsCancelling] = useState(false); + + // Animated progress state - moves 1% per second until 80%, then waits for actual data + const [animatedPercent, setAnimatedPercent] = useState(0); + const lastRealPercent = useRef(0); + const animationTimer = useRef | null>(null); + + const toast = useToast(); + + // Fetch processing state + useEffect(() => { + let isMounted = true; + + const fetchState = async () => { + try { + const state = await automationService.getCurrentProcessing(siteId, runId); + if (!isMounted) return; + setProcessingState(state); + + // If stage completed, trigger update + if (state && state.processed_items >= state.total_items && state.total_items > 0) { + onUpdate(); + } + } catch (err) { + console.error('Error fetching processing state:', err); + } + }; + + if (currentRun.status === 'running' || currentRun.status === 'paused') { + fetchState(); + const interval = setInterval(fetchState, 3000); + return () => { + isMounted = false; + clearInterval(interval); + }; + } + + return () => { isMounted = false; }; + }, [siteId, runId, currentRun.status, currentRun.current_stage, onUpdate]); + + // Get real values + const stageOverview = pipelineOverview?.find(s => s.number === currentRun.current_stage); + const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`]; + + const realProcessed = processingState?.processed_items ?? getProcessedFromResult(stageResult, currentRun.current_stage); + const realTotal = processingState?.total_items ?? getTotalFromResult(stageResult, currentRun.current_stage) ?? (stageOverview?.pending ?? 0) + realProcessed; + const realPercent = realTotal > 0 ? Math.round((realProcessed / realTotal) * 100) : 0; + + // Animated progress: moves 1% per second up to 80%, then follows real progress + useEffect(() => { + // Track when real percent changes + if (realPercent > lastRealPercent.current) { + lastRealPercent.current = realPercent; + setAnimatedPercent(realPercent); + } + + // Clear existing timer + if (animationTimer.current) { + clearInterval(animationTimer.current); + } + + // Only animate if running and not paused + if (currentRun.status !== 'running') { + return; + } + + // Animate 1% per second up to 80% of current ceiling + const ceiling = Math.min(80, realPercent + 20); // Don't go more than 20% ahead + animationTimer.current = setInterval(() => { + setAnimatedPercent(prev => { + if (prev >= ceiling || prev >= realPercent + 10) { + return prev; // Stop animation + } + return Math.min(prev + 1, ceiling); + }); + }, 1000); + + return () => { + if (animationTimer.current) { + clearInterval(animationTimer.current); + } + }; + }, [currentRun.status, realPercent]); + + // Use the higher of animated or real percent + const displayPercent = Math.max(animatedPercent, realPercent); + + const isPaused = currentRun.status === 'paused'; + const stageName = STAGE_NAMES[currentRun.current_stage] || `Stage ${currentRun.current_stage}`; + const outputLabel = STAGE_OUTPUT_LABELS[currentRun.current_stage] || 'Items'; + + const handlePause = async () => { + setIsPausing(true); + try { + await automationService.pause(siteId, runId); + toast?.success('Automation pausing...'); + setTimeout(onUpdate, 1000); + } catch (error: any) { + toast?.error(error?.message || 'Failed to pause'); + } finally { + setIsPausing(false); + } + }; + + const handleResume = async () => { + setIsResuming(true); + try { + await automationService.resume(siteId, runId); + toast?.success('Automation resumed'); + setTimeout(onUpdate, 1000); + } catch (error: any) { + toast?.error(error?.message || 'Failed to resume'); + } finally { + setIsResuming(false); + } + }; + + const handleCancel = async () => { + if (!confirm('Cancel this automation run? Progress will be saved.')) return; + setIsCancelling(true); + try { + await automationService.cancel(siteId, runId); + toast?.success('Automation cancelled'); + setTimeout(onUpdate, 500); + } catch (error: any) { + toast?.error(error?.message || 'Failed to cancel'); + } finally { + setIsCancelling(false); + } + }; + + return ( +
+ {/* Header Row */} +
+
+
+ {isPaused ? ( + + ) : ( + + )} +
+
+

+ {isPaused ? 'Automation Paused' : 'Automation In Progress'} +

+
+ Stage {currentRun.current_stage}: {stageName} + + {stageOverview?.type || 'AI'} + +
+
+
+ + {/* Close and Actions */} +
+ +
+
+ + {/* Progress Section */} +
+ {/* Main Progress - 70% width */} +
+ {/* Progress Text */} +
+
+ + {displayPercent}% + + + {realProcessed}/{realTotal} {outputLabel} + +
+ + {realTotal - realProcessed} remaining + +
+ + {/* Progress Bar */} +
+
+
+ + {/* Control Buttons */} +
+ {currentRun.status === 'running' ? ( + + ) : currentRun.status === 'paused' ? ( + + ) : null} + + +
+
+ + {/* Metrics - 30% width */} +
+
+
+ + Duration +
+ {formatDuration(currentRun.started_at)} +
+ +
+
+ + Credits +
+ {currentRun.total_credits_used} +
+ +
+ Stage + {currentRun.current_stage} of 7 +
+ +
+ Status + + {isPaused ? 'Paused' : 'Running'} + +
+
+
+
+ ); +}; + +export default CurrentProcessingCard; diff --git a/frontend/src/components/Automation/GlobalProgressBar.tsx b/frontend/src/components/Automation/GlobalProgressBar.tsx index d56f6118..d2a6b68c 100644 --- a/frontend/src/components/Automation/GlobalProgressBar.tsx +++ b/frontend/src/components/Automation/GlobalProgressBar.tsx @@ -58,6 +58,11 @@ const GlobalProgressBar: React.FC = ({ stages, initialSnapshot, }) => { + // Animated progress state - moves 1% every 10 seconds + const [animatedPercent, setAnimatedPercent] = React.useState(0); + const lastRealPercent = React.useRef(0); + const animationTimer = React.useRef | null>(null); + // Don't render if no run or run is completed with 100% if (!currentRun) { return null; @@ -99,7 +104,43 @@ const GlobalProgressBar: React.FC = ({ }; }; - const { percentage, completed, total } = calculateGlobalProgress(); + const { percentage: realPercent, completed, total } = calculateGlobalProgress(); + + // Animated progress: moves 1% per 10 seconds, within current stage bounds + React.useEffect(() => { + if (realPercent > lastRealPercent.current) { + lastRealPercent.current = realPercent; + setAnimatedPercent(realPercent); + } + + if (animationTimer.current) { + clearInterval(animationTimer.current); + } + + if (currentRun.status !== 'running') { + return; + } + + // Calculate ceiling based on current stage (each stage is ~14% of total) + const stageCeiling = Math.min(currentRun.current_stage * 14, realPercent + 10); + + animationTimer.current = setInterval(() => { + setAnimatedPercent(prev => { + if (prev >= stageCeiling || prev >= realPercent + 5) { + return prev; + } + return Math.min(prev + 1, stageCeiling); + }); + }, 10000); // 1% every 10 seconds + + return () => { + if (animationTimer.current) { + clearInterval(animationTimer.current); + } + }; + }, [currentRun.status, realPercent, currentRun.current_stage]); + + const percentage = Math.max(animatedPercent, realPercent); // Hide if completed and at 100% if (currentRun.status === 'completed' && percentage >= 100) { diff --git a/frontend/src/pages/Automation/AutomationPage.tsx b/frontend/src/pages/Automation/AutomationPage.tsx index a108ffe5..c6dc4a74 100644 --- a/frontend/src/pages/Automation/AutomationPage.tsx +++ b/frontend/src/pages/Automation/AutomationPage.tsx @@ -17,7 +17,7 @@ import { import ActivityLog from '../../components/Automation/ActivityLog'; import ConfigModal from '../../components/Automation/ConfigModal'; import RunHistory from '../../components/Automation/RunHistory'; -import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard'; +import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCardV2'; import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar'; import PageMeta from '../../components/common/PageMeta'; import PageHeader from '../../components/common/PageHeader'; @@ -960,35 +960,10 @@ const AutomationPage: React.FC = () => {
- {stage7.pending > 0 && ( -
-
{stage7.pending}
-
ready for review
-
- )} - -
- -
- -
+ {/* Simplified: Just show the count, no buttons */} +
+
{stage7.pending}
+
ready for review
);