automation fixes (part2)

This commit is contained in:
IGNY8 VPS (Salman)
2025-12-28 03:15:39 +00:00
parent d4b9c8693a
commit f92b3fba6e
4 changed files with 543 additions and 37 deletions

View File

@@ -284,6 +284,22 @@ class AutomationService:
stage_number, f"Batch {batch_num} complete"
)
# INCREMENTAL SAVE: Update stage result after each batch for real-time UI progress
clusters_so_far = Clusters.objects.filter(
site=self.site,
created_at__gte=self.run.started_at
).count()
self.run.stage_1_result = {
'keywords_processed': keywords_processed,
'keywords_total': len(keyword_ids),
'clusters_created': clusters_so_far,
'batches_run': batches_run,
'credits_used': self._get_credits_used() - credits_before,
'time_elapsed': self._format_time_elapsed(start_time),
'in_progress': True
}
self.run.save(update_fields=['stage_1_result'])
# Emit per-item trace event for UI progress tracking
try:
self.logger.append_trace(self.account.id, self.site.id, self.run.run_id, {
@@ -358,6 +374,10 @@ class AutomationService:
}
self.run.current_stage = 2
self.run.total_credits_used += credits_used
# UPDATE SNAPSHOT: Record new items created for Stage 2
self._update_snapshot_after_stage(1, {'stage_2_initial': clusters_created})
self.run.save()
logger.info(f"[AutomationService] Stage 1 complete: {keywords_processed} keywords → {clusters_created} clusters")
@@ -484,6 +504,21 @@ class AutomationService:
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Cluster '{cluster.name}' complete"
)
# INCREMENTAL SAVE: Update stage result after each cluster for real-time UI progress
ideas_so_far = ContentIdeas.objects.filter(
site=self.site,
created_at__gte=self.run.started_at
).count()
self.run.stage_2_result = {
'clusters_processed': clusters_processed,
'clusters_total': total_count,
'ideas_created': ideas_so_far,
'credits_used': self._get_credits_used() - credits_before,
'time_elapsed': self._format_time_elapsed(start_time),
'in_progress': True
}
self.run.save(update_fields=['stage_2_result'])
except Exception as e:
# FIXED: Log error but continue processing remaining clusters
error_msg = f"Failed to generate ideas for cluster '{cluster.name}': {str(e)}"
@@ -525,6 +560,10 @@ class AutomationService:
}
self.run.current_stage = 3
self.run.total_credits_used += credits_used
# UPDATE SNAPSHOT: Record new items created for Stage 3
self._update_snapshot_after_stage(2, {'stage_3_initial': ideas_created})
self.run.save()
logger.info(f"[AutomationService] Stage 2 complete: {clusters_processed} clusters → {ideas_created} ideas")
@@ -687,6 +726,10 @@ class AutomationService:
'time_elapsed': time_elapsed
}
self.run.current_stage = 4
# UPDATE SNAPSHOT: Record new items created for Stage 4
self._update_snapshot_after_stage(3, {'stage_4_initial': tasks_created})
self.run.save()
logger.info(f"[AutomationService] Stage 3 complete: {ideas_processed} ideas → {tasks_created} tasks")
@@ -809,6 +852,21 @@ class AutomationService:
stage_number, f"Task '{task.title}' complete ({tasks_processed}/{total_tasks})"
)
# INCREMENTAL SAVE: Update stage result after each item for real-time UI progress
content_created_so_far = Content.objects.filter(
site=self.site,
created_at__gte=self.run.started_at
).count()
self.run.stage_4_result = {
'tasks_processed': tasks_processed,
'tasks_total': total_tasks,
'content_created': content_created_so_far,
'credits_used': self._get_credits_used() - credits_before,
'time_elapsed': self._format_time_elapsed(start_time),
'in_progress': True
}
self.run.save(update_fields=['stage_4_result'])
# Emit per-item trace event for UI progress tracking
try:
self.logger.append_trace(self.account.id, self.site.id, self.run.run_id, {
@@ -1424,6 +1482,32 @@ class AutomationService:
logger.info(f"[AutomationService] Initial snapshot captured: {snapshot}")
return snapshot
def _update_snapshot_after_stage(self, completed_stage: int, updates: dict):
"""
Update snapshot after a stage completes with new items created.
This ensures accurate counts for cascading stages.
Args:
completed_stage: The stage number that just completed
updates: Dict of snapshot keys to update, e.g., {'stage_4_initial': 12}
"""
if not self.run or not self.run.initial_snapshot:
return
snapshot = self.run.initial_snapshot.copy()
old_total = snapshot.get('total_initial_items', 0)
for key, value in updates.items():
old_value = snapshot.get(key, 0)
snapshot[key] = value
# Adjust total
old_total = old_total - old_value + value
snapshot['total_initial_items'] = old_total
self.run.initial_snapshot = snapshot
logger.info(f"[AutomationService] Snapshot updated after Stage {completed_stage}: {updates}, new total: {old_total}")
# Helper methods
def _wait_for_task(self, task_id: str, stage_number: int, item_name: str, continue_on_error: bool = True):
@@ -1584,7 +1668,16 @@ class AutomationService:
).order_by('id')
processed = self._get_processed_count(1)
total = queue.count() + processed
remaining = queue.count()
# Use keywords_total from incremental result if available
result = getattr(self.run, 'stage_1_result', None)
if result and result.get('keywords_total'):
total = result.get('keywords_total')
elif self.run and self.run.initial_snapshot:
total = self.run.initial_snapshot.get('stage_1_initial', remaining + processed)
else:
total = remaining + processed
return {
'stage_number': 1,
@@ -1595,7 +1688,7 @@ class AutomationService:
'percentage': round((processed / total * 100) if total > 0 else 0),
'currently_processing': self._get_current_items(queue, 3),
'up_next': self._get_next_items(queue, 2, skip=3),
'remaining_count': queue.count()
'remaining_count': remaining
}
def _get_stage_2_state(self) -> dict:
@@ -1605,7 +1698,16 @@ class AutomationService:
).order_by('id')
processed = self._get_processed_count(2)
total = queue.count() + processed
remaining = queue.count()
# Use clusters_total from incremental result if available
result = getattr(self.run, 'stage_2_result', None)
if result and result.get('clusters_total'):
total = result.get('clusters_total')
elif self.run and self.run.initial_snapshot:
total = self.run.initial_snapshot.get('stage_2_initial', remaining + processed)
else:
total = remaining + processed
return {
'stage_number': 2,
@@ -1616,7 +1718,7 @@ class AutomationService:
'percentage': round((processed / total * 100) if total > 0 else 0),
'currently_processing': self._get_current_items(queue, 1),
'up_next': self._get_next_items(queue, 2, skip=1),
'remaining_count': queue.count()
'remaining_count': remaining
}
def _get_stage_3_state(self) -> dict:
@@ -1647,7 +1749,17 @@ class AutomationService:
).order_by('id')
processed = self._get_processed_count(4)
total = queue.count() + processed
remaining = queue.count()
# Use tasks_total from incremental result if available (during active processing)
result = getattr(self.run, 'stage_4_result', None)
if result and result.get('tasks_total'):
total = result.get('tasks_total')
elif self.run and self.run.initial_snapshot:
# Fall back to snapshot (may be updated after Stage 3)
total = self.run.initial_snapshot.get('stage_4_initial', remaining + processed)
else:
total = remaining + processed
return {
'stage_number': 4,
@@ -1658,7 +1770,7 @@ class AutomationService:
'percentage': round((processed / total * 100) if total > 0 else 0),
'currently_processing': self._get_current_items(queue, 1),
'up_next': self._get_next_items(queue, 2, skip=1),
'remaining_count': queue.count()
'remaining_count': remaining
}
def _get_stage_5_state(self) -> dict:

View File

@@ -0,0 +1,378 @@
/**
* Current Processing Card V2 - Simplified
* Shows real-time automation progress with animated progress bar
* Clean UI without cluttered "Currently Processing" and "Up Next" sections
*/
import React, { useEffect, useState, useRef } from 'react';
import { automationService, ProcessingState, AutomationRun, PipelineStage } from '../../services/automationService';
import { useToast } from '../ui/toast/ToastContainer';
import Button from '../ui/button/Button';
import {
PlayIcon,
PauseIcon,
XMarkIcon,
ClockIcon,
BoltIcon
} from '../../icons';
interface CurrentProcessingCardProps {
runId: string;
siteId: number;
currentRun: AutomationRun;
onUpdate: () => void;
onClose: () => void;
pipelineOverview?: PipelineStage[];
}
// Stage config matching AutomationPage
const STAGE_NAMES: Record<number, string> = {
1: 'Keywords → Clusters',
2: 'Clusters → Ideas',
3: 'Ideas → Tasks',
4: 'Tasks → Content',
5: 'Content → Image Prompts',
6: 'Image Prompts → Images',
7: 'Review Gate',
};
const STAGE_OUTPUT_LABELS: Record<number, string> = {
1: 'Clusters',
2: 'Ideas',
3: 'Tasks',
4: 'Content',
5: 'Image Prompts',
6: 'Images',
7: 'Items',
};
// Helper to get processed count from stage result using correct key
const getProcessedFromResult = (result: any, stageNumber: number): number => {
if (!result) return 0;
const keyMap: Record<number, string> = {
1: 'keywords_processed',
2: 'clusters_processed',
3: 'ideas_processed',
4: 'tasks_processed',
5: 'content_processed',
6: 'images_processed',
7: 'ready_for_review'
};
return result[keyMap[stageNumber]] ?? 0;
};
// Helper to get total from stage result
const getTotalFromResult = (result: any, stageNumber: number): number => {
if (!result) return 0;
const keyMap: Record<number, string> = {
1: 'keywords_total',
2: 'clusters_total',
3: 'ideas_total',
4: 'tasks_total',
5: 'content_total',
6: 'images_total',
7: 'review_total'
};
return result[keyMap[stageNumber]] ?? 0;
};
const formatDuration = (startedAt: string): string => {
if (!startedAt) return '0m';
const start = new Date(startedAt).getTime();
const now = Date.now();
const diffMs = now - start;
const diffMins = Math.floor(diffMs / 60000);
const diffHours = Math.floor(diffMins / 60);
if (diffHours > 0) {
return `${diffHours}h ${diffMins % 60}m`;
}
return `${diffMins}m`;
};
const CurrentProcessingCard: React.FC<CurrentProcessingCardProps> = ({
runId,
siteId,
currentRun,
onUpdate,
onClose,
pipelineOverview,
}) => {
const [processingState, setProcessingState] = useState<ProcessingState | null>(null);
const [isPausing, setIsPausing] = useState(false);
const [isResuming, setIsResuming] = useState(false);
const [isCancelling, setIsCancelling] = useState(false);
// Animated progress state - moves 1% per second until 80%, then waits for actual data
const [animatedPercent, setAnimatedPercent] = useState(0);
const lastRealPercent = useRef(0);
const animationTimer = useRef<ReturnType<typeof setInterval> | null>(null);
const toast = useToast();
// Fetch processing state
useEffect(() => {
let isMounted = true;
const fetchState = async () => {
try {
const state = await automationService.getCurrentProcessing(siteId, runId);
if (!isMounted) return;
setProcessingState(state);
// If stage completed, trigger update
if (state && state.processed_items >= state.total_items && state.total_items > 0) {
onUpdate();
}
} catch (err) {
console.error('Error fetching processing state:', err);
}
};
if (currentRun.status === 'running' || currentRun.status === 'paused') {
fetchState();
const interval = setInterval(fetchState, 3000);
return () => {
isMounted = false;
clearInterval(interval);
};
}
return () => { isMounted = false; };
}, [siteId, runId, currentRun.status, currentRun.current_stage, onUpdate]);
// Get real values
const stageOverview = pipelineOverview?.find(s => s.number === currentRun.current_stage);
const stageResult = (currentRun as any)[`stage_${currentRun.current_stage}_result`];
const realProcessed = processingState?.processed_items ?? getProcessedFromResult(stageResult, currentRun.current_stage);
const realTotal = processingState?.total_items ?? getTotalFromResult(stageResult, currentRun.current_stage) ?? (stageOverview?.pending ?? 0) + realProcessed;
const realPercent = realTotal > 0 ? Math.round((realProcessed / realTotal) * 100) : 0;
// Animated progress: moves 1% per second up to 80%, then follows real progress
useEffect(() => {
// Track when real percent changes
if (realPercent > lastRealPercent.current) {
lastRealPercent.current = realPercent;
setAnimatedPercent(realPercent);
}
// Clear existing timer
if (animationTimer.current) {
clearInterval(animationTimer.current);
}
// Only animate if running and not paused
if (currentRun.status !== 'running') {
return;
}
// Animate 1% per second up to 80% of current ceiling
const ceiling = Math.min(80, realPercent + 20); // Don't go more than 20% ahead
animationTimer.current = setInterval(() => {
setAnimatedPercent(prev => {
if (prev >= ceiling || prev >= realPercent + 10) {
return prev; // Stop animation
}
return Math.min(prev + 1, ceiling);
});
}, 1000);
return () => {
if (animationTimer.current) {
clearInterval(animationTimer.current);
}
};
}, [currentRun.status, realPercent]);
// Use the higher of animated or real percent
const displayPercent = Math.max(animatedPercent, realPercent);
const isPaused = currentRun.status === 'paused';
const stageName = STAGE_NAMES[currentRun.current_stage] || `Stage ${currentRun.current_stage}`;
const outputLabel = STAGE_OUTPUT_LABELS[currentRun.current_stage] || 'Items';
const handlePause = async () => {
setIsPausing(true);
try {
await automationService.pause(siteId, runId);
toast?.success('Automation pausing...');
setTimeout(onUpdate, 1000);
} catch (error: any) {
toast?.error(error?.message || 'Failed to pause');
} finally {
setIsPausing(false);
}
};
const handleResume = async () => {
setIsResuming(true);
try {
await automationService.resume(siteId, runId);
toast?.success('Automation resumed');
setTimeout(onUpdate, 1000);
} catch (error: any) {
toast?.error(error?.message || 'Failed to resume');
} finally {
setIsResuming(false);
}
};
const handleCancel = async () => {
if (!confirm('Cancel this automation run? Progress will be saved.')) return;
setIsCancelling(true);
try {
await automationService.cancel(siteId, runId);
toast?.success('Automation cancelled');
setTimeout(onUpdate, 500);
} catch (error: any) {
toast?.error(error?.message || 'Failed to cancel');
} finally {
setIsCancelling(false);
}
};
return (
<div className={`rounded-xl p-5 mb-6 border-2 transition-all ${
isPaused
? 'bg-gradient-to-r from-amber-50 to-amber-100 dark:from-amber-900/20 dark:to-amber-800/20 border-amber-400'
: 'bg-gradient-to-r from-blue-50 to-blue-100 dark:from-blue-900/20 dark:to-blue-800/20 border-blue-400'
}`}>
{/* Header Row */}
<div className="flex items-start justify-between mb-4">
<div className="flex items-center gap-3">
<div className={`size-10 rounded-lg flex items-center justify-center shadow-md ${
isPaused
? 'bg-gradient-to-br from-amber-500 to-amber-600'
: 'bg-gradient-to-br from-blue-500 to-blue-600'
}`}>
{isPaused ? (
<PauseIcon className="w-5 h-5 text-white" />
) : (
<BoltIcon className="w-5 h-5 text-white animate-pulse" />
)}
</div>
<div>
<h2 className="text-lg font-bold text-gray-900 dark:text-white">
{isPaused ? 'Automation Paused' : 'Automation In Progress'}
</h2>
<div className="text-sm text-gray-600 dark:text-gray-400">
Stage {currentRun.current_stage}: {stageName}
<span className={`ml-2 px-1.5 py-0.5 rounded text-xs ${
isPaused ? 'bg-amber-200 text-amber-800' : 'bg-blue-200 text-blue-800'
}`}>
{stageOverview?.type || 'AI'}
</span>
</div>
</div>
</div>
{/* Close and Actions */}
<div className="flex items-center gap-2">
<Button variant="ghost" size="sm" onClick={onClose}>
<XMarkIcon className="w-4 h-4" />
<span className="ml-1">Close</span>
</Button>
</div>
</div>
{/* Progress Section */}
<div className="flex gap-6">
{/* Main Progress - 70% width */}
<div className="flex-1">
{/* Progress Text */}
<div className="flex items-baseline justify-between mb-2">
<div className="flex items-baseline gap-3">
<span className={`text-4xl font-bold ${isPaused ? 'text-amber-600' : 'text-blue-600'}`}>
{displayPercent}%
</span>
<span className="text-sm text-gray-500">
{realProcessed}/{realTotal} {outputLabel}
</span>
</div>
<span className="text-xs text-gray-400">
{realTotal - realProcessed} remaining
</span>
</div>
{/* Progress Bar */}
<div className="w-full bg-gray-200 dark:bg-gray-700 rounded-full h-3 overflow-hidden">
<div
className={`h-3 rounded-full transition-all duration-300 ${
isPaused
? 'bg-gradient-to-r from-amber-400 to-amber-600'
: 'bg-gradient-to-r from-blue-400 to-blue-600'
} ${!isPaused && displayPercent < 100 ? 'animate-pulse' : ''}`}
style={{ width: `${Math.min(displayPercent, 100)}%` }}
/>
</div>
{/* Control Buttons */}
<div className="flex items-center gap-3 mt-4">
{currentRun.status === 'running' ? (
<Button
onClick={handlePause}
disabled={isPausing}
variant="secondary"
size="sm"
startIcon={<PauseIcon className="w-4 h-4" />}
>
{isPausing ? 'Pausing...' : 'Pause'}
</Button>
) : currentRun.status === 'paused' ? (
<Button
onClick={handleResume}
disabled={isResuming}
variant="primary"
size="sm"
startIcon={<PlayIcon className="w-4 h-4" />}
>
{isResuming ? 'Resuming...' : 'Resume'}
</Button>
) : null}
<button
onClick={handleCancel}
disabled={isCancelling}
className="text-sm text-gray-500 hover:text-red-600 transition-colors"
>
<XMarkIcon className="w-4 h-4 inline mr-1" />
{isCancelling ? 'Cancelling...' : 'Cancel'}
</button>
</div>
</div>
{/* Metrics - 30% width */}
<div className="w-44 space-y-2">
<div className="bg-white dark:bg-gray-800 rounded-lg px-3 py-2 border border-gray-200 dark:border-gray-700 flex items-center justify-between">
<div className="flex items-center gap-1.5">
<ClockIcon className="w-4 h-4 text-gray-400" />
<span className="text-xs text-gray-500 uppercase">Duration</span>
</div>
<span className="text-sm font-bold text-gray-900 dark:text-white">{formatDuration(currentRun.started_at)}</span>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg px-3 py-2 border border-gray-200 dark:border-gray-700 flex items-center justify-between">
<div className="flex items-center gap-1.5">
<BoltIcon className="w-4 h-4 text-amber-500" />
<span className="text-xs text-gray-500 uppercase">Credits</span>
</div>
<span className="text-sm font-bold text-amber-600">{currentRun.total_credits_used}</span>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg px-3 py-2 border border-gray-200 dark:border-gray-700 flex items-center justify-between">
<span className="text-xs text-gray-500 uppercase">Stage</span>
<span className="text-sm font-bold text-gray-900 dark:text-white">{currentRun.current_stage} of 7</span>
</div>
<div className="bg-white dark:bg-gray-800 rounded-lg px-3 py-2 border border-gray-200 dark:border-gray-700 flex items-center justify-between">
<span className="text-xs text-gray-500 uppercase">Status</span>
<span className={`text-sm font-bold ${isPaused ? 'text-amber-600' : 'text-green-600'}`}>
{isPaused ? 'Paused' : 'Running'}
</span>
</div>
</div>
</div>
</div>
);
};
export default CurrentProcessingCard;

View File

@@ -58,6 +58,11 @@ const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({
stages,
initialSnapshot,
}) => {
// Animated progress state - moves 1% every 10 seconds
const [animatedPercent, setAnimatedPercent] = React.useState(0);
const lastRealPercent = React.useRef(0);
const animationTimer = React.useRef<ReturnType<typeof setInterval> | null>(null);
// Don't render if no run or run is completed with 100%
if (!currentRun) {
return null;
@@ -99,7 +104,43 @@ const GlobalProgressBar: React.FC<GlobalProgressBarProps> = ({
};
};
const { percentage, completed, total } = calculateGlobalProgress();
const { percentage: realPercent, completed, total } = calculateGlobalProgress();
// Animated progress: moves 1% per 10 seconds, within current stage bounds
React.useEffect(() => {
if (realPercent > lastRealPercent.current) {
lastRealPercent.current = realPercent;
setAnimatedPercent(realPercent);
}
if (animationTimer.current) {
clearInterval(animationTimer.current);
}
if (currentRun.status !== 'running') {
return;
}
// Calculate ceiling based on current stage (each stage is ~14% of total)
const stageCeiling = Math.min(currentRun.current_stage * 14, realPercent + 10);
animationTimer.current = setInterval(() => {
setAnimatedPercent(prev => {
if (prev >= stageCeiling || prev >= realPercent + 5) {
return prev;
}
return Math.min(prev + 1, stageCeiling);
});
}, 10000); // 1% every 10 seconds
return () => {
if (animationTimer.current) {
clearInterval(animationTimer.current);
}
};
}, [currentRun.status, realPercent, currentRun.current_stage]);
const percentage = Math.max(animatedPercent, realPercent);
// Hide if completed and at 100%
if (currentRun.status === 'completed' && percentage >= 100) {

View File

@@ -17,7 +17,7 @@ import {
import ActivityLog from '../../components/Automation/ActivityLog';
import ConfigModal from '../../components/Automation/ConfigModal';
import RunHistory from '../../components/Automation/RunHistory';
import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCard';
import CurrentProcessingCard from '../../components/Automation/CurrentProcessingCardV2';
import GlobalProgressBar, { getProcessedFromResult } from '../../components/Automation/GlobalProgressBar';
import PageMeta from '../../components/common/PageMeta';
import PageHeader from '../../components/common/PageHeader';
@@ -960,35 +960,10 @@ const AutomationPage: React.FC = () => {
</div>
</div>
{stage7.pending > 0 && (
<div className="text-center py-4">
<div className="text-3xl font-bold text-amber-600 dark:text-amber-400">{stage7.pending}</div>
<div className="text-xs text-amber-700 dark:text-amber-300 mt-1">ready for review</div>
</div>
)}
<div className="mt-3 pt-3 border-t border-amber-200 dark:border-amber-700">
<Button
variant="primary"
tone="brand"
size="sm"
className="w-full text-xs"
disabled={stage7.pending === 0}
>
Go to Review
</Button>
<div className="mt-2">
<Button
variant="outline"
tone="success"
size="sm"
className="w-full text-xs"
disabled={stage7.pending === 0}
onClick={handlePublishAllWithoutReview}
>
Publish all Without Review
</Button>
</div>
{/* Simplified: Just show the count, no buttons */}
<div className="text-center py-4">
<div className="text-3xl font-bold text-amber-600 dark:text-amber-400">{stage7.pending}</div>
<div className="text-xs text-amber-700 dark:text-amber-300 mt-1">ready for review</div>
</div>
</div>
);