From b0522c2989d35e01d7ea978861f9e2a5c08297a0 Mon Sep 17 00:00:00 2001 From: "IGNY8 VPS (Salman)" Date: Wed, 3 Dec 2025 07:33:08 +0000 Subject: [PATCH] docs update --- AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md | 1185 +++++++++++++++ automation-plan.md | 1576 ++++++++++++++++++++ 2 files changed, 2761 insertions(+) create mode 100644 AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md create mode 100644 automation-plan.md diff --git a/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md b/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md new file mode 100644 index 00000000..44af3e0f --- /dev/null +++ b/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md @@ -0,0 +1,1185 @@ +# AI Automation Pipeline - Implementation Blueprint +**Date:** December 3, 2025 +**Purpose:** Site-level automation orchestrating existing AI functions into sequential 7-stage pipeline + +--- + +## 🎯 EXECUTIVE SUMMARY + +### What We're Building +A **site-level automation page** (`/automation`) that orchestrates 6 existing AI functions + 1 local function into a **strictly sequential** 7-stage pipeline for hands-free content generation from keywords to draft content ready for review. + +### Core Principles +βœ… **Zero Duplication** - Reuse all existing AI functions (auto_cluster, generate_ideas, generate_content, generate_image_prompts, generate_images) +βœ… **Strictly Sequential** - Stage N+1 ONLY starts when Stage N is 100% complete +βœ… **Batch Processing** - Within each stage, process items in configurable batches with queues visible +βœ… **Site-Level Scope** - NO sector filtering - operates on entire site's data +βœ… **Observable** - Real-time batch progress, detailed queue counts, stage-by-stage logs +βœ… **Safe Execution** - Distributed locks, credit reservations, idempotent stages, Celery task chaining + +### Sequential Stage Execution +- **Stage completes** β†’ Trigger next stage automatically +- **Within stage** β†’ Process batches sequentially until queue empty +- **Between stages** β†’ Hard stop, verify completion, then proceed +- **Never parallel** - Only 1 stage active at a time per site + +### Automation Stops Before Publishing +- **Stage 7 (Manual Review Gate)** - Automation ends when content is draft+images ready +- User manually reviews and publishes via existing bulk actions +- No automated publishing to WordPress (human oversight required) + +--- + +## πŸ—οΈ SCOPE & DATA MODEL + +### Site-Level Operation (NO Sector) +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Site: "example.com" β”‚ +β”‚ β”œβ”€ Keywords (ALL sectors combined) β”‚ +β”‚ β”œβ”€ Clusters (ALL sectors combined) β”‚ +β”‚ β”œβ”€ Ideas (ALL sectors combined) β”‚ +β”‚ └─ Tasks (ALL sectors combined) β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +UI: Only Site selector at top +Database queries: Filter by site_id only +No sector dropdown in automation page +``` + +### Why Site-Level? +- **Simplicity** - User manages automation per website, not per topic +- **Unified Progress** - See total content pipeline for entire site +- **Flexible Sectors** - Content can span multiple sectors naturally +- **Easier Scheduling** - One automation config per site + +--- + +## πŸ“‹ EXISTING AI FUNCTIONS (Reused, Not Duplicated) + +| Function | File | Input | Output | Credits | Already Works | +|----------|------|-------|--------|---------|---------------| +| **auto_cluster** | `ai/functions/auto_cluster.py` | Keyword IDs (max 20) | Clusters created | 1 per 5 keywords | βœ… Yes | +| **generate_ideas** | `ai/functions/generate_ideas.py` | Cluster IDs (max 5) | Ideas created | 2 per cluster | βœ… Yes | +| **generate_content** | `ai/functions/generate_content.py` | Task IDs (1 at a time) | Content draft | 1 per 500 words | βœ… Yes | +| **generate_image_prompts** | `ai/functions/generate_image_prompts.py` | Content IDs | Image prompts | 0.5 per prompt | βœ… Yes | +| **generate_images** | `ai/functions/generate_images.py` | Image prompt IDs | Generated images | 1-4 per image | βœ… Yes | +| **bulk_queue_to_writer** | `modules/planner/views.py#L1084` | Idea IDs | Tasks created | 0 (local) | βœ… Yes | + +**All functions already:** +- Have async Celery tasks +- Return task_id for progress tracking +- Deduct credits automatically +- Update model statuses (new β†’ mapped β†’ queued β†’ completed) +- Handle errors gracefully + +--- + +## πŸ—οΈ NEW COMPONENTS TO BUILD + +### Phase 1: Backend Infrastructure + +#### 1.1 Database Models +**File:** `backend/igny8_core/business/automation/models.py` + +```python +class AutomationRun(SiteSectorBaseModel): + """Track each automation run""" + run_id = models.CharField(max_length=100, unique=True, db_index=True) + # Format: run_20251203_140523_manual or run_20251204_020000_scheduled + + trigger_type = models.CharField(max_length=20, choices=[ + ('manual', 'Manual'), + ('scheduled', 'Scheduled') + ]) + + status = models.CharField(max_length=20, choices=[ + ('running', 'Running'), + ('paused', 'Paused'), + ('completed', 'Completed'), + ('failed', 'Failed') + ], default='running') + + current_stage = models.IntegerField(default=1) # 1-7 + + started_at = models.DateTimeField(auto_now_add=True) + completed_at = models.DateTimeField(null=True, blank=True) + + total_credits_used = models.IntegerField(default=0) + + # Stage results (JSON) + stage_1_result = models.JSONField(default=dict, blank=True) # {clusters_created: 8, keywords_processed: 47} + stage_2_result = models.JSONField(default=dict, blank=True) # {ideas_created: 56} + stage_3_result = models.JSONField(default=dict, blank=True) # {tasks_created: 56} + stage_4_result = models.JSONField(default=dict, blank=True) # {content_created: 56} + stage_5_result = models.JSONField(default=dict, blank=True) # {prompts_created: 224} + stage_6_result = models.JSONField(default=dict, blank=True) # {images_created: 224} + stage_7_result = models.JSONField(default=dict, blank=True) # {ready_for_review: 56} + + error_message = models.TextField(blank=True, null=True) + + class Meta: + ordering = ['-started_at'] + indexes = [ + models.Index(fields=['run_id']), + models.Index(fields=['site', 'sector', '-started_at']), + ] + + +class AutomationConfig(SiteSectorBaseModel): + """Store automation schedule and settings per site/sector""" + + is_enabled = models.BooleanField(default=False) + + # Schedule + frequency = models.CharField(max_length=20, choices=[ + ('daily', 'Daily'), + ('weekly', 'Weekly'), + ('monthly', 'Monthly') + ], default='daily') + + scheduled_time = models.TimeField(default='02:00') # 2:00 AM + + # Batch sizes (sensible defaults from plan) + stage_1_batch_size = models.IntegerField(default=20) # Keywords per batch + stage_2_batch_size = models.IntegerField(default=1) # Clusters at a time + stage_3_batch_size = models.IntegerField(default=20) # Ideas per batch + stage_4_batch_size = models.IntegerField(default=1) # Tasks (sequential) + stage_5_batch_size = models.IntegerField(default=1) # Content at a time + stage_6_batch_size = models.IntegerField(default=1) # Images (auto-handled) + + last_run_at = models.DateTimeField(null=True, blank=True) + next_run_at = models.DateTimeField(null=True, blank=True) + + class Meta: + unique_together = [['site', 'sector']] +``` + +#### 1.2 Logging Service +**File:** `backend/igny8_core/business/automation/services/automation_logger.py` + +```python +import os +import logging +from datetime import datetime +from django.conf import settings + +class AutomationLogger: + """File-based logging for automation runs""" + + def __init__(self): + self.base_path = os.path.join(settings.BASE_DIR, 'logs', 'automation') + + def start_run(self, account_id, site_id, sector_id, trigger_type): + """Create run_id and log directory""" + timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') + run_id = f"run_{timestamp}_{trigger_type}" + + # Create directory: logs/automation/{account_id}/{site_id}/{sector_id}/{run_id}/ + log_dir = os.path.join( + self.base_path, + str(account_id), + str(site_id), + str(sector_id), + run_id + ) + os.makedirs(log_dir, exist_ok=True) + + # Create main log file + main_log = os.path.join(log_dir, 'automation_run.log') + with open(main_log, 'w') as f: + f.write(f"{'='*60}\\n") + f.write(f"AUTOMATION RUN: {run_id}\\n") + f.write(f"Started: {datetime.now()}\\n") + f.write(f"Trigger: {trigger_type}\\n") + f.write(f"{'='*60}\\n\\n") + + return run_id + + def log_stage_start(self, run_id, stage_number, stage_name, pending_count, account_id, site_id, sector_id): + """Log start of a stage""" + stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) + timestamp = datetime.now().strftime('%H:%M:%S') + + with open(stage_file, 'a') as f: + f.write(f"\\n{'='*60}\\n") + f.write(f"STAGE {stage_number}: {stage_name}\\n") + f.write(f"Started: {datetime.now()}\\n") + f.write(f"{'='*60}\\n\\n") + f.write(f"{timestamp} - Found {pending_count} items to process\\n") + + def log_stage_progress(self, run_id, stage_number, message, account_id, site_id, sector_id): + """Log progress within a stage""" + stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) + timestamp = datetime.now().strftime('%H:%M:%S') + + with open(stage_file, 'a') as f: + f.write(f"{timestamp} - {message}\\n") + + # Also append to main log + main_file = self._get_main_log(run_id, account_id, site_id, sector_id) + with open(main_file, 'a') as f: + f.write(f"{timestamp} - Stage {stage_number}: {message}\\n") + + def log_stage_complete(self, run_id, stage_number, stage_name, processed_count, time_elapsed, credits_used, account_id, site_id, sector_id): + """Log completion of a stage""" + stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) + + with open(stage_file, 'a') as f: + f.write(f"\\n{'='*60}\\n") + f.write(f"STAGE {stage_number} COMPLETE\\n") + f.write(f"Total Time: {time_elapsed}\\n") + f.write(f"Processed: {processed_count} items\\n") + f.write(f"Credits Used: {credits_used}\\n") + f.write(f"{'='*60}\\n") + + def log_stage_error(self, run_id, stage_number, error_message, account_id, site_id, sector_id): + """Log error in a stage""" + stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) + timestamp = datetime.now().strftime('%H:%M:%S') + + with open(stage_file, 'a') as f: + f.write(f"\\n{timestamp} - ERROR: {error_message}\\n") + + def get_activity_log(self, run_id, account_id, site_id, sector_id, last_n=50): + """Get last N lines from main log""" + main_file = self._get_main_log(run_id, account_id, site_id, sector_id) + + if not os.path.exists(main_file): + return [] + + with open(main_file, 'r') as f: + lines = f.readlines() + return lines[-last_n:] # Last 50 lines + + def _get_stage_file(self, run_id, stage_number, account_id, site_id, sector_id): + """Get path to stage log file""" + log_dir = os.path.join( + self.base_path, + str(account_id), + str(site_id), + str(sector_id), + run_id + ) + return os.path.join(log_dir, f"stage_{stage_number}.log") + + def _get_main_log(self, run_id, account_id, site_id, sector_id): + """Get path to main log file""" + log_dir = os.path.join( + self.base_path, + str(account_id), + str(site_id), + str(sector_id), + run_id + ) + return os.path.join(log_dir, 'automation_run.log') +``` + +#### 1.3 Automation Service (Core Orchestrator) +**File:** `backend/igny8_core/business/automation/services/automation_service.py` + +```python +import time +from datetime import datetime, timedelta +from django.utils import timezone +from igny8_core.business.automation.models import AutomationRun, AutomationConfig +from igny8_core.business.automation.services.automation_logger import AutomationLogger +from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas +from igny8_core.business.content.models import Tasks, Content, Images +from igny8_core.business.billing.services.credit_service import CreditService + +# Import existing services (NO DUPLICATION) +from igny8_core.business.planning.services.clustering_service import ClusteringService +from igny8_core.business.planning.services.ideas_service import IdeasService +from igny8_core.ai.functions.generate_content import GenerateContentFunction +from igny8_core.ai.functions.generate_image_prompts import GenerateImagePromptsFunction +from igny8_core.ai.functions.generate_images import GenerateImagesFunction + + +class AutomationService: + """ + Orchestrates the 7-stage automation pipeline. + Reuses all existing AI functions - zero duplication. + """ + + def __init__(self, account, site, sector): + self.account = account + self.site = site + self.sector = sector + self.logger = AutomationLogger() + self.credit_service = CreditService() + + # Existing services + self.clustering_service = ClusteringService() + self.ideas_service = IdeasService() + self.content_function = GenerateContentFunction() + self.prompts_function = GenerateImagePromptsFunction() + self.images_function = GenerateImagesFunction() + + self.run = None + self.config = None + + def start_automation(self, trigger_type='manual'): + \"\"\" + Main entry point for automation. + Creates run record, executes stages sequentially. + \"\"\" + try: + # Create run record + run_id = self.logger.start_run( + self.account.id, + self.site.id, + self.sector.id, + trigger_type + ) + + self.run = AutomationRun.objects.create( + run_id=run_id, + trigger_type=trigger_type, + account=self.account, + site=self.site, + sector=self.sector, + status='running', + current_stage=1 + ) + + # Load config (for batch sizes) + self.config = AutomationConfig.objects.filter( + site=self.site, + sector=self.sector + ).first() + + if not self.config: + # Create default config + self.config = AutomationConfig.objects.create( + site=self.site, + sector=self.sector, + account=self.account + ) + + # Execute stages sequentially + self.run_stage_1() # Keywords β†’ Clusters + self.run_stage_2() # Clusters β†’ Ideas + self.run_stage_3() # Ideas β†’ Tasks + self.run_stage_4() # Tasks β†’ Content + self.run_stage_5() # Content β†’ Image Prompts + self.run_stage_6() # Image Prompts β†’ Images + self.run_stage_7() # Manual Review Gate + + # Mark complete + self.run.status = 'completed' + self.run.completed_at = timezone.now() + self.run.save() + + return { + 'success': True, + 'run_id': run_id, + 'message': 'Automation completed successfully' + } + + except Exception as e: + if self.run: + self.run.status = 'failed' + self.run.error_message = str(e) + self.run.save() + + return { + 'success': False, + 'error': str(e) + } + + def run_stage_1(self): + \"\"\"Stage 1: Keywords (status='new') β†’ Clusters (AI)\"\"\" + stage_start = time.time() + stage_number = 1 + stage_name = "Keywords β†’ Clusters (AI)" + + # Find pending keywords (status='new', cluster_id=null) + pending_keywords = Keywords.objects.filter( + site=self.site, + sector=self.sector, + status='new', + cluster__isnull=True, + disabled=False + ) + + total_count = pending_keywords.count() + + if total_count == 0: + self.logger.log_stage_progress( + self.run.run_id, stage_number, + "No pending keywords found - skipping stage", + self.account.id, self.site.id, self.sector.id + ) + self.run.current_stage = 2 + self.run.save() + return + + self.logger.log_stage_start( + self.run.run_id, stage_number, stage_name, total_count, + self.account.id, self.site.id, self.sector.id + ) + + # Process in batches (default 20 per batch) + batch_size = self.config.stage_1_batch_size + keyword_ids = list(pending_keywords.values_list('id', flat=True)) + + clusters_created = 0 + keywords_processed = 0 + credits_used = 0 + + for i in range(0, len(keyword_ids), batch_size): + batch = keyword_ids[i:i+batch_size] + batch_num = (i // batch_size) + 1 + total_batches = (len(keyword_ids) + batch_size - 1) // batch_size + + self.logger.log_stage_progress( + self.run.run_id, stage_number, + f"Processing batch {batch_num}/{total_batches} ({len(batch)} keywords)", + self.account.id, self.site.id, self.sector.id + ) + + # Call existing ClusteringService (REUSE - NO DUPLICATION) + result = self.clustering_service.cluster_keywords( + keyword_ids=batch, + account=self.account, + sector_id=self.sector.id + ) + + if result.get('success'): + clusters_created += result.get('clusters_created', 0) + keywords_processed += len(batch) + credits_used += result.get('credits_used', 0) + + self.logger.log_stage_progress( + self.run.run_id, stage_number, + f"Batch {batch_num} complete: {result.get('clusters_created', 0)} clusters created", + self.account.id, self.site.id, self.sector.id + ) + + # Save stage result + elapsed = time.time() - stage_start + self.run.stage_1_result = { + 'clusters_created': clusters_created, + 'keywords_processed': keywords_processed + } + self.run.total_credits_used += credits_used + self.run.current_stage = 2 + self.run.save() + + self.logger.log_stage_complete( + self.run.run_id, stage_number, stage_name, + keywords_processed, f"{elapsed:.0f}s", credits_used, + self.account.id, self.site.id, self.sector.id + ) + + def run_stage_2(self): + \"\"\"Stage 2: Clusters (status='new', no ideas) β†’ Ideas (AI)\"\"\" + # Similar structure to stage_1 + # Calls existing IdeasService.generate_ideas() + pass + + def run_stage_3(self): + \"\"\"Stage 3: Ideas (status='new') β†’ Tasks (Local queue)\"\"\" + # Calls existing bulk_queue_to_writer endpoint logic + pass + + def run_stage_4(self): + \"\"\"Stage 4: Tasks (status='queued') β†’ Content (AI)\"\"\" + # Calls existing GenerateContentFunction + # Process one task at a time (sequential) + pass + + def run_stage_5(self): + \"\"\"Stage 5: Content (draft) β†’ Image Prompts (AI)\"\"\" + # Calls existing GenerateImagePromptsFunction + pass + + def run_stage_6(self): + \"\"\"Stage 6: Image Prompts (pending) β†’ Images (AI)\"\"\" + # Calls existing GenerateImagesFunction + # Handles batching automatically + pass + + def run_stage_7(self): + \"\"\"Stage 7: Manual Review Gate (STOP)\"\"\" + # Just count content ready for review + # Log final status + # Automation ends here + pass +``` + +--- + +### Phase 2: API Endpoints + +#### 2.1 Automation ViewSet +**File:** `backend/igny8_core/modules/automation/views.py` (NEW MODULE) + +```python +from rest_framework import viewsets +from rest_framework.decorators import action +from igny8_core.api.base import SiteSectorModelViewSet +from igny8_core.api.response import success_response, error_response +from igny8_core.business.automation.models import AutomationRun, AutomationConfig +from igny8_core.business.automation.services.automation_service import AutomationService +from igny8_core.business.automation.services.automation_logger import AutomationLogger + +class AutomationViewSet(SiteSectorModelViewSet): + """API endpoints for automation""" + + @action(detail=False, methods=['post']) + def run_now(self, request): + """Trigger manual automation run""" + account = request.account + site = request.site + sector = request.sector + + # Start automation + service = AutomationService(account, site, sector) + result = service.start_automation(trigger_type='manual') + + if result['success']: + return success_response(data={'run_id': result['run_id']}) + else: + return error_response(message=result['error']) + + @action(detail=False, methods=['get']) + def current_run(self, request): + """Get current/latest automation run status""" + site = request.site + sector = request.sector + + run = AutomationRun.objects.filter( + site=site, + sector=sector + ).order_by('-started_at').first() + + if not run: + return success_response(data={'run': None}) + + # Get activity log + logger = AutomationLogger() + activity = logger.get_activity_log( + run.run_id, + run.account_id, + run.site_id, + run.sector_id, + last_n=50 + ) + + return success_response(data={ + 'run': { + 'run_id': run.run_id, + 'status': run.status, + 'current_stage': run.current_stage, + 'trigger_type': run.trigger_type, + 'started_at': run.started_at, + 'total_credits_used': run.total_credits_used, + 'stage_1_result': run.stage_1_result, + 'stage_2_result': run.stage_2_result, + 'stage_3_result': run.stage_3_result, + 'stage_4_result': run.stage_4_result, + 'stage_5_result': run.stage_5_result, + 'stage_6_result': run.stage_6_result, + 'stage_7_result': run.stage_7_result, + }, + 'activity_log': activity + }) + + @action(detail=False, methods=['get', 'put']) + def config(self, request): + """Get/Update automation configuration""" + site = request.site + sector = request.sector + account = request.account + + if request.method == 'GET': + config, created = AutomationConfig.objects.get_or_create( + site=site, + sector=sector, + defaults={'account': account} + ) + + return success_response(data={ + 'is_enabled': config.is_enabled, + 'frequency': config.frequency, + 'scheduled_time': config.scheduled_time, + 'next_run_at': config.next_run_at, + 'stage_1_batch_size': config.stage_1_batch_size, + 'stage_2_batch_size': config.stage_2_batch_size, + 'stage_3_batch_size': config.stage_3_batch_size, + 'stage_4_batch_size': config.stage_4_batch_size, + 'stage_5_batch_size': config.stage_5_batch_size, + 'stage_6_batch_size': config.stage_6_batch_size, + }) + + elif request.method == 'PUT': + # Update configuration + config, created = AutomationConfig.objects.get_or_create( + site=site, + sector=sector, + defaults={'account': account} + ) + + # Update fields from request + config.is_enabled = request.data.get('is_enabled', config.is_enabled) + config.frequency = request.data.get('frequency', config.frequency) + config.scheduled_time = request.data.get('scheduled_time', config.scheduled_time) + config.save() + + return success_response(message='Configuration updated') +``` + +#### 2.2 URL Configuration +**File:** `backend/igny8_core/urls/api_urls.py` (ADD) + +```python +# Add to router +router.register(r'automation', AutomationViewSet, basename='automation') +``` + +--- + +### Phase 3: Celery Scheduled Task + +#### 3.1 Periodic Task for Scheduled Runs +**File:** `backend/igny8_core/tasks/automation_tasks.py` (NEW) + +```python +from celery import shared_task +from django.utils import timezone +from datetime import datetime, timedelta +from igny8_core.business.automation.models import AutomationConfig +from igny8_core.business.automation.services.automation_service import AutomationService +from igny8_core.auth.models import Account, Site, Sector + +@shared_task(name='run_scheduled_automation') +def run_scheduled_automation(): + \"\"\" + Celery beat task - runs every hour, checks if any configs need to run + \"\"\" + now = timezone.now() + + # Find configs that: + # 1. Are enabled + # 2. Have next_run_at <= now + configs = AutomationConfig.objects.filter( + is_enabled=True, + next_run_at__lte=now + ) + + for config in configs: + try: + # Load related objects + account = config.account + site = config.site + sector = config.sector + + # Start automation + service = AutomationService(account, site, sector) + service.start_automation(trigger_type='scheduled') + + # Calculate next run time + if config.frequency == 'daily': + next_run = now + timedelta(days=1) + elif config.frequency == 'weekly': + next_run = now + timedelta(weeks=1) + elif config.frequency == 'monthly': + next_run = now + timedelta(days=30) + + # Set time to scheduled_time + next_run = next_run.replace( + hour=config.scheduled_time.hour, + minute=config.scheduled_time.minute, + second=0, + microsecond=0 + ) + + config.last_run_at = now + config.next_run_at = next_run + config.save() + + except Exception as e: + # Log error but continue with other configs + print(f"Error running scheduled automation for {config.id}: {e}") + continue +``` + +#### 3.2 Register Celery Beat Schedule +**File:** `backend/igny8_core/celery.py` (UPDATE) + +```python +app.conf.beat_schedule = { + # ... existing schedules ... + 'run-scheduled-automation': { + 'task': 'run_scheduled_automation', + 'schedule': crontab(minute=0), # Every hour on the hour + }, +} +``` + +--- + +### Phase 4: Frontend Components + +#### 4.1 Automation Page Component +**File:** `frontend/src/pages/Automation/Dashboard.tsx` (NEW) + +```tsx +import { useState, useEffect } from 'react'; +import { useInterval } from '../../hooks/useInterval'; +import { automationApi } from '../../services/api/automationApi'; +import { useSiteStore } from '../../stores/siteStore'; +import { useSectorStore } from '../../stores/sectorStore'; +import Button from '../../components/ui/button/Button'; +import { BoltIcon, PlayIcon, PauseIcon, SettingsIcon } from '../../icons'; +import StageCard from './components/StageCard'; +import ActivityLog from './components/ActivityLog'; +import ConfigModal from './components/ConfigModal'; + +export default function AutomationDashboard() { + const { activeSite } = useSiteStore(); + const { activeSector } = useSectorStore(); + + const [currentRun, setCurrentRun] = useState(null); + const [activityLog, setActivityLog] = useState([]); + const [config, setConfig] = useState(null); + const [showConfigModal, setShowConfigModal] = useState(false); + const [isRunning, setIsRunning] = useState(false); + + // Poll current run status every 3 seconds when running + useInterval(() => { + if (activeSite && activeSector) { + loadCurrentRun(); + } + }, currentRun?.status === 'running' ? 3000 : null); + + useEffect(() => { + if (activeSite && activeSector) { + loadCurrentRun(); + loadConfig(); + } + }, [activeSite?.id, activeSector?.id]); + + const loadCurrentRun = async () => { + const response = await automationApi.getCurrentRun(); + if (response.success) { + setCurrentRun(response.data.run); + setActivityLog(response.data.activity_log || []); + } + }; + + const loadConfig = async () => { + const response = await automationApi.getConfig(); + if (response.success) { + setConfig(response.data); + } + }; + + const handleRunNow = async () => { + setIsRunning(true); + const response = await automationApi.runNow(); + if (response.success) { + // Start polling + loadCurrentRun(); + } + setIsRunning(false); + }; + + return ( +
+ {/* Header */} +
+
+

+ + AI Automation Pipeline +

+

+ Automated content generation from keywords to review +

+
+ +
+ + + +
+
+ + {/* Schedule Info */} + {config && ( +
+
+
+
+ {config.is_enabled ? '⏰ Scheduled' : '⏸ Paused'} +
+
+ {config.is_enabled + ? `Next Run: ${config.next_run_at} (${config.frequency})` + : 'Scheduling disabled' + } +
+
+ +
+
+ {currentRun?.total_credits_used || 0} +
+
+ Credits Used +
+
+
+
+ )} + + {/* Pipeline Overview */} +
+

Pipeline Overview

+
+ + Keywords + + β†’ + + Clusters + + β†’ + + Ideas + + β†’ + + Tasks + + β†’ + + Content + + β†’ + + Review + +
+ + {currentRun && ( +
+
+
+
+ + Stage {currentRun.current_stage}/7 + +
+ )} +
+ + {/* Stage Cards */} +
+ + + + + + + + + + + + + +
+ + {/* Activity Log */} + + + {/* Config Modal */} + {showConfigModal && ( + setShowConfigModal(false)} + config={config} + onSave={loadConfig} + /> + )} +
+ ); +} + +function getStageStatus(run, stageNumber) { + if (!run) return 'waiting'; + if (run.status === 'running' && run.current_stage === stageNumber) return 'running'; + if (run.current_stage > stageNumber) return 'completed'; + return 'waiting'; +} +``` + +#### 4.2 Add to Sidebar Menu +**File:** `frontend/src/layouts/AppSidebar.tsx` (UPDATE) + +```tsx +// Add after Sites menu item +{ + path: '/automation', + label: 'Automation', + icon: , + badge: automationRunning ? { text: 'Running', color: 'green' } : null +}, +``` + +#### 4.3 API Service +**File:** `frontend/src/services/api/automationApi.ts` (NEW) + +```typescript +import { apiClient } from './apiClient'; + +export const automationApi = { + runNow: () => apiClient.post('/automation/run_now/'), + + getCurrentRun: () => apiClient.get('/automation/current_run/'), + + getConfig: () => apiClient.get('/automation/config/'), + + updateConfig: (data: any) => apiClient.put('/automation/config/', data), +}; +``` + +--- + +### Phase 5: Implementation Checklist + +#### Week 1: Backend Foundation +- [ ] Create `automation` module directory structure +- [ ] Implement `AutomationRun` and `AutomationConfig` models +- [ ] Run migrations +- [ ] Implement `AutomationLogger` service +- [ ] Test file logging manually + +#### Week 2: Core Service +- [ ] Implement `AutomationService` class +- [ ] Implement `run_stage_1()` (Keywords β†’ Clusters) +- [ ] Test stage 1 in isolation +- [ ] Implement `run_stage_2()` (Clusters β†’ Ideas) +- [ ] Test stage 2 in isolation +- [ ] Implement remaining stages 3-7 + +#### Week 3: API & Scheduling +- [ ] Create `AutomationViewSet` with endpoints +- [ ] Test manual run via API +- [ ] Implement Celery periodic task +- [ ] Test scheduled runs +- [ ] Add error handling and rollback + +#### Week 4: Frontend +- [ ] Create Automation page component +- [ ] Implement StageCard component +- [ ] Implement ActivityLog component +- [ ] Implement ConfigModal component +- [ ] Add to sidebar menu +- [ ] Test full UI flow + +#### Week 5: Testing & Polish +- [ ] End-to-end testing (manual + scheduled) +- [ ] Load testing (100+ keywords) +- [ ] Credit calculation verification +- [ ] Log file verification +- [ ] UI polish and responsiveness +- [ ] Documentation update + +--- + +### Phase 6: Safety Mechanisms + +#### 6.1 Pause/Resume +```python +# In AutomationService +def pause_run(self): + self.run.status = 'paused' + self.run.save() + +def resume_run(self): + self.run.status = 'running' + self.run.save() + # Resume from current_stage +``` + +#### 6.2 Rollback on Error +```python +# Each stage wraps in try-except +try: + self.run_stage_1() +except Exception as e: + self.logger.log_stage_error(self.run.run_id, 1, str(e)) + self.run.status = 'failed' + self.run.error_message = str(e) + self.run.save() + # Optionally: rollback created records + raise +``` + +#### 6.3 Credit Pre-Check +```python +# Before starting, estimate total credits needed +def estimate_credits(self): + keywords = Keywords.objects.filter(status='new', cluster__isnull=True).count() + clusters = Clusters.objects.filter(ideas__isnull=True).count() + # ... etc + + total_estimate = (keywords / 5) + (clusters * 2) + ... + + if self.account.credits_balance < total_estimate: + raise InsufficientCreditsError(f"Need ~{total_estimate} credits") +``` + +--- + +### Phase 7: Monitoring & Observability + +#### 7.1 Dashboard Metrics +- Total runs (today/week/month) +- Success rate +- Average credits per run +- Average time per stage +- Content pieces generated + +#### 7.2 Alerts +- Email when run completes +- Email on failure +- Slack notification (optional) + +--- + +## 🎨 UI/UX HIGHLIGHTS + +### Rich Visual Design +- **Stage Cards** with status badges (waiting/running/completed/failed) +- **Live Progress Bar** for current stage +- **Activity Feed** with timestamps and color-coded messages +- **Credit Counter** with real-time updates +- **Schedule Badge** showing next run time + +### User Experience +- **One-Click Run** - Single "Run Now" button +- **Real-Time Updates** - Auto-refreshes every 3 seconds when running +- **Clear Status** - Visual indicators for each stage +- **Easy Config** - Modal for schedule settings +- **Error Clarity** - Detailed error messages with stage number + +--- + +## πŸ”§ TROUBLESHOOTING GUIDE + +### Issue: Stage stuck in "running" +**Solution:** +1. Check `/logs/automation/{account}/{site}/{sector}/{run_id}/stage_X.log` +2. Look for last log entry +3. Check Celery worker logs +4. Manually mark stage complete or restart + +### Issue: Credits deducted but no results +**Solution:** +1. Check stage log for AI task_id +2. Query task progress endpoint +3. Verify AI function completed +4. Rollback transaction if needed + +### Issue: Duplicate clusters created +**Solution:** +1. Add unique constraint on cluster name per sector +2. Check deduplication logic in ClusteringService +3. Review stage_1 logs for batch processing + +--- + +## πŸ“Š SUCCESS METRICS + +After implementation, measure: +- **Automation adoption rate** (% of sites using scheduled runs) +- **Content generation volume** (pieces per day/week) +- **Time savings** (manual hours vs automated) +- **Credit efficiency** (credits per content piece) +- **Error rate** (failed runs / total runs) + +--- + +## πŸš€ FUTURE ENHANCEMENTS + +### Phase 8: Advanced Features +- **Conditional stages** (skip if no data) +- **Parallel processing** (multiple tasks at once in stage 4) +- **Smart scheduling** (avoid peak hours) +- **A/B testing** (test different prompts) +- **Content quality scoring** (auto-reject low scores) + +### Phase 9: Integrations +- **WordPress auto-publish** (with approval workflow) +- **Analytics tracking** (measure content performance) +- **Social media posting** (auto-share published content) + +--- + +**END OF IMPLEMENTATION PLAN** + +This plan provides a complete, production-ready automation system that: +βœ… Reuses all existing AI functions (zero duplication) +βœ… Modular and maintainable (each stage independent) +βœ… Observable and debuggable (file logs + database records) +βœ… Safe and reliable (error handling + rollback) +βœ… Rich UI/UX (real-time updates + visual feedback) +βœ… Scalable (handles 100+ keywords efficiently) diff --git a/automation-plan.md b/automation-plan.md new file mode 100644 index 00000000..c163e4c6 --- /dev/null +++ b/automation-plan.md @@ -0,0 +1,1576 @@ +# AI Automation Pipeline - Complete Implementation Plan +**Version:** 2.0 +**Date:** December 3, 2025 +**Scope:** Site-level automation orchestrating existing AI functions + +--- + +## 🎯 CORE ARCHITECTURE DECISIONS + +### Decision 1: Site-Level Automation (NO Sector) +**Rationale:** +- User manages automation per website, not per topic/sector +- Simpler UX - single site selector at top of page +- Database queries filter by `site_id` only (no sector_id filtering) +- Content naturally spans multiple sectors within a site +- One automation schedule per site (not per site/sector combination) + +**Implementation:** +- Remove sector dropdown from automation page UI +- AutomationRun model: Remove sector foreign key +- AutomationConfig model: One config per site (not per site+sector) +- All stage database queries: `.filter(site=site)` (no sector filter) + +--- + +### Decision 2: Single Global Automation Page +**Why:** +- Complete pipeline visibility in one place (Keywords β†’ Draft Content) +- Configure one schedule for entire lifecycle +- See exactly where pipeline is stuck or running +- Cleaner UX - no jumping between module pages + +**Location:** `/automation` (new route below Sites in sidebar) + +--- + +### Decision 3: Strictly Sequential Stages (Never Parallel) +**Critical Principle:** +- Stage N+1 ONLY starts when Stage N is 100% complete +- Within each stage: process items in batches sequentially +- Hard stop between stages to verify completion +- Only ONE stage active at a time per site + +**Example Flow:** +``` +Stage 1 starts β†’ processes ALL batches β†’ completes 100% + ↓ (trigger next) +Stage 2 starts β†’ processes ALL batches β†’ completes 100% + ↓ (trigger next) +Stage 3 starts β†’ ... +``` + +**Never:** +- Run stages in parallel +- Start next stage while current stage has pending items +- Skip verification between stages + +--- + +### Decision 4: Automation Stops Before Publishing +**Manual Review Gate (Stage 7):** +- Automation ends when content reaches `status='draft'` with all images generated +- User manually reviews content quality, accuracy, brand voice +- User manually publishes via existing bulk actions on Content page +- No automated WordPress publishing (requires human oversight) + +**Rationale:** +- Content quality control needed +- Publishing has real consequences (public-facing) +- Legal/compliance review may be required +- Brand voice verification essential + +--- + +## πŸ“Š EXISTING AI FUNCTIONS (Zero Duplication) + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ πŸ€– AI AUTOMATION PIPELINE β”‚ +β”‚ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ β”‚ +β”‚ β”‚ +β”‚ ⏰ SCHEDULE β”‚ +β”‚ Next Run: Tomorrow at 2:00 AM (in 16 hours) β”‚ +β”‚ Frequency: [Daily β–Ό] at [02:00 β–Ό] β”‚ +β”‚ Status: ● Scheduled β”‚ +β”‚ β”‚ +β”‚ [Run Now] [Pause Schedule] [Configure] β”‚ +β”‚ β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ πŸ“Š PIPELINE OVERVIEW β”‚ +β”‚ β”‚ +β”‚ Keywords ──→ Clusters ──→ Ideas ──→ Tasks ──→ Content β”‚ +β”‚ 47 pending 42 20 generating β”‚ +β”‚ pending Stage 1 ready queued Stage 5 β”‚ +β”‚ β”‚ +β”‚ Overall Progress: ━━━━━━━╸ 62% (Stage 5/7) β”‚ +β”‚ Estimated Completion: 2 hours 15 minutes β”‚ +β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 1: Keywords β†’ Clusters (AI) β”‚ +β”‚ Status: βœ“ Completed β”‚ +β”‚ β€’ Processed: 60 keywords β†’ 8 clusters β”‚ +β”‚ β€’ Time: 2m 30s | Credits: 12 β”‚ +β”‚ [View Details] [Retry Failed] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 2: Clusters β†’ Ideas (AI) β”‚ +β”‚ Status: βœ“ Completed β”‚ +β”‚ β€’ Processed: 8 clusters β†’ 56 ideas β”‚ +β”‚ β€’ Time: 8m 15s | Credits: 16 β”‚ +β”‚ [View Details] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 3: Ideas β†’ Tasks (Local Queue) β”‚ +β”‚ Status: βœ“ Completed β”‚ +β”‚ β€’ Processed: 42 ideas β†’ 42 tasks β”‚ +β”‚ β€’ Time: Instant | Credits: 0 β”‚ +β”‚ [View Details] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 4: Tasks β†’ Content (AI) β”‚ +β”‚ Status: ● Processing (Task 3/20) β”‚ +β”‚ β€’ Current: "Ultimate Coffee Bean Guide" ━━━━╸ 65% β”‚ +β”‚ β€’ Progress: 2 completed, 1 processing, 17 queued β”‚ +β”‚ β€’ Time: 45m elapsed | Credits: 38 used β”‚ +β”‚ β€’ ETA: 1h 30m remaining β”‚ +β”‚ [View Details] [Pause Stage] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 5: Content β†’ Image Prompts (AI) β”‚ +β”‚ Status: ⏸ Waiting (Stage 4 must complete) β”‚ +β”‚ β€’ Pending: 2 content pieces ready for prompts β”‚ +β”‚ β€’ Queue: Will process when Stage 4 completes β”‚ +β”‚ [View Details] [Trigger Now] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 6: Image Prompts β†’ Generated Images (AI) β”‚ +β”‚ Status: ⏸ Waiting β”‚ +β”‚ β€’ Pending: 0 prompts ready β”‚ +β”‚ [View Details] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 7: Content β†’ Review (Manual Gate) 🚫 STOPS HERE β”‚ +β”‚ Status: ⏸ Awaiting Manual Review β”‚ +β”‚ β€’ Ready for Review: 2 content pieces β”‚ +β”‚ β€’ Note: Automation stops here. User reviews manually. β”‚ +β”‚ [Go to Review Page] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +πŸ“‹ LIVE ACTIVITY LOG (Last 50 events) +β”œβ”€ 14:23:45 - Stage 4: Started content generation for Task 3 +β”œβ”€ 14:24:12 - Stage 4: Writing sections (65% complete) +β”œβ”€ 14:22:30 - Stage 4: Completed Task 2 β†’ Content created +β”œβ”€ 14:20:15 - Stage 4: Started content generation for Task 2 +β”œβ”€ 14:18:45 - Stage 4: Completed Task 1 β†’ Content created +└─ [View Full Log] + +πŸ’° TOTAL CREDITS USED THIS RUN: 66 credits +``` + +**All 6 AI Functions Already Exist and Work:** + +| Function | File Location | Input | Output | Credits | Status | +|----------|---------------|-------|--------|---------|--------| +| **auto_cluster** | `ai/functions/auto_cluster.py` | Keyword IDs (max 20) | Clusters created | 1 per 5 keywords | βœ… Working | +| **generate_ideas** | `ai/functions/generate_ideas.py` | Cluster IDs (max 5) | Ideas created | 2 per cluster | βœ… Working | +| **bulk_queue_to_writer** | `modules/planner/views.py` (line 1014) | Idea IDs | Tasks created | 0 (local) | βœ… Working | +| **generate_content** | `ai/functions/generate_content.py` | Task IDs (1 at a time) | Content draft | 1 per 500 words | βœ… Working | +| **generate_image_prompts** | `ai/functions/generate_image_prompts.py` | Content IDs | Image prompts | 0.5 per prompt | βœ… Working | +| **generate_images** | `ai/functions/generate_images.py` | Image prompt IDs | Generated images | 1-4 per image | βœ… Working | + +--- + +### 🚫 WHAT AI FUNCTIONS ALREADY DO (DO NOT DUPLICATE) + +**Credit Management** (Fully Automated in `ai/engine.py`): +```python +# Line 395 in AIEngine.execute(): +CreditService.deduct_credits_for_operation( + account=account, + operation_type=self._get_operation_type(), + amount=self._get_actual_amount(), + ... +) +``` +- βœ… Credits are AUTOMATICALLY deducted after successful save +- βœ… Credit calculation happens in `_get_actual_amount()` and `_get_operation_type()` +- ❌ Automation does NOT need to call `CreditService` manually +- ❌ Automation does NOT need to calculate credit costs + +**Status Updates** (Handled Inside AI Functions): +- βœ… Keywords: `status='new'` β†’ `status='mapped'` (in auto_cluster save_output) +- βœ… Clusters: Created with `status='new'` (in auto_cluster save_output) +- βœ… Ideas: `status='new'` β†’ `status='queued'` (in bulk_queue_to_writer) +- βœ… Tasks: Created with `status='queued'`, β†’ `status='completed'` (in generate_content) +- βœ… Content: Created with `status='draft'`, β†’ `status='review'` ONLY when all images complete (ai/tasks.py line 723) +- βœ… Images: `status='pending'` β†’ `status='generated'` (in generate_images save_output) +- ❌ Automation does NOT update these statuses directly + +**Progress Tracking** (Event-Based System Already Exists): +- βœ… `StepTracker` and `ProgressTracker` emit real-time events during AI execution +- βœ… Each AI function has 6 phases: `INIT`, `PREP`, `AI_CALL`, `PARSE`, `SAVE`, `DONE` +- βœ… Phase descriptions available in function metadata: `get_metadata()` +- ❌ Automation does NOT need to poll progress every 2 seconds +- ❌ Automation listens to existing phase events via Celery task status + +**Error Handling & Logging**: +- βœ… AIEngine wraps execution in try/catch, logs to `AIUsageLog` +- βœ… Failed operations rollback database changes automatically +- ❌ Automation only needs to check final task result (success/failure) + +--- + +**Automation Service ONLY Does:** +1. **Batch Selection**: Query database for items to process (by status and site) +2. **Function Calling**: Call existing AI functions with selected item IDs +3. **Stage Sequencing**: Wait for Stage N completion before starting Stage N+1 +4. **Scheduling**: Trigger automation runs on configurable schedules +5. **Aggregation**: Collect results from all batches and log totals per stage + +--- + +## πŸ—οΈ 7-STAGE PIPELINE ARCHITECTURE + +### Sequential Stage Flow + +| Stage | From | To | Function Used | Batch Size | Type | +|-------|------|-----|---------------|------------|------| +| **1** | Keywords (`status='new'`, `cluster_id=null`) | Clusters (`status='new'`) | `auto_cluster` | 20 keywords | AI | +| **2** | Clusters (`status='new'`, no ideas) | Ideas (`status='new'`) | `generate_ideas` | 1 cluster | AI | +| **3** | Ideas (`status='new'`) | Tasks (`status='queued'`) | `bulk_queue_to_writer` | 20 ideas | Local | +| **4** | Tasks (`status='queued'`) | Content (`status='draft'`) | `generate_content` | 1 task | AI | +| **5** | Content (`status='draft'`, no Images) | Images (`status='pending'` with prompts) | `generate_image_prompts` | 1 content | AI | +| **6** | Images (`status='pending'`) | Images (`status='generated'` with URLs) | `generate_images` | 1 image | AI | +| **7** | Content (`status='review'`) | Manual Review | None (gate) | N/A | Manual | + +--- + +### Stage 1: Keywords β†’ Clusters (AI) + +**Purpose:** Group semantically similar keywords into topic clusters + +**Database Query (Automation Orchestrator):** +```python +pending_keywords = Keywords.objects.filter( + site=site, + status='new', + cluster__isnull=True, + disabled=False +) +``` + +**Orchestration Logic (What Automation Does):** +1. **Select Batch**: Count pending keywords + - If 0 keywords β†’ Skip stage, log "No keywords to process" + - If 1-20 keywords β†’ Select all (batch_size = count) + - If >20 keywords β†’ Select first 20 (configurable batch_size) + +2. **Call AI Function**: + ```python + from igny8_core.ai.functions.auto_cluster import AutoCluster + + result = AutoCluster().execute( + payload={'ids': keyword_ids}, + account=account + ) + # Returns: {'task_id': 'celery_task_abc123'} + ``` + +3. **Monitor Progress**: Listen to Celery task status + - Use existing `StepTracker` phase events (INIT β†’ PREP β†’ AI_CALL β†’ PARSE β†’ SAVE β†’ DONE) + - OR poll: `AsyncResult(task_id).state` until SUCCESS/FAILURE + - Log phase progress: "AI analyzing keywords (65% complete)" + +4. **Collect Results**: When task completes + - AI function already updated Keywords.status β†’ 'mapped' + - AI function already created Cluster records with status='new' + - AI function already deducted credits via AIEngine + - Automation just logs: "Batch complete: N clusters created" + +5. **Repeat**: If more keywords remain, select next batch and go to step 2 + +**Stage Completion Criteria:** +- All keyword batches processed (pending_keywords.count() == 0) +- No critical errors + +**What AI Function Does (Already Implemented - DON'T DUPLICATE):** +- βœ… Groups keywords semantically using AI +- βœ… Creates Cluster records with `status='new'` +- βœ… Updates Keywords: `cluster_id=cluster.id`, `status='mapped'` +- βœ… Deducts credits automatically (AIEngine line 395) +- βœ… Logs to AIUsageLog +- βœ… Emits progress events via StepTracker + +**Stage Result Logged:** +```json +{ + "keywords_processed": 47, + "clusters_created": 8, + "batches_run": 3, + "credits_used": 10 // Read from AIUsageLog sum, not calculated +} +``` + +--- + +### Stage 2: Clusters β†’ Ideas (AI) + +**Purpose:** Generate content ideas for each cluster + +**Database Query:** +``` +Clusters.objects.filter( + site=site, + status='new', + disabled=False +).exclude( + ideas__isnull=False # Has no ideas yet +) +``` + +**Process:** +1. Count clusters without ideas +2. If 0 β†’ Skip stage +3. If > 0 β†’ Process one cluster at a time (configurable batch size = 1) +4. For each cluster: + - Log: "Generating ideas for cluster: {cluster.name}" + - Call `IdeasService.generate_ideas(cluster_ids=[cluster.id], account)` + - Function returns `{'task_id': 'xyz789'}` + - Monitor via Celery task status or StepTracker events + - Wait for completion + - Log: "Cluster '{name}' complete: N ideas created" +5. Log stage summary + +**Stage Completion Criteria:** +- All clusters processed +- Each cluster now has >=1 idea +- No errors + +**Updates:** +- ContentIdeas: New records created with `status='new'`, `keyword_cluster_id=cluster.id` +- Clusters: `status='mapped'` +- Stage result: `{clusters_processed: 8, ideas_created: 56}` + +**Credits:** ~16 credits (2 per cluster) + +--- + +### Stage 3: Ideas β†’ Tasks (Local Queue) + +**Purpose:** Convert content ideas to writer tasks (local, no AI) + +**Database Query:** +``` +ContentIdeas.objects.filter( + site=site, + status='new' +) +``` + +**Process:** +1. Count pending ideas +2. If 0 β†’ Skip stage +3. If > 0 β†’ Split into batches of 20 +4. For each batch: + - Log: "Queueing batch X/Y (20 ideas)" + - Call `bulk_queue_to_writer` view logic (NOT via HTTP, direct function call) + - For each idea: + - Create Tasks record with title=idea.idea_title, status='queued', cluster=idea.keyword_cluster + - Update idea status to 'queued' + - Log: "Batch X complete: 20 tasks created" +5. Log stage summary + +**Stage Completion Criteria:** +- All batches processed +- All ideas now have `status='queued'` +- Corresponding Tasks exist with `status='queued'` +- No errors + +**Updates:** +- Tasks: New records created with `status='queued'` +- ContentIdeas: `status` changed 'new' β†’ 'queued' +- Stage result: `{ideas_processed: 56, tasks_created: 56, batches: 3}` + +**Credits:** 0 (local operation) + +--- + +### Stage 4: Tasks β†’ Content (AI) + +**Purpose:** Generate full content drafts from tasks + +**Database Query (Automation Orchestrator):** +```python +pending_tasks = Tasks.objects.filter( + site=site, + status='queued', + content__isnull=True # No content generated yet +) +``` + +**Orchestration Logic:** +1. **Select Item**: Count queued tasks + - If 0 β†’ Skip stage + - If > 0 β†’ Select ONE task at a time (sequential processing) + +2. **Call AI Function**: + ```python + from igny8_core.ai.functions.generate_content import GenerateContent + + result = GenerateContent().execute( + payload={'ids': [task.id]}, + account=account + ) + # Returns: {'task_id': 'celery_task_xyz789'} + ``` + +3. **Monitor Progress**: Listen to Celery task status + - Use `StepTracker` phase events for real-time updates + - Log: "Writing sections (65% complete)" (from phase metadata) + - Content generation takes 5-15 minutes per task + +4. **Collect Results**: When task completes + - AI function already created Content with `status='draft'` + - AI function already updated Task.status β†’ 'completed' + - AI function already updated Idea.status β†’ 'completed' + - AI function already deducted credits based on word count + - Automation logs: "Content created (2500 words)" + +5. **Repeat**: Process next task sequentially + +**Stage Completion Criteria:** +- All tasks processed (pending_tasks.count() == 0) +- Each task has linked Content record + +**What AI Function Does (Already Implemented):** +- βœ… Generates article sections using AI +- βœ… Creates Content record with `status='draft'`, `task_id=task.id` +- βœ… Updates Task: `status='completed'` +- βœ… Updates linked Idea: `status='completed'` +- βœ… Deducts credits: 1 credit per 500 words (automatic) +- βœ… Logs to AIUsageLog with word count + +**Stage Result Logged:** +```json +{ + "tasks_processed": 56, + "content_created": 56, + "total_words": 140000, + "credits_used": 280 // From AIUsageLog, not calculated +} +``` + +--- + +### Stage 5: Content β†’ Image Prompts (AI) + +**Purpose:** Extract image prompts from content and create Images records with prompts + +**CRITICAL:** There is NO separate "ImagePrompts" model. Images records ARE the prompts (with `status='pending'`) until images are generated. + +**Database Query (Automation Orchestrator):** +```python +# Content that has NO Images records at all +content_without_images = Content.objects.filter( + site=site, + status='draft' +).annotate( + images_count=Count('images') +).filter( + images_count=0 # No Images records exist yet +) +``` + +**Orchestration Logic:** +1. **Select Item**: Count content without any Images records + - If 0 β†’ Skip stage + - If > 0 β†’ Select ONE content at a time (sequential) + +2. **Call AI Function**: + ```python + from igny8_core.ai.functions.generate_image_prompts import GenerateImagePromptsFunction + + result = GenerateImagePromptsFunction().execute( + payload={'ids': [content.id]}, + account=account + ) + # Returns: {'task_id': 'celery_task_prompts456'} + ``` + +3. **Monitor Progress**: Wait for completion + +4. **Collect Results**: When task completes + - AI function already created Images records with: + - `status='pending'` + - `prompt='...'` (AI-generated prompt text) + - `image_type='featured'` or `'in_article'` + - `content_id=content.id` + - Content.status stays `'draft'` (unchanged) + - Automation logs: "Content '{title}' complete: N prompts created" + +5. **Repeat**: Process next content sequentially + +**Stage Completion Criteria:** +- All content processed (content_without_images.count() == 0) +- Each content has >=1 Images record with `status='pending'` and prompt text + +**What AI Function Does (Already Implemented):** +- βœ… Extracts featured image prompt from title/intro +- βœ… Extracts in-article prompts from H2 headings +- βœ… Creates Images records with `status='pending'`, `prompt='...'` +- βœ… Deducts credits automatically (0.5 per prompt) +- βœ… Logs to AIUsageLog + +**Stage Result Logged:** +```json +{ + "content_processed": 56, + "prompts_created": 224, + "credits_used": 112 // From AIUsageLog +} +``` + +--- + +### Stage 6: Images (Prompts) β†’ Generated Images (AI) + +**Purpose:** Generate actual image URLs from Images records that contain prompts + +**CRITICAL:** Input is Images records with `status='pending'` (these contain the prompts). Output is same Images records updated with `status='generated'` and `image_url='https://...'` + +**Database Query (Automation Orchestrator):** +```python +# Images with prompts waiting to be generated +pending_images = Images.objects.filter( + site=site, + status='pending' # Has prompt text, needs image URL +) +``` + +**Orchestration Logic:** +1. **Select Item**: Count pending Images + - If 0 β†’ Skip stage + - If > 0 β†’ Select ONE Image at a time (sequential) + +2. **Call AI Function**: + ```python + from igny8_core.ai.functions.generate_images import GenerateImages + + result = GenerateImages().execute( + payload={'image_ids': [image.id]}, + account=account + ) + # Returns: {'task_id': 'celery_task_img789'} + ``` + +3. **Monitor Progress**: Wait for completion + +4. **Collect Results**: When task completes + - AI function already called image API using the `prompt` field + - AI function already updated Images: + - `status='pending'` β†’ `status='generated'` + - `image_url='https://...'` (populated with generated image URL) + - AI function already deducted credits (1-4 per image) + - Automation logs: "Image generated: {image_url}" + +5. **Automatic Content Status Change** (NOT done by automation): + - After each image generation, background task checks if ALL Images for that Content are now `status='generated'` + - When last image completes: Content.status changes `'draft'` β†’ `'review'` (in `ai/tasks.py` line 723) + - Automation does NOT trigger this - happens automatically + +6. **Repeat**: Process next pending Image sequentially + +**Stage Completion Criteria:** +- All pending Images processed (pending_images.count() == 0) +- All Images now have `image_url != null`, `status='generated'` + +**What AI Function Does (Already Implemented):** +- βœ… Reads `prompt` field from Images record +- βœ… Calls image generation API (OpenAI/Runware) with prompt +- βœ… Updates Images: `image_url=generated_url`, `status='generated'` +- βœ… Deducts credits automatically (1-4 per image) +- βœ… Logs to AIUsageLog + +**What Happens Automatically (ai/tasks.py:723):** +- βœ… Background task checks if all Images for a Content are `status='generated'` +- βœ… When complete: Content.status changes `'draft'` β†’ `'review'` +- βœ… This happens OUTSIDE automation orchestrator (in Celery task) + +**Stage Result Logged:** +```json +{ + "images_processed": 224, + "images_generated": 224, + "content_moved_to_review": 56, // Side effect (automatic) + "credits_used": 448 // From AIUsageLog +} +``` + +--- + +### Stage 7: Manual Review Gate (STOP) + +**Purpose:** Automation ends - content automatically moved to 'review' status ready for manual review + +**CRITICAL:** Content with `status='review'` was automatically set in Stage 6 when ALL images completed. Automation just counts them. + +**Database Query (Automation Orchestrator):** +```python +# Content that has ALL images generated (status already changed to 'review') +ready_for_review = Content.objects.filter( + site=site, + status='review' # Automatically set when all images complete +) +``` + +**Orchestration Logic:** +1. **Count Only**: Count content with `status='review'` + - No processing, just counting + - These Content records already have all Images with `status='generated'` + +2. **Log Results**: + - Log: "Automation complete. X content pieces ready for review" + - Log: "Content IDs ready: [123, 456, 789, ...]" + +3. **Mark Run Complete**: + - AutomationRun.status = 'completed' + - AutomationRun.completed_at = now() + +4. **Send Notification** (optional): + - Email/notification: "Your automation run completed. X content pieces ready for review" + +5. **STOP**: No further automation stages + +**Stage Completion Criteria:** +- Counting complete +- Automation run marked `status='completed'` + +**What AI Function Does:** +- N/A - No AI function called in this stage + +**Stage Result Logged:** +```json +{ + "ready_for_review": 56, + "content_ids": [123, 456, 789, ...] +} +``` + +**What Happens Next (Manual - User Action):** +1. User navigates to `/writer/content` page +2. Content page shows filter: `status='review'` +3. User sees 56 content pieces with all images generated +4. User manually reviews: + - Content quality + - Image relevance + - Brand voice + - Accuracy +5. User selects multiple content β†’ "Bulk Publish" action +6. Existing WordPress publishing workflow executes + +**Why Manual Review is Required:** +- Quality control before public publishing +- Legal/compliance verification +- Brand voice consistency check +- Final accuracy confirmation + +--- + +## πŸ”„ BATCH PROCESSING WITHIN STAGES + +### Critical Concepts + +**Batch vs Queue:** +- **Batch:** Group of items processed together in ONE AI call +- **Queue:** Total pending items waiting to be processed + +**Example - Stage 1 with 47 keywords:** +``` +Total Queue: 47 keywords +Batch Size: 20 + +Execution: + Batch 1: Keywords 1-20 β†’ Call auto_cluster β†’ Wait for completion + Batch 2: Keywords 21-40 β†’ Call auto_cluster β†’ Wait for completion + Batch 3: Keywords 41-47 β†’ Call auto_cluster β†’ Wait for completion + +Total Batches: 3 +Processing: Sequential (never parallel) +``` + +**UI Display:** +``` +Stage 1: Keywords β†’ Clusters +Status: ● Processing +Queue: 47 keywords total +Progress: Batch 2/3 (40 processed, 7 remaining) +Current: Processing keywords 21-40 +Time Elapsed: 4m 30s +Credits Used: 8 +``` + +### Batch Completion Triggers + +**Within Stage:** +- Batch completes β†’ Immediately start next batch +- Last batch completes β†’ Stage complete + +**Between Stages:** +- Stage N completes β†’ Trigger Stage N+1 automatically +- Hard verification: Ensure queue is empty before proceeding + +**Detailed Stage Processing Queues (UI Elements):** + +Each stage card should show: +1. **Total Queue Count** - How many items need processing in this stage +2. **Current Batch** - Which batch is being processed (e.g., "Batch 2/5") +3. **Processed Count** - How many items completed so far +4. **Remaining Count** - How many items left in queue +5. **Current Item** - What specific item is processing right now (for single-item batches) + +**Example UI for Stage 4:** +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ STAGE 4: Tasks β†’ Content (AI) β”‚ +β”‚ Status: ● Processing β”‚ +β”‚ β”‚ +β”‚ πŸ“Š QUEUE OVERVIEW: β”‚ +β”‚ β”œβ”€ Total Tasks: 56 β”‚ +β”‚ β”œβ”€ Processed: 23 β”‚ +β”‚ β”œβ”€ Remaining: 33 β”‚ +β”‚ └─ Progress: ━━━━━━━╸━━━━━━━━━━━━ 41% β”‚ +β”‚ β”‚ +β”‚ πŸ”„ CURRENT PROCESSING: β”‚ +β”‚ β”œβ”€ Item: Task 24/56 β”‚ +β”‚ β”œβ”€ Title: "Ultimate Coffee Bean Buying Guide" β”‚ +β”‚ β”œβ”€ Progress: Writing sections (65% complete) β”‚ +β”‚ └─ Time: 2m 15s elapsed β”‚ +β”‚ β”‚ +β”‚ πŸ’³ STAGE STATS: β”‚ +β”‚ β”œβ”€ Credits Used: 46 β”‚ +β”‚ β”œβ”€ Time Elapsed: 1h 23m β”‚ +β”‚ └─ ETA: 1h 15m remaining β”‚ +β”‚ β”‚ +β”‚ [View Details] [Pause Stage] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +--- + +## πŸ—„οΈ DATABASE STRUCTURE + +### New Models to Create + +**AutomationRun** (tracks each automation execution) +``` +Table: igny8_automation_runs +Fields: + - id: Integer (PK) + - run_id: String (unique, indexed) - Format: run_20251203_140523_manual + - account_id: ForeignKey(Account) + - site_id: ForeignKey(Site) + - trigger_type: String - Choices: 'manual', 'scheduled' + - status: String - Choices: 'running', 'paused', 'completed', 'failed' + - current_stage: Integer - Current stage number (1-7) + - started_at: DateTime + - completed_at: DateTime (nullable) + - total_credits_used: Integer + - stage_1_result: JSON - {keywords_processed, clusters_created, batches} + - stage_2_result: JSON - {clusters_processed, ideas_created} + - stage_3_result: JSON - {ideas_processed, tasks_created} + - stage_4_result: JSON - {tasks_processed, content_created, total_words} + - stage_5_result: JSON - {content_processed, prompts_created} + - stage_6_result: JSON - {prompts_processed, images_generated} + - stage_7_result: JSON - {ready_for_review} + - error_message: Text (nullable) + +Indexes: + - run_id (unique) + - site_id, started_at + - status, started_at +``` + +**AutomationConfig** (per-site configuration) +``` +Table: igny8_automation_configs +Fields: + - id: Integer (PK) + - account_id: ForeignKey(Account) + - site_id: ForeignKey(Site, unique) - ONE config per site + - is_enabled: Boolean - Whether scheduled automation is active + - frequency: String - Choices: 'daily', 'weekly', 'monthly' + - scheduled_time: Time - When to run (e.g., 02:00) + - stage_1_batch_size: Integer - Default 20 (keywords per batch) + - stage_2_batch_size: Integer - Default 1 (clusters at a time) + - stage_3_batch_size: Integer - Default 20 (ideas per batch) + - stage_4_batch_size: Integer - Default 1 (tasks - sequential) + - stage_5_batch_size: Integer - Default 1 (content at a time) + - stage_6_batch_size: Integer - Default 1 (images - sequential) + - last_run_at: DateTime (nullable) + - next_run_at: DateTime (nullable) - Calculated based on frequency + +Constraints: + - Unique: site_id (one config per site) +``` + +### File-Based Logging Structure + +**Directory Structure:** +``` +logs/ +└── automation/ + └── {account_id}/ + └── {site_id}/ + └── {run_id}/ + β”œβ”€β”€ automation_run.log (main activity log) + β”œβ”€β”€ stage_1.log (keywords β†’ clusters) + β”œβ”€β”€ stage_2.log (clusters β†’ ideas) + β”œβ”€β”€ stage_3.log (ideas β†’ tasks) + β”œβ”€β”€ stage_4.log (tasks β†’ content) + β”œβ”€β”€ stage_5.log (content β†’ prompts) + β”œβ”€β”€ stage_6.log (prompts β†’ images) + └── stage_7.log (review gate) +``` + +**Log File Format (automation_run.log):** +``` +======================================== +AUTOMATION RUN: run_20251203_140523_manual +Started: 2025-12-03 14:05:23 +Trigger: manual +Account: 5 +Site: 12 +======================================== + +14:05:23 - Automation started (trigger: manual) +14:05:23 - Credit check: Account has 1500 credits, estimated need: 866 credits +14:05:23 - Stage 1 starting: Keywords β†’ Clusters +14:05:24 - Stage 1: Found 47 pending keywords +14:05:24 - Stage 1: Processing batch 1/3 (20 keywords) +14:05:25 - Stage 1: AI task queued: task_id=abc123 +14:07:30 - Stage 1: Batch 1 complete - 3 clusters created +14:07:31 - Stage 1: Processing batch 2/3 (20 keywords) +[... continues ...] +``` + +**Stage-Specific Log (stage_1.log):** +``` +======================================== +STAGE 1: Keywords β†’ Clusters (AI) +Started: 2025-12-03 14:05:23 +======================================== + +14:05:24 - Query: Keywords.objects.filter(site=12, status='new', cluster__isnull=True) +14:05:24 - Found 47 pending keywords +14:05:24 - Batch size: 20 keywords +14:05:24 - Total batches: 3 + +--- Batch 1/3 --- +14:05:24 - Keyword IDs: [101, 102, 103, ..., 120] +14:05:25 - Calling ClusteringService.cluster_keywords(ids=[101..120], account=5, site_id=12) +14:05:25 - AI task queued: task_id=abc123 +14:05:26 - Monitoring task status... +14:05:28 - Phase: INIT - Initializing (StepTracker event) +14:05:45 - Phase: AI_CALL - AI analyzing keywords (StepTracker event) +14:07:15 - Phase: SAVE - Creating clusters (StepTracker event) +14:07:30 - Phase: DONE - Complete +14:07:30 - Result: 3 clusters created +14:07:30 - Clusters: ["Coffee Beans", "Brewing Methods", "Coffee Equipment"] +14:07:30 - Credits used: 4 (from AIUsageLog) + +--- Batch 2/3 --- +[... continues ...] + +======================================== +STAGE 1 COMPLETE +Total Time: 5m 30s +Processed: 47 keywords +Clusters Created: 8 +Credits Used: 10 +======================================== +``` + +--- + +## πŸ” SAFETY MECHANISMS + +### 1. Concurrency Control (Prevent Duplicate Runs) + +**Problem:** User clicks "Run Now" while scheduled task is running + +**Solution:** Distributed locking using Django cache + +**Implementation Logic:** +``` +When starting automation: + 1. Try to acquire lock: cache.add(f'automation_lock_{site.id}', 'locked', timeout=21600) + 2. If lock exists β†’ Return error: "Automation already running for this site" + 3. If lock acquired β†’ Proceed with run + 4. On completion/failure β†’ Release lock: cache.delete(f'automation_lock_{site.id}') + +Also check database: + - Query AutomationRun.objects.filter(site=site, status='running').exists() + - If exists β†’ Error: "Another automation is running" +``` + +**User sees:** +- "Automation already in progress. Started at 02:00 AM, currently on Stage 4." +- Link to view current run progress + +--- + +### 2. Credit Reservation (Prevent Mid-Run Failures) + +**Problem:** Account runs out of credits during Stage 4 + +**Solution:** Reserve estimated credits at start, deduct as used + +**Implementation Logic:** +``` +Before starting: + 1. Estimate total credits needed: + - Count keywords β†’ estimate clustering credits + - Count clusters β†’ estimate ideas credits + - Estimate content generation (assume avg word count) + - Estimate image generation (assume 4 images per content) + 2. Check: account.credits_balance >= estimated_credits * 1.2 (20% buffer) + 3. If insufficient β†’ Error: "Need ~866 credits, you have 500" + 4. Reserve credits: account.credits_reserved += estimated_credits + 5. As each stage completes β†’ Deduct actual: account.credits_balance -= actual_used + 6. On completion β†’ Release unused: account.credits_reserved -= unused + +Database fields needed: + - Account.credits_reserved (new field) +``` + +--- + +### 3. Stage Idempotency (Safe to Retry) + +**Problem:** User resumes paused run, Stage 1 runs again creating duplicate clusters + +**Solution:** Check if stage already completed before executing + +**Implementation Logic:** +``` +At start of each run_stage_N(): + 1. Check AutomationRun.stage_N_result + 2. If result exists and has processed_count > 0: + - Log: "Stage N already completed - skipping" + - return (skip to next stage) + 3. Else: Proceed with stage execution +``` + +--- + +### 4. Celery Task Chaining (Non-Blocking Workers) + +**Problem:** Synchronous execution blocks Celery worker for hours + +**Solution:** Chain stages as separate Celery tasks + +**Implementation Logic:** +``` +Instead of: + def start_automation(): + run_stage_1() # blocks for 30 min + run_stage_2() # blocks for 45 min + ... + +Do: + @shared_task + def run_stage_1_task(run_id): + service = AutomationService.from_run_id(run_id) + service.run_stage_1() + # Trigger next stage + run_stage_2_task.apply_async(args=[run_id], countdown=5) + + @shared_task + def run_stage_2_task(run_id): + service = AutomationService.from_run_id(run_id) + service.run_stage_2() + run_stage_3_task.apply_async(args=[run_id], countdown=5) + +Benefits: + - Workers not blocked for hours + - Can retry individual stages + - Better monitoring in Celery Flower + - Horizontal scaling possible +``` + +--- + +### 5. Pause/Resume Capability + +**User Can:** +- Pause automation at any point +- Resume from where it left off + +**Implementation Logic:** +``` +Pause: + - Update AutomationRun.status = 'paused' + - Current stage completes current batch then stops + - Celery task checks status before each batch + +Resume: + - Update AutomationRun.status = 'running' + - Restart from current_stage + - Use idempotency check to skip completed work +``` + +--- + +### 6. Error Handling Per Stage + +**If Stage Fails:** +``` +try: + run_stage_1() +except Exception as e: + - Log error to stage_1.log + - Update AutomationRun: + - status = 'failed' + - error_message = str(e) + - current_stage = 1 (where it failed) + - Send notification: "Automation failed at Stage 1" + - Stop execution (don't proceed to Stage 2) + +User can: + - View logs to see what went wrong + - Fix issue (e.g., add credits) + - Click "Resume" to retry from Stage 1 +``` + +--- + +### 7. Log Cleanup (Prevent Disk Bloat) + +**Problem:** After 1000 runs, logs occupy 80MB+ per site + +**Solution:** Celery periodic task to delete old logs + +**Implementation Logic:** +``` +@shared_task +def cleanup_old_automation_logs(): + cutoff = datetime.now() - timedelta(days=90) # Keep last 90 days + + old_runs = AutomationRun.objects.filter( + started_at__lt=cutoff, + status__in=['completed', 'failed'] + ) + + for run in old_runs: + log_dir = f'logs/automation/{run.account_id}/{run.site_id}/{run.run_id}/' + shutil.rmtree(log_dir) # Delete directory + run.delete() # Remove DB record + +Schedule: Weekly, Monday 3 AM +``` + +--- + +## 🎨 FRONTEND DESIGN + +### Page Structure: `/automation` + +**Layout:** +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ πŸ€– AI AUTOMATION PIPELINE β”‚ +β”‚ ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ β”‚ +β”‚ β”‚ +β”‚ β”‚ +β”‚ ⏰ SCHEDULE β”‚ +β”‚ Next Run: Tomorrow at 2:00 AM (in 16 hours) β”‚ +β”‚ Frequency: [Daily β–Ό] at [02:00 β–Ό] β”‚ +β”‚ Status: ● Scheduled β”‚ +β”‚ β”‚ +β”‚ [Run Now] [Pause Schedule] [Configure] β”‚ +β”‚ β”‚ +β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ +β”‚ πŸ“Š PIPELINE OVERVIEW β”‚ +β”‚ β”‚ +β”‚ Keywords ──→ Clusters ──→ Ideas ──→ Tasks ──→ Content β”‚ +β”‚ 47 8 42 20 generating β”‚ +β”‚ pending new ready queued Stage 5 β”‚ +β”‚ β”‚ +β”‚ Overall Progress: ━━━━━━━╸━━━━━━━━━ 62% (Stage 5/7) β”‚ +β”‚ Estimated Completion: 2 hours 15 minutes β”‚ +β”‚ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + +[STAGE 1 CARD - completed state] +[STAGE 2 CARD - completed state] +[STAGE 3 CARD - completed state] +[STAGE 4 CARD - running state with queue details] +[STAGE 5 CARD - waiting state] +[STAGE 6 CARD - waiting state] +[STAGE 7 CARD - gate state] + +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +πŸ“‹ LIVE ACTIVITY LOG (Last 50 events) +β”œβ”€ 14:23:45 - Stage 4: Started content generation for Task 3 +β”œβ”€ 14:24:12 - Stage 4: Writing sections (65% complete) +β”œβ”€ 14:22:30 - Stage 4: Completed Task 2 β†’ Content created +β”œβ”€ 14:20:15 - Stage 4: Started content generation for Task 2 +└─ [View Full Log] + +πŸ’° TOTAL CREDITS USED THIS RUN: 66 credits +``` + +**Components:** + +**StageCard.tsx** - Individual stage display component +- Props: stageNumber, stageName, status, queueData, result +- Shows: Status badge, queue overview, progress bar, stats +- Actions: "View Details", "Pause", "Retry Failed" + +**ActivityLog.tsx** - Live activity feed component +- Props: runId +- Fetches: `/api/v1/automation/activity_log/{runId}` every 3 seconds +- Shows: Timestamped log entries, color-coded by type (info/success/error) + +**ConfigModal.tsx** - Schedule configuration modal +- Fields: Frequency dropdown, Time picker, Batch sizes (advanced) +- Saves to: AutomationConfig model via `/api/v1/automation/config/` + +**Sidebar Menu Addition:** +``` +Sites + β”œβ”€ Site Management + └─ Site Settings + +Automation ← NEW + └─ Pipeline Dashboard + +Planner + β”œβ”€ Keywords + β”œβ”€ Clusters + └─ Ideas +``` + +--- + +### Real-Time Progress Updates + +**UI Update Strategy:** +- **Frontend Polling**: Poll automation status API every 3 seconds when run is active +- **Backend Progress**: Uses event-based `StepTracker` to capture AI function phases +- When automation is `status='running'` β†’ Poll every 3 seconds +- When `status='completed'` or `status='failed'` β†’ Stop polling +- When `status='paused'` β†’ Poll every 10 seconds + +**How Progress Works:** +1. **AI Function Execution**: Each AI function emits phase events (INIT, PREP, AI_CALL, PARSE, SAVE, DONE) +2. **StepTracker Captures**: Progress tracker records these events with metadata +3. **Automation Logs**: Orchestrator reads from StepTracker and logs to file +4. **UI Polls**: Frontend polls automation status API to read aggregated progress +5. **Display**: UI shows current phase and completion percentage per stage + +**API Endpoint:** +``` +GET /api/v1/automation/current_run/?site_id=12 + +Response: +{ + "run": { + "run_id": "run_20251203_140523_manual", + "status": "running", + "current_stage": 4, + "started_at": "2025-12-03T14:05:23Z", + "total_credits_used": 66, + "stage_1_result": {"keywords_processed": 47, "clusters_created": 8}, + "stage_2_result": {"clusters_processed": 8, "ideas_created": 56}, + "stage_3_result": {"ideas_processed": 56, "tasks_created": 56}, + "stage_4_result": {"tasks_processed": 23, "tasks_total": 56}, + ... + }, + "activity_log": [ + "14:23:45 - Stage 4: Started content generation for Task 3", + "14:24:12 - Stage 4: Writing sections (65% complete)", + ... + ], + "queues": { + "stage_1": {"total": 0, "pending": 0}, + "stage_2": {"total": 0, "pending": 0}, + "stage_3": {"total": 0, "pending": 0}, + "stage_4": {"total": 56, "pending": 33}, + "stage_5": {"total": 23, "pending": 23}, + "stage_6": {"total": 0, "pending": 0}, + "stage_7": {"total": 0, "pending": 0} + } +} +``` + +--- + +## πŸ”„ BACKEND IMPLEMENTATION FLOW + +### Service Layer Architecture + +**AutomationService** (core orchestrator) +- Location: `backend/igny8_core/business/automation/services/automation_service.py` +- Responsibility: Execute stages sequentially, manage run state +- Reuses: All existing AI function classes (NO duplication) + +**AutomationLogger** (file logging) +- Location: `backend/igny8_core/business/automation/services/automation_logger.py` +- Responsibility: Write timestamped logs to files +- Methods: start_run(), log_stage_start(), log_stage_progress(), log_stage_complete() + +**Key Service Methods:** + +``` +AutomationService: + - __init__(account, site) β†’ Initialize with site context (NO sector) + - start_automation(trigger_type) β†’ Main entry point + - run_stage_1() β†’ Keywords β†’ Clusters + - run_stage_2() β†’ Clusters β†’ Ideas + - run_stage_3() β†’ Ideas β†’ Tasks + - run_stage_4() β†’ Tasks β†’ Content + - run_stage_5() β†’ Content β†’ Prompts + - run_stage_6() β†’ Prompts β†’ Images + - run_stage_7() β†’ Review gate + - pause_automation() β†’ Pause current run + - resume_automation() β†’ Resume from current_stage + - estimate_credits() β†’ Calculate estimated credits needed + +AutomationLogger: + - start_run(account_id, site_id, trigger_type) β†’ Create log directory, return run_id + - log_stage_start(run_id, stage_number, stage_name, pending_count) + - log_stage_progress(run_id, stage_number, message) + - log_stage_complete(run_id, stage_number, processed_count, time_elapsed, credits_used) + - log_stage_error(run_id, stage_number, error_message) + - get_activity_log(run_id, last_n=50) β†’ Return last N log lines +``` + +--- + +### API Endpoints to Implement + +**AutomationViewSet** - Django REST Framework ViewSet +- Base URL: `/api/v1/automation/` +- Actions: + +``` +POST /api/v1/automation/run_now/ + - Body: {"site_id": 12} + - Action: Trigger manual automation run + - Returns: {"run_id": "run_...", "message": "Automation started"} + +GET /api/v1/automation/current_run/?site_id=12 + - Returns: Current/latest run status, activity log, queue counts + +POST /api/v1/automation/pause/ + - Body: {"run_id": "run_..."} + - Action: Pause running automation + +POST /api/v1/automation/resume/ + - Body: {"run_id": "run_..."} + - Action: Resume paused automation + +GET /api/v1/automation/config/?site_id=12 + - Returns: AutomationConfig for site + +PUT /api/v1/automation/config/ + - Body: {"site_id": 12, "is_enabled": true, "frequency": "daily", "scheduled_time": "02:00"} + - Action: Update automation schedule + +GET /api/v1/automation/history/?site_id=12&page=1 + - Returns: Paginated list of past runs + +GET /api/v1/automation/logs/{run_id}/ + - Returns: Full logs for a specific run (all stage files) +``` + +--- + +### Celery Tasks for Scheduling + +**Periodic Task** (runs every hour) +``` +@shared_task(name='check_scheduled_automations') +def check_scheduled_automations(): + """ + Runs every hour (via Celery Beat) + Checks if any AutomationConfig needs to run + """ + now = timezone.now() + + configs = AutomationConfig.objects.filter( + is_enabled=True, + next_run_at__lte=now + ) + + for config in configs: + # Check for concurrent run + if AutomationRun.objects.filter(site=config.site, status='running').exists(): + continue # Skip if already running + + # Start automation + run_automation_task.delay( + account_id=config.account_id, + site_id=config.site_id, + trigger_type='scheduled' + ) + + # Calculate next run time + if config.frequency == 'daily': + config.next_run_at = now + timedelta(days=1) + elif config.frequency == 'weekly': + config.next_run_at = now + timedelta(weeks=1) + elif config.frequency == 'monthly': + config.next_run_at = now + timedelta(days=30) + + config.last_run_at = now + config.save() + +Schedule in celery.py: + app.conf.beat_schedule['check-scheduled-automations'] = { + 'task': 'check_scheduled_automations', + 'schedule': crontab(minute=0), # Every hour on the hour + } +``` + +**Stage Task Chain** +``` +@shared_task +def run_automation_task(account_id, site_id, trigger_type): + """ + Main automation task - chains individual stage tasks + """ + service = AutomationService(account_id, site_id) + run_id = service.start_automation(trigger_type) + + # Chain stages as separate tasks for non-blocking execution + chain( + run_stage_1.si(run_id), + run_stage_2.si(run_id), + run_stage_3.si(run_id), + run_stage_4.si(run_id), + run_stage_5.si(run_id), + run_stage_6.si(run_id), + run_stage_7.si(run_id), + ).apply_async() + +@shared_task +def run_stage_1(run_id): + service = AutomationService.from_run_id(run_id) + service.run_stage_1() + return run_id # Pass to next task + +@shared_task +def run_stage_2(run_id): + service = AutomationService.from_run_id(run_id) + service.run_stage_2() + return run_id + +[... similar for stages 3-7 ...] +``` + +--- + +## πŸ§ͺ TESTING STRATEGY + +### Unit Tests + +**Test AutomationService:** +- test_estimate_credits_calculation() +- test_stage_1_processes_batches_correctly() +- test_stage_completion_triggers_next_stage() +- test_pause_stops_after_current_batch() +- test_resume_from_paused_state() +- test_idempotency_skips_completed_stages() + +**Test AutomationLogger:** +- test_creates_log_directory_structure() +- test_writes_timestamped_log_entries() +- test_get_activity_log_returns_last_n_lines() + +### Integration Tests + +**Test Full Pipeline:** +``` +def test_full_automation_pipeline(): + # Setup: Create 10 keywords + keywords = KeywordFactory.create_batch(10, site=site) + + # Execute + service = AutomationService(account, site) + result = service.start_automation(trigger_type='manual') + + # Assert Stage 1 + assert result['stage_1_result']['keywords_processed'] == 10 + assert result['stage_1_result']['clusters_created'] > 0 + + # Assert Stage 2 + assert result['stage_2_result']['ideas_created'] > 0 + + # Assert Stage 3 + assert result['stage_3_result']['tasks_created'] > 0 + + # Assert Stage 4 + assert result['stage_4_result']['content_created'] > 0 + + # Assert Stage 5 + assert result['stage_5_result']['prompts_created'] > 0 + + # Assert Stage 6 + assert result['stage_6_result']['images_generated'] > 0 + + # Assert final state + assert result['status'] == 'completed' + assert AutomationRun.objects.get(run_id=result['run_id']).status == 'completed' +``` + +**Test Error Scenarios:** +- test_insufficient_credits_prevents_start() +- test_concurrent_run_prevented() +- test_stage_failure_stops_pipeline() +- test_rollback_on_error() + +--- + +## πŸ“‹ IMPLEMENTATION CHECKLIST + +### Phase 1: Database & Models (Week 1) +- [ ] Create `automation` app directory structure +- [ ] Define AutomationRun model with all stage_result JSON fields +- [ ] Define AutomationConfig model (one per site, NO sector) +- [ ] Create migrations +- [ ] Test model creation and queries + +### Phase 2: Logging Service (Week 1) +- [ ] Create AutomationLogger class +- [ ] Implement start_run() with log directory creation +- [ ] Implement log_stage_start(), log_stage_progress(), log_stage_complete() +- [ ] Implement get_activity_log() +- [ ] Test file logging manually + +### Phase 3: Core Automation Service (Week 2) +- [ ] Create AutomationService class +- [ ] Implement estimate_credits() +- [ ] Implement start_automation() with credit check +- [ ] Implement run_stage_1() calling ClusteringService +- [ ] Test Stage 1 in isolation with real keywords +- [ ] Implement run_stage_2() calling IdeasService +- [ ] Test Stage 2 in isolation +- [ ] Implement run_stage_3() calling bulk_queue_to_writer logic +- [ ] Implement run_stage_4() calling GenerateContentFunction +- [ ] Implement run_stage_5() calling GenerateImagePromptsFunction +- [ ] Implement run_stage_6() calling GenerateImagesFunction +- [ ] Implement run_stage_7() review gate (count only) +- [ ] Implement pause_automation() and resume_automation() + +### Phase 4: API Endpoints (Week 3) +- [ ] Create AutomationViewSet +- [ ] Implement run_now() action +- [ ] Implement current_run() action +- [ ] Implement pause() and resume() actions +- [ ] Implement config GET/PUT actions +- [ ] Implement history() action +- [ ] Implement logs() action +- [ ] Add URL routing in api_urls.py +- [ ] Test all endpoints with Postman/curl + +### Phase 5: Celery Tasks & Scheduling (Week 3) +- [ ] Create check_scheduled_automations periodic task +- [ ] Create run_automation_task +- [ ] Create stage task chain (run_stage_1, run_stage_2, etc.) +- [ ] Register tasks in celery.py +- [ ] Add Celery Beat schedule +- [ ] Test scheduled execution + +### Phase 6: Frontend Components (Week 4) +- [ ] Create /automation route in React Router +- [ ] Create Dashboard.tsx page component +- [ ] Create StageCard.tsx with queue display +- [ ] Create ActivityLog.tsx with 3-second polling +- [ ] Create ConfigModal.tsx for schedule settings +- [ ] Add "Automation" to sidebar menu (below Sites) +- [ ] Implement "Run Now" button +- [ ] Implement "Pause" and "Resume" buttons +- [ ] Test full UI flow + +### Phase 7: Safety & Polish (Week 5) +- [ ] Implement distributed locking (prevent concurrent runs) +- [ ] Implement credit reservation system +- [ ] Implement stage idempotency checks +- [ ] Implement error handling and rollback +- [ ] Create cleanup_old_automation_logs task +- [ ] Add email/notification on completion/failure +- [ ] Load testing with 100+ keywords +- [ ] UI polish and responsiveness +- [ ] Documentation update + +--- + +## πŸš€ POST-LAUNCH ENHANCEMENTS + +### Future Features (Phase 8+) +- **Conditional Stages:** Skip stages if no data (e.g., skip Stage 1 if no keywords) +- **Parallel Task Processing:** Process multiple tasks simultaneously in Stage 4 (with worker limits) +- **Smart Scheduling:** Avoid peak hours, optimize for cost +- **A/B Testing:** Test different prompts, compare results +- **Content Quality Scoring:** Auto-reject low-quality AI content +- **WordPress Auto-Publish:** With approval workflow and staging +- **Analytics Integration:** Track content performance post-publish +- **Social Media Auto-Post:** Share published content to social channels + +--- + +## πŸ“– USER DOCUMENTATION + +### How to Use Automation + +**1. Configure Schedule:** +- Navigate to Automation page +- Click "Configure" button +- Set frequency (Daily/Weekly/Monthly) +- Set time (e.g., 2:00 AM) +- Optionally adjust batch sizes (advanced) +- Click "Save" + +**2. Manual Run:** +- Click "Run Now" button +- Monitor progress in real-time +- View activity log for details + +**3. Review Content:** +- Wait for automation to complete (or check next morning if scheduled) +- Navigate to Writer β†’ Content page +- Filter by "Draft" status with images generated +- Review content quality +- Select multiple β†’ Bulk Publish + +**4. Monitor History:** +- View past runs in History tab +- Click run to view detailed logs +- See credits used per run + +--- + +## βœ… SUCCESS CRITERIA + +**Automation is successful if:** +- βœ… Runs without manual intervention from Keywords β†’ Draft Content +- βœ… Processes 100+ keywords without errors +- βœ… Respects credit limits (pre-check + reservation) +- βœ… Stops at review gate (doesn't auto-publish) +- βœ… Completes within estimated time (6-12 hours for 100 keywords) +- βœ… UI shows real-time progress accurately +- βœ… Logs are detailed and troubleshoot-able +- βœ… Can pause/resume without data loss +- βœ… Scheduled runs trigger correctly +- βœ… No duplicate runs occur +- βœ… Reuses ALL existing AI functions (zero duplication) + +--- + +**END OF COMPLETE IMPLEMENTATION PLAN** + +This plan ensures a safe, modular, observable, and maintainable automation system that orchestrates the existing IGNY8 AI functions into a fully automated content pipeline. \ No newline at end of file