8 Phases refactor

backend/igny8_core/business/automation/migrations/0002_add_delay_configuration.py

@@ -0,0 +1,23 @@
+# Generated migration for delay configuration fields
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('automation', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='automationconfig',
+            name='within_stage_delay',
+            field=models.IntegerField(default=3, help_text='Delay between batches within a stage (seconds)'),
+        ),
+        migrations.AddField(
+            model_name='automationconfig',
+            name='between_stage_delay',
+            field=models.IntegerField(default=5, help_text='Delay between stage transitions (seconds)'),
+        ),
+    ]

backend/igny8_core/business/automation/migrations/0003_alter_automationconfig_options_and_more.py

@@ -0,0 +1,166 @@
+# Generated by Django 5.2.8 on 2025-12-03 16:06
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('automation', '0002_add_delay_configuration'),
+        ('igny8_core_auth', '0003_add_sync_event_model'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='automationconfig',
+            options={'verbose_name': 'Automation Config', 'verbose_name_plural': 'Automation Configs'},
+        ),
+        migrations.AlterModelOptions(
+            name='automationrun',
+            options={'ordering': ['-started_at'], 'verbose_name': 'Automation Run', 'verbose_name_plural': 'Automation Runs'},
+        ),
+        migrations.RemoveIndex(
+            model_name='automationrun',
+            name='automation_site_status_idx',
+        ),
+        migrations.RemoveIndex(
+            model_name='automationrun',
+            name='automation_site_started_idx',
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='account',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='automation_configs', to='igny8_core_auth.account'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='is_enabled',
+            field=models.BooleanField(default=False, help_text='Whether scheduled automation is active'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='next_run_at',
+            field=models.DateTimeField(blank=True, help_text='Calculated based on frequency', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='scheduled_time',
+            field=models.TimeField(default='02:00', help_text='Time to run (e.g., 02:00)'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_1_batch_size',
+            field=models.IntegerField(default=20, help_text='Keywords per batch'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_2_batch_size',
+            field=models.IntegerField(default=1, help_text='Clusters at a time'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_3_batch_size',
+            field=models.IntegerField(default=20, help_text='Ideas per batch'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_4_batch_size',
+            field=models.IntegerField(default=1, help_text='Tasks - sequential'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_5_batch_size',
+            field=models.IntegerField(default=1, help_text='Content at a time'),
+        ),
+        migrations.AlterField(
+            model_name='automationconfig',
+            name='stage_6_batch_size',
+            field=models.IntegerField(default=1, help_text='Images - sequential'),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='account',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='automation_runs', to='igny8_core_auth.account'),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='current_stage',
+            field=models.IntegerField(default=1, help_text='Current stage number (1-7)'),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='run_id',
+            field=models.CharField(db_index=True, help_text='Format: run_20251203_140523_manual', max_length=100, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_1_result',
+            field=models.JSONField(blank=True, help_text='{keywords_processed, clusters_created, batches}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_2_result',
+            field=models.JSONField(blank=True, help_text='{clusters_processed, ideas_created}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_3_result',
+            field=models.JSONField(blank=True, help_text='{ideas_processed, tasks_created}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_4_result',
+            field=models.JSONField(blank=True, help_text='{tasks_processed, content_created, total_words}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_5_result',
+            field=models.JSONField(blank=True, help_text='{content_processed, prompts_created}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_6_result',
+            field=models.JSONField(blank=True, help_text='{images_processed, images_generated}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='stage_7_result',
+            field=models.JSONField(blank=True, help_text='{ready_for_review}', null=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='started_at',
+            field=models.DateTimeField(auto_now_add=True, db_index=True),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='status',
+            field=models.CharField(choices=[('running', 'Running'), ('paused', 'Paused'), ('completed', 'Completed'), ('failed', 'Failed')], db_index=True, default='running', max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='automationrun',
+            name='trigger_type',
+            field=models.CharField(choices=[('manual', 'Manual'), ('scheduled', 'Scheduled')], max_length=20),
+        ),
+        migrations.AddIndex(
+            model_name='automationconfig',
+            index=models.Index(fields=['is_enabled', 'next_run_at'], name='igny8_autom_is_enab_038ce6_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='automationconfig',
+            index=models.Index(fields=['account', 'site'], name='igny8_autom_account_c6092f_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='automationrun',
+            index=models.Index(fields=['site', '-started_at'], name='igny8_autom_site_id_b5bf36_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='automationrun',
+            index=models.Index(fields=['status', '-started_at'], name='igny8_autom_status_1457b0_idx'),
+        ),
+        migrations.AddIndex(
+            model_name='automationrun',
+            index=models.Index(fields=['account', '-started_at'], name='igny8_autom_account_27cb3c_idx'),
+        ),
+    ]

backend/igny8_core/business/automation/models.py

@@ -31,6 +31,10 @@ class AutomationConfig(models.Model):
     stage_5_batch_size = models.IntegerField(default=1, help_text="Content at a time")
     stage_6_batch_size = models.IntegerField(default=1, help_text="Images - sequential")
     
+    # Delay configuration (in seconds)
+    within_stage_delay = models.IntegerField(default=3, help_text="Delay between batches within a stage (seconds)")
+    between_stage_delay = models.IntegerField(default=5, help_text="Delay between stage transitions (seconds)")
+    
     last_run_at = models.DateTimeField(null=True, blank=True)
     next_run_at = models.DateTimeField(null=True, blank=True, help_text="Calculated based on frequency")
     

backend/igny8_core/business/automation/services/automation_service.py

@@ -146,8 +146,11 @@ class AutomationService:
             self.run.save()
             return
         
-        # Process in batches
+        # Process in batches with dynamic sizing
         batch_size = self.config.stage_1_batch_size
+        # FIXED: Use min() for dynamic batch sizing
+        actual_batch_size = min(total_count, batch_size)
+        
         keywords_processed = 0
         clusters_created = 0
         batches_run = 0
@@ -155,10 +158,10 @@ class AutomationService:
         
         keyword_ids = list(pending_keywords.values_list('id', flat=True))
         
-        for i in range(0, len(keyword_ids), batch_size):
-            batch = keyword_ids[i:i + batch_size]
-            batch_num = (i // batch_size) + 1
-            total_batches = (len(keyword_ids) + batch_size - 1) // batch_size
+        for i in range(0, len(keyword_ids), actual_batch_size):
+            batch = keyword_ids[i:i + actual_batch_size]
+            batch_num = (i // actual_batch_size) + 1
+            total_batches = (len(keyword_ids) + actual_batch_size - 1) // actual_batch_size
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
@@ -185,6 +188,19 @@ class AutomationService:
                 self.run.run_id, self.account.id, self.site.id,
                 stage_number, f"Batch {batch_num} complete"
             )
+            
+            # ADDED: Within-stage delay (between batches)
+            if i + actual_batch_size < len(keyword_ids):  # Not the last batch
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next batch..."
+                )
+                time.sleep(delay)
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, "Delay complete, resuming processing"
+                )
         
         # Get clusters created count
         clusters_created = Clusters.objects.filter(
@@ -204,6 +220,12 @@ class AutomationService:
             stage_number, keywords_processed, time_elapsed, credits_used
         )
         
+        # ADDED: Post-stage validation
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Validation: {keywords_processed} keywords processed, {clusters_created} clusters created"
+        )
+        
         # Save results
         self.run.stage_1_result = {
             'keywords_processed': keywords_processed,
@@ -216,6 +238,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 1 complete: {keywords_processed} keywords → {clusters_created} clusters")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_2(self):
         """Stage 2: Clusters → Ideas"""
@@ -223,6 +253,32 @@ class AutomationService:
         stage_name = "Clusters → Ideas (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 1 completion
+        pending_keywords = Keywords.objects.filter(
+            site=self.site,
+            status='new',
+            cluster__isnull=True,
+            disabled=False
+        ).count()
+        
+        if pending_keywords > 0:
+            error_msg = f"Stage 1 incomplete: {pending_keywords} keywords still pending"
+            self.logger.log_stage_error(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, error_msg
+            )
+            logger.error(f"[AutomationService] {error_msg}")
+            # Continue anyway but log warning
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: Proceeding despite {pending_keywords} pending keywords from Stage 1"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 keywords pending from Stage 1"
+            )
+        
         # Query clusters without ideas
         pending_clusters = Clusters.objects.filter(
             site=self.site,
@@ -308,6 +364,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 2 complete: {clusters_processed} clusters → {ideas_created} ideas")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_3(self):
         """Stage 3: Ideas → Tasks (Local Queue)"""
@@ -315,6 +379,26 @@ class AutomationService:
         stage_name = "Ideas → Tasks (Local Queue)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 2 completion
+        pending_clusters = Clusters.objects.filter(
+            site=self.site,
+            status='new',
+            disabled=False
+        ).exclude(
+            ideas__isnull=False
+        ).count()
+        
+        if pending_clusters > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {pending_clusters} clusters from Stage 2 still pending"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 clusters pending from Stage 2"
+            )
+        
         # Query pending ideas
         pending_ideas = ContentIdeas.objects.filter(
             site=self.site,
@@ -414,6 +498,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 3 complete: {ideas_processed} ideas → {tasks_created} tasks")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_4(self):
         """Stage 4: Tasks → Content"""
@@ -421,6 +513,23 @@ class AutomationService:
         stage_name = "Tasks → Content (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 3 completion
+        pending_ideas = ContentIdeas.objects.filter(
+            site=self.site,
+            status='new'
+        ).count()
+        
+        if pending_ideas > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {pending_ideas} ideas from Stage 3 still pending"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 ideas pending from Stage 3"
+            )
+        
         # Query queued tasks (all queued tasks need content generated)
         pending_tasks = Tasks.objects.filter(
             site=self.site,
@@ -449,10 +558,14 @@ class AutomationService:
         tasks_processed = 0
         credits_before = self._get_credits_used()
         
-        for task in pending_tasks:
+        # FIXED: Ensure ALL tasks are processed by iterating over queryset list
+        task_list = list(pending_tasks)
+        total_tasks = len(task_list)
+        
+        for idx, task in enumerate(task_list, 1):
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Generating content for task: {task.title}"
+                stage_number, f"Generating content for task {idx}/{total_tasks}: {task.title}"
             )
             
             # Call AI function via AIEngine
@@ -469,10 +582,20 @@ class AutomationService:
             
             tasks_processed += 1
             
+            # Log progress
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Task '{task.title}' complete"
+                stage_number, f"Task '{task.title}' complete ({tasks_processed}/{total_tasks})"
             )
+            
+            # ADDED: Within-stage delay between tasks (if not last task)
+            if idx < total_tasks:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next task..."
+                )
+                time.sleep(delay)
         
         # Get content created count and total words
         content_created = Content.objects.filter(
@@ -497,6 +620,23 @@ class AutomationService:
             stage_number, tasks_processed, time_elapsed, credits_used
         )
         
+        # ADDED: Post-stage validation - verify all tasks processed
+        remaining_tasks = Tasks.objects.filter(
+            site=self.site,
+            status='queued'
+        ).count()
+        
+        if remaining_tasks > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {remaining_tasks} tasks still queued after Stage 4"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Post-stage validation passed: 0 tasks remaining"
+            )
+        
         # Save results
         self.run.stage_4_result = {
             'tasks_processed': tasks_processed,
@@ -509,6 +649,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 4 complete: {tasks_processed} tasks → {content_created} content")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_5(self):
         """Stage 5: Content → Image Prompts"""
@@ -516,10 +664,27 @@ class AutomationService:
         stage_name = "Content → Image Prompts (AI)"
         start_time = time.time()
         
-        # Query content without Images records
+        # ADDED: Pre-stage validation - verify Stage 4 completion
+        remaining_tasks = Tasks.objects.filter(
+            site=self.site,
+            status='queued'
+        ).count()
+        
+        if remaining_tasks > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {remaining_tasks} tasks from Stage 4 still queued"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 tasks pending from Stage 4"
+            )
+        
+        # FIXED: Query content without Images records (ensure status='draft')
         content_without_images = Content.objects.filter(
             site=self.site,
-            status='draft'
+            status='draft'  # Explicitly check for draft status
         ).annotate(
             images_count=Count('images')
         ).filter(
@@ -528,6 +693,12 @@ class AutomationService:
         
         total_count = content_without_images.count()
         
+        # ADDED: Enhanced logging
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage 5: Found {total_count} content pieces without images (status='draft', images_count=0)"
+        )
+        
         # Log stage start
         self.logger.log_stage_start(
             self.run.run_id, self.account.id, self.site.id,
@@ -548,10 +719,13 @@ class AutomationService:
         content_processed = 0
         credits_before = self._get_credits_used()
         
-        for content in content_without_images:
+        content_list = list(content_without_images)
+        total_content = len(content_list)
+        
+        for idx, content in enumerate(content_list, 1):
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Extracting prompts from: {content.title}"
+                stage_number, f"Extracting prompts {idx}/{total_content}: {content.title}"
             )
             
             # Call AI function via AIEngine
@@ -570,8 +744,17 @@ class AutomationService:
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Content '{content.title}' complete"
+                stage_number, f"Content '{content.title}' complete ({content_processed}/{total_content})"
             )
+            
+            # ADDED: Within-stage delay between content pieces
+            if idx < total_content:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next content..."
+                )
+                time.sleep(delay)
         
         # Get prompts created count
         prompts_created = Images.objects.filter(
@@ -603,6 +786,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 5 complete: {content_processed} content → {prompts_created} prompts")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_6(self):
         """Stage 6: Image Prompts → Generated Images"""
@@ -610,6 +801,27 @@ class AutomationService:
         stage_name = "Images (Prompts) → Generated Images (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 5 completion
+        content_without_images = Content.objects.filter(
+            site=self.site,
+            status='draft'
+        ).annotate(
+            images_count=Count('images')
+        ).filter(
+            images_count=0
+        ).count()
+        
+        if content_without_images > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {content_without_images} content pieces from Stage 5 still without images"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: All content has image prompts"
+            )
+        
         # Query pending images
         pending_images = Images.objects.filter(
             site=self.site,
@@ -638,11 +850,14 @@ class AutomationService:
         images_processed = 0
         credits_before = self._get_credits_used()
         
-        for image in pending_images:
+        image_list = list(pending_images)
+        total_images = len(image_list)
+        
+        for idx, image in enumerate(image_list, 1):
             content_title = image.content.title if image.content else 'Unknown'
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Generating image: {image.image_type} for '{content_title}'"
+                stage_number, f"Generating image {idx}/{total_images}: {image.image_type} for '{content_title}'"
             )
             
             # Call AI function via AIEngine
@@ -661,8 +876,17 @@ class AutomationService:
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Image generated for '{content_title}'"
+                stage_number, f"Image generated for '{content_title}' ({images_processed}/{total_images})"
             )
+            
+            # ADDED: Within-stage delay between images
+            if idx < total_images:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next image..."
+                )
+                time.sleep(delay)
         
         # Get images generated count
         images_generated = Images.objects.filter(
@@ -702,6 +926,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 6 complete: {images_processed} images generated, {content_moved_to_review} content moved to review")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before final stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_7(self):
         """Stage 7: Manual Review Gate (Count Only)"""