8 Phases refactor

backend/igny8_core/business/automation/services/automation_service.py

@@ -146,8 +146,11 @@ class AutomationService:
             self.run.save()
             return
         
-        # Process in batches
+        # Process in batches with dynamic sizing
         batch_size = self.config.stage_1_batch_size
+        # FIXED: Use min() for dynamic batch sizing
+        actual_batch_size = min(total_count, batch_size)
+        
         keywords_processed = 0
         clusters_created = 0
         batches_run = 0
@@ -155,10 +158,10 @@ class AutomationService:
         
         keyword_ids = list(pending_keywords.values_list('id', flat=True))
         
-        for i in range(0, len(keyword_ids), batch_size):
-            batch = keyword_ids[i:i + batch_size]
-            batch_num = (i // batch_size) + 1
-            total_batches = (len(keyword_ids) + batch_size - 1) // batch_size
+        for i in range(0, len(keyword_ids), actual_batch_size):
+            batch = keyword_ids[i:i + actual_batch_size]
+            batch_num = (i // actual_batch_size) + 1
+            total_batches = (len(keyword_ids) + actual_batch_size - 1) // actual_batch_size
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
@@ -185,6 +188,19 @@ class AutomationService:
                 self.run.run_id, self.account.id, self.site.id,
                 stage_number, f"Batch {batch_num} complete"
             )
+            
+            # ADDED: Within-stage delay (between batches)
+            if i + actual_batch_size < len(keyword_ids):  # Not the last batch
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next batch..."
+                )
+                time.sleep(delay)
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, "Delay complete, resuming processing"
+                )
         
         # Get clusters created count
         clusters_created = Clusters.objects.filter(
@@ -204,6 +220,12 @@ class AutomationService:
             stage_number, keywords_processed, time_elapsed, credits_used
         )
         
+        # ADDED: Post-stage validation
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Validation: {keywords_processed} keywords processed, {clusters_created} clusters created"
+        )
+        
         # Save results
         self.run.stage_1_result = {
             'keywords_processed': keywords_processed,
@@ -216,6 +238,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 1 complete: {keywords_processed} keywords → {clusters_created} clusters")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_2(self):
         """Stage 2: Clusters → Ideas"""
@@ -223,6 +253,32 @@ class AutomationService:
         stage_name = "Clusters → Ideas (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 1 completion
+        pending_keywords = Keywords.objects.filter(
+            site=self.site,
+            status='new',
+            cluster__isnull=True,
+            disabled=False
+        ).count()
+        
+        if pending_keywords > 0:
+            error_msg = f"Stage 1 incomplete: {pending_keywords} keywords still pending"
+            self.logger.log_stage_error(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, error_msg
+            )
+            logger.error(f"[AutomationService] {error_msg}")
+            # Continue anyway but log warning
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: Proceeding despite {pending_keywords} pending keywords from Stage 1"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 keywords pending from Stage 1"
+            )
+        
         # Query clusters without ideas
         pending_clusters = Clusters.objects.filter(
             site=self.site,
@@ -308,6 +364,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 2 complete: {clusters_processed} clusters → {ideas_created} ideas")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_3(self):
         """Stage 3: Ideas → Tasks (Local Queue)"""
@@ -315,6 +379,26 @@ class AutomationService:
         stage_name = "Ideas → Tasks (Local Queue)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 2 completion
+        pending_clusters = Clusters.objects.filter(
+            site=self.site,
+            status='new',
+            disabled=False
+        ).exclude(
+            ideas__isnull=False
+        ).count()
+        
+        if pending_clusters > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {pending_clusters} clusters from Stage 2 still pending"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 clusters pending from Stage 2"
+            )
+        
         # Query pending ideas
         pending_ideas = ContentIdeas.objects.filter(
             site=self.site,
@@ -414,6 +498,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 3 complete: {ideas_processed} ideas → {tasks_created} tasks")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_4(self):
         """Stage 4: Tasks → Content"""
@@ -421,6 +513,23 @@ class AutomationService:
         stage_name = "Tasks → Content (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 3 completion
+        pending_ideas = ContentIdeas.objects.filter(
+            site=self.site,
+            status='new'
+        ).count()
+        
+        if pending_ideas > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {pending_ideas} ideas from Stage 3 still pending"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 ideas pending from Stage 3"
+            )
+        
         # Query queued tasks (all queued tasks need content generated)
         pending_tasks = Tasks.objects.filter(
             site=self.site,
@@ -449,10 +558,14 @@ class AutomationService:
         tasks_processed = 0
         credits_before = self._get_credits_used()
         
-        for task in pending_tasks:
+        # FIXED: Ensure ALL tasks are processed by iterating over queryset list
+        task_list = list(pending_tasks)
+        total_tasks = len(task_list)
+        
+        for idx, task in enumerate(task_list, 1):
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Generating content for task: {task.title}"
+                stage_number, f"Generating content for task {idx}/{total_tasks}: {task.title}"
             )
             
             # Call AI function via AIEngine
@@ -469,10 +582,20 @@ class AutomationService:
             
             tasks_processed += 1
             
+            # Log progress
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Task '{task.title}' complete"
+                stage_number, f"Task '{task.title}' complete ({tasks_processed}/{total_tasks})"
             )
+            
+            # ADDED: Within-stage delay between tasks (if not last task)
+            if idx < total_tasks:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next task..."
+                )
+                time.sleep(delay)
         
         # Get content created count and total words
         content_created = Content.objects.filter(
@@ -497,6 +620,23 @@ class AutomationService:
             stage_number, tasks_processed, time_elapsed, credits_used
         )
         
+        # ADDED: Post-stage validation - verify all tasks processed
+        remaining_tasks = Tasks.objects.filter(
+            site=self.site,
+            status='queued'
+        ).count()
+        
+        if remaining_tasks > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {remaining_tasks} tasks still queued after Stage 4"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Post-stage validation passed: 0 tasks remaining"
+            )
+        
         # Save results
         self.run.stage_4_result = {
             'tasks_processed': tasks_processed,
@@ -509,6 +649,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 4 complete: {tasks_processed} tasks → {content_created} content")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_5(self):
         """Stage 5: Content → Image Prompts"""
@@ -516,10 +664,27 @@ class AutomationService:
         stage_name = "Content → Image Prompts (AI)"
         start_time = time.time()
         
-        # Query content without Images records
+        # ADDED: Pre-stage validation - verify Stage 4 completion
+        remaining_tasks = Tasks.objects.filter(
+            site=self.site,
+            status='queued'
+        ).count()
+        
+        if remaining_tasks > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {remaining_tasks} tasks from Stage 4 still queued"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: 0 tasks pending from Stage 4"
+            )
+        
+        # FIXED: Query content without Images records (ensure status='draft')
         content_without_images = Content.objects.filter(
             site=self.site,
-            status='draft'
+            status='draft'  # Explicitly check for draft status
         ).annotate(
             images_count=Count('images')
         ).filter(
@@ -528,6 +693,12 @@ class AutomationService:
         
         total_count = content_without_images.count()
         
+        # ADDED: Enhanced logging
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage 5: Found {total_count} content pieces without images (status='draft', images_count=0)"
+        )
+        
         # Log stage start
         self.logger.log_stage_start(
             self.run.run_id, self.account.id, self.site.id,
@@ -548,10 +719,13 @@ class AutomationService:
         content_processed = 0
         credits_before = self._get_credits_used()
         
-        for content in content_without_images:
+        content_list = list(content_without_images)
+        total_content = len(content_list)
+        
+        for idx, content in enumerate(content_list, 1):
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Extracting prompts from: {content.title}"
+                stage_number, f"Extracting prompts {idx}/{total_content}: {content.title}"
             )
             
             # Call AI function via AIEngine
@@ -570,8 +744,17 @@ class AutomationService:
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Content '{content.title}' complete"
+                stage_number, f"Content '{content.title}' complete ({content_processed}/{total_content})"
             )
+            
+            # ADDED: Within-stage delay between content pieces
+            if idx < total_content:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next content..."
+                )
+                time.sleep(delay)
         
         # Get prompts created count
         prompts_created = Images.objects.filter(
@@ -603,6 +786,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 5 complete: {content_processed} content → {prompts_created} prompts")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before next stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_6(self):
         """Stage 6: Image Prompts → Generated Images"""
@@ -610,6 +801,27 @@ class AutomationService:
         stage_name = "Images (Prompts) → Generated Images (AI)"
         start_time = time.time()
         
+        # ADDED: Pre-stage validation - verify Stage 5 completion
+        content_without_images = Content.objects.filter(
+            site=self.site,
+            status='draft'
+        ).annotate(
+            images_count=Count('images')
+        ).filter(
+            images_count=0
+        ).count()
+        
+        if content_without_images > 0:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, f"Warning: {content_without_images} content pieces from Stage 5 still without images"
+            )
+        else:
+            self.logger.log_stage_progress(
+                self.run.run_id, self.account.id, self.site.id,
+                stage_number, "Pre-stage validation passed: All content has image prompts"
+            )
+        
         # Query pending images
         pending_images = Images.objects.filter(
             site=self.site,
@@ -638,11 +850,14 @@ class AutomationService:
         images_processed = 0
         credits_before = self._get_credits_used()
         
-        for image in pending_images:
+        image_list = list(pending_images)
+        total_images = len(image_list)
+        
+        for idx, image in enumerate(image_list, 1):
             content_title = image.content.title if image.content else 'Unknown'
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Generating image: {image.image_type} for '{content_title}'"
+                stage_number, f"Generating image {idx}/{total_images}: {image.image_type} for '{content_title}'"
             )
             
             # Call AI function via AIEngine
@@ -661,8 +876,17 @@ class AutomationService:
             
             self.logger.log_stage_progress(
                 self.run.run_id, self.account.id, self.site.id,
-                stage_number, f"Image generated for '{content_title}'"
+                stage_number, f"Image generated for '{content_title}' ({images_processed}/{total_images})"
             )
+            
+            # ADDED: Within-stage delay between images
+            if idx < total_images:
+                delay = self.config.within_stage_delay
+                self.logger.log_stage_progress(
+                    self.run.run_id, self.account.id, self.site.id,
+                    stage_number, f"Waiting {delay} seconds before next image..."
+                )
+                time.sleep(delay)
         
         # Get images generated count
         images_generated = Images.objects.filter(
@@ -702,6 +926,14 @@ class AutomationService:
         self.run.save()
         
         logger.info(f"[AutomationService] Stage 6 complete: {images_processed} images generated, {content_moved_to_review} content moved to review")
+        
+        # ADDED: Between-stage delay
+        delay = self.config.between_stage_delay
+        self.logger.log_stage_progress(
+            self.run.run_id, self.account.id, self.site.id,
+            stage_number, f"Stage complete. Waiting {delay} seconds before final stage..."
+        )
+        time.sleep(delay)
     
     def run_stage_7(self):
         """Stage 7: Manual Review Gate (Count Only)"""