Add SEO fields to Tasks model, improve content generation response handling, and enhance progress bar animation

- Added primary_keyword, secondary_keywords, tags, and categories fields to Tasks model
- Updated generate_content function to handle full JSON response with all SEO fields
- Improved progress bar animation: smooth 1% increments every 300ms
- Enhanced step detection for content generation vs clustering vs ideas
- Fixed progress modal to show correct messages for each function type
- Added comprehensive logging to Keywords and Tasks pages for AI functions
- Fixed error handling to show meaningful error messages instead of generic failures

backend/igny8_core/modules/planner/tasks.py

@@ -7,6 +7,8 @@ from typing import List
 from django.db import transaction
 from igny8_core.modules.planner.models import Keywords, Clusters, ContentIdeas
 from igny8_core.utils.ai_processor import ai_processor
+from igny8_core.ai.functions.generate_ideas import generate_ideas_core
+from igny8_core.ai.tracker import ConsoleStepTracker
 
 logger = logging.getLogger(__name__)
 
@@ -23,9 +25,19 @@ except ImportError:
         return decorator
 
 
+# ============================================================================
+# DEPRECATED: This function is deprecated. Use the new AI framework instead.
+# New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
+# This function is kept for backward compatibility but should not be used.
+# ============================================================================
 def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, account_id: int = None, progress_callback=None):
     """
-    Core logic for clustering keywords. Can be called with or without Celery.
+    [DEPRECATED] Core logic for clustering keywords. Can be called with or without Celery.
+    
+    ⚠️  WARNING: This function is deprecated. Use the new AI framework instead:
+    - New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
+    - This function uses the old AIProcessor and does not use PromptRegistry
+    - Console logging may not work correctly in this path
     
     Args:
         keyword_ids: List of keyword IDs to cluster
@@ -33,7 +45,11 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
         account_id: Account ID for account isolation
         progress_callback: Optional function to call for progress updates (for Celery tasks)
     """
-    # Track request and response steps
+    # Initialize console step tracker for logging
+    tracker = ConsoleStepTracker('auto_cluster')
+    tracker.init(f"Starting keyword clustering for {len(keyword_ids)} keywords")
+    
+    # Track request and response steps (for Celery progress callbacks)
     request_steps = []
     response_steps = []
     
@@ -56,6 +72,7 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
             )
         
         # Step 4: Keyword Loading & Validation
+        tracker.prep(f"Loading {len(keyword_ids)} keywords from database")
         step_start = time.time()
         keywords_queryset = Keywords.objects.filter(id__in=keyword_ids)
         if account_id:
@@ -66,7 +83,9 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
         keywords = list(keywords_queryset.select_related('account', 'site', 'site__account', 'sector', 'sector__site'))
         
         if not keywords:
-            logger.warning(f"No keywords found for clustering: {keyword_ids}")
+            error_msg = f"No keywords found for clustering: {keyword_ids}"
+            logger.warning(error_msg)
+            tracker.error('Validation', error_msg)
             request_steps.append({
                 'stepNumber': 4,
                 'stepName': 'Keyword Loading & Validation',
@@ -83,6 +102,7 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
                 )
             return {'success': False, 'error': 'No keywords found', 'request_steps': request_steps, 'response_steps': response_steps}
         
+        tracker.prep(f"Loaded {len(keywords)} keywords successfully")
         request_steps.append({
             'stepNumber': 4,
             'stepName': 'Keyword Loading & Validation',
@@ -329,10 +349,20 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
             return {'success': False, 'error': f'Error preparing AI call: {str(e)}', 'request_steps': request_steps, 'response_steps': response_steps}
         
         # Call AI with step tracking
-        result = processor.cluster_keywords(keyword_data, sector_name=sector_name, account=account, response_steps=response_steps, progress_callback=progress_callback)
+        tracker.ai_call(f"Sending {len(keyword_data)} keywords to AI for clustering")
+        result = processor.cluster_keywords(
+            keyword_data, 
+            sector_name=sector_name, 
+            account=account, 
+            response_steps=response_steps, 
+            progress_callback=progress_callback,
+            tracker=tracker  # Pass tracker for console logging
+        )
         
         if result.get('error'):
-            logger.error(f"AI clustering error: {result['error']}")
+            error_msg = f"AI clustering error: {result['error']}"
+            logger.error(error_msg)
+            tracker.error('AI_CALL', error_msg)
             if progress_callback:
                 progress_callback(
                     state='FAILURE',
@@ -345,6 +375,9 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
                 )
             return {'success': False, 'error': result['error'], 'request_steps': request_steps, 'response_steps': response_steps}
         
+        # Parse response
+        tracker.parse("Parsing AI response into cluster data")
+        
         # Update response_steps from result if available
         if result.get('response_steps'):
             response_steps.extend(result.get('response_steps', []))
@@ -369,6 +402,7 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
         keywords_updated = 0
         
         # Step 13: Database Transaction Start
+        tracker.save(f"Creating {len(clusters_data)} clusters in database")
         step_start = time.time()
         # Create/update clusters and assign keywords
         # Note: account and sector are already extracted above to avoid database queries inside transaction
@@ -566,6 +600,7 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
         # Final progress update
         final_message = f"Clustering complete: {clusters_created} clusters created, {keywords_updated} keywords updated"
         logger.info(final_message)
+        tracker.done(final_message)
         
         if progress_callback:
             progress_callback(
@@ -587,7 +622,9 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
         }
         
     except Exception as e:
-        logger.error(f"Error in auto_cluster_keywords_core: {str(e)}", exc_info=True)
+        error_msg = f"Error in auto_cluster_keywords_core: {str(e)}"
+        logger.error(error_msg, exc_info=True)
+        tracker.error('Exception', error_msg, exception=e)
         if progress_callback:
             progress_callback(
                 state='FAILURE',
@@ -607,10 +644,18 @@ def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, a
 
 
 @shared_task(bind=True, max_retries=3)
+# ============================================================================
+# DEPRECATED: This Celery task is deprecated. Use run_ai_task instead.
+# New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
+# ============================================================================
 def auto_cluster_keywords_task(self, keyword_ids: List[int], sector_id: int = None, account_id: int = None):
     """
-    Celery task wrapper for clustering keywords using AI.
-    Calls the core function with progress callback.
+    [DEPRECATED] Celery task wrapper for clustering keywords using AI.
+    
+    ⚠️  WARNING: This task is deprecated. Use the new AI framework instead:
+    - New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
+    - This task uses the old _auto_cluster_keywords_core function
+    - Console logging may not work correctly in this path
     
     Args:
         keyword_ids: List of keyword IDs to cluster