Stage 1 & 2 refactor of AI engine

backend/igny8_core/ai/__init__.py

@@ -6,6 +6,25 @@ Unified framework for all AI functions with consistent lifecycle, progress track
 from igny8_core.ai.registry import register_function, get_function, list_functions
 from igny8_core.ai.engine import AIEngine
 from igny8_core.ai.base import BaseAIFunction
+from igny8_core.ai.ai_core import AICore
+from igny8_core.ai.validators import (
+    validate_ids,
+    validate_keywords_exist,
+    validate_cluster_limits,
+    validate_cluster_exists,
+    validate_tasks_exist,
+    validate_api_key,
+    validate_model,
+    validate_image_size,
+)
+from igny8_core.ai.constants import (
+    MODEL_RATES,
+    IMAGE_MODEL_RATES,
+    VALID_OPENAI_IMAGE_MODELS,
+    VALID_SIZES_BY_MODEL,
+    DEFAULT_AI_MODEL,
+    JSON_MODE_MODELS,
+)
 
 # Don't auto-import functions here - let apps.py handle it lazily
 # This prevents circular import issues during Django startup
@@ -13,8 +32,25 @@ from igny8_core.ai.base import BaseAIFunction
 __all__ = [
     'AIEngine',
     'BaseAIFunction',
+    'AICore',
     'register_function',
     'get_function',
     'list_functions',
+    # Validators
+    'validate_ids',
+    'validate_keywords_exist',
+    'validate_cluster_limits',
+    'validate_cluster_exists',
+    'validate_tasks_exist',
+    'validate_api_key',
+    'validate_model',
+    'validate_image_size',
+    # Constants
+    'MODEL_RATES',
+    'IMAGE_MODEL_RATES',
+    'VALID_OPENAI_IMAGE_MODELS',
+    'VALID_SIZES_BY_MODEL',
+    'DEFAULT_AI_MODEL',
+    'JSON_MODE_MODELS',
 ]
 

backend/igny8_core/ai/ai_core.py

@@ -0,0 +1,686 @@
+"""
+AI Core - Centralized execution and logging layer for all AI requests
+Handles API calls, model selection, response parsing, and console logging
+"""
+import logging
+import json
+import re
+import requests
+import time
+from typing import Dict, Any, Optional, List
+from django.conf import settings
+
+from .constants import (
+    DEFAULT_AI_MODEL,
+    JSON_MODE_MODELS,
+    MODEL_RATES,
+    IMAGE_MODEL_RATES,
+    VALID_OPENAI_IMAGE_MODELS,
+    VALID_SIZES_BY_MODEL,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AICore:
+    """
+    Centralized AI operations handler with console logging.
+    All AI requests go through run_ai_request() for consistent execution and logging.
+    """
+    
+    def __init__(self, account=None):
+        """
+        Initialize AICore with account context.
+        
+        Args:
+            account: Optional account object for API key/model loading
+        """
+        self.account = account
+        self._openai_api_key = None
+        self._runware_api_key = None
+        self._default_model = None
+        self._load_account_settings()
+    
+    def _load_account_settings(self):
+        """Load API keys and model from IntegrationSettings or Django settings"""
+        if self.account:
+            try:
+                from igny8_core.modules.system.models import IntegrationSettings
+                
+                # Load OpenAI settings
+                openai_settings = IntegrationSettings.objects.filter(
+                    integration_type='openai',
+                    account=self.account,
+                    is_active=True
+                ).first()
+                if openai_settings and openai_settings.config:
+                    self._openai_api_key = openai_settings.config.get('apiKey')
+                    model = openai_settings.config.get('model')
+                    if model and model in MODEL_RATES:
+                        self._default_model = model
+                
+                # Load Runware settings
+                runware_settings = IntegrationSettings.objects.filter(
+                    integration_type='runware',
+                    account=self.account,
+                    is_active=True
+                ).first()
+                if runware_settings and runware_settings.config:
+                    self._runware_api_key = runware_settings.config.get('apiKey')
+            except Exception as e:
+                logger.warning(f"Could not load account settings: {e}", exc_info=True)
+        
+        # Fallback to Django settings
+        if not self._openai_api_key:
+            self._openai_api_key = getattr(settings, 'OPENAI_API_KEY', None)
+        if not self._runware_api_key:
+            self._runware_api_key = getattr(settings, 'RUNWARE_API_KEY', None)
+        if not self._default_model:
+            self._default_model = getattr(settings, 'DEFAULT_AI_MODEL', DEFAULT_AI_MODEL)
+    
+    def get_api_key(self, integration_type: str = 'openai') -> Optional[str]:
+        """Get API key for integration type"""
+        if integration_type == 'openai':
+            return self._openai_api_key
+        elif integration_type == 'runware':
+            return self._runware_api_key
+        return None
+    
+    def get_model(self, integration_type: str = 'openai') -> str:
+        """Get model for integration type"""
+        if integration_type == 'openai':
+            return self._default_model
+        return DEFAULT_AI_MODEL
+    
+    def run_ai_request(
+        self,
+        prompt: str,
+        model: Optional[str] = None,
+        max_tokens: int = 4000,
+        temperature: float = 0.7,
+        response_format: Optional[Dict] = None,
+        api_key: Optional[str] = None,
+        function_name: str = 'ai_request'
+    ) -> Dict[str, Any]:
+        """
+        Centralized AI request handler with console logging.
+        All AI text generation requests go through this method.
+        
+        Args:
+            prompt: Prompt text
+            model: Model name (defaults to account's default)
+            max_tokens: Maximum tokens
+            temperature: Temperature (0-1)
+            response_format: Optional response format dict (for JSON mode)
+            api_key: Optional API key override
+            function_name: Function name for logging (e.g., 'cluster_keywords')
+        
+        Returns:
+            Dict with 'content', 'input_tokens', 'output_tokens', 'total_tokens',
+            'model', 'cost', 'error', 'api_id'
+        """
+        print(f"[AI][{function_name}] Step 1: Preparing request...")
+        
+        # Step 1: Validate API key
+        api_key = api_key or self._openai_api_key
+        if not api_key:
+            error_msg = 'OpenAI API key not configured'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'content': None,
+                'error': error_msg,
+                'input_tokens': 0,
+                'output_tokens': 0,
+                'total_tokens': 0,
+                'model': model or self._default_model,
+                'cost': 0.0,
+                'api_id': None,
+            }
+        
+        # Step 2: Determine model
+        active_model = model or self._default_model
+        print(f"[AI][{function_name}] Step 2: Using model: {active_model}")
+        
+        # Step 3: Auto-enable JSON mode for supported models
+        if response_format is None and active_model in JSON_MODE_MODELS:
+            response_format = {'type': 'json_object'}
+            print(f"[AI][{function_name}] Step 3: Auto-enabled JSON mode for {active_model}")
+        elif response_format:
+            print(f"[AI][{function_name}] Step 3: Using custom response format: {response_format}")
+        else:
+            print(f"[AI][{function_name}] Step 3: Using text response format")
+        
+        # Step 4: Validate prompt length
+        prompt_length = len(prompt)
+        print(f"[AI][{function_name}] Step 4: Prompt length: {prompt_length} characters")
+        
+        # Step 5: Build request payload
+        url = 'https://api.openai.com/v1/chat/completions'
+        headers = {
+            'Authorization': f'Bearer {api_key}',
+            'Content-Type': 'application/json',
+        }
+        
+        body_data = {
+            'model': active_model,
+            'messages': [{'role': 'user', 'content': prompt}],
+            'temperature': temperature,
+        }
+        
+        if max_tokens:
+            body_data['max_tokens'] = max_tokens
+        
+        if response_format:
+            body_data['response_format'] = response_format
+        
+        print(f"[AI][{function_name}] Step 5: Request payload prepared (model={active_model}, max_tokens={max_tokens}, temp={temperature})")
+        
+        # Step 6: Send request
+        print(f"[AI][{function_name}] Step 6: Sending request to OpenAI API...")
+        request_start = time.time()
+        
+        try:
+            response = requests.post(url, headers=headers, json=body_data, timeout=60)
+            request_duration = time.time() - request_start
+            print(f"[AI][{function_name}] Step 7: Received response in {request_duration:.2f}s (status={response.status_code})")
+            
+            # Step 7: Validate HTTP response
+            if response.status_code != 200:
+                error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {}
+                error_message = f"HTTP {response.status_code} error"
+                
+                if isinstance(error_data, dict) and 'error' in error_data:
+                    if isinstance(error_data['error'], dict) and 'message' in error_data['error']:
+                        error_message += f": {error_data['error']['message']}"
+                
+                # Check for rate limit
+                if response.status_code == 429:
+                    retry_after = response.headers.get('retry-after', '60')
+                    print(f"[AI][{function_name}][Error] OpenAI Rate Limit - waiting {retry_after}s")
+                    error_message += f" (Rate limit - retry after {retry_after}s)"
+                
+                print(f"[AI][{function_name}][Error] {error_message}")
+                logger.error(f"OpenAI API HTTP error {response.status_code}: {error_message}")
+                
+                return {
+                    'content': None,
+                    'error': error_message,
+                    'input_tokens': 0,
+                    'output_tokens': 0,
+                    'total_tokens': 0,
+                    'model': active_model,
+                    'cost': 0.0,
+                    'api_id': None,
+                }
+            
+            # Step 8: Parse response JSON
+            try:
+                data = response.json()
+            except json.JSONDecodeError as e:
+                error_msg = f'Failed to parse JSON response: {str(e)}'
+                print(f"[AI][{function_name}][Error] {error_msg}")
+                logger.error(error_msg)
+                return {
+                    'content': None,
+                    'error': error_msg,
+                    'input_tokens': 0,
+                    'output_tokens': 0,
+                    'total_tokens': 0,
+                    'model': active_model,
+                    'cost': 0.0,
+                    'api_id': None,
+                }
+            
+            api_id = data.get('id')
+            
+            # Step 9: Extract content
+            if 'choices' in data and len(data['choices']) > 0:
+                content = data['choices'][0]['message']['content']
+                usage = data.get('usage', {})
+                input_tokens = usage.get('prompt_tokens', 0)
+                output_tokens = usage.get('completion_tokens', 0)
+                total_tokens = usage.get('total_tokens', 0)
+                
+                print(f"[AI][{function_name}] Step 8: Received {total_tokens} tokens (input: {input_tokens}, output: {output_tokens})")
+                print(f"[AI][{function_name}] Step 9: Content length: {len(content)} characters")
+                
+                # Step 10: Calculate cost
+                rates = MODEL_RATES.get(active_model, {'input': 2.00, 'output': 8.00})
+                cost = (input_tokens * rates['input'] + output_tokens * rates['output']) / 1_000_000
+                print(f"[AI][{function_name}] Step 10: Cost calculated: ${cost:.6f}")
+                
+                print(f"[AI][{function_name}][Success] Request completed successfully")
+                
+                return {
+                    'content': content,
+                    'input_tokens': input_tokens,
+                    'output_tokens': output_tokens,
+                    'total_tokens': total_tokens,
+                    'model': active_model,
+                    'cost': cost,
+                    'error': None,
+                    'api_id': api_id,
+                }
+            else:
+                error_msg = 'No content in OpenAI response'
+                print(f"[AI][{function_name}][Error] {error_msg}")
+                logger.error(error_msg)
+                return {
+                    'content': None,
+                    'error': error_msg,
+                    'input_tokens': 0,
+                    'output_tokens': 0,
+                    'total_tokens': 0,
+                    'model': active_model,
+                    'cost': 0.0,
+                    'api_id': api_id,
+                }
+        
+        except requests.exceptions.Timeout:
+            error_msg = 'Request timeout (60s exceeded)'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            logger.error(error_msg)
+            return {
+                'content': None,
+                'error': error_msg,
+                'input_tokens': 0,
+                'output_tokens': 0,
+                'total_tokens': 0,
+                'model': active_model,
+                'cost': 0.0,
+                'api_id': None,
+            }
+        except requests.exceptions.RequestException as e:
+            error_msg = f'Request exception: {str(e)}'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            logger.error(f"OpenAI API error: {error_msg}", exc_info=True)
+            return {
+                'content': None,
+                'error': error_msg,
+                'input_tokens': 0,
+                'output_tokens': 0,
+                'total_tokens': 0,
+                'model': active_model,
+                'cost': 0.0,
+                'api_id': None,
+            }
+        except Exception as e:
+            error_msg = f'Unexpected error: {str(e)}'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            logger.error(error_msg, exc_info=True)
+            return {
+                'content': None,
+                'error': error_msg,
+                'input_tokens': 0,
+                'output_tokens': 0,
+                'total_tokens': 0,
+                'model': active_model,
+                'cost': 0.0,
+                'api_id': None,
+            }
+    
+    def extract_json(self, response_text: str) -> Optional[Dict]:
+        """
+        Extract JSON from response text.
+        Handles markdown code blocks, multiline JSON, etc.
+        
+        Args:
+            response_text: Raw response text from AI
+        
+        Returns:
+            Parsed JSON dict or None
+        """
+        if not response_text or not response_text.strip():
+            return None
+        
+        # Try direct JSON parse first
+        try:
+            return json.loads(response_text.strip())
+        except json.JSONDecodeError:
+            pass
+        
+        # Try to extract JSON from markdown code blocks
+        json_block_pattern = r'```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```'
+        matches = re.findall(json_block_pattern, response_text, re.DOTALL)
+        if matches:
+            try:
+                return json.loads(matches[0])
+            except json.JSONDecodeError:
+                pass
+        
+        # Try to find JSON object/array in text
+        json_pattern = r'(\{.*\}|\[.*\])'
+        matches = re.findall(json_pattern, response_text, re.DOTALL)
+        for match in matches:
+            try:
+                return json.loads(match)
+            except json.JSONDecodeError:
+                continue
+        
+        return None
+    
+    def generate_image(
+        self,
+        prompt: str,
+        provider: str = 'openai',
+        model: Optional[str] = None,
+        size: str = '1024x1024',
+        n: int = 1,
+        api_key: Optional[str] = None,
+        negative_prompt: Optional[str] = None,
+        function_name: str = 'generate_image'
+    ) -> Dict[str, Any]:
+        """
+        Generate image using AI with console logging.
+        
+        Args:
+            prompt: Image prompt
+            provider: 'openai' or 'runware'
+            model: Model name
+            size: Image size
+            n: Number of images
+            api_key: Optional API key override
+            negative_prompt: Optional negative prompt
+            function_name: Function name for logging
+        
+        Returns:
+            Dict with 'url', 'revised_prompt', 'cost', 'error', etc.
+        """
+        print(f"[AI][{function_name}] Step 1: Preparing image generation request...")
+        
+        if provider == 'openai':
+            return self._generate_image_openai(prompt, model, size, n, api_key, negative_prompt, function_name)
+        elif provider == 'runware':
+            return self._generate_image_runware(prompt, model, size, n, api_key, negative_prompt, function_name)
+        else:
+            error_msg = f'Unknown provider: {provider}'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': provider,
+                'cost': 0.0,
+                'error': error_msg,
+            }
+    
+    def _generate_image_openai(
+        self,
+        prompt: str,
+        model: Optional[str],
+        size: str,
+        n: int,
+        api_key: Optional[str],
+        negative_prompt: Optional[str],
+        function_name: str
+    ) -> Dict[str, Any]:
+        """Generate image using OpenAI DALL-E"""
+        print(f"[AI][{function_name}] Provider: OpenAI")
+        
+        api_key = api_key or self._openai_api_key
+        if not api_key:
+            error_msg = 'OpenAI API key not configured'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': 'openai',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        
+        model = model or 'dall-e-3'
+        print(f"[AI][{function_name}] Step 2: Using model: {model}, size: {size}")
+        
+        # Validate model
+        if model not in VALID_OPENAI_IMAGE_MODELS:
+            error_msg = f"Model '{model}' is not valid for OpenAI image generation. Only {', '.join(VALID_OPENAI_IMAGE_MODELS)} are supported."
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': 'openai',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        
+        # Validate size
+        valid_sizes = VALID_SIZES_BY_MODEL.get(model, [])
+        if size not in valid_sizes:
+            error_msg = f"Image size '{size}' is not valid for model '{model}'. Valid sizes: {', '.join(valid_sizes)}"
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': 'openai',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        
+        url = 'https://api.openai.com/v1/images/generations'
+        print(f"[AI][{function_name}] Step 3: Sending request to OpenAI Images API...")
+        
+        headers = {
+            'Authorization': f'Bearer {api_key}',
+            'Content-Type': 'application/json',
+        }
+        
+        data = {
+            'model': model,
+            'prompt': prompt,
+            'n': n,
+            'size': size
+        }
+        
+        if negative_prompt:
+            # Note: OpenAI DALL-E doesn't support negative_prompt in API, but we log it
+            print(f"[AI][{function_name}] Note: Negative prompt provided but OpenAI DALL-E doesn't support it")
+        
+        request_start = time.time()
+        try:
+            response = requests.post(url, headers=headers, json=data, timeout=150)
+            request_duration = time.time() - request_start
+            print(f"[AI][{function_name}] Step 4: Received response in {request_duration:.2f}s (status={response.status_code})")
+            
+            if response.status_code != 200:
+                error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {}
+                error_message = f"HTTP {response.status_code} error"
+                if isinstance(error_data, dict) and 'error' in error_data:
+                    if isinstance(error_data['error'], dict) and 'message' in error_data['error']:
+                        error_message += f": {error_data['error']['message']}"
+                
+                print(f"[AI][{function_name}][Error] {error_message}")
+                return {
+                    'url': None,
+                    'revised_prompt': None,
+                    'provider': 'openai',
+                    'cost': 0.0,
+                    'error': error_message,
+                }
+            
+            body = response.json()
+            if 'data' in body and len(body['data']) > 0:
+                image_data = body['data'][0]
+                image_url = image_data.get('url')
+                revised_prompt = image_data.get('revised_prompt')
+                
+                cost = IMAGE_MODEL_RATES.get(model, 0.040) * n
+                print(f"[AI][{function_name}] Step 5: Image generated successfully")
+                print(f"[AI][{function_name}] Step 6: Cost: ${cost:.4f}")
+                print(f"[AI][{function_name}][Success] Image generation completed")
+                
+                return {
+                    'url': image_url,
+                    'revised_prompt': revised_prompt,
+                    'provider': 'openai',
+                    'cost': cost,
+                    'error': None,
+                }
+            else:
+                error_msg = 'No image data in response'
+                print(f"[AI][{function_name}][Error] {error_msg}")
+                return {
+                    'url': None,
+                    'revised_prompt': None,
+                    'provider': 'openai',
+                    'cost': 0.0,
+                    'error': error_msg,
+                }
+        
+        except requests.exceptions.Timeout:
+            error_msg = 'Request timeout (150s exceeded)'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': 'openai',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        except Exception as e:
+            error_msg = f'Unexpected error: {str(e)}'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            logger.error(error_msg, exc_info=True)
+            return {
+                'url': None,
+                'revised_prompt': None,
+                'provider': 'openai',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+    
+    def _generate_image_runware(
+        self,
+        prompt: str,
+        model: Optional[str],
+        size: str,
+        n: int,
+        api_key: Optional[str],
+        negative_prompt: Optional[str],
+        function_name: str
+    ) -> Dict[str, Any]:
+        """Generate image using Runware"""
+        print(f"[AI][{function_name}] Provider: Runware")
+        
+        api_key = api_key or self._runware_api_key
+        if not api_key:
+            error_msg = 'Runware API key not configured'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'provider': 'runware',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        
+        runware_model = model or 'runware:97@1'
+        print(f"[AI][{function_name}] Step 2: Using model: {runware_model}, size: {size}")
+        
+        # Parse size
+        try:
+            width, height = map(int, size.split('x'))
+        except ValueError:
+            error_msg = f"Invalid size format: {size}. Expected format: WIDTHxHEIGHT"
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            return {
+                'url': None,
+                'provider': 'runware',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+        
+        url = 'https://api.runware.ai/v1'
+        print(f"[AI][{function_name}] Step 3: Sending request to Runware API...")
+        
+        # Runware uses array payload
+        payload = [{
+            'taskType': 'imageInference',
+            'model': runware_model,
+            'prompt': prompt,
+            'width': width,
+            'height': height,
+            'apiKey': api_key
+        }]
+        
+        if negative_prompt:
+            payload[0]['negativePrompt'] = negative_prompt
+        
+        request_start = time.time()
+        try:
+            response = requests.post(url, json=payload, timeout=150)
+            request_duration = time.time() - request_start
+            print(f"[AI][{function_name}] Step 4: Received response in {request_duration:.2f}s (status={response.status_code})")
+            
+            if response.status_code != 200:
+                error_msg = f"HTTP {response.status_code} error"
+                print(f"[AI][{function_name}][Error] {error_msg}")
+                return {
+                    'url': None,
+                    'provider': 'runware',
+                    'cost': 0.0,
+                    'error': error_msg,
+                }
+            
+            body = response.json()
+            # Runware returns array with image data
+            if isinstance(body, list) and len(body) > 0:
+                image_data = body[0]
+                image_url = image_data.get('imageURL') or image_data.get('url')
+                
+                cost = 0.036 * n  # Runware pricing
+                print(f"[AI][{function_name}] Step 5: Image generated successfully")
+                print(f"[AI][{function_name}] Step 6: Cost: ${cost:.4f}")
+                print(f"[AI][{function_name}][Success] Image generation completed")
+                
+                return {
+                    'url': image_url,
+                    'provider': 'runware',
+                    'cost': cost,
+                    'error': None,
+                }
+            else:
+                error_msg = 'No image data in Runware response'
+                print(f"[AI][{function_name}][Error] {error_msg}")
+                return {
+                    'url': None,
+                    'provider': 'runware',
+                    'cost': 0.0,
+                    'error': error_msg,
+                }
+        
+        except Exception as e:
+            error_msg = f'Unexpected error: {str(e)}'
+            print(f"[AI][{function_name}][Error] {error_msg}")
+            logger.error(error_msg, exc_info=True)
+            return {
+                'url': None,
+                'provider': 'runware',
+                'cost': 0.0,
+                'error': error_msg,
+            }
+    
+    def calculate_cost(self, model: str, input_tokens: int, output_tokens: int, model_type: str = 'text') -> float:
+        """Calculate cost for API call"""
+        if model_type == 'text':
+            rates = MODEL_RATES.get(model, {'input': 2.00, 'output': 8.00})
+            input_cost = (input_tokens / 1_000_000) * rates['input']
+            output_cost = (output_tokens / 1_000_000) * rates['output']
+            return input_cost + output_cost
+        elif model_type == 'image':
+            rate = IMAGE_MODEL_RATES.get(model, 0.040)
+            return rate * 1
+        return 0.0
+    
+    # Legacy method names for backward compatibility
+    def call_openai(self, prompt: str, model: Optional[str] = None, max_tokens: int = 4000,
+                    temperature: float = 0.7, response_format: Optional[Dict] = None,
+                    api_key: Optional[str] = None) -> Dict[str, Any]:
+        """Legacy method - redirects to run_ai_request()"""
+        return self.run_ai_request(
+            prompt=prompt,
+            model=model,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            response_format=response_format,
+            api_key=api_key,
+            function_name='call_openai'
+        )

backend/igny8_core/ai/constants.py

@@ -0,0 +1,37 @@
+"""
+AI Constants - Model pricing, valid models, and configuration constants
+"""
+# Model pricing (per 1M tokens) - EXACT from reference plugin model-rates-config.php
+MODEL_RATES = {
+    'gpt-4.1': {'input': 2.00, 'output': 8.00},
+    'gpt-4o-mini': {'input': 0.15, 'output': 0.60},
+    'gpt-4o': {'input': 2.50, 'output': 10.00},
+}
+
+# Image model pricing (per image) - EXACT from reference plugin
+IMAGE_MODEL_RATES = {
+    'dall-e-3': 0.040,
+    'dall-e-2': 0.020,
+    'gpt-image-1': 0.042,
+    'gpt-image-1-mini': 0.011,
+}
+
+# Valid OpenAI image generation models (only these work with /v1/images/generations endpoint)
+VALID_OPENAI_IMAGE_MODELS = {
+    'dall-e-3',
+    'dall-e-2',
+    # Note: gpt-image-1 and gpt-image-1-mini are NOT valid for OpenAI's /v1/images/generations endpoint
+}
+
+# Valid image sizes per model (from OpenAI official documentation)
+VALID_SIZES_BY_MODEL = {
+    'dall-e-3': ['1024x1024', '1024x1792', '1792x1024'],
+    'dall-e-2': ['256x256', '512x512', '1024x1024'],
+}
+
+# Default model
+DEFAULT_AI_MODEL = 'gpt-4.1'
+
+# JSON mode supported models
+JSON_MODE_MODELS = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview']
+

backend/igny8_core/ai/engine.py

@@ -5,7 +5,7 @@ import logging
 from typing import Dict, Any, Optional
 from igny8_core.ai.base import BaseAIFunction
 from igny8_core.ai.tracker import StepTracker, ProgressTracker, CostTracker
-from igny8_core.ai.processor import AIProcessor
+from igny8_core.ai.ai_core import AICore
 
 logger = logging.getLogger(__name__)
 
@@ -61,7 +61,7 @@ class AIEngine:
             self.tracker.update("PREP", 25, f"Data prepared: {data_count} items", meta=self.step_tracker.get_meta())
             
             # Phase 3: AI_CALL - Provider API Call (25-70%)
-            processor = AIProcessor(account=self.account)
+            ai_core = AICore(account=self.account)
             model = fn.get_model(self.account)
             
             # Track AI call start
@@ -69,15 +69,13 @@ class AIEngine:
             self.tracker.update("AI_CALL", 30, f"Sending to {model or 'default'}...", meta=self.step_tracker.get_meta())
             
             try:
-                raw_response = processor.call(
+                # Use centralized run_ai_request() with console logging
-                    prompt,
+                raw_response = ai_core.run_ai_request(
+                    prompt=prompt,
                     model=model,
-                    # Don't pass response_steps - the processor ignores it anyway
+                    max_tokens=4000,
-                    # Step tracking is handled by the engine
+                    temperature=0.7,
-                    progress_callback=lambda state, meta: self.tracker.update_ai_progress(state, {
+                    function_name=fn.get_name()
-                        **meta,
-                        **self.step_tracker.get_meta()
-                    })
                 )
             except Exception as e:
                 error_msg = f"AI call failed: {str(e)}"

backend/igny8_core/ai/functions/__init__.py

@@ -1,4 +1,17 @@
 """
 AI Function implementations
 """
+from igny8_core.ai.functions.auto_cluster import AutoClusterFunction
+from igny8_core.ai.functions.generate_ideas import GenerateIdeasFunction, generate_ideas_core
+from igny8_core.ai.functions.generate_content import GenerateContentFunction, generate_content_core
+from igny8_core.ai.functions.generate_images import GenerateImagesFunction, generate_images_core
 
+__all__ = [
+    'AutoClusterFunction',
+    'GenerateIdeasFunction',
+    'generate_ideas_core',
+    'GenerateContentFunction',
+    'generate_content_core',
+    'GenerateImagesFunction',
+    'generate_images_core',
+]

backend/igny8_core/ai/functions/auto_cluster.py

@@ -7,6 +7,7 @@ from django.db import transaction
 from igny8_core.ai.base import BaseAIFunction
 from igny8_core.modules.planner.models import Keywords, Clusters
 from igny8_core.modules.system.utils import get_prompt_value
+from igny8_core.ai.ai_core import AICore
 
 logger = logging.getLogger(__name__)
 
@@ -36,49 +37,23 @@ class AutoClusterFunction(BaseAIFunction):
     
     def validate(self, payload: dict, account=None) -> Dict:
         """Custom validation for clustering with plan limit checks"""
-        result = super().validate(payload, account)
+        from igny8_core.ai.validators import validate_ids, validate_keywords_exist, validate_cluster_limits
+        
+        # Base validation
+        result = validate_ids(payload, max_items=self.get_max_items())
         if not result['valid']:
             return result
         
-        # Additional validation: check keywords exist
+        # Check keywords exist
         ids = payload.get('ids', [])
-        queryset = Keywords.objects.filter(id__in=ids)
+        keywords_result = validate_keywords_exist(ids, account)
-        if account:
+        if not keywords_result['valid']:
-            queryset = queryset.filter(account=account)
+            return keywords_result
         
-        if queryset.count() == 0:
+        # Check plan limits
-            return {'valid': False, 'error': 'No keywords found'}
+        limit_result = validate_cluster_limits(account, operation_type='cluster')
-        
+        if not limit_result['valid']:
-        # Plan limit validation
+            return limit_result
-        if account:
-            plan = getattr(account, 'plan', None)
-            if plan:
-                from django.utils import timezone
-                from igny8_core.modules.planner.models import Clusters
-                
-                # Check daily cluster limit
-                now = timezone.now()
-                start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0)
-                clusters_today = Clusters.objects.filter(
-                    account=account,
-                    created_at__gte=start_of_day
-                ).count()
-                
-                if plan.daily_cluster_limit and clusters_today >= plan.daily_cluster_limit:
-                    return {
-                        'valid': False,
-                        'error': f'Daily cluster limit reached ({plan.daily_cluster_limit} clusters per day). Please try again tomorrow.'
-                    }
-                
-                # Check max clusters limit
-                total_clusters = Clusters.objects.filter(account=account).count()
-                if plan.max_clusters and total_clusters >= plan.max_clusters:
-                    return {
-                        'valid': False,
-                        'error': f'Maximum cluster limit reached ({plan.max_clusters} clusters). Please upgrade your plan or delete existing clusters.'
-                    }
-            else:
-                return {'valid': False, 'error': 'Account does not have an active plan'}
         
         return {'valid': True}
     
@@ -158,7 +133,7 @@ class AutoClusterFunction(BaseAIFunction):
     def parse_response(self, response: str, step_tracker=None) -> List[Dict]:
         """Parse AI response into cluster data"""
         import json
-        from igny8_core.ai.processor import AIProcessor
+        from igny8_core.ai.ai_core import AICore
         
         if not response or not response.strip():
             error_msg = "Empty response from AI"
@@ -172,8 +147,8 @@ class AutoClusterFunction(BaseAIFunction):
         except json.JSONDecodeError as e:
             logger.warning(f"parse_response: Direct JSON parse failed: {e}, trying extract_json method")
             # Fall back to extract_json method which handles markdown code blocks
-            processor = AIProcessor()
+            ai_core = AICore(account=getattr(self, 'account', None))
-            json_data = processor.extract_json(response)
+            json_data = ai_core.extract_json(response)
         
         if not json_data:
             error_msg = f"Failed to parse clustering response. Response: {response[:200]}..."

backend/igny8_core/ai/functions/generate_content.py

@@ -0,0 +1,263 @@
+"""
+Generate Content AI Function
+Extracted from modules/writer/tasks.py
+"""
+import logging
+import re
+from typing import Dict, List, Any
+from django.db import transaction
+from igny8_core.ai.base import BaseAIFunction
+from igny8_core.modules.writer.models import Tasks
+from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt
+from igny8_core.ai.ai_core import AICore
+from igny8_core.ai.validators import validate_tasks_exist
+
+logger = logging.getLogger(__name__)
+
+
+class GenerateContentFunction(BaseAIFunction):
+    """Generate content for tasks using AI"""
+    
+    def get_name(self) -> str:
+        return 'generate_content'
+    
+    def get_metadata(self) -> Dict:
+        return {
+            'display_name': 'Generate Content',
+            'description': 'Generate article content from task ideas',
+            'phases': {
+                'INIT': 'Initializing content generation...',
+                'PREP': 'Loading tasks and building prompts...',
+                'AI_CALL': 'Generating content with AI...',
+                'PARSE': 'Processing content...',
+                'SAVE': 'Saving content...',
+                'DONE': 'Content generated!'
+            }
+        }
+    
+    def get_max_items(self) -> int:
+        return 50  # Max tasks per batch
+    
+    def validate(self, payload: dict, account=None) -> Dict:
+        """Validate task IDs"""
+        result = super().validate(payload, account)
+        if not result['valid']:
+            return result
+        
+        # Check tasks exist
+        task_ids = payload.get('ids', [])
+        if task_ids:
+            task_result = validate_tasks_exist(task_ids, account)
+            if not task_result['valid']:
+                return task_result
+        
+        return {'valid': True}
+    
+    def prepare(self, payload: dict, account=None) -> List:
+        """Load tasks with all relationships"""
+        task_ids = payload.get('ids', [])
+        
+        queryset = Tasks.objects.filter(id__in=task_ids)
+        if account:
+            queryset = queryset.filter(account=account)
+        
+        # Preload all relationships to avoid N+1 queries
+        tasks = list(queryset.select_related(
+            'account', 'site', 'sector', 'cluster', 'idea'
+        ))
+        
+        if not tasks:
+            raise ValueError("No tasks found")
+        
+        return tasks
+    
+    def build_prompt(self, data: Any, account=None) -> str:
+        """Build content generation prompt for a single task"""
+        if isinstance(data, list):
+            # For now, handle single task (will be called per task)
+            if not data:
+                raise ValueError("No tasks provided")
+            task = data[0]
+        else:
+            task = data
+        
+        # Get prompt template
+        prompt_template = get_prompt_value(account or task.account, 'content_generation')
+        if not prompt_template:
+            prompt_template = get_default_prompt('content_generation')
+        
+        # Build idea data string
+        idea_data = f"Title: {task.title or 'Untitled'}\n"
+        if task.description:
+            idea_data += f"Description: {task.description}\n"
+        
+        # Handle idea description (might be JSON or plain text)
+        if task.idea and task.idea.description:
+            description = task.idea.description
+            try:
+                import json
+                parsed_desc = json.loads(description)
+                if isinstance(parsed_desc, dict):
+                    formatted_desc = "Content Outline:\n\n"
+                    if 'H2' in parsed_desc:
+                        for h2_section in parsed_desc['H2']:
+                            formatted_desc += f"## {h2_section.get('heading', '')}\n"
+                            if 'subsections' in h2_section:
+                                for h3_section in h2_section['subsections']:
+                                    formatted_desc += f"### {h3_section.get('subheading', '')}\n"
+                                    formatted_desc += f"Content Type: {h3_section.get('content_type', '')}\n"
+                                    formatted_desc += f"Details: {h3_section.get('details', '')}\n\n"
+                    description = formatted_desc
+            except (json.JSONDecodeError, TypeError):
+                pass  # Use as plain text
+            
+            idea_data += f"Outline: {description}\n"
+        
+        if task.idea:
+            idea_data += f"Structure: {task.idea.content_structure or task.content_structure or 'blog_post'}\n"
+            idea_data += f"Type: {task.idea.content_type or task.content_type or 'blog_post'}\n"
+            if task.idea.estimated_word_count:
+                idea_data += f"Estimated Word Count: {task.idea.estimated_word_count}\n"
+        
+        # Build cluster data string
+        cluster_data = ''
+        if task.cluster:
+            cluster_data = f"Cluster Name: {task.cluster.name or ''}\n"
+            if task.cluster.description:
+                cluster_data += f"Description: {task.cluster.description}\n"
+            cluster_data += f"Status: {task.cluster.status or 'active'}\n"
+        
+        # Build keywords string
+        keywords_data = task.keywords or ''
+        if not keywords_data and task.idea:
+            keywords_data = task.idea.target_keywords or ''
+        
+        # Replace placeholders
+        prompt = prompt_template.replace('[IGNY8_IDEA]', idea_data)
+        prompt = prompt.replace('[IGNY8_CLUSTER]', cluster_data)
+        prompt = prompt.replace('[IGNY8_KEYWORDS]', keywords_data)
+        
+        return prompt
+    
+    def parse_response(self, response: str, step_tracker=None) -> str:
+        """Parse and normalize content response"""
+        # Content is already text, just normalize it
+        try:
+            from igny8_core.utils.content_normalizer import normalize_content
+            normalized = normalize_content(response)
+            return normalized['normalized_content']
+        except Exception as e:
+            logger.warning(f"Content normalization failed: {e}, using original")
+            return response
+    
+    def save_output(
+        self,
+        parsed: str,
+        original_data: Any,
+        account=None,
+        progress_tracker=None,
+        step_tracker=None
+    ) -> Dict:
+        """Save content to task"""
+        if isinstance(original_data, list):
+            task = original_data[0] if original_data else None
+        else:
+            task = original_data
+        
+        if not task:
+            raise ValueError("No task provided for saving")
+        
+        # Calculate word count
+        text_for_counting = re.sub(r'<[^>]+>', '', parsed)
+        word_count = len(text_for_counting.split())
+        
+        # Update task
+        task.content = parsed
+        task.word_count = word_count
+        task.meta_title = task.title
+        task.meta_description = (task.description or '')[:160]
+        task.status = 'draft'
+        task.save()
+        
+        return {
+            'count': 1,
+            'tasks_updated': 1,
+            'word_count': word_count
+        }
+
+
+def generate_content_core(task_ids: List[int], account_id: int = None, progress_callback=None):
+    """
+    Core logic for generating content (legacy function signature for backward compatibility).
+    Can be called with or without Celery.
+    
+    Args:
+        task_ids: List of task IDs
+        account_id: Account ID for account isolation
+        progress_callback: Optional function to call for progress updates
+    
+    Returns:
+        Dict with 'success', 'tasks_updated', 'message', etc.
+    """
+    try:
+        from igny8_core.auth.models import Account
+        
+        account = None
+        if account_id:
+            account = Account.objects.get(id=account_id)
+        
+        # Use the new function class
+        fn = GenerateContentFunction()
+        fn.account = account
+        
+        # Prepare payload
+        payload = {'ids': task_ids}
+        
+        # Validate
+        validated = fn.validate(payload, account)
+        if not validated['valid']:
+            return {'success': False, 'error': validated['error']}
+        
+        # Prepare data
+        tasks = fn.prepare(payload, account)
+        
+        tasks_updated = 0
+        
+        # Process each task
+        for task in tasks:
+            # Build prompt for this task
+            prompt = fn.build_prompt([task], account)
+            
+            # Call AI using centralized request handler
+            ai_core = AICore(account=account)
+            result = ai_core.run_ai_request(
+                prompt=prompt,
+                max_tokens=4000,
+                function_name='generate_content'
+            )
+            
+            if result.get('error'):
+                logger.error(f"AI error for task {task.id}: {result['error']}")
+                continue
+            
+            # Parse response
+            content = fn.parse_response(result['content'])
+            
+            if not content:
+                logger.warning(f"No content generated for task {task.id}")
+                continue
+            
+            # Save output
+            save_result = fn.save_output(content, [task], account)
+            tasks_updated += save_result.get('tasks_updated', 0)
+        
+        return {
+            'success': True,
+            'tasks_updated': tasks_updated,
+            'message': f'Content generation complete: {tasks_updated} articles generated'
+        }
+        
+    except Exception as e:
+        logger.error(f"Error in generate_content_core: {str(e)}", exc_info=True)
+        return {'success': False, 'error': str(e)}
+

backend/igny8_core/ai/functions/generate_ideas.py

@@ -0,0 +1,256 @@
+"""
+Generate Ideas AI Function
+Extracted from modules/planner/tasks.py
+"""
+import logging
+import json
+from typing import Dict, List, Any
+from django.db import transaction
+from igny8_core.ai.base import BaseAIFunction
+from igny8_core.modules.planner.models import Clusters, ContentIdeas
+from igny8_core.modules.system.utils import get_prompt_value
+from igny8_core.ai.ai_core import AICore
+from igny8_core.ai.validators import validate_cluster_exists, validate_cluster_limits
+
+logger = logging.getLogger(__name__)
+
+
+class GenerateIdeasFunction(BaseAIFunction):
+    """Generate content ideas from clusters using AI"""
+    
+    def get_name(self) -> str:
+        return 'generate_ideas'
+    
+    def get_metadata(self) -> Dict:
+        return {
+            'display_name': 'Generate Ideas',
+            'description': 'Generate SEO-optimized content ideas from keyword clusters',
+            'phases': {
+                'INIT': 'Initializing idea generation...',
+                'PREP': 'Loading clusters...',
+                'AI_CALL': 'Generating ideas with AI...',
+                'PARSE': 'Parsing idea data...',
+                'SAVE': 'Saving ideas...',
+                'DONE': 'Ideas generated!'
+            }
+        }
+    
+    def get_max_items(self) -> int:
+        return 10  # Max clusters per idea generation
+    
+    def validate(self, payload: dict, account=None) -> Dict:
+        """Validate cluster IDs and plan limits"""
+        result = super().validate(payload, account)
+        if not result['valid']:
+            return result
+        
+        # Check cluster exists
+        cluster_ids = payload.get('ids', [])
+        if cluster_ids:
+            cluster_id = cluster_ids[0]  # For single cluster idea generation
+            cluster_result = validate_cluster_exists(cluster_id, account)
+            if not cluster_result['valid']:
+                return cluster_result
+        
+        # Check plan limits
+        limit_result = validate_cluster_limits(account, operation_type='idea')
+        if not limit_result['valid']:
+            return limit_result
+        
+        return {'valid': True}
+    
+    def prepare(self, payload: dict, account=None) -> Dict:
+        """Load cluster with keywords"""
+        cluster_ids = payload.get('ids', [])
+        if not cluster_ids:
+            raise ValueError("No cluster IDs provided")
+        
+        cluster_id = cluster_ids[0]  # Single cluster for now
+        
+        queryset = Clusters.objects.filter(id=cluster_id)
+        if account:
+            queryset = queryset.filter(account=account)
+        
+        cluster = queryset.select_related('sector', 'account', 'site').prefetch_related('keywords').first()
+        
+        if not cluster:
+            raise ValueError("Cluster not found")
+        
+        # Get keywords for this cluster
+        from igny8_core.modules.planner.models import Keywords
+        keywords = Keywords.objects.filter(cluster=cluster).values_list('keyword', flat=True)
+        
+        # Format cluster data for AI
+        cluster_data = [{
+            'id': cluster.id,
+            'name': cluster.name,
+            'description': cluster.description or '',
+            'keywords': list(keywords),
+        }]
+        
+        return {
+            'cluster': cluster,
+            'cluster_data': cluster_data,
+            'account': account or cluster.account
+        }
+    
+    def build_prompt(self, data: Dict, account=None) -> str:
+        """Build ideas generation prompt"""
+        cluster_data = data['cluster_data']
+        
+        # Get prompt template
+        prompt_template = get_prompt_value(account or data.get('account'), 'ideas')
+        
+        # Format clusters text
+        clusters_text = '\n'.join([
+            f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Description: {c.get('description', '')}"
+            for c in cluster_data
+        ])
+        
+        # Format cluster keywords
+        cluster_keywords_text = '\n'.join([
+            f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Keywords: {', '.join(c.get('keywords', []))}"
+            for c in cluster_data
+        ])
+        
+        # Replace placeholders
+        prompt = prompt_template.replace('[IGNY8_CLUSTERS]', clusters_text)
+        prompt = prompt.replace('[IGNY8_CLUSTER_KEYWORDS]', cluster_keywords_text)
+        
+        return prompt
+    
+    def parse_response(self, response: str, step_tracker=None) -> List[Dict]:
+        """Parse AI response into idea data"""
+        ai_core = AICore(account=self.account if hasattr(self, 'account') else None)
+        json_data = ai_core.extract_json(response)
+        
+        if not json_data or 'ideas' not in json_data:
+            error_msg = f"Failed to parse ideas response: {response[:200]}..."
+            logger.error(error_msg)
+            raise ValueError(error_msg)
+        
+        return json_data.get('ideas', [])
+    
+    def save_output(
+        self,
+        parsed: List[Dict],
+        original_data: Dict,
+        account=None,
+        progress_tracker=None,
+        step_tracker=None
+    ) -> Dict:
+        """Save ideas to database"""
+        cluster = original_data['cluster']
+        account = account or original_data.get('account')
+        
+        if not account:
+            raise ValueError("Account is required for idea creation")
+        
+        ideas_created = 0
+        
+        with transaction.atomic():
+            for idea_data in parsed:
+                # Handle description - might be dict or string
+                description = idea_data.get('description', '')
+                if isinstance(description, dict):
+                    description = json.dumps(description)
+                elif not isinstance(description, str):
+                    description = str(description)
+                
+                # Handle target_keywords
+                target_keywords = idea_data.get('covered_keywords', '') or idea_data.get('target_keywords', '')
+                
+                # Create ContentIdeas record
+                ContentIdeas.objects.create(
+                    idea_title=idea_data.get('title', 'Untitled Idea'),
+                    description=description,
+                    content_type=idea_data.get('content_type', 'blog_post'),
+                    content_structure=idea_data.get('content_structure', 'supporting_page'),
+                    target_keywords=target_keywords,
+                    keyword_cluster=cluster,
+                    estimated_word_count=idea_data.get('estimated_word_count', 1500),
+                    status='new',
+                    account=account,
+                    site=cluster.site,
+                    sector=cluster.sector,
+                )
+                ideas_created += 1
+        
+        return {
+            'count': ideas_created,
+            'ideas_created': ideas_created
+        }
+
+
+def generate_ideas_core(cluster_id: int, account_id: int = None, progress_callback=None):
+    """
+    Core logic for generating ideas (legacy function signature for backward compatibility).
+    Can be called with or without Celery.
+    
+    Args:
+        cluster_id: Cluster ID to generate idea for
+        account_id: Account ID for account isolation
+        progress_callback: Optional function to call for progress updates
+    
+    Returns:
+        Dict with 'success', 'idea_created', 'message', etc.
+    """
+    try:
+        from igny8_core.auth.models import Account
+        
+        account = None
+        if account_id:
+            account = Account.objects.get(id=account_id)
+        
+        # Use the new function class
+        fn = GenerateIdeasFunction()
+        # Store account for use in methods
+        fn.account = account
+        
+        # Prepare payload
+        payload = {'ids': [cluster_id]}
+        
+        # Validate
+        validated = fn.validate(payload, account)
+        if not validated['valid']:
+            return {'success': False, 'error': validated['error']}
+        
+        # Prepare data
+        data = fn.prepare(payload, account)
+        
+        # Build prompt
+        prompt = fn.build_prompt(data, account)
+        
+        # Call AI using centralized request handler
+        ai_core = AICore(account=account)
+        result = ai_core.run_ai_request(
+            prompt=prompt,
+            max_tokens=4000,
+            function_name='generate_ideas'
+        )
+        
+        if result.get('error'):
+            return {'success': False, 'error': result['error']}
+        
+        # Parse response
+        ideas_data = fn.parse_response(result['content'])
+        
+        if not ideas_data:
+            return {'success': False, 'error': 'No ideas generated by AI'}
+        
+        # Take first idea
+        idea_data = ideas_data[0]
+        
+        # Save output
+        save_result = fn.save_output(ideas_data, data, account)
+        
+        return {
+            'success': True,
+            'idea_created': save_result['ideas_created'],
+            'message': f"Idea '{idea_data.get('title', 'Untitled')}' created"
+        }
+        
+    except Exception as e:
+        logger.error(f"Error in generate_ideas_core: {str(e)}", exc_info=True)
+        return {'success': False, 'error': str(e)}
+

backend/igny8_core/ai/functions/generate_images.py

@@ -0,0 +1,275 @@
+"""
+Generate Images AI Function
+Extracted from modules/writer/tasks.py
+"""
+import logging
+from typing import Dict, List, Any
+from django.db import transaction
+from igny8_core.ai.base import BaseAIFunction
+from igny8_core.modules.writer.models import Tasks, Images
+from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt
+from igny8_core.ai.ai_core import AICore
+from igny8_core.ai.validators import validate_tasks_exist
+
+logger = logging.getLogger(__name__)
+
+
+class GenerateImagesFunction(BaseAIFunction):
+    """Generate images for tasks using AI"""
+    
+    def get_name(self) -> str:
+        return 'generate_images'
+    
+    def get_metadata(self) -> Dict:
+        return {
+            'display_name': 'Generate Images',
+            'description': 'Generate featured and in-article images for tasks',
+            'phases': {
+                'INIT': 'Initializing image generation...',
+                'PREP': 'Extracting image prompts...',
+                'AI_CALL': 'Generating images with AI...',
+                'PARSE': 'Processing image URLs...',
+                'SAVE': 'Saving images...',
+                'DONE': 'Images generated!'
+            }
+        }
+    
+    def get_max_items(self) -> int:
+        return 20  # Max tasks per batch
+    
+    def validate(self, payload: dict, account=None) -> Dict:
+        """Validate task IDs"""
+        result = super().validate(payload, account)
+        if not result['valid']:
+            return result
+        
+        # Check tasks exist
+        task_ids = payload.get('ids', [])
+        if task_ids:
+            task_result = validate_tasks_exist(task_ids, account)
+            if not task_result['valid']:
+                return task_result
+        
+        return {'valid': True}
+    
+    def prepare(self, payload: dict, account=None) -> Dict:
+        """Load tasks and image generation settings"""
+        task_ids = payload.get('ids', [])
+        
+        queryset = Tasks.objects.filter(id__in=task_ids)
+        if account:
+            queryset = queryset.filter(account=account)
+        
+        tasks = list(queryset.select_related('account', 'sector', 'site'))
+        
+        if not tasks:
+            raise ValueError("No tasks found")
+        
+        # Get image generation settings
+        image_settings = {}
+        if account:
+            try:
+                from igny8_core.modules.system.models import IntegrationSettings
+                integration = IntegrationSettings.objects.get(
+                    account=account,
+                    integration_type='image_generation',
+                    is_active=True
+                )
+                image_settings = integration.config or {}
+            except Exception:
+                pass
+        
+        # Extract settings with defaults
+        provider = image_settings.get('provider') or image_settings.get('service', 'openai')
+        if provider == 'runware':
+            model = image_settings.get('model') or image_settings.get('runwareModel', 'runware:97@1')
+        else:
+            model = image_settings.get('model', 'dall-e-3')
+        
+        return {
+            'tasks': tasks,
+            'account': account,
+            'provider': provider,
+            'model': model,
+            'image_type': image_settings.get('image_type', 'realistic'),
+            'max_in_article_images': int(image_settings.get('max_in_article_images', 2)),
+            'desktop_enabled': image_settings.get('desktop_enabled', True),
+            'mobile_enabled': image_settings.get('mobile_enabled', True),
+        }
+    
+    def build_prompt(self, data: Dict, account=None) -> Dict:
+        """Extract image prompts from task content"""
+        task = data.get('task')
+        max_images = data.get('max_in_article_images', 2)
+        
+        if not task or not task.content:
+            raise ValueError("Task has no content")
+        
+        # Use AI to extract image prompts
+        ai_core = AICore(account=account or data.get('account'))
+        
+        # Get prompt template
+        prompt_template = get_prompt_value(account or data.get('account'), 'image_prompt_extraction')
+        if not prompt_template:
+            prompt_template = get_default_prompt('image_prompt_extraction')
+        
+        # Format prompt
+        prompt = prompt_template.format(
+            title=task.title,
+            content=task.content[:5000],  # Limit content length
+            max_images=max_images
+        )
+        
+        # Call AI to extract prompts using centralized request handler
+        result = ai_core.run_ai_request(
+            prompt=prompt,
+            max_tokens=1000,
+            function_name='extract_image_prompts'
+        )
+        
+        if result.get('error'):
+            raise ValueError(f"Failed to extract image prompts: {result['error']}")
+        
+        # Parse JSON response
+        json_data = ai_core.extract_json(result['content'])
+        
+        if not json_data:
+            raise ValueError("Failed to parse image prompts response")
+        
+        return {
+            'featured_prompt': json_data.get('featured_prompt', ''),
+            'in_article_prompts': json_data.get('in_article_prompts', [])
+        }
+    
+    def parse_response(self, response: Dict, step_tracker=None) -> Dict:
+        """Parse image generation response (already parsed, just return)"""
+        return response
+    
+    def save_output(
+        self,
+        parsed: Dict,
+        original_data: Dict,
+        account=None,
+        progress_tracker=None,
+        step_tracker=None
+    ) -> Dict:
+        """Save images to database"""
+        task = original_data.get('task')
+        image_url = parsed.get('url')
+        image_type = parsed.get('image_type')  # 'featured', 'desktop', 'mobile'
+        
+        if not task or not image_url:
+            raise ValueError("Missing task or image URL")
+        
+        # Create Images record
+        image = Images.objects.create(
+            task=task,
+            image_url=image_url,
+            image_type=image_type,
+            account=account or task.account,
+            site=task.site,
+            sector=task.sector,
+        )
+        
+        return {
+            'count': 1,
+            'images_created': 1,
+            'image_id': image.id
+        }
+
+
+def generate_images_core(task_ids: List[int], account_id: int = None, progress_callback=None):
+    """
+    Core logic for generating images (legacy function signature for backward compatibility).
+    Can be called with or without Celery.
+    
+    Args:
+        task_ids: List of task IDs
+        account_id: Account ID for account isolation
+        progress_callback: Optional function to call for progress updates
+    
+    Returns:
+        Dict with 'success', 'images_created', 'message', etc.
+    """
+    try:
+        from igny8_core.auth.models import Account
+        
+        account = None
+        if account_id:
+            account = Account.objects.get(id=account_id)
+        
+        # Use the new function class
+        fn = GenerateImagesFunction()
+        fn.account = account
+        
+        # Prepare payload
+        payload = {'ids': task_ids}
+        
+        # Validate
+        validated = fn.validate(payload, account)
+        if not validated['valid']:
+            return {'success': False, 'error': validated['error']}
+        
+        # Prepare data
+        data = fn.prepare(payload, account)
+        tasks = data['tasks']
+        
+        # Get prompts
+        image_prompt_template = get_prompt_value(account, 'image_prompt_template')
+        if not image_prompt_template:
+            image_prompt_template = get_default_prompt('image_prompt_template')
+        
+        negative_prompt = get_prompt_value(account, 'negative_prompt')
+        if not negative_prompt:
+            negative_prompt = get_default_prompt('negative_prompt')
+        
+        ai_core = AICore(account=account)
+        images_created = 0
+        
+        # Process each task
+        for task in tasks:
+            if not task.content:
+                continue
+            
+            # Extract image prompts
+            prompts_data = fn.build_prompt({'task': task, **data}, account)
+            featured_prompt = prompts_data['featured_prompt']
+            in_article_prompts = prompts_data['in_article_prompts']
+            
+            # Format featured prompt
+            formatted_featured = image_prompt_template.format(
+                image_type=data['image_type'],
+                post_title=task.title,
+                image_prompt=featured_prompt
+            )
+            
+            # Generate featured image using centralized handler
+            featured_result = ai_core.generate_image(
+                prompt=formatted_featured,
+                provider=data['provider'],
+                model=data['model'],
+                negative_prompt=negative_prompt,
+                function_name='generate_images'
+            )
+            
+            if not featured_result.get('error') and featured_result.get('url'):
+                fn.save_output(
+                    {'url': featured_result['url'], 'image_type': 'featured'},
+                    {'task': task, **data},
+                    account
+                )
+                images_created += 1
+            
+            # Generate in-article images (desktop/mobile if enabled)
+            # ... (simplified for now, full logic in tasks.py)
+        
+        return {
+            'success': True,
+            'images_created': images_created,
+            'message': f'Image generation complete: {images_created} images created'
+        }
+        
+    except Exception as e:
+        logger.error(f"Error in generate_images_core: {str(e)}", exc_info=True)
+        return {'success': False, 'error': str(e)}
+

backend/igny8_core/ai/processor.py

@@ -1,20 +1,26 @@
 """
 AI Processor wrapper for the framework
-Reuses existing AIProcessor but provides framework-compatible interface
+DEPRECATED: Use AICore.run_ai_request() instead for all new code.
+This file is kept for backward compatibility only.
 """
 from typing import Dict, Any, Optional, List
 from igny8_core.utils.ai_processor import AIProcessor as BaseAIProcessor
+from igny8_core.ai.ai_core import AICore
 
 
 class AIProcessor:
     """
     Framework-compatible wrapper around existing AIProcessor.
-    Provides consistent interface for all AI functions.
+    DEPRECATED: Use AICore.run_ai_request() instead.
+    This class redirects to AICore for consistency.
     """
     
     def __init__(self, account=None):
-        self.processor = BaseAIProcessor(account=account)
+        # Use AICore internally for all requests
+        self.ai_core = AICore(account=account)
         self.account = account
+        # Keep old processor for backward compatibility only
+        self.processor = BaseAIProcessor(account=account)
     
     def call(
         self,
@@ -28,35 +34,25 @@ class AIProcessor:
     ) -> Dict[str, Any]:
         """
         Call AI provider with prompt.
+        DEPRECATED: Use AICore.run_ai_request() instead.
         
         Returns:
             Dict with 'content', 'error', 'input_tokens', 'output_tokens',
             'total_tokens', 'model', 'cost', 'api_id'
         """
-        # Use specified model or account's default
+        # Redirect to AICore for centralized execution
-        active_model = model or self.processor.default_model
+        return self.ai_core.run_ai_request(
-        
+            prompt=prompt,
-        # Check if model supports JSON mode
+            model=model,
-        json_models = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview']
-        if response_format is None and active_model in json_models:
-            response_format = {'type': 'json_object'}
-        
-        # Call OpenAI - don't pass response_steps to old processor
-        # The new framework handles all step tracking at the engine level
-        result = self.processor._call_openai(
-            prompt,
-            model=active_model,
             max_tokens=max_tokens,
             temperature=temperature,
             response_format=response_format,
-            response_steps=None  # Disable old processor's step tracking
+            function_name='AIProcessor.call'
         )
-        
-        return result
     
     def extract_json(self, response_text: str) -> Optional[Dict]:
         """Extract JSON from response text"""
-        return self.processor._extract_json_from_response(response_text)
+        return self.ai_core.extract_json(response_text)
     
     def generate_image(
         self,
@@ -67,11 +63,13 @@ class AIProcessor:
         account=None
     ) -> Dict[str, Any]:
         """Generate image using AI"""
-        return self.processor.generate_image(
+        return self.ai_core.generate_image(
             prompt=prompt,
+            provider='openai',
             model=model,
             size=size,
             n=n,
-            account=account or self.account
+            account=account or self.account,
+            function_name='AIProcessor.generate_image'
         )
 

backend/igny8_core/ai/registry.py

@@ -66,5 +66,23 @@ def _load_auto_cluster():
     from igny8_core.ai.functions.auto_cluster import AutoClusterFunction
     return AutoClusterFunction
 
-register_lazy_function('auto_cluster', _load_auto_cluster)
+def _load_generate_ideas():
+    """Lazy loader for generate_ideas function"""
+    from igny8_core.ai.functions.generate_ideas import GenerateIdeasFunction
+    return GenerateIdeasFunction
+
+def _load_generate_content():
+    """Lazy loader for generate_content function"""
+    from igny8_core.ai.functions.generate_content import GenerateContentFunction
+    return GenerateContentFunction
+
+def _load_generate_images():
+    """Lazy loader for generate_images function"""
+    from igny8_core.ai.functions.generate_images import GenerateImagesFunction
+    return GenerateImagesFunction
+
+register_lazy_function('auto_cluster', _load_auto_cluster)
+register_lazy_function('generate_ideas', _load_generate_ideas)
+register_lazy_function('generate_content', _load_generate_content)
+register_lazy_function('generate_images', _load_generate_images)
 

backend/igny8_core/ai/tests/test_run.py

@@ -0,0 +1,134 @@
+"""
+Test script for AI functions
+Run this to verify all AI functions work with console logging
+"""
+import os
+import sys
+import django
+
+# Setup Django
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../../../'))
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'igny8.settings')
+django.setup()
+
+from igny8_core.ai.functions.auto_cluster import AutoClusterFunction
+from igny8_core.ai.functions.generate_ideas import generate_ideas_core
+from igny8_core.ai.functions.generate_content import generate_content_core
+from igny8_core.ai.functions.generate_images import generate_images_core
+from igny8_core.ai.ai_core import AICore
+
+
+def test_ai_core():
+    """Test AICore.run_ai_request() directly"""
+    print("\n" + "="*80)
+    print("TEST 1: AICore.run_ai_request() - Direct API Call")
+    print("="*80)
+    
+    ai_core = AICore()
+    result = ai_core.run_ai_request(
+        prompt="Say 'Hello, World!' in JSON format: {\"message\": \"your message\"}",
+        max_tokens=100,
+        function_name='test_ai_core'
+    )
+    
+    if result.get('error'):
+        print(f"❌ Error: {result['error']}")
+    else:
+        print(f"✅ Success! Content: {result.get('content', '')[:100]}")
+        print(f"   Tokens: {result.get('total_tokens')}, Cost: ${result.get('cost', 0):.6f}")
+
+
+def test_auto_cluster():
+    """Test auto cluster function"""
+    print("\n" + "="*80)
+    print("TEST 2: Auto Cluster Function")
+    print("="*80)
+    print("Note: This requires actual keyword IDs in the database")
+    print("Skipping - requires database setup")
+    # Uncomment to test with real data:
+    # fn = AutoClusterFunction()
+    # result = fn.validate({'ids': [1, 2, 3]})
+    # print(f"Validation result: {result}")
+
+
+def test_generate_ideas():
+    """Test generate ideas function"""
+    print("\n" + "="*80)
+    print("TEST 3: Generate Ideas Function")
+    print("="*80)
+    print("Note: This requires actual cluster ID in the database")
+    print("Skipping - requires database setup")
+    # Uncomment to test with real data:
+    # result = generate_ideas_core(cluster_id=1, account_id=1)
+    # print(f"Result: {result}")
+
+
+def test_generate_content():
+    """Test generate content function"""
+    print("\n" + "="*80)
+    print("TEST 4: Generate Content Function")
+    print("="*80)
+    print("Note: This requires actual task IDs in the database")
+    print("Skipping - requires database setup")
+    # Uncomment to test with real data:
+    # result = generate_content_core(task_ids=[1], account_id=1)
+    # print(f"Result: {result}")
+
+
+def test_generate_images():
+    """Test generate images function"""
+    print("\n" + "="*80)
+    print("TEST 5: Generate Images Function")
+    print("="*80)
+    print("Note: This requires actual task IDs in the database")
+    print("Skipping - requires database setup")
+    # Uncomment to test with real data:
+    # result = generate_images_core(task_ids=[1], account_id=1)
+    # print(f"Result: {result}")
+
+
+def test_json_extraction():
+    """Test JSON extraction"""
+    print("\n" + "="*80)
+    print("TEST 6: JSON Extraction")
+    print("="*80)
+    
+    ai_core = AICore()
+    
+    # Test 1: Direct JSON
+    json_text = '{"clusters": [{"name": "Test", "keywords": ["test"]}]}'
+    result = ai_core.extract_json(json_text)
+    print(f"✅ Direct JSON: {result is not None}")
+    
+    # Test 2: JSON in markdown
+    json_markdown = '```json\n{"clusters": [{"name": "Test"}]}\n```'
+    result = ai_core.extract_json(json_markdown)
+    print(f"✅ JSON in markdown: {result is not None}")
+    
+    # Test 3: Invalid JSON
+    invalid_json = "This is not JSON"
+    result = ai_core.extract_json(invalid_json)
+    print(f"✅ Invalid JSON handled: {result is None}")
+
+
+if __name__ == '__main__':
+    print("\n" + "="*80)
+    print("AI FUNCTIONS TEST SUITE")
+    print("="*80)
+    print("Testing all AI functions with console logging enabled")
+    print("="*80)
+    
+    # Run tests
+    test_ai_core()
+    test_json_extraction()
+    test_auto_cluster()
+    test_generate_ideas()
+    test_generate_content()
+    test_generate_images()
+    
+    print("\n" + "="*80)
+    print("TEST SUITE COMPLETE")
+    print("="*80)
+    print("\nAll console logging should be visible above.")
+    print("Check for [AI][function_name] Step X: messages")
+

backend/igny8_core/ai/validators.py

@@ -0,0 +1,217 @@
+"""
+AI Validators - Consolidated validation logic for all AI functions
+"""
+import logging
+from typing import Dict, Any, Optional
+from django.utils import timezone
+
+logger = logging.getLogger(__name__)
+
+
+def validate_ids(payload: dict, max_items: Optional[int] = None) -> Dict[str, Any]:
+    """
+    Base validation: checks for 'ids' array and max_items limit.
+    
+    Args:
+        payload: Request payload containing 'ids' array
+        max_items: Maximum number of items allowed (None = no limit)
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    ids = payload.get('ids', [])
+    if not ids:
+        return {'valid': False, 'error': 'No IDs provided'}
+    
+    if max_items and len(ids) > max_items:
+        return {'valid': False, 'error': f'Maximum {max_items} items allowed'}
+    
+    return {'valid': True}
+
+
+def validate_keywords_exist(ids: list, account=None) -> Dict[str, Any]:
+    """
+    Validate that keywords exist in database.
+    
+    Args:
+        ids: List of keyword IDs
+        account: Optional account for filtering
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    from igny8_core.modules.planner.models import Keywords
+    
+    queryset = Keywords.objects.filter(id__in=ids)
+    if account:
+        queryset = queryset.filter(account=account)
+    
+    if queryset.count() == 0:
+        return {'valid': False, 'error': 'No keywords found'}
+    
+    return {'valid': True}
+
+
+def validate_cluster_limits(account, operation_type: str = 'cluster') -> Dict[str, Any]:
+    """
+    Validate plan limits for cluster operations.
+    
+    Args:
+        account: Account object
+        operation_type: Type of operation ('cluster', 'idea', etc.)
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    if not account:
+        return {'valid': False, 'error': 'Account is required'}
+    
+    plan = getattr(account, 'plan', None)
+    if not plan:
+        return {'valid': False, 'error': 'Account does not have an active plan'}
+    
+    if operation_type == 'cluster':
+        from igny8_core.modules.planner.models import Clusters
+        
+        # Check daily cluster limit
+        now = timezone.now()
+        start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0)
+        clusters_today = Clusters.objects.filter(
+            account=account,
+            created_at__gte=start_of_day
+        ).count()
+        
+        if plan.daily_cluster_limit and clusters_today >= plan.daily_cluster_limit:
+            return {
+                'valid': False,
+                'error': f'Daily cluster limit reached ({plan.daily_cluster_limit} clusters per day). Please try again tomorrow.'
+            }
+        
+        # Check max clusters limit
+        total_clusters = Clusters.objects.filter(account=account).count()
+        if plan.max_clusters and total_clusters >= plan.max_clusters:
+            return {
+                'valid': False,
+                'error': f'Maximum cluster limit reached ({plan.max_clusters} clusters). Please upgrade your plan or delete existing clusters.'
+            }
+    
+    return {'valid': True}
+
+
+def validate_cluster_exists(cluster_id: int, account=None) -> Dict[str, Any]:
+    """
+    Validate that a cluster exists.
+    
+    Args:
+        cluster_id: Cluster ID
+        account: Optional account for filtering
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    from igny8_core.modules.planner.models import Clusters
+    
+    queryset = Clusters.objects.filter(id=cluster_id)
+    if account:
+        queryset = queryset.filter(account=account)
+    
+    if not queryset.exists():
+        return {'valid': False, 'error': 'Cluster not found'}
+    
+    return {'valid': True}
+
+
+def validate_tasks_exist(task_ids: list, account=None) -> Dict[str, Any]:
+    """
+    Validate that tasks exist in database.
+    
+    Args:
+        task_ids: List of task IDs
+        account: Optional account for filtering
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    from igny8_core.modules.writer.models import Tasks
+    
+    queryset = Tasks.objects.filter(id__in=task_ids)
+    if account:
+        queryset = queryset.filter(account=account)
+    
+    if queryset.count() == 0:
+        return {'valid': False, 'error': 'No tasks found'}
+    
+    return {'valid': True}
+
+
+def validate_api_key(api_key: Optional[str], integration_type: str = 'openai') -> Dict[str, Any]:
+    """
+    Validate that API key is configured.
+    
+    Args:
+        api_key: API key to validate
+        integration_type: Type of integration ('openai', 'runware')
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    if not api_key:
+        return {
+            'valid': False,
+            'error': f'{integration_type.upper()} API key not configured'
+        }
+    
+    return {'valid': True}
+
+
+def validate_model(model: str, model_type: str = 'text') -> Dict[str, Any]:
+    """
+    Validate that model is in supported list.
+    
+    Args:
+        model: Model name to validate
+        model_type: Type of model ('text' or 'image')
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    from .constants import MODEL_RATES, VALID_OPENAI_IMAGE_MODELS
+    
+    if model_type == 'text':
+        if model not in MODEL_RATES:
+            return {
+                'valid': False,
+                'error': f'Model "{model}" is not in supported models list'
+            }
+    elif model_type == 'image':
+        if model not in VALID_OPENAI_IMAGE_MODELS:
+            return {
+                'valid': False,
+                'error': f'Model "{model}" is not valid for OpenAI image generation. Only {", ".join(VALID_OPENAI_IMAGE_MODELS)} are supported.'
+            }
+    
+    return {'valid': True}
+
+
+def validate_image_size(size: str, model: str) -> Dict[str, Any]:
+    """
+    Validate that image size is valid for the selected model.
+    
+    Args:
+        size: Image size (e.g., '1024x1024')
+        model: Model name
+    
+    Returns:
+        Dict with 'valid' (bool) and optional 'error' (str)
+    """
+    from .constants import VALID_SIZES_BY_MODEL
+    
+    valid_sizes = VALID_SIZES_BY_MODEL.get(model, [])
+    if size not in valid_sizes:
+        return {
+            'valid': False,
+            'error': f'Image size "{size}" is not valid for model "{model}". Valid sizes are: {", ".join(valid_sizes)}'
+        }
+    
+    return {'valid': True}
+

backend/igny8_core/modules/planner/tasks.py

@@ -7,6 +7,7 @@ from typing import List
 from django.db import transaction
 from igny8_core.modules.planner.models import Keywords, Clusters, ContentIdeas
 from igny8_core.utils.ai_processor import ai_processor
+from igny8_core.ai.functions.generate_ideas import generate_ideas_core
 
 logger = logging.getLogger(__name__)
 

backend/igny8_core/modules/planner/views.py

@@ -977,8 +977,8 @@ class ContentIdeasViewSet(SiteSectorModelViewSet):
                     except (KombuOperationalError, ConnectionError) as e:
                         # Celery connection failed - execute synchronously
                         logger.warning(f"Celery connection failed, executing synchronously: {e}")
-                        from .tasks import _generate_single_idea_core
+                        from igny8_core.ai.functions.generate_ideas import generate_ideas_core
-                        result = _generate_single_idea_core(cluster_id, account_id=account_id, progress_callback=None)
+                        result = generate_ideas_core(cluster_id, account_id=account_id, progress_callback=None)
                         if result.get('success'):
                             return Response({
                                 'success': True,

backend/igny8_core/modules/writer/tasks.py

@@ -8,6 +8,8 @@ from django.db import transaction
 from igny8_core.modules.writer.models import Tasks, Images, Content
 from igny8_core.utils.ai_processor import ai_processor
 from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt
+from igny8_core.ai.functions.generate_content import generate_content_core
+from igny8_core.ai.functions.generate_images import generate_images_core
 
 logger = logging.getLogger(__name__)
 

docs/ActiveDocs/STAGE1-REORGANIZATION-COMPLETE.md

@@ -0,0 +1,191 @@
+# Stage 1 - AI Folder Structure & Functional Split - COMPLETE ✅
+
+## Summary
+
+Successfully reorganized the AI backend into a clean, modular structure where every AI function lives inside its own file within `/ai/functions/`.
+
+## ✅ Completed Deliverables
+
+### 1. Folder Structure Created
+
+```
+backend/igny8_core/ai/
+├── functions/
+│   ├── __init__.py ✅
+│   ├── auto_cluster.py ✅
+│   ├── generate_ideas.py ✅
+│   ├── generate_content.py ✅
+│   └── generate_images.py ✅
+├── ai_core.py ✅ (Shared operations)
+├── validators.py ✅ (Consolidated validation)
+├── constants.py ✅ (Model pricing, valid models)
+├── engine.py ✅ (Updated to use AICore)
+├── tracker.py ✅ (Existing)
+├── base.py ✅ (Existing)
+├── processor.py ✅ (Existing wrapper)
+├── registry.py ✅ (Updated with new functions)
+└── __init__.py ✅ (Updated exports)
+```
+
+### 2. Shared Modules Created
+
+#### `ai_core.py`
+- **Purpose**: Shared operations for all AI functions
+- **Features**:
+  - API call construction (`call_openai`)
+  - Model selection (`get_model`, `get_api_key`)
+  - Response parsing (`extract_json`)
+  - Image generation (`generate_image`)
+  - Cost calculation (`calculate_cost`)
+- **Status**: ✅ Complete
+
+#### `validators.py`
+- **Purpose**: Consolidated validation logic
+- **Functions**:
+  - `validate_ids()` - Base ID validation
+  - `validate_keywords_exist()` - Keyword existence check
+  - `validate_cluster_limits()` - Plan limit checks
+  - `validate_cluster_exists()` - Cluster existence
+  - `validate_tasks_exist()` - Task existence
+  - `validate_api_key()` - API key validation
+  - `validate_model()` - Model validation
+  - `validate_image_size()` - Image size validation
+- **Status**: ✅ Complete
+
+#### `constants.py`
+- **Purpose**: AI-related constants
+- **Constants**:
+  - `MODEL_RATES` - Text model pricing
+  - `IMAGE_MODEL_RATES` - Image model pricing
+  - `VALID_OPENAI_IMAGE_MODELS` - Valid image models
+  - `VALID_SIZES_BY_MODEL` - Valid sizes per model
+  - `DEFAULT_AI_MODEL` - Default model name
+  - `JSON_MODE_MODELS` - Models supporting JSON mode
+- **Status**: ✅ Complete
+
+### 3. Function Files Created
+
+#### `functions/auto_cluster.py`
+- **Status**: ✅ Updated to use new validators and AICore
+- **Changes**:
+  - Uses `validate_ids()`, `validate_keywords_exist()`, `validate_cluster_limits()` from validators
+  - Uses `AICore.extract_json()` for JSON parsing
+  - Maintains backward compatibility
+
+#### `functions/generate_ideas.py`
+- **Status**: ✅ Created
+- **Features**:
+  - `GenerateIdeasFunction` class (BaseAIFunction)
+  - `generate_ideas_core()` legacy function for backward compatibility
+  - Uses AICore for API calls
+  - Uses validators for validation
+
+#### `functions/generate_content.py`
+- **Status**: ✅ Created
+- **Features**:
+  - `GenerateContentFunction` class (BaseAIFunction)
+  - `generate_content_core()` legacy function for backward compatibility
+  - Uses AICore for API calls
+  - Uses validators for validation
+
+#### `functions/generate_images.py`
+- **Status**: ✅ Created
+- **Features**:
+  - `GenerateImagesFunction` class (BaseAIFunction)
+  - `generate_images_core()` legacy function for backward compatibility
+  - Uses AICore for image generation
+  - Uses validators for validation
+
+### 4. Import Paths Updated
+
+#### Updated Files:
+- ✅ `modules/planner/views.py` - Uses `generate_ideas_core` from new location
+- ✅ `modules/planner/tasks.py` - Imports `generate_ideas_core` from new location
+- ✅ `modules/writer/tasks.py` - Imports `generate_content_core` and `generate_images_core` from new locations
+- ✅ `ai/engine.py` - Uses `AICore` instead of `AIProcessor`
+- ✅ `ai/functions/auto_cluster.py` - Uses new validators and AICore
+- ✅ `ai/registry.py` - Registered all new functions
+- ✅ `ai/__init__.py` - Exports all new modules
+
+### 5. Dependencies Verified
+
+#### No Circular Dependencies ✅
+- Functions depend on: `ai_core`, `validators`, `constants`, `base`
+- `ai_core` depends on: `utils.ai_processor` (legacy, will be refactored later)
+- `validators` depends on: `constants`, models
+- `engine` depends on: `ai_core`, `base`, `tracker`
+- All imports are clean and modular
+
+#### Modular Structure ✅
+- Each function file is self-contained
+- Shared logic in `ai_core.py`
+- Validation logic in `validators.py`
+- Constants in `constants.py`
+- No scattered or duplicated logic
+
+## 📋 File Structure Details
+
+### Core AI Modules
+
+| File | Purpose | Dependencies |
+|------|---------|--------------|
+| `ai_core.py` | Shared AI operations | `utils.ai_processor` (legacy) |
+| `validators.py` | All validation logic | `constants`, models |
+| `constants.py` | AI constants | None |
+| `engine.py` | Execution orchestrator | `ai_core`, `base`, `tracker` |
+| `base.py` | Base function class | None |
+| `tracker.py` | Progress/step tracking | None |
+| `registry.py` | Function registry | `base`, function modules |
+
+### Function Files
+
+| File | Function Class | Legacy Function | Status |
+|------|----------------|-----------------|--------|
+| `auto_cluster.py` | `AutoClusterFunction` | N/A (uses engine) | ✅ Updated |
+| `generate_ideas.py` | `GenerateIdeasFunction` | `generate_ideas_core()` | ✅ Created |
+| `generate_content.py` | `GenerateContentFunction` | `generate_content_core()` | ✅ Created |
+| `generate_images.py` | `GenerateImagesFunction` | `generate_images_core()` | ✅ Created |
+
+## 🔄 Import Path Changes
+
+### Old Imports (Still work, but deprecated)
+```python
+from igny8_core.utils.ai_processor import AIProcessor
+from igny8_core.modules.planner.tasks import _generate_single_idea_core
+```
+
+### New Imports (Recommended)
+```python
+from igny8_core.ai.functions.generate_ideas import generate_ideas_core
+from igny8_core.ai.functions.generate_content import generate_content_core
+from igny8_core.ai.functions.generate_images import generate_images_core
+from igny8_core.ai.ai_core import AICore
+from igny8_core.ai.validators import validate_ids, validate_cluster_limits
+from igny8_core.ai.constants import MODEL_RATES, DEFAULT_AI_MODEL
+```
+
+## ✅ Verification Checklist
+
+- [x] All function files created in `ai/functions/`
+- [x] Shared modules (`ai_core`, `validators`, `constants`) created
+- [x] No circular dependencies
+- [x] All imports updated in views and tasks
+- [x] Functions registered in registry
+- [x] `__init__.py` files updated
+- [x] Backward compatibility maintained (legacy functions still work)
+- [x] No linting errors
+- [x] Structure matches required layout
+
+## 🎯 Next Steps (Future Stages)
+
+- **Stage 2**: Inject tracker into all functions
+- **Stage 3**: Simplify logging
+- **Stage 4**: Clean up legacy code
+
+## 📝 Notes
+
+- Legacy `AIProcessor` from `utils.ai_processor` is still used by `ai_core.py` as a wrapper
+- This will be refactored in later stages
+- All existing API endpoints continue to work
+- No functional changes - only structural reorganization
+

docs/ActiveDocs/STAGE2-EXECUTION-LOGGING-COMPLETE.md

@@ -0,0 +1,220 @@
+# Stage 2 - AI Execution & Logging Layer - COMPLETE ✅
+
+## Summary
+
+Successfully created a centralized, consistent, and traceable execution layer for all AI requests with unified request handler and clean console-based logging.
+
+## ✅ Completed Deliverables
+
+### 1. Centralized Execution in `ai_core.py`
+
+#### `run_ai_request()` Method
+- **Purpose**: Single entry point for all AI text generation requests
+- **Features**:
+  - Step-by-step console logging with `print()` statements
+  - Standardized request payload construction
+  - Error handling with detailed logging
+  - Token counting and cost calculation
+  - Rate limit detection and logging
+  - Timeout handling
+  - JSON mode auto-enablement for supported models
+
+#### Console Logging Format
+```
+[AI][function_name] Step 1: Preparing request...
+[AI][function_name] Step 2: Using model: gpt-4o
+[AI][function_name] Step 3: Auto-enabled JSON mode for gpt-4o
+[AI][function_name] Step 4: Prompt length: 1234 characters
+[AI][function_name] Step 5: Request payload prepared (model=gpt-4o, max_tokens=4000, temp=0.7)
+[AI][function_name] Step 6: Sending request to OpenAI API...
+[AI][function_name] Step 7: Received response in 2.34s (status=200)
+[AI][function_name] Step 8: Received 150 tokens (input: 50, output: 100)
+[AI][function_name] Step 9: Content length: 450 characters
+[AI][function_name] Step 10: Cost calculated: $0.000123
+[AI][function_name][Success] Request completed successfully
+```
+
+#### Error Logging Format
+```
+[AI][function_name][Error] OpenAI Rate Limit - waiting 60s
+[AI][function_name][Error] HTTP 429 error: Rate limit exceeded (Rate limit - retry after 60s)
+[AI][function_name][Error] Request timeout (60s exceeded)
+[AI][function_name][Error] Failed to parse JSON response: ...
+```
+
+### 2. Image Generation with Logging
+
+#### `generate_image()` Method
+- **Purpose**: Centralized image generation with console logging
+- **Features**:
+  - Supports OpenAI DALL-E and Runware
+  - Model and size validation
+  - Step-by-step console logging
+  - Error handling with detailed messages
+  - Cost calculation
+
+#### Console Logging Format
+```
+[AI][generate_images] Step 1: Preparing image generation request...
+[AI][generate_images] Provider: OpenAI
+[AI][generate_images] Step 2: Using model: dall-e-3, size: 1024x1024
+[AI][generate_images] Step 3: Sending request to OpenAI Images API...
+[AI][generate_images] Step 4: Received response in 5.67s (status=200)
+[AI][generate_images] Step 5: Image generated successfully
+[AI][generate_images] Step 6: Cost: $0.0400
+[AI][generate_images][Success] Image generation completed
+```
+
+### 3. Updated All Function Files
+
+#### `functions/auto_cluster.py`
+- ✅ Uses `AICore.extract_json()` for JSON parsing
+- ✅ Engine calls `run_ai_request()` (via engine.py)
+
+#### `functions/generate_ideas.py`
+- ✅ Updated `generate_ideas_core()` to use `run_ai_request()`
+- ✅ Console logging enabled with function name
+
+#### `functions/generate_content.py`
+- ✅ Updated `generate_content_core()` to use `run_ai_request()`
+- ✅ Console logging enabled with function name
+
+#### `functions/generate_images.py`
+- ✅ Updated to use `run_ai_request()` for prompt extraction
+- ✅ Updated to use `generate_image()` with logging
+- ✅ Console logging enabled
+
+### 4. Updated Engine
+
+#### `engine.py`
+- ✅ Updated to use `run_ai_request()` instead of `call_openai()`
+- ✅ Passes function name for logging context
+- ✅ Maintains backward compatibility
+
+### 5. Deprecated Old Code
+
+#### `processor.py`
+- ✅ Marked as DEPRECATED
+- ✅ Redirects all calls to `AICore`
+- ✅ Kept for backward compatibility only
+- ✅ All methods now use `AICore` internally
+
+### 6. Edge Case Handling
+
+#### Implemented in `run_ai_request()`:
+- ✅ **API Key Validation**: Logs error if not configured
+- ✅ **Prompt Length**: Logs character count
+- ✅ **Rate Limits**: Detects and logs retry-after time
+- ✅ **Timeouts**: Handles 60s timeout with clear error
+- ✅ **JSON Parsing Errors**: Logs decode errors with context
+- ✅ **Empty Responses**: Validates content exists
+- ✅ **Token Overflow**: Max tokens enforced
+- ✅ **Model Validation**: Auto-selects JSON mode for supported models
+
+### 7. Standardized Request Schema
+
+#### OpenAI Request Payload
+```python
+{
+    "model": "gpt-4o",
+    "messages": [{"role": "user", "content": prompt}],
+    "temperature": 0.7,
+    "max_tokens": 4000,
+    "response_format": {"type": "json_object"}  # Auto-enabled for supported models
+}
+```
+
+#### All Functions Use Same Logic:
+- Model selection (account default or override)
+- JSON mode auto-enablement
+- Token limits
+- Temperature settings
+- Error handling
+
+### 8. Test Script Created
+
+#### `ai/tests/test_run.py`
+- ✅ Test script for all AI functions
+- ✅ Tests `run_ai_request()` directly
+- ✅ Tests JSON extraction
+- ✅ Placeholder tests for all functions
+- ✅ Can be run standalone to verify logging
+
+## 📋 File Changes Summary
+
+| File | Changes | Status |
+|------|---------|--------|
+| `ai_core.py` | Complete rewrite with `run_ai_request()` and console logging | ✅ Complete |
+| `engine.py` | Updated to use `run_ai_request()` | ✅ Complete |
+| `processor.py` | Marked deprecated, redirects to AICore | ✅ Complete |
+| `functions/auto_cluster.py` | Uses AICore methods | ✅ Complete |
+| `functions/generate_ideas.py` | Uses `run_ai_request()` | ✅ Complete |
+| `functions/generate_content.py` | Uses `run_ai_request()` | ✅ Complete |
+| `functions/generate_images.py` | Uses `run_ai_request()` and `generate_image()` | ✅ Complete |
+| `tests/test_run.py` | Test script created | ✅ Complete |
+
+## 🔄 Migration Path
+
+### Old Code (Deprecated)
+```python
+from igny8_core.utils.ai_processor import AIProcessor
+processor = AIProcessor(account=account)
+result = processor._call_openai(prompt, model=model)
+```
+
+### New Code (Recommended)
+```python
+from igny8_core.ai.ai_core import AICore
+ai_core = AICore(account=account)
+result = ai_core.run_ai_request(
+    prompt=prompt,
+    model=model,
+    function_name='my_function'
+)
+```
+
+## ✅ Verification Checklist
+
+- [x] `run_ai_request()` created with console logging
+- [x] All function files updated to use `run_ai_request()`
+- [x] Engine updated to use `run_ai_request()`
+- [x] Old processor code deprecated
+- [x] Edge cases handled with logging
+- [x] Request schema standardized
+- [x] Test script created
+- [x] No linting errors
+- [x] Backward compatibility maintained
+
+## 🎯 Benefits Achieved
+
+1. **Centralized Execution**: All AI requests go through one method
+2. **Consistent Logging**: Every request logs steps to console
+3. **Better Debugging**: Clear step-by-step visibility
+4. **Error Handling**: Comprehensive error detection and logging
+5. **Reduced Duplication**: No scattered AI call logic
+6. **Easy Testing**: Single point to test/mock
+7. **Future Ready**: Easy to add retry logic, backoff, etc.
+
+## 📝 Console Output Example
+
+When running any AI function, you'll see:
+```
+[AI][generate_ideas] Step 1: Preparing request...
+[AI][generate_ideas] Step 2: Using model: gpt-4o
+[AI][generate_ideas] Step 3: Auto-enabled JSON mode for gpt-4o
+[AI][generate_ideas] Step 4: Prompt length: 2345 characters
+[AI][generate_ideas] Step 5: Request payload prepared (model=gpt-4o, max_tokens=4000, temp=0.7)
+[AI][generate_ideas] Step 6: Sending request to OpenAI API...
+[AI][generate_ideas] Step 7: Received response in 3.45s (status=200)
+[AI][generate_ideas] Step 8: Received 250 tokens (input: 100, output: 150)
+[AI][generate_ideas] Step 9: Content length: 600 characters
+[AI][generate_ideas] Step 10: Cost calculated: $0.000250
+[AI][generate_ideas][Success] Request completed successfully
+```
+
+## 🚀 Next Steps (Future Stages)
+
+- **Stage 3**: Simplify logging (optional - console logging already implemented)
+- **Stage 4**: Clean up legacy code (remove old processor completely)
+- **Future**: Add retry logic, exponential backoff, request queuing
+