"""
AI Processor - Unified AI interface for content generation, images, clustering
Based on reference plugin's OpenAI integration (ai/openai-api.php)
Matches exact endpoints and request formats from reference plugin.
"""
import logging
import json
import re
import requests
import time
from typing import Dict, Any, Optional, List
from django.conf import settings
from django.core.cache import cache

logger = logging.getLogger(__name__)


# Import constants from unified location
from igny8_core.ai.constants import (
    MODEL_RATES,
    IMAGE_MODEL_RATES,
    VALID_OPENAI_IMAGE_MODELS,
    VALID_SIZES_BY_MODEL,
    DEFAULT_AI_MODEL,
    JSON_MODE_MODELS,
)


class AIProcessor:
    """
    Unified AI interface that routes requests to appropriate AI providers.
    Supports OpenAI (GPT models) and DALL-E/Runware for images.
    Matches reference plugin's openai-api.php implementation exactly.
    """
    
    def __init__(self, account=None):
        """
        Initialize AIProcessor. Can optionally load API keys and model from IntegrationSettings for account.
        
        Args:
            account: Optional account object to load API keys and model from IntegrationSettings
        """
        self.account = account
        
        # Get API keys from IntegrationSettings if account provided, else fallback to Django settings
        self.openai_api_key = self._get_api_key('openai', self.account)
        self.runware_api_key = self._get_api_key('runware', self.account)
        # Get model from IntegrationSettings if account provided, else fallback to Django settings
        self.default_model = self._get_model('openai', self.account)
        
        # Use global model rates
        self.model_rates = MODEL_RATES
        self.image_model_rates = IMAGE_MODEL_RATES
    
    def _get_api_key(self, integration_type: str, account=None) -> Optional[str]:
        """Get API key from IntegrationSettings or Django settings"""
        if account:
            try:
                from igny8_core.modules.system.models import IntegrationSettings
                settings_obj = IntegrationSettings.objects.filter(
                    integration_type=integration_type,
                    account=account,
                    is_active=True
                ).first()
                if settings_obj and settings_obj.config:
                    api_key = settings_obj.config.get('apiKey')
                    if api_key:
                        logger.info(f"Loaded {integration_type} API key from IntegrationSettings for account {account.id}")
                        return api_key
            except Exception as e:
                logger.warning(f"Could not load {integration_type} API key from IntegrationSettings: {e}", exc_info=True)
        
        # Fallback to Django settings
        if integration_type == 'openai':
            return getattr(settings, 'OPENAI_API_KEY', None)
        elif integration_type == 'runware':
            return getattr(settings, 'RUNWARE_API_KEY', None)
        return None
    
    def _get_model(self, integration_type: str, account=None) -> str:
        """Get model from IntegrationSettings or Django settings"""
        if account and integration_type == 'openai':
            try:
                from igny8_core.modules.system.models import IntegrationSettings
                settings_obj = IntegrationSettings.objects.filter(
                    integration_type=integration_type,
                    account=account,
                    is_active=True
                ).first()
                if settings_obj and settings_obj.config:
                    model = settings_obj.config.get('model')
                    if model:
                        # Validate model is in our supported list
                        if model in MODEL_RATES:
                            logger.info(f"Using model '{model}' from IntegrationSettings for account {account.id}")
                            return model
                        else:
                            logger.warning(f"Model '{model}' from IntegrationSettings not in supported models, using default")
            except Exception as e:
                logger.warning(f"Could not load {integration_type} model from IntegrationSettings: {e}", exc_info=True)
        
        # Fallback to Django settings or default
        default_model = getattr(settings, 'DEFAULT_AI_MODEL', 'gpt-4.1')
        logger.info(f"Using default model '{default_model}' (no IntegrationSettings found)")
        return default_model
    
    def _call_openai(
        self,
        prompt: str,
        model: Optional[str] = None,
        max_tokens: int = 4000,
        temperature: float = 0.7,
        response_format: Optional[Dict] = None,
        api_key: Optional[str] = None,
        function_id: Optional[str] = None,
        response_steps=None
    ) -> Dict[str, Any]:
        """
        Internal method to call OpenAI API.
        EXACT match to reference plugin's igny8_call_openai() function.
        Endpoint: https://api.openai.com/v1/chat/completions
        
        Returns:
            Dict with 'content', 'input_tokens', 'output_tokens', 'total_tokens', 'model', 'cost', 'error', 'api_id'
        """
        api_key = api_key or self.openai_api_key
        if not api_key:
            return {
                'content': None,
                'error': 'OpenAI API key not configured',
                'input_tokens': 0,
                'output_tokens': 0,
                'total_tokens': 0,
                'model': model or self.default_model,
                'cost': 0.0,
                'api_id': None,
            }
        
        model = model or self.default_model
        # Log which model is being used (helpful for debugging)
        if self.account:
            logger.info(f"Using model '{model}' for account {self.account.id} (OpenAI API request)")
        else:
            logger.info(f"Using model '{model}' (OpenAI API request, no account context)")
        
        url = 'https://api.openai.com/v1/chat/completions'
        
        headers = {
            'Authorization': f'Bearer {api_key}',
            'Content-Type': 'application/json',
        }
        
        # Add function_id to prompt if provided (for tracking)
        final_prompt = prompt
        if function_id:
            function_id_prefix = f'function_id: "{function_id}"\n\n'
            final_prompt = function_id_prefix + prompt
            logger.info(f"Added function_id to prompt: {function_id}")
        
        # EXACT request format from reference plugin (openai-api.php line 402-404)
        body_data = {
            'model': model,
            'messages': [{'role': 'user', 'content': final_prompt}],
            'temperature': temperature,
        }
        
        # Add max_tokens if specified (reference plugin uses 4000 default)
        if max_tokens:
            body_data['max_tokens'] = max_tokens
        
        # Add response_format if specified (for JSON mode)
        if response_format:
            body_data['response_format'] = response_format
        
        try:
            logger.info(f"Calling OpenAI API: model={model}, prompt_length={len(prompt)}")
            response = requests.post(url, headers=headers, json=body_data, timeout=60)
            
            # Step 8: HTTP Response Validation
            step_start = time.time()
            if response.status_code != 200:
                error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {}
                error_message = f"HTTP {response.status_code} error"
                if isinstance(error_data, dict) and 'error' in error_data:
                    if isinstance(error_data['error'], dict) and 'message' in error_data['error']:
                        error_message += f": {error_data['error']['message']}"
                
                logger.error(f"OpenAI API HTTP error {response.status_code}: {error_message}")
                if response_steps is not None:
                    response_steps.append({
                        'stepNumber': 8,
                        'stepName': 'HTTP Response Validation',
                        'functionName': '_call_openai',
                        'status': 'error',
                        'message': f'HTTP {response.status_code} error',
                        'error': error_message,
                        'duration': int((time.time() - step_start) * 1000)
                    })
                return {
                    'content': None,
                    'error': error_message,
                    'input_tokens': 0,
                    'output_tokens': 0,
                    'total_tokens': 0,
                    'model': model,
                    'cost': 0.0,
                    'api_id': None,
                }
            
            if response_steps is not None:
                response_steps.append({
                    'stepNumber': 8,
                    'stepName': 'HTTP Response Validation',
                    'functionName': '_call_openai',
                    'status': 'success',
                    'message': f'HTTP {response.status_code} OK',
                    'duration': int((time.time() - step_start) * 1000)
                })
            
            # Step 9: Content Extraction
            step_start = time.time()
            try:
                data = response.json()
            except Exception as e:
                if response_steps is not None:
                    response_steps.append({
                        'stepNumber': 9,
                        'stepName': 'Content Extraction',
                        'functionName': '_call_openai',
                        'status': 'error',
                        'message': 'Failed to parse JSON response',
                        'error': str(e),
                        'duration': int((time.time() - step_start) * 1000)
                    })
                return {
                    'content': None,
                    'error': f'Failed to parse JSON response: {str(e)}',
                    'input_tokens': 0,
                    'output_tokens': 0,
                    'total_tokens': 0,
                    'model': model,
                    'cost': 0.0,
                    'api_id': None,
                }
            
            api_id = data.get('id')
            
            if 'choices' in data and len(data['choices']) > 0:
                content = data['choices'][0]['message']['content']
                usage = data.get('usage', {})
                input_tokens = usage.get('prompt_tokens', 0)
                output_tokens = usage.get('completion_tokens', 0)
                total_tokens = usage.get('total_tokens', 0)
                
                if response_steps is not None:
                    response_steps.append({
                        'stepNumber': 9,
                        'stepName': 'Content Extraction',
                        'functionName': '_call_openai',
                        'status': 'success',
                        'message': f'Extracted content: {len(content)} chars, {total_tokens} tokens',
                        'duration': int((time.time() - step_start) * 1000)
                    })
                
                # Step 10: Cost Calculation
                step_start = time.time()
                rates = self.model_rates.get(model, {'input': 2.00, 'output': 8.00})
                cost = (input_tokens * rates['input'] + output_tokens * rates['output']) / 1000000
                
                if response_steps is not None:
                    response_steps.append({
                        'stepNumber': 10,
                        'stepName': 'Cost Calculation',
                        'functionName': '_call_openai',
                        'status': 'success',
                        'message': f'Cost: ${cost:.6f}',
                        'duration': int((time.time() - step_start) * 1000)
                    })
                
                return {
                    'content': content,
                    'input_tokens': input_tokens,
                    'output_tokens': output_tokens,
                    'total_tokens': total_tokens,
                    'model': model,
                    'cost': cost,
                    'error': None,
                    'api_id': api_id,
                }
            else:
                if response_steps is not None:
                    response_steps.append({
                        'stepNumber': 9,
                        'stepName': 'Content Extraction',
                        'functionName': '_call_openai',
                        'status': 'error',
                        'message': 'No content in OpenAI response',
                        'error': 'No content in OpenAI response',
                        'duration': int((time.time() - step_start) * 1000)
                    })
                return {
                    'content': None,
                    'error': 'No content in OpenAI response',
                    'input_tokens': 0,
                    'output_tokens': 0,
                    'total_tokens': 0,
                    'model': model,
                    'cost': 0.0,
                    'api_id': api_id,
                }
        except requests.exceptions.RequestException as e:
            logger.error(f"OpenAI API error: {str(e)}")
            return {
                'content': None,
                'error': f'OpenAI API error: {str(e)}',
                'input_tokens': 0,
                'output_tokens': 0,
                'total_tokens': 0,
                'model': model,
                'cost': 0.0,
                'api_id': None,
            }
    
    def _extract_json_from_response(self, response_text: str) -> Optional[Dict]:
        """
        Extract JSON from OpenAI response, handling various formats.
        Supports:
        1. Plain JSON: {"clusters": [...]}
        2. Markdown code blocks: ```json {"clusters": [...]} ```
        3. Markdown with language: ```json\n{...}\n```
        4. Text with JSON: text before {json} text after
        """
        if not response_text or not response_text.strip():
            return None
        
        # Clean up the response text
        response_text = response_text.strip()
        
        # Try direct JSON parse first (most common case)
        try:
            return json.loads(response_text)
        except json.JSONDecodeError:
            pass
        
        # Try to find JSON in markdown code blocks (```json ... ```)
        # Handle both ```json and ``` with newlines - use non-greedy match with DOTALL
        json_block_patterns = [
            r'```json\s*(\{.*?\})\s*```',  # ```json {...} ```
            r'```json\s*(\[.*?\])\s*```',  # ```json [...] ```
            r'```\s*(\{.*?\})\s*```',      # ``` {...} ```
            r'```\s*(\[.*?\])\s*```',      # ``` [...] ```
        ]
        
        for pattern in json_block_patterns:
            json_match = re.search(pattern, response_text, re.DOTALL | re.MULTILINE)
            if json_match:
                try:
                    json_str = json_match.group(1).strip()
                    return json.loads(json_str)
                except json.JSONDecodeError:
                    continue
        
        # Try to find code blocks and extract JSON from them (handles multiline JSON)
        # Pattern: ```json\n{...}\n``` or ```\n{...}\n```
        code_block_match = re.search(r'```(?:json)?\s*\n?([\s\S]*?)\n?```', response_text, re.MULTILINE)
        text_to_search = response_text
        if code_block_match:
            code_content = code_block_match.group(1).strip()
            # Try to parse as JSON directly
            try:
                return json.loads(code_content)
            except json.JSONDecodeError:
                # If direct parse fails, use balanced brace matching on code block content
                # This handles cases where there's extra text before/after JSON in the code block
                text_to_search = code_content
        
        # Try to find JSON object between braces (handles cases where JSON is embedded in text)
        # Look for the outermost { } pair using balanced brace matching
        brace_depth = 0
        start_idx = -1
        for i, char in enumerate(text_to_search):
            if char == '{':
                if brace_depth == 0:
                    start_idx = i
                brace_depth += 1
            elif char == '}':
                brace_depth -= 1
                if brace_depth == 0 and start_idx >= 0:
                    # Found complete JSON object
                    try:
                        json_str = text_to_search[start_idx:i+1]
                        return json.loads(json_str)
                    except json.JSONDecodeError:
                        # Reset and continue searching
                        start_idx = -1
                        brace_depth = 0
                        pass
        
        # Try to find JSON array between brackets
        bracket_depth = 0
        start_idx = -1
        for i, char in enumerate(text_to_search):
            if char == '[':
                if bracket_depth == 0:
                    start_idx = i
                bracket_depth += 1
            elif char == ']':
                bracket_depth -= 1
                if bracket_depth == 0 and start_idx >= 0:
                    # Found complete JSON array
                    try:
                        json_str = response_text[start_idx:i+1]
                        # Wrap in object if it's just an array
                        parsed = json.loads(json_str)
                        if isinstance(parsed, list):
                            # If we get a list, try to wrap it or return as-is
                            return parsed
                    except json.JSONDecodeError:
                        pass
        
        # Last resort: try to extract anything that looks like JSON
        # Look for { followed by } with reasonable content
        json_match = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', text_to_search, re.DOTALL)
        if json_match:
            try:
                return json.loads(json_match.group(0))
            except json.JSONDecodeError:
                pass
        
        logger.warning(f"Could not extract JSON from response: {response_text[:200]}...")
        return None
    
    def generate_content(
        self,
        prompt: str,
        model: Optional[str] = None,
        max_tokens: int = 4000,
        temperature: float = 0.7,
        **kwargs
    ) -> Dict[str, Any]:
        """
        Generate text content using OpenAI GPT models.
        
        Args:
            prompt: The prompt text
            model: Model to use (defaults to self.default_model)
            max_tokens: Maximum tokens to generate
            temperature: Sampling temperature (0-1)
        
        Returns:
            Dict with 'content', 'tokens_used', 'model', 'cost', 'error'
        """
        # Generate function_id for tracking (ai-generate-content-03 for AIProcessor path)
        function_id = "ai-generate-content-03"
        # Get response_format from settings for generate_content
        from igny8_core.ai.settings import get_model_config
        model_config = get_model_config('generate_content')
        response_format = model_config.get('response_format')
        result = self._call_openai(prompt, model, max_tokens, temperature, response_format=response_format, function_id=function_id)
        
        return {
            'content': result.get('content', ''),
            'tokens_used': result.get('total_tokens', 0),
            'input_tokens': result.get('input_tokens', 0),
            'output_tokens': result.get('output_tokens', 0),
            'model': result.get('model', model or self.default_model),
            'cost': result.get('cost', 0.0),
            'error': result.get('error'),
        }
    
    def extract_image_prompts(
        self,
        content: str,
        title: str,
        max_images: int = 3,
        account: Optional[Any] = None
    ) -> Dict[str, Any]:
        """
        Extract image prompts from content using AI.
        
        Args:
            content: The article content to extract prompts from
            title: The article title
            max_images: Maximum number of image prompts to extract (default 3)
            account: Optional account for getting custom prompts
        
        Returns:
            Dict with 'featured_prompt' and 'in_article_prompts' list, or 'error'
        """
        try:
            from igny8_core.modules.system.utils import get_prompt_value
            
            account_obj = account
            
            # Get prompt template for image extraction
            prompt_template = get_prompt_value(account_obj, 'image_prompt_extraction') if account_obj else None
            
            if not prompt_template:
                # Default prompt for image extraction
                prompt_template = """Extract image prompts from the following article content.

ARTICLE TITLE: {title}

ARTICLE CONTENT:
{content}

Extract image prompts for:
1. Featured Image: One main image that represents the article topic
2. In-Article Images: Up to {max_images} images that would be useful within the article content

Return a JSON object with this structure:
{{
    "featured_prompt": "Detailed description of the featured image",
    "in_article_prompts": [
        "Description of first in-article image",
        "Description of second in-article image",
        ...
    ]
}}

Make sure each prompt is detailed enough for image generation, describing the visual elements, style, mood, and composition."""
            
            # Format the prompt
            prompt = prompt_template.format(
                title=title,
                content=content[:5000],  # Limit content length
                max_images=max_images
            )
            
            # Call OpenAI to extract prompts
            result = self.generate_content(prompt, max_tokens=1000, temperature=0.7)
            
            if result.get('error'):
                return {'error': result['error']}
            
            response_text = result.get('content', '')
            
            # Try to extract JSON from response
            json_data = self._extract_json_from_response(response_text)
            
            if json_data:
                featured_prompt = json_data.get('featured_prompt', '')
                in_article_prompts = json_data.get('in_article_prompts', [])
                
                # Ensure we don't exceed max_images
                if len(in_article_prompts) > max_images:
                    in_article_prompts = in_article_prompts[:max_images]
                
                return {
                    'featured_prompt': featured_prompt,
                    'in_article_prompts': in_article_prompts,
                    'error': None
                }
            else:
                # Fallback: try to parse manually
                # Look for featured_prompt and in_article_prompts in the text
                featured_match = re.search(r'featured[_\s]*prompt["\']?\s*:\s*["\']([^"\']+)["\']', response_text, re.IGNORECASE)
                featured_prompt = featured_match.group(1) if featured_match else ''
                
                # Extract in-article prompts
                in_article_match = re.search(r'in[_\s]*article[_\s]*prompts["\']?\s*:\s*\[([^\]]+)\]', response_text, re.IGNORECASE)
                in_article_prompts = []
                if in_article_match:
                    prompts_text = in_article_match.group(1)
                    # Extract quoted strings
                    prompts = re.findall(r'["\']([^"\']+)["\']', prompts_text)
                    in_article_prompts = prompts[:max_images]
                
                if featured_prompt or in_article_prompts:
                    return {
                        'featured_prompt': featured_prompt,
                        'in_article_prompts': in_article_prompts,
                        'error': None
                    }
                else:
                    # Last resort: create simple prompts from title
                    featured_prompt = f"A professional, high-quality image representing: {title}"
                    return {
                        'featured_prompt': featured_prompt,
                        'in_article_prompts': [],
                        'error': None
                    }
                    
        except Exception as e:
            logger.error(f"Error extracting image prompts: {str(e)}", exc_info=True)
            return {
                'error': f'Failed to extract image prompts: {str(e)}'
            }
    
    def check_moderation(self, text: str, api_key: Optional[str] = None) -> Dict[str, Any]:
        """
        Check content for moderation violations using OpenAI's moderation API.
        EXACT match to reference plugin's igny8_check_moderation() function.
        Endpoint: https://api.openai.com/v1/moderations
        
        Args:
            text: Text to check for moderation violations
            api_key: Optional API key (uses instance key if not provided)
        
        Returns:
            Dict with 'flagged' (bool), 'categories' (dict), 'error' (optional)
        """
        api_key = api_key or self.openai_api_key
        if not api_key:
            return {
                'flagged': False,
                'categories': {},
                'error': 'OpenAI API key not configured',
            }
        
        url = 'https://api.openai.com/v1/moderations'
        headers = {
            'Authorization': f'Bearer {api_key}',
            'Content-Type': 'application/json',
        }
        
        try:
            response = requests.post(
                url,
                headers=headers,
                json={'input': text},
                timeout=20
            )
            
            if response.status_code != 200:
                return {
                    'flagged': False,
                    'categories': {},
                    'error': f'HTTP {response.status_code}: {response.text[:200]}',
                }
            
            body = response.json()
            # Reference plugin: $body['results'][0]['flagged'] ?? false
            result = body.get('results', [{}])[0]
            
            return {
                'flagged': result.get('flagged', False),
                'categories': result.get('categories', {}),
                'error': None,
            }
        except requests.exceptions.RequestException as e:
            logger.error(f"Moderation API error: {str(e)}")
            return {
                'flagged': False,
                'categories': {},
                'error': str(e),
            }
    
    def generate_image(
        self,
        prompt: str,
        provider: str = 'openai',  # 'openai' or 'runware'
        model: Optional[str] = None,
        size: str = '1024x1024',
        n: int = 1,
        api_key: Optional[str] = None,
        **kwargs
    ) -> Dict[str, Any]:
        """
        Generate image using DALL-E (OpenAI) or Runware.
        EXACT match to reference plugin's image generation endpoints.
        
        For OpenAI: https://api.openai.com/v1/images/generations
        Request format (reference plugin line 196-201):
        {
            'model': image_model,
            'prompt': prompt,
            'n': 1,
            'size': '1024x1024'  // OpenAI only supports square
        }
        
        Args:
            prompt: Image description prompt
            provider: 'openai' or 'runware'
            model: Model name (e.g., 'dall-e-3', 'dall-e-2')
            size: Image size (e.g., '1024x1024') - OpenAI only supports square
            n: Number of images to generate (default 1)
            api_key: Optional API key (uses instance key if not provided)
        
        Returns:
            Dict with 'url', 'revised_prompt' (OpenAI), 'provider', 'cost', 'error'
        """
        if provider == 'openai':
            logger.info("[AIProcessor.generate_image] Provider: OpenAI")
            api_key = api_key or self.openai_api_key
            if not api_key:
                logger.error("[AIProcessor.generate_image] ERROR: OpenAI API key not configured")
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'openai',
                    'cost': 0.0,
                    'error': 'OpenAI API key not configured',
                }
            
            model = model or 'dall-e-3'
            
            # Validate model - OpenAI /v1/images/generations only supports dall-e-3 and dall-e-2
            # gpt-image-1 and gpt-image-1-mini are NOT valid for OpenAI's image generation API
            if model not in VALID_OPENAI_IMAGE_MODELS:
                error_msg = (
                    f"Model '{model}' is not valid for OpenAI's /v1/images/generations endpoint. "
                    f"Only 'dall-e-3' and 'dall-e-2' are supported. "
                    f"Models like 'gpt-image-1' and 'gpt-image-1-mini' are not available through OpenAI's image generation API."
                )
                logger.error(f"[AIProcessor.generate_image] ERROR: {error_msg}")
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'openai',
                    'cost': 0.0,
                    'error': error_msg,
                }
            
            # Validate image size for the selected model
            valid_sizes = VALID_SIZES_BY_MODEL.get(model, [])
            if size not in valid_sizes:
                error_msg = (
                    f"Image size '{size}' is not valid for model '{model}'. "
                    f"Valid sizes for {model} are: {', '.join(valid_sizes)}"
                )
                logger.error(f"[AIProcessor.generate_image] ERROR: {error_msg}")
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'openai',
                    'cost': 0.0,
                    'error': error_msg,
                }
            
            url = 'https://api.openai.com/v1/images/generations'
            logger.info(f"[AIProcessor.generate_image] OpenAI API URL: {url}, model: {model}")
            
            headers = {
                'Content-Type': 'application/json',
                'Authorization': f'Bearer {api_key[:10]}...',  # Log partial key for security
            }
            logger.info(f"[AIProcessor.generate_image] Request headers: Content-Type={headers.get('Content-Type')}, Authorization={headers.get('Authorization')}")
            
            # EXACT request format from reference plugin (image-generation.php line 461-466)
            data = {
                'model': model,
                'prompt': prompt,
                'n': n,
                'size': size  # OpenAI only supports square formats like '1024x1024'
            }
            logger.info(f"[AIProcessor.generate_image] Request data: model={data['model']}, n={data['n']}, size={data['size']}, prompt_length={len(data['prompt'])}")
            
            try:
                logger.info(f"[AIProcessor.generate_image] Making POST request to OpenAI API: {url}")
                # Use full API key in actual request (not the logged one)
                headers['Authorization'] = f'Bearer {api_key}'
                response = requests.post(url, headers=headers, json=data, timeout=60)
                logger.info(f"[AIProcessor.generate_image] OpenAI API response received: status={response.status_code}")
                
                logger.info(f"[AIProcessor.generate_image] Response status: {response.status_code}, headers: {dict(response.headers)}")
                
                if response.status_code != 200:
                    logger.error(f"[AIProcessor.generate_image] ERROR: Non-200 status code: {response.status_code}")
                    error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {}
                    logger.error(f"[AIProcessor.generate_image] Error response data: {error_data}")
                    error_message = f"HTTP {response.status_code} error"
                    if isinstance(error_data, dict) and 'error' in error_data:
                        if isinstance(error_data['error'], dict) and 'message' in error_data['error']:
                            error_message += f": {error_data['error']['message']}"
                    
                    return {
                        'url': None,
                        'revised_prompt': None,
                        'provider': 'openai',
                        'cost': 0.0,
                        'error': error_message,
                    }
                
                body = response.json()
                logger.info(f"[AIProcessor.generate_image] Response body keys: {list(body.keys()) if isinstance(body, dict) else 'Not a dict'}")
                
                # Reference plugin: $body['data'][0]['url'] and $body['data'][0]['revised_prompt']
                if 'data' in body and len(body['data']) > 0:
                    image_data = body['data'][0]
                    image_url = image_data.get('url')
                    revised_prompt = image_data.get('revised_prompt')
                    logger.info(f"[AIProcessor.generate_image] Image data extracted: has_url={bool(image_url)}, has_revised_prompt={bool(revised_prompt)}")
                    
                    # Calculate cost (reference plugin: igny8_calculate_image_cost)
                    cost = self.image_model_rates.get(model, 0.040) * n
                    logger.info(f"[AIProcessor.generate_image] Cost calculated: ${cost:.4f}")
                    
                    result = {
                        'url': image_url,
                        'revised_prompt': revised_prompt,
                        'provider': 'openai',
                        'cost': cost,
                        'error': None,
                    }
                    logger.info("[AIProcessor.generate_image] Returning success result")
                    return result
                else:
                    logger.error(f"[AIProcessor.generate_image] ERROR: No image data in response. Body structure: {body}")
                    return {
                        'url': None,
                        'revised_prompt': None,
                        'provider': 'openai',
                        'cost': 0.0,
                        'error': 'No image data in response',
                    }
            except requests.exceptions.RequestException as e:
                logger.error(f"[AIProcessor.generate_image] EXCEPTION: OpenAI API request error: {str(e)}", exc_info=True)
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'openai',
                    'cost': 0.0,
                    'error': f'OpenAI API error: {str(e)}',
                }
        
        elif provider == 'runware':
            # Runware implementation (from reference plugin)
            # Reference: ai/writer/images/image-generation.php lines 77-193
            # Endpoint: https://api.runware.ai/v1
            logger.info("[AIProcessor.generate_image] Provider: Runware")
            api_key = api_key or self.runware_api_key
            if not api_key:
                logger.error("[AIProcessor.generate_image] ERROR: Runware API key not configured")
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'runware',
                    'cost': 0.0,
                    'error': 'Runware API key not configured',
                }
            
            # Get model from kwargs or use default
            runware_model = kwargs.get('model', 'runware:97@1')
            negative_prompt = kwargs.get('negative_prompt', '')
            
            # Parse size (format: "1024x1024" or "64x64")
            width, height = map(int, size.split('x'))
            
            url = 'https://api.runware.ai/v1'
            logger.info(f"[AIProcessor.generate_image] Runware API URL: {url}, model: {runware_model}, size: {width}x{height}")
            
            # Runware uses array payload with authentication and imageInference tasks
            # Reference: image-generation.php lines 79-97
            import uuid
            logger.info(f"[AIProcessor.generate_image] Runware API key check: has_key={bool(api_key)}, key_length={len(api_key) if api_key else 0}, key_preview={api_key[:10] + '...' + api_key[-4:] if api_key and len(api_key) > 14 else 'N/A'}")
            payload = [
                {
                    'taskType': 'authentication',
                    'apiKey': api_key
                },
                {
                    'taskType': 'imageInference',
                    'taskUUID': str(uuid.uuid4()),
                    'positivePrompt': prompt,
                    'negativePrompt': negative_prompt,
                    'model': runware_model,
                    'width': width,
                    'height': height,
                    'steps': 30,
                    'CFGScale': 7.5,
                    'numberResults': 1,
                    'outputFormat': kwargs.get('format', 'webp')
                }
            ]
            
            logger.info(f"[AIProcessor.generate_image] Runware request payload: model={runware_model}, width={width}, height={height}, prompt_length={len(prompt)}")
            
            try:
                logger.info(f"[AIProcessor.generate_image] Making POST request to Runware API: {url}")
                response = requests.post(
                    url,
                    headers={'Content-Type': 'application/json'},
                    json=payload,
                    timeout=150  # Runware can take longer
                )
                logger.info(f"[AIProcessor.generate_image] Runware API response received: status={response.status_code}")
                
                if response.status_code != 200:
                    logger.error(f"[AIProcessor.generate_image] ERROR: Non-200 status code: {response.status_code}")
                    error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {}
                    logger.error(f"[AIProcessor.generate_image] Error response data: {error_data}")
                    error_message = f"HTTP {response.status_code} error"
                    if isinstance(error_data, list) and len(error_data) > 0:
                        error_obj = error_data[0]
                        if isinstance(error_obj, dict) and 'error' in error_obj:
                            error_message += f": {error_obj['error']}"
                        elif isinstance(error_obj, dict) and 'errors' in error_obj:
                            errors = error_obj['errors']
                            if isinstance(errors, list) and len(errors) > 0:
                                error_message += f": {errors[0].get('message', 'Unknown error')}"
                    
                    return {
                        'url': None,
                        'revised_prompt': None,
                        'provider': 'runware',
                        'cost': 0.0,
                        'error': error_message,
                    }
                
                body = response.json()
                logger.info(f"[AIProcessor.generate_image] Response body type: {type(body)}")
                logger.info(f"[AIProcessor.generate_image] Response body (full): {json.dumps(body, indent=2)[:1000]}")
                
                # Runware API response format analysis:
                # When sending array payload: [auth_task, image_task]
                # Response can be:
                # 1. Array with task results: [auth_result, image_result]
                #    - auth_result: {'success': True, ...}
                #    - image_result: {'data': [{'imageURL': '...'}], ...} or {'errors': [...]}
                # 2. Single dict at root: {'data': [{'imageURL': '...'}]}
                # Reference: image-generation.php line 151 shows: $response_data['data'][0]['imageURL']
                
                image_url = None
                error_msg = None
                
                if isinstance(body, list):
                    # Case 1: Array response - find the imageInference result
                    logger.info(f"[AIProcessor.generate_image] Response is array with {len(body)} elements")
                    for idx, item in enumerate(body):
                        logger.info(f"[AIProcessor.generate_image] Array element {idx}: {type(item)}, keys: {list(item.keys()) if isinstance(item, dict) else 'N/A'}")
                        if isinstance(item, dict):
                            # Check if this is the image result with 'data' key
                            if 'data' in item:
                                data = item['data']
                                logger.info(f"[AIProcessor.generate_image] Found 'data' key, type: {type(data)}")
                                if isinstance(data, list) and len(data) > 0:
                                    first_item = data[0]
                                    logger.info(f"[AIProcessor.generate_image] First data item keys: {list(first_item.keys()) if isinstance(first_item, dict) else 'N/A'}")
                                    image_url = first_item.get('imageURL') or first_item.get('image_url')
                                    if image_url:
                                        logger.info(f"[AIProcessor.generate_image] Found imageURL: {image_url[:50]}...")
                                        break
                            # Check for errors
                            if 'errors' in item:
                                errors = item['errors']
                                logger.info(f"[AIProcessor.generate_image] Found 'errors' key, type: {type(errors)}")
                                if isinstance(errors, list) and len(errors) > 0:
                                    error_obj = errors[0]
                                    error_msg = error_obj.get('message') or error_obj.get('error') or str(error_obj)
                                    logger.error(f"[AIProcessor.generate_image] Error in response: {error_msg}")
                                    break
                            # Check for error at root level
                            if 'error' in item:
                                error_msg = item['error']
                                logger.error(f"[AIProcessor.generate_image] Error at root level: {error_msg}")
                                break
                elif isinstance(body, dict):
                    # Case 2: Direct dict response
                    logger.info(f"[AIProcessor.generate_image] Response is dict with keys: {list(body.keys())}")
                    if 'data' in body:
                        data = body['data']
                        logger.info(f"[AIProcessor.generate_image] Found 'data' key, type: {type(data)}")
                        if isinstance(data, list) and len(data) > 0:
                            first_item = data[0]
                            logger.info(f"[AIProcessor.generate_image] First data item keys: {list(first_item.keys()) if isinstance(first_item, dict) else 'N/A'}")
                            image_url = first_item.get('imageURL') or first_item.get('image_url')
                    elif 'errors' in body:
                        errors = body['errors']
                        logger.info(f"[AIProcessor.generate_image] Found 'errors' key, type: {type(errors)}")
                        if isinstance(errors, list) and len(errors) > 0:
                            error_obj = errors[0]
                            error_msg = error_obj.get('message') or error_obj.get('error') or str(error_obj)
                            logger.error(f"[AIProcessor.generate_image] Error in response: {error_msg}")
                    elif 'error' in body:
                        error_msg = body['error']
                        logger.error(f"[AIProcessor.generate_image] Error at root level: {error_msg}")
                
                if error_msg:
                    return {
                        'url': None,
                        'revised_prompt': None,
                        'provider': 'runware',
                        'cost': 0.0,
                        'error': error_msg,
                    }
                
                if image_url:
                    logger.info(f"[AIProcessor.generate_image] Image URL extracted: {image_url[:50]}...")
                    # Calculate cost (Runware: $0.009 per image)
                    cost = 0.009
                    logger.info(f"[AIProcessor.generate_image] Cost calculated: ${cost:.4f}")
                    
                    return {
                        'url': image_url,
                        'revised_prompt': None,  # Runware doesn't provide revised prompts
                        'provider': 'runware',
                        'cost': cost,
                        'error': None,
                    }
                
                # If we get here, we couldn't parse the response
                logger.error(f"[AIProcessor.generate_image] ERROR: Unexpected response format. Body structure: {type(body)}")
                logger.error(f"[AIProcessor.generate_image] Full response: {json.dumps(body, indent=2)}")
                
                # Try to provide a helpful error message
                error_detail = f"Response type: {type(body).__name__}"
                if isinstance(body, (dict, list)):
                    error_detail += f", Structure: {str(body)[:200]}"
                
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'runware',
                    'cost': 0.0,
                    'error': f'Unexpected response format from Runware API. {error_detail}. Check logs for full response.',
                }
            except requests.exceptions.RequestException as e:
                logger.error(f"[AIProcessor.generate_image] EXCEPTION: Runware API request error: {str(e)}", exc_info=True)
                return {
                    'url': None,
                    'revised_prompt': None,
                    'provider': 'runware',
                    'cost': 0.0,
                    'error': f'Runware API error: {str(e)}',
                }
        
        else:
            return {
                'url': None,
                'revised_prompt': None,
                'provider': provider,
                'cost': 0.0,
                'error': f'Unknown provider: {provider}',
            }
    
    def get_prompt(self, prompt_type: str, account=None) -> str:
        """Get prompt value from database or default"""
        try:
            account_obj = account
            if account_obj:
                from igny8_core.modules.system.utils import get_prompt_value
                return get_prompt_value(account_obj, prompt_type)
            else:
                from igny8_core.modules.system.utils import get_default_prompt
                return get_default_prompt(prompt_type)
        except Exception as e:
            logger.warning(f"Could not load prompt from database, using default: {e}")
            from igny8_core.modules.system.utils import get_default_prompt
            return get_default_prompt(prompt_type)
    
    def cluster_keywords(
        self,
        keywords: List[Dict[str, Any]],  # List of keyword dicts with 'keyword', 'volume', 'difficulty', 'intent'
        sector_name: Optional[str] = None,
        account=None,
        response_steps=None,
        progress_callback=None,
        tracker=None,  # Optional ConsoleStepTracker for logging
        **kwargs
    ) -> Dict[str, Any]:
        """
        [DEPRECATED] Cluster keywords using AI-based semantic similarity.
        
        ⚠️  WARNING: This method is deprecated. Use the new AI framework instead:
        - New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
        - This method uses the old prompt system and does not use PromptRegistry
        - Console logging may not work correctly in this path
        
        Based on reference plugin's clustering prompt.
        
        Args:
            keywords: List of keyword dicts with keyword, volume, difficulty, intent
            sector_name: Optional sector name for context
            account: Optional account for getting custom prompts
        
        Returns:
            Dict with 'clusters' (list of cluster dicts with name, description, keywords)
        """
        logger.warning("AIProcessor.cluster_keywords is deprecated. Use the new AI framework (AutoClusterFunction) instead.")
        if not keywords:
            return {
                'clusters': [],
                'error': 'No keywords provided',
            }
        
        # Format keywords for prompt
        keywords_text = '\n'.join([
            f"- {kw.get('keyword', '')} (Volume: {kw.get('volume', 0)}, Difficulty: {kw.get('difficulty', 0)}, Intent: {kw.get('intent', 'unknown')})"
            for kw in keywords
        ])
        
        if tracker:
            tracker.prep(f"Formatted {len(keywords)} keywords for prompt")
        
        account_obj = account or self.account
        
        # Get prompt template from database or default
        # NOTE: This is legacy code. New code should use PromptRegistry.get_prompt()
        # Keeping this for backward compatibility with old tasks
        prompt_template = self.get_prompt('clustering', account=account_obj)
        
        # Replace placeholders in prompt template
        if '[IGNY8_KEYWORDS]' not in prompt_template:
            error_msg = "Prompt template missing [IGNY8_KEYWORDS] placeholder"
            logger.error(error_msg)
            if tracker:
                tracker.error('Prompt', error_msg)
            return {
                'clusters': [],
                'error': error_msg,
            }
        
        prompt = prompt_template.replace('[IGNY8_KEYWORDS]', keywords_text)
        
        if tracker:
            tracker.prep(f"Prompt prepared: {len(prompt)} characters (keywords: {len(keywords_text)} chars)")
        
        if sector_name:
            prompt += f"\n\nNote: These keywords are for the '{sector_name}' sector."
        
        logger.info(f"Clustering {len(keywords)} keywords using AI")
        logger.info(f"AIProcessor.cluster_keywords: About to call OpenAI API with {len(keywords)} keywords")
        
        if tracker:
            tracker.ai_call(f"Calling OpenAI API with model: {self.default_model}")
        
        # Initialize response_steps if not provided
        if response_steps is None:
            response_steps = []
        
        # Call OpenAI with JSON response format - Steps 8-10 happen in _call_openai
        # Use the active model from integration settings (self.default_model)
        # Check if model supports JSON mode
        json_models = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview']
        active_model = self.default_model  # This is the active model from IntegrationSettings
        response_format = {'type': 'json_object'} if active_model in json_models else None
        
        try:
            result = self._call_openai(
                prompt,
                model=active_model,  # Explicitly pass to ensure consistency
                max_tokens=3000,
                temperature=0.7,
                response_format=response_format,
                response_steps=response_steps
            )
            
            if tracker:
                if result.get('error'):
                    tracker.error('AI_CALL', f"OpenAI API error: {result['error']}")
                else:
                    tracker.ai_call(f"Received response: {result.get('total_tokens', 0)} tokens")
            logger.info(f"AIProcessor.cluster_keywords: OpenAI API call completed. Error: {result.get('error')}, Has content: {bool(result.get('content'))}")
        except Exception as e:
            logger.error(f"AIProcessor.cluster_keywords: Exception calling OpenAI API: {type(e).__name__}: {str(e)}", exc_info=True)
            if response_steps is not None:
                response_steps.append({
                    'stepNumber': 8,
                    'stepName': 'HTTP Response Validation',
                    'functionName': 'cluster_keywords',
                    'status': 'error',
                    'message': f'Exception: {type(e).__name__}',
                    'error': str(e)
                })
            if progress_callback:
                progress_callback(
                    state='PROGRESS',
                    meta={'response_steps': response_steps}
                )
            return {
                'clusters': [],
                'error': f'Exception calling OpenAI API: {str(e)}',
                'response_steps': response_steps
            }
        
        if result.get('error'):
            logger.error(f"AIProcessor.cluster_keywords: OpenAI API returned error: {result['error']}")
            return {
                'clusters': [],
                'error': result['error'],
                'response_steps': response_steps
            }
        
        # Step 11: JSON Extraction & Parsing
        if tracker:
            tracker.parse("Extracting JSON from AI response")
        step_start = time.time()
        json_data = self._extract_json_from_response(result['content'])
        
        if not json_data:
            error_msg = f"Failed to parse clustering response: {result.get('content', '')[:200]}"
            logger.error(error_msg)
            if tracker:
                tracker.error('Parse', error_msg)
            if response_steps is not None:
                response_steps.append({
                    'stepNumber': 11,
                    'stepName': 'JSON Extraction & Parsing',
                    'functionName': 'cluster_keywords',
                    'status': 'error',
                    'message': 'Failed to parse JSON from response',
                    'error': 'Failed to parse clustering response',
                    'duration': int((time.time() - step_start) * 1000)
                })
            if progress_callback:
                progress_callback(
                    state='PROGRESS',
                    meta={'response_steps': response_steps}
                )
            return {
                'clusters': [],
                'error': 'Failed to parse clustering response',
                'response_steps': response_steps
            }
        
        if response_steps is not None:
            response_steps.append({
                'stepNumber': 11,
                'stepName': 'JSON Extraction & Parsing',
                'functionName': 'cluster_keywords',
                'status': 'success',
                'message': 'Successfully parsed JSON response',
                'duration': int((time.time() - step_start) * 1000)
            })
        
        # Step 12: Cluster Data Validation
        step_start = time.time()
        clusters = []
        if isinstance(json_data, dict):
            if 'clusters' in json_data:
                clusters = json_data.get('clusters', [])
            else:
                for key, value in json_data.items():
                    if isinstance(value, list) and len(value) > 0:
                        if isinstance(value[0], dict) and ('name' in value[0] or 'keywords' in value[0]):
                            clusters = value
                            logger.info(f"Found clusters array in key '{key}' instead of 'clusters'")
                            break
        elif isinstance(json_data, list):
            clusters = json_data
        
        if not clusters:
            error_msg = f"No clusters found in response: {json_data}"
            logger.error(error_msg)
            if tracker:
                tracker.error('Parse', error_msg)
            if response_steps is not None:
                response_steps.append({
                    'stepNumber': 12,
                    'stepName': 'Cluster Data Validation',
                    'functionName': 'cluster_keywords',
                    'status': 'error',
                    'message': 'No clusters found in response',
                    'error': 'No clusters found in AI response',
                    'duration': int((time.time() - step_start) * 1000)
                })
            if progress_callback:
                progress_callback(
                    state='PROGRESS',
                    meta={'response_steps': response_steps}
                )
            return {
                'clusters': [],
                'error': 'No clusters found in AI response',
                'response_steps': response_steps
            }
        
        logger.info(f"Successfully parsed {len(clusters)} clusters from AI response")
        if tracker:
            tracker.parse(f"Successfully extracted {len(clusters)} clusters from response")
        
        if response_steps is not None:
            response_steps.append({
                'stepNumber': 12,
                'stepName': 'Cluster Data Validation',
                'functionName': 'cluster_keywords',
                'status': 'success',
                'message': f'Validated {len(clusters)} clusters',
                'duration': int((time.time() - step_start) * 1000)
            })
        
        if progress_callback:
            progress_callback(
                state='PROGRESS',
                meta={'response_steps': response_steps}
            )
        
        return {
            'clusters': clusters,
            'error': None,
            'response_steps': response_steps
        }
    
    def generate_ideas(
        self,
        clusters: List[Dict[str, Any]],  # List of cluster dicts with cluster data and keywords
        account=None,
        **kwargs
    ) -> Dict[str, Any]:
        """
        Generate content ideas for clusters using AI.
        Based on reference plugin's ideas prompt.
        
        Args:
            clusters: List of cluster dicts with cluster info and keywords
            account: Optional account for getting custom prompts
        
        Returns:
            Dict with 'ideas' (list of idea dicts)
        """
        if not clusters:
            return {
                'ideas': [],
                'error': 'No clusters provided',
            }
        
        # Format clusters for prompt (include ID for AI to reference)
        clusters_text = '\n'.join([
            f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Description: {c.get('description', '')}"
            for c in clusters
        ])
        
        # Format cluster keywords (include ID for reference)
        cluster_keywords_text = '\n'.join([
            f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Keywords: {', '.join([kw for kw in c.get('keywords', [])])}"
            for c in clusters
        ])
        
        account_obj = account or self.account
        
        # Get prompt template from database or default
        prompt_template = self.get_prompt('ideas', account=account_obj)
        
        # Replace placeholders in prompt template
        prompt = prompt_template.replace('[IGNY8_CLUSTERS]', clusters_text)
        prompt = prompt.replace('[IGNY8_CLUSTER_KEYWORDS]', cluster_keywords_text)
        
        logger.info(f"Generating ideas for {len(clusters)} clusters using AI")
        
        # Use the active model from integration settings (self.default_model)
        # Check if model supports JSON mode
        json_models = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview']
        active_model = self.default_model  # This is the active model from IntegrationSettings
        response_format = {'type': 'json_object'} if active_model in json_models else None
        
        result = self._call_openai(
            prompt,
            model=active_model,  # Explicitly pass to ensure consistency
            max_tokens=4000,
            temperature=0.7,
            response_format=response_format
        )
        
        if result.get('error'):
            return {
                'ideas': [],
                'error': result['error'],
            }
        
        # Parse JSON response
        json_data = self._extract_json_from_response(result['content'])
        
        if not json_data or 'ideas' not in json_data:
            logger.error(f"Failed to parse ideas response: {result.get('content', '')[:200]}")
            return {
                'ideas': [],
                'error': 'Failed to parse ideas response',
            }
        
        return {
            'ideas': json_data.get('ideas', []),
            'error': None,
        }
    
    def estimate_cost(
        self,
        operation: str,
        tokens_or_prompt: int | str,
        model: Optional[str] = None
    ) -> float:
        """
        Estimate cost for an AI operation.
        
        Args:
            operation: 'content', 'image', 'clustering'
            tokens_or_prompt: Token count or prompt string
            model: Model name (for content generation)
        
        Returns:
            Estimated cost in USD
        """
        # TODO: Implement cost estimation based on provider pricing
        return 0.0


# Singleton instance
ai_processor = AIProcessor()

