diff --git a/backend/igny8_core/ai/__init__.py b/backend/igny8_core/ai/__init__.py index 83df9af6..77d5d000 100644 --- a/backend/igny8_core/ai/__init__.py +++ b/backend/igny8_core/ai/__init__.py @@ -6,6 +6,25 @@ Unified framework for all AI functions with consistent lifecycle, progress track from igny8_core.ai.registry import register_function, get_function, list_functions from igny8_core.ai.engine import AIEngine from igny8_core.ai.base import BaseAIFunction +from igny8_core.ai.ai_core import AICore +from igny8_core.ai.validators import ( + validate_ids, + validate_keywords_exist, + validate_cluster_limits, + validate_cluster_exists, + validate_tasks_exist, + validate_api_key, + validate_model, + validate_image_size, +) +from igny8_core.ai.constants import ( + MODEL_RATES, + IMAGE_MODEL_RATES, + VALID_OPENAI_IMAGE_MODELS, + VALID_SIZES_BY_MODEL, + DEFAULT_AI_MODEL, + JSON_MODE_MODELS, +) # Don't auto-import functions here - let apps.py handle it lazily # This prevents circular import issues during Django startup @@ -13,8 +32,25 @@ from igny8_core.ai.base import BaseAIFunction __all__ = [ 'AIEngine', 'BaseAIFunction', + 'AICore', 'register_function', 'get_function', 'list_functions', + # Validators + 'validate_ids', + 'validate_keywords_exist', + 'validate_cluster_limits', + 'validate_cluster_exists', + 'validate_tasks_exist', + 'validate_api_key', + 'validate_model', + 'validate_image_size', + # Constants + 'MODEL_RATES', + 'IMAGE_MODEL_RATES', + 'VALID_OPENAI_IMAGE_MODELS', + 'VALID_SIZES_BY_MODEL', + 'DEFAULT_AI_MODEL', + 'JSON_MODE_MODELS', ] diff --git a/backend/igny8_core/ai/ai_core.py b/backend/igny8_core/ai/ai_core.py new file mode 100644 index 00000000..6a0761d4 --- /dev/null +++ b/backend/igny8_core/ai/ai_core.py @@ -0,0 +1,686 @@ +""" +AI Core - Centralized execution and logging layer for all AI requests +Handles API calls, model selection, response parsing, and console logging +""" +import logging +import json +import re +import requests +import time +from typing import Dict, Any, Optional, List +from django.conf import settings + +from .constants import ( + DEFAULT_AI_MODEL, + JSON_MODE_MODELS, + MODEL_RATES, + IMAGE_MODEL_RATES, + VALID_OPENAI_IMAGE_MODELS, + VALID_SIZES_BY_MODEL, +) + +logger = logging.getLogger(__name__) + + +class AICore: + """ + Centralized AI operations handler with console logging. + All AI requests go through run_ai_request() for consistent execution and logging. + """ + + def __init__(self, account=None): + """ + Initialize AICore with account context. + + Args: + account: Optional account object for API key/model loading + """ + self.account = account + self._openai_api_key = None + self._runware_api_key = None + self._default_model = None + self._load_account_settings() + + def _load_account_settings(self): + """Load API keys and model from IntegrationSettings or Django settings""" + if self.account: + try: + from igny8_core.modules.system.models import IntegrationSettings + + # Load OpenAI settings + openai_settings = IntegrationSettings.objects.filter( + integration_type='openai', + account=self.account, + is_active=True + ).first() + if openai_settings and openai_settings.config: + self._openai_api_key = openai_settings.config.get('apiKey') + model = openai_settings.config.get('model') + if model and model in MODEL_RATES: + self._default_model = model + + # Load Runware settings + runware_settings = IntegrationSettings.objects.filter( + integration_type='runware', + account=self.account, + is_active=True + ).first() + if runware_settings and runware_settings.config: + self._runware_api_key = runware_settings.config.get('apiKey') + except Exception as e: + logger.warning(f"Could not load account settings: {e}", exc_info=True) + + # Fallback to Django settings + if not self._openai_api_key: + self._openai_api_key = getattr(settings, 'OPENAI_API_KEY', None) + if not self._runware_api_key: + self._runware_api_key = getattr(settings, 'RUNWARE_API_KEY', None) + if not self._default_model: + self._default_model = getattr(settings, 'DEFAULT_AI_MODEL', DEFAULT_AI_MODEL) + + def get_api_key(self, integration_type: str = 'openai') -> Optional[str]: + """Get API key for integration type""" + if integration_type == 'openai': + return self._openai_api_key + elif integration_type == 'runware': + return self._runware_api_key + return None + + def get_model(self, integration_type: str = 'openai') -> str: + """Get model for integration type""" + if integration_type == 'openai': + return self._default_model + return DEFAULT_AI_MODEL + + def run_ai_request( + self, + prompt: str, + model: Optional[str] = None, + max_tokens: int = 4000, + temperature: float = 0.7, + response_format: Optional[Dict] = None, + api_key: Optional[str] = None, + function_name: str = 'ai_request' + ) -> Dict[str, Any]: + """ + Centralized AI request handler with console logging. + All AI text generation requests go through this method. + + Args: + prompt: Prompt text + model: Model name (defaults to account's default) + max_tokens: Maximum tokens + temperature: Temperature (0-1) + response_format: Optional response format dict (for JSON mode) + api_key: Optional API key override + function_name: Function name for logging (e.g., 'cluster_keywords') + + Returns: + Dict with 'content', 'input_tokens', 'output_tokens', 'total_tokens', + 'model', 'cost', 'error', 'api_id' + """ + print(f"[AI][{function_name}] Step 1: Preparing request...") + + # Step 1: Validate API key + api_key = api_key or self._openai_api_key + if not api_key: + error_msg = 'OpenAI API key not configured' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': model or self._default_model, + 'cost': 0.0, + 'api_id': None, + } + + # Step 2: Determine model + active_model = model or self._default_model + print(f"[AI][{function_name}] Step 2: Using model: {active_model}") + + # Step 3: Auto-enable JSON mode for supported models + if response_format is None and active_model in JSON_MODE_MODELS: + response_format = {'type': 'json_object'} + print(f"[AI][{function_name}] Step 3: Auto-enabled JSON mode for {active_model}") + elif response_format: + print(f"[AI][{function_name}] Step 3: Using custom response format: {response_format}") + else: + print(f"[AI][{function_name}] Step 3: Using text response format") + + # Step 4: Validate prompt length + prompt_length = len(prompt) + print(f"[AI][{function_name}] Step 4: Prompt length: {prompt_length} characters") + + # Step 5: Build request payload + url = 'https://api.openai.com/v1/chat/completions' + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json', + } + + body_data = { + 'model': active_model, + 'messages': [{'role': 'user', 'content': prompt}], + 'temperature': temperature, + } + + if max_tokens: + body_data['max_tokens'] = max_tokens + + if response_format: + body_data['response_format'] = response_format + + print(f"[AI][{function_name}] Step 5: Request payload prepared (model={active_model}, max_tokens={max_tokens}, temp={temperature})") + + # Step 6: Send request + print(f"[AI][{function_name}] Step 6: Sending request to OpenAI API...") + request_start = time.time() + + try: + response = requests.post(url, headers=headers, json=body_data, timeout=60) + request_duration = time.time() - request_start + print(f"[AI][{function_name}] Step 7: Received response in {request_duration:.2f}s (status={response.status_code})") + + # Step 7: Validate HTTP response + if response.status_code != 200: + error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {} + error_message = f"HTTP {response.status_code} error" + + if isinstance(error_data, dict) and 'error' in error_data: + if isinstance(error_data['error'], dict) and 'message' in error_data['error']: + error_message += f": {error_data['error']['message']}" + + # Check for rate limit + if response.status_code == 429: + retry_after = response.headers.get('retry-after', '60') + print(f"[AI][{function_name}][Error] OpenAI Rate Limit - waiting {retry_after}s") + error_message += f" (Rate limit - retry after {retry_after}s)" + + print(f"[AI][{function_name}][Error] {error_message}") + logger.error(f"OpenAI API HTTP error {response.status_code}: {error_message}") + + return { + 'content': None, + 'error': error_message, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': None, + } + + # Step 8: Parse response JSON + try: + data = response.json() + except json.JSONDecodeError as e: + error_msg = f'Failed to parse JSON response: {str(e)}' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg) + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': None, + } + + api_id = data.get('id') + + # Step 9: Extract content + if 'choices' in data and len(data['choices']) > 0: + content = data['choices'][0]['message']['content'] + usage = data.get('usage', {}) + input_tokens = usage.get('prompt_tokens', 0) + output_tokens = usage.get('completion_tokens', 0) + total_tokens = usage.get('total_tokens', 0) + + print(f"[AI][{function_name}] Step 8: Received {total_tokens} tokens (input: {input_tokens}, output: {output_tokens})") + print(f"[AI][{function_name}] Step 9: Content length: {len(content)} characters") + + # Step 10: Calculate cost + rates = MODEL_RATES.get(active_model, {'input': 2.00, 'output': 8.00}) + cost = (input_tokens * rates['input'] + output_tokens * rates['output']) / 1_000_000 + print(f"[AI][{function_name}] Step 10: Cost calculated: ${cost:.6f}") + + print(f"[AI][{function_name}][Success] Request completed successfully") + + return { + 'content': content, + 'input_tokens': input_tokens, + 'output_tokens': output_tokens, + 'total_tokens': total_tokens, + 'model': active_model, + 'cost': cost, + 'error': None, + 'api_id': api_id, + } + else: + error_msg = 'No content in OpenAI response' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg) + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': api_id, + } + + except requests.exceptions.Timeout: + error_msg = 'Request timeout (60s exceeded)' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg) + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': None, + } + except requests.exceptions.RequestException as e: + error_msg = f'Request exception: {str(e)}' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(f"OpenAI API error: {error_msg}", exc_info=True) + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': None, + } + except Exception as e: + error_msg = f'Unexpected error: {str(e)}' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg, exc_info=True) + return { + 'content': None, + 'error': error_msg, + 'input_tokens': 0, + 'output_tokens': 0, + 'total_tokens': 0, + 'model': active_model, + 'cost': 0.0, + 'api_id': None, + } + + def extract_json(self, response_text: str) -> Optional[Dict]: + """ + Extract JSON from response text. + Handles markdown code blocks, multiline JSON, etc. + + Args: + response_text: Raw response text from AI + + Returns: + Parsed JSON dict or None + """ + if not response_text or not response_text.strip(): + return None + + # Try direct JSON parse first + try: + return json.loads(response_text.strip()) + except json.JSONDecodeError: + pass + + # Try to extract JSON from markdown code blocks + json_block_pattern = r'```(?:json)?\s*(\{.*?\}|\[.*?\])\s*```' + matches = re.findall(json_block_pattern, response_text, re.DOTALL) + if matches: + try: + return json.loads(matches[0]) + except json.JSONDecodeError: + pass + + # Try to find JSON object/array in text + json_pattern = r'(\{.*\}|\[.*\])' + matches = re.findall(json_pattern, response_text, re.DOTALL) + for match in matches: + try: + return json.loads(match) + except json.JSONDecodeError: + continue + + return None + + def generate_image( + self, + prompt: str, + provider: str = 'openai', + model: Optional[str] = None, + size: str = '1024x1024', + n: int = 1, + api_key: Optional[str] = None, + negative_prompt: Optional[str] = None, + function_name: str = 'generate_image' + ) -> Dict[str, Any]: + """ + Generate image using AI with console logging. + + Args: + prompt: Image prompt + provider: 'openai' or 'runware' + model: Model name + size: Image size + n: Number of images + api_key: Optional API key override + negative_prompt: Optional negative prompt + function_name: Function name for logging + + Returns: + Dict with 'url', 'revised_prompt', 'cost', 'error', etc. + """ + print(f"[AI][{function_name}] Step 1: Preparing image generation request...") + + if provider == 'openai': + return self._generate_image_openai(prompt, model, size, n, api_key, negative_prompt, function_name) + elif provider == 'runware': + return self._generate_image_runware(prompt, model, size, n, api_key, negative_prompt, function_name) + else: + error_msg = f'Unknown provider: {provider}' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': provider, + 'cost': 0.0, + 'error': error_msg, + } + + def _generate_image_openai( + self, + prompt: str, + model: Optional[str], + size: str, + n: int, + api_key: Optional[str], + negative_prompt: Optional[str], + function_name: str + ) -> Dict[str, Any]: + """Generate image using OpenAI DALL-E""" + print(f"[AI][{function_name}] Provider: OpenAI") + + api_key = api_key or self._openai_api_key + if not api_key: + error_msg = 'OpenAI API key not configured' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + + model = model or 'dall-e-3' + print(f"[AI][{function_name}] Step 2: Using model: {model}, size: {size}") + + # Validate model + if model not in VALID_OPENAI_IMAGE_MODELS: + error_msg = f"Model '{model}' is not valid for OpenAI image generation. Only {', '.join(VALID_OPENAI_IMAGE_MODELS)} are supported." + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + + # Validate size + valid_sizes = VALID_SIZES_BY_MODEL.get(model, []) + if size not in valid_sizes: + error_msg = f"Image size '{size}' is not valid for model '{model}'. Valid sizes: {', '.join(valid_sizes)}" + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + + url = 'https://api.openai.com/v1/images/generations' + print(f"[AI][{function_name}] Step 3: Sending request to OpenAI Images API...") + + headers = { + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json', + } + + data = { + 'model': model, + 'prompt': prompt, + 'n': n, + 'size': size + } + + if negative_prompt: + # Note: OpenAI DALL-E doesn't support negative_prompt in API, but we log it + print(f"[AI][{function_name}] Note: Negative prompt provided but OpenAI DALL-E doesn't support it") + + request_start = time.time() + try: + response = requests.post(url, headers=headers, json=data, timeout=150) + request_duration = time.time() - request_start + print(f"[AI][{function_name}] Step 4: Received response in {request_duration:.2f}s (status={response.status_code})") + + if response.status_code != 200: + error_data = response.json() if response.headers.get('content-type', '').startswith('application/json') else {} + error_message = f"HTTP {response.status_code} error" + if isinstance(error_data, dict) and 'error' in error_data: + if isinstance(error_data['error'], dict) and 'message' in error_data['error']: + error_message += f": {error_data['error']['message']}" + + print(f"[AI][{function_name}][Error] {error_message}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_message, + } + + body = response.json() + if 'data' in body and len(body['data']) > 0: + image_data = body['data'][0] + image_url = image_data.get('url') + revised_prompt = image_data.get('revised_prompt') + + cost = IMAGE_MODEL_RATES.get(model, 0.040) * n + print(f"[AI][{function_name}] Step 5: Image generated successfully") + print(f"[AI][{function_name}] Step 6: Cost: ${cost:.4f}") + print(f"[AI][{function_name}][Success] Image generation completed") + + return { + 'url': image_url, + 'revised_prompt': revised_prompt, + 'provider': 'openai', + 'cost': cost, + 'error': None, + } + else: + error_msg = 'No image data in response' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + + except requests.exceptions.Timeout: + error_msg = 'Request timeout (150s exceeded)' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + except Exception as e: + error_msg = f'Unexpected error: {str(e)}' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg, exc_info=True) + return { + 'url': None, + 'revised_prompt': None, + 'provider': 'openai', + 'cost': 0.0, + 'error': error_msg, + } + + def _generate_image_runware( + self, + prompt: str, + model: Optional[str], + size: str, + n: int, + api_key: Optional[str], + negative_prompt: Optional[str], + function_name: str + ) -> Dict[str, Any]: + """Generate image using Runware""" + print(f"[AI][{function_name}] Provider: Runware") + + api_key = api_key or self._runware_api_key + if not api_key: + error_msg = 'Runware API key not configured' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'provider': 'runware', + 'cost': 0.0, + 'error': error_msg, + } + + runware_model = model or 'runware:97@1' + print(f"[AI][{function_name}] Step 2: Using model: {runware_model}, size: {size}") + + # Parse size + try: + width, height = map(int, size.split('x')) + except ValueError: + error_msg = f"Invalid size format: {size}. Expected format: WIDTHxHEIGHT" + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'provider': 'runware', + 'cost': 0.0, + 'error': error_msg, + } + + url = 'https://api.runware.ai/v1' + print(f"[AI][{function_name}] Step 3: Sending request to Runware API...") + + # Runware uses array payload + payload = [{ + 'taskType': 'imageInference', + 'model': runware_model, + 'prompt': prompt, + 'width': width, + 'height': height, + 'apiKey': api_key + }] + + if negative_prompt: + payload[0]['negativePrompt'] = negative_prompt + + request_start = time.time() + try: + response = requests.post(url, json=payload, timeout=150) + request_duration = time.time() - request_start + print(f"[AI][{function_name}] Step 4: Received response in {request_duration:.2f}s (status={response.status_code})") + + if response.status_code != 200: + error_msg = f"HTTP {response.status_code} error" + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'provider': 'runware', + 'cost': 0.0, + 'error': error_msg, + } + + body = response.json() + # Runware returns array with image data + if isinstance(body, list) and len(body) > 0: + image_data = body[0] + image_url = image_data.get('imageURL') or image_data.get('url') + + cost = 0.036 * n # Runware pricing + print(f"[AI][{function_name}] Step 5: Image generated successfully") + print(f"[AI][{function_name}] Step 6: Cost: ${cost:.4f}") + print(f"[AI][{function_name}][Success] Image generation completed") + + return { + 'url': image_url, + 'provider': 'runware', + 'cost': cost, + 'error': None, + } + else: + error_msg = 'No image data in Runware response' + print(f"[AI][{function_name}][Error] {error_msg}") + return { + 'url': None, + 'provider': 'runware', + 'cost': 0.0, + 'error': error_msg, + } + + except Exception as e: + error_msg = f'Unexpected error: {str(e)}' + print(f"[AI][{function_name}][Error] {error_msg}") + logger.error(error_msg, exc_info=True) + return { + 'url': None, + 'provider': 'runware', + 'cost': 0.0, + 'error': error_msg, + } + + def calculate_cost(self, model: str, input_tokens: int, output_tokens: int, model_type: str = 'text') -> float: + """Calculate cost for API call""" + if model_type == 'text': + rates = MODEL_RATES.get(model, {'input': 2.00, 'output': 8.00}) + input_cost = (input_tokens / 1_000_000) * rates['input'] + output_cost = (output_tokens / 1_000_000) * rates['output'] + return input_cost + output_cost + elif model_type == 'image': + rate = IMAGE_MODEL_RATES.get(model, 0.040) + return rate * 1 + return 0.0 + + # Legacy method names for backward compatibility + def call_openai(self, prompt: str, model: Optional[str] = None, max_tokens: int = 4000, + temperature: float = 0.7, response_format: Optional[Dict] = None, + api_key: Optional[str] = None) -> Dict[str, Any]: + """Legacy method - redirects to run_ai_request()""" + return self.run_ai_request( + prompt=prompt, + model=model, + max_tokens=max_tokens, + temperature=temperature, + response_format=response_format, + api_key=api_key, + function_name='call_openai' + ) diff --git a/backend/igny8_core/ai/constants.py b/backend/igny8_core/ai/constants.py new file mode 100644 index 00000000..6bf91e76 --- /dev/null +++ b/backend/igny8_core/ai/constants.py @@ -0,0 +1,37 @@ +""" +AI Constants - Model pricing, valid models, and configuration constants +""" +# Model pricing (per 1M tokens) - EXACT from reference plugin model-rates-config.php +MODEL_RATES = { + 'gpt-4.1': {'input': 2.00, 'output': 8.00}, + 'gpt-4o-mini': {'input': 0.15, 'output': 0.60}, + 'gpt-4o': {'input': 2.50, 'output': 10.00}, +} + +# Image model pricing (per image) - EXACT from reference plugin +IMAGE_MODEL_RATES = { + 'dall-e-3': 0.040, + 'dall-e-2': 0.020, + 'gpt-image-1': 0.042, + 'gpt-image-1-mini': 0.011, +} + +# Valid OpenAI image generation models (only these work with /v1/images/generations endpoint) +VALID_OPENAI_IMAGE_MODELS = { + 'dall-e-3', + 'dall-e-2', + # Note: gpt-image-1 and gpt-image-1-mini are NOT valid for OpenAI's /v1/images/generations endpoint +} + +# Valid image sizes per model (from OpenAI official documentation) +VALID_SIZES_BY_MODEL = { + 'dall-e-3': ['1024x1024', '1024x1792', '1792x1024'], + 'dall-e-2': ['256x256', '512x512', '1024x1024'], +} + +# Default model +DEFAULT_AI_MODEL = 'gpt-4.1' + +# JSON mode supported models +JSON_MODE_MODELS = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview'] + diff --git a/backend/igny8_core/ai/engine.py b/backend/igny8_core/ai/engine.py index eb33af8c..20c67753 100644 --- a/backend/igny8_core/ai/engine.py +++ b/backend/igny8_core/ai/engine.py @@ -5,7 +5,7 @@ import logging from typing import Dict, Any, Optional from igny8_core.ai.base import BaseAIFunction from igny8_core.ai.tracker import StepTracker, ProgressTracker, CostTracker -from igny8_core.ai.processor import AIProcessor +from igny8_core.ai.ai_core import AICore logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ class AIEngine: self.tracker.update("PREP", 25, f"Data prepared: {data_count} items", meta=self.step_tracker.get_meta()) # Phase 3: AI_CALL - Provider API Call (25-70%) - processor = AIProcessor(account=self.account) + ai_core = AICore(account=self.account) model = fn.get_model(self.account) # Track AI call start @@ -69,15 +69,13 @@ class AIEngine: self.tracker.update("AI_CALL", 30, f"Sending to {model or 'default'}...", meta=self.step_tracker.get_meta()) try: - raw_response = processor.call( - prompt, + # Use centralized run_ai_request() with console logging + raw_response = ai_core.run_ai_request( + prompt=prompt, model=model, - # Don't pass response_steps - the processor ignores it anyway - # Step tracking is handled by the engine - progress_callback=lambda state, meta: self.tracker.update_ai_progress(state, { - **meta, - **self.step_tracker.get_meta() - }) + max_tokens=4000, + temperature=0.7, + function_name=fn.get_name() ) except Exception as e: error_msg = f"AI call failed: {str(e)}" diff --git a/backend/igny8_core/ai/functions/__init__.py b/backend/igny8_core/ai/functions/__init__.py index c27a57bc..bad2ef31 100644 --- a/backend/igny8_core/ai/functions/__init__.py +++ b/backend/igny8_core/ai/functions/__init__.py @@ -1,4 +1,17 @@ """ AI Function implementations """ +from igny8_core.ai.functions.auto_cluster import AutoClusterFunction +from igny8_core.ai.functions.generate_ideas import GenerateIdeasFunction, generate_ideas_core +from igny8_core.ai.functions.generate_content import GenerateContentFunction, generate_content_core +from igny8_core.ai.functions.generate_images import GenerateImagesFunction, generate_images_core +__all__ = [ + 'AutoClusterFunction', + 'GenerateIdeasFunction', + 'generate_ideas_core', + 'GenerateContentFunction', + 'generate_content_core', + 'GenerateImagesFunction', + 'generate_images_core', +] diff --git a/backend/igny8_core/ai/functions/auto_cluster.py b/backend/igny8_core/ai/functions/auto_cluster.py index 0842e668..f96997da 100644 --- a/backend/igny8_core/ai/functions/auto_cluster.py +++ b/backend/igny8_core/ai/functions/auto_cluster.py @@ -7,6 +7,7 @@ from django.db import transaction from igny8_core.ai.base import BaseAIFunction from igny8_core.modules.planner.models import Keywords, Clusters from igny8_core.modules.system.utils import get_prompt_value +from igny8_core.ai.ai_core import AICore logger = logging.getLogger(__name__) @@ -36,49 +37,23 @@ class AutoClusterFunction(BaseAIFunction): def validate(self, payload: dict, account=None) -> Dict: """Custom validation for clustering with plan limit checks""" - result = super().validate(payload, account) + from igny8_core.ai.validators import validate_ids, validate_keywords_exist, validate_cluster_limits + + # Base validation + result = validate_ids(payload, max_items=self.get_max_items()) if not result['valid']: return result - # Additional validation: check keywords exist + # Check keywords exist ids = payload.get('ids', []) - queryset = Keywords.objects.filter(id__in=ids) - if account: - queryset = queryset.filter(account=account) + keywords_result = validate_keywords_exist(ids, account) + if not keywords_result['valid']: + return keywords_result - if queryset.count() == 0: - return {'valid': False, 'error': 'No keywords found'} - - # Plan limit validation - if account: - plan = getattr(account, 'plan', None) - if plan: - from django.utils import timezone - from igny8_core.modules.planner.models import Clusters - - # Check daily cluster limit - now = timezone.now() - start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0) - clusters_today = Clusters.objects.filter( - account=account, - created_at__gte=start_of_day - ).count() - - if plan.daily_cluster_limit and clusters_today >= plan.daily_cluster_limit: - return { - 'valid': False, - 'error': f'Daily cluster limit reached ({plan.daily_cluster_limit} clusters per day). Please try again tomorrow.' - } - - # Check max clusters limit - total_clusters = Clusters.objects.filter(account=account).count() - if plan.max_clusters and total_clusters >= plan.max_clusters: - return { - 'valid': False, - 'error': f'Maximum cluster limit reached ({plan.max_clusters} clusters). Please upgrade your plan or delete existing clusters.' - } - else: - return {'valid': False, 'error': 'Account does not have an active plan'} + # Check plan limits + limit_result = validate_cluster_limits(account, operation_type='cluster') + if not limit_result['valid']: + return limit_result return {'valid': True} @@ -158,7 +133,7 @@ class AutoClusterFunction(BaseAIFunction): def parse_response(self, response: str, step_tracker=None) -> List[Dict]: """Parse AI response into cluster data""" import json - from igny8_core.ai.processor import AIProcessor + from igny8_core.ai.ai_core import AICore if not response or not response.strip(): error_msg = "Empty response from AI" @@ -172,8 +147,8 @@ class AutoClusterFunction(BaseAIFunction): except json.JSONDecodeError as e: logger.warning(f"parse_response: Direct JSON parse failed: {e}, trying extract_json method") # Fall back to extract_json method which handles markdown code blocks - processor = AIProcessor() - json_data = processor.extract_json(response) + ai_core = AICore(account=getattr(self, 'account', None)) + json_data = ai_core.extract_json(response) if not json_data: error_msg = f"Failed to parse clustering response. Response: {response[:200]}..." diff --git a/backend/igny8_core/ai/functions/generate_content.py b/backend/igny8_core/ai/functions/generate_content.py new file mode 100644 index 00000000..d539d95d --- /dev/null +++ b/backend/igny8_core/ai/functions/generate_content.py @@ -0,0 +1,263 @@ +""" +Generate Content AI Function +Extracted from modules/writer/tasks.py +""" +import logging +import re +from typing import Dict, List, Any +from django.db import transaction +from igny8_core.ai.base import BaseAIFunction +from igny8_core.modules.writer.models import Tasks +from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt +from igny8_core.ai.ai_core import AICore +from igny8_core.ai.validators import validate_tasks_exist + +logger = logging.getLogger(__name__) + + +class GenerateContentFunction(BaseAIFunction): + """Generate content for tasks using AI""" + + def get_name(self) -> str: + return 'generate_content' + + def get_metadata(self) -> Dict: + return { + 'display_name': 'Generate Content', + 'description': 'Generate article content from task ideas', + 'phases': { + 'INIT': 'Initializing content generation...', + 'PREP': 'Loading tasks and building prompts...', + 'AI_CALL': 'Generating content with AI...', + 'PARSE': 'Processing content...', + 'SAVE': 'Saving content...', + 'DONE': 'Content generated!' + } + } + + def get_max_items(self) -> int: + return 50 # Max tasks per batch + + def validate(self, payload: dict, account=None) -> Dict: + """Validate task IDs""" + result = super().validate(payload, account) + if not result['valid']: + return result + + # Check tasks exist + task_ids = payload.get('ids', []) + if task_ids: + task_result = validate_tasks_exist(task_ids, account) + if not task_result['valid']: + return task_result + + return {'valid': True} + + def prepare(self, payload: dict, account=None) -> List: + """Load tasks with all relationships""" + task_ids = payload.get('ids', []) + + queryset = Tasks.objects.filter(id__in=task_ids) + if account: + queryset = queryset.filter(account=account) + + # Preload all relationships to avoid N+1 queries + tasks = list(queryset.select_related( + 'account', 'site', 'sector', 'cluster', 'idea' + )) + + if not tasks: + raise ValueError("No tasks found") + + return tasks + + def build_prompt(self, data: Any, account=None) -> str: + """Build content generation prompt for a single task""" + if isinstance(data, list): + # For now, handle single task (will be called per task) + if not data: + raise ValueError("No tasks provided") + task = data[0] + else: + task = data + + # Get prompt template + prompt_template = get_prompt_value(account or task.account, 'content_generation') + if not prompt_template: + prompt_template = get_default_prompt('content_generation') + + # Build idea data string + idea_data = f"Title: {task.title or 'Untitled'}\n" + if task.description: + idea_data += f"Description: {task.description}\n" + + # Handle idea description (might be JSON or plain text) + if task.idea and task.idea.description: + description = task.idea.description + try: + import json + parsed_desc = json.loads(description) + if isinstance(parsed_desc, dict): + formatted_desc = "Content Outline:\n\n" + if 'H2' in parsed_desc: + for h2_section in parsed_desc['H2']: + formatted_desc += f"## {h2_section.get('heading', '')}\n" + if 'subsections' in h2_section: + for h3_section in h2_section['subsections']: + formatted_desc += f"### {h3_section.get('subheading', '')}\n" + formatted_desc += f"Content Type: {h3_section.get('content_type', '')}\n" + formatted_desc += f"Details: {h3_section.get('details', '')}\n\n" + description = formatted_desc + except (json.JSONDecodeError, TypeError): + pass # Use as plain text + + idea_data += f"Outline: {description}\n" + + if task.idea: + idea_data += f"Structure: {task.idea.content_structure or task.content_structure or 'blog_post'}\n" + idea_data += f"Type: {task.idea.content_type or task.content_type or 'blog_post'}\n" + if task.idea.estimated_word_count: + idea_data += f"Estimated Word Count: {task.idea.estimated_word_count}\n" + + # Build cluster data string + cluster_data = '' + if task.cluster: + cluster_data = f"Cluster Name: {task.cluster.name or ''}\n" + if task.cluster.description: + cluster_data += f"Description: {task.cluster.description}\n" + cluster_data += f"Status: {task.cluster.status or 'active'}\n" + + # Build keywords string + keywords_data = task.keywords or '' + if not keywords_data and task.idea: + keywords_data = task.idea.target_keywords or '' + + # Replace placeholders + prompt = prompt_template.replace('[IGNY8_IDEA]', idea_data) + prompt = prompt.replace('[IGNY8_CLUSTER]', cluster_data) + prompt = prompt.replace('[IGNY8_KEYWORDS]', keywords_data) + + return prompt + + def parse_response(self, response: str, step_tracker=None) -> str: + """Parse and normalize content response""" + # Content is already text, just normalize it + try: + from igny8_core.utils.content_normalizer import normalize_content + normalized = normalize_content(response) + return normalized['normalized_content'] + except Exception as e: + logger.warning(f"Content normalization failed: {e}, using original") + return response + + def save_output( + self, + parsed: str, + original_data: Any, + account=None, + progress_tracker=None, + step_tracker=None + ) -> Dict: + """Save content to task""" + if isinstance(original_data, list): + task = original_data[0] if original_data else None + else: + task = original_data + + if not task: + raise ValueError("No task provided for saving") + + # Calculate word count + text_for_counting = re.sub(r'<[^>]+>', '', parsed) + word_count = len(text_for_counting.split()) + + # Update task + task.content = parsed + task.word_count = word_count + task.meta_title = task.title + task.meta_description = (task.description or '')[:160] + task.status = 'draft' + task.save() + + return { + 'count': 1, + 'tasks_updated': 1, + 'word_count': word_count + } + + +def generate_content_core(task_ids: List[int], account_id: int = None, progress_callback=None): + """ + Core logic for generating content (legacy function signature for backward compatibility). + Can be called with or without Celery. + + Args: + task_ids: List of task IDs + account_id: Account ID for account isolation + progress_callback: Optional function to call for progress updates + + Returns: + Dict with 'success', 'tasks_updated', 'message', etc. + """ + try: + from igny8_core.auth.models import Account + + account = None + if account_id: + account = Account.objects.get(id=account_id) + + # Use the new function class + fn = GenerateContentFunction() + fn.account = account + + # Prepare payload + payload = {'ids': task_ids} + + # Validate + validated = fn.validate(payload, account) + if not validated['valid']: + return {'success': False, 'error': validated['error']} + + # Prepare data + tasks = fn.prepare(payload, account) + + tasks_updated = 0 + + # Process each task + for task in tasks: + # Build prompt for this task + prompt = fn.build_prompt([task], account) + + # Call AI using centralized request handler + ai_core = AICore(account=account) + result = ai_core.run_ai_request( + prompt=prompt, + max_tokens=4000, + function_name='generate_content' + ) + + if result.get('error'): + logger.error(f"AI error for task {task.id}: {result['error']}") + continue + + # Parse response + content = fn.parse_response(result['content']) + + if not content: + logger.warning(f"No content generated for task {task.id}") + continue + + # Save output + save_result = fn.save_output(content, [task], account) + tasks_updated += save_result.get('tasks_updated', 0) + + return { + 'success': True, + 'tasks_updated': tasks_updated, + 'message': f'Content generation complete: {tasks_updated} articles generated' + } + + except Exception as e: + logger.error(f"Error in generate_content_core: {str(e)}", exc_info=True) + return {'success': False, 'error': str(e)} + diff --git a/backend/igny8_core/ai/functions/generate_ideas.py b/backend/igny8_core/ai/functions/generate_ideas.py new file mode 100644 index 00000000..79661764 --- /dev/null +++ b/backend/igny8_core/ai/functions/generate_ideas.py @@ -0,0 +1,256 @@ +""" +Generate Ideas AI Function +Extracted from modules/planner/tasks.py +""" +import logging +import json +from typing import Dict, List, Any +from django.db import transaction +from igny8_core.ai.base import BaseAIFunction +from igny8_core.modules.planner.models import Clusters, ContentIdeas +from igny8_core.modules.system.utils import get_prompt_value +from igny8_core.ai.ai_core import AICore +from igny8_core.ai.validators import validate_cluster_exists, validate_cluster_limits + +logger = logging.getLogger(__name__) + + +class GenerateIdeasFunction(BaseAIFunction): + """Generate content ideas from clusters using AI""" + + def get_name(self) -> str: + return 'generate_ideas' + + def get_metadata(self) -> Dict: + return { + 'display_name': 'Generate Ideas', + 'description': 'Generate SEO-optimized content ideas from keyword clusters', + 'phases': { + 'INIT': 'Initializing idea generation...', + 'PREP': 'Loading clusters...', + 'AI_CALL': 'Generating ideas with AI...', + 'PARSE': 'Parsing idea data...', + 'SAVE': 'Saving ideas...', + 'DONE': 'Ideas generated!' + } + } + + def get_max_items(self) -> int: + return 10 # Max clusters per idea generation + + def validate(self, payload: dict, account=None) -> Dict: + """Validate cluster IDs and plan limits""" + result = super().validate(payload, account) + if not result['valid']: + return result + + # Check cluster exists + cluster_ids = payload.get('ids', []) + if cluster_ids: + cluster_id = cluster_ids[0] # For single cluster idea generation + cluster_result = validate_cluster_exists(cluster_id, account) + if not cluster_result['valid']: + return cluster_result + + # Check plan limits + limit_result = validate_cluster_limits(account, operation_type='idea') + if not limit_result['valid']: + return limit_result + + return {'valid': True} + + def prepare(self, payload: dict, account=None) -> Dict: + """Load cluster with keywords""" + cluster_ids = payload.get('ids', []) + if not cluster_ids: + raise ValueError("No cluster IDs provided") + + cluster_id = cluster_ids[0] # Single cluster for now + + queryset = Clusters.objects.filter(id=cluster_id) + if account: + queryset = queryset.filter(account=account) + + cluster = queryset.select_related('sector', 'account', 'site').prefetch_related('keywords').first() + + if not cluster: + raise ValueError("Cluster not found") + + # Get keywords for this cluster + from igny8_core.modules.planner.models import Keywords + keywords = Keywords.objects.filter(cluster=cluster).values_list('keyword', flat=True) + + # Format cluster data for AI + cluster_data = [{ + 'id': cluster.id, + 'name': cluster.name, + 'description': cluster.description or '', + 'keywords': list(keywords), + }] + + return { + 'cluster': cluster, + 'cluster_data': cluster_data, + 'account': account or cluster.account + } + + def build_prompt(self, data: Dict, account=None) -> str: + """Build ideas generation prompt""" + cluster_data = data['cluster_data'] + + # Get prompt template + prompt_template = get_prompt_value(account or data.get('account'), 'ideas') + + # Format clusters text + clusters_text = '\n'.join([ + f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Description: {c.get('description', '')}" + for c in cluster_data + ]) + + # Format cluster keywords + cluster_keywords_text = '\n'.join([ + f"Cluster ID: {c.get('id', '')} | Name: {c.get('name', '')} | Keywords: {', '.join(c.get('keywords', []))}" + for c in cluster_data + ]) + + # Replace placeholders + prompt = prompt_template.replace('[IGNY8_CLUSTERS]', clusters_text) + prompt = prompt.replace('[IGNY8_CLUSTER_KEYWORDS]', cluster_keywords_text) + + return prompt + + def parse_response(self, response: str, step_tracker=None) -> List[Dict]: + """Parse AI response into idea data""" + ai_core = AICore(account=self.account if hasattr(self, 'account') else None) + json_data = ai_core.extract_json(response) + + if not json_data or 'ideas' not in json_data: + error_msg = f"Failed to parse ideas response: {response[:200]}..." + logger.error(error_msg) + raise ValueError(error_msg) + + return json_data.get('ideas', []) + + def save_output( + self, + parsed: List[Dict], + original_data: Dict, + account=None, + progress_tracker=None, + step_tracker=None + ) -> Dict: + """Save ideas to database""" + cluster = original_data['cluster'] + account = account or original_data.get('account') + + if not account: + raise ValueError("Account is required for idea creation") + + ideas_created = 0 + + with transaction.atomic(): + for idea_data in parsed: + # Handle description - might be dict or string + description = idea_data.get('description', '') + if isinstance(description, dict): + description = json.dumps(description) + elif not isinstance(description, str): + description = str(description) + + # Handle target_keywords + target_keywords = idea_data.get('covered_keywords', '') or idea_data.get('target_keywords', '') + + # Create ContentIdeas record + ContentIdeas.objects.create( + idea_title=idea_data.get('title', 'Untitled Idea'), + description=description, + content_type=idea_data.get('content_type', 'blog_post'), + content_structure=idea_data.get('content_structure', 'supporting_page'), + target_keywords=target_keywords, + keyword_cluster=cluster, + estimated_word_count=idea_data.get('estimated_word_count', 1500), + status='new', + account=account, + site=cluster.site, + sector=cluster.sector, + ) + ideas_created += 1 + + return { + 'count': ideas_created, + 'ideas_created': ideas_created + } + + +def generate_ideas_core(cluster_id: int, account_id: int = None, progress_callback=None): + """ + Core logic for generating ideas (legacy function signature for backward compatibility). + Can be called with or without Celery. + + Args: + cluster_id: Cluster ID to generate idea for + account_id: Account ID for account isolation + progress_callback: Optional function to call for progress updates + + Returns: + Dict with 'success', 'idea_created', 'message', etc. + """ + try: + from igny8_core.auth.models import Account + + account = None + if account_id: + account = Account.objects.get(id=account_id) + + # Use the new function class + fn = GenerateIdeasFunction() + # Store account for use in methods + fn.account = account + + # Prepare payload + payload = {'ids': [cluster_id]} + + # Validate + validated = fn.validate(payload, account) + if not validated['valid']: + return {'success': False, 'error': validated['error']} + + # Prepare data + data = fn.prepare(payload, account) + + # Build prompt + prompt = fn.build_prompt(data, account) + + # Call AI using centralized request handler + ai_core = AICore(account=account) + result = ai_core.run_ai_request( + prompt=prompt, + max_tokens=4000, + function_name='generate_ideas' + ) + + if result.get('error'): + return {'success': False, 'error': result['error']} + + # Parse response + ideas_data = fn.parse_response(result['content']) + + if not ideas_data: + return {'success': False, 'error': 'No ideas generated by AI'} + + # Take first idea + idea_data = ideas_data[0] + + # Save output + save_result = fn.save_output(ideas_data, data, account) + + return { + 'success': True, + 'idea_created': save_result['ideas_created'], + 'message': f"Idea '{idea_data.get('title', 'Untitled')}' created" + } + + except Exception as e: + logger.error(f"Error in generate_ideas_core: {str(e)}", exc_info=True) + return {'success': False, 'error': str(e)} + diff --git a/backend/igny8_core/ai/functions/generate_images.py b/backend/igny8_core/ai/functions/generate_images.py new file mode 100644 index 00000000..4a9ba1b8 --- /dev/null +++ b/backend/igny8_core/ai/functions/generate_images.py @@ -0,0 +1,275 @@ +""" +Generate Images AI Function +Extracted from modules/writer/tasks.py +""" +import logging +from typing import Dict, List, Any +from django.db import transaction +from igny8_core.ai.base import BaseAIFunction +from igny8_core.modules.writer.models import Tasks, Images +from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt +from igny8_core.ai.ai_core import AICore +from igny8_core.ai.validators import validate_tasks_exist + +logger = logging.getLogger(__name__) + + +class GenerateImagesFunction(BaseAIFunction): + """Generate images for tasks using AI""" + + def get_name(self) -> str: + return 'generate_images' + + def get_metadata(self) -> Dict: + return { + 'display_name': 'Generate Images', + 'description': 'Generate featured and in-article images for tasks', + 'phases': { + 'INIT': 'Initializing image generation...', + 'PREP': 'Extracting image prompts...', + 'AI_CALL': 'Generating images with AI...', + 'PARSE': 'Processing image URLs...', + 'SAVE': 'Saving images...', + 'DONE': 'Images generated!' + } + } + + def get_max_items(self) -> int: + return 20 # Max tasks per batch + + def validate(self, payload: dict, account=None) -> Dict: + """Validate task IDs""" + result = super().validate(payload, account) + if not result['valid']: + return result + + # Check tasks exist + task_ids = payload.get('ids', []) + if task_ids: + task_result = validate_tasks_exist(task_ids, account) + if not task_result['valid']: + return task_result + + return {'valid': True} + + def prepare(self, payload: dict, account=None) -> Dict: + """Load tasks and image generation settings""" + task_ids = payload.get('ids', []) + + queryset = Tasks.objects.filter(id__in=task_ids) + if account: + queryset = queryset.filter(account=account) + + tasks = list(queryset.select_related('account', 'sector', 'site')) + + if not tasks: + raise ValueError("No tasks found") + + # Get image generation settings + image_settings = {} + if account: + try: + from igny8_core.modules.system.models import IntegrationSettings + integration = IntegrationSettings.objects.get( + account=account, + integration_type='image_generation', + is_active=True + ) + image_settings = integration.config or {} + except Exception: + pass + + # Extract settings with defaults + provider = image_settings.get('provider') or image_settings.get('service', 'openai') + if provider == 'runware': + model = image_settings.get('model') or image_settings.get('runwareModel', 'runware:97@1') + else: + model = image_settings.get('model', 'dall-e-3') + + return { + 'tasks': tasks, + 'account': account, + 'provider': provider, + 'model': model, + 'image_type': image_settings.get('image_type', 'realistic'), + 'max_in_article_images': int(image_settings.get('max_in_article_images', 2)), + 'desktop_enabled': image_settings.get('desktop_enabled', True), + 'mobile_enabled': image_settings.get('mobile_enabled', True), + } + + def build_prompt(self, data: Dict, account=None) -> Dict: + """Extract image prompts from task content""" + task = data.get('task') + max_images = data.get('max_in_article_images', 2) + + if not task or not task.content: + raise ValueError("Task has no content") + + # Use AI to extract image prompts + ai_core = AICore(account=account or data.get('account')) + + # Get prompt template + prompt_template = get_prompt_value(account or data.get('account'), 'image_prompt_extraction') + if not prompt_template: + prompt_template = get_default_prompt('image_prompt_extraction') + + # Format prompt + prompt = prompt_template.format( + title=task.title, + content=task.content[:5000], # Limit content length + max_images=max_images + ) + + # Call AI to extract prompts using centralized request handler + result = ai_core.run_ai_request( + prompt=prompt, + max_tokens=1000, + function_name='extract_image_prompts' + ) + + if result.get('error'): + raise ValueError(f"Failed to extract image prompts: {result['error']}") + + # Parse JSON response + json_data = ai_core.extract_json(result['content']) + + if not json_data: + raise ValueError("Failed to parse image prompts response") + + return { + 'featured_prompt': json_data.get('featured_prompt', ''), + 'in_article_prompts': json_data.get('in_article_prompts', []) + } + + def parse_response(self, response: Dict, step_tracker=None) -> Dict: + """Parse image generation response (already parsed, just return)""" + return response + + def save_output( + self, + parsed: Dict, + original_data: Dict, + account=None, + progress_tracker=None, + step_tracker=None + ) -> Dict: + """Save images to database""" + task = original_data.get('task') + image_url = parsed.get('url') + image_type = parsed.get('image_type') # 'featured', 'desktop', 'mobile' + + if not task or not image_url: + raise ValueError("Missing task or image URL") + + # Create Images record + image = Images.objects.create( + task=task, + image_url=image_url, + image_type=image_type, + account=account or task.account, + site=task.site, + sector=task.sector, + ) + + return { + 'count': 1, + 'images_created': 1, + 'image_id': image.id + } + + +def generate_images_core(task_ids: List[int], account_id: int = None, progress_callback=None): + """ + Core logic for generating images (legacy function signature for backward compatibility). + Can be called with or without Celery. + + Args: + task_ids: List of task IDs + account_id: Account ID for account isolation + progress_callback: Optional function to call for progress updates + + Returns: + Dict with 'success', 'images_created', 'message', etc. + """ + try: + from igny8_core.auth.models import Account + + account = None + if account_id: + account = Account.objects.get(id=account_id) + + # Use the new function class + fn = GenerateImagesFunction() + fn.account = account + + # Prepare payload + payload = {'ids': task_ids} + + # Validate + validated = fn.validate(payload, account) + if not validated['valid']: + return {'success': False, 'error': validated['error']} + + # Prepare data + data = fn.prepare(payload, account) + tasks = data['tasks'] + + # Get prompts + image_prompt_template = get_prompt_value(account, 'image_prompt_template') + if not image_prompt_template: + image_prompt_template = get_default_prompt('image_prompt_template') + + negative_prompt = get_prompt_value(account, 'negative_prompt') + if not negative_prompt: + negative_prompt = get_default_prompt('negative_prompt') + + ai_core = AICore(account=account) + images_created = 0 + + # Process each task + for task in tasks: + if not task.content: + continue + + # Extract image prompts + prompts_data = fn.build_prompt({'task': task, **data}, account) + featured_prompt = prompts_data['featured_prompt'] + in_article_prompts = prompts_data['in_article_prompts'] + + # Format featured prompt + formatted_featured = image_prompt_template.format( + image_type=data['image_type'], + post_title=task.title, + image_prompt=featured_prompt + ) + + # Generate featured image using centralized handler + featured_result = ai_core.generate_image( + prompt=formatted_featured, + provider=data['provider'], + model=data['model'], + negative_prompt=negative_prompt, + function_name='generate_images' + ) + + if not featured_result.get('error') and featured_result.get('url'): + fn.save_output( + {'url': featured_result['url'], 'image_type': 'featured'}, + {'task': task, **data}, + account + ) + images_created += 1 + + # Generate in-article images (desktop/mobile if enabled) + # ... (simplified for now, full logic in tasks.py) + + return { + 'success': True, + 'images_created': images_created, + 'message': f'Image generation complete: {images_created} images created' + } + + except Exception as e: + logger.error(f"Error in generate_images_core: {str(e)}", exc_info=True) + return {'success': False, 'error': str(e)} + diff --git a/backend/igny8_core/ai/processor.py b/backend/igny8_core/ai/processor.py index f8c82792..671a569a 100644 --- a/backend/igny8_core/ai/processor.py +++ b/backend/igny8_core/ai/processor.py @@ -1,20 +1,26 @@ """ AI Processor wrapper for the framework -Reuses existing AIProcessor but provides framework-compatible interface +DEPRECATED: Use AICore.run_ai_request() instead for all new code. +This file is kept for backward compatibility only. """ from typing import Dict, Any, Optional, List from igny8_core.utils.ai_processor import AIProcessor as BaseAIProcessor +from igny8_core.ai.ai_core import AICore class AIProcessor: """ Framework-compatible wrapper around existing AIProcessor. - Provides consistent interface for all AI functions. + DEPRECATED: Use AICore.run_ai_request() instead. + This class redirects to AICore for consistency. """ def __init__(self, account=None): - self.processor = BaseAIProcessor(account=account) + # Use AICore internally for all requests + self.ai_core = AICore(account=account) self.account = account + # Keep old processor for backward compatibility only + self.processor = BaseAIProcessor(account=account) def call( self, @@ -28,35 +34,25 @@ class AIProcessor: ) -> Dict[str, Any]: """ Call AI provider with prompt. + DEPRECATED: Use AICore.run_ai_request() instead. Returns: Dict with 'content', 'error', 'input_tokens', 'output_tokens', 'total_tokens', 'model', 'cost', 'api_id' """ - # Use specified model or account's default - active_model = model or self.processor.default_model - - # Check if model supports JSON mode - json_models = ['gpt-4o', 'gpt-4o-mini', 'gpt-4-turbo-preview'] - if response_format is None and active_model in json_models: - response_format = {'type': 'json_object'} - - # Call OpenAI - don't pass response_steps to old processor - # The new framework handles all step tracking at the engine level - result = self.processor._call_openai( - prompt, - model=active_model, + # Redirect to AICore for centralized execution + return self.ai_core.run_ai_request( + prompt=prompt, + model=model, max_tokens=max_tokens, temperature=temperature, response_format=response_format, - response_steps=None # Disable old processor's step tracking + function_name='AIProcessor.call' ) - - return result def extract_json(self, response_text: str) -> Optional[Dict]: """Extract JSON from response text""" - return self.processor._extract_json_from_response(response_text) + return self.ai_core.extract_json(response_text) def generate_image( self, @@ -67,11 +63,13 @@ class AIProcessor: account=None ) -> Dict[str, Any]: """Generate image using AI""" - return self.processor.generate_image( + return self.ai_core.generate_image( prompt=prompt, + provider='openai', model=model, size=size, n=n, - account=account or self.account + account=account or self.account, + function_name='AIProcessor.generate_image' ) diff --git a/backend/igny8_core/ai/registry.py b/backend/igny8_core/ai/registry.py index a0ba67ba..ae784d37 100644 --- a/backend/igny8_core/ai/registry.py +++ b/backend/igny8_core/ai/registry.py @@ -66,5 +66,23 @@ def _load_auto_cluster(): from igny8_core.ai.functions.auto_cluster import AutoClusterFunction return AutoClusterFunction -register_lazy_function('auto_cluster', _load_auto_cluster) +def _load_generate_ideas(): + """Lazy loader for generate_ideas function""" + from igny8_core.ai.functions.generate_ideas import GenerateIdeasFunction + return GenerateIdeasFunction + +def _load_generate_content(): + """Lazy loader for generate_content function""" + from igny8_core.ai.functions.generate_content import GenerateContentFunction + return GenerateContentFunction + +def _load_generate_images(): + """Lazy loader for generate_images function""" + from igny8_core.ai.functions.generate_images import GenerateImagesFunction + return GenerateImagesFunction + +register_lazy_function('auto_cluster', _load_auto_cluster) +register_lazy_function('generate_ideas', _load_generate_ideas) +register_lazy_function('generate_content', _load_generate_content) +register_lazy_function('generate_images', _load_generate_images) diff --git a/backend/igny8_core/ai/tests/test_run.py b/backend/igny8_core/ai/tests/test_run.py new file mode 100644 index 00000000..700b0620 --- /dev/null +++ b/backend/igny8_core/ai/tests/test_run.py @@ -0,0 +1,134 @@ +""" +Test script for AI functions +Run this to verify all AI functions work with console logging +""" +import os +import sys +import django + +# Setup Django +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '../../../../')) +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'igny8.settings') +django.setup() + +from igny8_core.ai.functions.auto_cluster import AutoClusterFunction +from igny8_core.ai.functions.generate_ideas import generate_ideas_core +from igny8_core.ai.functions.generate_content import generate_content_core +from igny8_core.ai.functions.generate_images import generate_images_core +from igny8_core.ai.ai_core import AICore + + +def test_ai_core(): + """Test AICore.run_ai_request() directly""" + print("\n" + "="*80) + print("TEST 1: AICore.run_ai_request() - Direct API Call") + print("="*80) + + ai_core = AICore() + result = ai_core.run_ai_request( + prompt="Say 'Hello, World!' in JSON format: {\"message\": \"your message\"}", + max_tokens=100, + function_name='test_ai_core' + ) + + if result.get('error'): + print(f"❌ Error: {result['error']}") + else: + print(f"✅ Success! Content: {result.get('content', '')[:100]}") + print(f" Tokens: {result.get('total_tokens')}, Cost: ${result.get('cost', 0):.6f}") + + +def test_auto_cluster(): + """Test auto cluster function""" + print("\n" + "="*80) + print("TEST 2: Auto Cluster Function") + print("="*80) + print("Note: This requires actual keyword IDs in the database") + print("Skipping - requires database setup") + # Uncomment to test with real data: + # fn = AutoClusterFunction() + # result = fn.validate({'ids': [1, 2, 3]}) + # print(f"Validation result: {result}") + + +def test_generate_ideas(): + """Test generate ideas function""" + print("\n" + "="*80) + print("TEST 3: Generate Ideas Function") + print("="*80) + print("Note: This requires actual cluster ID in the database") + print("Skipping - requires database setup") + # Uncomment to test with real data: + # result = generate_ideas_core(cluster_id=1, account_id=1) + # print(f"Result: {result}") + + +def test_generate_content(): + """Test generate content function""" + print("\n" + "="*80) + print("TEST 4: Generate Content Function") + print("="*80) + print("Note: This requires actual task IDs in the database") + print("Skipping - requires database setup") + # Uncomment to test with real data: + # result = generate_content_core(task_ids=[1], account_id=1) + # print(f"Result: {result}") + + +def test_generate_images(): + """Test generate images function""" + print("\n" + "="*80) + print("TEST 5: Generate Images Function") + print("="*80) + print("Note: This requires actual task IDs in the database") + print("Skipping - requires database setup") + # Uncomment to test with real data: + # result = generate_images_core(task_ids=[1], account_id=1) + # print(f"Result: {result}") + + +def test_json_extraction(): + """Test JSON extraction""" + print("\n" + "="*80) + print("TEST 6: JSON Extraction") + print("="*80) + + ai_core = AICore() + + # Test 1: Direct JSON + json_text = '{"clusters": [{"name": "Test", "keywords": ["test"]}]}' + result = ai_core.extract_json(json_text) + print(f"✅ Direct JSON: {result is not None}") + + # Test 2: JSON in markdown + json_markdown = '```json\n{"clusters": [{"name": "Test"}]}\n```' + result = ai_core.extract_json(json_markdown) + print(f"✅ JSON in markdown: {result is not None}") + + # Test 3: Invalid JSON + invalid_json = "This is not JSON" + result = ai_core.extract_json(invalid_json) + print(f"✅ Invalid JSON handled: {result is None}") + + +if __name__ == '__main__': + print("\n" + "="*80) + print("AI FUNCTIONS TEST SUITE") + print("="*80) + print("Testing all AI functions with console logging enabled") + print("="*80) + + # Run tests + test_ai_core() + test_json_extraction() + test_auto_cluster() + test_generate_ideas() + test_generate_content() + test_generate_images() + + print("\n" + "="*80) + print("TEST SUITE COMPLETE") + print("="*80) + print("\nAll console logging should be visible above.") + print("Check for [AI][function_name] Step X: messages") + diff --git a/backend/igny8_core/ai/validators.py b/backend/igny8_core/ai/validators.py new file mode 100644 index 00000000..8b64680a --- /dev/null +++ b/backend/igny8_core/ai/validators.py @@ -0,0 +1,217 @@ +""" +AI Validators - Consolidated validation logic for all AI functions +""" +import logging +from typing import Dict, Any, Optional +from django.utils import timezone + +logger = logging.getLogger(__name__) + + +def validate_ids(payload: dict, max_items: Optional[int] = None) -> Dict[str, Any]: + """ + Base validation: checks for 'ids' array and max_items limit. + + Args: + payload: Request payload containing 'ids' array + max_items: Maximum number of items allowed (None = no limit) + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + ids = payload.get('ids', []) + if not ids: + return {'valid': False, 'error': 'No IDs provided'} + + if max_items and len(ids) > max_items: + return {'valid': False, 'error': f'Maximum {max_items} items allowed'} + + return {'valid': True} + + +def validate_keywords_exist(ids: list, account=None) -> Dict[str, Any]: + """ + Validate that keywords exist in database. + + Args: + ids: List of keyword IDs + account: Optional account for filtering + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + from igny8_core.modules.planner.models import Keywords + + queryset = Keywords.objects.filter(id__in=ids) + if account: + queryset = queryset.filter(account=account) + + if queryset.count() == 0: + return {'valid': False, 'error': 'No keywords found'} + + return {'valid': True} + + +def validate_cluster_limits(account, operation_type: str = 'cluster') -> Dict[str, Any]: + """ + Validate plan limits for cluster operations. + + Args: + account: Account object + operation_type: Type of operation ('cluster', 'idea', etc.) + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + if not account: + return {'valid': False, 'error': 'Account is required'} + + plan = getattr(account, 'plan', None) + if not plan: + return {'valid': False, 'error': 'Account does not have an active plan'} + + if operation_type == 'cluster': + from igny8_core.modules.planner.models import Clusters + + # Check daily cluster limit + now = timezone.now() + start_of_day = now.replace(hour=0, minute=0, second=0, microsecond=0) + clusters_today = Clusters.objects.filter( + account=account, + created_at__gte=start_of_day + ).count() + + if plan.daily_cluster_limit and clusters_today >= plan.daily_cluster_limit: + return { + 'valid': False, + 'error': f'Daily cluster limit reached ({plan.daily_cluster_limit} clusters per day). Please try again tomorrow.' + } + + # Check max clusters limit + total_clusters = Clusters.objects.filter(account=account).count() + if plan.max_clusters and total_clusters >= plan.max_clusters: + return { + 'valid': False, + 'error': f'Maximum cluster limit reached ({plan.max_clusters} clusters). Please upgrade your plan or delete existing clusters.' + } + + return {'valid': True} + + +def validate_cluster_exists(cluster_id: int, account=None) -> Dict[str, Any]: + """ + Validate that a cluster exists. + + Args: + cluster_id: Cluster ID + account: Optional account for filtering + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + from igny8_core.modules.planner.models import Clusters + + queryset = Clusters.objects.filter(id=cluster_id) + if account: + queryset = queryset.filter(account=account) + + if not queryset.exists(): + return {'valid': False, 'error': 'Cluster not found'} + + return {'valid': True} + + +def validate_tasks_exist(task_ids: list, account=None) -> Dict[str, Any]: + """ + Validate that tasks exist in database. + + Args: + task_ids: List of task IDs + account: Optional account for filtering + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + from igny8_core.modules.writer.models import Tasks + + queryset = Tasks.objects.filter(id__in=task_ids) + if account: + queryset = queryset.filter(account=account) + + if queryset.count() == 0: + return {'valid': False, 'error': 'No tasks found'} + + return {'valid': True} + + +def validate_api_key(api_key: Optional[str], integration_type: str = 'openai') -> Dict[str, Any]: + """ + Validate that API key is configured. + + Args: + api_key: API key to validate + integration_type: Type of integration ('openai', 'runware') + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + if not api_key: + return { + 'valid': False, + 'error': f'{integration_type.upper()} API key not configured' + } + + return {'valid': True} + + +def validate_model(model: str, model_type: str = 'text') -> Dict[str, Any]: + """ + Validate that model is in supported list. + + Args: + model: Model name to validate + model_type: Type of model ('text' or 'image') + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + from .constants import MODEL_RATES, VALID_OPENAI_IMAGE_MODELS + + if model_type == 'text': + if model not in MODEL_RATES: + return { + 'valid': False, + 'error': f'Model "{model}" is not in supported models list' + } + elif model_type == 'image': + if model not in VALID_OPENAI_IMAGE_MODELS: + return { + 'valid': False, + 'error': f'Model "{model}" is not valid for OpenAI image generation. Only {", ".join(VALID_OPENAI_IMAGE_MODELS)} are supported.' + } + + return {'valid': True} + + +def validate_image_size(size: str, model: str) -> Dict[str, Any]: + """ + Validate that image size is valid for the selected model. + + Args: + size: Image size (e.g., '1024x1024') + model: Model name + + Returns: + Dict with 'valid' (bool) and optional 'error' (str) + """ + from .constants import VALID_SIZES_BY_MODEL + + valid_sizes = VALID_SIZES_BY_MODEL.get(model, []) + if size not in valid_sizes: + return { + 'valid': False, + 'error': f'Image size "{size}" is not valid for model "{model}". Valid sizes are: {", ".join(valid_sizes)}' + } + + return {'valid': True} + diff --git a/backend/igny8_core/modules/planner/tasks.py b/backend/igny8_core/modules/planner/tasks.py index 409f2f53..9e88a048 100644 --- a/backend/igny8_core/modules/planner/tasks.py +++ b/backend/igny8_core/modules/planner/tasks.py @@ -7,6 +7,7 @@ from typing import List from django.db import transaction from igny8_core.modules.planner.models import Keywords, Clusters, ContentIdeas from igny8_core.utils.ai_processor import ai_processor +from igny8_core.ai.functions.generate_ideas import generate_ideas_core logger = logging.getLogger(__name__) diff --git a/backend/igny8_core/modules/planner/views.py b/backend/igny8_core/modules/planner/views.py index aadbf6d9..b65d57cd 100644 --- a/backend/igny8_core/modules/planner/views.py +++ b/backend/igny8_core/modules/planner/views.py @@ -977,8 +977,8 @@ class ContentIdeasViewSet(SiteSectorModelViewSet): except (KombuOperationalError, ConnectionError) as e: # Celery connection failed - execute synchronously logger.warning(f"Celery connection failed, executing synchronously: {e}") - from .tasks import _generate_single_idea_core - result = _generate_single_idea_core(cluster_id, account_id=account_id, progress_callback=None) + from igny8_core.ai.functions.generate_ideas import generate_ideas_core + result = generate_ideas_core(cluster_id, account_id=account_id, progress_callback=None) if result.get('success'): return Response({ 'success': True, diff --git a/backend/igny8_core/modules/writer/tasks.py b/backend/igny8_core/modules/writer/tasks.py index 11f6b1fe..c7a1a718 100644 --- a/backend/igny8_core/modules/writer/tasks.py +++ b/backend/igny8_core/modules/writer/tasks.py @@ -8,6 +8,8 @@ from django.db import transaction from igny8_core.modules.writer.models import Tasks, Images, Content from igny8_core.utils.ai_processor import ai_processor from igny8_core.modules.system.utils import get_prompt_value, get_default_prompt +from igny8_core.ai.functions.generate_content import generate_content_core +from igny8_core.ai.functions.generate_images import generate_images_core logger = logging.getLogger(__name__) diff --git a/docs/ActiveDocs/STAGE1-REORGANIZATION-COMPLETE.md b/docs/ActiveDocs/STAGE1-REORGANIZATION-COMPLETE.md new file mode 100644 index 00000000..c0eeeadc --- /dev/null +++ b/docs/ActiveDocs/STAGE1-REORGANIZATION-COMPLETE.md @@ -0,0 +1,191 @@ +# Stage 1 - AI Folder Structure & Functional Split - COMPLETE ✅ + +## Summary + +Successfully reorganized the AI backend into a clean, modular structure where every AI function lives inside its own file within `/ai/functions/`. + +## ✅ Completed Deliverables + +### 1. Folder Structure Created + +``` +backend/igny8_core/ai/ +├── functions/ +│ ├── __init__.py ✅ +│ ├── auto_cluster.py ✅ +│ ├── generate_ideas.py ✅ +│ ├── generate_content.py ✅ +│ └── generate_images.py ✅ +├── ai_core.py ✅ (Shared operations) +├── validators.py ✅ (Consolidated validation) +├── constants.py ✅ (Model pricing, valid models) +├── engine.py ✅ (Updated to use AICore) +├── tracker.py ✅ (Existing) +├── base.py ✅ (Existing) +├── processor.py ✅ (Existing wrapper) +├── registry.py ✅ (Updated with new functions) +└── __init__.py ✅ (Updated exports) +``` + +### 2. Shared Modules Created + +#### `ai_core.py` +- **Purpose**: Shared operations for all AI functions +- **Features**: + - API call construction (`call_openai`) + - Model selection (`get_model`, `get_api_key`) + - Response parsing (`extract_json`) + - Image generation (`generate_image`) + - Cost calculation (`calculate_cost`) +- **Status**: ✅ Complete + +#### `validators.py` +- **Purpose**: Consolidated validation logic +- **Functions**: + - `validate_ids()` - Base ID validation + - `validate_keywords_exist()` - Keyword existence check + - `validate_cluster_limits()` - Plan limit checks + - `validate_cluster_exists()` - Cluster existence + - `validate_tasks_exist()` - Task existence + - `validate_api_key()` - API key validation + - `validate_model()` - Model validation + - `validate_image_size()` - Image size validation +- **Status**: ✅ Complete + +#### `constants.py` +- **Purpose**: AI-related constants +- **Constants**: + - `MODEL_RATES` - Text model pricing + - `IMAGE_MODEL_RATES` - Image model pricing + - `VALID_OPENAI_IMAGE_MODELS` - Valid image models + - `VALID_SIZES_BY_MODEL` - Valid sizes per model + - `DEFAULT_AI_MODEL` - Default model name + - `JSON_MODE_MODELS` - Models supporting JSON mode +- **Status**: ✅ Complete + +### 3. Function Files Created + +#### `functions/auto_cluster.py` +- **Status**: ✅ Updated to use new validators and AICore +- **Changes**: + - Uses `validate_ids()`, `validate_keywords_exist()`, `validate_cluster_limits()` from validators + - Uses `AICore.extract_json()` for JSON parsing + - Maintains backward compatibility + +#### `functions/generate_ideas.py` +- **Status**: ✅ Created +- **Features**: + - `GenerateIdeasFunction` class (BaseAIFunction) + - `generate_ideas_core()` legacy function for backward compatibility + - Uses AICore for API calls + - Uses validators for validation + +#### `functions/generate_content.py` +- **Status**: ✅ Created +- **Features**: + - `GenerateContentFunction` class (BaseAIFunction) + - `generate_content_core()` legacy function for backward compatibility + - Uses AICore for API calls + - Uses validators for validation + +#### `functions/generate_images.py` +- **Status**: ✅ Created +- **Features**: + - `GenerateImagesFunction` class (BaseAIFunction) + - `generate_images_core()` legacy function for backward compatibility + - Uses AICore for image generation + - Uses validators for validation + +### 4. Import Paths Updated + +#### Updated Files: +- ✅ `modules/planner/views.py` - Uses `generate_ideas_core` from new location +- ✅ `modules/planner/tasks.py` - Imports `generate_ideas_core` from new location +- ✅ `modules/writer/tasks.py` - Imports `generate_content_core` and `generate_images_core` from new locations +- ✅ `ai/engine.py` - Uses `AICore` instead of `AIProcessor` +- ✅ `ai/functions/auto_cluster.py` - Uses new validators and AICore +- ✅ `ai/registry.py` - Registered all new functions +- ✅ `ai/__init__.py` - Exports all new modules + +### 5. Dependencies Verified + +#### No Circular Dependencies ✅ +- Functions depend on: `ai_core`, `validators`, `constants`, `base` +- `ai_core` depends on: `utils.ai_processor` (legacy, will be refactored later) +- `validators` depends on: `constants`, models +- `engine` depends on: `ai_core`, `base`, `tracker` +- All imports are clean and modular + +#### Modular Structure ✅ +- Each function file is self-contained +- Shared logic in `ai_core.py` +- Validation logic in `validators.py` +- Constants in `constants.py` +- No scattered or duplicated logic + +## 📋 File Structure Details + +### Core AI Modules + +| File | Purpose | Dependencies | +|------|---------|--------------| +| `ai_core.py` | Shared AI operations | `utils.ai_processor` (legacy) | +| `validators.py` | All validation logic | `constants`, models | +| `constants.py` | AI constants | None | +| `engine.py` | Execution orchestrator | `ai_core`, `base`, `tracker` | +| `base.py` | Base function class | None | +| `tracker.py` | Progress/step tracking | None | +| `registry.py` | Function registry | `base`, function modules | + +### Function Files + +| File | Function Class | Legacy Function | Status | +|------|----------------|-----------------|--------| +| `auto_cluster.py` | `AutoClusterFunction` | N/A (uses engine) | ✅ Updated | +| `generate_ideas.py` | `GenerateIdeasFunction` | `generate_ideas_core()` | ✅ Created | +| `generate_content.py` | `GenerateContentFunction` | `generate_content_core()` | ✅ Created | +| `generate_images.py` | `GenerateImagesFunction` | `generate_images_core()` | ✅ Created | + +## 🔄 Import Path Changes + +### Old Imports (Still work, but deprecated) +```python +from igny8_core.utils.ai_processor import AIProcessor +from igny8_core.modules.planner.tasks import _generate_single_idea_core +``` + +### New Imports (Recommended) +```python +from igny8_core.ai.functions.generate_ideas import generate_ideas_core +from igny8_core.ai.functions.generate_content import generate_content_core +from igny8_core.ai.functions.generate_images import generate_images_core +from igny8_core.ai.ai_core import AICore +from igny8_core.ai.validators import validate_ids, validate_cluster_limits +from igny8_core.ai.constants import MODEL_RATES, DEFAULT_AI_MODEL +``` + +## ✅ Verification Checklist + +- [x] All function files created in `ai/functions/` +- [x] Shared modules (`ai_core`, `validators`, `constants`) created +- [x] No circular dependencies +- [x] All imports updated in views and tasks +- [x] Functions registered in registry +- [x] `__init__.py` files updated +- [x] Backward compatibility maintained (legacy functions still work) +- [x] No linting errors +- [x] Structure matches required layout + +## 🎯 Next Steps (Future Stages) + +- **Stage 2**: Inject tracker into all functions +- **Stage 3**: Simplify logging +- **Stage 4**: Clean up legacy code + +## 📝 Notes + +- Legacy `AIProcessor` from `utils.ai_processor` is still used by `ai_core.py` as a wrapper +- This will be refactored in later stages +- All existing API endpoints continue to work +- No functional changes - only structural reorganization + diff --git a/docs/ActiveDocs/STAGE2-EXECUTION-LOGGING-COMPLETE.md b/docs/ActiveDocs/STAGE2-EXECUTION-LOGGING-COMPLETE.md new file mode 100644 index 00000000..31fd5ac7 --- /dev/null +++ b/docs/ActiveDocs/STAGE2-EXECUTION-LOGGING-COMPLETE.md @@ -0,0 +1,220 @@ +# Stage 2 - AI Execution & Logging Layer - COMPLETE ✅ + +## Summary + +Successfully created a centralized, consistent, and traceable execution layer for all AI requests with unified request handler and clean console-based logging. + +## ✅ Completed Deliverables + +### 1. Centralized Execution in `ai_core.py` + +#### `run_ai_request()` Method +- **Purpose**: Single entry point for all AI text generation requests +- **Features**: + - Step-by-step console logging with `print()` statements + - Standardized request payload construction + - Error handling with detailed logging + - Token counting and cost calculation + - Rate limit detection and logging + - Timeout handling + - JSON mode auto-enablement for supported models + +#### Console Logging Format +``` +[AI][function_name] Step 1: Preparing request... +[AI][function_name] Step 2: Using model: gpt-4o +[AI][function_name] Step 3: Auto-enabled JSON mode for gpt-4o +[AI][function_name] Step 4: Prompt length: 1234 characters +[AI][function_name] Step 5: Request payload prepared (model=gpt-4o, max_tokens=4000, temp=0.7) +[AI][function_name] Step 6: Sending request to OpenAI API... +[AI][function_name] Step 7: Received response in 2.34s (status=200) +[AI][function_name] Step 8: Received 150 tokens (input: 50, output: 100) +[AI][function_name] Step 9: Content length: 450 characters +[AI][function_name] Step 10: Cost calculated: $0.000123 +[AI][function_name][Success] Request completed successfully +``` + +#### Error Logging Format +``` +[AI][function_name][Error] OpenAI Rate Limit - waiting 60s +[AI][function_name][Error] HTTP 429 error: Rate limit exceeded (Rate limit - retry after 60s) +[AI][function_name][Error] Request timeout (60s exceeded) +[AI][function_name][Error] Failed to parse JSON response: ... +``` + +### 2. Image Generation with Logging + +#### `generate_image()` Method +- **Purpose**: Centralized image generation with console logging +- **Features**: + - Supports OpenAI DALL-E and Runware + - Model and size validation + - Step-by-step console logging + - Error handling with detailed messages + - Cost calculation + +#### Console Logging Format +``` +[AI][generate_images] Step 1: Preparing image generation request... +[AI][generate_images] Provider: OpenAI +[AI][generate_images] Step 2: Using model: dall-e-3, size: 1024x1024 +[AI][generate_images] Step 3: Sending request to OpenAI Images API... +[AI][generate_images] Step 4: Received response in 5.67s (status=200) +[AI][generate_images] Step 5: Image generated successfully +[AI][generate_images] Step 6: Cost: $0.0400 +[AI][generate_images][Success] Image generation completed +``` + +### 3. Updated All Function Files + +#### `functions/auto_cluster.py` +- ✅ Uses `AICore.extract_json()` for JSON parsing +- ✅ Engine calls `run_ai_request()` (via engine.py) + +#### `functions/generate_ideas.py` +- ✅ Updated `generate_ideas_core()` to use `run_ai_request()` +- ✅ Console logging enabled with function name + +#### `functions/generate_content.py` +- ✅ Updated `generate_content_core()` to use `run_ai_request()` +- ✅ Console logging enabled with function name + +#### `functions/generate_images.py` +- ✅ Updated to use `run_ai_request()` for prompt extraction +- ✅ Updated to use `generate_image()` with logging +- ✅ Console logging enabled + +### 4. Updated Engine + +#### `engine.py` +- ✅ Updated to use `run_ai_request()` instead of `call_openai()` +- ✅ Passes function name for logging context +- ✅ Maintains backward compatibility + +### 5. Deprecated Old Code + +#### `processor.py` +- ✅ Marked as DEPRECATED +- ✅ Redirects all calls to `AICore` +- ✅ Kept for backward compatibility only +- ✅ All methods now use `AICore` internally + +### 6. Edge Case Handling + +#### Implemented in `run_ai_request()`: +- ✅ **API Key Validation**: Logs error if not configured +- ✅ **Prompt Length**: Logs character count +- ✅ **Rate Limits**: Detects and logs retry-after time +- ✅ **Timeouts**: Handles 60s timeout with clear error +- ✅ **JSON Parsing Errors**: Logs decode errors with context +- ✅ **Empty Responses**: Validates content exists +- ✅ **Token Overflow**: Max tokens enforced +- ✅ **Model Validation**: Auto-selects JSON mode for supported models + +### 7. Standardized Request Schema + +#### OpenAI Request Payload +```python +{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.7, + "max_tokens": 4000, + "response_format": {"type": "json_object"} # Auto-enabled for supported models +} +``` + +#### All Functions Use Same Logic: +- Model selection (account default or override) +- JSON mode auto-enablement +- Token limits +- Temperature settings +- Error handling + +### 8. Test Script Created + +#### `ai/tests/test_run.py` +- ✅ Test script for all AI functions +- ✅ Tests `run_ai_request()` directly +- ✅ Tests JSON extraction +- ✅ Placeholder tests for all functions +- ✅ Can be run standalone to verify logging + +## 📋 File Changes Summary + +| File | Changes | Status | +|------|---------|--------| +| `ai_core.py` | Complete rewrite with `run_ai_request()` and console logging | ✅ Complete | +| `engine.py` | Updated to use `run_ai_request()` | ✅ Complete | +| `processor.py` | Marked deprecated, redirects to AICore | ✅ Complete | +| `functions/auto_cluster.py` | Uses AICore methods | ✅ Complete | +| `functions/generate_ideas.py` | Uses `run_ai_request()` | ✅ Complete | +| `functions/generate_content.py` | Uses `run_ai_request()` | ✅ Complete | +| `functions/generate_images.py` | Uses `run_ai_request()` and `generate_image()` | ✅ Complete | +| `tests/test_run.py` | Test script created | ✅ Complete | + +## 🔄 Migration Path + +### Old Code (Deprecated) +```python +from igny8_core.utils.ai_processor import AIProcessor +processor = AIProcessor(account=account) +result = processor._call_openai(prompt, model=model) +``` + +### New Code (Recommended) +```python +from igny8_core.ai.ai_core import AICore +ai_core = AICore(account=account) +result = ai_core.run_ai_request( + prompt=prompt, + model=model, + function_name='my_function' +) +``` + +## ✅ Verification Checklist + +- [x] `run_ai_request()` created with console logging +- [x] All function files updated to use `run_ai_request()` +- [x] Engine updated to use `run_ai_request()` +- [x] Old processor code deprecated +- [x] Edge cases handled with logging +- [x] Request schema standardized +- [x] Test script created +- [x] No linting errors +- [x] Backward compatibility maintained + +## 🎯 Benefits Achieved + +1. **Centralized Execution**: All AI requests go through one method +2. **Consistent Logging**: Every request logs steps to console +3. **Better Debugging**: Clear step-by-step visibility +4. **Error Handling**: Comprehensive error detection and logging +5. **Reduced Duplication**: No scattered AI call logic +6. **Easy Testing**: Single point to test/mock +7. **Future Ready**: Easy to add retry logic, backoff, etc. + +## 📝 Console Output Example + +When running any AI function, you'll see: +``` +[AI][generate_ideas] Step 1: Preparing request... +[AI][generate_ideas] Step 2: Using model: gpt-4o +[AI][generate_ideas] Step 3: Auto-enabled JSON mode for gpt-4o +[AI][generate_ideas] Step 4: Prompt length: 2345 characters +[AI][generate_ideas] Step 5: Request payload prepared (model=gpt-4o, max_tokens=4000, temp=0.7) +[AI][generate_ideas] Step 6: Sending request to OpenAI API... +[AI][generate_ideas] Step 7: Received response in 3.45s (status=200) +[AI][generate_ideas] Step 8: Received 250 tokens (input: 100, output: 150) +[AI][generate_ideas] Step 9: Content length: 600 characters +[AI][generate_ideas] Step 10: Cost calculated: $0.000250 +[AI][generate_ideas][Success] Request completed successfully +``` + +## 🚀 Next Steps (Future Stages) + +- **Stage 3**: Simplify logging (optional - console logging already implemented) +- **Stage 4**: Clean up legacy code (remove old processor completely) +- **Future**: Add retry logic, exponential backoff, request queuing +