Stage 3 & stage 4
This commit is contained in:
Desktop
@@ -1,75 +0,0 @@
|
|||||||
"""
|
|
||||||
AI Processor wrapper for the framework
|
|
||||||
DEPRECATED: Use AICore.run_ai_request() instead for all new code.
|
|
||||||
This file is kept for backward compatibility only.
|
|
||||||
"""
|
|
||||||
from typing import Dict, Any, Optional, List
|
|
||||||
from igny8_core.utils.ai_processor import AIProcessor as BaseAIProcessor
|
|
||||||
from igny8_core.ai.ai_core import AICore
|
|
||||||
|
|
||||||
|
|
||||||
class AIProcessor:
|
|
||||||
"""
|
|
||||||
Framework-compatible wrapper around existing AIProcessor.
|
|
||||||
DEPRECATED: Use AICore.run_ai_request() instead.
|
|
||||||
This class redirects to AICore for consistency.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, account=None):
|
|
||||||
# Use AICore internally for all requests
|
|
||||||
self.ai_core = AICore(account=account)
|
|
||||||
self.account = account
|
|
||||||
# Keep old processor for backward compatibility only
|
|
||||||
self.processor = BaseAIProcessor(account=account)
|
|
||||||
|
|
||||||
def call(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
model: Optional[str] = None,
|
|
||||||
max_tokens: int = 4000,
|
|
||||||
temperature: float = 0.7,
|
|
||||||
response_format: Optional[Dict] = None,
|
|
||||||
response_steps: Optional[List] = None,
|
|
||||||
progress_callback=None
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Call AI provider with prompt.
|
|
||||||
DEPRECATED: Use AICore.run_ai_request() instead.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with 'content', 'error', 'input_tokens', 'output_tokens',
|
|
||||||
'total_tokens', 'model', 'cost', 'api_id'
|
|
||||||
"""
|
|
||||||
# Redirect to AICore for centralized execution
|
|
||||||
return self.ai_core.run_ai_request(
|
|
||||||
prompt=prompt,
|
|
||||||
model=model,
|
|
||||||
max_tokens=max_tokens,
|
|
||||||
temperature=temperature,
|
|
||||||
response_format=response_format,
|
|
||||||
function_name='AIProcessor.call'
|
|
||||||
)
|
|
||||||
|
|
||||||
def extract_json(self, response_text: str) -> Optional[Dict]:
|
|
||||||
"""Extract JSON from response text"""
|
|
||||||
return self.ai_core.extract_json(response_text)
|
|
||||||
|
|
||||||
def generate_image(
|
|
||||||
self,
|
|
||||||
prompt: str,
|
|
||||||
model: str = 'dall-e-3',
|
|
||||||
size: str = '1024x1024',
|
|
||||||
n: int = 1,
|
|
||||||
account=None
|
|
||||||
) -> Dict[str, Any]:
|
|
||||||
"""Generate image using AI"""
|
|
||||||
return self.ai_core.generate_image(
|
|
||||||
prompt=prompt,
|
|
||||||
provider='openai',
|
|
||||||
model=model,
|
|
||||||
size=size,
|
|
||||||
n=n,
|
|
||||||
account=account or self.account,
|
|
||||||
function_name='AIProcessor.generate_image'
|
|
||||||
)
|
|
||||||
|
|
||||||
@@ -1,735 +0,0 @@
|
|||||||
"""
|
|
||||||
Celery tasks for Planner module - AI clustering and idea generation
|
|
||||||
"""
|
|
||||||
import logging
|
|
||||||
import time
|
|
||||||
from typing import List
|
|
||||||
from django.db import transaction
|
|
||||||
from igny8_core.modules.planner.models import Keywords, Clusters, ContentIdeas
|
|
||||||
from igny8_core.utils.ai_processor import ai_processor
|
|
||||||
from igny8_core.ai.tracker import ConsoleStepTracker
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
# Try to import Celery, fall back to synchronous execution if not available
|
|
||||||
try:
|
|
||||||
from celery import shared_task
|
|
||||||
CELERY_AVAILABLE = True
|
|
||||||
except ImportError:
|
|
||||||
CELERY_AVAILABLE = False
|
|
||||||
# Create a mock decorator for synchronous execution
|
|
||||||
def shared_task(*args, **kwargs):
|
|
||||||
def decorator(func):
|
|
||||||
return func
|
|
||||||
return decorator
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================================
|
|
||||||
# DEPRECATED: This function is deprecated. Use the new AI framework instead.
|
|
||||||
# New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
|
|
||||||
# This function is kept for backward compatibility but should not be used.
|
|
||||||
# ============================================================================
|
|
||||||
def _auto_cluster_keywords_core(keyword_ids: List[int], sector_id: int = None, account_id: int = None, progress_callback=None):
|
|
||||||
"""
|
|
||||||
[DEPRECATED] Core logic for clustering keywords. Can be called with or without Celery.
|
|
||||||
|
|
||||||
⚠️ WARNING: This function is deprecated. Use the new AI framework instead:
|
|
||||||
- New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
|
|
||||||
- This function uses the old AIProcessor and does not use PromptRegistry
|
|
||||||
- Console logging may not work correctly in this path
|
|
||||||
|
|
||||||
Args:
|
|
||||||
keyword_ids: List of keyword IDs to cluster
|
|
||||||
sector_id: Sector ID for the keywords
|
|
||||||
account_id: Account ID for account isolation
|
|
||||||
progress_callback: Optional function to call for progress updates (for Celery tasks)
|
|
||||||
"""
|
|
||||||
# Initialize console step tracker for logging
|
|
||||||
tracker = ConsoleStepTracker('auto_cluster')
|
|
||||||
tracker.init(f"Starting keyword clustering for {len(keyword_ids)} keywords")
|
|
||||||
|
|
||||||
# Track request and response steps (for Celery progress callbacks)
|
|
||||||
request_steps = []
|
|
||||||
response_steps = []
|
|
||||||
|
|
||||||
try:
|
|
||||||
from igny8_core.auth.models import Sector
|
|
||||||
|
|
||||||
# Initialize progress if callback provided
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': 0,
|
|
||||||
'total': len(keyword_ids),
|
|
||||||
'percentage': 0,
|
|
||||||
'message': 'Initializing keyword clustering...',
|
|
||||||
'phase': 'initializing',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 4: Keyword Loading & Validation
|
|
||||||
tracker.prep(f"Loading {len(keyword_ids)} keywords from database")
|
|
||||||
step_start = time.time()
|
|
||||||
keywords_queryset = Keywords.objects.filter(id__in=keyword_ids)
|
|
||||||
if account_id:
|
|
||||||
keywords_queryset = keywords_queryset.filter(account_id=account_id)
|
|
||||||
if sector_id:
|
|
||||||
keywords_queryset = keywords_queryset.filter(sector_id=sector_id)
|
|
||||||
|
|
||||||
keywords = list(keywords_queryset.select_related('account', 'site', 'site__account', 'sector', 'sector__site'))
|
|
||||||
|
|
||||||
if not keywords:
|
|
||||||
error_msg = f"No keywords found for clustering: {keyword_ids}"
|
|
||||||
logger.warning(error_msg)
|
|
||||||
tracker.error('Validation', error_msg)
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 4,
|
|
||||||
'stepName': 'Keyword Loading & Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': 'No keywords found',
|
|
||||||
'error': 'No keywords found',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': 'No keywords found', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
tracker.prep(f"Loaded {len(keywords)} keywords successfully")
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 4,
|
|
||||||
'stepName': 'Keyword Loading & Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Loaded {len(keywords)} keywords',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
|
|
||||||
total_keywords = len(keywords)
|
|
||||||
|
|
||||||
# Step 5: Relationship Validation
|
|
||||||
step_start = time.time()
|
|
||||||
try:
|
|
||||||
first_keyword = keywords[0]
|
|
||||||
account = getattr(first_keyword, 'account', None)
|
|
||||||
site = getattr(first_keyword, 'site', None)
|
|
||||||
|
|
||||||
# If account is None, try to get it from site
|
|
||||||
if not account and site:
|
|
||||||
try:
|
|
||||||
account = getattr(site, 'account', None)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
sector = getattr(first_keyword, 'sector', None)
|
|
||||||
|
|
||||||
# If site is None, try to get it from sector
|
|
||||||
if not site and sector:
|
|
||||||
try:
|
|
||||||
site = getattr(sector, 'site', None)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error accessing keyword relationships: {str(e)}")
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 5,
|
|
||||||
'stepName': 'Relationship Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': f'Error accessing relationships: {str(e)}',
|
|
||||||
'error': str(e),
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': f'Invalid keyword data: {str(e)}', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
if not account:
|
|
||||||
logger.error(f"No account found for keywords: {keyword_ids}. Keyword site: {getattr(first_keyword, 'site', None)}, Keyword account: {getattr(first_keyword, 'account', None)}")
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 5,
|
|
||||||
'stepName': 'Relationship Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': 'No account found',
|
|
||||||
'error': 'No account found for keywords',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': 'No account found for keywords. Please ensure keywords are properly associated with a site and account.', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
if not site:
|
|
||||||
logger.error(f"No site found for keywords: {keyword_ids}. Keyword site: {getattr(first_keyword, 'site', None)}, Sector site: {getattr(sector, 'site', None) if sector else None}")
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 5,
|
|
||||||
'stepName': 'Relationship Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': 'No site found',
|
|
||||||
'error': 'No site found for keywords',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': 'No site found for keywords. Please ensure keywords are properly associated with a site.', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 5,
|
|
||||||
'stepName': 'Relationship Validation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Account: {account.id if account else None}, Site: {site.id if site else None}, Sector: {sector.id if sector else None}',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
|
|
||||||
# Update progress: Analyzing keywords (0-40%)
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': 0,
|
|
||||||
'total': total_keywords,
|
|
||||||
'percentage': 5,
|
|
||||||
'message': f'Preparing to analyze {total_keywords} keywords...',
|
|
||||||
'phase': 'preparing',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get sector name if available
|
|
||||||
sector_name = sector.name if sector else None
|
|
||||||
|
|
||||||
# Format keywords for AI
|
|
||||||
keyword_data = [
|
|
||||||
{
|
|
||||||
'keyword': kw.keyword,
|
|
||||||
'volume': kw.volume,
|
|
||||||
'difficulty': kw.difficulty,
|
|
||||||
'intent': kw.intent,
|
|
||||||
}
|
|
||||||
for kw in keywords
|
|
||||||
]
|
|
||||||
|
|
||||||
# Update progress: Sending to AI (10-40%)
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': 0,
|
|
||||||
'total': total_keywords,
|
|
||||||
'percentage': 10,
|
|
||||||
'message': 'Analyzing keyword relationships with AI...',
|
|
||||||
'phase': 'analyzing',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Step 6: AIProcessor Creation
|
|
||||||
step_start = time.time()
|
|
||||||
from igny8_core.utils.ai_processor import AIProcessor
|
|
||||||
try:
|
|
||||||
# Log account info for debugging
|
|
||||||
account_id = account.id if account else None
|
|
||||||
account_name = account.name if account else None
|
|
||||||
logger.info(f"Creating AIProcessor with account: id={account_id}, name={account_name}")
|
|
||||||
|
|
||||||
processor = AIProcessor(account=account)
|
|
||||||
|
|
||||||
# Log API key status
|
|
||||||
has_api_key = bool(processor.openai_api_key)
|
|
||||||
api_key_preview = processor.openai_api_key[:10] + "..." if processor.openai_api_key else "None"
|
|
||||||
logger.info(f"AIProcessor created. Has API key: {has_api_key}, Preview: {api_key_preview}, Model: {processor.default_model}")
|
|
||||||
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 6,
|
|
||||||
'stepName': 'AIProcessor Creation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'AIProcessor created with account context (Account ID: {account_id}, Has API Key: {has_api_key})',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"Error creating AIProcessor: {type(e).__name__}: {str(e)}", exc_info=True)
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 6,
|
|
||||||
'stepName': 'AIProcessor Creation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': f'Error creating AIProcessor: {str(e)}',
|
|
||||||
'error': str(e),
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': f'Error creating AIProcessor: {str(e)}', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
# Step 7: AI Call Preparation
|
|
||||||
step_start = time.time()
|
|
||||||
try:
|
|
||||||
# Check if API key is available
|
|
||||||
if not processor.openai_api_key:
|
|
||||||
# Try to debug why API key is missing
|
|
||||||
logger.error(f"OpenAI API key not found for account {account.id if account else None}")
|
|
||||||
# Check IntegrationSettings directly
|
|
||||||
try:
|
|
||||||
from igny8_core.modules.system.models import IntegrationSettings
|
|
||||||
settings_obj = IntegrationSettings.objects.filter(
|
|
||||||
integration_type='openai',
|
|
||||||
account=account,
|
|
||||||
is_active=True
|
|
||||||
).first()
|
|
||||||
if settings_obj:
|
|
||||||
logger.error(f"IntegrationSettings found but API key missing. Config keys: {list(settings_obj.config.keys()) if settings_obj.config else 'None'}")
|
|
||||||
else:
|
|
||||||
logger.error(f"No IntegrationSettings found for account {account.id if account else None}, integration_type='openai', is_active=True")
|
|
||||||
except Exception as debug_error:
|
|
||||||
logger.error(f"Error checking IntegrationSettings: {str(debug_error)}", exc_info=True)
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 7,
|
|
||||||
'stepName': 'AI Call Preparation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': 'OpenAI API key not configured',
|
|
||||||
'error': 'OpenAI API key not configured',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': 'OpenAI API key not configured', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 7,
|
|
||||||
'stepName': 'AI Call Preparation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Prepared {len(keyword_data)} keywords for AI analysis',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
except Exception as e:
|
|
||||||
request_steps.append({
|
|
||||||
'stepNumber': 7,
|
|
||||||
'stepName': 'AI Call Preparation',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'error',
|
|
||||||
'message': f'Error preparing AI call: {str(e)}',
|
|
||||||
'error': str(e),
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': f'Error preparing AI call: {str(e)}', 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
# Call AI with step tracking
|
|
||||||
tracker.ai_call(f"Sending {len(keyword_data)} keywords to AI for clustering")
|
|
||||||
result = processor.cluster_keywords(
|
|
||||||
keyword_data,
|
|
||||||
sector_name=sector_name,
|
|
||||||
account=account,
|
|
||||||
response_steps=response_steps,
|
|
||||||
progress_callback=progress_callback,
|
|
||||||
tracker=tracker # Pass tracker for console logging
|
|
||||||
)
|
|
||||||
|
|
||||||
if result.get('error'):
|
|
||||||
error_msg = f"AI clustering error: {result['error']}"
|
|
||||||
logger.error(error_msg)
|
|
||||||
tracker.error('AI_CALL', error_msg)
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='FAILURE',
|
|
||||||
meta={
|
|
||||||
'error': result['error'],
|
|
||||||
'message': f"Error: {result['error']}",
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return {'success': False, 'error': result['error'], 'request_steps': request_steps, 'response_steps': response_steps}
|
|
||||||
|
|
||||||
# Parse response
|
|
||||||
tracker.parse("Parsing AI response into cluster data")
|
|
||||||
|
|
||||||
# Update response_steps from result if available
|
|
||||||
if result.get('response_steps'):
|
|
||||||
response_steps.extend(result.get('response_steps', []))
|
|
||||||
|
|
||||||
# Update progress: Creating clusters (40-90%)
|
|
||||||
clusters_data = result.get('clusters', [])
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': 0,
|
|
||||||
'total': total_keywords,
|
|
||||||
'percentage': 40,
|
|
||||||
'message': f'Creating {len(clusters_data)} clusters...',
|
|
||||||
'phase': 'creating_clusters',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
clusters_created = 0
|
|
||||||
keywords_updated = 0
|
|
||||||
|
|
||||||
# Step 13: Database Transaction Start
|
|
||||||
tracker.save(f"Creating {len(clusters_data)} clusters in database")
|
|
||||||
step_start = time.time()
|
|
||||||
# Create/update clusters and assign keywords
|
|
||||||
# Note: account and sector are already extracted above to avoid database queries inside transaction
|
|
||||||
with transaction.atomic():
|
|
||||||
if response_steps is not None:
|
|
||||||
response_steps.append({
|
|
||||||
'stepNumber': 13,
|
|
||||||
'stepName': 'Database Transaction Start',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': 'Transaction started',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
|
|
||||||
# Step 14: Cluster Creation/Update
|
|
||||||
cluster_step_start = time.time()
|
|
||||||
for idx, cluster_data in enumerate(clusters_data):
|
|
||||||
cluster_name = cluster_data.get('name', '')
|
|
||||||
cluster_keywords = cluster_data.get('keywords', [])
|
|
||||||
|
|
||||||
if not cluster_name or not cluster_keywords:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# Update progress for each cluster
|
|
||||||
if progress_callback:
|
|
||||||
progress_pct = 40 + int((idx / len(clusters_data)) * 50)
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': idx + 1,
|
|
||||||
'total': len(clusters_data),
|
|
||||||
'percentage': progress_pct,
|
|
||||||
'message': f"Creating cluster '{cluster_name}' ({idx + 1} of {len(clusters_data)})...",
|
|
||||||
'phase': 'creating_clusters',
|
|
||||||
'current_item': cluster_name,
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Get or create cluster
|
|
||||||
# Note: Clusters model (SiteSectorBaseModel) requires both site and sector
|
|
||||||
# Ensure site is always set (can be from sector.site if sector exists)
|
|
||||||
cluster_site = site if site else (sector.site if sector and hasattr(sector, 'site') else None)
|
|
||||||
|
|
||||||
if not cluster_site:
|
|
||||||
logger.error(f"Cannot create cluster '{cluster_name}': No site available. Keywords: {keyword_ids}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
if sector:
|
|
||||||
cluster, created = Clusters.objects.get_or_create(
|
|
||||||
name=cluster_name,
|
|
||||||
account=account,
|
|
||||||
site=cluster_site,
|
|
||||||
sector=sector,
|
|
||||||
defaults={
|
|
||||||
'description': cluster_data.get('description', ''),
|
|
||||||
'status': 'active',
|
|
||||||
}
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# If no sector, create cluster without sector filter but still require site
|
|
||||||
cluster, created = Clusters.objects.get_or_create(
|
|
||||||
name=cluster_name,
|
|
||||||
account=account,
|
|
||||||
site=cluster_site,
|
|
||||||
sector__isnull=True,
|
|
||||||
defaults={
|
|
||||||
'description': cluster_data.get('description', ''),
|
|
||||||
'status': 'active',
|
|
||||||
'sector': None,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
if created:
|
|
||||||
clusters_created += 1
|
|
||||||
|
|
||||||
# Step 15: Keyword Matching & Assignment
|
|
||||||
kw_step_start = time.time()
|
|
||||||
# Assign keywords to cluster
|
|
||||||
# Match keywords by keyword string (case-insensitive) from the already-loaded keywords list
|
|
||||||
# Also create a mapping for fuzzy matching (handles minor variations)
|
|
||||||
matched_keyword_objects = []
|
|
||||||
unmatched_keywords = []
|
|
||||||
|
|
||||||
# Create normalized versions for exact matching
|
|
||||||
cluster_keywords_normalized = {}
|
|
||||||
for kw in cluster_keywords:
|
|
||||||
normalized = kw.strip().lower()
|
|
||||||
cluster_keywords_normalized[normalized] = kw.strip() # Keep original for logging
|
|
||||||
|
|
||||||
# Create a mapping of all available keywords (normalized)
|
|
||||||
available_keywords_normalized = {
|
|
||||||
kw_obj.keyword.strip().lower(): kw_obj
|
|
||||||
for kw_obj in keywords
|
|
||||||
}
|
|
||||||
|
|
||||||
# First pass: exact matches (case-insensitive)
|
|
||||||
for cluster_kw_normalized, cluster_kw_original in cluster_keywords_normalized.items():
|
|
||||||
if cluster_kw_normalized in available_keywords_normalized:
|
|
||||||
matched_keyword_objects.append(available_keywords_normalized[cluster_kw_normalized])
|
|
||||||
else:
|
|
||||||
unmatched_keywords.append(cluster_kw_original)
|
|
||||||
|
|
||||||
# Log unmatched keywords for debugging
|
|
||||||
if unmatched_keywords:
|
|
||||||
logger.warning(
|
|
||||||
f"Some keywords in cluster '{cluster_name}' were not matched: {unmatched_keywords}. "
|
|
||||||
f"Available keywords: {[kw.keyword for kw in keywords]}"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Update matched keywords
|
|
||||||
if matched_keyword_objects:
|
|
||||||
matched_ids = [kw.id for kw in matched_keyword_objects]
|
|
||||||
# Rebuild queryset inside transaction to avoid database connection issues
|
|
||||||
# Handle sector=None case
|
|
||||||
keyword_filter = Keywords.objects.filter(
|
|
||||||
id__in=matched_ids,
|
|
||||||
account=account
|
|
||||||
)
|
|
||||||
if sector:
|
|
||||||
keyword_filter = keyword_filter.filter(sector=sector)
|
|
||||||
else:
|
|
||||||
keyword_filter = keyword_filter.filter(sector__isnull=True)
|
|
||||||
|
|
||||||
updated_count = keyword_filter.update(
|
|
||||||
cluster=cluster,
|
|
||||||
status='mapped' # Update status from pending to mapped
|
|
||||||
)
|
|
||||||
keywords_updated += updated_count
|
|
||||||
|
|
||||||
# Log steps 14 and 15 after all clusters are processed
|
|
||||||
if response_steps is not None:
|
|
||||||
response_steps.append({
|
|
||||||
'stepNumber': 14,
|
|
||||||
'stepName': 'Cluster Creation/Update',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Created/updated {clusters_created} clusters',
|
|
||||||
'duration': int((time.time() - cluster_step_start) * 1000)
|
|
||||||
})
|
|
||||||
response_steps.append({
|
|
||||||
'stepNumber': 15,
|
|
||||||
'stepName': 'Keyword Matching & Assignment',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Assigned {keywords_updated} keywords to clusters',
|
|
||||||
'duration': 0 # Duration already included in step 14
|
|
||||||
})
|
|
||||||
|
|
||||||
# Step 16: Metrics Recalculation & Commit
|
|
||||||
step_start = time.time()
|
|
||||||
# Update progress: Recalculating metrics (90-95%)
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='PROGRESS',
|
|
||||||
meta={
|
|
||||||
'current': clusters_created,
|
|
||||||
'total': clusters_created,
|
|
||||||
'percentage': 90,
|
|
||||||
'message': 'Recalculating cluster metrics...',
|
|
||||||
'phase': 'finalizing',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
# Recalculate cluster metrics
|
|
||||||
from django.db.models import Sum
|
|
||||||
cluster_filter = Clusters.objects.filter(account=account)
|
|
||||||
if sector:
|
|
||||||
cluster_filter = cluster_filter.filter(sector=sector)
|
|
||||||
else:
|
|
||||||
cluster_filter = cluster_filter.filter(sector__isnull=True)
|
|
||||||
|
|
||||||
for cluster in cluster_filter:
|
|
||||||
cluster.keywords_count = Keywords.objects.filter(cluster=cluster).count()
|
|
||||||
volume_sum = Keywords.objects.filter(cluster=cluster).aggregate(
|
|
||||||
total=Sum('volume')
|
|
||||||
)['total']
|
|
||||||
cluster.volume = volume_sum or 0
|
|
||||||
cluster.save()
|
|
||||||
|
|
||||||
# Transaction commits here automatically
|
|
||||||
if response_steps is not None:
|
|
||||||
response_steps.append({
|
|
||||||
'stepNumber': 16,
|
|
||||||
'stepName': 'Metrics Recalculation & Commit',
|
|
||||||
'functionName': '_auto_cluster_keywords_core',
|
|
||||||
'status': 'success',
|
|
||||||
'message': f'Recalculated metrics for {cluster_filter.count()} clusters, transaction committed',
|
|
||||||
'duration': int((time.time() - step_start) * 1000)
|
|
||||||
})
|
|
||||||
|
|
||||||
# Final progress update
|
|
||||||
final_message = f"Clustering complete: {clusters_created} clusters created, {keywords_updated} keywords updated"
|
|
||||||
logger.info(final_message)
|
|
||||||
tracker.done(final_message)
|
|
||||||
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='SUCCESS',
|
|
||||||
meta={
|
|
||||||
'message': final_message,
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
'success': True,
|
|
||||||
'clusters_created': clusters_created,
|
|
||||||
'keywords_updated': keywords_updated,
|
|
||||||
'message': final_message,
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps,
|
|
||||||
}
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
error_msg = f"Error in auto_cluster_keywords_core: {str(e)}"
|
|
||||||
logger.error(error_msg, exc_info=True)
|
|
||||||
tracker.error('Exception', error_msg, exception=e)
|
|
||||||
if progress_callback:
|
|
||||||
progress_callback(
|
|
||||||
state='FAILURE',
|
|
||||||
meta={
|
|
||||||
'error': str(e),
|
|
||||||
'message': f'Error: {str(e)}',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return {
|
|
||||||
'success': False,
|
|
||||||
'error': str(e),
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
@shared_task(bind=True, max_retries=3)
|
|
||||||
# ============================================================================
|
|
||||||
# DEPRECATED: This Celery task is deprecated. Use run_ai_task instead.
|
|
||||||
# New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
|
|
||||||
# ============================================================================
|
|
||||||
def auto_cluster_keywords_task(self, keyword_ids: List[int], sector_id: int = None, account_id: int = None):
|
|
||||||
"""
|
|
||||||
[DEPRECATED] Celery task wrapper for clustering keywords using AI.
|
|
||||||
|
|
||||||
⚠️ WARNING: This task is deprecated. Use the new AI framework instead:
|
|
||||||
- New path: views.py -> run_ai_task -> AIEngine -> AutoClusterFunction
|
|
||||||
- This task uses the old _auto_cluster_keywords_core function
|
|
||||||
- Console logging may not work correctly in this path
|
|
||||||
|
|
||||||
Args:
|
|
||||||
keyword_ids: List of keyword IDs to cluster
|
|
||||||
sector_id: Sector ID for the keywords
|
|
||||||
account_id: Account ID for account isolation
|
|
||||||
"""
|
|
||||||
logger.info("=" * 80)
|
|
||||||
logger.info("auto_cluster_keywords_task STARTED")
|
|
||||||
logger.info(f" - Task ID: {self.request.id}")
|
|
||||||
logger.info(f" - keyword_ids: {keyword_ids}")
|
|
||||||
logger.info(f" - sector_id: {sector_id}")
|
|
||||||
logger.info(f" - account_id: {account_id}")
|
|
||||||
logger.info("=" * 80)
|
|
||||||
|
|
||||||
# Initialize request_steps and response_steps for error reporting
|
|
||||||
request_steps = []
|
|
||||||
response_steps = []
|
|
||||||
|
|
||||||
def progress_callback(state, meta):
|
|
||||||
# Capture request_steps and response_steps from meta if available
|
|
||||||
nonlocal request_steps, response_steps
|
|
||||||
if isinstance(meta, dict):
|
|
||||||
if 'request_steps' in meta:
|
|
||||||
request_steps = meta['request_steps']
|
|
||||||
if 'response_steps' in meta:
|
|
||||||
response_steps = meta['response_steps']
|
|
||||||
self.update_state(state=state, meta=meta)
|
|
||||||
|
|
||||||
try:
|
|
||||||
result = _auto_cluster_keywords_core(keyword_ids, sector_id, account_id, progress_callback)
|
|
||||||
logger.info(f"auto_cluster_keywords_task COMPLETED: {result}")
|
|
||||||
return result
|
|
||||||
except Exception as e:
|
|
||||||
error_type = type(e).__name__
|
|
||||||
error_msg = str(e)
|
|
||||||
|
|
||||||
# Log full error details
|
|
||||||
logger.error("=" * 80)
|
|
||||||
logger.error(f"auto_cluster_keywords_task FAILED: {error_type}: {error_msg}")
|
|
||||||
logger.error(f" - Task ID: {self.request.id}")
|
|
||||||
logger.error(f" - keyword_ids: {keyword_ids}")
|
|
||||||
logger.error(f" - sector_id: {sector_id}")
|
|
||||||
logger.error(f" - account_id: {account_id}")
|
|
||||||
logger.error("=" * 80, exc_info=True)
|
|
||||||
|
|
||||||
# Create detailed error dict that Celery can serialize
|
|
||||||
error_dict = {
|
|
||||||
'error': error_msg,
|
|
||||||
'error_type': error_type,
|
|
||||||
'error_class': error_type,
|
|
||||||
'message': f'{error_type}: {error_msg}',
|
|
||||||
'request_steps': request_steps,
|
|
||||||
'response_steps': response_steps,
|
|
||||||
'task_id': str(self.request.id),
|
|
||||||
'keyword_ids': keyword_ids,
|
|
||||||
'sector_id': sector_id,
|
|
||||||
'account_id': account_id
|
|
||||||
}
|
|
||||||
|
|
||||||
# Update task state with detailed error
|
|
||||||
try:
|
|
||||||
self.update_state(
|
|
||||||
state='FAILURE',
|
|
||||||
meta=error_dict
|
|
||||||
)
|
|
||||||
except Exception as update_error:
|
|
||||||
# If update_state fails, log it but continue
|
|
||||||
logger.error(f"Failed to update task state: {str(update_error)}")
|
|
||||||
|
|
||||||
# Return error result
|
|
||||||
return error_dict
|
|
||||||
|
|
||||||
|
|
||||||
# REMOVED: All idea generation functions removed
|
|
||||||
# - auto_generate_ideas_task
|
|
||||||
# - _generate_single_idea_core
|
|
||||||
# - generate_single_idea_task
|
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -178,19 +178,37 @@ class TasksViewSet(SiteSectorModelViewSet):
|
|||||||
|
|
||||||
# Try to queue Celery task, fall back to synchronous if Celery not available
|
# Try to queue Celery task, fall back to synchronous if Celery not available
|
||||||
try:
|
try:
|
||||||
from .tasks import auto_generate_content_task
|
from igny8_core.ai.tasks import run_ai_task
|
||||||
|
from kombu.exceptions import OperationalError as KombuOperationalError
|
||||||
|
|
||||||
if hasattr(auto_generate_content_task, 'delay'):
|
if hasattr(run_ai_task, 'delay'):
|
||||||
# Celery is available - queue async task
|
# Celery is available - queue async task
|
||||||
logger.info(f"auto_generate_content: Queuing Celery task for {len(ids)} tasks")
|
logger.info(f"auto_generate_content: Queuing Celery task for {len(ids)} tasks")
|
||||||
try:
|
try:
|
||||||
task = auto_generate_content_task.delay(ids, account_id=account_id)
|
task = run_ai_task.delay(
|
||||||
|
function_name='generate_content',
|
||||||
|
payload={'ids': ids},
|
||||||
|
account_id=account_id
|
||||||
|
)
|
||||||
logger.info(f"auto_generate_content: Celery task queued successfully: {task.id}")
|
logger.info(f"auto_generate_content: Celery task queued successfully: {task.id}")
|
||||||
return Response({
|
return Response({
|
||||||
'success': True,
|
'success': True,
|
||||||
'task_id': str(task.id),
|
'task_id': str(task.id),
|
||||||
'message': 'Content generation started'
|
'message': 'Content generation started'
|
||||||
}, status=status.HTTP_200_OK)
|
}, status=status.HTTP_200_OK)
|
||||||
|
except KombuOperationalError as celery_error:
|
||||||
|
logger.error("=" * 80)
|
||||||
|
logger.error("CELERY ERROR: Failed to queue task")
|
||||||
|
logger.error(f" - Error type: {type(celery_error).__name__}")
|
||||||
|
logger.error(f" - Error message: {str(celery_error)}")
|
||||||
|
logger.error(f" - Task IDs: {ids}")
|
||||||
|
logger.error(f" - Account ID: {account_id}")
|
||||||
|
logger.error("=" * 80, exc_info=True)
|
||||||
|
|
||||||
|
return Response({
|
||||||
|
'error': 'Task queue unavailable. Please try again.',
|
||||||
|
'type': 'QueueError'
|
||||||
|
}, status=status.HTTP_503_SERVICE_UNAVAILABLE)
|
||||||
except Exception as celery_error:
|
except Exception as celery_error:
|
||||||
logger.error("=" * 80)
|
logger.error("=" * 80)
|
||||||
logger.error("CELERY ERROR: Failed to queue task")
|
logger.error("CELERY ERROR: Failed to queue task")
|
||||||
@@ -202,11 +220,15 @@ class TasksViewSet(SiteSectorModelViewSet):
|
|||||||
|
|
||||||
# Fall back to synchronous execution
|
# Fall back to synchronous execution
|
||||||
logger.info("auto_generate_content: Falling back to synchronous execution")
|
logger.info("auto_generate_content: Falling back to synchronous execution")
|
||||||
result = auto_generate_content_task(ids, account_id=account_id)
|
result = run_ai_task(
|
||||||
|
function_name='generate_content',
|
||||||
|
payload={'ids': ids},
|
||||||
|
account_id=account_id
|
||||||
|
)
|
||||||
if result.get('success'):
|
if result.get('success'):
|
||||||
return Response({
|
return Response({
|
||||||
'success': True,
|
'success': True,
|
||||||
'tasks_updated': result.get('tasks_updated', 0),
|
'tasks_updated': result.get('count', 0),
|
||||||
'message': 'Content generated successfully (synchronous)'
|
'message': 'Content generated successfully (synchronous)'
|
||||||
}, status=status.HTTP_200_OK)
|
}, status=status.HTTP_200_OK)
|
||||||
else:
|
else:
|
||||||
@@ -217,12 +239,16 @@ class TasksViewSet(SiteSectorModelViewSet):
|
|||||||
else:
|
else:
|
||||||
# Celery not available - execute synchronously
|
# Celery not available - execute synchronously
|
||||||
logger.info(f"auto_generate_content: Executing synchronously (Celery not available)")
|
logger.info(f"auto_generate_content: Executing synchronously (Celery not available)")
|
||||||
result = auto_generate_content_task(ids, account_id=account_id)
|
result = run_ai_task(
|
||||||
|
function_name='generate_content',
|
||||||
|
payload={'ids': ids},
|
||||||
|
account_id=account_id
|
||||||
|
)
|
||||||
if result.get('success'):
|
if result.get('success'):
|
||||||
logger.info(f"auto_generate_content: Synchronous execution successful: {result.get('tasks_updated', 0)} tasks updated")
|
logger.info(f"auto_generate_content: Synchronous execution successful: {result.get('count', 0)} tasks updated")
|
||||||
return Response({
|
return Response({
|
||||||
'success': True,
|
'success': True,
|
||||||
'tasks_updated': result.get('tasks_updated', 0),
|
'tasks_updated': result.get('count', 0),
|
||||||
'message': 'Content generated successfully'
|
'message': 'Content generated successfully'
|
||||||
}, status=status.HTTP_200_OK)
|
}, status=status.HTTP_200_OK)
|
||||||
else:
|
else:
|
||||||
@@ -356,10 +382,16 @@ class ImagesViewSet(SiteSectorModelViewSet):
|
|||||||
|
|
||||||
# Try to queue Celery task, fall back to synchronous if Celery not available
|
# Try to queue Celery task, fall back to synchronous if Celery not available
|
||||||
try:
|
try:
|
||||||
from .tasks import auto_generate_images_task
|
from igny8_core.ai.tasks import run_ai_task
|
||||||
if hasattr(auto_generate_images_task, 'delay'):
|
from kombu.exceptions import OperationalError as KombuOperationalError
|
||||||
|
|
||||||
|
if hasattr(run_ai_task, 'delay'):
|
||||||
# Celery is available - queue async task
|
# Celery is available - queue async task
|
||||||
task = auto_generate_images_task.delay(task_ids, account_id=account_id)
|
task = run_ai_task.delay(
|
||||||
|
function_name='generate_images',
|
||||||
|
payload={'ids': task_ids},
|
||||||
|
account_id=account_id
|
||||||
|
)
|
||||||
return Response({
|
return Response({
|
||||||
'success': True,
|
'success': True,
|
||||||
'task_id': str(task.id),
|
'task_id': str(task.id),
|
||||||
@@ -367,22 +399,39 @@ class ImagesViewSet(SiteSectorModelViewSet):
|
|||||||
}, status=status.HTTP_200_OK)
|
}, status=status.HTTP_200_OK)
|
||||||
else:
|
else:
|
||||||
# Celery not available - execute synchronously
|
# Celery not available - execute synchronously
|
||||||
result = auto_generate_images_task(task_ids, account_id=account_id)
|
result = run_ai_task(
|
||||||
|
function_name='generate_images',
|
||||||
|
payload={'ids': task_ids},
|
||||||
|
account_id=account_id
|
||||||
|
)
|
||||||
if result.get('success'):
|
if result.get('success'):
|
||||||
return Response({
|
return Response({
|
||||||
'success': True,
|
'success': True,
|
||||||
'images_created': result.get('images_created', 0),
|
'images_created': result.get('count', 0),
|
||||||
'message': result.get('message', 'Image generation completed')
|
'message': result.get('message', 'Image generation completed')
|
||||||
}, status=status.HTTP_200_OK)
|
}, status=status.HTTP_200_OK)
|
||||||
else:
|
else:
|
||||||
return Response({
|
return Response({
|
||||||
'error': result.get('error', 'Image generation failed')
|
'error': result.get('error', 'Image generation failed')
|
||||||
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||||
|
except KombuOperationalError as e:
|
||||||
|
return Response({
|
||||||
|
'error': 'Task queue unavailable. Please try again.',
|
||||||
|
'type': 'QueueError'
|
||||||
|
}, status=status.HTTP_503_SERVICE_UNAVAILABLE)
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# Tasks module not available
|
# Tasks module not available
|
||||||
return Response({
|
return Response({
|
||||||
'error': 'Image generation task not available'
|
'error': 'Image generation task not available'
|
||||||
}, status=status.HTTP_503_SERVICE_UNAVAILABLE)
|
}, status=status.HTTP_503_SERVICE_UNAVAILABLE)
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logger.error(f"Error queuing image generation task: {str(e)}", exc_info=True)
|
||||||
|
return Response({
|
||||||
|
'error': f'Failed to start image generation: {str(e)}',
|
||||||
|
'type': 'TaskError'
|
||||||
|
}, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
|
||||||
|
|
||||||
|
|
||||||
class ContentViewSet(SiteSectorModelViewSet):
|
class ContentViewSet(SiteSectorModelViewSet):
|
||||||
|
|||||||
@@ -15,33 +15,15 @@ from django.core.cache import cache
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
# Model pricing (per 1M tokens) - EXACT from reference plugin model-rates-config.php
|
# Import constants from unified location
|
||||||
MODEL_RATES = {
|
from igny8_core.ai.constants import (
|
||||||
'gpt-4.1': {'input': 2.00, 'output': 8.00},
|
MODEL_RATES,
|
||||||
'gpt-4o-mini': {'input': 0.15, 'output': 0.60},
|
IMAGE_MODEL_RATES,
|
||||||
'gpt-4o': {'input': 2.50, 'output': 10.00},
|
VALID_OPENAI_IMAGE_MODELS,
|
||||||
}
|
VALID_SIZES_BY_MODEL,
|
||||||
|
DEFAULT_AI_MODEL,
|
||||||
# Image model pricing (per image) - EXACT from reference plugin
|
JSON_MODE_MODELS,
|
||||||
IMAGE_MODEL_RATES = {
|
)
|
||||||
'dall-e-3': 0.040,
|
|
||||||
'dall-e-2': 0.020,
|
|
||||||
'gpt-image-1': 0.042,
|
|
||||||
'gpt-image-1-mini': 0.011,
|
|
||||||
}
|
|
||||||
|
|
||||||
# Valid OpenAI image generation models (only these work with /v1/images/generations endpoint)
|
|
||||||
VALID_OPENAI_IMAGE_MODELS = {
|
|
||||||
'dall-e-3',
|
|
||||||
'dall-e-2',
|
|
||||||
# Note: gpt-image-1 and gpt-image-1-mini are NOT valid for OpenAI's /v1/images/generations endpoint
|
|
||||||
}
|
|
||||||
|
|
||||||
# Valid image sizes per model (from OpenAI official documentation)
|
|
||||||
VALID_SIZES_BY_MODEL = {
|
|
||||||
'dall-e-3': ['1024x1024', '1024x1792', '1792x1024'],
|
|
||||||
'dall-e-2': ['256x256', '512x512', '1024x1024'],
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
class AIProcessor:
|
class AIProcessor:
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
// Centralized API configuration and functions
|
// Centralized API configuration and functions
|
||||||
// Auto-detect API URL based on current origin (supports both IP and subdomain access)
|
// Auto-detect API URL based on current origin (supports both IP and subdomain access)
|
||||||
import { useAuthStore } from '../store/authStore';
|
import { useAuthStore } from '../store/authStore';
|
||||||
import { useAIRequestLogsStore } from '../store/aiRequestLogsStore';
|
|
||||||
|
|
||||||
function getApiBaseUrl(): string {
|
function getApiBaseUrl(): string {
|
||||||
// First check environment variables
|
// First check environment variables
|
||||||
@@ -575,43 +574,15 @@ export async function bulkUpdateClustersStatus(ids: number[], status: string): P
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function autoClusterKeywords(keywordIds: number[], sectorId?: number): Promise<{ success: boolean; task_id?: string; clusters_created?: number; keywords_updated?: number; message?: string; error?: string }> {
|
export async function autoClusterKeywords(keywordIds: number[], sectorId?: number): Promise<{ success: boolean; task_id?: string; clusters_created?: number; keywords_updated?: number; message?: string; error?: string }> {
|
||||||
const startTime = Date.now();
|
|
||||||
const addLog = useAIRequestLogsStore.getState().addLog;
|
|
||||||
|
|
||||||
const endpoint = `/v1/planner/keywords/auto_cluster/`;
|
const endpoint = `/v1/planner/keywords/auto_cluster/`;
|
||||||
const requestBody = { ids: keywordIds, sector_id: sectorId };
|
const requestBody = { ids: keywordIds, sector_id: sectorId };
|
||||||
|
|
||||||
const pendingLogId = addLog({
|
|
||||||
function: 'autoClusterKeywords',
|
|
||||||
endpoint,
|
|
||||||
request: {
|
|
||||||
method: 'POST',
|
|
||||||
body: requestBody,
|
|
||||||
},
|
|
||||||
status: 'pending',
|
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetchAPI(endpoint, {
|
const response = await fetchAPI(endpoint, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
body: JSON.stringify(requestBody),
|
body: JSON.stringify(requestBody),
|
||||||
});
|
});
|
||||||
|
|
||||||
const duration = Date.now() - startTime;
|
|
||||||
const updateLog = useAIRequestLogsStore.getState().updateLog;
|
|
||||||
|
|
||||||
// Update log with response data (including task_id for progress tracking)
|
|
||||||
if (pendingLogId && response) {
|
|
||||||
updateLog(pendingLogId, {
|
|
||||||
response: {
|
|
||||||
status: 200,
|
|
||||||
data: response,
|
|
||||||
},
|
|
||||||
status: response.success === false ? 'error' : 'success',
|
|
||||||
duration,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if response indicates an error (success: false)
|
// Check if response indicates an error (success: false)
|
||||||
if (response && response.success === false) {
|
if (response && response.success === false) {
|
||||||
// Return error response as-is so caller can check result.success
|
// Return error response as-is so caller can check result.success
|
||||||
@@ -620,108 +591,7 @@ export async function autoClusterKeywords(keywordIds: number[], sectorId?: numbe
|
|||||||
|
|
||||||
return response;
|
return response;
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
const duration = Date.now() - startTime;
|
throw error;
|
||||||
|
|
||||||
// Try to extract error response data if available
|
|
||||||
let errorResponseData = null;
|
|
||||||
let errorRequestSteps = null;
|
|
||||||
|
|
||||||
// Check if error has response data (from fetchAPI)
|
|
||||||
if (error.response || error.data) {
|
|
||||||
errorResponseData = error.response || error.data;
|
|
||||||
errorRequestSteps = errorResponseData?.request_steps;
|
|
||||||
} else if ((error as any).response) {
|
|
||||||
// Error object from fetchAPI has response attached
|
|
||||||
errorResponseData = (error as any).response;
|
|
||||||
errorRequestSteps = errorResponseData?.request_steps;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Parse error message to extract error type
|
|
||||||
let errorType = 'UNKNOWN_ERROR';
|
|
||||||
let errorMessage = error.message || 'Unknown error';
|
|
||||||
|
|
||||||
// Check if error response contains JSON with error field
|
|
||||||
if (error.message && error.message.includes('API Error')) {
|
|
||||||
// Try to extract structured error from API response
|
|
||||||
const apiErrorMatch = error.message.match(/API Error \(\d+\): ([^-]+) - (.+)/);
|
|
||||||
if (apiErrorMatch) {
|
|
||||||
errorType = apiErrorMatch[1].trim();
|
|
||||||
errorMessage = apiErrorMatch[2].trim();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (errorMessage.includes('OperationalError')) {
|
|
||||||
errorType = 'DATABASE_ERROR';
|
|
||||||
errorMessage = errorMessage.replace(/API Error \(\d+\): /, '').replace(/ - .*OperationalError.*/, ' - Database operation failed');
|
|
||||||
} else if (errorMessage.includes('ValidationError')) {
|
|
||||||
errorType = 'VALIDATION_ERROR';
|
|
||||||
} else if (errorMessage.includes('PermissionDenied')) {
|
|
||||||
errorType = 'PERMISSION_ERROR';
|
|
||||||
} else if (errorMessage.includes('NotFound')) {
|
|
||||||
errorType = 'NOT_FOUND_ERROR';
|
|
||||||
} else if (errorMessage.includes('IntegrityError')) {
|
|
||||||
errorType = 'DATABASE_ERROR';
|
|
||||||
} else if (errorMessage.includes('RelatedObjectDoesNotExist')) {
|
|
||||||
errorType = 'RELATED_OBJECT_ERROR';
|
|
||||||
// Extract clean error message
|
|
||||||
errorMessage = errorMessage.replace(/API Error \(\d+\): [^-]+ - /, '').trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update existing log or create new one
|
|
||||||
const updateLog = useAIRequestLogsStore.getState().updateLog;
|
|
||||||
const addRequestStep = useAIRequestLogsStore.getState().addRequestStep;
|
|
||||||
|
|
||||||
if (pendingLogId) {
|
|
||||||
updateLog(pendingLogId, {
|
|
||||||
response: {
|
|
||||||
status: errorResponseData?.status || 500,
|
|
||||||
error: errorMessage,
|
|
||||||
errorType,
|
|
||||||
data: errorResponseData,
|
|
||||||
},
|
|
||||||
status: 'error',
|
|
||||||
duration,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Add request steps from error response if available
|
|
||||||
if (errorRequestSteps && Array.isArray(errorRequestSteps)) {
|
|
||||||
errorRequestSteps.forEach((step: any) => {
|
|
||||||
addRequestStep(pendingLogId, step);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// Create new log if pendingLogId doesn't exist
|
|
||||||
const errorLogId = addLog({
|
|
||||||
function: 'autoClusterKeywords',
|
|
||||||
endpoint,
|
|
||||||
request: {
|
|
||||||
method: 'POST',
|
|
||||||
body: requestBody,
|
|
||||||
},
|
|
||||||
response: {
|
|
||||||
status: errorResponseData?.status || 500,
|
|
||||||
error: errorMessage,
|
|
||||||
errorType,
|
|
||||||
data: errorResponseData,
|
|
||||||
},
|
|
||||||
status: 'error',
|
|
||||||
duration,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (errorLogId && errorRequestSteps && Array.isArray(errorRequestSteps)) {
|
|
||||||
errorRequestSteps.forEach((step: any) => {
|
|
||||||
addRequestStep(errorLogId, step);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Return error response in same format as successful response
|
|
||||||
// This allows the caller to check result.success === false
|
|
||||||
return {
|
|
||||||
success: false,
|
|
||||||
error: errorMessage,
|
|
||||||
errorType,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,122 +0,0 @@
|
|||||||
import { create } from 'zustand';
|
|
||||||
|
|
||||||
export interface AIStepLog {
|
|
||||||
stepNumber: number;
|
|
||||||
stepName: string;
|
|
||||||
functionName: string;
|
|
||||||
status: 'pending' | 'success' | 'error';
|
|
||||||
timestamp: Date;
|
|
||||||
message?: string;
|
|
||||||
error?: string;
|
|
||||||
duration?: number; // milliseconds
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface AIRequestLog {
|
|
||||||
id: string;
|
|
||||||
timestamp: Date;
|
|
||||||
function: string; // e.g., 'autoClusterKeywords', 'autoGenerateIdeas', 'autoGenerateContent', 'autoGenerateImages'
|
|
||||||
endpoint: string;
|
|
||||||
request: {
|
|
||||||
method: string;
|
|
||||||
body?: any;
|
|
||||||
params?: any;
|
|
||||||
};
|
|
||||||
response?: {
|
|
||||||
status: number;
|
|
||||||
data?: any;
|
|
||||||
error?: string;
|
|
||||||
errorType?: string; // e.g., 'DATABASE_ERROR', 'VALIDATION_ERROR', 'PERMISSION_ERROR'
|
|
||||||
};
|
|
||||||
status: 'pending' | 'success' | 'error';
|
|
||||||
duration?: number; // milliseconds
|
|
||||||
requestSteps: AIStepLog[]; // Request steps (INIT, PREP, SAVE, DONE)
|
|
||||||
responseSteps: AIStepLog[]; // Response steps (AI_CALL, PARSE)
|
|
||||||
}
|
|
||||||
|
|
||||||
interface AIRequestLogsStore {
|
|
||||||
logs: AIRequestLog[];
|
|
||||||
addLog: (log: Omit<AIRequestLog, 'id' | 'timestamp' | 'requestSteps' | 'responseSteps'>) => string;
|
|
||||||
updateLog: (logId: string, updates: Partial<AIRequestLog>) => void;
|
|
||||||
addRequestStep: (logId: string, step: Omit<AIStepLog, 'timestamp'>) => void;
|
|
||||||
addResponseStep: (logId: string, step: Omit<AIStepLog, 'timestamp'>) => void;
|
|
||||||
clearLogs: () => void;
|
|
||||||
maxLogs: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const useAIRequestLogsStore = create<AIRequestLogsStore>((set, get) => ({
|
|
||||||
logs: [],
|
|
||||||
maxLogs: 20, // Keep last 20 logs
|
|
||||||
|
|
||||||
addLog: (log) => {
|
|
||||||
const newLog: AIRequestLog = {
|
|
||||||
...log,
|
|
||||||
id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
|
|
||||||
timestamp: new Date(),
|
|
||||||
requestSteps: [],
|
|
||||||
responseSteps: [],
|
|
||||||
};
|
|
||||||
|
|
||||||
set((state) => {
|
|
||||||
const updatedLogs = [newLog, ...state.logs].slice(0, state.maxLogs);
|
|
||||||
return { logs: updatedLogs };
|
|
||||||
});
|
|
||||||
|
|
||||||
// Return the log ID so callers can add steps
|
|
||||||
return newLog.id;
|
|
||||||
},
|
|
||||||
|
|
||||||
updateLog: (logId, updates) => {
|
|
||||||
set((state) => {
|
|
||||||
const updatedLogs = state.logs.map((log) => {
|
|
||||||
if (log.id === logId) {
|
|
||||||
return { ...log, ...updates };
|
|
||||||
}
|
|
||||||
return log;
|
|
||||||
});
|
|
||||||
return { logs: updatedLogs };
|
|
||||||
});
|
|
||||||
},
|
|
||||||
|
|
||||||
addRequestStep: (logId, step) => {
|
|
||||||
set((state) => {
|
|
||||||
const updatedLogs = state.logs.map((log) => {
|
|
||||||
if (log.id === logId) {
|
|
||||||
const stepWithTimestamp: AIStepLog = {
|
|
||||||
...step,
|
|
||||||
timestamp: new Date(),
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
...log,
|
|
||||||
requestSteps: [...log.requestSteps, stepWithTimestamp],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return log;
|
|
||||||
});
|
|
||||||
return { logs: updatedLogs };
|
|
||||||
});
|
|
||||||
},
|
|
||||||
|
|
||||||
addResponseStep: (logId, step) => {
|
|
||||||
set((state) => {
|
|
||||||
const updatedLogs = state.logs.map((log) => {
|
|
||||||
if (log.id === logId) {
|
|
||||||
const stepWithTimestamp: AIStepLog = {
|
|
||||||
...step,
|
|
||||||
timestamp: new Date(),
|
|
||||||
};
|
|
||||||
return {
|
|
||||||
...log,
|
|
||||||
responseSteps: [...log.responseSteps, stepWithTimestamp],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return log;
|
|
||||||
});
|
|
||||||
return { logs: updatedLogs };
|
|
||||||
});
|
|
||||||
},
|
|
||||||
|
|
||||||
clearLogs: () => {
|
|
||||||
set({ logs: [] });
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
|
|
||||||
Reference in New Issue
Block a user