salman/igny8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

alot of othe mess fro autoamtion overview an ddetiaeld run apge sonly

Browse Source
This commit is contained in:
IGNY8 VPS (Salman)
2026-01-17 10:13:08 +00:00
parent 6b1fa0c1ee
commit 0435a5cf70
14 changed files with 1727 additions and 470 deletions
Download Patch File Download Diff File Expand all files Collapse all files

416
backend/igny8_core/business/automation/views.py
View File

@@ -312,20 +312,43 @@ class AutomationViewSet(viewsets.ViewSet):
def _calculate_historical_averages(self, site, completed_runs):
"""Calculate historical averages from completed runs"""
if completed_runs.count() < 3:
# Not enough data, return defaults
run_count = completed_runs.count()
default_stage_averages = {
1: {'avg_credits': 0.2, 'avg_items_created': 0, 'avg_output_ratio': 0.125},
2: {'avg_credits': 2.0, 'avg_items_created': 0, 'avg_output_ratio': 8.7},
3: {'avg_credits': 0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
4: {'avg_credits': 5.0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
5: {'avg_credits': 2.0, 'avg_items_created': 0, 'avg_output_ratio': 4.0},
6: {'avg_credits': 2.0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
7: {'avg_credits': 0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
}
if run_count < 3:
return {
'period_days': 30,
'runs_analyzed': completed_runs.count(),
'avg_credits_stage_1': 0.2,
'avg_credits_stage_2': 2.0,
'avg_credits_stage_4': 5.0,
'avg_credits_stage_5': 2.0,
'avg_credits_stage_6': 2.0,
'avg_output_ratio_stage_1': 0.125,
'avg_output_ratio_stage_2': 8.7,
'avg_output_ratio_stage_5': 4.0,
'avg_output_ratio_stage_6': 1.0,
'runs_analyzed': run_count,
'avg_total_credits': 0,
'avg_duration_seconds': 0,
'avg_credits_per_item': 0,
'total_runs_analyzed': run_count,
'has_sufficient_data': False,
'stages': [
{
'stage_number': stage_number,
'stage_name': f"Stage {stage_number}",
**averages,
}
for stage_number, averages in default_stage_averages.items()
],
'avg_credits_stage_1': default_stage_averages[1]['avg_credits'],
'avg_credits_stage_2': default_stage_averages[2]['avg_credits'],
'avg_credits_stage_4': default_stage_averages[4]['avg_credits'],
'avg_credits_stage_5': default_stage_averages[5]['avg_credits'],
'avg_credits_stage_6': default_stage_averages[6]['avg_credits'],
'avg_output_ratio_stage_1': default_stage_averages[1]['avg_output_ratio'],
'avg_output_ratio_stage_2': default_stage_averages[2]['avg_output_ratio'],
'avg_output_ratio_stage_5': default_stage_averages[5]['avg_output_ratio'],
'avg_output_ratio_stage_6': default_stage_averages[6]['avg_output_ratio'],
}
# Calculate per-stage averages
@@ -340,6 +363,9 @@ class AutomationViewSet(viewsets.ViewSet):
output_ratios_5 = []
output_ratios_6 = []
total_created_items = 0
total_credits_used = 0
for run in completed_runs[:10]: # Last 10 runs
if run.stage_1_result:
processed = run.stage_1_result.get('keywords_processed', 0)
@@ -349,6 +375,8 @@ class AutomationViewSet(viewsets.ViewSet):
stage_1_credits.append(credits / processed)
if created > 0 and processed > 0:
output_ratios_1.append(created / processed)
total_created_items += created
total_credits_used += credits
if run.stage_2_result:
processed = run.stage_2_result.get('clusters_processed', 0)
@@ -358,12 +386,16 @@ class AutomationViewSet(viewsets.ViewSet):
stage_2_credits.append(credits / processed)
if created > 0 and processed > 0:
output_ratios_2.append(created / processed)
total_created_items += created
total_credits_used += credits
if run.stage_4_result:
processed = run.stage_4_result.get('tasks_processed', 0)
credits = run.stage_4_result.get('credits_used', 0)
if processed > 0:
stage_4_credits.append(credits / processed)
total_created_items += run.stage_4_result.get('content_created', 0)
total_credits_used += credits
if run.stage_5_result:
processed = run.stage_5_result.get('content_processed', 0)
@@ -373,6 +405,8 @@ class AutomationViewSet(viewsets.ViewSet):
stage_5_credits.append(credits / processed)
if created > 0 and processed > 0:
output_ratios_5.append(created / processed)
total_created_items += created
total_credits_used += credits
if run.stage_6_result:
processed = run.stage_6_result.get('images_processed', 0)
@@ -382,22 +416,57 @@ class AutomationViewSet(viewsets.ViewSet):
stage_6_credits.append(credits / processed)
if created > 0 and processed > 0:
output_ratios_6.append(created / processed)
total_created_items += created
total_credits_used += credits
def avg(lst):
return sum(lst) / len(lst) if lst else 0
avg_total_credits = completed_runs.aggregate(avg=Avg('total_credits_used'))['avg'] or 0
avg_duration = completed_runs.annotate(
duration=F('completed_at') - F('started_at')
).aggregate(avg=Avg('duration'))['avg']
avg_duration_seconds = int(avg_duration.total_seconds()) if avg_duration else 0
derived_stage_averages = {
1: {'avg_credits': round(avg(stage_1_credits), 2), 'avg_items_created': 0, 'avg_output_ratio': round(avg(output_ratios_1), 3)},
2: {'avg_credits': round(avg(stage_2_credits), 2), 'avg_items_created': 0, 'avg_output_ratio': round(avg(output_ratios_2), 1)},
3: {'avg_credits': 0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
4: {'avg_credits': round(avg(stage_4_credits), 2), 'avg_items_created': 0, 'avg_output_ratio': 1.0},
5: {'avg_credits': round(avg(stage_5_credits), 2), 'avg_items_created': 0, 'avg_output_ratio': round(avg(output_ratios_5), 1)},
6: {'avg_credits': round(avg(stage_6_credits), 2), 'avg_items_created': 0, 'avg_output_ratio': round(avg(output_ratios_6), 1)},
7: {'avg_credits': 0, 'avg_items_created': 0, 'avg_output_ratio': 1.0},
}
avg_credits_per_item = 0
if total_created_items > 0:
avg_credits_per_item = total_credits_used / total_created_items
return {
'period_days': 30,
'runs_analyzed': min(completed_runs.count(), 10),
'avg_credits_stage_1': round(avg(stage_1_credits), 2),
'avg_credits_stage_2': round(avg(stage_2_credits), 2),
'avg_credits_stage_4': round(avg(stage_4_credits), 2),
'avg_credits_stage_5': round(avg(stage_5_credits), 2),
'avg_credits_stage_6': round(avg(stage_6_credits), 2),
'avg_output_ratio_stage_1': round(avg(output_ratios_1), 3),
'avg_output_ratio_stage_2': round(avg(output_ratios_2), 1),
'avg_output_ratio_stage_5': round(avg(output_ratios_5), 1),
'avg_output_ratio_stage_6': round(avg(output_ratios_6), 1),
'runs_analyzed': min(run_count, 10),
'avg_total_credits': round(avg_total_credits, 1),
'avg_duration_seconds': avg_duration_seconds,
'avg_credits_per_item': round(avg_credits_per_item, 2),
'total_runs_analyzed': min(run_count, 10),
'has_sufficient_data': run_count >= 3,
'stages': [
{
'stage_number': stage_number,
'stage_name': f"Stage {stage_number}",
**averages,
}
for stage_number, averages in derived_stage_averages.items()
],
'avg_credits_stage_1': derived_stage_averages[1]['avg_credits'],
'avg_credits_stage_2': derived_stage_averages[2]['avg_credits'],
'avg_credits_stage_4': derived_stage_averages[4]['avg_credits'],
'avg_credits_stage_5': derived_stage_averages[5]['avg_credits'],
'avg_credits_stage_6': derived_stage_averages[6]['avg_credits'],
'avg_output_ratio_stage_1': derived_stage_averages[1]['avg_output_ratio'],
'avg_output_ratio_stage_2': derived_stage_averages[2]['avg_output_ratio'],
'avg_output_ratio_stage_5': derived_stage_averages[5]['avg_output_ratio'],
'avg_output_ratio_stage_6': derived_stage_averages[6]['avg_output_ratio'],
}
def _calculate_predictive_analysis(self, site, historical_averages):
@@ -430,86 +499,62 @@ class AutomationViewSet(viewsets.ViewSet):
return {
'stages': [
{
'stage': 1,
'name': 'Keywords → Clusters',
'stage_number': 1,
'stage_name': 'Keywords → Clusters',
'pending_items': pending_keywords,
'avg_credits_per_item': historical_averages['avg_credits_stage_1'],
'estimated_credits': stage_1_credits,
'avg_output_ratio': historical_averages['avg_output_ratio_stage_1'],
'estimated_output': expected_clusters,
'output_type': 'clusters'
},
{
'stage': 2,
'name': 'Clusters → Ideas',
'stage_number': 2,
'stage_name': 'Clusters → Ideas',
'pending_items': pending_clusters,
'avg_credits_per_item': historical_averages['avg_credits_stage_2'],
'estimated_credits': stage_2_credits,
'avg_output_ratio': historical_averages['avg_output_ratio_stage_2'],
'estimated_output': expected_ideas,
'output_type': 'ideas'
},
{
'stage': 3,
'name': 'Ideas → Tasks',
'stage_number': 3,
'stage_name': 'Ideas → Tasks',
'pending_items': pending_ideas,
'avg_credits_per_item': 0,
'estimated_credits': 0,
'avg_output_ratio': 1.0,
'estimated_output': pending_ideas,
'output_type': 'tasks'
},
{
'stage': 4,
'name': 'Tasks → Content',
'stage_number': 4,
'stage_name': 'Tasks → Content',
'pending_items': pending_tasks,
'avg_credits_per_item': historical_averages['avg_credits_stage_4'],
'estimated_credits': stage_4_credits,
'avg_output_ratio': 1.0,
'estimated_output': pending_tasks,
'output_type': 'content'
},
{
'stage': 5,
'name': 'Content → Image Prompts',
'stage_number': 5,
'stage_name': 'Content → Image Prompts',
'pending_items': pending_content,
'avg_credits_per_item': historical_averages['avg_credits_stage_5'],
'estimated_credits': stage_5_credits,
'avg_output_ratio': historical_averages['avg_output_ratio_stage_5'],
'estimated_output': expected_prompts,
'output_type': 'prompts'
},
{
'stage': 6,
'name': 'Image Prompts → Images',
'stage_number': 6,
'stage_name': 'Image Prompts → Images',
'pending_items': pending_images,
'avg_credits_per_item': historical_averages['avg_credits_stage_6'],
'estimated_credits': stage_6_credits,
'avg_output_ratio': historical_averages['avg_output_ratio_stage_6'],
'estimated_output': expected_images,
'output_type': 'images'
},
{
'stage': 7,
'name': 'Review → Approved',
'stage_number': 7,
'stage_name': 'Review → Approved',
'pending_items': pending_review,
'avg_credits_per_item': 0,
'estimated_credits': 0,
'avg_output_ratio': 1.0,
'estimated_output': pending_review,
'output_type': 'approved'
},
],
'total_estimated_credits': total_estimated,
'recommended_buffer': recommended_buffer,
'current_balance': site.account.credits,
'is_sufficient': site.account.credits >= recommended_buffer,
'expected_outputs': {
'clusters': expected_clusters,
'ideas': expected_ideas,
'content': pending_tasks,
'images': expected_images,
}
'totals': {
'total_pending_items': pending_keywords + pending_clusters + pending_ideas + pending_tasks + pending_content + pending_images + pending_review,
'total_estimated_credits': total_estimated,
'total_estimated_output': expected_clusters + expected_ideas + pending_tasks + expected_images + pending_review,
'recommended_buffer_credits': recommended_buffer,
},
'confidence': 'high' if historical_averages.get('has_sufficient_data') else 'low',
}
def _get_attention_items(self, site):
@@ -551,7 +596,7 @@ class AutomationViewSet(viewsets.ViewSet):
failed_runs = recent_runs.filter(status='failed')
# Calculate averages from completed runs
avg_duration = completed_runs.annotate(
avg_duration = this_week_runs.filter(status='completed').annotate(
duration=F('completed_at') - F('started_at')
).aggregate(avg=Avg('duration'))['avg']
@@ -578,12 +623,11 @@ class AutomationViewSet(viewsets.ViewSet):
'total_runs': all_runs.count(),
'completed_runs': completed_runs.count(),
'failed_runs': failed_runs.count(),
'success_rate': round(completed_runs.count() / recent_runs.count() * 100, 1) if recent_runs.count() > 0 else 0,
'avg_duration_seconds': int(avg_duration.total_seconds()) if avg_duration else 0,
'running_runs': all_runs.filter(status__in=['running', 'paused']).count(),
'total_credits_used': all_runs.aggregate(total=Sum('total_credits_used'))['total'] or 0,
'total_credits_last_30_days': recent_runs.aggregate(total=Sum('total_credits_used'))['total'] or 0,
'avg_credits_per_run': round(avg_credits, 1),
'runs_this_week': this_week_runs.count(),
'runs_last_week': last_week_runs.count(),
'credits_trend': credits_trend,
'avg_duration_last_7_days_seconds': int(avg_duration.total_seconds()) if avg_duration else 0,
},
'predictive_analysis': predictive_analysis,
'attention_items': attention_items,
@@ -1151,6 +1195,232 @@ class AutomationViewSet(viewsets.ViewSet):
'message': None if is_eligible else 'This site has no data yet. Add keywords in the Planner module to get started with automation.'
})
@extend_schema(tags=['Automation'])
@action(detail=False, methods=['get'], url_path='trend_data')
def trend_data(self, request):
"""
GET /api/v1/automation/trend_data/?site_id=123&limit=10
Get trend data for credits usage visualization
Returns last N runs with credits and output metrics
"""
site, error_response = self._get_site(request)
if error_response:
return error_response
limit = int(request.query_params.get('limit', 10))
limit = min(limit, 50) # Cap at 50 runs
runs = AutomationRun.objects.filter(site=site).order_by('-started_at')[:limit]
trend_data = []
for run in reversed(list(runs)): # Oldest first for chart
run_number = self._calculate_run_number(site, run)
# Calculate items created from stage results
items_created = 0
if run.stage_1_result:
items_created += run.stage_1_result.get('clusters_created', 0)
if run.stage_2_result:
items_created += run.stage_2_result.get('ideas_created', 0)
if run.stage_4_result:
items_created += run.stage_4_result.get('content_created', 0)
if run.stage_6_result:
items_created += run.stage_6_result.get('images_generated', 0)
trend_data.append({
'run_id': run.run_id,
'run_number': run_number,
'credits_used': run.total_credits_used,
'items_created': items_created,
'date': run.started_at.isoformat() if run.started_at else None,
'status': run.status,
})
# Calculate summary stats
total_credits = sum(d['credits_used'] for d in trend_data)
total_items = sum(d['items_created'] for d in trend_data)
avg_credits = total_credits / len(trend_data) if trend_data else 0
return Response({
'trend_data': trend_data,
'summary': {
'total_runs': len(trend_data),
'total_credits': total_credits,
'total_items': total_items,
'avg_credits_per_run': round(avg_credits, 1),
'avg_credits_per_item': round(total_credits / total_items, 2) if total_items > 0 else 0,
}
})
@extend_schema(tags=['Automation'])
@action(detail=False, methods=['get'], url_path='production_stats')
def production_stats(self, request):
"""
GET /api/v1/automation/production_stats/?site_id=123
Get actual production statistics - what was really created across all runs
"""
site, error_response = self._get_site(request)
if error_response:
return error_response
# Get actual entity counts from database (ground truth)
from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas
from igny8_core.business.content.models import Tasks, Content, Images
actual_counts = {
'keywords': Keywords.objects.filter(site=site).count(),
'clusters': Clusters.objects.filter(site=site).count(),
'ideas': ContentIdeas.objects.filter(site=site).count(),
'tasks': Tasks.objects.filter(site=site).count(),
'content': Content.objects.filter(site=site).count(),
'images': Images.objects.filter(site=site).count(),
}
# Get all runs for this site
all_runs = AutomationRun.objects.filter(site=site)
# Aggregate actual production from stage results
totals = {
'total_runs': all_runs.count(),
'runs_with_output': 0,
'total_credits': 0,
# Use actual database counts for current state
'clusters_total': actual_counts['clusters'],
'ideas_total': actual_counts['ideas'],
'content_total': actual_counts['content'],
'images_total': actual_counts['images'],
# Track what was created via automation (from run results)
'clusters_created': 0,
'ideas_created': 0,
'content_created': 0,
'images_created': 0,
'approved_via_automation': 0,
}
# Build meaningful runs list (credits > 0 or output > 0)
meaningful_runs = []
for run in all_runs.order_by('-started_at')[:15]: # Last 15 runs
run_data = {
'run_id': run.run_id,
'run_number': self._calculate_run_number(site, run),
'status': run.status,
'started_at': run.started_at.isoformat() if run.started_at else None,
'duration_seconds': 0,
'total_credits': run.total_credits_used,
'stages': [],
}
if run.completed_at and run.started_at:
run_data['duration_seconds'] = int((run.completed_at - run.started_at).total_seconds())
totals['total_credits'] += run.total_credits_used
has_output = False
# Stage 1: Keywords → Clusters
if run.stage_1_result:
inp = run.stage_1_result.get('keywords_processed', 0)
out = run.stage_1_result.get('clusters_created', 0)
cr = run.stage_1_result.get('credits_used', 0)
totals['clusters_created'] += out
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 1, 'name': 'Keywords→Clusters',
'input': inp, 'output': out, 'credits': cr
})
# Stage 2: Clusters → Ideas
if run.stage_2_result:
inp = run.stage_2_result.get('clusters_processed', 0)
out = run.stage_2_result.get('ideas_created', 0)
cr = run.stage_2_result.get('credits_used', 0)
totals['ideas_created'] += out
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 2, 'name': 'Clusters→Ideas',
'input': inp, 'output': out, 'credits': cr
})
# Stage 3: Ideas → Tasks (1:1 always)
if run.stage_3_result:
out = run.stage_3_result.get('tasks_created', 0)
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 3, 'name': 'Ideas→Tasks',
'input': out, 'output': out, 'credits': 0
})
# Stage 4: Tasks → Content (1:1 always)
if run.stage_4_result:
out = run.stage_4_result.get('content_created', 0)
cr = run.stage_4_result.get('credits_used', 0)
totals['content_created'] += out
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 4, 'name': 'Tasks→Content',
'input': out, 'output': out, 'credits': cr
})
# Stage 5: Content → Prompts (can be multiple prompts per content)
if run.stage_5_result:
inp = run.stage_5_result.get('content_processed', 0)
out = run.stage_5_result.get('prompts_created', 0)
cr = run.stage_5_result.get('credits_used', 0)
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 5, 'name': 'Content→Prompts',
'input': inp, 'output': out, 'credits': cr
})
# Stage 6: Prompts → Images
if run.stage_6_result:
inp = run.stage_6_result.get('images_processed', 0)
out = run.stage_6_result.get('images_generated', 0)
cr = run.stage_6_result.get('credits_used', 0)
totals['images_created'] += out
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 6, 'name': 'Prompts→Images',
'input': inp, 'output': out, 'credits': cr
})
# Stage 7: Review → Approved
if run.stage_7_result:
out = run.stage_7_result.get('approved_count', 0)
totals['approved_via_automation'] += out
if out > 0:
has_output = True
run_data['stages'].append({
'stage': 7, 'name': 'Review→Approved',
'input': run.stage_7_result.get('ready_for_review', 0) or run.stage_7_result.get('review_total', 0),
'output': out, 'credits': 0
})
# Add to meaningful runs if has output or credits
if has_output or run.total_credits_used > 0:
totals['runs_with_output'] += 1
meaningful_runs.append(run_data)
# Calculate efficiency metrics using actual counts
total_created = totals['clusters_created'] + totals['ideas_created'] + totals['content_created'] + totals['images_created']
credits_per_item = round(totals['total_credits'] / total_created, 2) if total_created > 0 else 0
return Response({
'totals': totals,
'actual_counts': actual_counts,
'efficiency': {
'total_items_created': total_created,
'credits_per_item': credits_per_item,
},
'meaningful_runs': meaningful_runs[:10], # Top 10 most recent meaningful runs
})
@extend_schema(tags=['Automation'])
@action(detail=False, methods=['get'], url_path='current_processing')
def current_processing(self, request):