salman/igny8
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI AUtomtaion, Schudelign and publishign fromt and backe end refoactr

Browse Source
This commit is contained in:
IGNY8 VPS (Salman)
2026-01-17 15:52:46 +00:00
parent 0435a5cf70
commit d3b3e1c0d4
34 changed files with 4715 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
backend/igny8_core/business/automation/migrations/0008_automationconfig_max_approvals_per_run_and_more.py Normal file
View File

@@ -0,0 +1,53 @@
# Generated by Django 5.2.10 on 2026-01-17 14:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('automation', '0007_add_stage_enabled_toggles'),
]
operations = [
migrations.AddField(
model_name='automationconfig',
name='max_approvals_per_run',
field=models.IntegerField(default=0, help_text='Max content pieces to auto-approve in stage 7 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_clusters_per_run',
field=models.IntegerField(default=0, help_text='Max clusters to process in stage 2 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_content_per_run',
field=models.IntegerField(default=0, help_text='Max content pieces for image prompts in stage 5 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_credits_per_run',
field=models.IntegerField(default=0, help_text='Max credits to use per run (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_ideas_per_run',
field=models.IntegerField(default=0, help_text='Max ideas to process in stage 3 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_images_per_run',
field=models.IntegerField(default=0, help_text='Max images to generate in stage 6 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_keywords_per_run',
field=models.IntegerField(default=0, help_text='Max keywords to process in stage 1 (0=unlimited)'),
),
migrations.AddField(
model_name='automationconfig',
name='max_tasks_per_run',
field=models.IntegerField(default=0, help_text='Max tasks to process in stage 4 (0=unlimited)'),
),
]

13
backend/igny8_core/business/automation/models.py
View File

@@ -44,6 +44,19 @@ class AutomationConfig(models.Model):
within_stage_delay = models.IntegerField(default=3, help_text="Delay between batches within a stage (seconds)")
between_stage_delay = models.IntegerField(default=5, help_text="Delay between stage transitions (seconds)")
# Per-run item limits (0 = unlimited, processes all available)
# These prevent runaway automation and control resource usage
max_keywords_per_run = models.IntegerField(default=0, help_text="Max keywords to process in stage 1 (0=unlimited)")
max_clusters_per_run = models.IntegerField(default=0, help_text="Max clusters to process in stage 2 (0=unlimited)")
max_ideas_per_run = models.IntegerField(default=0, help_text="Max ideas to process in stage 3 (0=unlimited)")
max_tasks_per_run = models.IntegerField(default=0, help_text="Max tasks to process in stage 4 (0=unlimited)")
max_content_per_run = models.IntegerField(default=0, help_text="Max content pieces for image prompts in stage 5 (0=unlimited)")
max_images_per_run = models.IntegerField(default=0, help_text="Max images to generate in stage 6 (0=unlimited)")
max_approvals_per_run = models.IntegerField(default=0, help_text="Max content pieces to auto-approve in stage 7 (0=unlimited)")
# Credit budget limit per run (0 = use site's full credit balance)
max_credits_per_run = models.IntegerField(default=0, help_text="Max credits to use per run (0=unlimited)")
last_run_at = models.DateTimeField(null=True, blank=True)
next_run_at = models.DateTimeField(null=True, blank=True, help_text="Calculated based on frequency")

158
backend/igny8_core/business/automation/services/automation_service.py
View File

@@ -63,7 +63,7 @@ class AutomationService:
def _check_should_stop(self) -> tuple[bool, str]:
"""
Check if automation should stop (paused or cancelled)
Check if automation should stop (paused, cancelled, or credit budget exceeded)
Returns:
(should_stop, reason)
@@ -79,6 +79,83 @@ class AutomationService:
elif self.run.status == 'cancelled':
return True, "cancelled"
# Check credit budget
budget_exceeded, budget_reason = self._check_credit_budget()
if budget_exceeded:
return True, f"credit_budget_exceeded: {budget_reason}"
return False, ""
def _get_per_run_limit(self, stage: int) -> int:
"""
Get the per-run item limit for a stage from config.
Args:
stage: Stage number (1-7)
Returns:
Max items to process (0 = unlimited)
"""
limit_map = {
1: self.config.max_keywords_per_run,
2: self.config.max_clusters_per_run,
3: self.config.max_ideas_per_run,
4: self.config.max_tasks_per_run,
5: self.config.max_content_per_run,
6: self.config.max_images_per_run,
7: self.config.max_approvals_per_run,
}
return limit_map.get(stage, 0)
def _apply_per_run_limit(self, queryset, stage: int, log_prefix: str = ""):
"""
Apply per-run limit to queryset if configured.
Args:
queryset: Django queryset to limit
stage: Stage number (1-7)
log_prefix: Prefix for log messages
Returns:
Limited queryset (or list if limit applied)
"""
limit = self._get_per_run_limit(stage)
if limit > 0:
total = queryset.count()
if total > limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage, f"{log_prefix}Applying per-run limit: {limit} of {total} items (limit set in automation config)"
)
return list(queryset[:limit])
return queryset
def _check_credit_budget(self) -> tuple[bool, str]:
"""
Check if credit budget for this run has been exceeded.
Returns:
(exceeded, reason) - If exceeded is True, automation should stop
"""
if not self.run or not self.config:
return False, ""
max_credits = self.config.max_credits_per_run
if max_credits <= 0: # 0 = unlimited
return False, ""
credits_used = self._get_credits_used()
if credits_used >= max_credits:
reason = f"Credit budget exhausted: {credits_used}/{max_credits} credits used"
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
self.run.current_stage, reason
)
return True, reason
return False, ""
def start_automation(self, trigger_type: str = 'manual') -> str:
@@ -170,6 +247,19 @@ class AutomationService:
disabled=False
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = pending_keywords.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} keywords"
)
# Get limited keyword IDs first, then filter queryset
limited_ids = list(pending_keywords.values_list('id', flat=True)[:per_run_limit])
pending_keywords = pending_keywords.filter(id__in=limited_ids)
total_count = pending_keywords.count()
# IMPORTANT: Group keywords by sector to avoid mixing sectors in clustering
@@ -480,6 +570,17 @@ class AutomationService:
disabled=False
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = pending_clusters.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} clusters"
)
pending_clusters = pending_clusters[:per_run_limit]
total_count = pending_clusters.count()
# Log stage start
@@ -674,6 +775,17 @@ class AutomationService:
status='new'
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = pending_ideas.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} ideas"
)
pending_ideas = pending_ideas[:per_run_limit]
total_count = pending_ideas.count()
# Log stage start
@@ -837,6 +949,17 @@ class AutomationService:
status='queued'
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = pending_tasks.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} tasks"
)
pending_tasks = pending_tasks[:per_run_limit]
total_count = pending_tasks.count()
# Log stage start
@@ -1078,6 +1201,17 @@ class AutomationService:
images_count=0
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = content_without_images.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} content items"
)
content_without_images = content_without_images[:per_run_limit]
total_count = content_without_images.count()
# ADDED: Enhanced logging
@@ -1291,6 +1425,17 @@ class AutomationService:
status='pending'
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = pending_images.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Processing {per_run_limit} of {total_available} images"
)
pending_images = pending_images[:per_run_limit]
total_count = pending_images.count()
# Log stage start
@@ -1538,6 +1683,17 @@ class AutomationService:
status='review'
)
# Apply per-run limit (0 = unlimited)
per_run_limit = self._get_per_run_limit(stage_number)
total_available = ready_for_review.count()
if per_run_limit > 0 and total_available > per_run_limit:
self.logger.log_stage_progress(
self.run.run_id, self.account.id, self.site.id,
stage_number, f"Per-run limit: Approving {per_run_limit} of {total_available} content items"
)
ready_for_review = ready_for_review[:per_run_limit]
total_count = ready_for_review.count()
# Log stage start

76
backend/igny8_core/business/automation/tasks.py
View File

@@ -49,9 +49,9 @@ def check_scheduled_automations():
logger.info(f"[AutomationTask] Skipping site {config.site.id} - already ran today")
continue
# Check if already running
if AutomationRun.objects.filter(site=config.site, status='running').exists():
logger.info(f"[AutomationTask] Skipping site {config.site.id} - already running")
# Check if already running OR paused (don't start new if existing in progress)
if AutomationRun.objects.filter(site=config.site, status__in=['running', 'paused']).exists():
logger.info(f"[AutomationTask] Skipping site {config.site.id} - automation in progress (running/paused)")
continue
logger.info(f"[AutomationTask] Starting scheduled automation for site {config.site.id}")
@@ -162,13 +162,50 @@ def run_automation_task(self, run_id: str):
@shared_task(name='automation.resume_automation_task', bind=True, max_retries=0)
def resume_automation_task(self, run_id: str):
"""
Resume paused automation run from current stage
Resume paused automation run from current stage.
CRITICAL FIXES:
- Verifies run status is 'running' before processing
- Reacquires lock in case it expired during long pause
- Checks pause/cancel status after each stage
- Releases lock on failure
"""
logger.info(f"[AutomationTask] Resuming automation run: {run_id}")
try:
from django.core.cache import cache
# Load run and verify status
run = AutomationRun.objects.get(run_id=run_id)
# CRITICAL FIX: Verify run is actually in 'running' status
# (status is set to 'running' by views.resume before calling this task)
if run.status != 'running':
logger.warning(f"[AutomationTask] Run {run_id} status is '{run.status}', not 'running'. Aborting resume.")
return
# CRITICAL FIX: Reacquire lock in case it expired during long pause (6hr timeout)
lock_key = f'automation_lock_{run.site.id}'
lock_acquired = cache.add(lock_key, run_id, timeout=21600) # 6 hours
if not lock_acquired:
# Lock exists - check if it's ours (from original run start)
existing_lock = cache.get(lock_key)
# If lock exists but isn't our run_id, another run may have started
if existing_lock and existing_lock != run_id and existing_lock != 'locked':
logger.warning(f"[AutomationTask] Lock held by different run ({existing_lock}). Aborting resume for {run_id}")
run.status = 'failed'
run.error_message = f'Lock acquired by another run ({existing_lock}) during pause'
run.completed_at = timezone.now()
run.save()
return
# Lock exists and is either 'locked' (our old format) or our run_id - proceed
logger.info(f"[AutomationTask] Existing lock found, proceeding with resume")
else:
# We acquired a new lock (old one expired)
logger.info(f"[AutomationTask] Reacquired lock after expiry for run {run_id}")
service = AutomationService.from_run_id(run_id)
run = service.run
config = service.config
# Continue from current stage
@@ -196,20 +233,35 @@ def resume_automation_task(self, run_id: str):
for stage in range(run.current_stage - 1, 7):
if stage_enabled[stage]:
stage_methods[stage]()
# CRITICAL FIX: Check for pause/cancel AFTER each stage (same as run_automation_task)
service.run.refresh_from_db()
if service.run.status in ['paused', 'cancelled']:
logger.info(f"[AutomationTask] Resumed automation {service.run.status} after stage {stage + 1}")
return
else:
logger.info(f"[AutomationTask] Stage {stage + 1} is disabled, skipping")
logger.info(f"[AutomationTask] Resumed automation run: {run_id}")
logger.info(f"[AutomationTask] Resumed automation completed: {run_id}")
except Exception as e:
logger.error(f"[AutomationTask] Failed to resume automation run {run_id}: {e}")
# Mark as failed
run = AutomationRun.objects.get(run_id=run_id)
run.status = 'failed'
run.error_message = str(e)
run.completed_at = timezone.now()
run.save()
# Mark as failed and release lock
try:
run = AutomationRun.objects.get(run_id=run_id)
run.status = 'failed'
run.error_message = str(e)
run.completed_at = timezone.now()
run.save()
# Release lock on failure
from django.core.cache import cache
cache.delete(f'automation_lock_{run.site.id}')
except Exception as cleanup_err:
logger.error(f"[AutomationTask] Failed to cleanup after resume failure: {cleanup_err}")
raise
# Alias for continue_automation_task (same as resume)

57
backend/igny8_core/business/automation/views.py
View File

@@ -77,6 +77,15 @@ class AutomationViewSet(viewsets.ViewSet):
'stage_6_batch_size': config.stage_6_batch_size,
'within_stage_delay': config.within_stage_delay,
'between_stage_delay': config.between_stage_delay,
# Per-run limits (0 = unlimited)
'max_keywords_per_run': config.max_keywords_per_run,
'max_clusters_per_run': config.max_clusters_per_run,
'max_ideas_per_run': config.max_ideas_per_run,
'max_tasks_per_run': config.max_tasks_per_run,
'max_content_per_run': config.max_content_per_run,
'max_images_per_run': config.max_images_per_run,
'max_approvals_per_run': config.max_approvals_per_run,
'max_credits_per_run': config.max_credits_per_run,
'last_run_at': config.last_run_at,
'next_run_at': config.next_run_at,
})
@@ -153,6 +162,18 @@ class AutomationViewSet(viewsets.ViewSet):
except (TypeError, ValueError):
pass
# Per-run limits (0 = unlimited)
for field in ['max_keywords_per_run', 'max_clusters_per_run', 'max_ideas_per_run',
'max_tasks_per_run', 'max_content_per_run', 'max_images_per_run',
'max_approvals_per_run', 'max_credits_per_run']:
if field in request.data:
try:
value = int(request.data[field])
if value >= 0: # Allow 0 (unlimited) or positive numbers
setattr(config, field, value)
except (TypeError, ValueError):
pass
config.save()
return Response({
@@ -175,6 +196,15 @@ class AutomationViewSet(viewsets.ViewSet):
'stage_6_batch_size': config.stage_6_batch_size,
'within_stage_delay': config.within_stage_delay,
'between_stage_delay': config.between_stage_delay,
# Per-run limits (0 = unlimited)
'max_keywords_per_run': config.max_keywords_per_run,
'max_clusters_per_run': config.max_clusters_per_run,
'max_ideas_per_run': config.max_ideas_per_run,
'max_tasks_per_run': config.max_tasks_per_run,
'max_content_per_run': config.max_content_per_run,
'max_images_per_run': config.max_images_per_run,
'max_approvals_per_run': config.max_approvals_per_run,
'max_credits_per_run': config.max_credits_per_run,
'last_run_at': config.last_run_at,
'next_run_at': config.next_run_at,
})
@@ -267,6 +297,17 @@ class AutomationViewSet(viewsets.ViewSet):
try:
service = AutomationService.from_run_id(run_id)
service.pause_automation()
# CRITICAL FIX: Log pause to automation log files
try:
service.logger.log_stage_progress(
service.run.run_id, service.account.id, service.site.id,
service.run.current_stage, f"Automation paused by user at stage {service.run.current_stage}"
)
except Exception as log_err:
# Don't fail the pause if logging fails
pass
return Response({'message': 'Automation paused'})
except AutomationRun.DoesNotExist:
return Response(
@@ -1613,6 +1654,22 @@ class AutomationViewSet(viewsets.ViewSet):
run.completed_at = timezone.now()
run.save(update_fields=['status', 'cancelled_at', 'completed_at'])
# CRITICAL FIX: Release the lock so user can start new automation
from django.core.cache import cache
cache.delete(f'automation_lock_{run.site.id}')
# Log the cancellation to automation log files
try:
from igny8_core.business.automation.services.automation_logger import AutomationLogger
logger = AutomationLogger()
logger.log_stage_progress(
run.run_id, run.account.id, run.site.id, run.current_stage,
f"Automation cancelled by user at stage {run.current_stage}"
)
except Exception as log_err:
# Don't fail the cancellation if logging fails
pass
return Response({
'message': 'Automation cancelled',
'status': run.status,