Automation Part 1
This commit is contained in:
IGNY8 VPS (Salman)
299
AUTOMATION-DEPLOYMENT-CHECKLIST.md
Normal file
299
AUTOMATION-DEPLOYMENT-CHECKLIST.md
Normal file
@@ -0,0 +1,299 @@
|
||||
# Automation Implementation - Deployment Checklist
|
||||
|
||||
## ✅ Completed Components
|
||||
|
||||
### Backend
|
||||
- [x] Database models created (`AutomationConfig`, `AutomationRun`)
|
||||
- [x] AutomationLogger service (file-based logging)
|
||||
- [x] AutomationService orchestrator (7-stage pipeline)
|
||||
- [x] API endpoints (`AutomationViewSet`)
|
||||
- [x] Celery tasks (scheduled checks, run execution, resume)
|
||||
- [x] URL routing registered
|
||||
- [x] Celery beat schedule configured
|
||||
- [x] Migration file created
|
||||
|
||||
### Frontend
|
||||
- [x] TypeScript API service (`automationService.ts`)
|
||||
- [x] Main dashboard page (`AutomationPage.tsx`)
|
||||
- [x] StageCard component
|
||||
- [x] ActivityLog component
|
||||
- [x] ConfigModal component
|
||||
- [x] RunHistory component
|
||||
|
||||
### Documentation
|
||||
- [x] Comprehensive README (`AUTOMATION-IMPLEMENTATION-README.md`)
|
||||
- [x] Original plan corrected (`automation-plan.md`)
|
||||
|
||||
## ⏳ Remaining Tasks
|
||||
|
||||
### 1. Run Database Migration
|
||||
|
||||
```bash
|
||||
cd /data/app/igny8/backend
|
||||
python manage.py migrate
|
||||
```
|
||||
|
||||
This will create the `automation_config` and `automation_run` tables.
|
||||
|
||||
### 2. Register Frontend Route
|
||||
|
||||
Add to your React Router configuration:
|
||||
|
||||
```typescript
|
||||
import AutomationPage from './pages/Automation/AutomationPage';
|
||||
|
||||
// In your route definitions:
|
||||
{
|
||||
path: '/automation',
|
||||
element: <AutomationPage />,
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Add Navigation Link
|
||||
|
||||
Add link to main navigation menu:
|
||||
|
||||
```typescript
|
||||
{
|
||||
name: 'Automation',
|
||||
href: '/automation',
|
||||
icon: /* automation icon */
|
||||
}
|
||||
```
|
||||
|
||||
### 4. Verify Infrastructure
|
||||
|
||||
**Celery Worker**
|
||||
```bash
|
||||
# Check if running
|
||||
docker ps | grep celery
|
||||
|
||||
# Start if needed
|
||||
docker-compose up -d celery
|
||||
```
|
||||
|
||||
**Celery Beat**
|
||||
```bash
|
||||
# Check if running
|
||||
docker ps | grep beat
|
||||
|
||||
# Start if needed
|
||||
docker-compose up -d celery-beat
|
||||
```
|
||||
|
||||
**Redis/Cache**
|
||||
```bash
|
||||
# Verify cache backend in settings.py
|
||||
CACHES = {
|
||||
'default': {
|
||||
'BACKEND': 'django.core.cache.backends.redis.RedisCache',
|
||||
'LOCATION': 'redis://redis:6379/1',
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 5. Create Log Directory
|
||||
|
||||
```bash
|
||||
mkdir -p /data/app/igny8/backend/logs/automation
|
||||
chmod 755 /data/app/igny8/backend/logs/automation
|
||||
```
|
||||
|
||||
### 6. Test API Endpoints
|
||||
|
||||
```bash
|
||||
# Get config (should return default config)
|
||||
curl -X GET "http://localhost:8000/api/v1/automation/config/?site_id=1" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN"
|
||||
|
||||
# Estimate credits
|
||||
curl -X GET "http://localhost:8000/api/v1/automation/estimate/?site_id=1" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN"
|
||||
```
|
||||
|
||||
### 7. Test Frontend
|
||||
|
||||
1. Navigate to `/automation` page
|
||||
2. Click [Configure] - modal should open
|
||||
3. Save configuration
|
||||
4. Click [Run Now] - should trigger automation
|
||||
5. Verify real-time updates in stage cards
|
||||
6. Check activity log is streaming
|
||||
|
||||
### 8. Test Scheduled Automation
|
||||
|
||||
1. Enable automation in config
|
||||
2. Set scheduled time to 1 minute from now
|
||||
3. Wait for next hour (beat checks hourly at :00)
|
||||
4. Verify automation starts automatically
|
||||
|
||||
### 9. Monitor First Run
|
||||
|
||||
Watch logs in real-time:
|
||||
|
||||
```bash
|
||||
# Backend logs
|
||||
tail -f /data/app/igny8/backend/logs/automation/{account_id}/{site_id}/{run_id}/automation_run.log
|
||||
|
||||
# Celery worker logs
|
||||
docker logs -f <celery_container>
|
||||
|
||||
# Django logs
|
||||
docker logs -f <backend_container>
|
||||
```
|
||||
|
||||
### 10. Verify Database Records
|
||||
|
||||
```python
|
||||
from igny8_core.business.automation.models import AutomationConfig, AutomationRun
|
||||
|
||||
# Check config created
|
||||
AutomationConfig.objects.all()
|
||||
|
||||
# Check runs recorded
|
||||
AutomationRun.objects.all()
|
||||
|
||||
# View stage results
|
||||
run = AutomationRun.objects.latest('started_at')
|
||||
print(run.stage_1_result)
|
||||
print(run.stage_2_result)
|
||||
# ... etc
|
||||
```
|
||||
|
||||
## Quick Start Commands
|
||||
|
||||
```bash
|
||||
# 1. Run migration
|
||||
cd /data/app/igny8/backend
|
||||
python manage.py migrate
|
||||
|
||||
# 2. Create log directory
|
||||
mkdir -p logs/automation
|
||||
chmod 755 logs/automation
|
||||
|
||||
# 3. Restart services
|
||||
docker-compose restart celery celery-beat
|
||||
|
||||
# 4. Verify Celery beat schedule
|
||||
docker exec <celery_container> celery -A igny8_core inspect scheduled
|
||||
|
||||
# 5. Test automation (Django shell)
|
||||
python manage.py shell
|
||||
>>> from igny8_core.business.automation.services import AutomationService
|
||||
>>> from igny8_core.modules.system.models import Account, Site
|
||||
>>> account = Account.objects.first()
|
||||
>>> site = Site.objects.first()
|
||||
>>> service = AutomationService(account, site)
|
||||
>>> service.estimate_credits() # Should return number
|
||||
>>> # Don't run start_automation() yet - test via UI first
|
||||
```
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
### First Successful Run
|
||||
|
||||
1. **Stage 1**: Process keywords → create clusters (2-5 min)
|
||||
2. **Stage 2**: Generate ideas from clusters (1-2 min per cluster)
|
||||
3. **Stage 3**: Create tasks from ideas (instant)
|
||||
4. **Stage 4**: Generate content from tasks (3-5 min per task)
|
||||
5. **Stage 5**: Extract image prompts (1-2 min per content)
|
||||
6. **Stage 6**: Generate images (2-3 min per image)
|
||||
7. **Stage 7**: Count content ready for review (instant)
|
||||
|
||||
Total time: 15-45 minutes depending on batch sizes
|
||||
|
||||
### Stage Results Example
|
||||
|
||||
```json
|
||||
{
|
||||
"stage_1_result": {
|
||||
"keywords_processed": 20,
|
||||
"clusters_created": 4,
|
||||
"batches_run": 1,
|
||||
"credits_used": 4
|
||||
},
|
||||
"stage_2_result": {
|
||||
"clusters_processed": 4,
|
||||
"ideas_created": 16,
|
||||
"credits_used": 8
|
||||
},
|
||||
"stage_3_result": {
|
||||
"ideas_processed": 16,
|
||||
"tasks_created": 16,
|
||||
"batches_run": 1
|
||||
},
|
||||
"stage_4_result": {
|
||||
"tasks_processed": 16,
|
||||
"content_created": 16,
|
||||
"total_words": 40000,
|
||||
"credits_used": 80
|
||||
},
|
||||
"stage_5_result": {
|
||||
"content_processed": 16,
|
||||
"prompts_created": 64,
|
||||
"credits_used": 32
|
||||
},
|
||||
"stage_6_result": {
|
||||
"images_processed": 64,
|
||||
"images_generated": 64,
|
||||
"content_moved_to_review": 16,
|
||||
"credits_used": 128
|
||||
},
|
||||
"stage_7_result": {
|
||||
"ready_for_review": 16,
|
||||
"content_ids": [1, 2, 3, ...]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### "Module not found" errors
|
||||
- Restart Django server after adding new models
|
||||
- Run `python manage.py collectstatic` if needed
|
||||
|
||||
### "Table does not exist" errors
|
||||
- Run migration: `python manage.py migrate`
|
||||
|
||||
### "No module named automation"
|
||||
- Check `__init__.py` files exist in all directories
|
||||
- Verify imports in `urls.py`
|
||||
|
||||
### Celery tasks not running
|
||||
- Check worker is running: `docker ps | grep celery`
|
||||
- Check beat is running: `docker ps | grep beat`
|
||||
- Verify tasks registered: `celery -A igny8_core inspect registered`
|
||||
|
||||
### Logs not appearing
|
||||
- Check directory permissions: `ls -la logs/automation`
|
||||
- Check AutomationLogger.start_run() creates directories
|
||||
- Verify log file path in code matches actual filesystem
|
||||
|
||||
### Frontend errors
|
||||
- Check API service imported correctly
|
||||
- Verify route registered in router
|
||||
- Check for TypeScript compilation errors
|
||||
- Verify API endpoints returning expected data
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] Migration runs without errors
|
||||
- [ ] Frontend `/automation` page loads
|
||||
- [ ] Config modal opens and saves
|
||||
- [ ] Credit estimate shows reasonable number
|
||||
- [ ] "Run Now" starts automation successfully
|
||||
- [ ] Stage cards update in real-time
|
||||
- [ ] Activity log shows progress
|
||||
- [ ] All 7 stages complete successfully
|
||||
- [ ] Content moved to review status
|
||||
- [ ] Run History table shows completed run
|
||||
- [ ] Scheduled automation triggers at configured time
|
||||
|
||||
## Post-Deployment
|
||||
|
||||
1. Monitor first few runs closely
|
||||
2. Adjust batch sizes based on performance
|
||||
3. Set up alerts for failed runs
|
||||
4. Document any issues encountered
|
||||
5. Train users on automation features
|
||||
6. Gather feedback for improvements
|
||||
383
AUTOMATION-IMPLEMENTATION-README.md
Normal file
383
AUTOMATION-IMPLEMENTATION-README.md
Normal file
@@ -0,0 +1,383 @@
|
||||
# AI Automation Pipeline - Implementation Complete
|
||||
|
||||
## Overview
|
||||
|
||||
The IGNY8 AI Automation Pipeline is a fully automated content creation system that orchestrates existing AI functions into a 7-stage pipeline, transforming keywords into published content without manual intervention.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Backend Components
|
||||
|
||||
#### 1. Models (`/backend/igny8_core/business/automation/models.py`)
|
||||
|
||||
**AutomationConfig**
|
||||
- Per-site configuration for automation
|
||||
- Fields: `is_enabled`, `frequency` (daily/weekly/monthly), `scheduled_time`, batch sizes for all 7 stages
|
||||
- OneToOne relationship with Site model
|
||||
|
||||
**AutomationRun**
|
||||
- Tracks execution of automation runs
|
||||
- Fields: `run_id`, `status`, `current_stage`, `stage_1_result` through `stage_7_result` (JSON), `total_credits_used`
|
||||
- Status choices: running, paused, completed, failed
|
||||
|
||||
#### 2. Services
|
||||
|
||||
**AutomationLogger** (`services/automation_logger.py`)
|
||||
- File-based logging system
|
||||
- Log structure: `logs/automation/{account_id}/{site_id}/{run_id}/`
|
||||
- Files: `automation_run.log`, `stage_1.log` through `stage_7.log`
|
||||
- Methods: `start_run()`, `log_stage_start()`, `log_stage_progress()`, `log_stage_complete()`, `log_stage_error()`
|
||||
|
||||
**AutomationService** (`services/automation_service.py`)
|
||||
- Core orchestrator for automation pipeline
|
||||
- Methods:
|
||||
- `start_automation()` - Initialize new run with credit check
|
||||
- `run_stage_1()` through `run_stage_7()` - Execute each pipeline stage
|
||||
- `pause_automation()`, `resume_automation()` - Control run execution
|
||||
- `estimate_credits()` - Pre-run credit estimation
|
||||
- `from_run_id()` - Create service from existing run
|
||||
|
||||
#### 3. API Endpoints (`views.py`)
|
||||
|
||||
All endpoints at `/api/v1/automation/`:
|
||||
|
||||
- `GET /config/?site_id=123` - Get automation configuration
|
||||
- `PUT /update_config/?site_id=123` - Update configuration
|
||||
- `POST /run_now/?site_id=123` - Trigger immediate run
|
||||
- `GET /current_run/?site_id=123` - Get active run status
|
||||
- `POST /pause/?run_id=abc` - Pause running automation
|
||||
- `POST /resume/?run_id=abc` - Resume paused automation
|
||||
- `GET /history/?site_id=123` - Get past runs (last 20)
|
||||
- `GET /logs/?run_id=abc&lines=100` - Get run logs
|
||||
- `GET /estimate/?site_id=123` - Estimate credits needed
|
||||
|
||||
#### 4. Celery Tasks (`tasks.py`)
|
||||
|
||||
**check_scheduled_automations**
|
||||
- Runs hourly via Celery Beat
|
||||
- Checks AutomationConfig records for scheduled runs
|
||||
- Triggers automation based on frequency and scheduled_time
|
||||
|
||||
**run_automation_task**
|
||||
- Main background task that executes all 7 stages sequentially
|
||||
- Called by `run_now` API endpoint or scheduled trigger
|
||||
- Handles errors and updates AutomationRun status
|
||||
|
||||
**resume_automation_task**
|
||||
- Resumes paused automation from `current_stage`
|
||||
- Called by `resume` API endpoint
|
||||
|
||||
#### 5. Database Migration
|
||||
|
||||
Located at `/backend/igny8_core/business/automation/migrations/0001_initial.py`
|
||||
|
||||
Run with: `python manage.py migrate`
|
||||
|
||||
### Frontend Components
|
||||
|
||||
#### 1. Service (`/frontend/src/services/automationService.ts`)
|
||||
|
||||
TypeScript API client with methods matching backend endpoints:
|
||||
- `getConfig()`, `updateConfig()`, `runNow()`, `getCurrentRun()`
|
||||
- `pause()`, `resume()`, `getHistory()`, `getLogs()`, `estimate()`
|
||||
|
||||
#### 2. Pages
|
||||
|
||||
**AutomationPage** (`pages/Automation/AutomationPage.tsx`)
|
||||
- Main dashboard at `/automation`
|
||||
- Displays current run status, stage progress, activity log, history
|
||||
- Real-time polling (5s interval when run is active)
|
||||
- Controls: Run Now, Pause, Resume, Configure
|
||||
|
||||
#### 3. Components
|
||||
|
||||
**StageCard** (`components/Automation/StageCard.tsx`)
|
||||
- Visual representation of each stage (1-7)
|
||||
- Shows status: pending (⏳), active (🔄), complete (✅)
|
||||
- Displays stage results (items processed, credits used, etc.)
|
||||
|
||||
**ActivityLog** (`components/Automation/ActivityLog.tsx`)
|
||||
- Real-time log viewer with terminal-style display
|
||||
- Auto-refreshes every 3 seconds
|
||||
- Configurable line count (50, 100, 200, 500)
|
||||
|
||||
**ConfigModal** (`components/Automation/ConfigModal.tsx`)
|
||||
- Modal for editing automation settings
|
||||
- Fields: Enable/disable, frequency, scheduled time, batch sizes
|
||||
- Form validation and save
|
||||
|
||||
**RunHistory** (`components/Automation/RunHistory.tsx`)
|
||||
- Table of past automation runs
|
||||
- Columns: run_id, status, trigger, started, completed, credits, stage
|
||||
- Status badges with color coding
|
||||
|
||||
## 7-Stage Pipeline
|
||||
|
||||
### Stage 1: Keywords → Clusters (AI)
|
||||
- **Query**: `Keywords` with `status='new'`, `cluster__isnull=True`, `disabled=False`
|
||||
- **Batch Size**: Default 20 keywords
|
||||
- **AI Function**: `AutoCluster().execute()`
|
||||
- **Output**: Creates `Clusters` records
|
||||
- **Credits**: ~1 per 5 keywords
|
||||
|
||||
### Stage 2: Clusters → Ideas (AI)
|
||||
- **Query**: `Clusters` with `status='new'`, exclude those with existing ideas
|
||||
- **Batch Size**: Default 1 cluster
|
||||
- **AI Function**: `GenerateIdeas().execute()`
|
||||
- **Output**: Creates `ContentIdeas` records
|
||||
- **Credits**: ~2 per cluster
|
||||
|
||||
### Stage 3: Ideas → Tasks (Local Queue)
|
||||
- **Query**: `ContentIdeas` with `status='new'`
|
||||
- **Batch Size**: Default 20 ideas
|
||||
- **Operation**: Local database creation (no AI)
|
||||
- **Output**: Creates `Tasks` records with status='queued'
|
||||
- **Credits**: 0 (local operation)
|
||||
|
||||
### Stage 4: Tasks → Content (AI)
|
||||
- **Query**: `Tasks` with `status='queued'`, `content__isnull=True`
|
||||
- **Batch Size**: Default 1 task
|
||||
- **AI Function**: `GenerateContent().execute()`
|
||||
- **Output**: Creates `Content` records with status='draft'
|
||||
- **Credits**: ~5 per content (2500 words avg)
|
||||
|
||||
### Stage 5: Content → Image Prompts (AI)
|
||||
- **Query**: `Content` with `status='draft'`, `images_count=0` (annotated)
|
||||
- **Batch Size**: Default 1 content
|
||||
- **AI Function**: `GenerateImagePromptsFunction().execute()`
|
||||
- **Output**: Creates `Images` records with status='pending' (contains prompts)
|
||||
- **Credits**: ~2 per content (4 prompts avg)
|
||||
|
||||
### Stage 6: Image Prompts → Generated Images (AI)
|
||||
- **Query**: `Images` with `status='pending'`
|
||||
- **Batch Size**: Default 1 image
|
||||
- **AI Function**: `GenerateImages().execute()`
|
||||
- **Output**: Updates `Images` to status='generated' with `image_url`
|
||||
- **Side Effect**: Automatically sets `Content.status='review'` when all images complete (via `ai/tasks.py:723`)
|
||||
- **Credits**: ~2 per image
|
||||
|
||||
### Stage 7: Manual Review Gate
|
||||
- **Query**: `Content` with `status='review'`
|
||||
- **Operation**: Count only, no processing
|
||||
- **Output**: Returns list of content IDs ready for review
|
||||
- **Credits**: 0
|
||||
|
||||
## Key Design Principles
|
||||
|
||||
### 1. NO Duplication of AI Function Logic
|
||||
|
||||
The automation system ONLY handles:
|
||||
- Batch selection and sequencing
|
||||
- Stage orchestration
|
||||
- Credit estimation and checking
|
||||
- Progress tracking and logging
|
||||
- Scheduling and triggers
|
||||
|
||||
It does NOT handle:
|
||||
- Credit deduction (done by `AIEngine.execute()` at line 395)
|
||||
- Status updates (done within AI functions)
|
||||
- Progress tracking (StepTracker emits events automatically)
|
||||
|
||||
### 2. Correct Image Model Understanding
|
||||
|
||||
- **NO separate ImagePrompts model** - this was a misunderstanding
|
||||
- `Images` model serves dual purpose:
|
||||
- `status='pending'` = has prompt, needs image URL
|
||||
- `status='generated'` = has image_url
|
||||
- Stage 5 creates Images records with prompts
|
||||
- Stage 6 updates same records with URLs
|
||||
|
||||
### 3. Automatic Content Status Changes
|
||||
|
||||
- `Content.status` changes from 'draft' to 'review' automatically
|
||||
- Happens in `ai/tasks.py:723` when all images complete
|
||||
- Automation does NOT manually update this status
|
||||
|
||||
### 4. Distributed Locking
|
||||
|
||||
- Uses Django cache with `automation_lock_{site.id}` key
|
||||
- 6-hour timeout to prevent deadlocks
|
||||
- Released on completion, pause, or failure
|
||||
|
||||
## Configuration
|
||||
|
||||
### Schedule Configuration UI
|
||||
|
||||
Located at `/automation` page → [Configure] button
|
||||
|
||||
**Options:**
|
||||
- **Enable/Disable**: Toggle automation on/off
|
||||
- **Frequency**: Daily, Weekly (Mondays), Monthly (1st)
|
||||
- **Scheduled Time**: Time of day to run (24-hour format)
|
||||
- **Batch Sizes**: Per-stage item counts
|
||||
|
||||
**Defaults:**
|
||||
- Stage 1: 20 keywords
|
||||
- Stage 2: 1 cluster
|
||||
- Stage 3: 20 ideas
|
||||
- Stage 4: 1 task
|
||||
- Stage 5: 1 content
|
||||
- Stage 6: 1 image
|
||||
|
||||
### Credit Estimation
|
||||
|
||||
Before starting, system estimates:
|
||||
- Stage 1: keywords_count / 5
|
||||
- Stage 2: clusters_count * 2
|
||||
- Stage 4: tasks_count * 5
|
||||
- Stage 5: content_count * 2
|
||||
- Stage 6: content_count * 8 (4 images * 2 credits avg)
|
||||
|
||||
Requires 20% buffer: `account.credits_balance >= estimated * 1.2`
|
||||
|
||||
## Deployment Checklist
|
||||
|
||||
### Backend
|
||||
|
||||
1. ✅ Models created in `business/automation/models.py`
|
||||
2. ✅ Services created (`AutomationLogger`, `AutomationService`)
|
||||
3. ✅ Views created (`AutomationViewSet`)
|
||||
4. ✅ URLs registered in `igny8_core/urls.py`
|
||||
5. ✅ Celery tasks created (`check_scheduled_automations`, `run_automation_task`, `resume_automation_task`)
|
||||
6. ✅ Celery beat schedule updated in `celery.py`
|
||||
7. ⏳ Migration created (needs to run: `python manage.py migrate`)
|
||||
|
||||
### Frontend
|
||||
|
||||
8. ✅ API service created (`services/automationService.ts`)
|
||||
9. ✅ Main page created (`pages/Automation/AutomationPage.tsx`)
|
||||
10. ✅ Components created (`StageCard`, `ActivityLog`, `ConfigModal`, `RunHistory`)
|
||||
11. ⏳ Route registration (add to router: `/automation` → `AutomationPage`)
|
||||
|
||||
### Infrastructure
|
||||
|
||||
12. ⏳ Celery worker running (for background tasks)
|
||||
13. ⏳ Celery beat running (for scheduled checks)
|
||||
14. ⏳ Redis/cache backend configured (for distributed locks)
|
||||
15. ⏳ Log directory writable: `/data/app/igny8/backend/logs/automation/`
|
||||
|
||||
## Usage
|
||||
|
||||
### Manual Trigger
|
||||
|
||||
1. Navigate to `/automation` page
|
||||
2. Verify credit balance is sufficient (shows in header)
|
||||
3. Click [Run Now] button
|
||||
4. Monitor progress in real-time:
|
||||
- Stage cards show current progress
|
||||
- Activity log shows detailed logs
|
||||
- Credits used updates live
|
||||
|
||||
### Scheduled Automation
|
||||
|
||||
1. Navigate to `/automation` page
|
||||
2. Click [Configure] button
|
||||
3. Enable automation
|
||||
4. Set frequency and time
|
||||
5. Configure batch sizes
|
||||
6. Save configuration
|
||||
7. Automation will run automatically at scheduled time
|
||||
|
||||
### Pause/Resume
|
||||
|
||||
- During active run, click [Pause] to halt execution
|
||||
- Click [Resume] to continue from current stage
|
||||
- Useful for credit management or issue investigation
|
||||
|
||||
### Viewing History
|
||||
|
||||
- Run History table shows last 20 runs
|
||||
- Filter by status, date, trigger type
|
||||
- Click run_id to view detailed logs
|
||||
|
||||
## Monitoring
|
||||
|
||||
### Log Files
|
||||
|
||||
Located at: `logs/automation/{account_id}/{site_id}/{run_id}/`
|
||||
|
||||
- `automation_run.log` - Main activity log
|
||||
- `stage_1.log` through `stage_7.log` - Stage-specific logs
|
||||
|
||||
### Database Records
|
||||
|
||||
**AutomationRun** table tracks:
|
||||
- Current status and stage
|
||||
- Stage results (JSON)
|
||||
- Credits used
|
||||
- Error messages
|
||||
- Timestamps
|
||||
|
||||
**AutomationConfig** table tracks:
|
||||
- Last run timestamp
|
||||
- Next scheduled run
|
||||
- Configuration changes
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Run stuck in "running" status
|
||||
|
||||
1. Check Celery worker logs: `docker logs <celery_container>`
|
||||
2. Check for cache lock: `redis-cli GET automation_lock_<site_id>`
|
||||
3. Manually release lock if needed: `redis-cli DEL automation_lock_<site_id>`
|
||||
4. Update run status: `AutomationRun.objects.filter(run_id='...').update(status='failed')`
|
||||
|
||||
### Insufficient credits
|
||||
|
||||
1. Check estimate: GET `/api/v1/automation/estimate/?site_id=123`
|
||||
2. Add credits via billing page
|
||||
3. Retry run
|
||||
|
||||
### Stage failures
|
||||
|
||||
1. View logs: GET `/api/v1/automation/logs/?run_id=...`
|
||||
2. Check `error_message` field in AutomationRun
|
||||
3. Verify AI function is working: test individually via existing UI
|
||||
4. Check credit balance mid-run
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
1. Email notifications on completion/failure
|
||||
2. Slack/webhook integrations
|
||||
3. Per-stage retry logic
|
||||
4. Partial run resumption after failure
|
||||
5. Advanced scheduling (specific days, multiple times)
|
||||
6. Content preview before Stage 7
|
||||
7. Auto-publish to WordPress option
|
||||
8. Credit usage analytics and forecasting
|
||||
|
||||
## File Locations Summary
|
||||
|
||||
```
|
||||
backend/igny8_core/business/automation/
|
||||
├── __init__.py
|
||||
├── models.py # AutomationConfig, AutomationRun
|
||||
├── views.py # AutomationViewSet (API endpoints)
|
||||
├── tasks.py # Celery tasks
|
||||
├── urls.py # URL routing
|
||||
├── migrations/
|
||||
│ ├── __init__.py
|
||||
│ └── 0001_initial.py # Database schema
|
||||
└── services/
|
||||
├── __init__.py
|
||||
├── automation_logger.py # File logging service
|
||||
└── automation_service.py # Core orchestrator
|
||||
|
||||
frontend/src/
|
||||
├── services/
|
||||
│ └── automationService.ts # API client
|
||||
├── pages/Automation/
|
||||
│ └── AutomationPage.tsx # Main dashboard
|
||||
└── components/Automation/
|
||||
├── StageCard.tsx # Stage status display
|
||||
├── ActivityLog.tsx # Log viewer
|
||||
├── ConfigModal.tsx # Settings modal
|
||||
└── RunHistory.tsx # Past runs table
|
||||
```
|
||||
|
||||
## Credits
|
||||
|
||||
Implemented according to `automation-plan.md` with corrections for:
|
||||
- Image model structure (no separate ImagePrompts)
|
||||
- AI function internal logic (no duplication)
|
||||
- Content status changes (automatic in background)
|
||||
116
backend/deploy_automation.sh
Normal file
116
backend/deploy_automation.sh
Normal file
@@ -0,0 +1,116 @@
|
||||
#!/bin/bash
|
||||
# Automation System Deployment Script
|
||||
# Run this script to complete the automation system deployment
|
||||
|
||||
set -e # Exit on error
|
||||
|
||||
echo "========================================="
|
||||
echo "IGNY8 Automation System Deployment"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
# Colors for output
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Check if running from correct directory
|
||||
if [ ! -f "manage.py" ]; then
|
||||
echo -e "${RED}Error: Please run this script from the backend directory${NC}"
|
||||
echo "cd /data/app/igny8/backend && ./deploy_automation.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo -e "${YELLOW}Step 1: Creating log directory...${NC}"
|
||||
mkdir -p logs/automation
|
||||
chmod 755 logs/automation
|
||||
echo -e "${GREEN}✓ Log directory created${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Step 2: Running database migrations...${NC}"
|
||||
python3 manage.py makemigrations
|
||||
python3 manage.py migrate
|
||||
echo -e "${GREEN}✓ Migrations complete${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Step 3: Checking Celery services...${NC}"
|
||||
if docker ps | grep -q celery; then
|
||||
echo -e "${GREEN}✓ Celery worker is running${NC}"
|
||||
else
|
||||
echo -e "${RED}⚠ Celery worker is NOT running${NC}"
|
||||
echo "Start with: docker-compose up -d celery"
|
||||
fi
|
||||
|
||||
if docker ps | grep -q beat; then
|
||||
echo -e "${GREEN}✓ Celery beat is running${NC}"
|
||||
else
|
||||
echo -e "${RED}⚠ Celery beat is NOT running${NC}"
|
||||
echo "Start with: docker-compose up -d celery-beat"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Step 4: Verifying cache backend...${NC}"
|
||||
python3 -c "
|
||||
from django.core.cache import cache
|
||||
try:
|
||||
cache.set('test_key', 'test_value', 10)
|
||||
if cache.get('test_key') == 'test_value':
|
||||
print('${GREEN}✓ Cache backend working${NC}')
|
||||
else:
|
||||
print('${RED}⚠ Cache backend not working properly${NC}')
|
||||
except Exception as e:
|
||||
print('${RED}⚠ Cache backend error:', str(e), '${NC}')
|
||||
" || echo -e "${RED}⚠ Could not verify cache backend${NC}"
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Step 5: Testing automation API...${NC}"
|
||||
python3 manage.py shell << EOF
|
||||
from igny8_core.business.automation.services import AutomationService
|
||||
from igny8_core.modules.system.models import Account, Site
|
||||
|
||||
try:
|
||||
account = Account.objects.first()
|
||||
site = Site.objects.first()
|
||||
if account and site:
|
||||
service = AutomationService(account, site)
|
||||
estimate = service.estimate_credits()
|
||||
print('${GREEN}✓ AutomationService working - Estimated credits:', estimate, '${NC}')
|
||||
else:
|
||||
print('${YELLOW}⚠ No account or site found - create one first${NC}')
|
||||
except Exception as e:
|
||||
print('${RED}⚠ AutomationService error:', str(e), '${NC}')
|
||||
EOF
|
||||
echo ""
|
||||
|
||||
echo -e "${YELLOW}Step 6: Checking Celery beat schedule...${NC}"
|
||||
if docker ps | grep -q celery; then
|
||||
CELERY_CONTAINER=$(docker ps | grep celery | grep -v beat | awk '{print $1}')
|
||||
docker exec $CELERY_CONTAINER celery -A igny8_core inspect scheduled 2>/dev/null | grep -q "check-scheduled-automations" && \
|
||||
echo -e "${GREEN}✓ Automation task scheduled in Celery beat${NC}" || \
|
||||
echo -e "${YELLOW}⚠ Automation task not found in schedule (may need restart)${NC}"
|
||||
else
|
||||
echo -e "${YELLOW}⚠ Celery worker not running - cannot check schedule${NC}"
|
||||
fi
|
||||
echo ""
|
||||
|
||||
echo "========================================="
|
||||
echo -e "${GREEN}Deployment Steps Completed!${NC}"
|
||||
echo "========================================="
|
||||
echo ""
|
||||
|
||||
echo "Next steps:"
|
||||
echo "1. Restart Celery services to pick up new tasks:"
|
||||
echo " docker-compose restart celery celery-beat"
|
||||
echo ""
|
||||
echo "2. Access the frontend at /automation page"
|
||||
echo ""
|
||||
echo "3. Test the automation:"
|
||||
echo " - Click [Configure] to set up schedule"
|
||||
echo " - Click [Run Now] to start automation"
|
||||
echo " - Monitor progress in real-time"
|
||||
echo ""
|
||||
echo "4. Check logs:"
|
||||
echo " tail -f logs/automation/{account_id}/{site_id}/{run_id}/automation_run.log"
|
||||
echo ""
|
||||
echo -e "${YELLOW}For troubleshooting, see: AUTOMATION-DEPLOYMENT-CHECKLIST.md${NC}"
|
||||
4
backend/igny8_core/business/automation/__init__.py
Normal file
4
backend/igny8_core/business/automation/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
"""
|
||||
Automation Business Logic
|
||||
Orchestrates AI functions into automated pipelines
|
||||
"""
|
||||
@@ -0,0 +1,89 @@
|
||||
# Generated migration for automation models
|
||||
|
||||
from django.db import migrations, models
|
||||
import django.db.models.deletion
|
||||
import uuid
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
initial = True
|
||||
|
||||
dependencies = [
|
||||
('system', '__latest__'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.CreateModel(
|
||||
name='AutomationConfig',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('is_enabled', models.BooleanField(default=False, help_text='Enable/disable automation for this site')),
|
||||
('frequency', models.CharField(
|
||||
choices=[('daily', 'Daily'), ('weekly', 'Weekly'), ('monthly', 'Monthly')],
|
||||
default='daily',
|
||||
max_length=20
|
||||
)),
|
||||
('scheduled_time', models.TimeField(default='02:00', help_text='Time of day to run automation (HH:MM)')),
|
||||
('stage_1_batch_size', models.IntegerField(default=20, help_text='Keywords → Clusters batch size')),
|
||||
('stage_2_batch_size', models.IntegerField(default=1, help_text='Clusters → Ideas batch size')),
|
||||
('stage_3_batch_size', models.IntegerField(default=20, help_text='Ideas → Tasks batch size')),
|
||||
('stage_4_batch_size', models.IntegerField(default=1, help_text='Tasks → Content batch size')),
|
||||
('stage_5_batch_size', models.IntegerField(default=1, help_text='Content → Image Prompts batch size')),
|
||||
('stage_6_batch_size', models.IntegerField(default=1, help_text='Image Prompts → Images batch size')),
|
||||
('last_run_at', models.DateTimeField(blank=True, null=True)),
|
||||
('next_run_at', models.DateTimeField(blank=True, null=True)),
|
||||
('created_at', models.DateTimeField(auto_now_add=True)),
|
||||
('updated_at', models.DateTimeField(auto_now=True)),
|
||||
('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='system.account')),
|
||||
('site', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='automation_config', to='system.site')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'automation_config',
|
||||
},
|
||||
),
|
||||
migrations.CreateModel(
|
||||
name='AutomationRun',
|
||||
fields=[
|
||||
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
||||
('run_id', models.CharField(max_length=100, unique=True)),
|
||||
('trigger_type', models.CharField(
|
||||
choices=[('manual', 'Manual'), ('scheduled', 'Scheduled')],
|
||||
default='manual',
|
||||
max_length=20
|
||||
)),
|
||||
('status', models.CharField(
|
||||
choices=[
|
||||
('running', 'Running'),
|
||||
('paused', 'Paused'),
|
||||
('completed', 'Completed'),
|
||||
('failed', 'Failed')
|
||||
],
|
||||
default='running',
|
||||
max_length=20
|
||||
)),
|
||||
('current_stage', models.IntegerField(default=1, help_text='Current stage (1-7)')),
|
||||
('stage_1_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_2_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_3_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_4_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_5_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_6_result', models.JSONField(blank=True, null=True)),
|
||||
('stage_7_result', models.JSONField(blank=True, null=True)),
|
||||
('total_credits_used', models.IntegerField(default=0)),
|
||||
('error_message', models.TextField(blank=True, null=True)),
|
||||
('started_at', models.DateTimeField(auto_now_add=True)),
|
||||
('completed_at', models.DateTimeField(blank=True, null=True)),
|
||||
('account', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='system.account')),
|
||||
('site', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='automation_runs', to='system.site')),
|
||||
],
|
||||
options={
|
||||
'db_table': 'automation_run',
|
||||
'ordering': ['-started_at'],
|
||||
'indexes': [
|
||||
models.Index(fields=['site', 'status'], name='automation_site_status_idx'),
|
||||
models.Index(fields=['site', 'started_at'], name='automation_site_started_idx'),
|
||||
],
|
||||
},
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1 @@
|
||||
"""Automation migrations"""
|
||||
104
backend/igny8_core/business/automation/models.py
Normal file
104
backend/igny8_core/business/automation/models.py
Normal file
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
Automation Models
|
||||
Tracks automation runs and configuration
|
||||
"""
|
||||
from django.db import models
|
||||
from django.utils import timezone
|
||||
from igny8_core.modules.system.models import Account, Site
|
||||
|
||||
|
||||
class AutomationConfig(models.Model):
|
||||
"""Per-site automation configuration"""
|
||||
|
||||
FREQUENCY_CHOICES = [
|
||||
('daily', 'Daily'),
|
||||
('weekly', 'Weekly'),
|
||||
('monthly', 'Monthly'),
|
||||
]
|
||||
|
||||
account = models.ForeignKey(Account, on_delete=models.CASCADE, related_name='automation_configs')
|
||||
site = models.OneToOneField(Site, on_delete=models.CASCADE, related_name='automation_config')
|
||||
|
||||
is_enabled = models.BooleanField(default=False, help_text="Whether scheduled automation is active")
|
||||
frequency = models.CharField(max_length=20, choices=FREQUENCY_CHOICES, default='daily')
|
||||
scheduled_time = models.TimeField(default='02:00', help_text="Time to run (e.g., 02:00)")
|
||||
|
||||
# Batch sizes per stage
|
||||
stage_1_batch_size = models.IntegerField(default=20, help_text="Keywords per batch")
|
||||
stage_2_batch_size = models.IntegerField(default=1, help_text="Clusters at a time")
|
||||
stage_3_batch_size = models.IntegerField(default=20, help_text="Ideas per batch")
|
||||
stage_4_batch_size = models.IntegerField(default=1, help_text="Tasks - sequential")
|
||||
stage_5_batch_size = models.IntegerField(default=1, help_text="Content at a time")
|
||||
stage_6_batch_size = models.IntegerField(default=1, help_text="Images - sequential")
|
||||
|
||||
last_run_at = models.DateTimeField(null=True, blank=True)
|
||||
next_run_at = models.DateTimeField(null=True, blank=True, help_text="Calculated based on frequency")
|
||||
|
||||
created_at = models.DateTimeField(auto_now_add=True)
|
||||
updated_at = models.DateTimeField(auto_now=True)
|
||||
|
||||
class Meta:
|
||||
db_table = 'igny8_automation_configs'
|
||||
verbose_name = 'Automation Config'
|
||||
verbose_name_plural = 'Automation Configs'
|
||||
indexes = [
|
||||
models.Index(fields=['is_enabled', 'next_run_at']),
|
||||
models.Index(fields=['account', 'site']),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"Automation Config: {self.site.domain} ({self.frequency})"
|
||||
|
||||
|
||||
class AutomationRun(models.Model):
|
||||
"""Tracks each automation execution"""
|
||||
|
||||
TRIGGER_TYPE_CHOICES = [
|
||||
('manual', 'Manual'),
|
||||
('scheduled', 'Scheduled'),
|
||||
]
|
||||
|
||||
STATUS_CHOICES = [
|
||||
('running', 'Running'),
|
||||
('paused', 'Paused'),
|
||||
('completed', 'Completed'),
|
||||
('failed', 'Failed'),
|
||||
]
|
||||
|
||||
run_id = models.CharField(max_length=100, unique=True, db_index=True, help_text="Format: run_20251203_140523_manual")
|
||||
account = models.ForeignKey(Account, on_delete=models.CASCADE, related_name='automation_runs')
|
||||
site = models.ForeignKey(Site, on_delete=models.CASCADE, related_name='automation_runs')
|
||||
|
||||
trigger_type = models.CharField(max_length=20, choices=TRIGGER_TYPE_CHOICES)
|
||||
status = models.CharField(max_length=20, choices=STATUS_CHOICES, default='running', db_index=True)
|
||||
current_stage = models.IntegerField(default=1, help_text="Current stage number (1-7)")
|
||||
|
||||
started_at = models.DateTimeField(auto_now_add=True, db_index=True)
|
||||
completed_at = models.DateTimeField(null=True, blank=True)
|
||||
|
||||
total_credits_used = models.IntegerField(default=0)
|
||||
|
||||
# JSON results per stage
|
||||
stage_1_result = models.JSONField(null=True, blank=True, help_text="{keywords_processed, clusters_created, batches}")
|
||||
stage_2_result = models.JSONField(null=True, blank=True, help_text="{clusters_processed, ideas_created}")
|
||||
stage_3_result = models.JSONField(null=True, blank=True, help_text="{ideas_processed, tasks_created}")
|
||||
stage_4_result = models.JSONField(null=True, blank=True, help_text="{tasks_processed, content_created, total_words}")
|
||||
stage_5_result = models.JSONField(null=True, blank=True, help_text="{content_processed, prompts_created}")
|
||||
stage_6_result = models.JSONField(null=True, blank=True, help_text="{images_processed, images_generated}")
|
||||
stage_7_result = models.JSONField(null=True, blank=True, help_text="{ready_for_review}")
|
||||
|
||||
error_message = models.TextField(null=True, blank=True)
|
||||
|
||||
class Meta:
|
||||
db_table = 'igny8_automation_runs'
|
||||
verbose_name = 'Automation Run'
|
||||
verbose_name_plural = 'Automation Runs'
|
||||
ordering = ['-started_at']
|
||||
indexes = [
|
||||
models.Index(fields=['site', '-started_at']),
|
||||
models.Index(fields=['status', '-started_at']),
|
||||
models.Index(fields=['account', '-started_at']),
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.run_id} - {self.site.domain} ({self.status})"
|
||||
@@ -0,0 +1,7 @@
|
||||
"""
|
||||
Automation Services
|
||||
"""
|
||||
from .automation_service import AutomationService
|
||||
from .automation_logger import AutomationLogger
|
||||
|
||||
__all__ = ['AutomationService', 'AutomationLogger']
|
||||
@@ -0,0 +1,153 @@
|
||||
"""
|
||||
Automation Logger Service
|
||||
Handles file-based logging for automation runs
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AutomationLogger:
|
||||
"""File-based logging for automation runs"""
|
||||
|
||||
def __init__(self, base_log_dir: str = 'logs/automation'):
|
||||
self.base_log_dir = base_log_dir
|
||||
|
||||
def start_run(self, account_id: int, site_id: int, trigger_type: str) -> str:
|
||||
"""
|
||||
Create log directory structure and return run_id
|
||||
|
||||
Returns:
|
||||
run_id in format: run_20251203_140523_manual
|
||||
"""
|
||||
# Generate run_id
|
||||
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
run_id = f"run_{timestamp}_{trigger_type}"
|
||||
|
||||
# Create directory structure
|
||||
run_dir = self._get_run_dir(account_id, site_id, run_id)
|
||||
os.makedirs(run_dir, exist_ok=True)
|
||||
|
||||
# Create main log file
|
||||
log_file = os.path.join(run_dir, 'automation_run.log')
|
||||
with open(log_file, 'w') as f:
|
||||
f.write("=" * 80 + "\n")
|
||||
f.write(f"AUTOMATION RUN: {run_id}\n")
|
||||
f.write(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
f.write(f"Trigger: {trigger_type}\n")
|
||||
f.write(f"Account: {account_id}\n")
|
||||
f.write(f"Site: {site_id}\n")
|
||||
f.write("=" * 80 + "\n\n")
|
||||
|
||||
logger.info(f"[AutomationLogger] Created run: {run_id}")
|
||||
return run_id
|
||||
|
||||
def log_stage_start(self, run_id: str, account_id: int, site_id: int, stage_number: int, stage_name: str, pending_count: int):
|
||||
"""Log stage start"""
|
||||
timestamp = self._timestamp()
|
||||
|
||||
# Main log
|
||||
self._append_to_main_log(account_id, site_id, run_id,
|
||||
f"{timestamp} - Stage {stage_number} starting: {stage_name}")
|
||||
self._append_to_main_log(account_id, site_id, run_id,
|
||||
f"{timestamp} - Stage {stage_number}: Found {pending_count} pending items")
|
||||
|
||||
# Stage-specific log
|
||||
stage_log = self._get_stage_log_path(account_id, site_id, run_id, stage_number)
|
||||
with open(stage_log, 'w') as f:
|
||||
f.write("=" * 80 + "\n")
|
||||
f.write(f"STAGE {stage_number}: {stage_name}\n")
|
||||
f.write(f"Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
f.write("=" * 80 + "\n\n")
|
||||
f.write(f"{timestamp} - Found {pending_count} pending items\n")
|
||||
|
||||
def log_stage_progress(self, run_id: str, account_id: int, site_id: int, stage_number: int, message: str):
|
||||
"""Log stage progress"""
|
||||
timestamp = self._timestamp()
|
||||
log_message = f"{timestamp} - Stage {stage_number}: {message}"
|
||||
|
||||
# Main log
|
||||
self._append_to_main_log(account_id, site_id, run_id, log_message)
|
||||
|
||||
# Stage-specific log
|
||||
stage_log = self._get_stage_log_path(account_id, site_id, run_id, stage_number)
|
||||
with open(stage_log, 'a') as f:
|
||||
f.write(f"{log_message}\n")
|
||||
|
||||
def log_stage_complete(self, run_id: str, account_id: int, site_id: int, stage_number: int,
|
||||
processed_count: int, time_elapsed: str, credits_used: int):
|
||||
"""Log stage completion"""
|
||||
timestamp = self._timestamp()
|
||||
|
||||
# Main log
|
||||
self._append_to_main_log(account_id, site_id, run_id,
|
||||
f"{timestamp} - Stage {stage_number} complete: {processed_count} items processed")
|
||||
|
||||
# Stage-specific log
|
||||
stage_log = self._get_stage_log_path(account_id, site_id, run_id, stage_number)
|
||||
with open(stage_log, 'a') as f:
|
||||
f.write("\n" + "=" * 80 + "\n")
|
||||
f.write(f"STAGE {stage_number} COMPLETE\n")
|
||||
f.write(f"Total Time: {time_elapsed}\n")
|
||||
f.write(f"Processed: {processed_count} items\n")
|
||||
f.write(f"Credits Used: {credits_used}\n")
|
||||
f.write("=" * 80 + "\n")
|
||||
|
||||
def log_stage_error(self, run_id: str, account_id: int, site_id: int, stage_number: int, error_message: str):
|
||||
"""Log stage error"""
|
||||
timestamp = self._timestamp()
|
||||
log_message = f"{timestamp} - Stage {stage_number} ERROR: {error_message}"
|
||||
|
||||
# Main log
|
||||
self._append_to_main_log(account_id, site_id, run_id, log_message)
|
||||
|
||||
# Stage-specific log
|
||||
stage_log = self._get_stage_log_path(account_id, site_id, run_id, stage_number)
|
||||
with open(stage_log, 'a') as f:
|
||||
f.write(f"\n{log_message}\n")
|
||||
|
||||
def get_activity_log(self, account_id: int, site_id: int, run_id: str, last_n: int = 50) -> List[str]:
|
||||
"""
|
||||
Get last N lines from main activity log
|
||||
|
||||
Returns:
|
||||
List of log lines (newest first)
|
||||
"""
|
||||
log_file = os.path.join(self._get_run_dir(account_id, site_id, run_id), 'automation_run.log')
|
||||
|
||||
if not os.path.exists(log_file):
|
||||
return []
|
||||
|
||||
with open(log_file, 'r') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
# Filter out header lines and empty lines
|
||||
activity_lines = [line.strip() for line in lines if line.strip() and not line.startswith('=')]
|
||||
|
||||
# Return last N lines (newest first)
|
||||
return list(reversed(activity_lines[-last_n:]))
|
||||
|
||||
# Helper methods
|
||||
|
||||
def _get_run_dir(self, account_id: int, site_id: int, run_id: str) -> str:
|
||||
"""Get run directory path"""
|
||||
return os.path.join(self.base_log_dir, str(account_id), str(site_id), run_id)
|
||||
|
||||
def _get_stage_log_path(self, account_id: int, site_id: int, run_id: str, stage_number: int) -> str:
|
||||
"""Get stage log file path"""
|
||||
run_dir = self._get_run_dir(account_id, site_id, run_id)
|
||||
return os.path.join(run_dir, f'stage_{stage_number}.log')
|
||||
|
||||
def _append_to_main_log(self, account_id: int, site_id: int, run_id: str, message: str):
|
||||
"""Append message to main log file"""
|
||||
log_file = os.path.join(self._get_run_dir(account_id, site_id, run_id), 'automation_run.log')
|
||||
with open(log_file, 'a') as f:
|
||||
f.write(f"{message}\n")
|
||||
|
||||
def _timestamp(self) -> str:
|
||||
"""Get formatted timestamp"""
|
||||
return datetime.now().strftime('%H:%M:%S')
|
||||
@@ -0,0 +1,824 @@
|
||||
"""
|
||||
Automation Service
|
||||
Core orchestrator that executes AI function stages sequentially
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, Optional, List
|
||||
from django.db import transaction
|
||||
from django.db.models import Count, Q, F
|
||||
from django.core.cache import cache
|
||||
from celery.result import AsyncResult
|
||||
|
||||
from igny8_core.business.automation.models import AutomationRun, AutomationConfig
|
||||
from igny8_core.business.automation.services.automation_logger import AutomationLogger
|
||||
from igny8_core.modules.system.models import Account, Site
|
||||
from igny8_core.modules.planner.models import Keywords, Clusters, ContentIdeas
|
||||
from igny8_core.modules.writer.models import Tasks, Content, Images
|
||||
from igny8_core.business.content.models import AIUsageLog
|
||||
|
||||
# AI Functions
|
||||
from igny8_core.ai.functions.auto_cluster import AutoCluster
|
||||
from igny8_core.ai.functions.generate_ideas import GenerateIdeas
|
||||
from igny8_core.ai.functions.generate_content import GenerateContent
|
||||
from igny8_core.ai.functions.generate_image_prompts import GenerateImagePromptsFunction
|
||||
from igny8_core.ai.functions.generate_images import GenerateImages
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AutomationService:
|
||||
"""Orchestrates AI functions into automated pipeline"""
|
||||
|
||||
def __init__(self, account: Account, site: Site):
|
||||
self.account = account
|
||||
self.site = site
|
||||
self.logger = AutomationLogger()
|
||||
self.run = None
|
||||
self.config = None
|
||||
|
||||
# Load or create config
|
||||
self.config, _ = AutomationConfig.objects.get_or_create(
|
||||
account=account,
|
||||
site=site,
|
||||
defaults={
|
||||
'is_enabled': False,
|
||||
'frequency': 'daily',
|
||||
'scheduled_time': '02:00',
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_run_id(cls, run_id: str) -> 'AutomationService':
|
||||
"""Create service instance from run_id"""
|
||||
run = AutomationRun.objects.get(run_id=run_id)
|
||||
service = cls(run.account, run.site)
|
||||
service.run = run
|
||||
return service
|
||||
|
||||
def start_automation(self, trigger_type: str = 'manual') -> str:
|
||||
"""
|
||||
Start automation run
|
||||
|
||||
Returns:
|
||||
run_id
|
||||
"""
|
||||
# Check for concurrent run
|
||||
if AutomationRun.objects.filter(site=self.site, status='running').exists():
|
||||
raise ValueError("Automation already running for this site")
|
||||
|
||||
# Acquire distributed lock
|
||||
lock_key = f'automation_lock_{self.site.id}'
|
||||
if not cache.add(lock_key, 'locked', timeout=21600): # 6 hours
|
||||
raise ValueError("Automation already running for this site (cache lock)")
|
||||
|
||||
try:
|
||||
# Estimate credits needed
|
||||
estimated_credits = self.estimate_credits()
|
||||
|
||||
# Check credit balance (with 20% buffer)
|
||||
required_credits = int(estimated_credits * 1.2)
|
||||
if self.account.credits_balance < required_credits:
|
||||
raise ValueError(f"Insufficient credits. Need ~{required_credits}, you have {self.account.credits_balance}")
|
||||
|
||||
# Create run_id and log files
|
||||
run_id = self.logger.start_run(self.account.id, self.site.id, trigger_type)
|
||||
|
||||
# Create AutomationRun record
|
||||
self.run = AutomationRun.objects.create(
|
||||
run_id=run_id,
|
||||
account=self.account,
|
||||
site=self.site,
|
||||
trigger_type=trigger_type,
|
||||
status='running',
|
||||
current_stage=1,
|
||||
)
|
||||
|
||||
# Log start
|
||||
self.logger.log_stage_progress(
|
||||
run_id, self.account.id, self.site.id, 0,
|
||||
f"Automation started (trigger: {trigger_type})"
|
||||
)
|
||||
self.logger.log_stage_progress(
|
||||
run_id, self.account.id, self.site.id, 0,
|
||||
f"Credit check: Account has {self.account.credits_balance} credits, estimated need: {estimated_credits} credits"
|
||||
)
|
||||
|
||||
logger.info(f"[AutomationService] Started run: {run_id}")
|
||||
return run_id
|
||||
|
||||
except Exception as e:
|
||||
# Release lock on failure
|
||||
cache.delete(lock_key)
|
||||
raise
|
||||
|
||||
def run_stage_1(self):
|
||||
"""Stage 1: Keywords → Clusters"""
|
||||
stage_number = 1
|
||||
stage_name = "Keywords → Clusters (AI)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query pending keywords
|
||||
pending_keywords = Keywords.objects.filter(
|
||||
site=self.site,
|
||||
status='new',
|
||||
cluster__isnull=True,
|
||||
disabled=False
|
||||
)
|
||||
|
||||
total_count = pending_keywords.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No keywords to process - skipping stage"
|
||||
)
|
||||
self.run.stage_1_result = {'keywords_processed': 0, 'clusters_created': 0, 'batches_run': 0, 'credits_used': 0}
|
||||
self.run.current_stage = 2
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process in batches
|
||||
batch_size = self.config.stage_1_batch_size
|
||||
keywords_processed = 0
|
||||
clusters_created = 0
|
||||
batches_run = 0
|
||||
credits_before = self._get_credits_used()
|
||||
|
||||
keyword_ids = list(pending_keywords.values_list('id', flat=True))
|
||||
|
||||
for i in range(0, len(keyword_ids), batch_size):
|
||||
batch = keyword_ids[i:i + batch_size]
|
||||
batch_num = (i // batch_size) + 1
|
||||
total_batches = (len(keyword_ids) + batch_size - 1) // batch_size
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Processing batch {batch_num}/{total_batches} ({len(batch)} keywords)"
|
||||
)
|
||||
|
||||
# Call AI function
|
||||
result = AutoCluster().execute(
|
||||
payload={'ids': batch},
|
||||
account=self.account
|
||||
)
|
||||
|
||||
# Monitor task
|
||||
task_id = result.get('task_id')
|
||||
if task_id:
|
||||
self._wait_for_task(task_id, stage_number, f"Batch {batch_num}")
|
||||
|
||||
keywords_processed += len(batch)
|
||||
batches_run += 1
|
||||
|
||||
# Log progress
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Batch {batch_num} complete"
|
||||
)
|
||||
|
||||
# Get clusters created count
|
||||
clusters_created = Clusters.objects.filter(
|
||||
site=self.site,
|
||||
created_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
# Calculate credits used
|
||||
credits_used = self._get_credits_used() - credits_before
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, keywords_processed, time_elapsed, credits_used
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_1_result = {
|
||||
'keywords_processed': keywords_processed,
|
||||
'clusters_created': clusters_created,
|
||||
'batches_run': batches_run,
|
||||
'credits_used': credits_used
|
||||
}
|
||||
self.run.current_stage = 2
|
||||
self.run.total_credits_used += credits_used
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 1 complete: {keywords_processed} keywords → {clusters_created} clusters")
|
||||
|
||||
def run_stage_2(self):
|
||||
"""Stage 2: Clusters → Ideas"""
|
||||
stage_number = 2
|
||||
stage_name = "Clusters → Ideas (AI)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query clusters without ideas
|
||||
pending_clusters = Clusters.objects.filter(
|
||||
site=self.site,
|
||||
status='new',
|
||||
disabled=False
|
||||
).exclude(
|
||||
ideas__isnull=False
|
||||
)
|
||||
|
||||
total_count = pending_clusters.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No clusters to process - skipping stage"
|
||||
)
|
||||
self.run.stage_2_result = {'clusters_processed': 0, 'ideas_created': 0, 'credits_used': 0}
|
||||
self.run.current_stage = 3
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process one at a time
|
||||
clusters_processed = 0
|
||||
credits_before = self._get_credits_used()
|
||||
|
||||
for cluster in pending_clusters:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Generating ideas for cluster: {cluster.name}"
|
||||
)
|
||||
|
||||
# Call AI function
|
||||
result = GenerateIdeas().execute(
|
||||
payload={'ids': [cluster.id]},
|
||||
account=self.account
|
||||
)
|
||||
|
||||
# Monitor task
|
||||
task_id = result.get('task_id')
|
||||
if task_id:
|
||||
self._wait_for_task(task_id, stage_number, f"Cluster '{cluster.name}'")
|
||||
|
||||
clusters_processed += 1
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Cluster '{cluster.name}' complete"
|
||||
)
|
||||
|
||||
# Get ideas created count
|
||||
ideas_created = ContentIdeas.objects.filter(
|
||||
site=self.site,
|
||||
created_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
# Calculate credits used
|
||||
credits_used = self._get_credits_used() - credits_before
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, clusters_processed, time_elapsed, credits_used
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_2_result = {
|
||||
'clusters_processed': clusters_processed,
|
||||
'ideas_created': ideas_created,
|
||||
'credits_used': credits_used
|
||||
}
|
||||
self.run.current_stage = 3
|
||||
self.run.total_credits_used += credits_used
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 2 complete: {clusters_processed} clusters → {ideas_created} ideas")
|
||||
|
||||
def run_stage_3(self):
|
||||
"""Stage 3: Ideas → Tasks (Local Queue)"""
|
||||
stage_number = 3
|
||||
stage_name = "Ideas → Tasks (Local Queue)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query pending ideas
|
||||
pending_ideas = ContentIdeas.objects.filter(
|
||||
site=self.site,
|
||||
status='new'
|
||||
)
|
||||
|
||||
total_count = pending_ideas.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No ideas to process - skipping stage"
|
||||
)
|
||||
self.run.stage_3_result = {'ideas_processed': 0, 'tasks_created': 0, 'batches_run': 0}
|
||||
self.run.current_stage = 4
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process in batches
|
||||
batch_size = self.config.stage_3_batch_size
|
||||
ideas_processed = 0
|
||||
tasks_created = 0
|
||||
batches_run = 0
|
||||
|
||||
idea_list = list(pending_ideas)
|
||||
|
||||
for i in range(0, len(idea_list), batch_size):
|
||||
batch = idea_list[i:i + batch_size]
|
||||
batch_num = (i // batch_size) + 1
|
||||
total_batches = (len(idea_list) + batch_size - 1) // batch_size
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Queueing batch {batch_num}/{total_batches} ({len(batch)} ideas)"
|
||||
)
|
||||
|
||||
# Create tasks (local operation)
|
||||
for idea in batch:
|
||||
# Build keywords string
|
||||
keywords_str = ''
|
||||
if idea.keyword_objects.exists():
|
||||
keywords_str = ', '.join([kw.keyword for kw in idea.keyword_objects.all()])
|
||||
elif idea.target_keywords:
|
||||
keywords_str = idea.target_keywords
|
||||
|
||||
# Create task
|
||||
task = Tasks.objects.create(
|
||||
title=idea.idea_title,
|
||||
description=idea.description or '',
|
||||
cluster=idea.keyword_cluster,
|
||||
content_type=idea.content_type or 'post',
|
||||
content_structure=idea.content_structure or 'article',
|
||||
keywords=keywords_str,
|
||||
status='queued',
|
||||
account=idea.account,
|
||||
site=idea.site,
|
||||
sector=idea.sector,
|
||||
idea=idea,
|
||||
)
|
||||
|
||||
# Update idea status
|
||||
idea.status = 'queued'
|
||||
idea.save()
|
||||
|
||||
tasks_created += 1
|
||||
|
||||
ideas_processed += len(batch)
|
||||
batches_run += 1
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Batch {batch_num} complete: {len(batch)} tasks created"
|
||||
)
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, ideas_processed, time_elapsed, 0 # No credits for local operation
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_3_result = {
|
||||
'ideas_processed': ideas_processed,
|
||||
'tasks_created': tasks_created,
|
||||
'batches_run': batches_run
|
||||
}
|
||||
self.run.current_stage = 4
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 3 complete: {ideas_processed} ideas → {tasks_created} tasks")
|
||||
|
||||
def run_stage_4(self):
|
||||
"""Stage 4: Tasks → Content"""
|
||||
stage_number = 4
|
||||
stage_name = "Tasks → Content (AI)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query queued tasks
|
||||
pending_tasks = Tasks.objects.filter(
|
||||
site=self.site,
|
||||
status='queued',
|
||||
content__isnull=True
|
||||
)
|
||||
|
||||
total_count = pending_tasks.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No tasks to process - skipping stage"
|
||||
)
|
||||
self.run.stage_4_result = {'tasks_processed': 0, 'content_created': 0, 'total_words': 0, 'credits_used': 0}
|
||||
self.run.current_stage = 5
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process one at a time
|
||||
tasks_processed = 0
|
||||
credits_before = self._get_credits_used()
|
||||
|
||||
for task in pending_tasks:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Generating content for task: {task.title}"
|
||||
)
|
||||
|
||||
# Call AI function
|
||||
result = GenerateContent().execute(
|
||||
payload={'ids': [task.id]},
|
||||
account=self.account
|
||||
)
|
||||
|
||||
# Monitor task
|
||||
task_id = result.get('task_id')
|
||||
if task_id:
|
||||
self._wait_for_task(task_id, stage_number, f"Task '{task.title}'")
|
||||
|
||||
tasks_processed += 1
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Task '{task.title}' complete"
|
||||
)
|
||||
|
||||
# Get content created count and total words
|
||||
content_created = Content.objects.filter(
|
||||
site=self.site,
|
||||
created_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
total_words = Content.objects.filter(
|
||||
site=self.site,
|
||||
created_at__gte=self.run.started_at
|
||||
).aggregate(total=Count('id'))['total'] * 2500 # Estimate
|
||||
|
||||
# Calculate credits used
|
||||
credits_used = self._get_credits_used() - credits_before
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, tasks_processed, time_elapsed, credits_used
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_4_result = {
|
||||
'tasks_processed': tasks_processed,
|
||||
'content_created': content_created,
|
||||
'total_words': total_words,
|
||||
'credits_used': credits_used
|
||||
}
|
||||
self.run.current_stage = 5
|
||||
self.run.total_credits_used += credits_used
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 4 complete: {tasks_processed} tasks → {content_created} content")
|
||||
|
||||
def run_stage_5(self):
|
||||
"""Stage 5: Content → Image Prompts"""
|
||||
stage_number = 5
|
||||
stage_name = "Content → Image Prompts (AI)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query content without Images records
|
||||
content_without_images = Content.objects.filter(
|
||||
site=self.site,
|
||||
status='draft'
|
||||
).annotate(
|
||||
images_count=Count('images')
|
||||
).filter(
|
||||
images_count=0
|
||||
)
|
||||
|
||||
total_count = content_without_images.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No content to process - skipping stage"
|
||||
)
|
||||
self.run.stage_5_result = {'content_processed': 0, 'prompts_created': 0, 'credits_used': 0}
|
||||
self.run.current_stage = 6
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process one at a time
|
||||
content_processed = 0
|
||||
credits_before = self._get_credits_used()
|
||||
|
||||
for content in content_without_images:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Extracting prompts from: {content.title}"
|
||||
)
|
||||
|
||||
# Call AI function
|
||||
result = GenerateImagePromptsFunction().execute(
|
||||
payload={'ids': [content.id]},
|
||||
account=self.account
|
||||
)
|
||||
|
||||
# Monitor task
|
||||
task_id = result.get('task_id')
|
||||
if task_id:
|
||||
self._wait_for_task(task_id, stage_number, f"Content '{content.title}'")
|
||||
|
||||
content_processed += 1
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Content '{content.title}' complete"
|
||||
)
|
||||
|
||||
# Get prompts created count
|
||||
prompts_created = Images.objects.filter(
|
||||
site=self.site,
|
||||
status='pending',
|
||||
created_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
# Calculate credits used
|
||||
credits_used = self._get_credits_used() - credits_before
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, content_processed, time_elapsed, credits_used
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_5_result = {
|
||||
'content_processed': content_processed,
|
||||
'prompts_created': prompts_created,
|
||||
'credits_used': credits_used
|
||||
}
|
||||
self.run.current_stage = 6
|
||||
self.run.total_credits_used += credits_used
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 5 complete: {content_processed} content → {prompts_created} prompts")
|
||||
|
||||
def run_stage_6(self):
|
||||
"""Stage 6: Image Prompts → Generated Images"""
|
||||
stage_number = 6
|
||||
stage_name = "Images (Prompts) → Generated Images (AI)"
|
||||
start_time = time.time()
|
||||
|
||||
# Query pending images
|
||||
pending_images = Images.objects.filter(
|
||||
site=self.site,
|
||||
status='pending'
|
||||
)
|
||||
|
||||
total_count = pending_images.count()
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
if total_count == 0:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, "No images to process - skipping stage"
|
||||
)
|
||||
self.run.stage_6_result = {'images_processed': 0, 'images_generated': 0, 'content_moved_to_review': 0, 'credits_used': 0}
|
||||
self.run.current_stage = 7
|
||||
self.run.save()
|
||||
return
|
||||
|
||||
# Process one at a time
|
||||
images_processed = 0
|
||||
credits_before = self._get_credits_used()
|
||||
|
||||
for image in pending_images:
|
||||
content_title = image.content.title if image.content else 'Unknown'
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Generating image: {image.image_type} for '{content_title}'"
|
||||
)
|
||||
|
||||
# Call AI function
|
||||
result = GenerateImages().execute(
|
||||
payload={'image_ids': [image.id]},
|
||||
account=self.account
|
||||
)
|
||||
|
||||
# Monitor task
|
||||
task_id = result.get('task_id')
|
||||
if task_id:
|
||||
self._wait_for_task(task_id, stage_number, f"Image for '{content_title}'")
|
||||
|
||||
images_processed += 1
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Image generated for '{content_title}'"
|
||||
)
|
||||
|
||||
# Get images generated count
|
||||
images_generated = Images.objects.filter(
|
||||
site=self.site,
|
||||
status='generated',
|
||||
updated_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
# Count content moved to review (automatic side effect)
|
||||
content_moved_to_review = Content.objects.filter(
|
||||
site=self.site,
|
||||
status='review',
|
||||
updated_at__gte=self.run.started_at
|
||||
).count()
|
||||
|
||||
# Calculate credits used
|
||||
credits_used = self._get_credits_used() - credits_before
|
||||
|
||||
# Calculate time elapsed
|
||||
time_elapsed = self._format_time_elapsed(start_time)
|
||||
|
||||
# Log completion
|
||||
self.logger.log_stage_complete(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, images_processed, time_elapsed, credits_used
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_6_result = {
|
||||
'images_processed': images_processed,
|
||||
'images_generated': images_generated,
|
||||
'content_moved_to_review': content_moved_to_review,
|
||||
'credits_used': credits_used
|
||||
}
|
||||
self.run.current_stage = 7
|
||||
self.run.total_credits_used += credits_used
|
||||
self.run.save()
|
||||
|
||||
logger.info(f"[AutomationService] Stage 6 complete: {images_processed} images generated, {content_moved_to_review} content moved to review")
|
||||
|
||||
def run_stage_7(self):
|
||||
"""Stage 7: Manual Review Gate (Count Only)"""
|
||||
stage_number = 7
|
||||
stage_name = "Manual Review Gate"
|
||||
|
||||
# Query content ready for review
|
||||
ready_for_review = Content.objects.filter(
|
||||
site=self.site,
|
||||
status='review'
|
||||
)
|
||||
|
||||
total_count = ready_for_review.count()
|
||||
content_ids = list(ready_for_review.values_list('id', flat=True))
|
||||
|
||||
# Log stage start
|
||||
self.logger.log_stage_start(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, stage_name, total_count
|
||||
)
|
||||
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Automation complete. {total_count} content pieces ready for review"
|
||||
)
|
||||
|
||||
if content_ids:
|
||||
self.logger.log_stage_progress(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, f"Content IDs ready: {content_ids[:10]}..." if len(content_ids) > 10 else f"Content IDs ready: {content_ids}"
|
||||
)
|
||||
|
||||
# Save results
|
||||
self.run.stage_7_result = {
|
||||
'ready_for_review': total_count,
|
||||
'content_ids': content_ids
|
||||
}
|
||||
self.run.status = 'completed'
|
||||
self.run.completed_at = datetime.now()
|
||||
self.run.save()
|
||||
|
||||
# Release lock
|
||||
cache.delete(f'automation_lock_{self.site.id}')
|
||||
|
||||
logger.info(f"[AutomationService] Stage 7 complete: Automation ended, {total_count} content ready for review")
|
||||
|
||||
def pause_automation(self):
|
||||
"""Pause current automation run"""
|
||||
if self.run:
|
||||
self.run.status = 'paused'
|
||||
self.run.save()
|
||||
logger.info(f"[AutomationService] Paused run: {self.run.run_id}")
|
||||
|
||||
def resume_automation(self):
|
||||
"""Resume paused automation run"""
|
||||
if self.run and self.run.status == 'paused':
|
||||
self.run.status = 'running'
|
||||
self.run.save()
|
||||
logger.info(f"[AutomationService] Resumed run: {self.run.run_id}")
|
||||
|
||||
def estimate_credits(self) -> int:
|
||||
"""Estimate total credits needed for automation"""
|
||||
# Count items
|
||||
keywords_count = Keywords.objects.filter(site=self.site, status='new', cluster__isnull=True).count()
|
||||
clusters_count = Clusters.objects.filter(site=self.site, status='new').exclude(ideas__isnull=False).count()
|
||||
ideas_count = ContentIdeas.objects.filter(site=self.site, status='new').count()
|
||||
tasks_count = Tasks.objects.filter(site=self.site, status='queued', content__isnull=True).count()
|
||||
content_count = Content.objects.filter(site=self.site, status='draft').annotate(images_count=Count('images')).filter(images_count=0).count()
|
||||
|
||||
# Estimate credits
|
||||
clustering_credits = (keywords_count // 5) + 1 # 1 credit per 5 keywords
|
||||
ideas_credits = clusters_count * 2 # 2 credits per cluster
|
||||
content_credits = tasks_count * 5 # Assume 2500 words avg = 5 credits
|
||||
prompts_credits = content_count * 2 # Assume 4 prompts per content = 2 credits
|
||||
images_credits = content_count * 8 # Assume 4 images * 2 credits avg
|
||||
|
||||
total = clustering_credits + ideas_credits + content_credits + prompts_credits + images_credits
|
||||
|
||||
logger.info(f"[AutomationService] Estimated credits: {total}")
|
||||
return total
|
||||
|
||||
# Helper methods
|
||||
|
||||
def _wait_for_task(self, task_id: str, stage_number: int, item_name: str):
|
||||
"""Wait for Celery task to complete"""
|
||||
result = AsyncResult(task_id)
|
||||
|
||||
while not result.ready():
|
||||
time.sleep(3) # Poll every 3 seconds
|
||||
|
||||
# Check for pause
|
||||
self.run.refresh_from_db()
|
||||
if self.run.status == 'paused':
|
||||
logger.info(f"[AutomationService] Paused during {item_name}")
|
||||
# Wait until resumed
|
||||
while self.run.status == 'paused':
|
||||
time.sleep(5)
|
||||
self.run.refresh_from_db()
|
||||
|
||||
if result.failed():
|
||||
error_msg = f"Task failed for {item_name}"
|
||||
self.logger.log_stage_error(
|
||||
self.run.run_id, self.account.id, self.site.id,
|
||||
stage_number, error_msg
|
||||
)
|
||||
raise Exception(error_msg)
|
||||
|
||||
def _get_credits_used(self) -> int:
|
||||
"""Get total credits used by this run so far"""
|
||||
if not self.run:
|
||||
return 0
|
||||
|
||||
total = AIUsageLog.objects.filter(
|
||||
account=self.account,
|
||||
created_at__gte=self.run.started_at
|
||||
).aggregate(total=Count('id'))['total'] or 0
|
||||
|
||||
return total
|
||||
|
||||
def _format_time_elapsed(self, start_time: float) -> str:
|
||||
"""Format elapsed time"""
|
||||
elapsed = time.time() - start_time
|
||||
minutes = int(elapsed // 60)
|
||||
seconds = int(elapsed % 60)
|
||||
return f"{minutes}m {seconds}s"
|
||||
195
backend/igny8_core/business/automation/tasks.py
Normal file
195
backend/igny8_core/business/automation/tasks.py
Normal file
@@ -0,0 +1,195 @@
|
||||
"""
|
||||
Automation Celery Tasks
|
||||
Background tasks for automation pipeline
|
||||
"""
|
||||
from celery import shared_task, chain
|
||||
from celery.utils.log import get_task_logger
|
||||
from datetime import datetime, timedelta
|
||||
from django.utils import timezone
|
||||
|
||||
from igny8_core.business.automation.models import AutomationConfig, AutomationRun
|
||||
from igny8_core.business.automation.services import AutomationService
|
||||
|
||||
logger = get_task_logger(__name__)
|
||||
|
||||
|
||||
@shared_task(name='automation.check_scheduled_automations')
|
||||
def check_scheduled_automations():
|
||||
"""
|
||||
Check for scheduled automation runs (runs every hour)
|
||||
"""
|
||||
logger.info("[AutomationTask] Checking scheduled automations")
|
||||
|
||||
now = timezone.now()
|
||||
current_time = now.time()
|
||||
|
||||
# Find configs that should run now
|
||||
for config in AutomationConfig.objects.filter(is_enabled=True):
|
||||
# Check if it's time to run
|
||||
should_run = False
|
||||
|
||||
if config.frequency == 'daily':
|
||||
# Run if current time matches scheduled_time
|
||||
if current_time.hour == config.scheduled_time.hour and current_time.minute < 60:
|
||||
should_run = True
|
||||
elif config.frequency == 'weekly':
|
||||
# Run on Mondays at scheduled_time
|
||||
if now.weekday() == 0 and current_time.hour == config.scheduled_time.hour and current_time.minute < 60:
|
||||
should_run = True
|
||||
elif config.frequency == 'monthly':
|
||||
# Run on 1st of month at scheduled_time
|
||||
if now.day == 1 and current_time.hour == config.scheduled_time.hour and current_time.minute < 60:
|
||||
should_run = True
|
||||
|
||||
if should_run:
|
||||
# Check if already ran today
|
||||
if config.last_run_at:
|
||||
time_since_last_run = now - config.last_run_at
|
||||
if time_since_last_run < timedelta(hours=23):
|
||||
logger.info(f"[AutomationTask] Skipping site {config.site.id} - already ran today")
|
||||
continue
|
||||
|
||||
# Check if already running
|
||||
if AutomationRun.objects.filter(site=config.site, status='running').exists():
|
||||
logger.info(f"[AutomationTask] Skipping site {config.site.id} - already running")
|
||||
continue
|
||||
|
||||
logger.info(f"[AutomationTask] Starting scheduled automation for site {config.site.id}")
|
||||
|
||||
try:
|
||||
service = AutomationService(config.account, config.site)
|
||||
run_id = service.start_automation(trigger_type='scheduled')
|
||||
|
||||
# Update config
|
||||
config.last_run_at = now
|
||||
config.next_run_at = _calculate_next_run(config, now)
|
||||
config.save()
|
||||
|
||||
# Start async processing
|
||||
run_automation_task.delay(run_id)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[AutomationTask] Failed to start automation for site {config.site.id}: {e}")
|
||||
|
||||
|
||||
@shared_task(name='automation.run_automation_task', bind=True, max_retries=0)
|
||||
def run_automation_task(self, run_id: str):
|
||||
"""
|
||||
Run automation pipeline (chains all stages)
|
||||
"""
|
||||
logger.info(f"[AutomationTask] Starting automation run: {run_id}")
|
||||
|
||||
try:
|
||||
service = AutomationService.from_run_id(run_id)
|
||||
|
||||
# Run all stages sequentially
|
||||
service.run_stage_1()
|
||||
service.run_stage_2()
|
||||
service.run_stage_3()
|
||||
service.run_stage_4()
|
||||
service.run_stage_5()
|
||||
service.run_stage_6()
|
||||
service.run_stage_7()
|
||||
|
||||
logger.info(f"[AutomationTask] Completed automation run: {run_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[AutomationTask] Failed automation run {run_id}: {e}")
|
||||
|
||||
# Mark as failed
|
||||
run = AutomationRun.objects.get(run_id=run_id)
|
||||
run.status = 'failed'
|
||||
run.error_message = str(e)
|
||||
run.completed_at = timezone.now()
|
||||
run.save()
|
||||
|
||||
# Release lock
|
||||
from django.core.cache import cache
|
||||
cache.delete(f'automation_lock_{run.site.id}')
|
||||
|
||||
raise
|
||||
|
||||
|
||||
@shared_task(name='automation.resume_automation_task', bind=True, max_retries=0)
|
||||
def resume_automation_task(self, run_id: str):
|
||||
"""
|
||||
Resume paused automation run from current stage
|
||||
"""
|
||||
logger.info(f"[AutomationTask] Resuming automation run: {run_id}")
|
||||
|
||||
try:
|
||||
service = AutomationService.from_run_id(run_id)
|
||||
run = service.run
|
||||
|
||||
# Continue from current stage
|
||||
stage_methods = [
|
||||
service.run_stage_1,
|
||||
service.run_stage_2,
|
||||
service.run_stage_3,
|
||||
service.run_stage_4,
|
||||
service.run_stage_5,
|
||||
service.run_stage_6,
|
||||
service.run_stage_7,
|
||||
]
|
||||
|
||||
# Run from current_stage to end
|
||||
for stage in range(run.current_stage - 1, 7):
|
||||
stage_methods[stage]()
|
||||
|
||||
logger.info(f"[AutomationTask] Resumed automation run: {run_id}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"[AutomationTask] Failed to resume automation run {run_id}: {e}")
|
||||
|
||||
# Mark as failed
|
||||
run = AutomationRun.objects.get(run_id=run_id)
|
||||
run.status = 'failed'
|
||||
run.error_message = str(e)
|
||||
run.completed_at = timezone.now()
|
||||
run.save()
|
||||
|
||||
# Release lock
|
||||
from django.core.cache import cache
|
||||
cache.delete(f'automation_lock_{run.site.id}')
|
||||
|
||||
raise
|
||||
|
||||
|
||||
def _calculate_next_run(config: AutomationConfig, now: datetime) -> datetime:
|
||||
"""Calculate next run time based on frequency"""
|
||||
if config.frequency == 'daily':
|
||||
next_run = now + timedelta(days=1)
|
||||
next_run = next_run.replace(
|
||||
hour=config.scheduled_time.hour,
|
||||
minute=config.scheduled_time.minute,
|
||||
second=0,
|
||||
microsecond=0
|
||||
)
|
||||
elif config.frequency == 'weekly':
|
||||
# Next Monday
|
||||
days_until_monday = (7 - now.weekday()) % 7
|
||||
if days_until_monday == 0:
|
||||
days_until_monday = 7
|
||||
next_run = now + timedelta(days=days_until_monday)
|
||||
next_run = next_run.replace(
|
||||
hour=config.scheduled_time.hour,
|
||||
minute=config.scheduled_time.minute,
|
||||
second=0,
|
||||
microsecond=0
|
||||
)
|
||||
elif config.frequency == 'monthly':
|
||||
# Next 1st of month
|
||||
if now.month == 12:
|
||||
next_run = now.replace(year=now.year + 1, month=1, day=1)
|
||||
else:
|
||||
next_run = now.replace(month=now.month + 1, day=1)
|
||||
next_run = next_run.replace(
|
||||
hour=config.scheduled_time.hour,
|
||||
minute=config.scheduled_time.minute,
|
||||
second=0,
|
||||
microsecond=0
|
||||
)
|
||||
else:
|
||||
next_run = now + timedelta(days=1)
|
||||
|
||||
return next_run
|
||||
13
backend/igny8_core/business/automation/urls.py
Normal file
13
backend/igny8_core/business/automation/urls.py
Normal file
@@ -0,0 +1,13 @@
|
||||
"""
|
||||
Automation URLs
|
||||
"""
|
||||
from django.urls import path, include
|
||||
from rest_framework.routers import DefaultRouter
|
||||
from igny8_core.business.automation.views import AutomationViewSet
|
||||
|
||||
router = DefaultRouter()
|
||||
router.register(r'', AutomationViewSet, basename='automation')
|
||||
|
||||
urlpatterns = [
|
||||
path('', include(router.urls)),
|
||||
]
|
||||
313
backend/igny8_core/business/automation/views.py
Normal file
313
backend/igny8_core/business/automation/views.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""
|
||||
Automation API Views
|
||||
REST API endpoints for automation management
|
||||
"""
|
||||
from rest_framework import viewsets, status
|
||||
from rest_framework.decorators import action
|
||||
from rest_framework.response import Response
|
||||
from rest_framework.permissions import IsAuthenticated
|
||||
from django.shortcuts import get_object_or_404
|
||||
from django.utils import timezone
|
||||
|
||||
from igny8_core.business.automation.models import AutomationConfig, AutomationRun
|
||||
from igny8_core.business.automation.services import AutomationService
|
||||
from igny8_core.modules.system.models import Account, Site
|
||||
|
||||
|
||||
class AutomationViewSet(viewsets.ViewSet):
|
||||
"""API endpoints for automation"""
|
||||
permission_classes = [IsAuthenticated]
|
||||
|
||||
def _get_site(self, request):
|
||||
"""Get site from request"""
|
||||
site_id = request.query_params.get('site_id')
|
||||
if not site_id:
|
||||
return None, Response(
|
||||
{'error': 'site_id required'},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
site = get_object_or_404(Site, id=site_id, account__user=request.user)
|
||||
return site, None
|
||||
|
||||
@action(detail=False, methods=['get'])
|
||||
def config(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/config/?site_id=123
|
||||
Get automation configuration for site
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
config, _ = AutomationConfig.objects.get_or_create(
|
||||
account=site.account,
|
||||
site=site,
|
||||
defaults={
|
||||
'is_enabled': False,
|
||||
'frequency': 'daily',
|
||||
'scheduled_time': '02:00',
|
||||
}
|
||||
)
|
||||
|
||||
return Response({
|
||||
'is_enabled': config.is_enabled,
|
||||
'frequency': config.frequency,
|
||||
'scheduled_time': str(config.scheduled_time),
|
||||
'stage_1_batch_size': config.stage_1_batch_size,
|
||||
'stage_2_batch_size': config.stage_2_batch_size,
|
||||
'stage_3_batch_size': config.stage_3_batch_size,
|
||||
'stage_4_batch_size': config.stage_4_batch_size,
|
||||
'stage_5_batch_size': config.stage_5_batch_size,
|
||||
'stage_6_batch_size': config.stage_6_batch_size,
|
||||
'last_run_at': config.last_run_at,
|
||||
'next_run_at': config.next_run_at,
|
||||
})
|
||||
|
||||
@action(detail=False, methods=['put'])
|
||||
def update_config(self, request):
|
||||
"""
|
||||
PUT /api/v1/automation/update_config/?site_id=123
|
||||
Update automation configuration
|
||||
|
||||
Body:
|
||||
{
|
||||
"is_enabled": true,
|
||||
"frequency": "daily",
|
||||
"scheduled_time": "02:00",
|
||||
"stage_1_batch_size": 20,
|
||||
...
|
||||
}
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
config, _ = AutomationConfig.objects.get_or_create(
|
||||
account=site.account,
|
||||
site=site
|
||||
)
|
||||
|
||||
# Update fields
|
||||
if 'is_enabled' in request.data:
|
||||
config.is_enabled = request.data['is_enabled']
|
||||
if 'frequency' in request.data:
|
||||
config.frequency = request.data['frequency']
|
||||
if 'scheduled_time' in request.data:
|
||||
config.scheduled_time = request.data['scheduled_time']
|
||||
if 'stage_1_batch_size' in request.data:
|
||||
config.stage_1_batch_size = request.data['stage_1_batch_size']
|
||||
if 'stage_2_batch_size' in request.data:
|
||||
config.stage_2_batch_size = request.data['stage_2_batch_size']
|
||||
if 'stage_3_batch_size' in request.data:
|
||||
config.stage_3_batch_size = request.data['stage_3_batch_size']
|
||||
if 'stage_4_batch_size' in request.data:
|
||||
config.stage_4_batch_size = request.data['stage_4_batch_size']
|
||||
if 'stage_5_batch_size' in request.data:
|
||||
config.stage_5_batch_size = request.data['stage_5_batch_size']
|
||||
if 'stage_6_batch_size' in request.data:
|
||||
config.stage_6_batch_size = request.data['stage_6_batch_size']
|
||||
|
||||
config.save()
|
||||
|
||||
return Response({'message': 'Config updated'})
|
||||
|
||||
@action(detail=False, methods=['post'])
|
||||
def run_now(self, request):
|
||||
"""
|
||||
POST /api/v1/automation/run_now/?site_id=123
|
||||
Trigger automation run immediately
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
try:
|
||||
service = AutomationService(site.account, site)
|
||||
run_id = service.start_automation(trigger_type='manual')
|
||||
|
||||
# Start async processing
|
||||
from igny8_core.business.automation.tasks import run_automation_task
|
||||
run_automation_task.delay(run_id)
|
||||
|
||||
return Response({
|
||||
'run_id': run_id,
|
||||
'message': 'Automation started'
|
||||
})
|
||||
except ValueError as e:
|
||||
return Response(
|
||||
{'error': str(e)},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
except Exception as e:
|
||||
return Response(
|
||||
{'error': f'Failed to start automation: {str(e)}'},
|
||||
status=status.HTTP_500_INTERNAL_SERVER_ERROR
|
||||
)
|
||||
|
||||
@action(detail=False, methods=['get'])
|
||||
def current_run(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/current_run/?site_id=123
|
||||
Get current automation run status
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
run = AutomationRun.objects.filter(
|
||||
site=site,
|
||||
status__in=['running', 'paused']
|
||||
).order_by('-started_at').first()
|
||||
|
||||
if not run:
|
||||
return Response({'run': None})
|
||||
|
||||
return Response({
|
||||
'run': {
|
||||
'run_id': run.run_id,
|
||||
'status': run.status,
|
||||
'current_stage': run.current_stage,
|
||||
'trigger_type': run.trigger_type,
|
||||
'started_at': run.started_at,
|
||||
'total_credits_used': run.total_credits_used,
|
||||
'stage_1_result': run.stage_1_result,
|
||||
'stage_2_result': run.stage_2_result,
|
||||
'stage_3_result': run.stage_3_result,
|
||||
'stage_4_result': run.stage_4_result,
|
||||
'stage_5_result': run.stage_5_result,
|
||||
'stage_6_result': run.stage_6_result,
|
||||
'stage_7_result': run.stage_7_result,
|
||||
}
|
||||
})
|
||||
|
||||
@action(detail=False, methods=['post'])
|
||||
def pause(self, request):
|
||||
"""
|
||||
POST /api/v1/automation/pause/?run_id=abc123
|
||||
Pause automation run
|
||||
"""
|
||||
run_id = request.query_params.get('run_id')
|
||||
if not run_id:
|
||||
return Response(
|
||||
{'error': 'run_id required'},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
try:
|
||||
service = AutomationService.from_run_id(run_id)
|
||||
service.pause_automation()
|
||||
return Response({'message': 'Automation paused'})
|
||||
except AutomationRun.DoesNotExist:
|
||||
return Response(
|
||||
{'error': 'Run not found'},
|
||||
status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
@action(detail=False, methods=['post'])
|
||||
def resume(self, request):
|
||||
"""
|
||||
POST /api/v1/automation/resume/?run_id=abc123
|
||||
Resume paused automation run
|
||||
"""
|
||||
run_id = request.query_params.get('run_id')
|
||||
if not run_id:
|
||||
return Response(
|
||||
{'error': 'run_id required'},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
try:
|
||||
service = AutomationService.from_run_id(run_id)
|
||||
service.resume_automation()
|
||||
|
||||
# Resume async processing
|
||||
from igny8_core.business.automation.tasks import resume_automation_task
|
||||
resume_automation_task.delay(run_id)
|
||||
|
||||
return Response({'message': 'Automation resumed'})
|
||||
except AutomationRun.DoesNotExist:
|
||||
return Response(
|
||||
{'error': 'Run not found'},
|
||||
status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
@action(detail=False, methods=['get'])
|
||||
def history(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/history/?site_id=123
|
||||
Get automation run history
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
runs = AutomationRun.objects.filter(
|
||||
site=site
|
||||
).order_by('-started_at')[:20]
|
||||
|
||||
return Response({
|
||||
'runs': [
|
||||
{
|
||||
'run_id': run.run_id,
|
||||
'status': run.status,
|
||||
'trigger_type': run.trigger_type,
|
||||
'started_at': run.started_at,
|
||||
'completed_at': run.completed_at,
|
||||
'total_credits_used': run.total_credits_used,
|
||||
'current_stage': run.current_stage,
|
||||
}
|
||||
for run in runs
|
||||
]
|
||||
})
|
||||
|
||||
@action(detail=False, methods=['get'])
|
||||
def logs(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/logs/?run_id=abc123&lines=100
|
||||
Get automation run logs
|
||||
"""
|
||||
run_id = request.query_params.get('run_id')
|
||||
if not run_id:
|
||||
return Response(
|
||||
{'error': 'run_id required'},
|
||||
status=status.HTTP_400_BAD_REQUEST
|
||||
)
|
||||
|
||||
try:
|
||||
run = AutomationRun.objects.get(run_id=run_id)
|
||||
service = AutomationService(run.account, run.site)
|
||||
|
||||
lines = int(request.query_params.get('lines', 100))
|
||||
log_text = service.logger.get_activity_log(
|
||||
run_id, run.account.id, run.site.id, lines
|
||||
)
|
||||
|
||||
return Response({
|
||||
'run_id': run_id,
|
||||
'log': log_text
|
||||
})
|
||||
except AutomationRun.DoesNotExist:
|
||||
return Response(
|
||||
{'error': 'Run not found'},
|
||||
status=status.HTTP_404_NOT_FOUND
|
||||
)
|
||||
|
||||
@action(detail=False, methods=['get'])
|
||||
def estimate(self, request):
|
||||
"""
|
||||
GET /api/v1/automation/estimate/?site_id=123
|
||||
Estimate credits needed for automation
|
||||
"""
|
||||
site, error_response = self._get_site(request)
|
||||
if error_response:
|
||||
return error_response
|
||||
|
||||
service = AutomationService(site.account, site)
|
||||
estimated_credits = service.estimate_credits()
|
||||
|
||||
return Response({
|
||||
'estimated_credits': estimated_credits,
|
||||
'current_balance': site.account.credits_balance,
|
||||
'sufficient': site.account.credits_balance >= (estimated_credits * 1.2)
|
||||
})
|
||||
@@ -38,6 +38,11 @@ app.conf.beat_schedule = {
|
||||
'task': 'igny8_core.tasks.wordpress_publishing.retry_failed_wordpress_publications',
|
||||
'schedule': crontab(hour='*/6', minute=30), # Every 6 hours at :30
|
||||
},
|
||||
# Automation Tasks
|
||||
'check-scheduled-automations': {
|
||||
'task': 'automation.check_scheduled_automations',
|
||||
'schedule': crontab(minute=0), # Every hour at :00
|
||||
},
|
||||
}
|
||||
|
||||
@app.task(bind=True, ignore_result=True)
|
||||
|
||||
@@ -42,7 +42,7 @@ urlpatterns = [
|
||||
# Site Builder module removed - legacy blueprint functionality deprecated
|
||||
path('api/v1/system/', include('igny8_core.modules.system.urls')),
|
||||
path('api/v1/billing/', include('igny8_core.modules.billing.urls')), # Billing endpoints
|
||||
# path('api/v1/automation/', include('igny8_core.modules.automation.urls')), # Automation endpoints - REMOVED
|
||||
path('api/v1/automation/', include('igny8_core.business.automation.urls')), # Automation endpoints
|
||||
path('api/v1/linker/', include('igny8_core.modules.linker.urls')), # Linker endpoints
|
||||
path('api/v1/optimizer/', include('igny8_core.modules.optimizer.urls')), # Optimizer endpoints
|
||||
path('api/v1/publisher/', include('igny8_core.modules.publisher.urls')), # Publisher endpoints
|
||||
|
||||
@@ -35,6 +35,9 @@ const Images = lazy(() => import("./pages/Writer/Images"));
|
||||
const Review = lazy(() => import("./pages/Writer/Review"));
|
||||
const Published = lazy(() => import("./pages/Writer/Published"));
|
||||
|
||||
// Automation Module - Lazy loaded
|
||||
const AutomationPage = lazy(() => import("./pages/Automation/AutomationPage"));
|
||||
|
||||
// Linker Module - Lazy loaded
|
||||
const LinkerDashboard = lazy(() => import("./pages/Linker/Dashboard"));
|
||||
const LinkerContentList = lazy(() => import("./pages/Linker/ContentList"));
|
||||
@@ -249,6 +252,12 @@ export default function App() {
|
||||
</Suspense>
|
||||
} />
|
||||
|
||||
{/* Automation Module */}
|
||||
<Route path="/automation" element={
|
||||
<Suspense fallback={null}>
|
||||
<AutomationPage />
|
||||
</Suspense>
|
||||
} />
|
||||
|
||||
{/* Linker Module - Redirect dashboard to content */}
|
||||
<Route path="/linker" element={<Navigate to="/linker/content" replace />} />
|
||||
|
||||
58
frontend/src/components/Automation/ActivityLog.tsx
Normal file
58
frontend/src/components/Automation/ActivityLog.tsx
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Activity Log Component
|
||||
* Real-time log viewer for automation runs
|
||||
*/
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { automationService } from '../../services/automationService';
|
||||
|
||||
interface ActivityLogProps {
|
||||
runId: string;
|
||||
}
|
||||
|
||||
const ActivityLog: React.FC<ActivityLogProps> = ({ runId }) => {
|
||||
const [logs, setLogs] = useState<string>('');
|
||||
const [lines, setLines] = useState<number>(100);
|
||||
|
||||
useEffect(() => {
|
||||
loadLogs();
|
||||
|
||||
// Poll every 3 seconds
|
||||
const interval = setInterval(loadLogs, 3000);
|
||||
return () => clearInterval(interval);
|
||||
}, [runId, lines]);
|
||||
|
||||
const loadLogs = async () => {
|
||||
try {
|
||||
const logText = await automationService.getLogs(runId, lines);
|
||||
setLogs(logText);
|
||||
} catch (error) {
|
||||
console.error('Failed to load logs', error);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="bg-white p-4 rounded-lg shadow">
|
||||
<div className="flex items-center justify-between mb-3">
|
||||
<h2 className="text-xl font-bold">Activity Log</h2>
|
||||
<div className="flex items-center gap-2">
|
||||
<label className="text-sm">Lines:</label>
|
||||
<select
|
||||
value={lines}
|
||||
onChange={(e) => setLines(parseInt(e.target.value))}
|
||||
className="border rounded px-2 py-1 text-sm"
|
||||
>
|
||||
<option value={50}>50</option>
|
||||
<option value={100}>100</option>
|
||||
<option value={200}>200</option>
|
||||
<option value={500}>500</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div className="bg-gray-900 text-green-400 p-4 rounded font-mono text-xs overflow-auto max-h-96">
|
||||
<pre className="whitespace-pre-wrap">{logs || 'No logs available'}</pre>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ActivityLog;
|
||||
237
frontend/src/components/Automation/ConfigModal.tsx
Normal file
237
frontend/src/components/Automation/ConfigModal.tsx
Normal file
@@ -0,0 +1,237 @@
|
||||
/**
|
||||
* Config Modal Component
|
||||
* Modal for configuring automation settings
|
||||
*/
|
||||
import React, { useState } from 'react';
|
||||
import { AutomationConfig } from '../../services/automationService';
|
||||
|
||||
interface ConfigModalProps {
|
||||
config: AutomationConfig;
|
||||
onSave: (config: Partial<AutomationConfig>) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
|
||||
const ConfigModal: React.FC<ConfigModalProps> = ({ config, onSave, onCancel }) => {
|
||||
const [formData, setFormData] = useState<Partial<AutomationConfig>>({
|
||||
is_enabled: config.is_enabled,
|
||||
frequency: config.frequency,
|
||||
scheduled_time: config.scheduled_time,
|
||||
stage_1_batch_size: config.stage_1_batch_size,
|
||||
stage_2_batch_size: config.stage_2_batch_size,
|
||||
stage_3_batch_size: config.stage_3_batch_size,
|
||||
stage_4_batch_size: config.stage_4_batch_size,
|
||||
stage_5_batch_size: config.stage_5_batch_size,
|
||||
stage_6_batch_size: config.stage_6_batch_size,
|
||||
});
|
||||
|
||||
const handleSubmit = (e: React.FormEvent) => {
|
||||
e.preventDefault();
|
||||
onSave(formData);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-50">
|
||||
<div className="bg-white rounded-lg p-6 max-w-2xl w-full max-h-screen overflow-y-auto">
|
||||
<h2 className="text-2xl font-bold mb-4">Automation Configuration</h2>
|
||||
|
||||
<form onSubmit={handleSubmit}>
|
||||
{/* Enable/Disable */}
|
||||
<div className="mb-4">
|
||||
<label className="flex items-center">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={formData.is_enabled || false}
|
||||
onChange={(e) =>
|
||||
setFormData({ ...formData, is_enabled: e.target.checked })
|
||||
}
|
||||
className="mr-2"
|
||||
/>
|
||||
<span className="font-semibold">Enable Automation</span>
|
||||
</label>
|
||||
<p className="text-sm text-gray-600 ml-6">
|
||||
When enabled, automation will run on the configured schedule
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Frequency */}
|
||||
<div className="mb-4">
|
||||
<label className="block font-semibold mb-1">Frequency</label>
|
||||
<select
|
||||
value={formData.frequency || 'daily'}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
frequency: e.target.value as 'daily' | 'weekly' | 'monthly',
|
||||
})
|
||||
}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
>
|
||||
<option value="daily">Daily</option>
|
||||
<option value="weekly">Weekly (Mondays)</option>
|
||||
<option value="monthly">Monthly (1st of month)</option>
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{/* Scheduled Time */}
|
||||
<div className="mb-4">
|
||||
<label className="block font-semibold mb-1">Scheduled Time</label>
|
||||
<input
|
||||
type="time"
|
||||
value={formData.scheduled_time || '02:00'}
|
||||
onChange={(e) =>
|
||||
setFormData({ ...formData, scheduled_time: e.target.value })
|
||||
}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
<p className="text-sm text-gray-600 mt-1">
|
||||
Time of day to run automation (24-hour format)
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Batch Sizes */}
|
||||
<div className="mb-4">
|
||||
<h3 className="font-semibold mb-2">Batch Sizes</h3>
|
||||
<p className="text-sm text-gray-600 mb-3">
|
||||
Configure how many items to process in each stage
|
||||
</p>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 1: Keywords → Clusters
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_1_batch_size || 20}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_1_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={100}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 2: Clusters → Ideas
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_2_batch_size || 1}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_2_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={10}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 3: Ideas → Tasks
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_3_batch_size || 20}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_3_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={100}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 4: Tasks → Content
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_4_batch_size || 1}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_4_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={10}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 5: Content → Image Prompts
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_5_batch_size || 1}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_5_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={10}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label className="block text-sm mb-1">
|
||||
Stage 6: Image Prompts → Images
|
||||
</label>
|
||||
<input
|
||||
type="number"
|
||||
value={formData.stage_6_batch_size || 1}
|
||||
onChange={(e) =>
|
||||
setFormData({
|
||||
...formData,
|
||||
stage_6_batch_size: parseInt(e.target.value),
|
||||
})
|
||||
}
|
||||
min={1}
|
||||
max={10}
|
||||
className="border rounded px-3 py-2 w-full"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Buttons */}
|
||||
<div className="flex justify-end gap-2 mt-6">
|
||||
<button
|
||||
type="button"
|
||||
onClick={onCancel}
|
||||
className="px-4 py-2 bg-gray-200 text-gray-800 rounded hover:bg-gray-300"
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
className="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600"
|
||||
>
|
||||
Save Configuration
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default ConfigModal;
|
||||
114
frontend/src/components/Automation/RunHistory.tsx
Normal file
114
frontend/src/components/Automation/RunHistory.tsx
Normal file
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* Run History Component
|
||||
* Shows past automation runs
|
||||
*/
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { automationService, RunHistoryItem } from '../../services/automationService';
|
||||
|
||||
interface RunHistoryProps {
|
||||
siteId: number;
|
||||
}
|
||||
|
||||
const RunHistory: React.FC<RunHistoryProps> = ({ siteId }) => {
|
||||
const [history, setHistory] = useState<RunHistoryItem[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
|
||||
useEffect(() => {
|
||||
loadHistory();
|
||||
}, [siteId]);
|
||||
|
||||
const loadHistory = async () => {
|
||||
try {
|
||||
setLoading(true);
|
||||
const data = await automationService.getHistory(siteId);
|
||||
setHistory(data);
|
||||
} catch (error) {
|
||||
console.error('Failed to load history', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const getStatusBadge = (status: string) => {
|
||||
const colors: Record<string, string> = {
|
||||
completed: 'bg-green-100 text-green-800',
|
||||
running: 'bg-blue-100 text-blue-800',
|
||||
paused: 'bg-yellow-100 text-yellow-800',
|
||||
failed: 'bg-red-100 text-red-800',
|
||||
};
|
||||
return colors[status] || 'bg-gray-100 text-gray-800';
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return <div className="text-center py-4">Loading history...</div>;
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="bg-white p-4 rounded-lg shadow">
|
||||
<h2 className="text-xl font-bold mb-4">Run History</h2>
|
||||
|
||||
{history.length === 0 ? (
|
||||
<div className="text-center text-gray-600 py-8">No automation runs yet</div>
|
||||
) : (
|
||||
<div className="overflow-x-auto">
|
||||
<table className="min-w-full">
|
||||
<thead className="bg-gray-50">
|
||||
<tr>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Run ID
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Status
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Trigger
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Started
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Completed
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Credits Used
|
||||
</th>
|
||||
<th className="px-4 py-2 text-left text-xs font-medium text-gray-500 uppercase">
|
||||
Stage
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-gray-200">
|
||||
{history.map((run) => (
|
||||
<tr key={run.run_id} className="hover:bg-gray-50">
|
||||
<td className="px-4 py-3 text-sm font-mono">{run.run_id.slice(0, 8)}...</td>
|
||||
<td className="px-4 py-3">
|
||||
<span
|
||||
className={`px-2 py-1 rounded-full text-xs font-semibold ${getStatusBadge(
|
||||
run.status
|
||||
)}`}
|
||||
>
|
||||
{run.status}
|
||||
</span>
|
||||
</td>
|
||||
<td className="px-4 py-3 text-sm capitalize">{run.trigger_type}</td>
|
||||
<td className="px-4 py-3 text-sm">
|
||||
{new Date(run.started_at).toLocaleString()}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-sm">
|
||||
{run.completed_at
|
||||
? new Date(run.completed_at).toLocaleString()
|
||||
: '-'}
|
||||
</td>
|
||||
<td className="px-4 py-3 text-sm">{run.total_credits_used}</td>
|
||||
<td className="px-4 py-3 text-sm">{run.current_stage}/7</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default RunHistory;
|
||||
58
frontend/src/components/Automation/StageCard.tsx
Normal file
58
frontend/src/components/Automation/StageCard.tsx
Normal file
@@ -0,0 +1,58 @@
|
||||
/**
|
||||
* Stage Card Component
|
||||
* Shows status and results for each automation stage
|
||||
*/
|
||||
import React from 'react';
|
||||
import { StageResult } from '../../services/automationService';
|
||||
|
||||
interface StageCardProps {
|
||||
stageNumber: number;
|
||||
stageName: string;
|
||||
currentStage: number;
|
||||
result: StageResult | null;
|
||||
}
|
||||
|
||||
const StageCard: React.FC<StageCardProps> = ({
|
||||
stageNumber,
|
||||
stageName,
|
||||
currentStage,
|
||||
result,
|
||||
}) => {
|
||||
const isPending = stageNumber > currentStage;
|
||||
const isActive = stageNumber === currentStage;
|
||||
const isComplete = stageNumber < currentStage || (result !== null && stageNumber <= 7);
|
||||
|
||||
const getStatusColor = () => {
|
||||
if (isActive) return 'border-blue-500 bg-blue-50';
|
||||
if (isComplete) return 'border-green-500 bg-green-50';
|
||||
return 'border-gray-300 bg-gray-50';
|
||||
};
|
||||
|
||||
const getStatusIcon = () => {
|
||||
if (isActive) return '🔄';
|
||||
if (isComplete) return '✅';
|
||||
return '⏳';
|
||||
};
|
||||
|
||||
return (
|
||||
<div className={`border-2 rounded-lg p-3 ${getStatusColor()}`}>
|
||||
<div className="flex items-center justify-between mb-2">
|
||||
<div className="text-sm font-bold">Stage {stageNumber}</div>
|
||||
<div className="text-xl">{getStatusIcon()}</div>
|
||||
</div>
|
||||
<div className="text-xs text-gray-700 mb-2">{stageName}</div>
|
||||
{result && (
|
||||
<div className="text-xs space-y-1">
|
||||
{Object.entries(result).map(([key, value]) => (
|
||||
<div key={key} className="flex justify-between">
|
||||
<span className="text-gray-600">{key.replace(/_/g, ' ')}:</span>
|
||||
<span className="font-semibold">{value}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default StageCard;
|
||||
@@ -125,6 +125,15 @@ const AppSidebar: React.FC = () => {
|
||||
});
|
||||
}
|
||||
|
||||
// Add Automation (always available if Writer is enabled)
|
||||
if (moduleEnabled('writer')) {
|
||||
workflowItems.push({
|
||||
icon: <BoltIcon />,
|
||||
name: "Automation",
|
||||
path: "/automation",
|
||||
});
|
||||
}
|
||||
|
||||
// Add Linker if enabled (single item, no dropdown)
|
||||
if (moduleEnabled('linker')) {
|
||||
workflowItems.push({
|
||||
|
||||
308
frontend/src/pages/Automation/AutomationPage.tsx
Normal file
308
frontend/src/pages/Automation/AutomationPage.tsx
Normal file
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* Automation Dashboard Page
|
||||
* Main page for managing AI automation pipeline
|
||||
*/
|
||||
import React, { useState, useEffect } from 'react';
|
||||
import { useToast } from '../../components/ui/toast/ToastContainer';
|
||||
import { useSiteStore } from '../../store/siteStore';
|
||||
import { automationService, AutomationRun, AutomationConfig } from '../../services/automationService';
|
||||
import StageCard from '../../components/Automation/StageCard';
|
||||
import ActivityLog from '../../components/Automation/ActivityLog';
|
||||
import ConfigModal from '../../components/Automation/ConfigModal';
|
||||
import RunHistory from '../../components/Automation/RunHistory';
|
||||
|
||||
const STAGE_NAMES = [
|
||||
'Keywords → Clusters',
|
||||
'Clusters → Ideas',
|
||||
'Ideas → Tasks',
|
||||
'Tasks → Content',
|
||||
'Content → Image Prompts',
|
||||
'Image Prompts → Images',
|
||||
'Manual Review Gate',
|
||||
];
|
||||
|
||||
const AutomationPage: React.FC = () => {
|
||||
const { activeSite } = useSiteStore();
|
||||
const { showToast } = useToast();
|
||||
const [config, setConfig] = useState<AutomationConfig | null>(null);
|
||||
const [currentRun, setCurrentRun] = useState<AutomationRun | null>(null);
|
||||
const [showConfigModal, setShowConfigModal] = useState(false);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [estimate, setEstimate] = useState<{ estimated_credits: number; current_balance: number; sufficient: boolean } | null>(null);
|
||||
|
||||
// Poll for current run updates
|
||||
useEffect(() => {
|
||||
if (!activeSite) return;
|
||||
|
||||
loadData();
|
||||
|
||||
// Poll every 5 seconds when run is active
|
||||
const interval = setInterval(() => {
|
||||
if (currentRun && (currentRun.status === 'running' || currentRun.status === 'paused')) {
|
||||
loadCurrentRun();
|
||||
}
|
||||
}, 5000);
|
||||
|
||||
return () => clearInterval(interval);
|
||||
}, [activeSite, currentRun?.status]);
|
||||
|
||||
const loadData = async () => {
|
||||
if (!activeSite) return;
|
||||
|
||||
try {
|
||||
setLoading(true);
|
||||
const [configData, runData, estimateData] = await Promise.all([
|
||||
automationService.getConfig(activeSite.id),
|
||||
automationService.getCurrentRun(activeSite.id),
|
||||
automationService.estimate(activeSite.id),
|
||||
]);
|
||||
setConfig(configData);
|
||||
setCurrentRun(runData.run);
|
||||
setEstimate(estimateData);
|
||||
} catch (error: any) {
|
||||
showToast('Failed to load automation data', 'error');
|
||||
console.error(error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
const loadCurrentRun = async () => {
|
||||
if (!activeSite) return;
|
||||
|
||||
try {
|
||||
const data = await automationService.getCurrentRun(activeSite.id);
|
||||
setCurrentRun(data.run);
|
||||
} catch (error) {
|
||||
console.error('Failed to poll current run', error);
|
||||
}
|
||||
};
|
||||
|
||||
const handleRunNow = async () => {
|
||||
if (!activeSite) return;
|
||||
|
||||
// Check credit balance
|
||||
if (estimate && !estimate.sufficient) {
|
||||
showToast(`Insufficient credits. Need ~${estimate.estimated_credits}, you have ${estimate.current_balance}`, 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await automationService.runNow(activeSite.id);
|
||||
showToast('Automation started', 'success');
|
||||
loadCurrentRun();
|
||||
} catch (error: any) {
|
||||
showToast(error.response?.data?.error || 'Failed to start automation', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
const handlePause = async () => {
|
||||
if (!currentRun) return;
|
||||
|
||||
try {
|
||||
await automationService.pause(currentRun.run_id);
|
||||
showToast('Automation paused', 'success');
|
||||
loadCurrentRun();
|
||||
} catch (error) {
|
||||
showToast('Failed to pause automation', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
const handleResume = async () => {
|
||||
if (!currentRun) return;
|
||||
|
||||
try {
|
||||
await automationService.resume(currentRun.run_id);
|
||||
showToast('Automation resumed', 'success');
|
||||
loadCurrentRun();
|
||||
} catch (error) {
|
||||
showToast('Failed to resume automation', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
const handleSaveConfig = async (newConfig: Partial<AutomationConfig>) => {
|
||||
if (!activeSite) return;
|
||||
|
||||
try {
|
||||
await automationService.updateConfig(activeSite.id, newConfig);
|
||||
showToast('Configuration saved', 'success');
|
||||
setShowConfigModal(false);
|
||||
loadData();
|
||||
} catch (error) {
|
||||
showToast('Failed to save configuration', 'error');
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="flex items-center justify-center h-screen">
|
||||
<div className="text-xl">Loading automation...</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
if (!activeSite) {
|
||||
return (
|
||||
<div className="flex items-center justify-center h-screen">
|
||||
<div className="text-xl">Please select a site</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="container mx-auto px-4 py-6">
|
||||
{/* Header */}
|
||||
<div className="mb-6">
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<div>
|
||||
<h1 className="text-3xl font-bold">AI Automation Pipeline</h1>
|
||||
<p className="text-gray-600 mt-1">
|
||||
Automated content creation from keywords to published articles
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => setShowConfigModal(true)}
|
||||
className="px-4 py-2 bg-gray-200 text-gray-800 rounded hover:bg-gray-300"
|
||||
>
|
||||
Configure
|
||||
</button>
|
||||
{currentRun?.status === 'running' && (
|
||||
<button
|
||||
onClick={handlePause}
|
||||
className="px-4 py-2 bg-yellow-500 text-white rounded hover:bg-yellow-600"
|
||||
>
|
||||
Pause
|
||||
</button>
|
||||
)}
|
||||
{currentRun?.status === 'paused' && (
|
||||
<button
|
||||
onClick={handleResume}
|
||||
className="px-4 py-2 bg-blue-500 text-white rounded hover:bg-blue-600"
|
||||
>
|
||||
Resume
|
||||
</button>
|
||||
)}
|
||||
{!currentRun && (
|
||||
<button
|
||||
onClick={handleRunNow}
|
||||
className="px-4 py-2 bg-green-500 text-white rounded hover:bg-green-600 disabled:opacity-50"
|
||||
disabled={!config?.is_enabled}
|
||||
>
|
||||
Run Now
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Status Bar */}
|
||||
{config && (
|
||||
<div className="bg-white p-4 rounded-lg shadow">
|
||||
<div className="grid grid-cols-4 gap-4">
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Status</div>
|
||||
<div className="font-semibold">
|
||||
{config.is_enabled ? (
|
||||
<span className="text-green-600">Enabled</span>
|
||||
) : (
|
||||
<span className="text-gray-600">Disabled</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Schedule</div>
|
||||
<div className="font-semibold capitalize">
|
||||
{config.frequency} at {config.scheduled_time}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Last Run</div>
|
||||
<div className="font-semibold">
|
||||
{config.last_run_at
|
||||
? new Date(config.last_run_at).toLocaleString()
|
||||
: 'Never'}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Estimated Credits</div>
|
||||
<div className="font-semibold">
|
||||
{estimate?.estimated_credits || 0} credits
|
||||
{estimate && !estimate.sufficient && (
|
||||
<span className="text-red-600 ml-2">(Insufficient)</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Current Run Status */}
|
||||
{currentRun && (
|
||||
<div className="mb-6">
|
||||
<div className="bg-white p-4 rounded-lg shadow">
|
||||
<h2 className="text-xl font-bold mb-4">
|
||||
Current Run: {currentRun.run_id}
|
||||
</h2>
|
||||
<div className="grid grid-cols-4 gap-4 mb-4">
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Status</div>
|
||||
<div className="font-semibold capitalize">{currentRun.status}</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Current Stage</div>
|
||||
<div className="font-semibold">
|
||||
Stage {currentRun.current_stage}: {STAGE_NAMES[currentRun.current_stage - 1]}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Started</div>
|
||||
<div className="font-semibold">
|
||||
{new Date(currentRun.started_at).toLocaleString()}
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div className="text-sm text-gray-600">Credits Used</div>
|
||||
<div className="font-semibold">{currentRun.total_credits_used}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Stage Progress */}
|
||||
<div className="grid grid-cols-7 gap-2">
|
||||
{STAGE_NAMES.map((name, index) => (
|
||||
<StageCard
|
||||
key={index}
|
||||
stageNumber={index + 1}
|
||||
stageName={name}
|
||||
currentStage={currentRun.current_stage}
|
||||
result={currentRun[`stage_${index + 1}_result` as keyof AutomationRun] as any}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Activity Log */}
|
||||
{currentRun && (
|
||||
<div className="mb-6">
|
||||
<ActivityLog runId={currentRun.run_id} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Run History */}
|
||||
<RunHistory siteId={activeSite.id} />
|
||||
|
||||
{/* Config Modal */}
|
||||
{showConfigModal && config && (
|
||||
<ConfigModal
|
||||
config={config}
|
||||
onSave={handleSaveConfig}
|
||||
onCancel={() => setShowConfigModal(false)}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default AutomationPage;
|
||||
144
frontend/src/services/automationService.ts
Normal file
144
frontend/src/services/automationService.ts
Normal file
@@ -0,0 +1,144 @@
|
||||
/**
|
||||
* Automation API Service
|
||||
*/
|
||||
import { fetchAPI } from './api';
|
||||
|
||||
export interface AutomationConfig {
|
||||
is_enabled: boolean;
|
||||
frequency: 'daily' | 'weekly' | 'monthly';
|
||||
scheduled_time: string;
|
||||
stage_1_batch_size: number;
|
||||
stage_2_batch_size: number;
|
||||
stage_3_batch_size: number;
|
||||
stage_4_batch_size: number;
|
||||
stage_5_batch_size: number;
|
||||
stage_6_batch_size: number;
|
||||
last_run_at: string | null;
|
||||
next_run_at: string | null;
|
||||
}
|
||||
|
||||
export interface StageResult {
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
export interface AutomationRun {
|
||||
run_id: string;
|
||||
status: 'running' | 'paused' | 'completed' | 'failed';
|
||||
current_stage: number;
|
||||
trigger_type: 'manual' | 'scheduled';
|
||||
started_at: string;
|
||||
total_credits_used: number;
|
||||
stage_1_result: StageResult | null;
|
||||
stage_2_result: StageResult | null;
|
||||
stage_3_result: StageResult | null;
|
||||
stage_4_result: StageResult | null;
|
||||
stage_5_result: StageResult | null;
|
||||
stage_6_result: StageResult | null;
|
||||
stage_7_result: StageResult | null;
|
||||
}
|
||||
|
||||
export interface RunHistoryItem {
|
||||
run_id: string;
|
||||
status: string;
|
||||
trigger_type: string;
|
||||
started_at: string;
|
||||
completed_at: string | null;
|
||||
total_credits_used: number;
|
||||
current_stage: number;
|
||||
}
|
||||
|
||||
function buildUrl(endpoint: string, params?: Record<string, any>): string {
|
||||
let url = `/v1/automation${endpoint}`;
|
||||
if (params) {
|
||||
const query = new URLSearchParams(
|
||||
Object.entries(params)
|
||||
.filter(([, v]) => v != null)
|
||||
.map(([k, v]) => [k, String(v)])
|
||||
);
|
||||
const queryStr = query.toString();
|
||||
if (queryStr) {
|
||||
url += `?${queryStr}`;
|
||||
}
|
||||
}
|
||||
return url;
|
||||
}
|
||||
|
||||
export const automationService = {
|
||||
/**
|
||||
* Get automation configuration for site
|
||||
*/
|
||||
getConfig: async (siteId: number): Promise<AutomationConfig> => {
|
||||
return fetchAPI(buildUrl('/config/', { site_id: siteId }));
|
||||
},
|
||||
|
||||
/**
|
||||
* Update automation configuration
|
||||
*/
|
||||
updateConfig: async (siteId: number, config: Partial<AutomationConfig>): Promise<void> => {
|
||||
await fetchAPI(buildUrl('/update_config/', { site_id: siteId }), {
|
||||
method: 'PUT',
|
||||
body: JSON.stringify(config),
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Trigger automation run now
|
||||
*/
|
||||
runNow: async (siteId: number): Promise<{ run_id: string; message: string }> => {
|
||||
return fetchAPI(buildUrl('/run_now/', { site_id: siteId }), {
|
||||
method: 'POST',
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Get current automation run status
|
||||
*/
|
||||
getCurrentRun: async (siteId: number): Promise<{ run: AutomationRun | null }> => {
|
||||
return fetchAPI(buildUrl('/current_run/', { site_id: siteId }));
|
||||
},
|
||||
|
||||
/**
|
||||
* Pause automation run
|
||||
*/
|
||||
pause: async (runId: string): Promise<void> => {
|
||||
await fetchAPI(buildUrl('/pause/', { run_id: runId }), {
|
||||
method: 'POST',
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Resume paused automation run
|
||||
*/
|
||||
resume: async (runId: string): Promise<void> => {
|
||||
await fetchAPI(buildUrl('/resume/', { run_id: runId }), {
|
||||
method: 'POST',
|
||||
});
|
||||
},
|
||||
|
||||
/**
|
||||
* Get automation run history
|
||||
*/
|
||||
getHistory: async (siteId: number): Promise<RunHistoryItem[]> => {
|
||||
const response = await fetchAPI(buildUrl('/history/', { site_id: siteId }));
|
||||
return response.runs;
|
||||
},
|
||||
|
||||
/**
|
||||
* Get automation run logs
|
||||
*/
|
||||
getLogs: async (runId: string, lines: number = 100): Promise<string> => {
|
||||
const response = await fetchAPI(buildUrl('/logs/', { run_id: runId, lines }));
|
||||
return response.log;
|
||||
},
|
||||
|
||||
/**
|
||||
* Estimate credits needed
|
||||
*/
|
||||
estimate: async (siteId: number): Promise<{
|
||||
estimated_credits: number;
|
||||
current_balance: number;
|
||||
sufficient: boolean;
|
||||
}> => {
|
||||
return fetchAPI(buildUrl('/estimate/', { site_id: siteId }));
|
||||
},
|
||||
};
|
||||
Reference in New Issue
Block a user