diff --git a/backend/igny8_core/business/automation/services/automation_service.py b/backend/igny8_core/business/automation/services/automation_service.py index c46bce64..1466e51c 100644 --- a/backend/igny8_core/business/automation/services/automation_service.py +++ b/backend/igny8_core/business/automation/services/automation_service.py @@ -98,6 +98,18 @@ class AutomationService: raise ValueError("Automation already running for this site (cache lock)") try: + # Capture initial queue snapshot for accurate progress tracking + # Do this BEFORE credit check to validate there's work to do + initial_snapshot = self._capture_initial_snapshot() + + # Check if there are any items to process across all stages + total_pending = initial_snapshot.get('total_initial_items', 0) + if total_pending == 0: + raise ValueError( + "No items available to process. Add keywords, clusters, ideas, tasks, " + "or content to the pipeline before running automation." + ) + # Estimate credits needed estimated_credits = self.estimate_credits() @@ -109,9 +121,6 @@ class AutomationService: # Create run_id and log files run_id = self.logger.start_run(self.account.id, self.site.id, trigger_type) - # Capture initial queue snapshot for accurate progress tracking - initial_snapshot = self._capture_initial_snapshot() - # Create AutomationRun record self.run = AutomationRun.objects.create( run_id=run_id, @@ -1609,14 +1618,6 @@ class AutomationService: cache.delete(f'automation_lock_{self.site.id}') logger.info(f"[AutomationService] Stage 7 complete: {approved_count} content pieces approved and published") - self.run.status = 'completed' - self.run.completed_at = datetime.now() - self.run.save() - - # Release lock - cache.delete(f'automation_lock_{self.site.id}') - - logger.info(f"[AutomationService] Stage 7 complete: Automation ended, {total_count} content ready for review") def pause_automation(self): """Pause current automation run""" diff --git a/docs/50-DEPLOYMENT/WORDPRESS-INTEGRATION-FLOW.md b/docs/50-DEPLOYMENT/WORDPRESS-INTEGRATION-FLOW.md new file mode 100644 index 00000000..f643d782 --- /dev/null +++ b/docs/50-DEPLOYMENT/WORDPRESS-INTEGRATION-FLOW.md @@ -0,0 +1,574 @@ +# WordPress Integration & Publishing Flow - Complete Technical Documentation + +**Last Updated:** January 1, 2026 +**Version:** 1.3.0 +**Status:** Production Active + +--- + +## Table of Contents + +1. [System Overview](#1-system-overview) +2. [Integration Setup Flow](#2-integration-setup-flow) +3. [Manual Publishing Flow](#3-manual-publishing-flow) +4. [Automation Publishing Flow](#4-automation-publishing-flow) +5. [Webhook Sync Flow (WordPress → IGNY8)](#5-webhook-sync-flow-wordpress--igny8) +6. [Metadata Sync Flow](#6-metadata-sync-flow) +7. [Data Models & Storage](#7-data-models--storage) +8. [Current Implementation Gaps](#8-current-implementation-gaps) +9. [Flow Diagrams](#9-flow-diagrams) + +--- + +## 1. System Overview + +### Architecture Summary + +IGNY8 integrates with WordPress sites through a **custom WordPress plugin** (`igny8-wp-bridge`) that: +- Receives content from IGNY8 via a custom REST endpoint (`/wp-json/igny8/v1/publish`) +- Sends status updates back to IGNY8 via webhooks +- Authenticates using API keys stored in both systems + +### Communication Pattern + +``` +IGNY8 App ←→ WordPress Site + │ │ + │ HTTP POST │ + ├─────────────→│ Publish content via /wp-json/igny8/v1/publish + │ │ + │ HTTP POST │ + │←─────────────┤ Webhook status updates via /api/v1/integration/webhooks/wordpress/status/ + │ │ +``` + +### Key Components + +| Component | Location | Purpose | +|-----------|----------|---------| +| SiteIntegration Model | `business/integration/models.py` | Stores WordPress credentials & config | +| SyncEvent Model | `business/integration/models.py` | Logs all sync operations | +| Celery Task | `tasks/wordpress_publishing.py` | Background publishing worker | +| Webhook Handler | `modules/integration/webhooks.py` | Receives WordPress status updates | +| Frontend Form | `components/sites/WordPressIntegrationForm.tsx` | User setup UI | + +--- + +## 2. Integration Setup Flow + +### 2.1 Pre-requisites (WordPress Side) + +1. WordPress 5.6+ with REST API enabled +2. Pretty permalinks enabled (Settings → Permalinks) +3. IGNY8 WordPress Bridge plugin installed and activated +4. No security plugins blocking REST API + +### 2.2 Setup Steps (User Flow) + +**Step 1: User navigates to Site Settings** +- Frontend: `/sites/{id}/settings` → WordPress Integration section +- Component: `WordPressIntegrationForm.tsx` + +**Step 2: User clicks "Generate API Key"** +- Frontend calls: `POST /v1/integration/integrations/generate-api-key/` +- Body: `{ "site_id": 123 }` +- Backend creates/updates `SiteIntegration` record with new API key + +**Step 3: User copies API key and configures WordPress plugin** +- User downloads plugin from GitHub releases +- Installs in WordPress: Plugins → Add New → Upload +- Configures plugin with: + - IGNY8 API URL: `https://app.igny8.com` + - Site API Key: (copied from IGNY8) + - Site ID: (shown in IGNY8) + +**Step 4: Test Connection** +- User clicks "Test Connection" in either app +- IGNY8 calls: `GET {wordpress_url}/wp-json/wp/v2/users/me` +- Uses API key in `X-IGNY8-API-KEY` header +- Success: Connection verified, `is_active` set to true +- Failure: Error message displayed + +### 2.3 Data Created During Setup + +**SiteIntegration Record:** +```json +{ + "id": 1, + "site_id": 123, + "account_id": 456, + "platform": "wordpress", + "platform_type": "cms", + "config_json": { + "site_url": "https://example.com" + }, + "credentials_json": { + "api_key": "igny8_xxxxxxxxxxxxxxxxxxxx" + }, + "is_active": true, + "sync_enabled": true, + "sync_status": "pending" +} +``` + +--- + +## 3. Manual Publishing Flow + +### 3.1 Trigger Points + +1. **Content Review Page** - "Publish to WordPress" button +2. **Content Approved Page** - Publish action in context menu +3. **Bulk Actions** - Select multiple, publish all + +### 3.2 Detailed Flow + +**Step 1: User clicks "Publish to WordPress"** +- Frontend: `ContentViewSet.publish` action called +- Endpoint: `POST /api/v1/writer/content/{id}/publish/` +- Optional body: `{ "site_integration_id": 123 }` + +**Step 2: Backend validates content** +- Checks content exists and belongs to user's site +- Checks content not already published (`external_id` must be null) +- Finds active WordPress integration for the site +- Error if no integration found + +**Step 3: Optimistic status update** +- Content status changed to `published` immediately +- User sees success message +- Actual WordPress publishing happens async + +**Step 4: Celery task queued** +- Task: `publish_content_to_wordpress.delay(content_id, site_integration_id)` +- Task name: `igny8_core.tasks.wordpress_publishing` + +**Step 5: Celery task execution (Background)** + +The task performs these steps: + +1. **Load data** - Get Content and SiteIntegration from database +2. **Check if already published** - Skip if `external_id` exists +3. **Generate excerpt** - Strip HTML, take first 150 chars +4. **Load taxonomy terms** - Categories from `taxonomy_terms` M2M field, fallback to cluster name +5. **Load tags** - From `taxonomy_terms` + primary/secondary keywords +6. **Load images** - Featured image and gallery images, convert paths to URLs +7. **Build payload:** + ```json + { + "content_id": 123, + "title": "Post Title", + "content_html": "

Full HTML content...

", + "excerpt": "Short excerpt...", + "status": "publish", + "seo_title": "SEO Title", + "seo_description": "Meta description", + "primary_keyword": "main keyword", + "secondary_keywords": ["kw1", "kw2"], + "featured_image_url": "https://app.igny8.com/images/...", + "gallery_images": [{ "url": "...", "alt": "", "caption": "" }], + "categories": ["Category Name"], + "tags": ["tag1", "tag2"] + } + ``` +8. **Send to WordPress:** + - URL: `{site_url}/wp-json/igny8/v1/publish` + - Headers: `X-IGNY8-API-Key: {api_key}` + - Method: POST + - Timeout: 30 seconds + +**Step 6: Process WordPress response** + +| HTTP Code | Meaning | Action | +|-----------|---------|--------| +| 201 | Created | Update content with `external_id`, `external_url`, log success | +| 409 | Already exists | Update content with existing WordPress post data | +| 4xx/5xx | Error | Log failure, retry up to 3 times with exponential backoff | + +**Step 7: Update IGNY8 content record** +```python +content.external_id = str(wp_post_id) +content.external_url = wp_post_url +content.status = 'published' +content.metadata['wordpress_status'] = 'publish' +content.save() +``` + +**Step 8: Create SyncEvent record** +```python +SyncEvent.objects.create( + integration=site_integration, + site=content.site, + account=content.account, + event_type='publish', + action='content_publish', + description=f"Published content '{title}' to WordPress", + success=True, + content_id=content.id, + external_id=external_id, + details={...} +) +``` + +### 3.3 Error Handling + +- **Timeout**: Retry after 60 seconds (first attempt), then exponentially +- **Connection Error**: Same retry logic +- **Max Retries**: 3 attempts total +- **Final Failure**: SyncEvent created with `success=False`, error logged + +--- + +## 4. Automation Publishing Flow + +### 4.1 Automation Stage 7: Review → Published + +Automation Stage 7 handles auto-approval and publishing: + +**Current Implementation:** +- Content in `review` status is changed to `published` status +- Status change only - **NO automatic WordPress push currently implemented** +- Lock released, automation run marked complete + +**What DOES NOT happen automatically:** +- No `publish_content_to_wordpress` task is triggered +- Content sits in `published` status waiting for manual publish or scheduled task + +### 4.2 Scheduled Publishing Task (NOT CURRENTLY SCHEDULED) + +There exists a task `process_pending_wordpress_publications()` that: +- Finds content with `status='published'` and `external_id=NULL` +- Queues each item to `publish_content_to_wordpress` task +- Processes max 50 items per run + +**CURRENT STATUS: This task is NOT in Celery Beat schedule!** + +Looking at `celery.py`, the beat schedule includes: +- `check-scheduled-automations` (hourly) +- `replenish-monthly-credits` (monthly) +- Various maintenance tasks + +**Missing:** +- `process_pending_wordpress_publications` is NOT scheduled + +### 4.3 To Enable Auto-Publishing After Automation + +Two options: + +**Option A: Add to Celery Beat Schedule** +```python +'process-pending-wordpress-publications': { + 'task': 'igny8_core.tasks.wordpress_publishing.process_pending_wordpress_publications', + 'schedule': crontab(minute='*/5'), # Every 5 minutes +}, +``` + +**Option B: Call directly from Stage 7** +Modify `run_stage_7()` to queue publish tasks for each approved content item. + +--- + +## 5. Webhook Sync Flow (WordPress → IGNY8) + +### 5.1 Purpose + +Keeps IGNY8 in sync when content is modified directly in WordPress: +- Post status changed (published → draft, etc.) +- Post deleted/trashed +- Post metadata updated + +### 5.2 Webhook Endpoints + +| Endpoint | Purpose | +|----------|---------| +| `POST /api/v1/integration/webhooks/wordpress/status/` | Status changes | +| `POST /api/v1/integration/webhooks/wordpress/metadata/` | Metadata updates | + +### 5.3 Status Webhook Flow + +**Step 1: WordPress plugin detects status change** +- Hooks into `transition_post_status` action +- Collects: post_id, content_id, new_status, post_url + +**Step 2: Plugin sends webhook to IGNY8** +``` +POST https://app.igny8.com/api/v1/integration/webhooks/wordpress/status/ + +Headers: + X-IGNY8-API-KEY: {api_key} + +Body: +{ + "post_id": 123, + "content_id": 456, + "post_status": "publish", + "post_url": "https://example.com/my-post/", + "post_title": "My Post Title", + "site_url": "https://example.com" +} +``` + +**Step 3: IGNY8 validates and processes** + +1. Extract API key from header +2. Find Content by `content_id` +3. Find SiteIntegration by site_url + platform +4. Verify API key matches stored key +5. Map WordPress status to IGNY8 status: + | WordPress | IGNY8 | + |-----------|-------| + | publish | published | + | draft | draft | + | pending | review | + | private | published | + | trash | draft | + | future | review | +6. Update Content record: + ```python + content.external_id = str(post_id) + content.external_url = post_url + content.status = mapped_status + content.metadata['wordpress_status'] = post_status + content.metadata['last_wp_sync'] = now + content.save() + ``` +7. Create SyncEvent log + +### 5.4 Metadata Webhook Flow + +Similar to status webhook but updates `content.metadata['wp_metadata']` with: +- Categories +- Tags +- Author info +- Modified date + +--- + +## 6. Metadata Sync Flow + +### 6.1 Manual Sync (User-Initiated) + +**Trigger:** User clicks "Sync Now" in Site Settings + +**Endpoint:** `POST /api/v1/integration/integrations/{id}/sync/` + +**What it does:** +- Calls `SyncMetadataService.sync_wordpress_structure()` +- Fetches from WordPress: + - Post types and counts + - Categories list + - Tags list + - Site metadata +- Updates integration's `last_sync_at` +- Does NOT push/pull content + +### 6.2 Structure Update + +**Endpoint:** `POST /api/v1/integration/integrations/{id}/update_structure/` + +Refreshes understanding of WordPress site: +- Available post types +- Taxonomy structures +- Capabilities + +--- + +## 7. Data Models & Storage + +### 7.1 SiteIntegration + +| Field | Type | Purpose | +|-------|------|---------| +| id | AutoField | Primary key | +| account | FK(Account) | Owner account | +| site | FK(Site) | IGNY8 site | +| platform | CharField | 'wordpress' | +| platform_type | CharField | 'cms' | +| config_json | JSONField | `{ "site_url": "https://..." }` | +| credentials_json | JSONField | `{ "api_key": "igny8_xxx" }` | +| is_active | Boolean | Connection enabled | +| sync_enabled | Boolean | Two-way sync enabled | +| last_sync_at | DateTime | Last successful sync | +| sync_status | CharField | pending/success/failed/syncing | +| sync_error | TextField | Last error message | + +### 7.2 SyncEvent + +| Field | Type | Purpose | +|-------|------|---------| +| id | AutoField | Primary key | +| integration | FK(SiteIntegration) | Parent integration | +| site | FK(Site) | Related site | +| account | FK(Account) | Owner account | +| event_type | CharField | publish/sync/error/webhook/metadata_sync | +| action | CharField | content_publish/status_update/etc | +| description | TextField | Human-readable event description | +| success | Boolean | Event outcome | +| content_id | Integer | IGNY8 content ID | +| external_id | CharField | WordPress post ID | +| error_message | TextField | Error details if failed | +| details | JSONField | Additional event data | +| duration_ms | Integer | Operation duration | +| created_at | DateTime | Event timestamp | + +### 7.3 Content Fields (Publishing Related) + +| Field | Purpose | +|-------|---------| +| external_id | WordPress post ID (string) | +| external_url | WordPress post URL | +| status | IGNY8 status (draft/review/published) | +| metadata['wordpress_status'] | WordPress status (publish/draft/etc) | +| metadata['last_wp_sync'] | Last webhook sync timestamp | + +--- + +## 8. Current Implementation Gaps + +### 8.1 Missing: Scheduled Auto-Publishing + +**Problem:** Content approved by Automation Stage 7 is not automatically pushed to WordPress. + +**Current State:** +- `process_pending_wordpress_publications()` task exists +- Task is NOT in Celery Beat schedule +- Content remains in `published` status with `external_id=NULL` + +**Impact:** Users must manually publish each content item even after automation completes. + +### 8.2 Missing: Pull Sync (WordPress → IGNY8 Content) + +**Problem:** No way to import existing WordPress posts into IGNY8. + +**Current State:** +- Webhooks only handle status updates for content that originated from IGNY8 +- No "Import from WordPress" feature +- No scheduled pull sync + +### 8.3 Missing: Content Update Sync + +**Problem:** Editing content in IGNY8 after publishing doesn't update WordPress. + +**Current State:** +- Only initial publish is supported +- No "republish" or "update" functionality +- `external_id` check prevents re-publishing + +### 8.4 Missing: Image Upload to WordPress + +**Problem:** Featured images are passed as URLs, not uploaded to WordPress media library. + +**Current State:** +- Image URL is sent in payload +- WordPress plugin must download and create attachment +- If plugin doesn't handle this, no featured image + +### 8.5 Missing: Conflict Resolution + +**Problem:** If content is edited in both IGNY8 and WordPress, there's no merge strategy. + +**Current State:** +- Last write wins +- No version tracking +- No conflict detection + +--- + +## 9. Flow Diagrams + +### 9.1 Integration Setup + +``` +┌──────────┐ ┌──────────────┐ ┌───────────────┐ +│ User │ │ IGNY8 App │ │ WordPress │ +└────┬─────┘ └──────┬───────┘ └───────┬───────┘ + │ │ │ + │ 1. Open Site Settings │ + ├─────────────────>│ │ + │ │ │ + │ 2. Generate API Key │ + ├─────────────────>│ │ + │ │ │ + │<─────────────────┤ │ + │ 3. Display API Key │ + │ │ │ + │ 4. Install Plugin──────────────────────┼──────────> + │ │ │ + │ 5. Enter API Key in Plugin─────────────┼──────────> + │ │ │ + │ 6. Test Connection │ + ├─────────────────>│ │ + │ │ 7. GET /wp-json/... │ + │ ├────────────────────>│ + │ │<────────────────────┤ + │<─────────────────┤ 8. Success │ + │ │ │ +``` + +### 9.2 Manual Publishing + +``` +┌──────────┐ ┌──────────────┐ ┌──────────┐ ┌───────────────┐ +│ User │ │ IGNY8 API │ │ Celery │ │ WordPress │ +└────┬─────┘ └──────┬───────┘ └────┬─────┘ └───────┬───────┘ + │ │ │ │ + │ 1. Click Publish │ │ │ + ├─────────────────>│ │ │ + │ │ │ │ + │ │ 2. Validate │ │ + │ │ 3. Update status │ │ + │ │ 4. Queue task │ │ + │ ├─────────────────>│ │ + │<─────────────────┤ │ │ + │ 5. "Publishing..." │ │ + │ │ │ │ + │ │ │ 6. POST /publish │ + │ │ ├──────────────────>│ + │ │ │ │ + │ │ │<──────────────────┤ + │ │ │ 7. 201 Created │ + │ │ │ │ + │ │ 8. Update Content│ │ + │ │<─────────────────┤ │ + │ │ 9. Create SyncEvent │ + │ │ │ │ +``` + +### 9.3 Webhook Status Sync + +``` +┌───────────────┐ ┌──────────────┐ +│ WordPress │ │ IGNY8 API │ +└───────┬───────┘ └──────┬───────┘ + │ │ + │ 1. User changes status in WP + │ │ + │ 2. POST /webhooks/wordpress/status/ + ├───────────────────>│ + │ │ + │ │ 3. Validate API key + │ │ 4. Find Content + │ │ 5. Map status + │ │ 6. Update Content + │ │ 7. Create SyncEvent + │ │ + │<───────────────────┤ + │ 8. 200 OK │ + │ │ +``` + +--- + +## Summary + +| Flow | Status | Notes | +|------|--------|-------| +| Integration Setup | ✅ Working | API key based | +| Manual Publish | ✅ Working | Via Celery task | +| Automation Publish | ⚠️ Partial | Stage 7 sets status but doesn't trigger WordPress push | +| Webhook Status Sync | ✅ Working | WordPress → IGNY8 status updates | +| Webhook Metadata Sync | ✅ Working | WordPress → IGNY8 metadata | +| Scheduled Auto-Publish | ❌ Not Active | Task exists but not scheduled | +| Content Pull Sync | ❌ Not Implemented | No import from WordPress | +| Content Update Sync | ❌ Not Implemented | No republish capability | + diff --git a/frontend/src/layout/AppSidebar.tsx b/frontend/src/layout/AppSidebar.tsx index 0bd98a4d..a9a7bffa 100644 --- a/frontend/src/layout/AppSidebar.tsx +++ b/frontend/src/layout/AppSidebar.tsx @@ -70,14 +70,12 @@ const AppSidebar: React.FC = () => { // SETUP section items - Ordered: Sites → Add Keywords → Content Settings → Thinker const setupItems: NavItem[] = []; - // Add Sites first (if enabled) - if (isModuleEnabled('site_builder')) { - setupItems.push({ - icon: , - name: "Sites", - path: "/sites", - }); - } + // Sites is always visible - it's core functionality for managing sites + setupItems.push({ + icon: , + name: "Sites", + path: "/sites", + }); // Add Keywords second setupItems.push({