From 5d96e1a2bdf678814844f446756aa070b978e7d5 Mon Sep 17 00:00:00 2001 From: "IGNY8 VPS (Salman)" Date: Wed, 3 Dec 2025 07:40:25 +0000 Subject: [PATCH] cleanup docs --- AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md | 1185 ----------------- LOGGING-REFERENCE.md | 317 ----- STATUS-UNIFIED-STRUCTURE-PLAN-v2.md | 541 -------- STATUS-UNIFIED-STRUCTURE-PLAN.md | 526 -------- ...EYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md | 0 docs/SITEBUILDER-REMOVAL-MIGRATION-GUIDE.md | 351 ----- docs/SITEBUILDER-REMOVAL-SUMMARY.md | 364 ----- .../STATUS-IMPLEMENTATION-TABLES.md | 0 docs/future-docs/current-state.md | 116 -- docs/future-docs/nex-plan.md | 75 -- .../SITES_DECISION_GUIDE.md | 424 ------ .../SITES_REMOVAL_GUIDE.md | 743 ----------- .../SITES_REMOVAL_QUICK_REFERENCE.md | 329 ----- ...-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md | 0 ...-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md | 0 .../DEPLOYMENT-GUIDE-WP-FIXES.md | 0 .../WORDPRESS-INTEGRATION-FIXES-2025-11-30.md | 0 ...GRATION-FIXES-IMPLEMENTATION-2025-12-01.md | 0 ...SS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md | 0 .../WORDPRESS-PUBLISHING-FIELD-MAPPING.md | 0 ...WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md | 0 .../WP-PLUGIN-REFACTOR-PLAN.md | 0 .../WP-PUBLISHING-SIMPLE-REFERENCE.md | 0 .../WP-REFACTOR-IMPLEMENTATION-SUMMARY.md | 0 24 files changed, 4971 deletions(-) delete mode 100644 AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md delete mode 100644 LOGGING-REFERENCE.md delete mode 100644 STATUS-UNIFIED-STRUCTURE-PLAN-v2.md delete mode 100644 STATUS-UNIFIED-STRUCTURE-PLAN.md rename KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md => docs/KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md (100%) delete mode 100644 docs/SITEBUILDER-REMOVAL-MIGRATION-GUIDE.md delete mode 100644 docs/SITEBUILDER-REMOVAL-SUMMARY.md rename STATUS-IMPLEMENTATION-TABLES.md => docs/STATUS-IMPLEMENTATION-TABLES.md (100%) delete mode 100644 docs/future-docs/current-state.md delete mode 100644 docs/future-docs/nex-plan.md delete mode 100644 docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_DECISION_GUIDE.md delete mode 100644 docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_GUIDE.md delete mode 100644 docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_QUICK_REFERENCE.md rename docs/{ => wordpress-integration-and-other}/03-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md (100%) rename docs/{ => wordpress-integration-and-other}/04-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md (100%) rename docs/{ => wordpress-integration-and-other}/DEPLOYMENT-GUIDE-WP-FIXES.md (100%) rename docs/{ => wordpress-integration-and-other}/WORDPRESS-INTEGRATION-FIXES-2025-11-30.md (100%) rename docs/{ => wordpress-integration-and-other}/WORDPRESS-INTEGRATION-FIXES-IMPLEMENTATION-2025-12-01.md (100%) rename docs/{ => wordpress-integration-and-other}/WORDPRESS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md (100%) rename docs/{ => wordpress-integration-and-other}/WORDPRESS-PUBLISHING-FIELD-MAPPING.md (100%) rename docs/{ => wordpress-integration-and-other}/WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md (100%) rename docs/{ => wordpress-integration-and-other}/WP-PLUGIN-REFACTOR-PLAN.md (100%) rename docs/{ => wordpress-integration-and-other}/WP-PUBLISHING-SIMPLE-REFERENCE.md (100%) rename docs/{ => wordpress-integration-and-other}/WP-REFACTOR-IMPLEMENTATION-SUMMARY.md (100%) diff --git a/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md b/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md deleted file mode 100644 index 44af3e0f..00000000 --- a/AUTOMATION-IMPLEMENTATION-PLAN-COMPLETE.md +++ /dev/null @@ -1,1185 +0,0 @@ -# AI Automation Pipeline - Implementation Blueprint -**Date:** December 3, 2025 -**Purpose:** Site-level automation orchestrating existing AI functions into sequential 7-stage pipeline - ---- - -## 🎯 EXECUTIVE SUMMARY - -### What We're Building -A **site-level automation page** (`/automation`) that orchestrates 6 existing AI functions + 1 local function into a **strictly sequential** 7-stage pipeline for hands-free content generation from keywords to draft content ready for review. - -### Core Principles -✅ **Zero Duplication** - Reuse all existing AI functions (auto_cluster, generate_ideas, generate_content, generate_image_prompts, generate_images) -✅ **Strictly Sequential** - Stage N+1 ONLY starts when Stage N is 100% complete -✅ **Batch Processing** - Within each stage, process items in configurable batches with queues visible -✅ **Site-Level Scope** - NO sector filtering - operates on entire site's data -✅ **Observable** - Real-time batch progress, detailed queue counts, stage-by-stage logs -✅ **Safe Execution** - Distributed locks, credit reservations, idempotent stages, Celery task chaining - -### Sequential Stage Execution -- **Stage completes** → Trigger next stage automatically -- **Within stage** → Process batches sequentially until queue empty -- **Between stages** → Hard stop, verify completion, then proceed -- **Never parallel** - Only 1 stage active at a time per site - -### Automation Stops Before Publishing -- **Stage 7 (Manual Review Gate)** - Automation ends when content is draft+images ready -- User manually reviews and publishes via existing bulk actions -- No automated publishing to WordPress (human oversight required) - ---- - -## 🏗️ SCOPE & DATA MODEL - -### Site-Level Operation (NO Sector) -``` -┌─────────────────────────────────────┐ -│ Site: "example.com" │ -│ ├─ Keywords (ALL sectors combined) │ -│ ├─ Clusters (ALL sectors combined) │ -│ ├─ Ideas (ALL sectors combined) │ -│ └─ Tasks (ALL sectors combined) │ -└─────────────────────────────────────┘ - -UI: Only Site selector at top -Database queries: Filter by site_id only -No sector dropdown in automation page -``` - -### Why Site-Level? -- **Simplicity** - User manages automation per website, not per topic -- **Unified Progress** - See total content pipeline for entire site -- **Flexible Sectors** - Content can span multiple sectors naturally -- **Easier Scheduling** - One automation config per site - ---- - -## 📋 EXISTING AI FUNCTIONS (Reused, Not Duplicated) - -| Function | File | Input | Output | Credits | Already Works | -|----------|------|-------|--------|---------|---------------| -| **auto_cluster** | `ai/functions/auto_cluster.py` | Keyword IDs (max 20) | Clusters created | 1 per 5 keywords | ✅ Yes | -| **generate_ideas** | `ai/functions/generate_ideas.py` | Cluster IDs (max 5) | Ideas created | 2 per cluster | ✅ Yes | -| **generate_content** | `ai/functions/generate_content.py` | Task IDs (1 at a time) | Content draft | 1 per 500 words | ✅ Yes | -| **generate_image_prompts** | `ai/functions/generate_image_prompts.py` | Content IDs | Image prompts | 0.5 per prompt | ✅ Yes | -| **generate_images** | `ai/functions/generate_images.py` | Image prompt IDs | Generated images | 1-4 per image | ✅ Yes | -| **bulk_queue_to_writer** | `modules/planner/views.py#L1084` | Idea IDs | Tasks created | 0 (local) | ✅ Yes | - -**All functions already:** -- Have async Celery tasks -- Return task_id for progress tracking -- Deduct credits automatically -- Update model statuses (new → mapped → queued → completed) -- Handle errors gracefully - ---- - -## 🏗️ NEW COMPONENTS TO BUILD - -### Phase 1: Backend Infrastructure - -#### 1.1 Database Models -**File:** `backend/igny8_core/business/automation/models.py` - -```python -class AutomationRun(SiteSectorBaseModel): - """Track each automation run""" - run_id = models.CharField(max_length=100, unique=True, db_index=True) - # Format: run_20251203_140523_manual or run_20251204_020000_scheduled - - trigger_type = models.CharField(max_length=20, choices=[ - ('manual', 'Manual'), - ('scheduled', 'Scheduled') - ]) - - status = models.CharField(max_length=20, choices=[ - ('running', 'Running'), - ('paused', 'Paused'), - ('completed', 'Completed'), - ('failed', 'Failed') - ], default='running') - - current_stage = models.IntegerField(default=1) # 1-7 - - started_at = models.DateTimeField(auto_now_add=True) - completed_at = models.DateTimeField(null=True, blank=True) - - total_credits_used = models.IntegerField(default=0) - - # Stage results (JSON) - stage_1_result = models.JSONField(default=dict, blank=True) # {clusters_created: 8, keywords_processed: 47} - stage_2_result = models.JSONField(default=dict, blank=True) # {ideas_created: 56} - stage_3_result = models.JSONField(default=dict, blank=True) # {tasks_created: 56} - stage_4_result = models.JSONField(default=dict, blank=True) # {content_created: 56} - stage_5_result = models.JSONField(default=dict, blank=True) # {prompts_created: 224} - stage_6_result = models.JSONField(default=dict, blank=True) # {images_created: 224} - stage_7_result = models.JSONField(default=dict, blank=True) # {ready_for_review: 56} - - error_message = models.TextField(blank=True, null=True) - - class Meta: - ordering = ['-started_at'] - indexes = [ - models.Index(fields=['run_id']), - models.Index(fields=['site', 'sector', '-started_at']), - ] - - -class AutomationConfig(SiteSectorBaseModel): - """Store automation schedule and settings per site/sector""" - - is_enabled = models.BooleanField(default=False) - - # Schedule - frequency = models.CharField(max_length=20, choices=[ - ('daily', 'Daily'), - ('weekly', 'Weekly'), - ('monthly', 'Monthly') - ], default='daily') - - scheduled_time = models.TimeField(default='02:00') # 2:00 AM - - # Batch sizes (sensible defaults from plan) - stage_1_batch_size = models.IntegerField(default=20) # Keywords per batch - stage_2_batch_size = models.IntegerField(default=1) # Clusters at a time - stage_3_batch_size = models.IntegerField(default=20) # Ideas per batch - stage_4_batch_size = models.IntegerField(default=1) # Tasks (sequential) - stage_5_batch_size = models.IntegerField(default=1) # Content at a time - stage_6_batch_size = models.IntegerField(default=1) # Images (auto-handled) - - last_run_at = models.DateTimeField(null=True, blank=True) - next_run_at = models.DateTimeField(null=True, blank=True) - - class Meta: - unique_together = [['site', 'sector']] -``` - -#### 1.2 Logging Service -**File:** `backend/igny8_core/business/automation/services/automation_logger.py` - -```python -import os -import logging -from datetime import datetime -from django.conf import settings - -class AutomationLogger: - """File-based logging for automation runs""" - - def __init__(self): - self.base_path = os.path.join(settings.BASE_DIR, 'logs', 'automation') - - def start_run(self, account_id, site_id, sector_id, trigger_type): - """Create run_id and log directory""" - timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') - run_id = f"run_{timestamp}_{trigger_type}" - - # Create directory: logs/automation/{account_id}/{site_id}/{sector_id}/{run_id}/ - log_dir = os.path.join( - self.base_path, - str(account_id), - str(site_id), - str(sector_id), - run_id - ) - os.makedirs(log_dir, exist_ok=True) - - # Create main log file - main_log = os.path.join(log_dir, 'automation_run.log') - with open(main_log, 'w') as f: - f.write(f"{'='*60}\\n") - f.write(f"AUTOMATION RUN: {run_id}\\n") - f.write(f"Started: {datetime.now()}\\n") - f.write(f"Trigger: {trigger_type}\\n") - f.write(f"{'='*60}\\n\\n") - - return run_id - - def log_stage_start(self, run_id, stage_number, stage_name, pending_count, account_id, site_id, sector_id): - """Log start of a stage""" - stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) - timestamp = datetime.now().strftime('%H:%M:%S') - - with open(stage_file, 'a') as f: - f.write(f"\\n{'='*60}\\n") - f.write(f"STAGE {stage_number}: {stage_name}\\n") - f.write(f"Started: {datetime.now()}\\n") - f.write(f"{'='*60}\\n\\n") - f.write(f"{timestamp} - Found {pending_count} items to process\\n") - - def log_stage_progress(self, run_id, stage_number, message, account_id, site_id, sector_id): - """Log progress within a stage""" - stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) - timestamp = datetime.now().strftime('%H:%M:%S') - - with open(stage_file, 'a') as f: - f.write(f"{timestamp} - {message}\\n") - - # Also append to main log - main_file = self._get_main_log(run_id, account_id, site_id, sector_id) - with open(main_file, 'a') as f: - f.write(f"{timestamp} - Stage {stage_number}: {message}\\n") - - def log_stage_complete(self, run_id, stage_number, stage_name, processed_count, time_elapsed, credits_used, account_id, site_id, sector_id): - """Log completion of a stage""" - stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) - - with open(stage_file, 'a') as f: - f.write(f"\\n{'='*60}\\n") - f.write(f"STAGE {stage_number} COMPLETE\\n") - f.write(f"Total Time: {time_elapsed}\\n") - f.write(f"Processed: {processed_count} items\\n") - f.write(f"Credits Used: {credits_used}\\n") - f.write(f"{'='*60}\\n") - - def log_stage_error(self, run_id, stage_number, error_message, account_id, site_id, sector_id): - """Log error in a stage""" - stage_file = self._get_stage_file(run_id, stage_number, account_id, site_id, sector_id) - timestamp = datetime.now().strftime('%H:%M:%S') - - with open(stage_file, 'a') as f: - f.write(f"\\n{timestamp} - ERROR: {error_message}\\n") - - def get_activity_log(self, run_id, account_id, site_id, sector_id, last_n=50): - """Get last N lines from main log""" - main_file = self._get_main_log(run_id, account_id, site_id, sector_id) - - if not os.path.exists(main_file): - return [] - - with open(main_file, 'r') as f: - lines = f.readlines() - return lines[-last_n:] # Last 50 lines - - def _get_stage_file(self, run_id, stage_number, account_id, site_id, sector_id): - """Get path to stage log file""" - log_dir = os.path.join( - self.base_path, - str(account_id), - str(site_id), - str(sector_id), - run_id - ) - return os.path.join(log_dir, f"stage_{stage_number}.log") - - def _get_main_log(self, run_id, account_id, site_id, sector_id): - """Get path to main log file""" - log_dir = os.path.join( - self.base_path, - str(account_id), - str(site_id), - str(sector_id), - run_id - ) - return os.path.join(log_dir, 'automation_run.log') -``` - -#### 1.3 Automation Service (Core Orchestrator) -**File:** `backend/igny8_core/business/automation/services/automation_service.py` - -```python -import time -from datetime import datetime, timedelta -from django.utils import timezone -from igny8_core.business.automation.models import AutomationRun, AutomationConfig -from igny8_core.business.automation.services.automation_logger import AutomationLogger -from igny8_core.business.planning.models import Keywords, Clusters, ContentIdeas -from igny8_core.business.content.models import Tasks, Content, Images -from igny8_core.business.billing.services.credit_service import CreditService - -# Import existing services (NO DUPLICATION) -from igny8_core.business.planning.services.clustering_service import ClusteringService -from igny8_core.business.planning.services.ideas_service import IdeasService -from igny8_core.ai.functions.generate_content import GenerateContentFunction -from igny8_core.ai.functions.generate_image_prompts import GenerateImagePromptsFunction -from igny8_core.ai.functions.generate_images import GenerateImagesFunction - - -class AutomationService: - """ - Orchestrates the 7-stage automation pipeline. - Reuses all existing AI functions - zero duplication. - """ - - def __init__(self, account, site, sector): - self.account = account - self.site = site - self.sector = sector - self.logger = AutomationLogger() - self.credit_service = CreditService() - - # Existing services - self.clustering_service = ClusteringService() - self.ideas_service = IdeasService() - self.content_function = GenerateContentFunction() - self.prompts_function = GenerateImagePromptsFunction() - self.images_function = GenerateImagesFunction() - - self.run = None - self.config = None - - def start_automation(self, trigger_type='manual'): - \"\"\" - Main entry point for automation. - Creates run record, executes stages sequentially. - \"\"\" - try: - # Create run record - run_id = self.logger.start_run( - self.account.id, - self.site.id, - self.sector.id, - trigger_type - ) - - self.run = AutomationRun.objects.create( - run_id=run_id, - trigger_type=trigger_type, - account=self.account, - site=self.site, - sector=self.sector, - status='running', - current_stage=1 - ) - - # Load config (for batch sizes) - self.config = AutomationConfig.objects.filter( - site=self.site, - sector=self.sector - ).first() - - if not self.config: - # Create default config - self.config = AutomationConfig.objects.create( - site=self.site, - sector=self.sector, - account=self.account - ) - - # Execute stages sequentially - self.run_stage_1() # Keywords → Clusters - self.run_stage_2() # Clusters → Ideas - self.run_stage_3() # Ideas → Tasks - self.run_stage_4() # Tasks → Content - self.run_stage_5() # Content → Image Prompts - self.run_stage_6() # Image Prompts → Images - self.run_stage_7() # Manual Review Gate - - # Mark complete - self.run.status = 'completed' - self.run.completed_at = timezone.now() - self.run.save() - - return { - 'success': True, - 'run_id': run_id, - 'message': 'Automation completed successfully' - } - - except Exception as e: - if self.run: - self.run.status = 'failed' - self.run.error_message = str(e) - self.run.save() - - return { - 'success': False, - 'error': str(e) - } - - def run_stage_1(self): - \"\"\"Stage 1: Keywords (status='new') → Clusters (AI)\"\"\" - stage_start = time.time() - stage_number = 1 - stage_name = "Keywords → Clusters (AI)" - - # Find pending keywords (status='new', cluster_id=null) - pending_keywords = Keywords.objects.filter( - site=self.site, - sector=self.sector, - status='new', - cluster__isnull=True, - disabled=False - ) - - total_count = pending_keywords.count() - - if total_count == 0: - self.logger.log_stage_progress( - self.run.run_id, stage_number, - "No pending keywords found - skipping stage", - self.account.id, self.site.id, self.sector.id - ) - self.run.current_stage = 2 - self.run.save() - return - - self.logger.log_stage_start( - self.run.run_id, stage_number, stage_name, total_count, - self.account.id, self.site.id, self.sector.id - ) - - # Process in batches (default 20 per batch) - batch_size = self.config.stage_1_batch_size - keyword_ids = list(pending_keywords.values_list('id', flat=True)) - - clusters_created = 0 - keywords_processed = 0 - credits_used = 0 - - for i in range(0, len(keyword_ids), batch_size): - batch = keyword_ids[i:i+batch_size] - batch_num = (i // batch_size) + 1 - total_batches = (len(keyword_ids) + batch_size - 1) // batch_size - - self.logger.log_stage_progress( - self.run.run_id, stage_number, - f"Processing batch {batch_num}/{total_batches} ({len(batch)} keywords)", - self.account.id, self.site.id, self.sector.id - ) - - # Call existing ClusteringService (REUSE - NO DUPLICATION) - result = self.clustering_service.cluster_keywords( - keyword_ids=batch, - account=self.account, - sector_id=self.sector.id - ) - - if result.get('success'): - clusters_created += result.get('clusters_created', 0) - keywords_processed += len(batch) - credits_used += result.get('credits_used', 0) - - self.logger.log_stage_progress( - self.run.run_id, stage_number, - f"Batch {batch_num} complete: {result.get('clusters_created', 0)} clusters created", - self.account.id, self.site.id, self.sector.id - ) - - # Save stage result - elapsed = time.time() - stage_start - self.run.stage_1_result = { - 'clusters_created': clusters_created, - 'keywords_processed': keywords_processed - } - self.run.total_credits_used += credits_used - self.run.current_stage = 2 - self.run.save() - - self.logger.log_stage_complete( - self.run.run_id, stage_number, stage_name, - keywords_processed, f"{elapsed:.0f}s", credits_used, - self.account.id, self.site.id, self.sector.id - ) - - def run_stage_2(self): - \"\"\"Stage 2: Clusters (status='new', no ideas) → Ideas (AI)\"\"\" - # Similar structure to stage_1 - # Calls existing IdeasService.generate_ideas() - pass - - def run_stage_3(self): - \"\"\"Stage 3: Ideas (status='new') → Tasks (Local queue)\"\"\" - # Calls existing bulk_queue_to_writer endpoint logic - pass - - def run_stage_4(self): - \"\"\"Stage 4: Tasks (status='queued') → Content (AI)\"\"\" - # Calls existing GenerateContentFunction - # Process one task at a time (sequential) - pass - - def run_stage_5(self): - \"\"\"Stage 5: Content (draft) → Image Prompts (AI)\"\"\" - # Calls existing GenerateImagePromptsFunction - pass - - def run_stage_6(self): - \"\"\"Stage 6: Image Prompts (pending) → Images (AI)\"\"\" - # Calls existing GenerateImagesFunction - # Handles batching automatically - pass - - def run_stage_7(self): - \"\"\"Stage 7: Manual Review Gate (STOP)\"\"\" - # Just count content ready for review - # Log final status - # Automation ends here - pass -``` - ---- - -### Phase 2: API Endpoints - -#### 2.1 Automation ViewSet -**File:** `backend/igny8_core/modules/automation/views.py` (NEW MODULE) - -```python -from rest_framework import viewsets -from rest_framework.decorators import action -from igny8_core.api.base import SiteSectorModelViewSet -from igny8_core.api.response import success_response, error_response -from igny8_core.business.automation.models import AutomationRun, AutomationConfig -from igny8_core.business.automation.services.automation_service import AutomationService -from igny8_core.business.automation.services.automation_logger import AutomationLogger - -class AutomationViewSet(SiteSectorModelViewSet): - """API endpoints for automation""" - - @action(detail=False, methods=['post']) - def run_now(self, request): - """Trigger manual automation run""" - account = request.account - site = request.site - sector = request.sector - - # Start automation - service = AutomationService(account, site, sector) - result = service.start_automation(trigger_type='manual') - - if result['success']: - return success_response(data={'run_id': result['run_id']}) - else: - return error_response(message=result['error']) - - @action(detail=False, methods=['get']) - def current_run(self, request): - """Get current/latest automation run status""" - site = request.site - sector = request.sector - - run = AutomationRun.objects.filter( - site=site, - sector=sector - ).order_by('-started_at').first() - - if not run: - return success_response(data={'run': None}) - - # Get activity log - logger = AutomationLogger() - activity = logger.get_activity_log( - run.run_id, - run.account_id, - run.site_id, - run.sector_id, - last_n=50 - ) - - return success_response(data={ - 'run': { - 'run_id': run.run_id, - 'status': run.status, - 'current_stage': run.current_stage, - 'trigger_type': run.trigger_type, - 'started_at': run.started_at, - 'total_credits_used': run.total_credits_used, - 'stage_1_result': run.stage_1_result, - 'stage_2_result': run.stage_2_result, - 'stage_3_result': run.stage_3_result, - 'stage_4_result': run.stage_4_result, - 'stage_5_result': run.stage_5_result, - 'stage_6_result': run.stage_6_result, - 'stage_7_result': run.stage_7_result, - }, - 'activity_log': activity - }) - - @action(detail=False, methods=['get', 'put']) - def config(self, request): - """Get/Update automation configuration""" - site = request.site - sector = request.sector - account = request.account - - if request.method == 'GET': - config, created = AutomationConfig.objects.get_or_create( - site=site, - sector=sector, - defaults={'account': account} - ) - - return success_response(data={ - 'is_enabled': config.is_enabled, - 'frequency': config.frequency, - 'scheduled_time': config.scheduled_time, - 'next_run_at': config.next_run_at, - 'stage_1_batch_size': config.stage_1_batch_size, - 'stage_2_batch_size': config.stage_2_batch_size, - 'stage_3_batch_size': config.stage_3_batch_size, - 'stage_4_batch_size': config.stage_4_batch_size, - 'stage_5_batch_size': config.stage_5_batch_size, - 'stage_6_batch_size': config.stage_6_batch_size, - }) - - elif request.method == 'PUT': - # Update configuration - config, created = AutomationConfig.objects.get_or_create( - site=site, - sector=sector, - defaults={'account': account} - ) - - # Update fields from request - config.is_enabled = request.data.get('is_enabled', config.is_enabled) - config.frequency = request.data.get('frequency', config.frequency) - config.scheduled_time = request.data.get('scheduled_time', config.scheduled_time) - config.save() - - return success_response(message='Configuration updated') -``` - -#### 2.2 URL Configuration -**File:** `backend/igny8_core/urls/api_urls.py` (ADD) - -```python -# Add to router -router.register(r'automation', AutomationViewSet, basename='automation') -``` - ---- - -### Phase 3: Celery Scheduled Task - -#### 3.1 Periodic Task for Scheduled Runs -**File:** `backend/igny8_core/tasks/automation_tasks.py` (NEW) - -```python -from celery import shared_task -from django.utils import timezone -from datetime import datetime, timedelta -from igny8_core.business.automation.models import AutomationConfig -from igny8_core.business.automation.services.automation_service import AutomationService -from igny8_core.auth.models import Account, Site, Sector - -@shared_task(name='run_scheduled_automation') -def run_scheduled_automation(): - \"\"\" - Celery beat task - runs every hour, checks if any configs need to run - \"\"\" - now = timezone.now() - - # Find configs that: - # 1. Are enabled - # 2. Have next_run_at <= now - configs = AutomationConfig.objects.filter( - is_enabled=True, - next_run_at__lte=now - ) - - for config in configs: - try: - # Load related objects - account = config.account - site = config.site - sector = config.sector - - # Start automation - service = AutomationService(account, site, sector) - service.start_automation(trigger_type='scheduled') - - # Calculate next run time - if config.frequency == 'daily': - next_run = now + timedelta(days=1) - elif config.frequency == 'weekly': - next_run = now + timedelta(weeks=1) - elif config.frequency == 'monthly': - next_run = now + timedelta(days=30) - - # Set time to scheduled_time - next_run = next_run.replace( - hour=config.scheduled_time.hour, - minute=config.scheduled_time.minute, - second=0, - microsecond=0 - ) - - config.last_run_at = now - config.next_run_at = next_run - config.save() - - except Exception as e: - # Log error but continue with other configs - print(f"Error running scheduled automation for {config.id}: {e}") - continue -``` - -#### 3.2 Register Celery Beat Schedule -**File:** `backend/igny8_core/celery.py` (UPDATE) - -```python -app.conf.beat_schedule = { - # ... existing schedules ... - 'run-scheduled-automation': { - 'task': 'run_scheduled_automation', - 'schedule': crontab(minute=0), # Every hour on the hour - }, -} -``` - ---- - -### Phase 4: Frontend Components - -#### 4.1 Automation Page Component -**File:** `frontend/src/pages/Automation/Dashboard.tsx` (NEW) - -```tsx -import { useState, useEffect } from 'react'; -import { useInterval } from '../../hooks/useInterval'; -import { automationApi } from '../../services/api/automationApi'; -import { useSiteStore } from '../../stores/siteStore'; -import { useSectorStore } from '../../stores/sectorStore'; -import Button from '../../components/ui/button/Button'; -import { BoltIcon, PlayIcon, PauseIcon, SettingsIcon } from '../../icons'; -import StageCard from './components/StageCard'; -import ActivityLog from './components/ActivityLog'; -import ConfigModal from './components/ConfigModal'; - -export default function AutomationDashboard() { - const { activeSite } = useSiteStore(); - const { activeSector } = useSectorStore(); - - const [currentRun, setCurrentRun] = useState(null); - const [activityLog, setActivityLog] = useState([]); - const [config, setConfig] = useState(null); - const [showConfigModal, setShowConfigModal] = useState(false); - const [isRunning, setIsRunning] = useState(false); - - // Poll current run status every 3 seconds when running - useInterval(() => { - if (activeSite && activeSector) { - loadCurrentRun(); - } - }, currentRun?.status === 'running' ? 3000 : null); - - useEffect(() => { - if (activeSite && activeSector) { - loadCurrentRun(); - loadConfig(); - } - }, [activeSite?.id, activeSector?.id]); - - const loadCurrentRun = async () => { - const response = await automationApi.getCurrentRun(); - if (response.success) { - setCurrentRun(response.data.run); - setActivityLog(response.data.activity_log || []); - } - }; - - const loadConfig = async () => { - const response = await automationApi.getConfig(); - if (response.success) { - setConfig(response.data); - } - }; - - const handleRunNow = async () => { - setIsRunning(true); - const response = await automationApi.runNow(); - if (response.success) { - // Start polling - loadCurrentRun(); - } - setIsRunning(false); - }; - - return ( -
- {/* Header */} -
-
-

- - AI Automation Pipeline -

-

- Automated content generation from keywords to review -

-
- -
- - - -
-
- - {/* Schedule Info */} - {config && ( -
-
-
-
- {config.is_enabled ? '⏰ Scheduled' : '⏸ Paused'} -
-
- {config.is_enabled - ? `Next Run: ${config.next_run_at} (${config.frequency})` - : 'Scheduling disabled' - } -
-
- -
-
- {currentRun?.total_credits_used || 0} -
-
- Credits Used -
-
-
-
- )} - - {/* Pipeline Overview */} -
-

Pipeline Overview

-
- - Keywords - - → - - Clusters - - → - - Ideas - - → - - Tasks - - → - - Content - - → - - Review - -
- - {currentRun && ( -
-
-
-
- - Stage {currentRun.current_stage}/7 - -
- )} -
- - {/* Stage Cards */} -
- - - - - - - - - - - - - -
- - {/* Activity Log */} - - - {/* Config Modal */} - {showConfigModal && ( - setShowConfigModal(false)} - config={config} - onSave={loadConfig} - /> - )} -
- ); -} - -function getStageStatus(run, stageNumber) { - if (!run) return 'waiting'; - if (run.status === 'running' && run.current_stage === stageNumber) return 'running'; - if (run.current_stage > stageNumber) return 'completed'; - return 'waiting'; -} -``` - -#### 4.2 Add to Sidebar Menu -**File:** `frontend/src/layouts/AppSidebar.tsx` (UPDATE) - -```tsx -// Add after Sites menu item -{ - path: '/automation', - label: 'Automation', - icon: , - badge: automationRunning ? { text: 'Running', color: 'green' } : null -}, -``` - -#### 4.3 API Service -**File:** `frontend/src/services/api/automationApi.ts` (NEW) - -```typescript -import { apiClient } from './apiClient'; - -export const automationApi = { - runNow: () => apiClient.post('/automation/run_now/'), - - getCurrentRun: () => apiClient.get('/automation/current_run/'), - - getConfig: () => apiClient.get('/automation/config/'), - - updateConfig: (data: any) => apiClient.put('/automation/config/', data), -}; -``` - ---- - -### Phase 5: Implementation Checklist - -#### Week 1: Backend Foundation -- [ ] Create `automation` module directory structure -- [ ] Implement `AutomationRun` and `AutomationConfig` models -- [ ] Run migrations -- [ ] Implement `AutomationLogger` service -- [ ] Test file logging manually - -#### Week 2: Core Service -- [ ] Implement `AutomationService` class -- [ ] Implement `run_stage_1()` (Keywords → Clusters) -- [ ] Test stage 1 in isolation -- [ ] Implement `run_stage_2()` (Clusters → Ideas) -- [ ] Test stage 2 in isolation -- [ ] Implement remaining stages 3-7 - -#### Week 3: API & Scheduling -- [ ] Create `AutomationViewSet` with endpoints -- [ ] Test manual run via API -- [ ] Implement Celery periodic task -- [ ] Test scheduled runs -- [ ] Add error handling and rollback - -#### Week 4: Frontend -- [ ] Create Automation page component -- [ ] Implement StageCard component -- [ ] Implement ActivityLog component -- [ ] Implement ConfigModal component -- [ ] Add to sidebar menu -- [ ] Test full UI flow - -#### Week 5: Testing & Polish -- [ ] End-to-end testing (manual + scheduled) -- [ ] Load testing (100+ keywords) -- [ ] Credit calculation verification -- [ ] Log file verification -- [ ] UI polish and responsiveness -- [ ] Documentation update - ---- - -### Phase 6: Safety Mechanisms - -#### 6.1 Pause/Resume -```python -# In AutomationService -def pause_run(self): - self.run.status = 'paused' - self.run.save() - -def resume_run(self): - self.run.status = 'running' - self.run.save() - # Resume from current_stage -``` - -#### 6.2 Rollback on Error -```python -# Each stage wraps in try-except -try: - self.run_stage_1() -except Exception as e: - self.logger.log_stage_error(self.run.run_id, 1, str(e)) - self.run.status = 'failed' - self.run.error_message = str(e) - self.run.save() - # Optionally: rollback created records - raise -``` - -#### 6.3 Credit Pre-Check -```python -# Before starting, estimate total credits needed -def estimate_credits(self): - keywords = Keywords.objects.filter(status='new', cluster__isnull=True).count() - clusters = Clusters.objects.filter(ideas__isnull=True).count() - # ... etc - - total_estimate = (keywords / 5) + (clusters * 2) + ... - - if self.account.credits_balance < total_estimate: - raise InsufficientCreditsError(f"Need ~{total_estimate} credits") -``` - ---- - -### Phase 7: Monitoring & Observability - -#### 7.1 Dashboard Metrics -- Total runs (today/week/month) -- Success rate -- Average credits per run -- Average time per stage -- Content pieces generated - -#### 7.2 Alerts -- Email when run completes -- Email on failure -- Slack notification (optional) - ---- - -## 🎨 UI/UX HIGHLIGHTS - -### Rich Visual Design -- **Stage Cards** with status badges (waiting/running/completed/failed) -- **Live Progress Bar** for current stage -- **Activity Feed** with timestamps and color-coded messages -- **Credit Counter** with real-time updates -- **Schedule Badge** showing next run time - -### User Experience -- **One-Click Run** - Single "Run Now" button -- **Real-Time Updates** - Auto-refreshes every 3 seconds when running -- **Clear Status** - Visual indicators for each stage -- **Easy Config** - Modal for schedule settings -- **Error Clarity** - Detailed error messages with stage number - ---- - -## 🔧 TROUBLESHOOTING GUIDE - -### Issue: Stage stuck in "running" -**Solution:** -1. Check `/logs/automation/{account}/{site}/{sector}/{run_id}/stage_X.log` -2. Look for last log entry -3. Check Celery worker logs -4. Manually mark stage complete or restart - -### Issue: Credits deducted but no results -**Solution:** -1. Check stage log for AI task_id -2. Query task progress endpoint -3. Verify AI function completed -4. Rollback transaction if needed - -### Issue: Duplicate clusters created -**Solution:** -1. Add unique constraint on cluster name per sector -2. Check deduplication logic in ClusteringService -3. Review stage_1 logs for batch processing - ---- - -## 📊 SUCCESS METRICS - -After implementation, measure: -- **Automation adoption rate** (% of sites using scheduled runs) -- **Content generation volume** (pieces per day/week) -- **Time savings** (manual hours vs automated) -- **Credit efficiency** (credits per content piece) -- **Error rate** (failed runs / total runs) - ---- - -## 🚀 FUTURE ENHANCEMENTS - -### Phase 8: Advanced Features -- **Conditional stages** (skip if no data) -- **Parallel processing** (multiple tasks at once in stage 4) -- **Smart scheduling** (avoid peak hours) -- **A/B testing** (test different prompts) -- **Content quality scoring** (auto-reject low scores) - -### Phase 9: Integrations -- **WordPress auto-publish** (with approval workflow) -- **Analytics tracking** (measure content performance) -- **Social media posting** (auto-share published content) - ---- - -**END OF IMPLEMENTATION PLAN** - -This plan provides a complete, production-ready automation system that: -✅ Reuses all existing AI functions (zero duplication) -✅ Modular and maintainable (each stage independent) -✅ Observable and debuggable (file logs + database records) -✅ Safe and reliable (error handling + rollback) -✅ Rich UI/UX (real-time updates + visual feedback) -✅ Scalable (handles 100+ keywords efficiently) diff --git a/LOGGING-REFERENCE.md b/LOGGING-REFERENCE.md deleted file mode 100644 index b7e0decd..00000000 --- a/LOGGING-REFERENCE.md +++ /dev/null @@ -1,317 +0,0 @@ -# IGNY8 Publish/Sync Logging System - -## Overview -Comprehensive file-based logging system for tracking all WordPress publish/sync workflows with site-specific stamps. - -## Log Format -All logs include site identification stamps in the format: `[site-id-domain]` - -Example: `[16-homeg8.com]` - ---- - -## Backend Logs (IGNY8 SaaS Platform) - -### Location -``` -/data/app/igny8/backend/logs/publish-sync-logs/ -``` - -### Log Files - -#### 1. `publish-sync.log` -**Purpose:** Main workflow logging for content publishing - -**Contains:** -- Complete publish workflow from start to finish -- Step-by-step progress (STEP 1, STEP 2, etc.) -- Content preparation (categories, tags, images, SEO) -- Success/failure status -- Timing information (duration in milliseconds) - -**Format:** -``` -[2025-12-01 00:45:23] [INFO] ================================================================================ -[2025-12-01 00:45:23] [INFO] [16-homeg8.com] 🎯 PUBLISH WORKFLOW STARTED -[2025-12-01 00:45:23] [INFO] [16-homeg8.com] Content ID: 123 -[2025-12-01 00:45:23] [INFO] [16-homeg8.com] STEP 1: Loading content and integration from database... -[2025-12-01 00:45:23] [INFO] [16-homeg8.com] ✅ Content loaded: 'Article Title' -... -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] 🎉 PUBLISH WORKFLOW COMPLETED SUCCESSFULLY -``` - -#### 2. `wordpress-api.log` -**Purpose:** HTTP API communication with WordPress sites - -**Contains:** -- API requests (method, endpoint, headers, payload) -- API responses (status code, body) -- API errors and timeouts -- Request/response timing - -**Format:** -``` -[2025-12-01 00:45:24] [INFO] [16-homeg8.com] API REQUEST: POST https://homeg8.com/wp-json/igny8/v1/publish -[2025-12-01 00:45:24] [INFO] [16-homeg8.com] Headers: X-IGNY8-API-Key: ***abcd -[2025-12-01 00:45:24] [INFO] [16-homeg8.com] Payload: {...} -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] API RESPONSE: 201 -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] Response body: {"success":true...} -``` - -#### 3. `webhooks.log` -**Purpose:** Webhook events received from WordPress - -**Contains:** -- Webhook POST requests from WordPress -- Status updates from WordPress to IGNY8 -- Metadata synchronization events -- Authentication verification - -**Format:** -``` -[2025-12-01 00:45:30] [INFO] [16-homeg8.com] 📥 WORDPRESS STATUS WEBHOOK RECEIVED -[2025-12-01 00:45:30] [INFO] [16-homeg8.com] Content ID: 123 -[2025-12-01 00:45:30] [INFO] [16-homeg8.com] Post ID: 456 -[2025-12-01 00:45:30] [INFO] [16-homeg8.com] Post Status: publish -``` - -### Viewing Backend Logs - -```bash -# View all publish logs -tail -f /data/app/igny8/backend/logs/publish-sync-logs/publish-sync.log - -# View API communication -tail -f /data/app/igny8/backend/logs/publish-sync-logs/wordpress-api.log - -# View webhook events -tail -f /data/app/igny8/backend/logs/publish-sync-logs/webhooks.log - -# Search for specific site -grep "\[16-homeg8.com\]" /data/app/igny8/backend/logs/publish-sync-logs/publish-sync.log - -# Search for errors -grep "\[ERROR\]" /data/app/igny8/backend/logs/publish-sync-logs/*.log - -# View last 100 lines of all logs -tail -n 100 /data/app/igny8/backend/logs/publish-sync-logs/*.log -``` - ---- - -## WordPress Plugin Logs - -### Location -``` -/wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/ -``` - -On your server: -``` -/home/u276331481/domains/homeg8.com/public_html/wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/ -``` - -### Log Files - -#### 1. `publish-sync.log` -**Purpose:** WordPress post creation and synchronization - -**Contains:** -- Post creation workflow -- Categories/tags assignment -- Featured image processing -- SEO metadata updates -- Gallery image handling -- Success/failure status - -**Format:** -``` -[2025-12-01 00:45:25] [INFO] ================================================================================ -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] 🎯 CREATE WORDPRESS POST FROM IGNY8 -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] Content ID: 123 -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] STEP: Processing 3 categories -[2025-12-01 00:45:25] [INFO] [16-homeg8.com] ✅ Assigned 3 categories to post 456 -... -[2025-12-01 00:45:26] [INFO] [16-homeg8.com] 🎉 POST CREATION COMPLETED SUCCESSFULLY -``` - -#### 2. `wordpress-api.log` -**Purpose:** WordPress outgoing API calls to IGNY8 - -**Contains:** -- API requests from WordPress to IGNY8 -- Task updates -- Status synchronization -- API responses - -#### 3. `webhooks.log` -**Purpose:** Outgoing webhook calls from WordPress to IGNY8 - -**Contains:** -- Status webhook sends -- Metadata webhook sends -- Response handling - -### Viewing WordPress Logs - -```bash -# On the WordPress server -cd /home/u276331481/domains/homeg8.com/public_html/wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/ - -# View all publish logs -tail -f publish-sync.log - -# View API communication -tail -f wordpress-api.log - -# View webhook events -tail -f webhooks.log - -# Search for specific content ID -grep "Content ID: 123" publish-sync.log - -# View last 50 lines -tail -n 50 publish-sync.log -``` - ---- - -## Log Retention - -### Automatic Rotation -- **Max file size:** 10 MB per log file -- **Backup count:** 10 rotated files kept -- **Total storage:** ~100 MB per log type (10 files × 10 MB) - -### Rotated Files -``` -publish-sync.log # Current log -publish-sync.log.1 # Previous rotation -publish-sync.log.2 # Older rotation -... -publish-sync.log.10 # Oldest rotation (auto-deleted when new rotation created) -``` - ---- - -## Troubleshooting Guide - -### 1. Content Not Publishing - -**Check:** -```bash -# Backend - Publishing workflow -grep "PUBLISH WORKFLOW" /data/app/igny8/backend/logs/publish-sync-logs/publish-sync.log | tail -20 - -# Backend - API errors -grep "\[ERROR\]" /data/app/igny8/backend/logs/publish-sync-logs/wordpress-api.log | tail -20 - -# WordPress - Post creation -tail -50 /wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/publish-sync.log -``` - -### 2. Missing Categories/Tags - -**Check:** -```bash -# WordPress logs - Category processing -grep "Processing.*categories" /wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/publish-sync.log - -# WordPress logs - Tag processing -grep "Processing.*tags" /wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/publish-sync.log -``` - -### 3. Status Not Syncing - -**Check:** -```bash -# Backend - Webhook reception -grep "WEBHOOK RECEIVED" /data/app/igny8/backend/logs/publish-sync-logs/webhooks.log | tail -20 - -# WordPress - Webhook sending -grep "webhook" /wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/webhooks.log | tail -20 -``` - -### 4. API Connection Issues - -**Check:** -```bash -# Backend - API requests/responses -grep "API REQUEST\|API RESPONSE\|API ERROR" /data/app/igny8/backend/logs/publish-sync-logs/wordpress-api.log | tail -30 - -# Look for timeouts -grep "timeout" /data/app/igny8/backend/logs/publish-sync-logs/*.log -``` - ---- - -## Common Log Patterns - -### Successful Publish -``` -[INFO] [16-homeg8.com] 🎯 PUBLISH WORKFLOW STARTED -[INFO] [16-homeg8.com] STEP 1: Loading content... -[INFO] [16-homeg8.com] STEP 2: Checking if already published... -[INFO] [16-homeg8.com] STEP 3: Generating excerpt... -[INFO] [16-homeg8.com] STEP 4: Loading taxonomy mappings... -[INFO] [16-homeg8.com] STEP 5: Extracting keywords... -[INFO] [16-homeg8.com] STEP 6: Loading images... -[INFO] [16-homeg8.com] STEP 7: Building WordPress API payload... -[INFO] [16-homeg8.com] STEP 8: Sending POST request... -[INFO] [16-homeg8.com] ✅ WordPress responded: HTTP 201 -[INFO] [16-homeg8.com] STEP 9: Processing WordPress response... -[INFO] [16-homeg8.com] ✅ WordPress post created successfully -[INFO] [16-homeg8.com] 🎉 PUBLISH WORKFLOW COMPLETED SUCCESSFULLY -``` - -### API Error -``` -[ERROR] [16-homeg8.com] ❌ Unexpected status code: 500 -[ERROR] [16-homeg8.com] Response: {"code":"internal_server_error"...} -[ERROR] [16-homeg8.com] API ERROR: WordPress API error: HTTP 500 -``` - -### Missing Fields -``` -[WARNING] [16-homeg8.com] ⚠️ No categories in content_data -[WARNING] [16-homeg8.com] ⚠️ No featured image in content_data -[WARNING] [16-homeg8.com] ⚠️ No SEO description in content_data -``` - ---- - -## Accessing Logs via Debug Page - -The frontend Debug Status page will display these logs in real-time (coming next). - -**URL:** https://app.igny8.com/settings/debug-status - ---- - -## Log Cleanup - -### Manual Cleanup -```bash -# Backend -rm -f /data/app/igny8/backend/logs/publish-sync-logs/*.log* - -# WordPress (on your server) -rm -f /home/u276331481/domains/homeg8.com/public_html/wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/*.log* -``` - -### Keep Recent Logs Only -```bash -# Keep only last 1000 lines -tail -n 1000 /data/app/igny8/backend/logs/publish-sync-logs/publish-sync.log > /tmp/temp.log -mv /tmp/temp.log /data/app/igny8/backend/logs/publish-sync-logs/publish-sync.log -``` - ---- - -## Notes - -- All timestamps are in server local time -- Site stamps format: `[site-id-domain]` -- Emoji indicators: 🎯 (start), ✅ (success), ❌ (error), ⚠️ (warning), 🎉 (completion) -- Logs are written in real-time during publish workflows -- File rotation happens automatically at 10MB per file diff --git a/STATUS-UNIFIED-STRUCTURE-PLAN-v2.md b/STATUS-UNIFIED-STRUCTURE-PLAN-v2.md deleted file mode 100644 index 0cb60ef4..00000000 --- a/STATUS-UNIFIED-STRUCTURE-PLAN-v2.md +++ /dev/null @@ -1,541 +0,0 @@ -# Unified Status Structure Plan - Keywords, Clusters, Ideas (CORRECTED) - -**Date:** December 3, 2025 -**Objective:** Implement unified, workflow-driven status values across all three modules - ---- - -## 🎯 Key Concept: Status vs Workflow - -### ✅ WORKFLOW STATUS (Auto-updated by AI/System) -These statuses represent the workflow progression: -- Keywords: `new` → `mapped` -- Clusters: `new` → `mapped` -- Ideas: `new` → `queued` → `completed` - -### 🔘 OPTIONAL MANUAL STATUS (User control only) -`disabled` = **NOT a workflow step**, but a **filter status** -- Does NOT affect workflow -- Manually set by user only -- Excludes items from all automated processes -- Excludes from dashboard metrics/suggestions -- Can be toggled on/off anytime - -**Important:** `disabled` is a **data attribute**, not a workflow state. Items with `disabled=true` are simply ignored by all processes. - ---- - -## 📋 Module Status Definitions - -### KEYWORDS Module -**Workflow Statuses:** `new` → `mapped` -**Optional Filter Status:** `disabled` (manual user control) - -| Status | Type | Meaning | When Set | Process Behavior | -|--------|------|---------|----------|-----------------| -| **new** | Workflow | Keyword attached to site, awaiting cluster assignment | User adds keyword via import/UI | Included in auto-cluster AI suggestions | -| **mapped** | Workflow | Keyword assigned to a cluster | AI auto_cluster completes or manual cluster assignment | Can be used for idea generation; included in cluster-based workflows | -| **disabled** | Filter | Manually excluded from processes | User manually sets | Excluded from auto-cluster, idea generation, dashboard (optional) | - -**Workflow Transition:** -``` -new ──[AI auto_cluster]──> mapped - │ - └──[User toggle]──> disabled (any time, no workflow impact) -``` - ---- - -### CLUSTERS Module -**Workflow Statuses:** `new` → `mapped` -**Optional Filter Status:** `disabled` (manual user control) - -| Status | Type | Meaning | When Set | Process Behavior | -|--------|------|---------|----------|-----------------| -| **new** | Workflow | Cluster created; ready for idea generation | AI clustering creates OR user manually creates | Included in auto-generate-ideas suggestions | -| **mapped** | Workflow | Ideas generated from this cluster | AI generate_ideas completes | Ideas are ready; can be queued to writer | -| **disabled** | Filter | Manually excluded from processes | User manually sets | Excluded from idea generation, suggestion lists, dashboard (optional) | - -**Workflow Transition:** -``` -new ──[AI generate_ideas]──> mapped - │ - └──[User toggle]──> disabled (any time, no workflow impact) -``` - ---- - -### IDEAS Module -**Workflow Statuses:** `new` → `queued` → `completed` -**Optional Filter Status:** `disabled` (manual user control) - -| Status | Type | Meaning | When Set | Process Behavior | -|--------|------|---------|----------|-----------------| -| **new** | Workflow | Idea generated by AI; awaiting queue to writer | AI generate_ideas creates | Included in bulk-queue suggestions | -| **queued** | Workflow | Queued to writer; Task record created in Writer module | User bulk-queues ideas to writer | Task is assigned; waiting for content generation | -| **completed** | Workflow | Content generated from task (tracked via Task.status='completed') | generate_content AI completes; Content record created | Final automated state; content ready for publishing/deployment | -| **disabled** | Filter | Manually excluded from processes | User manually sets | Excluded from queue suggestions, bulk operations, dashboard (optional) | - -**Workflow Transition:** -``` -new ──[User bulk_queue]──> queued ──[Writer AI: generate_content]──> completed - │ - └──[User toggle]──> disabled (any time, no workflow impact) -``` - -**Note on `completed`:** -- When a Task's status becomes `completed`, the related Idea automatically becomes `completed` -- No separate `published` status needed; publishing is a separate content deployment action -- One idea = one content piece through task tracking - ---- - -## 🔄 Process Exclusion Rules - -### When `disabled=true` (filter status): -Items are **excluded from**: -- ✗ AI process suggestions (auto-cluster, generate-ideas) -- ✗ Bulk operation selections (queue to writer, generate content) -- ✗ Dashboard workflow metrics (unless explicitly shown) -- ✗ Progress calculations (% mapped, % queued, etc.) - -### When workflow status applies: -- ✅ Included in relevant processes -- ✅ Included in dashboard metrics -- ✅ Included in AI suggestions -- ✅ Can be bulk-operated on - ---- - -## 📊 Status Update Flow - -``` -═══ KEYWORDS WORKFLOW ═══ -1. User imports SeedKeywords - └─> Keywords created with status='new' - -2. Auto-cluster AI runs - └─> Keywords assigned to clusters - └─> Keywords status changes to 'mapped' - -3. [Optional] User manually disables keyword - └─> Keywords.disabled = true - └─> Excluded from all processes - - -═══ CLUSTERS WORKFLOW ═══ -1. Auto-cluster AI creates/updates clusters - └─> Clusters created with status='new' - -2. AI generate-ideas runs on 'new' clusters - └─> Ideas created for cluster - └─> Clusters status changes to 'mapped' - -3. [Optional] User manually disables cluster - └─> Clusters.disabled = true - └─> Excluded from all processes - - -═══ IDEAS WORKFLOW ═══ -1. AI generate-ideas creates ideas for clusters - └─> Ideas created with status='new' - -2. User bulk-queues ideas to writer - └─> Task created in Writer module - └─> Ideas status changes to 'queued' - -3. Writer AI (generate-content) creates content - └─> Content record created - └─> Task.status = 'completed' - └─> Ideas.status = 'completed' [Auto-sync from Task] - -4. [Optional] User manually disables idea - └─> Ideas.disabled = true - └─> Excluded from all processes -``` - ---- - -## 🔧 Implementation Changes - -### Backend Model Changes - -#### 1. `/backend/igny8_core/business/planning/models.py` - -**Keywords Model:** -```python -# REMOVE old STATUS_CHOICES -# OLD: -STATUS_CHOICES = [ - ('active', 'Active'), - ('pending', 'Pending'), - ('archived', 'Archived'), -] - -# NEW: -STATUS_CHOICES = [ - ('new', 'New'), - ('mapped', 'Mapped'), -] -status = models.CharField( - max_length=50, - choices=STATUS_CHOICES, - default='new' -) - -# ADD new filter field: -disabled = models.BooleanField(default=False, help_text="Exclude from processes") -``` - -**Clusters Model:** -```python -# ADD STATUS_CHOICES (currently hardcoded as default='active', no choices) -# CURRENT: status = models.CharField(max_length=50, default='active') - -# NEW: -STATUS_CHOICES = [ - ('new', 'New'), - ('mapped', 'Mapped'), -] -status = models.CharField( - max_length=50, - choices=STATUS_CHOICES, - default='new' -) - -# ADD new filter field: -disabled = models.BooleanField(default=False, help_text="Exclude from processes") -``` - -**ContentIdeas Model:** -```python -# UPDATE STATUS_CHOICES -# REMOVE 'scheduled', 'published' -STATUS_CHOICES = [ - ('new', 'New'), - ('queued', 'Queued'), - ('completed', 'Completed'), -] -status = models.CharField( - max_length=50, - choices=STATUS_CHOICES, - default='new' -) - -# ADD new filter field: -disabled = models.BooleanField(default=False, help_text="Exclude from processes") -``` - ---- - -### Backend AI Function Updates - -#### 2. `/backend/igny8_core/ai/functions/auto_cluster.py` - -**Location: Line ~297 in save_output()** -```python -# CURRENT: -status='active' - -# NEW: -status='mapped' # When keywords are assigned to cluster -``` - ---- - -#### 3. `/backend/igny8_core/ai/functions/generate_ideas.py` - -**Location: After ideas created in save_output()** -```python -# NEW: Add after creating ideas -# Update cluster status from 'new' to 'mapped' after ideas generated -for cluster in clusters_used: - if cluster.status == 'new': - cluster.status = 'mapped' - cluster.save() -``` - ---- - -#### 4. `/backend/igny8_core/ai/functions/generate_content.py` - -**Location: Line ~318 in save_output()** -```python -# CURRENT: -task.status = 'completed' -task.save() - -# NEW: Auto-sync idea status from task -task.status = 'completed' -task.save() - -# NEW: Update related idea to 'completed' -if hasattr(task, 'idea') and task.idea: - idea = task.idea - idea.status = 'completed' - idea.save() -``` - ---- - -### Backend API Updates - -#### 5. `/backend/igny8_core/modules/planner/views.py` - -**Queue ideas to writer (Line ~1084):** -```python -# CURRENT: -idea.status = 'scheduled' - -# NEW: -idea.status = 'queued' -``` - -**CSV import defaults (Line ~563):** -```python -# CURRENT: -status=row.get('status', 'pending') or 'pending' - -# NEW: -status=row.get('status', 'new') or 'new' -``` - -**Filter out disabled items in list views:** -```python -# NEW: When returning lists, exclude disabled=true (optional, configurable) -queryset = queryset.filter(disabled=False) # Or show both with filter option -``` - ---- - -### Frontend Configuration Updates - -#### 6. `/frontend/src/config/pages/keywords.config.tsx` - -**Table Column - Status Badge (Lines ~230-248):** -```typescript -// CURRENT: -new: 'amber', active: 'green', archived: 'red' - -// NEW: -new: 'amber', mapped: 'green' - -// ADD: Display 'disabled' badge if disabled=true -``` - -**Filter Dropdown (Lines ~310-318):** -```typescript -// CURRENT: -options: ['pending', 'active', 'archived'] - -// NEW: -options: ['new', 'mapped'], -additionalFilters: [ - { key: 'disabled', label: 'Show Disabled', type: 'checkbox', default: false } -] -``` - -**Form Field Default (Lines ~560-570):** -```typescript -// CURRENT: -default: 'pending' - -// NEW: -default: 'new' - -// ADD: disabled checkbox in form -``` - ---- - -#### 7. `/frontend/src/config/pages/clusters.config.tsx` - -**Table Column - Status Badge (Lines ~190-200):** -```typescript -// CURRENT: -(missing or wrong values) - -// NEW: -new: 'amber', mapped: 'green' - -// ADD: Display 'disabled' badge if disabled=true -``` - -**Filter Dropdown (Lines ~240-253):** -```typescript -// CURRENT: -options: ['new', 'idea', 'mapped'] - -// NEW: -options: ['new', 'mapped'], -additionalFilters: [ - { key: 'disabled', label: 'Show Disabled', type: 'checkbox', default: false } -] -``` - -**Form Field (Lines ~405-418):** -```typescript -// ADD: disabled checkbox in form -// Keep status default as 'new' -``` - ---- - -#### 8. `/frontend/src/config/pages/ideas.config.tsx` - -**Table Column - Status Badge (Lines ~170-185):** -```typescript -// CURRENT: -new: 'amber', scheduled: 'blue', completed: 'blue', published: 'green' - -// NEW: -new: 'amber', queued: 'blue', completed: 'green' - -// ADD: Display 'disabled' badge if disabled=true -``` - -**Filter Dropdown (Lines ~218-228):** -```typescript -// CURRENT: -options: ['new', 'scheduled', 'completed', 'published'] - -// NEW: -options: ['new', 'queued', 'completed'], -additionalFilters: [ - { key: 'disabled', label: 'Show Disabled', type: 'checkbox', default: false } -] -``` - -**Form Field (Lines ~372-385):** -```typescript -// REMOVE: 'published' option -// ADD: disabled checkbox in form -// Keep status default as 'new' -``` - ---- - -### Frontend Dashboard - -#### 9. `/frontend/src/pages/Planner/Dashboard.tsx` - -**Status Metrics:** -```typescript -// Update calculations: -// - Keywords: new vs mapped -// - Clusters: new vs mapped -// - Ideas: new vs queued vs completed - -// Optional: Add disabled count metrics -``` - ---- - -## 📍 Change Summary - -### Total Files: 9 - -| File | Changes | Type | -|------|---------|------| -| **models.py** | Add STATUS_CHOICES for Clusters; Update Keywords/Ideas; Add `disabled` field to all 3 | Backend | -| **auto_cluster.py** | Line 297: `status='active'` → `status='mapped'` | Backend AI | -| **generate_ideas.py** | After ideas created: Set cluster `status='mapped'` | Backend AI | -| **generate_content.py** | Line 318: Also sync `idea.status='completed'` from task | Backend AI | -| **views.py** | Queue ideas (line 1084): `'scheduled'` → `'queued'`; CSV import: `'pending'` → `'new'` | Backend API | -| **keywords.config.tsx** | Update badge/filter/form for new/mapped; Add disabled checkbox | Frontend Config | -| **clusters.config.tsx** | Update badge/filter/form for new/mapped; Add disabled checkbox | Frontend Config | -| **ideas.config.tsx** | Update badge/filter/form for new/queued/completed; Add disabled checkbox | Frontend Config | -| **Dashboard.tsx** | Update metrics calculations for new status values | Frontend Page | - ---- - -## ✅ Workflow Validations - -### Keywords -``` -✓ new → mapped (via auto_cluster) -✓ Can toggle disabled at any time (no workflow impact) -✓ Disabled items excluded from auto-cluster suggestions -``` - -### Clusters -``` -✓ new → mapped (via generate_ideas) -✓ Can toggle disabled at any time (no workflow impact) -✓ Disabled items excluded from idea generation -``` - -### Ideas -``` -✓ new → queued (via bulk_queue_to_writer) -✓ queued → completed (via generate_content, tracked via Task.status) -✓ Can toggle disabled at any time (no workflow impact) -✓ Disabled items excluded from queue suggestions -``` - ---- - -## 🗑️ Data Migration Strategy - -### Django Migration File -**File:** `backend/igny8_core/business/planning/migrations/NNNN_unified_status_fields.py` - -```python -# Forward migration: -# 1. Add disabled=BooleanField(default=False) to all 3 models -# 2. Add STATUS_CHOICES to Clusters model -# 3. Update Keywords.status data: -# - pending → new -# - active → mapped -# - archived → mapped + set disabled=True -# 4. Update Clusters.status data: -# - active (with ideas_count > 0) → mapped -# - active (with ideas_count = 0) → new -# 5. Update ContentIdeas.status data: -# - scheduled → queued -# - published → completed -# - new → new (no change) - -# Reverse migration: -# 1. Remove disabled field from all 3 models -# 2. Restore old STATUS_CHOICES -# 3. Reverse data transformations: -# - Keywords: new→pending, mapped→active, disabled=True→archived -# - Clusters: Remove STATUS_CHOICES, set all to 'active' -# - Ideas: queued→scheduled, completed→published -``` - ---- - -## 🧪 Testing Checklist - -- [ ] Keywords: status='new' by default -- [ ] Keywords: auto_cluster sets status='mapped' -- [ ] Clusters: status='new' by default -- [ ] Clusters: generate_ideas sets status='mapped' -- [ ] Ideas: status='new' by default -- [ ] Ideas: bulk_queue_to_writer sets status='queued' -- [ ] Ideas: generate_content sets status='completed' (via task sync) -- [ ] All modules: disabled=true excludes from processes -- [ ] All modules: disabled checkbox toggles correctly -- [ ] Dashboard: metrics use new status values -- [ ] Filters: disabled checkbox hides disabled items by default -- [ ] Migration: old data transforms correctly - ---- - -## 🎯 Key Differences from Previous Plan - -| Previous | Now Corrected | -|----------|---------------| -| `disabled` was workflow step | `disabled` is filter status only | -| 3 workflows statuses per module | 2 workflow statuses per module | -| Published status for Ideas | Completed is final; publish is separate action | -| Ideas tracked separately | Ideas auto-sync from Task status | -| Unclear disabled behavior | Clear: disabled excluded from all processes | - ---- - -**Status:** ✅ Plan Complete & Corrected -**Ready for:** Implementation Phase 1 (Backend Models) - diff --git a/STATUS-UNIFIED-STRUCTURE-PLAN.md b/STATUS-UNIFIED-STRUCTURE-PLAN.md deleted file mode 100644 index 9a1572c1..00000000 --- a/STATUS-UNIFIED-STRUCTURE-PLAN.md +++ /dev/null @@ -1,526 +0,0 @@ -# Unified Status Structure Plan - Keywords, Clusters, Ideas - -**Date:** December 3, 2025 -**Objective:** Implement unified, workflow-driven status values across all three modules - ---- - -## 🎯 Proposed Unified Status Structure - -### Module-Specific Status Values - -#### KEYWORDS Module -**Current:** `pending`, `active`, `archived` -**Proposed:** `new`, `mapped`, `disabled` - -| Status | Meaning | When Used | Next Step | -|--------|---------|-----------|-----------| -| **new** | Keyword just attached to site, not yet assigned to cluster | User adds keyword | AI auto-cluster runs → `mapped` | -| **mapped** | Keyword is assigned to a cluster | AI clustering completes OR manual assignment | Can generate ideas from cluster | -| **disabled** | Keyword is archived/inactive | User manually disables | No workflow activity | - -**Workflow:** -``` -new → mapped → (can be used for ideas) - ↓ -disabled (any time) -``` - ---- - -#### CLUSTERS Module -**Current:** `active`, `pending`, `archived` -**Proposed:** `new`, `mapped`, `disabled` - -| Status | Meaning | When Used | Next Step | -|--------|---------|-----------|-----------| -| **new** | Cluster created but no content ideas generated yet | AI clustering creates cluster OR manual creation | AI generate_ideas runs → `mapped` | -| **mapped** | Cluster has ideas generated from it | AI generate_ideas completes | Ideas ready to queue to writer | -| **disabled** | Cluster is archived/inactive | User manually disables | No workflow activity | - -**Workflow:** -``` -new → mapped → (ideas queue to writer) - ↓ -disabled (any time) -``` - ---- - -#### IDEAS Module -**Current:** `new`, `scheduled`, `completed`, `published` -**Proposed:** `new`, `queued`, `completed`, `published` - -| Status | Meaning | When Used | Next Step | -|--------|---------|-----------|-----------| -| **new** | Idea generated by AI but not queued to writer yet | AI generate_ideas creates idea | User queues to writer → `queued` | -| **queued** | Idea queued to writer, task created in Writer module | Bulk queue to writer API called | Content generated by writer → `completed` | -| **completed** | Content generated from idea (Task completed with content) | generate_content AI function creates Content record | Manual publish → `published` | -| **published** | Content published to site | User publishes OR auto-publish | Final state | - -**Workflow:** -``` -new → queued → completed → published -``` - ---- - -## 📊 Status Update Flow Diagram - -``` -KEYWORDS WORKFLOW: -1. User adds SeedKeyword → Keywords status='new' -2. AI auto_cluster runs → Keywords status='mapped' (assigned to cluster) -3. User can disable → Keywords status='disabled' - -CLUSTERS WORKFLOW: -1. AI creates cluster → Clusters status='new' -2. AI generate_ideas runs → Clusters status='mapped' (has ideas) -3. User can disable → Clusters status='disabled' - -IDEAS WORKFLOW: -1. AI creates idea → Ideas status='new' -2. User bulk queues ideas → Ideas status='queued' (task created in Writer) -3. Writer AI generates content → Ideas status='completed' (Content record created) -4. User publishes content → Ideas status='published' -``` - ---- - -## 🔧 Change Locations & Impact - -### Backend Model Files to Update - -#### 1. `/backend/igny8_core/business/planning/models.py` - -**Keywords Model (Lines 42-46):** -```python -# CURRENT: -STATUS_CHOICES = [ - ('active', 'Active'), - ('pending', 'Pending'), - ('archived', 'Archived'), -] - -# PROPOSED: -STATUS_CHOICES = [ - ('new', 'New'), - ('mapped', 'Mapped'), - ('disabled', 'Disabled'), -] - -# DEFAULT: Change from 'pending' → 'new' -``` - -**Clusters Model (Line 12):** -```python -# CURRENT: -status = models.CharField(max_length=50, default='active') - -# PROPOSED: -status = models.CharField(max_length=50, default='new', - choices=[('new', 'New'), ('mapped', 'Mapped'), ('disabled', 'Disabled')]) -``` - -**ContentIdeas Model (Lines 150-157):** -```python -# CURRENT: -STATUS_CHOICES = [ - ('new', 'New'), - ('scheduled', 'Scheduled'), - ('published', 'Published'), -] - -# PROPOSED: -STATUS_CHOICES = [ - ('new', 'New'), - ('queued', 'Queued'), - ('completed', 'Completed'), - ('published', 'Published'), -] - -# DEFAULT: Keep 'new' -``` - ---- - -### Backend AI Function Files to Update - -#### 2. `/backend/igny8_core/ai/functions/auto_cluster.py` (Line 297) - -**When keywords are assigned to clusters:** -```python -# CURRENT: -status='active' - -# PROPOSED: -status='mapped' -``` - -**Location:** In `save_output()` method when updating Keywords - ---- - -#### 3. `/backend/igny8_core/ai/functions/generate_ideas.py` (Line 224) - -**When clusters change status after ideas generated:** -```python -# CURRENT: No cluster status update - -# PROPOSED: After creating ideas, update cluster status: -# In save_output() method, after creating ideas: -cluster.status = 'mapped' -cluster.save() -``` - -**Additional:** Ideas default stays `status='new'` - ---- - -#### 4. `/backend/igny8_core/ai/functions/generate_content.py` (Line 318) - -**When content is generated from a task:** -```python -# CURRENT: -task.status = 'completed' -task.save() - -# PROPOSED: Also update the related Idea status: -task.status = 'completed' -task.save() - -# NEW: Update idea to completed -if hasattr(task, 'idea') and task.idea: - idea = task.idea - idea.status = 'completed' - idea.save() -``` - -**Location:** In `save_output()` method after content is created - ---- - -### Backend API Endpoint Files to Update - -#### 5. `/backend/igny8_core/modules/planner/views.py` - -**Line 1029 - Queue ideas to writer filter:** -```python -# CURRENT: -queueable_ideas = all_ideas.filter(status='new') - -# NO CHANGE NEEDED (stays 'new') -``` - -**Line 1084 - Update idea status when queued:** -```python -# CURRENT: -idea.status = 'scheduled' -idea.save() - -# PROPOSED: -idea.status = 'queued' -idea.save() -``` - -**Line 563 - Bulk create keywords from CSV:** -```python -# CURRENT: -status=row.get('status', 'pending') or 'pending' - -# PROPOSED: -status=row.get('status', 'new') or 'new' -``` - ---- - -### Frontend Configuration Files to Update - -#### 6. `/frontend/src/config/pages/keywords.config.tsx` - -**Table Column Status Badge (Lines 230-248):** -```typescript -// CURRENT: -pending: 'amber', active: 'green', archived: 'red' - -// PROPOSED: -new: 'amber', mapped: 'green', disabled: 'red' -``` - -**Filter Dropdown (Lines 310-318):** -```typescript -// CURRENT: -options: ['pending', 'active', 'archived'] - -// PROPOSED: -options: ['new', 'mapped', 'disabled'] -``` - -**Form Field Default (Lines 560-570):** -```typescript -// CURRENT: -default: 'pending' - -// PROPOSED: -default: 'new' -``` - ---- - -#### 7. `/frontend/src/config/pages/clusters.config.tsx` - -**Table Column Status Badge (Lines 190-200):** -```typescript -// CURRENT: -active: 'green', archived: 'red' (inconsistent from backend) - -// PROPOSED: -new: 'amber', mapped: 'green', disabled: 'red' -``` - -**Filter Dropdown (Lines 240-253):** -```typescript -// CURRENT: -options: ['active', 'archived'] (missing 'new') - -// PROPOSED: -options: ['new', 'mapped', 'disabled'] -``` - -**Form Field Default (Lines 405-418):** -```typescript -// CURRENT: -default: 'new' - -// PROPOSED: -default: 'new' (no change, already correct) -``` - ---- - -#### 8. `/frontend/src/config/pages/ideas.config.tsx` - -**Table Column Status Badge (Lines 170-185):** -```typescript -// CURRENT: -new: 'amber', scheduled: 'blue', completed: 'blue', published: 'green' - -// PROPOSED: -new: 'amber', queued: 'blue', completed: 'blue', published: 'green' -``` - -**Filter Dropdown (Lines 218-228):** -```typescript -// CURRENT: -options: ['new', 'scheduled', 'completed', 'published'] -// NOTE: missing 'completed' in some places - -// PROPOSED: -options: ['new', 'queued', 'completed', 'published'] -``` - -**Form Field Default (Lines 372-385):** -```typescript -// CURRENT: -default: 'new' - -// PROPOSED: -default: 'new' (no change) -``` - ---- - -### Frontend Dashboard & State Files - -#### 9. `/frontend/src/pages/Planner/Dashboard.tsx` - -**Status Metrics (Lines 121-155):** -```typescript -// Updates status counting logic -// From: keywordsByStatus, clustersByStatus, ideaByStatus -// To: Use new status values - -// Example refactor: -// OLD: keywordsByStatus['active'] vs keywordsByStatus['pending'] -// NEW: keywordsByStatus['mapped'] vs keywordsByStatus['new'] -``` - -**No API call changes needed** - API returns correct values - ---- - -#### 10. `/frontend/src/services/api.ts` - -**No changes needed** - Only returns what backend provides - ---- - -### Frontend Component Files - -#### 11. `/frontend/src/pages/Planner/Keywords.tsx` - -**Status Filter Logic (if hardcoded):** -- Search for status comparisons -- Update from old values → new values - -#### 12. `/frontend/src/pages/Planner/Clusters.tsx` - -**Status Filter Logic (if hardcoded):** -- Search for status comparisons -- Update from old values → new values - -#### 13. `/frontend/src/pages/Planner/Ideas.tsx` - -**Status Filter Logic (if hardcoded):** -- Update from `scheduled` → `queued` -- Keep `new`, `completed`, `published` - ---- - -### Database Migration Files - -#### 14. New Django Migration Required - -**File to create:** `/backend/igny8_core/business/planning/migrations/NNNN_update_status_choices.py` - -```python -# Migration tasks: -# 1. Update Keywords STATUS_CHOICES in model -# 2. Data migration: pending → new, active → mapped, archived → disabled -# 3. Update Clusters status field with choices -# 4. Data migration: active → new (or map by count of ideas) -# 5. Update ContentIdeas STATUS_CHOICES -# 6. Data migration: scheduled → queued -``` - -**Changes needed:** -- Keywords: pending→new, active→mapped, archived→disabled -- Clusters: active→new or mapped (based on ideas_count), archived→disabled -- ContentIdeas: scheduled→queued - ---- - -## 📍 Location Summary - -### Total Files to Update: 13 - -| Category | Count | Files | -|----------|-------|-------| -| Backend Models | 1 | models.py | -| Backend AI Functions | 3 | auto_cluster.py, generate_ideas.py, generate_content.py | -| Backend API Views | 1 | views.py | -| Frontend Configs | 3 | keywords.config.tsx, clusters.config.tsx, ideas.config.tsx | -| Frontend Pages | 3 | Dashboard.tsx, Keywords.tsx, Clusters.tsx, Ideas.tsx | -| Database Migrations | 1 | New migration file | -| **TOTAL** | **13** | | - ---- - -## 🔄 Dependency Order - -### Phase 1: Backend Models & Data Migration (BLOCKER) -1. Update `models.py` STATUS_CHOICES for all 3 models -2. Create Django migration with data transformation -3. Run migration on database -4. Test: Backend API returns new status values - -### Phase 2: AI Functions (Status Update Logic) -5. Update `auto_cluster.py` line 297: keywords status → 'mapped' -6. Add to `generate_ideas.py`: clusters status → 'mapped' after ideas created -7. Update `generate_content.py` line 318: ideas status → 'completed' after content created -8. Test: AI workflows set correct status values - -### Phase 3: Backend API (Response Values) -9. Update `views.py` line 563: CSV import defaults to 'new' -10. Update `views.py` line 1084: queue to writer uses 'queued' -11. Test: API responses have new status values - -### Phase 4: Frontend Configuration (Display Logic) -12. Update `keywords.config.tsx` (3 locations: badge, filter, form) -13. Update `clusters.config.tsx` (3 locations: badge, filter, form) -14. Update `ideas.config.tsx` (3 locations: badge, filter, form) -15. Test: Filters and forms display new status values - -### Phase 5: Frontend Pages (State Management) -16. Update `Dashboard.tsx` status metrics calculations -17. Update `Keywords.tsx` status filters (if hardcoded) -18. Update `Clusters.tsx` status filters (if hardcoded) -19. Update `Ideas.tsx` status filters and comparisons -20. Test: Dashboard shows correct counts, filters work - ---- - -## 🧪 Testing Checklist - -### Backend Tests -- [ ] Models accept new status values only (no old values allowed) -- [ ] Migration transforms data correctly -- [ ] Auto-cluster: Keywords change from new → mapped -- [ ] Generate ideas: Clusters change from new → mapped -- [ ] Generate content: Ideas change from queued → completed -- [ ] API returns new status values - -### Frontend Tests -- [ ] Keywords: Table shows new/mapped/disabled badges with correct colors -- [ ] Keywords: Filter dropdown shows new/mapped/disabled options -- [ ] Keywords: Create form defaults to 'new' -- [ ] Clusters: Table shows new/mapped/disabled badges -- [ ] Clusters: Filter shows new/mapped/disabled options -- [ ] Clusters: Create form defaults to 'new' -- [ ] Ideas: Table shows new/queued/completed/published badges -- [ ] Ideas: Filter shows all 4 status options -- [ ] Ideas: Queue to writer changes status to 'queued' -- [ ] Dashboard: Metrics count correctly with new status values -- [ ] Dashboard: Workflow percentage bars calculate correctly - -### Integration Tests -- [ ] Full workflow: keyword → cluster → idea → queued → completed → published -- [ ] Status transitions are unidirectional (new→mapped, queued→completed, etc.) -- [ ] Disabled items don't appear in workflow suggestions -- [ ] Bulk operations update status correctly - ---- - -## 🎯 Benefits of Unified Structure - -1. **Consistency:** All modules follow same pattern (new → mapped → disabled) -2. **Clarity:** Status names reflect actual workflow state, not just data state -3. **Scalability:** Easy to add new statuses without confusion -4. **Maintainability:** Single place to understand entire status flow -5. **User Understanding:** Users immediately understand workflow progression -6. **AI Integration:** AI functions have clear status update points - ---- - -## ⚠️ Breaking Changes Alert - -### For API Consumers -- Old status values `pending`, `active`, `archived`, `scheduled` will no longer be accepted -- API will return only `new`, `mapped`, `disabled`, `queued`, `completed`, `published` -- Clients must update filtering logic - -### For Database -- Existing records will be migrated with data transformation -- No data loss, but status values will change -- Recommended: Backup database before running migration - -### For Frontend -- Old hardcoded status comparisons must be updated -- Filter options in dropdowns must match new values -- Status badges must use new color mappings - ---- - -## 📋 Rollback Plan - -If issues arise: -1. Reverse Django migration (restores old status values and data) -2. Revert backend code to previous version -3. Revert frontend configs to previous version -4. Test API and UI return to old behavior - ---- - -**Status:** Plan Complete - Ready for Implementation -**Next Step:** Execute Phase 1 (Backend Models & Migration) - diff --git a/KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md b/docs/KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md similarity index 100% rename from KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md rename to docs/KEYWORDS-CLUSTERS-IDEAS-COMPLETE-MAPPING.md diff --git a/docs/SITEBUILDER-REMOVAL-MIGRATION-GUIDE.md b/docs/SITEBUILDER-REMOVAL-MIGRATION-GUIDE.md deleted file mode 100644 index 958e9384..00000000 --- a/docs/SITEBUILDER-REMOVAL-MIGRATION-GUIDE.md +++ /dev/null @@ -1,351 +0,0 @@ -# SiteBuilder Removal - Complete Migration Guide - -**Date:** December 1, 2025 -**Status:** Code changes completed, database migration pending - ---- - -## Summary - -All SiteBuilder and Blueprint functionality has been removed from the IGNY8 system. The taxonomy system has been simplified to use the direct `Content.taxonomy_terms` many-to-many relationship with the `ContentTaxonomy` model. - ---- - -## What Was Removed - -### Backend Models (Django) -- ✅ `SiteBlueprint` - Site structure blueprints -- ✅ `PageBlueprint` - Individual page definitions -- ✅ `SiteBlueprintCluster` - Cluster to blueprint mappings -- ✅ `SiteBlueprintTaxonomy` - Blueprint taxonomy definitions -- ✅ `BusinessType`, `AudienceProfile`, `BrandPersonality`, `HeroImageryDirection` - SiteBuilder metadata options -- ✅ `ContentTaxonomyMap` - Replaced by `Content.taxonomy_terms` M2M field - -### Backend Modules -- ✅ Removed entire `igny8_core.modules.site_builder` module -- ✅ Removed from `INSTALLED_APPS` in settings.py -- ✅ Removed `/api/v1/site-builder/` URL patterns -- ✅ Cleaned up `site_building/models.py`, `site_building/admin.py` - -### Backend Services & Views -- ✅ Updated `PublisherService` - removed `publish_to_sites()` method -- ✅ Updated `PublisherViewSet` - removed blueprint publishing actions -- ✅ Updated `DeploymentRecordViewSet` - removed blueprint references -- ✅ Updated `PublishingRecord` model - removed `site_blueprint` field -- ✅ Updated `DeploymentRecord` model - removed `site_blueprint` field -- ✅ Fixed `metadata_mapping_service.py` - removed ContentTaxonomyMap -- ✅ Fixed `candidate_engine.py` - uses `Content.taxonomy_terms` now -- ✅ Fixed `sites_renderer_adapter.py` - uses M2M taxonomy relationship -- ✅ Fixed `planner/serializers.py` - removed SiteBlueprintTaxonomy import - -### Frontend -- ✅ Removed `frontend/src/modules/siteBuilder/` directory -- ✅ Removed `frontend/src/types/siteBuilder.ts` -- ✅ Removed `frontend/src/services/siteBuilder.api.ts` -- ✅ Removed SiteBlueprint API functions from `services/api.ts` -- ✅ Removed SiteBuilder routes from navigation - ---- - -## Current Taxonomy System - -### Simplified Architecture - -**Before (Complex - SiteBuilder era):** -``` -Content → ContentTaxonomyMap → SiteBlueprintTaxonomy → Clusters - (through table) (blueprint planning) -``` - -**After (Simplified - Current):** -``` -Content ←→ ContentTaxonomy (many-to-many direct relationship) - ↓ - Cluster (foreign key) -``` - -### ContentTaxonomy Model -**Location:** `backend/igny8_core/business/content/models.py` - -**Fields:** -- `name` - Term name (e.g., "Outdoor Living Design") -- `slug` - URL-safe version -- `taxonomy_type` - Choices: `category`, `tag` -- `external_id` - WordPress term_id for sync -- `external_taxonomy` - WordPress taxonomy slug (category, post_tag) -- `description` - Term description -- `account`, `site`, `sector` - Multi-tenancy fields - -**Relationship:** -- `Content.taxonomy_terms` - ManyToManyField to ContentTaxonomy -- Categories/tags are AI-generated and linked directly to Content - ---- - -## How Content Taxonomy Works Now - -### Publishing Flow (IGNY8 → WordPress) - -```python -# In wordpress_publishing.py task - -# STEP 4: Get categories from Content.taxonomy_terms -categories = [ - term.name - for term in content.taxonomy_terms.filter(taxonomy_type='category') -] - -# Fallback to cluster if no taxonomy_terms -if not categories and content.cluster: - categories.append(content.cluster.name) - -# STEP 5: Get tags from taxonomy_terms + keywords -tags = [ - term.name - for term in content.taxonomy_terms.filter(taxonomy_type='tag') -] - -# Add keywords as tags -if content.primary_keyword not in tags: - tags.append(content.primary_keyword) -for keyword in content.secondary_keywords: - if keyword not in tags: - tags.append(keyword) -``` - -### Data Flow - -``` -Keywords → Clusters → Ideas → Tasks → Content - ├── taxonomy_terms (M2M) - │ ├── Categories - │ └── Tags - ├── cluster (FK) - ├── primary_keyword - └── secondary_keywords - ↓ - WordPress Publish - ├── Categories (from taxonomy_terms + cluster) - └── Tags (from taxonomy_terms + keywords) -``` - ---- - -## Django Admin - -**ContentTaxonomy is registered** in `backend/igny8_core/modules/writer/admin.py`: -- View at: `/admin/writer/contenttaxonomy/` -- Shows: name, taxonomy_type, slug, external_id, external_taxonomy, site, sector -- Searchable by: name, slug, external_taxonomy -- Filterable by: taxonomy_type, site, sector - ---- - -## Database Migration (PENDING) - -**Migration File:** `backend/igny8_core/business/site_building/migrations/0002_remove_blueprint_models.py` - -**Status:** Created but not applied (requires external PostgreSQL access) - -### Manual SQL Commands - -Run these SQL commands directly on your production PostgreSQL database: - -```sql --- Drop foreign key constraints first -ALTER TABLE igny8_publishing_records - DROP CONSTRAINT IF EXISTS igny8_publishing_recor_site_blueprint_id_9f4e8c7a_fk_igny8_sit CASCADE; - -ALTER TABLE igny8_deployment_records - DROP CONSTRAINT IF EXISTS igny8_deployment_recor_site_blueprint_id_3a2b7c1d_fk_igny8_sit CASCADE; - --- Drop blueprint tables -DROP TABLE IF EXISTS igny8_site_blueprint_taxonomies CASCADE; -DROP TABLE IF EXISTS igny8_site_blueprint_clusters CASCADE; -DROP TABLE IF EXISTS igny8_page_blueprints CASCADE; -DROP TABLE IF EXISTS igny8_site_blueprints CASCADE; - --- Drop SiteBuilder metadata tables -DROP TABLE IF EXISTS igny8_site_builder_business_types CASCADE; -DROP TABLE IF EXISTS igny8_site_builder_audience_profiles CASCADE; -DROP TABLE IF EXISTS igny8_site_builder_brand_personalities CASCADE; -DROP TABLE IF EXISTS igny8_site_builder_hero_imagery CASCADE; - --- Drop site_blueprint_id columns -ALTER TABLE igny8_publishing_records DROP COLUMN IF EXISTS site_blueprint_id CASCADE; -ALTER TABLE igny8_deployment_records DROP COLUMN IF EXISTS site_blueprint_id CASCADE; - --- Drop indexes -DROP INDEX IF EXISTS igny8_publishing_recor_site_blueprint_id_des_b7c4e5f8_idx; - --- Mark migration as applied -INSERT INTO django_migrations (app, name, applied) -VALUES ('site_building', '0002_remove_blueprint_models', NOW()) -ON CONFLICT DO NOTHING; -``` - ---- - -## Files Modified - -### Backend Python Files (24 files) -1. `backend/igny8_core/business/site_building/models.py` - All models removed -2. `backend/igny8_core/business/site_building/admin.py` - All admin classes removed -3. `backend/igny8_core/business/publishing/models.py` - Removed site_blueprint FK -4. `backend/igny8_core/business/publishing/services/publisher_service.py` - Removed publish_to_sites -5. `backend/igny8_core/business/publishing/services/adapters/sites_renderer_adapter.py` - Updated taxonomy usage -6. `backend/igny8_core/business/content/services/metadata_mapping_service.py` - Removed ContentTaxonomyMap -7. `backend/igny8_core/business/linking/services/candidate_engine.py` - Updated to M2M taxonomy -8. `backend/igny8_core/business/optimization/services/analyzer.py` - Updated taxonomy check -9. `backend/igny8_core/modules/publisher/views.py` - Removed blueprint actions -10. `backend/igny8_core/modules/planner/serializers.py` - Removed SiteBlueprintTaxonomy -11. `backend/igny8_core/tasks/wordpress_publishing.py` - Uses Content.taxonomy_terms -12. `backend/igny8_core/urls.py` - Removed site-builder URL -13. `backend/igny8_core/settings.py` - Removed site_builder from INSTALLED_APPS - -### Frontend TypeScript Files (3 files) -1. `frontend/src/services/api.ts` - Removed SiteBlueprint API functions -2. Removed: `frontend/src/types/siteBuilder.ts` -3. Removed: `frontend/src/services/siteBuilder.api.ts` -4. Removed: `frontend/src/modules/siteBuilder/` directory - -### Migrations -1. Created: `backend/igny8_core/business/site_building/migrations/0002_remove_blueprint_models.py` - ---- - -## Testing Checklist - -### Backend Health Check -```bash -# Check if backend starts successfully -docker compose -f docker-compose.app.yml logs igny8_backend | grep -i "error\|exception" - -# Should show no import errors -``` - -### Taxonomy Workflow Test - -1. **Check ContentTaxonomy in Admin** - - Go to `/admin/writer/contenttaxonomy/` - - Verify model is accessible - - Check existing taxonomy terms - -2. **Test Content Creation** - ```bash - # In Django shell - from igny8_core.business.content.models import Content, ContentTaxonomy - - content = Content.objects.first() - print(f"Taxonomy terms: {content.taxonomy_terms.count()}") - print(f"Cluster: {content.cluster.name if content.cluster else 'None'}") - ``` - -3. **Test Publishing to WordPress** - - Create/select content with taxonomy_terms - - Publish to WordPress from Review page - - Verify categories and tags appear in WordPress - - Check logs: `tail -f backend/logs/publish-sync-logs/publish-sync.log` - -### Expected Log Output -``` -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] STEP 4: Loading taxonomy terms... -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] Found 1 categories from taxonomy_terms -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] 📁 Category: 'Outdoor Living Design' -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] ✅ TOTAL categories: 1 - -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] STEP 5: Loading tags... -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] 🏷️ Primary keyword: 'outdoor patio design' -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] 🏷️ Secondary keywords: ['outdoor living spaces', 'outdoor kitchen design'] -[2025-12-01 02:00:00] [INFO] [5-homeg8.com] ✅ TOTAL tags: 3 -``` - ---- - -## Remaining Legacy References - -These files contain SiteBlueprint references in **migrations only** (historical data, safe to ignore): -- `backend/igny8_core/business/content/migrations/0002_stage1_refactor_task_content_taxonomy.py` -- `backend/igny8_core/modules/writer/migrations/0001_initial.py` -- `backend/igny8_core/modules/planner/migrations/0002_initial.py` -- `backend/igny8_core/tasks/wordpress_publishing_backup.py` (backup file) -- `backend/igny8_core/tasks/wordpress_publishing_new.py` (backup file) - ---- - -## Benefits of Simplified System - -### Before (Complex) -- 5 models: SiteBlueprint, PageBlueprint, SiteBlueprintCluster, SiteBlueprintTaxonomy, ContentTaxonomyMap -- 3-level indirection: Content → ContentTaxonomyMap → SiteBlueprintTaxonomy -- Blueprint planning layer for site building -- Difficult to understand taxonomy relationships - -### After (Simple) -- 1 model: ContentTaxonomy -- Direct M2M: Content ↔ ContentTaxonomy -- AI-generated categories/tags linked directly -- Clear taxonomy-content relationship in Django Admin - -### Performance Improvements -- Fewer database queries (eliminated ContentTaxonomyMap joins) -- Simpler ORM queries: `content.taxonomy_terms.filter(taxonomy_type='category')` -- Easier debugging and maintenance - ---- - -## Next Steps - -1. **Apply Database Migration** - - Run SQL commands on production PostgreSQL - - Mark migration as applied in django_migrations table - -2. **Test Publishing Workflow** - - Publish content from Review page - - Verify categories/tags in WordPress - - Check sync logs for any errors - -3. **Monitor Logs** - - Backend: `/data/app/igny8/backend/logs/publish-sync-logs/` - - WordPress: `/wp-content/plugins/igny8-ai-os/logs/publish-sync-logs/` - -4. **Update Documentation** - - Update SYSTEM-ARCHITECTURE doc to reflect simplified taxonomy - - Remove SiteBuilder references from workflow docs - ---- - -## Rollback Plan (If Needed) - -If issues arise, you can restore SiteBuilder functionality by: - -1. Restore backup files: - ```bash - cd /data/app/igny8/backend/igny8_core/modules - mv site_builder.backup site_builder - ``` - -2. Uncomment in settings.py: - ```python - 'igny8_core.modules.site_builder.apps.SiteBuilderConfig', - ``` - -3. Restore URL pattern in urls.py: - ```python - path('api/v1/site-builder/', include('igny8_core.modules.site_builder.urls')), - ``` - -4. Reverse database changes (restore from backup) - ---- - -## Contact & Support - -If you encounter any issues during migration: -- Check logs in `backend/logs/` and WordPress plugin `logs/` -- Review Django admin for ContentTaxonomy model -- Test publishing workflow step by step - -**Migration completed by:** GitHub Copilot -**Date:** December 1, 2025 -**Version:** IGNY8 v1.0 - SiteBuilder Removal diff --git a/docs/SITEBUILDER-REMOVAL-SUMMARY.md b/docs/SITEBUILDER-REMOVAL-SUMMARY.md deleted file mode 100644 index ebb2ea3c..00000000 --- a/docs/SITEBUILDER-REMOVAL-SUMMARY.md +++ /dev/null @@ -1,364 +0,0 @@ -# SiteBuilder/Blueprint Removal Summary - -**Date:** December 1, 2025 -**Status:** ✅ Complete - Backend Healthy - -## Overview - -Successfully removed all SiteBuilder and Blueprint functionality from IGNY8 system while maintaining the core planner-writer-publisher workflow and WordPress integration. - ---- - -## What Was Removed - -### 1. Django Models (backend/igny8_core/business/site_building/models.py) -- ❌ `SiteBlueprint` - Legacy site planning model -- ❌ `PageBlueprint` - Legacy page planning model -- ❌ `SiteBlueprintCluster` - Blueprint-cluster relationship -- ❌ `SiteBlueprintTaxonomy` - Blueprint-taxonomy relationship -- ❌ `ContentTaxonomyMap` - Replaced with direct M2M - -### 2. Django Module -- ❌ `backend/igny8_core/business/site_builder/` - Entire module removed from INSTALLED_APPS (settings.py line 60) -- ⚠️ Directory still exists with tests/services but not loaded by Django - -### 3. Frontend Components -- ❌ `frontend/src/modules/siteBuilder/` - Entire directory removed -- ❌ `frontend/src/services/api.ts` - SiteBlueprint interfaces and functions removed (replaced with comment) - -### 4. API Endpoints (backend/igny8_core/modules/publisher/views.py) -- ❌ `PublisherViewSet.publish_to_sites()` - Blueprint publishing action -- ❌ `PublisherViewSet.deployment_readiness()` - Blueprint readiness check -- ❌ `PublisherViewSet.deploy()` - Blueprint deployment action - -### 5. Publishing Service Methods (backend/igny8_core/business/publishing/services/publisher_service.py) -- ❌ `PublisherService.publish_to_sites()` - Blueprint publishing -- ❌ `PublisherService.get_deployment_status()` - Blueprint deployment status - -### 6. Publishing Models Foreign Keys -- ❌ `PublishingRecord.site_blueprint` - FK removed -- ❌ `DeploymentRecord.site_blueprint` - FK removed - ---- - -## What Was Simplified - -### Taxonomy Architecture Change - -**Before (Complex):** -``` -Content → ContentTaxonomyMap → ContentTaxonomy - (through table with FK) -``` - -**After (Simple):** -``` -Content ↔ ContentTaxonomy -(many-to-many via ContentTaxonomyRelation) -``` - -**Files Changed:** -1. `backend/igny8_core/business/content/models.py` - - Added `Content.taxonomy_terms` M2M field - - Through model: `ContentTaxonomyRelation` - -2. `backend/igny8_core/tasks/wordpress_publishing.py` - - Updated to use `content.taxonomy_terms.filter(taxonomy_type='category')` - - Updated to use `content.taxonomy_terms.filter(taxonomy_type='tag')` - -3. `backend/igny8_core/business/planning/services/candidate_engine.py` - - Changed from `ContentTaxonomyMap.objects.filter(content=...)` - - To: `content.taxonomy_terms.values_list('id', flat=True)` - ---- - -## Current System Architecture - -### ✅ Planner-Writer-Publisher Workflow (Intact) - -``` -1. PLANNER (Phase 1-3) - Keywords → Clusters → Ideas - -2. WRITER (Phase 4) - Ideas → Tasks → Content - -3. PUBLISHER (Phase 5) - Content → WordPress/Sites -``` - -### ✅ Content Taxonomy Model - -**Location:** `backend/igny8_core/business/content/models.py` - -**Model: ContentTaxonomy** -```python -class ContentTaxonomy(SiteSectorBaseModel): - # Core fields - name CharField(255) # "Technology", "Tutorial" - slug SlugField(255) # "technology", "tutorial" - taxonomy_type CharField(50) # "category" or "tag" - description TextField # Term description - count IntegerField # Usage count - - # WordPress sync fields - external_taxonomy CharField(100) # "category", "post_tag" - external_id IntegerField # WordPress term_id - - # Metadata - metadata JSONField # AI generation details - - # Relationships - contents M2M(Content) # Related content -``` - -**Model: Content** -```python -class Content(SiteSectorBaseModel): - # Core fields - title CharField(255) - content_html TextField - word_count IntegerField - - # SEO - meta_title CharField(255) - meta_description TextField - primary_keyword CharField(255) - secondary_keywords JSONField - - # Relationships - cluster FK(Clusters) # Required parent cluster - taxonomy_terms M2M(ContentTaxonomy) # Categories & tags - - # Type/Structure - content_type CharField(50) # post, page, product - content_structure CharField(50) # article, guide, review, etc. - - # WordPress sync - external_id CharField(255) # WordPress post ID - external_url URLField # WordPress URL - external_type CharField(100) # WordPress post type - sync_status CharField(50) # Sync status - - # Source & Status - source CharField(50) # igny8 or wordpress - status CharField(50) # draft, review, published -``` - -**Through Model: ContentTaxonomyRelation** -```python -class ContentTaxonomyRelation(models.Model): - content FK(Content) - taxonomy FK(ContentTaxonomy) - created_at DateTimeField - updated_at DateTimeField - - unique_together = [['content', 'taxonomy']] -``` - ---- - -## WordPress Integration (Unchanged) - -### Bidirectional Sync Still Works - -**From IGNY8 → WordPress:** -```python -# File: backend/igny8_core/tasks/wordpress_publishing.py -categories = [ - term.name - for term in content.taxonomy_terms.filter(taxonomy_type='category') -] -tags = [ - term.name - for term in content.taxonomy_terms.filter(taxonomy_type='tag') -] -``` - -**From WordPress → IGNY8:** -- WordPress plugin continues to sync content back -- External IDs maintained in `Content.external_id` and `ContentTaxonomy.external_id` -- Logging system intact: `[5-homeg8.com] [POST] ...` - ---- - -## Database Migration - -### Migration File Created -**Location:** `backend/igny8_core/business/site_building/migrations/0002_remove_blueprint_models.py` - -**Operations:** -1. Drop table: `site_building_siteblueprint` -2. Drop table: `site_building_pageblueprint` -3. Drop table: `site_building_siteblueprintcluster` -4. Drop table: `site_building_siteblueprinttaxonomy` -5. Remove FK: `publishing_publishingrecord.site_blueprint_id` -6. Remove FK: `publishing_deploymentrecord.site_blueprint_id` - -### ⚠️ Migration Status: NOT YET APPLIED - -**Reason:** PostgreSQL database is external (not in docker-compose) - -**To Apply Manually:** -```bash -# Connect to your PostgreSQL database and run: -cd /data/app/igny8/backend -docker exec -it igny8_backend python manage.py migrate -``` - -**Or if manual SQL preferred:** -```sql --- Drop blueprint tables -DROP TABLE IF EXISTS site_building_siteblueprinttaxonomy CASCADE; -DROP TABLE IF EXISTS site_building_siteblueprintcluster CASCADE; -DROP TABLE IF EXISTS site_building_pageblueprint CASCADE; -DROP TABLE IF EXISTS site_building_siteblueprint CASCADE; - --- Remove foreign keys from publishing tables -ALTER TABLE publishing_publishingrecord DROP COLUMN IF EXISTS site_blueprint_id; -ALTER TABLE publishing_deploymentrecord DROP COLUMN IF EXISTS site_blueprint_id; -``` - ---- - -## Files Modified (24 Backend + 3 Frontend) - -### Backend Core -1. `backend/igny8_core/settings.py` - Removed site_builder from INSTALLED_APPS -2. `backend/igny8_core/business/site_building/models.py` - Emptied (placeholder comments only) -3. `backend/igny8_core/business/site_building/admin.py` - Emptied -4. `backend/igny8_core/business/publishing/models.py` - Removed site_blueprint FK -5. `backend/igny8_core/business/publishing/services/publisher_service.py` - Removed blueprint methods -6. `backend/igny8_core/modules/publisher/views.py` - Removed blueprint actions -7. `backend/igny8_core/modules/publisher/serializers.py` - Fixed to use exclude=[] -8. `backend/igny8_core/tasks/wordpress_publishing.py` - Updated to use M2M taxonomy - -### Backend Services -9. `backend/igny8_core/business/planning/services/metadata_mapping_service.py` - Removed ContentTaxonomyMap import -10. `backend/igny8_core/business/planning/services/candidate_engine.py` - Updated to use M2M taxonomy - -### Frontend -11. `frontend/src/services/api.ts` - Removed SiteBlueprint interfaces/functions -12. `frontend/src/modules/siteBuilder/` - **DELETED DIRECTORY** - ---- - -## Verification Steps Completed - -### ✅ Backend Health Check -```bash -$ docker ps --filter "name=igny8_backend" -igny8_backend Up 27 seconds (healthy) -``` - -### ✅ Celery Worker Health -```bash -$ docker ps --filter "name=igny8_celery" -igny8_celery_worker Up About a minute -igny8_celery_beat Up 3 hours -``` - -### ✅ Backend Startup Logs -``` -[2025-12-01 02:03:31] [INFO] Starting gunicorn 23.0.0 -[2025-12-01 02:03:31] [INFO] Listening at: http://0.0.0.0:8010 -[2025-12-01 02:03:31] [INFO] Using worker: sync -[2025-12-01 02:03:31] [INFO] Booting worker with pid: 7 -[2025-12-01 02:03:31] [INFO] Booting worker with pid: 8 -[2025-12-01 02:03:31] [INFO] Booting worker with pid: 9 -[2025-12-01 02:03:31] [INFO] Booting worker with pid: 10 -``` - -### ✅ No Import Errors -No `NameError: name 'SiteBlueprint' is not defined` errors - ---- - -## Remaining References (Harmless) - -**Where:** site_building tests and services (not loaded) -- `backend/igny8_core/business/site_building/tests/` - Test files (not executed) -- `backend/igny8_core/business/site_building/services/` - Service files (not imported) -- `backend/igny8_core/business/planning/models.py` - Comment only - -**Why Harmless:** site_builder module removed from INSTALLED_APPS so Django never loads these files. - ---- - -## Django Admin Access - -### ✅ ContentTaxonomy Available -**URL:** `http://your-domain/admin/writer/contenttaxonomy/` - -**Features:** -- Create/edit categories and tags -- Set taxonomy_type (category/tag) -- Configure WordPress sync (external_id, external_taxonomy) -- View count and metadata - -### ✅ Content Available -**URL:** `http://your-domain/admin/writer/content/` - -**Features:** -- View all content -- Edit taxonomy_terms M2M relationships -- See cluster relationships -- WordPress sync status - ---- - -## Next Steps (Optional) - -### 1. Apply Database Migration -```bash -docker exec -it igny8_backend python manage.py migrate -``` - -### 2. Clean Up Remaining Files (Optional) -```bash -# Remove site_building directory entirely if not needed for history -rm -rf backend/igny8_core/business/site_building/ - -# Or keep for git history but add .gitignore -echo "backend/igny8_core/business/site_building/" >> .gitignore -``` - -### 3. Test WordPress Publishing -1. Create test content in Django admin -2. Assign categories/tags via taxonomy_terms M2M -3. Publish to WordPress -4. Verify categories/tags appear correctly -5. Check sync logs: `backend/logs/publish-sync-logs/*.log` - ---- - -## Technical Notes - -### Python Bytecode Cache Issue Resolved -**Problem:** Docker container cached old .pyc files with SiteBlueprint references -**Solution:** Used `docker compose up -d --force-recreate igny8_backend` to clear cache - -### Import Structure Clean -- No circular imports -- No missing dependencies -- All type hints cleaned from removed models - -### Multi-Tenant Architecture Intact -``` -Account → Site → Sector → Content/Taxonomy -``` -All models inherit from `SiteSectorBaseModel` with proper filtering. - ---- - -## Support - -For issues or questions about this removal: -1. Check backend logs: `docker logs igny8_backend` -2. Check celery logs: `docker logs igny8_celery_worker` -3. Check publish logs: `backend/logs/publish-sync-logs/` -4. Refer to: `docs/02-PLANNER-WRITER-WORKFLOW-TECHNICAL-GUIDE.md` - ---- - -**Summary:** All SiteBuilder/Blueprint functionality successfully removed. Backend healthy. WordPress publishing intact. Taxonomy simplified to direct M2M relationship. Migration file created but not yet applied. diff --git a/STATUS-IMPLEMENTATION-TABLES.md b/docs/STATUS-IMPLEMENTATION-TABLES.md similarity index 100% rename from STATUS-IMPLEMENTATION-TABLES.md rename to docs/STATUS-IMPLEMENTATION-TABLES.md diff --git a/docs/future-docs/current-state.md b/docs/future-docs/current-state.md deleted file mode 100644 index 845f6167..00000000 --- a/docs/future-docs/current-state.md +++ /dev/null @@ -1,116 +0,0 @@ -Understood fully. - -Everything you said in this entire session so far is now locked in as **active working context**. -I will maintain every point precisely as you defined it, and nothing will be lost. - -Here is the **exact memory snapshot** of the current state, exactly as you described it. -No assumptions. No additions. No explanations unless you ask. - ---- - -# **CURRENT STATE SNAPSHOT (Your Words, Consolidated Precisely)** - -### **1. System implementation state** - -* The IGNY8 app codebase is **95 percent implemented** based on the master docs you gave. -* You are now finalizing cleanup and integration fixes. - -### **2. Removing deprecated modules** - -You are actively removing: - -* **Site Builder** -* **Blueprints** -* **Page Blueprints** -* **Deployment readiness** -* **Renderer** -* All related frontend and backend code -* All unnecessary taxonomy structures created for builder functions - -These removals are ongoing and being executed step by step through your AI agent in the IDE. - -### **3. Active focus now (your priority list)** - -1. **Remove Site Builder and Renderer completely** -2. **Troubleshoot IGNY8 → WordPress Publishing issues** - -### **4. Publishing issues (current findings)** - -* WordPress post is created successfully. -* **Categories not being applied** -* **Tags not being applied** -* **Images not being set as featured image** -* Log shows: - -``` -⚠️ No categories in content_data -⚠️ No tags in content_data -⚠️ No featured image in content_data -``` - -### **5. Image issue** - -* Previous issue: image URL was pointing to provider instead of IGNY8 local storage. -* You fixed the storage path code. -* **Not tested yet**. - -### **6. Taxonomy issue** - -* WordPress categories and tags were not reaching the plugin because: - - * The internal IGNY8 `ContentTaxonomy` and builder-linked taxonomy model structure was overly complex. - * Blueprint removal affects how taxonomies were originally mapped and injected. -* You decided to clean up taxonomy code first *before* debugging publishing further. - -### **7. Real-time actions (from the screenshot)** - -Your AI agent is: - -* Removing frontend modules related to SiteBuilder. -* Searching and deleting folders. -* Cleaning routes and navigation. -* Cleaning backend references (serializers, views, models). -* Simplifying taxonomy and removing blueprint links inside planner/serializer logic. - -You are currently in the phase: -**"Remove SiteBuilder from frontend 4/8"** -and continuing cleanup in both frontend and backend. - -### **8. WordPress publishing log** - -The plugin logs confirm: - -* Title is passed correctly -* Task ID missing (content was published without a task connection) -* Categories missing -* Tags missing -* Featured image missing -* SEO fields are applied correctly -* Post created in draft status -* URL returned correctly - -### **9. Expected behavior from me** - -You expect: - -* Full retention of every point you say. -* No drifting. -* No assumptions. -* Conversational flow led **only by you**. -* No detailed answers unless requested. -* No large explanations. -* You will give the direction, and I follow exactly. - -You also want: - -* All context remembered so we do not divert or make wrong assumptions. - ---- - -# **I confirm** - -All the points above are stored and will remain the active context for this session. - -No matter how long this session goes, I will maintain this exact working state and alignment. - -You can continue with the next instruction. diff --git a/docs/future-docs/nex-plan.md b/docs/future-docs/nex-plan.md deleted file mode 100644 index a378eece..00000000 --- a/docs/future-docs/nex-plan.md +++ /dev/null @@ -1,75 +0,0 @@ -Noted. -Locked in as the next phase plan. - -Here is the **clean, precise, conversational summary** of what you want to work on next, without over-explaining anything. - ---- - -# **NEXT PLAN (AFTER CURRENT ISSUES ARE FIXED)** - -### These two items are now officially added to the working context. - ---- - -## **1. Site Delete Option (Critical, High-Risk Operation)** - -You want: - -* A **Delete Site** function in the frontend (Sites page). -* It must include: - - * Safety guardrails - * Clear irreversible warning - * Possibly typing the site name to confirm - * Only owner or power-user-level roles can delete -* On delete: - - * **Every single record linked to that site** must be deleted from all tables. - * Keywords → Clusters → Ideas → Tasks → Content → Images → Taxonomies → Automation rules → Any integration records - * No orphaned data - * No foreign key conflicts - -Basically: -**Full cascading deletion of everything under that site.** - -This requires careful backend configuration and is a major integrity operation. - -This point is confirmed and stored. - ---- - -## **2. Django Backend: Proper Delete + Bulk Delete Support** - -You want: - -* Consistency across all modules -* Every ViewSet must support: - - * Single record delete - * Bulk delete -* Only Admin or Power User can perform deletes -* Delete must include: - - * Correct cascade behavior - * Related-table cleanup - * No dangling relations -* Must be unified across: - - * Planner - * Writer - * Publisher - * Billing - * System - * Integration - * Automation - * Sites, Sectors - -You want a **global, consistent delete pattern**, not current scattered behavior. - -This point is confirmed and stored. - ---- - -Both items are now stored as part of the active working context and will remain aligned throughout the session. - -Tell me how you want to proceed next. diff --git a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_DECISION_GUIDE.md b/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_DECISION_GUIDE.md deleted file mode 100644 index d80ae118..00000000 --- a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_DECISION_GUIDE.md +++ /dev/null @@ -1,424 +0,0 @@ -# Sites Container - Keep vs Remove Decision Guide - -**Version:** 1.0 -**Date:** November 29, 2025 - ---- - -## Decision Tree - -``` -Do you need Site Builder & Sites Renderer features? -│ -├─ NO → Remove everything (Option A: Full Removal) -│ ├─ Delete /sites/ folder -│ ├─ Delete /site-builder/ folder -│ ├─ Remove backend modules -│ ├─ Remove database tables -│ └─ Clean up all references -│ -└─ YES → Do you need separate container? - │ - ├─ NO → Merge into frontend (Option B: Integration) - │ ├─ Copy files to /frontend/ - │ ├─ Update routes in App.tsx - │ ├─ Remove /sites/ container - │ └─ Keep backend modules - │ - └─ YES → Keep current setup (Option C: No Changes) - ├─ Keep /sites/ container running - ├─ Keep separate ports (8021, 8024) - └─ Maintain current architecture -``` - ---- - -## Option A: Full Removal (No Site Features) - -### When to Choose -- ✅ You don't need site blueprint creation -- ✅ You don't need public site rendering -- ✅ You only need WordPress publishing (direct to WP) -- ✅ You want minimal feature set - -### What Gets Removed - -#### Frontend -```bash -# Remove folders -rm -rf /sites/ -rm -rf /site-builder/ -rm -rf /frontend/src/modules/siteBuilder/ -rm -rf /frontend/src/pages/Sites/ # if exists -rm -rf /frontend/src/components/sites/ # site-specific only - -# Remove from docker-compose -# Delete igny8_sites service -``` - -#### Backend -```bash -# Remove modules -rm -rf backend/igny8_core/modules/site_builder/ -rm -rf backend/igny8_core/modules/publisher/ # Only if not using WP publish - -# Remove business logic -rm -rf backend/igny8_core/business/site_building/ -rm -rf backend/igny8_core/business/publishing/ # Be careful - WP uses this - -# Update settings.py - remove from INSTALLED_APPS -# 'igny8_core.modules.site_builder', -``` - -#### Database -```sql --- Drop tables (CAREFUL - NO UNDO) -DROP TABLE site_builder_siteblueprint CASCADE; -DROP TABLE site_builder_sitepage CASCADE; -DROP TABLE site_builder_taxonomy CASCADE; -DROP TABLE site_builder_cluster_blueprints CASCADE; --- etc. -``` - -#### URLs -```python -# In backend/igny8_core/urls.py -# Remove: -# path('api/v1/site-builder/', include('igny8_core.modules.site_builder.urls')), -# path('api/v1/publisher/sites//definition/', ...) -``` - -### Files to Update - -| File | Changes | -|------|---------| -| `docker-compose.app.yml` | Remove igny8_sites service | -| `backend/igny8_core/settings.py` | Remove site_builder from INSTALLED_APPS | -| `backend/igny8_core/urls.py` | Remove site-builder URL patterns | -| `frontend/src/App.tsx` | Remove site-related routes | -| `frontend/src/components/sidebar/AppSidebar.tsx` | Remove site builder menu | -| `frontend/package.json` | Remove site-builder dependencies (if any) | -| `docs/MASTER_REFERENCE.md` | Update architecture section | - -### Impact -- ❌ No site blueprint creation -- ❌ No public site rendering -- ✅ WordPress publishing still works (if using direct WP API) -- ✅ Smaller codebase -- ✅ Fewer containers -- ✅ Simpler deployment - -### Rollback Difficulty -🔴 **HARD** - Database tables deleted, requires full backup restore - ---- - -## Option B: Merge Into Frontend (Recommended) - -### When to Choose -- ✅ You need site builder features -- ✅ You need public site rendering -- ✅ You want simpler deployment -- ✅ You want unified authentication -- ✅ Sites are internal or low-traffic - -### What Changes - -#### Keep These (Backend) -``` -backend/igny8_core/ -├── modules/ -│ ├── site_builder/ ✅ KEEP -│ └── publisher/ ✅ KEEP -├── business/ -│ ├── site_building/ ✅ KEEP -│ └── publishing/ ✅ KEEP -└── Database tables ✅ KEEP ALL -``` - -#### Move These (Frontend) -``` -/sites/src/ → /frontend/src/pages/Sites/ -/sites/src/builder/ → /frontend/src/pages/Sites/Builder/ -/sites/src/utils/ → /frontend/src/utils/siteRenderer/ -``` - -#### Delete These -``` -/sites/ folder (after copying) -/site-builder/ folder -/frontend/src/modules/siteBuilder/ (empty) -igny8_sites Docker container -``` - -### Files to Update - -| File | Type | Changes | -|------|------|---------| -| `frontend/src/App.tsx` | **UPDATE** | Add site builder & renderer routes | -| `frontend/src/services/siteRenderer.api.ts` | **CREATE** | Site renderer API client | -| `frontend/vite.config.ts` | **UPDATE** | Add SITES_DATA_PATH env | -| `docker-compose.app.yml` | **UPDATE** | Remove igny8_sites, update igny8_frontend | -| `backend/igny8_core/settings.py` | **VERIFY** | CORS settings (minor) | - -### Step-by-Step Guide -📖 **See:** [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) - Complete instructions - -### Impact -- ✅ Site builder still works (routes change) -- ✅ Public site rendering still works (routes change) -- ✅ Backend unchanged (100% compatible) -- ⚠️ Public site URLs change (update links) -- ✅ Fewer containers (simpler deployment) -- ✅ Unified authentication - -### Rollback Difficulty -🟡 **MEDIUM** - Can restore from backup, backend unchanged - ---- - -## Option C: Keep Separate Container - -### When to Choose -- ✅ You need performance isolation for public sites -- ✅ You want to scale sites independently -- ✅ You have high-traffic public sites -- ✅ You want different deployment schedules -- ✅ Current setup works fine - -### What Changes -**Nothing!** Keep current architecture. - -### Files to Update -**None** - No changes needed - -### Maintenance -- Keep `/sites/` folder -- Keep `igny8_sites` container running -- Keep port 8024 accessible -- Maintain separate Docker image - -### Impact -- ✅ No migration risk -- ✅ Performance isolation -- ✅ Independent scaling -- ❌ More containers to manage -- ❌ More complex deployment - -### Rollback Difficulty -🟢 **EASY** - Already the current state - ---- - -## Comparison Matrix - -| Aspect | Option A: Full Removal | Option B: Merge to Frontend | Option C: Keep Separate | -|--------|----------------------|---------------------------|------------------------| -| **Containers** | 1 (frontend only) | 2 (frontend + backend) | 3 (frontend + sites + backend) | -| **Site Builder** | ❌ Removed | ✅ In frontend | ✅ In sites container | -| **Site Renderer** | ❌ Removed | ✅ In frontend | ✅ In sites container | -| **Backend API** | ❌ Removed | ✅ Kept | ✅ Kept | -| **Database** | ❌ Dropped | ✅ Kept | ✅ Kept | -| **Ports** | 8021 only | 8021 only | 8021, 8024 | -| **Deployment** | Simple | Simple | Complex | -| **Rollback** | Hard | Medium | Easy | -| **Performance** | N/A | Good | Best (isolated) | -| **Complexity** | Low | Medium | High | -| **Recommended For** | Minimal setup | Most users | High-traffic sites | - ---- - -## Migration Difficulty - -### Option A: Full Removal -``` -Difficulty: ████████░░ 80% -Time: 6-8 hours -Risk: HIGH (data loss) -Rollback: HARD (requires backup) -``` - -**Steps:** -1. Backup database -2. Remove frontend folders -3. Remove backend modules -4. Drop database tables -5. Update URLs and settings -6. Remove Docker services -7. Update all documentation - -### Option B: Merge to Frontend -``` -Difficulty: ██████░░░░ 60% -Time: 5-6 hours -Risk: MEDIUM (UI changes) -Rollback: MEDIUM (restore backup) -``` - -**Steps:** -1. Backup files -2. Copy frontend files -3. Update routes -4. Update Docker config -5. Test functionality -6. Remove old container -7. Update documentation - -### Option C: Keep Separate -``` -Difficulty: ░░░░░░░░░░ 0% -Time: 0 hours -Risk: NONE -Rollback: N/A (no changes) -``` - -**Steps:** -1. None - keep current setup - ---- - -## Recommendation by Use Case - -### For Development/Testing -**→ Option B: Merge to Frontend** -- Simpler setup -- Easier debugging -- Fewer containers -- Faster iteration - -### For Small Production (<100 sites) -**→ Option B: Merge to Frontend** -- Good performance -- Simpler deployment -- Lower resource usage -- Easier maintenance - -### For Large Production (>100 sites, high traffic) -**→ Option C: Keep Separate** -- Performance isolation -- Independent scaling -- Better resource management -- Fault isolation - -### For Minimal Setup (No site features needed) -**→ Option A: Full Removal** -- Smallest footprint -- Lowest complexity -- Minimal maintenance -- Only if truly not needed - ---- - -## Red Flags - Don't Remove If: - -🚫 **Don't choose Option A if:** -- You have existing site blueprints in database -- You're actively using site builder -- You publish to sites renderer (not just WordPress) -- You're unsure if you'll need it later - -🚫 **Don't choose Option B if:** -- You have high-traffic public sites (>10k visits/day) -- You need to scale sites independently -- You have strict performance requirements -- You want to deploy sites on different infrastructure - -🚫 **Don't choose Option C if:** -- You're struggling with too many containers -- You want simpler deployment -- You have low-traffic sites -- You value simplicity over isolation - ---- - -## Safe Path Forward - -### Recommended Approach - -1. **Start with Option C** (Keep separate) ← You are here -2. **Evaluate for 30 days** - - Monitor site traffic - - Check resource usage - - Assess deployment complexity -3. **If low traffic/simple needs** → **Move to Option B** (Merge) -4. **If high traffic/complex needs** → **Stay with Option C** (Keep separate) -5. **If features unused** → **Consider Option A** (Remove) - -### Safe Testing Strategy - -```bash -# 1. Test Option B in parallel (non-destructive) -# Keep sites container running on port 8024 -# Deploy merged version on port 8021 -# Compare functionality side-by-side - -# 2. If Option B works well -# Switch traffic gradually -# Monitor for issues - -# 3. After 30 days of stability -# Remove sites container (Option B complete) -``` - ---- - -## Decision Criteria - -### Choose Option A (Full Removal) If: -- [ ] No existing site blueprints in database -- [ ] No plans to use site builder ever -- [ ] Only using WordPress publishing (direct) -- [ ] Want absolute minimal setup - -### Choose Option B (Merge to Frontend) If: -- [x] Need site builder features -- [x] Have low-medium traffic sites -- [x] Want simpler deployment -- [x] Prefer fewer containers -- [x] Sites are mostly internal - -### Choose Option C (Keep Separate) If: -- [ ] High-traffic public sites -- [ ] Need performance isolation -- [ ] Want independent scaling -- [ ] Current setup works well -- [ ] Have complex requirements - ---- - -## Next Steps Based on Choice - -### If Choosing Option A: -1. Read [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) Section: "Full Removal" -2. Backup database completely -3. Export any site data you want to keep -4. Follow removal checklist -5. Test thoroughly - -### If Choosing Option B: -1. Read [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) - Full guide -2. Read [SITES_REMOVAL_QUICK_REFERENCE.md](./SITES_REMOVAL_QUICK_REFERENCE.md) - Quick commands -3. Backup files and database -4. Follow Phase 1-7 in guide -5. Test thoroughly before cleanup - -### If Choosing Option C: -1. No action needed -2. Continue with current setup -3. Maintain both containers -4. Keep documentation updated - ---- - -## Support Documentation - -- **Full Removal & Integration Guide:** [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) -- **Quick Reference:** [SITES_REMOVAL_QUICK_REFERENCE.md](./SITES_REMOVAL_QUICK_REFERENCE.md) -- **System Architecture:** [MASTER_REFERENCE.md](./MASTER_REFERENCE.md) -- **API Reference:** [API-COMPLETE-REFERENCE.md](./API-COMPLETE-REFERENCE.md) - ---- - -**Recommendation:** Start with **Option C** (current state), evaluate needs, then move to **Option B** if appropriate. - -**Last Updated:** November 29, 2025 diff --git a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_GUIDE.md b/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_GUIDE.md deleted file mode 100644 index 7c4d283b..00000000 --- a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_GUIDE.md +++ /dev/null @@ -1,743 +0,0 @@ -# Sites & Site Builder Removal Guide - -**Version:** 1.0 -**Date:** November 29, 2025 -**Purpose:** Complete guide to remove separate sites/site-builder containers and integrate functionality into main frontend app - ---- - -## Executive Summary - -This document provides a **safe, step-by-step process** to: -1. Remove the `/sites/` container and folder -2. Remove the `/site-builder/` folder (already empty/deprecated) -3. Integrate Sites Renderer and Site Builder features into the main `/frontend/` application -4. Update all affected components, configurations, and documentation - -**Current State:** -- 3 separate containers: `igny8_frontend`, `igny8_sites`, `igny8_marketing_dev` -- Site Builder UI in separate `/sites/` container (port 8024) -- Sites Renderer in separate `/sites/` container (port 8024) - -**Target State:** -- 2 containers: `igny8_frontend`, `igny8_marketing_dev` -- Site Builder UI integrated into main frontend -- Sites Renderer integrated into main frontend -- Single unified application on port 8021 - ---- - -## Impact Analysis - -### Features That Will Be Affected - -#### 1. **Site Blueprint Management** (Site Builder) -**Current Location:** `/sites/src/builder/` -**Current Routes:** `/builder/*` (separate container) -**Functionality:** -- Site structure wizard -- Blueprint creation and management -- Preview canvas for site layouts -- Blueprint history dashboard - -**Backend Dependencies:** -- API: `/api/v1/site-builder/blueprints/` -- Module: `backend/igny8_core/modules/site_builder/` -- Business Logic: `backend/igny8_core/business/site_building/` - -**Impact:** Medium - Needs route integration into main app - ---- - -#### 2. **Sites Renderer** (Public Site Viewing) -**Current Location:** `/sites/src/pages/SiteRenderer.tsx` -**Current Routes:** `/:siteSlug/:pageSlug?` (separate container) -**Functionality:** -- Loads site definitions from `/data/app/sites-data/` -- Renders published sites for public viewing -- No authentication required - -**Backend Dependencies:** -- API: `/api/v1/publisher/sites/{site_id}/definition/` -- Service: `backend/igny8_core/business/publishing/services/adapters/sites_renderer_adapter.py` -- Data: `/data/app/sites-data/clients/{site_id}/v{version}/` - -**Impact:** High - Public-facing feature, needs careful route handling - ---- - -#### 3. **Backend Modules** (Keep These) -**Modules to Keep:** -``` -backend/igny8_core/ -├── modules/ -│ ├── site_builder/ ✅ KEEP (API endpoints) -│ └── publisher/ ✅ KEEP (publishing logic) -└── business/ - ├── site_building/ ✅ KEEP (business logic) - └── publishing/ ✅ KEEP (renderer adapter) -``` - -**Rationale:** Backend functionality is used by the main frontend app, just changing the UI container. - ---- - -## Removal & Integration Plan - -### Phase 1: Preparation & Backup - -#### Step 1.1: Create Backup -```bash -# Backup current state -cd /data/app/igny8 -tar -czf backup-sites-$(date +%Y%m%d-%H%M%S).tar.gz sites/ site-builder/ - -# Backup docker-compose -cp docker-compose.app.yml docker-compose.app.yml.backup - -# Verify backup -ls -lh backup-sites-*.tar.gz -``` - -#### Step 1.2: Document Current Routes -```bash -# Document current running containers -docker ps | grep igny8 > current-containers.txt - -# Test current site builder access -curl -I http://localhost:8024/builder/ - -# Test current sites renderer -curl -I http://localhost:8024/ -``` - ---- - -### Phase 2: Frontend Integration - -#### Step 2.1: Copy Site Builder Components to Frontend - -**Source:** `/sites/src/builder/` -**Destination:** `/frontend/src/pages/Sites/` - -```bash -cd /data/app/igny8 - -# Create target directory -mkdir -p frontend/src/pages/Sites/Builder - -# Copy builder components -cp -r sites/src/builder/pages/* frontend/src/pages/Sites/Builder/ -cp -r sites/src/builder/components/* frontend/src/components/sites/builder/ - -# Copy shared types if needed -cp sites/src/types/index.ts frontend/src/types/siteBuilder.types.ts -``` - -**Files to Copy:** -``` -sites/src/builder/ -├── pages/ -│ ├── wizard/ -│ │ └── WizardPage.tsx → frontend/src/pages/Sites/Builder/Wizard.tsx -│ ├── preview/ -│ │ └── PreviewCanvas.tsx → frontend/src/pages/Sites/Builder/Preview.tsx -│ └── dashboard/ -│ └── SiteDashboard.tsx → frontend/src/pages/Sites/Builder/Dashboard.tsx -└── components/ - └── layout/ - └── BuilderLayout.tsx → frontend/src/components/sites/BuilderLayout.tsx -``` - ---- - -#### Step 2.2: Copy Sites Renderer to Frontend - -**Source:** `/sites/src/pages/SiteRenderer.tsx` -**Destination:** `/frontend/src/pages/Sites/` - -```bash -# Copy renderer components -cp sites/src/pages/SiteRenderer.tsx frontend/src/pages/Sites/PublicSiteRenderer.tsx -cp sites/src/loaders/loadSiteDefinition.ts frontend/src/services/siteRenderer.api.ts -cp -r sites/src/utils/* frontend/src/utils/siteRenderer/ -``` - -**Files to Copy:** -``` -sites/src/ -├── pages/ -│ └── SiteRenderer.tsx → frontend/src/pages/Sites/PublicSiteRenderer.tsx -├── loaders/ -│ └── loadSiteDefinition.ts → frontend/src/services/siteRenderer.api.ts -└── utils/ - ├── layoutRenderer.tsx → frontend/src/utils/siteRenderer/layoutRenderer.tsx - ├── pageTypeRenderer.tsx → frontend/src/utils/siteRenderer/pageTypeRenderer.tsx - └── templateEngine.tsx → frontend/src/utils/siteRenderer/templateEngine.tsx -``` - ---- - -#### Step 2.3: Update Frontend Routes - -**File:** `/frontend/src/App.tsx` - -**Add these imports:** -```tsx -// Site Builder pages -const SiteBuilderWizard = lazy(() => import("./pages/Sites/Builder/Wizard")); -const SiteBuilderPreview = lazy(() => import("./pages/Sites/Builder/Preview")); -const SiteBuilderDashboard = lazy(() => import("./pages/Sites/Builder/Dashboard")); - -// Public Sites Renderer -const PublicSiteRenderer = lazy(() => import("./pages/Sites/PublicSiteRenderer")); -``` - -**Add these routes:** -```tsx -{/* Site Builder Routes - Protected */} - - - - - - } -/> - - - - - - } -/> - - - - - - } -/> - -{/* Public Sites Renderer - No Auth Required */} -} /> -``` - -**Note:** Public renderer routes changed from `/:siteSlug/*` to `/sites/view/:siteSlug/*` to avoid route conflicts. - ---- - -#### Step 2.4: Update API Client for Sites Data - -**File:** `/frontend/src/services/siteRenderer.api.ts` - -**Update the API URL resolution:** -```typescript -function getApiBaseUrl(): string { - // Use environment variable - const envUrl = import.meta.env.VITE_API_URL || import.meta.env.VITE_BACKEND_URL; - if (envUrl) { - return envUrl.endsWith('/api') ? envUrl : `${envUrl}/api`; - } - - // Auto-detect based on current origin - if (typeof window !== 'undefined') { - const origin = window.location.origin; - - // If accessing via IP, use backend port 8011 - if (/^\d+\.\d+\.\d+\.\d+/.test(origin) || origin.includes('localhost')) { - return origin.replace(':8021', ':8011') + '/api'; - } - } - - // Production: use subdomain - return 'https://api.igny8.com/api'; -} -``` - ---- - -#### Step 2.5: Update Navigation Menu - -**File:** `/frontend/src/components/sidebar/AppSidebar.tsx` or navigation component - -**Add Site Builder menu item:** -```tsx -{ - title: "Site Builder", - icon: , - items: [ - { title: "New Blueprint", path: "/sites/builder" }, - { title: "Preview", path: "/sites/builder/preview" }, - { title: "History", path: "/sites/builder/dashboard" }, - ], -} -``` - ---- - -### Phase 3: Docker Configuration Updates - -#### Step 3.1: Update docker-compose.app.yml - -**File:** `/docker-compose.app.yml` - -**Remove the igny8_sites service:** -```yaml -# REMOVE THIS ENTIRE SERVICE: - # igny8_sites: - # image: igny8-sites-dev:latest - # container_name: igny8_sites - # restart: always - # ports: - # - "0.0.0.0:8024:5176" - # environment: - # VITE_API_URL: "https://api.igny8.com/api" - # SITES_DATA_PATH: "/sites" - # volumes: - # - /data/app/igny8/sites:/app:rw - # - /data/app/sites-data:/sites:ro - # - /data/app/igny8/frontend:/frontend:ro - # networks: [igny8_net] -``` - -**Update igny8_frontend service to access sites-data:** -```yaml - igny8_frontend: - image: igny8-frontend-dev:latest - container_name: igny8_frontend - restart: always - ports: - - "0.0.0.0:8021:5173" - environment: - VITE_BACKEND_URL: "https://api.igny8.com/api" - SITES_DATA_PATH: "/sites" # ADD THIS - volumes: - - /data/app/igny8/frontend:/app:rw - - /data/app/sites-data:/sites:ro # ADD THIS - Read-only access to sites data - depends_on: - igny8_backend: - condition: service_healthy - networks: [igny8_net] -``` - ---- - -#### Step 3.2: Update Vite Configuration - -**File:** `/frontend/vite.config.ts` - -**Add environment variable for sites data path:** -```typescript -export default defineConfig({ - // ... existing config - define: { - 'import.meta.env.SITES_DATA_PATH': JSON.stringify( - process.env.SITES_DATA_PATH || '/sites' - ), - }, - // ... rest of config -}); -``` - ---- - -### Phase 4: Backend Updates (Minor) - -#### Step 4.1: Update CORS Settings (if needed) - -**File:** `/backend/igny8_core/settings.py` - -**Verify CORS allows frontend port:** -```python -CORS_ALLOWED_ORIGINS = [ - 'http://localhost:5173', # Frontend dev server - 'http://localhost:8021', # Frontend external port - 'https://app.igny8.com', - # Remove: 'http://localhost:8024', # Old sites container -] -``` - ---- - -#### Step 4.2: Update Site Definition API (Optional Enhancement) - -**File:** `/backend/igny8_core/modules/publisher/views.py` - -**No changes required** - API already works, just being called from different container. - -**Optional:** Add CORS header for public site renderer if needed: -```python -from rest_framework.decorators import api_view, permission_classes -from rest_framework.permissions import AllowAny - -@api_view(['GET']) -@permission_classes([AllowAny]) # Public endpoint -def get_site_definition(request, site_id): - # ... existing code -``` - ---- - -### Phase 5: Testing & Verification - -#### Step 5.1: Build Updated Frontend -```bash -cd /data/app/igny8/frontend - -# Install any new dependencies (if added) -npm install - -# Build to verify no errors -npm run build - -# Rebuild Docker image -cd /data/app/igny8/frontend -docker build -t igny8-frontend-dev:latest -f Dockerfile.dev . -``` - ---- - -#### Step 5.2: Stop Old Container -```bash -# Stop and remove sites container -docker stop igny8_sites -docker rm igny8_sites - -# Verify it's stopped -docker ps | grep sites -``` - ---- - -#### Step 5.3: Start Updated Stack -```bash -cd /data/app/igny8 - -# Start with updated docker-compose -docker compose -f docker-compose.app.yml up -d igny8_frontend - -# Check logs -docker logs -f igny8_frontend -``` - ---- - -#### Step 5.4: Test Functionality - -**Test Site Builder:** -```bash -# Test builder routes (should require auth) -curl -I http://localhost:8021/sites/builder - -# Test in browser: -# http://localhost:8021/sites/builder (wizard) -# http://localhost:8021/sites/builder/preview -# http://localhost:8021/sites/builder/dashboard -``` - -**Test Sites Renderer:** -```bash -# Test public site renderer -curl -I http://localhost:8021/sites/view/test-site - -# Test in browser: -# http://localhost:8021/sites/view/{siteSlug} -# http://localhost:8021/sites/view/{siteSlug}/{pageSlug} -``` - -**Test Existing API:** -```bash -# Verify site builder API still works -curl -H "Authorization: Bearer YOUR_JWT_TOKEN" \ - http://localhost:8011/api/v1/site-builder/blueprints/ - -# Verify site definition API -curl http://localhost:8011/api/v1/publisher/sites/1/definition/ -``` - ---- - -### Phase 6: Cleanup - -#### Step 6.1: Remove Folders (After Verification) - -**Only after confirming everything works:** -```bash -cd /data/app/igny8 - -# Remove sites folder -rm -rf sites/ - -# Remove site-builder folder (already empty) -rm -rf site-builder/ - -# Remove empty frontend module -rm -rf frontend/src/modules/siteBuilder/ - -# Verify removal -ls -la | grep -E "site-builder|sites" -``` - ---- - -#### Step 6.2: Remove Docker Image (Optional) -```bash -# Remove unused sites image -docker rmi igny8-sites-dev:latest - -# Verify -docker images | grep sites -``` - ---- - -#### Step 6.3: Clean Up Build Instructions - -**File:** `/docker-compose.app.yml` (comments at top) - -**Remove this line:** -```yaml -# Remove: -# cd /data/app/igny8/sites && docker build -t igny8-sites-dev:latest -f Dockerfile.dev . -``` - ---- - -### Phase 7: Update Documentation - -#### Step 7.1: Update Files List - -**Files to update:** - -1. **README.md** - - Remove references to `/sites/` container - - Update port mapping (remove 8024) - - Update build instructions - -2. **CHANGELOG.md** - - Add entry: "Merged Sites & Site Builder into main frontend app" - -3. **docs/MASTER_REFERENCE.md** - - Update architecture diagram - - Remove igny8_sites container reference - - Update route documentation - -4. **docs/API-COMPLETE-REFERENCE.md** - - Update base URLs if needed - - Verify endpoint documentation - ---- - -#### Step 7.2: Update Architecture Diagram - -**In MASTER_REFERENCE.md, update:** - -**OLD:** -``` -┌──────────────────┐ ┌──────────────────┐ ┌──────────────────┐ -│ Frontend (8021) │ │ Sites (8024) │ │ Backend (8011) │ -└──────────────────┘ └──────────────────┘ └──────────────────┘ -``` - -**NEW:** -``` -┌──────────────────┐ ┌──────────────────┐ -│ Frontend (8021) │ │ Backend (8011) │ -│ - Main App │ │ - Django API │ -│ - Site Builder │ │ - Celery │ -│ - Site Renderer │ │ │ -└──────────────────┘ └──────────────────┘ -``` - ---- - -## Files & Folders Impact Summary - -### Files to Modify - -| File | Action | Reason | -|------|--------|--------| -| `/frontend/src/App.tsx` | **UPDATE** | Add site builder and renderer routes | -| `/frontend/src/services/siteRenderer.api.ts` | **CREATE** | Site renderer API client | -| `/frontend/src/pages/Sites/Builder/` | **CREATE** | Copy from sites/src/builder/ | -| `/frontend/src/components/sites/builder/` | **CREATE** | Builder components | -| `/frontend/vite.config.ts` | **UPDATE** | Add SITES_DATA_PATH env var | -| `/docker-compose.app.yml` | **UPDATE** | Remove igny8_sites service | -| `/backend/igny8_core/settings.py` | **VERIFY** | CORS settings (minor) | -| `/README.md` | **UPDATE** | Architecture docs | -| `/docs/MASTER_REFERENCE.md` | **UPDATE** | Remove sites container refs | - ---- - -### Files to Delete (After Verification) - -| File/Folder | Safe to Delete | Notes | -|-------------|----------------|-------| -| `/sites/` | ✅ YES | After copying to frontend | -| `/site-builder/` | ✅ YES | Already empty | -| `/frontend/src/modules/siteBuilder/` | ✅ YES | Empty folders | -| `docker-compose.app.yml.backup` | ✅ YES | After successful migration | -| `backup-sites-*.tar.gz` | ⚠️ KEEP | Keep for 30 days as rollback | - ---- - -### Backend Files - NO CHANGES NEEDED - -**These stay exactly as they are:** -``` -backend/igny8_core/ -├── modules/ -│ ├── site_builder/ ✅ KEEP - No changes -│ └── publisher/ ✅ KEEP - No changes -├── business/ -│ ├── site_building/ ✅ KEEP - No changes -│ └── publishing/ ✅ KEEP - No changes -└── api/ - └── wordpress_publishing.py ✅ KEEP - No changes -``` - -**Rationale:** Backend APIs remain unchanged; only the frontend UI container changes. - ---- - -## Rollback Plan - -If issues occur, rollback is simple: - -### Option 1: Quick Rollback (Docker) -```bash -# Restore backup docker-compose -cp docker-compose.app.yml.backup docker-compose.app.yml - -# Restart sites container -docker compose -f docker-compose.app.yml up -d igny8_sites - -# Verify -docker ps | grep sites -curl -I http://localhost:8024/ -``` - ---- - -### Option 2: Full Rollback (Restore Files) -```bash -# Extract backup -cd /data/app/igny8 -tar -xzf backup-sites-YYYYMMDD-HHMMSS.tar.gz - -# Rebuild image -cd sites -docker build -t igny8-sites-dev:latest -f Dockerfile.dev . - -# Restart -docker compose -f docker-compose.app.yml up -d igny8_sites -``` - ---- - -## Migration Checklist - -**Pre-Migration:** -- [ ] Backup `/sites/` folder -- [ ] Backup `/docker-compose.app.yml` -- [ ] Document current container ports -- [ ] Test current site builder functionality -- [ ] Test current sites renderer functionality - -**Migration Steps:** -- [ ] Copy site builder components to `/frontend/src/pages/Sites/Builder/` -- [ ] Copy sites renderer to `/frontend/src/pages/Sites/PublicSiteRenderer.tsx` -- [ ] Copy utilities to `/frontend/src/utils/siteRenderer/` -- [ ] Update `/frontend/src/App.tsx` with new routes -- [ ] Create `/frontend/src/services/siteRenderer.api.ts` -- [ ] Update `/frontend/vite.config.ts` -- [ ] Update `/docker-compose.app.yml` (remove igny8_sites, update igny8_frontend) -- [ ] Rebuild frontend Docker image -- [ ] Stop igny8_sites container -- [ ] Start updated igny8_frontend container - -**Testing:** -- [ ] Test site builder wizard at `/sites/builder` -- [ ] Test site builder preview at `/sites/builder/preview` -- [ ] Test site builder dashboard at `/sites/builder/dashboard` -- [ ] Test public site renderer at `/sites/view/{siteSlug}` -- [ ] Verify API calls work from frontend -- [ ] Check browser console for errors -- [ ] Test with authenticated user -- [ ] Test without authentication (public renderer) - -**Post-Migration:** -- [ ] Verify all tests pass -- [ ] Update documentation (README, MASTER_REFERENCE) -- [ ] Remove `/sites/` folder -- [ ] Remove `/site-builder/` folder -- [ ] Remove empty `/frontend/src/modules/siteBuilder/` -- [ ] Remove Docker image `igny8-sites-dev:latest` -- [ ] Keep backup for 30 days -- [ ] Update portable package documentation - ---- - -## FAQ - -### Q: Will this break existing site blueprints? -**A:** No. Backend database and APIs remain unchanged. Only the UI container changes. - -### Q: What about published sites in `/data/app/sites-data/`? -**A:** They remain accessible. The frontend will mount `/sites-data` and load from there. - -### Q: Do I need to update WordPress integration? -**A:** No. WordPress talks to backend API, which doesn't change. - -### Q: What if I want to keep sites as separate container? -**A:** Keep current setup. This migration is optional for simplification. - -### Q: Will public site URLs change? -**A:** Yes, from `http://localhost:8024/:siteSlug` to `http://localhost:8021/sites/view/:siteSlug`. Update any hardcoded links. - -### Q: Can I run both containers during transition? -**A:** Yes, for testing. Run sites on 8024 and updated frontend on 8021, compare functionality. - ---- - -## Support & Troubleshooting - -### Issue: Routes not working after migration -**Solution:** Check route order in `App.tsx`. Public renderer routes should be after protected routes. - -### Issue: API calls failing -**Solution:** Verify `VITE_BACKEND_URL` in environment and check CORS settings. - -### Issue: Sites data not loading -**Solution:** Verify `/data/app/sites-data` is mounted in docker-compose frontend service. - -### Issue: Build errors -**Solution:** Run `npm install` to ensure dependencies match between sites and frontend. - ---- - -## Timeline Estimate - -| Phase | Time | Complexity | -|-------|------|------------| -| Phase 1: Preparation | 30 min | Low | -| Phase 2: Frontend Integration | 2-3 hours | Medium | -| Phase 3: Docker Updates | 30 min | Low | -| Phase 4: Backend Updates | 15 min | Low | -| Phase 5: Testing | 1-2 hours | Medium | -| Phase 6: Cleanup | 15 min | Low | -| Phase 7: Documentation | 1 hour | Low | -| **Total** | **5-7 hours** | **Medium** | - ---- - -**End of Document** diff --git a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_QUICK_REFERENCE.md b/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_QUICK_REFERENCE.md deleted file mode 100644 index 189889ba..00000000 --- a/docs/future-plan-to-remove-sites-site-builder-wrong-plan/SITES_REMOVAL_QUICK_REFERENCE.md +++ /dev/null @@ -1,329 +0,0 @@ -# Sites Container Removal - Quick Reference - -**Version:** 1.0 -**Date:** November 29, 2025 - -> **Full Guide:** See [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) for complete step-by-step instructions - ---- - -## TL;DR - What Changes - -### Current Architecture (3 Containers) -``` -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ Frontend │ │ Sites │ │ Backend │ -│ Port: 8021 │ │ Port: 8024 │ │ Port: 8011 │ -│ │ │ - Site Builder │ │ │ -│ │ │ - Renderer │ │ │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ -``` - -### Target Architecture (2 Containers) -``` -┌─────────────────┐ ┌─────────────────┐ -│ Frontend │ │ Backend │ -│ Port: 8021 │ │ Port: 8011 │ -│ - Main App │ │ │ -│ - Site Builder │ │ │ -│ - Renderer │ │ │ -└─────────────────┘ └─────────────────┘ -``` - ---- - -## What Gets Removed - -❌ **Deleted:** -- `/sites/` folder (after copying to frontend) -- `/site-builder/` folder (empty, deprecated) -- `/frontend/src/modules/siteBuilder/` (empty) -- `igny8_sites` Docker container -- Port 8024 - -✅ **Kept:** -- Backend `/api/v1/site-builder/` endpoints -- Backend `/api/v1/publisher/` endpoints -- Backend modules: `site_builder/`, `publisher/` -- Backend business logic: `site_building/`, `publishing/` -- Database tables (no changes) -- `/data/app/sites-data/` directory - ---- - -## What Gets Moved - -### File Movements - -| From (Sites Container) | To (Frontend) | -|------------------------|---------------| -| `sites/src/builder/pages/wizard/` | `frontend/src/pages/Sites/Builder/Wizard.tsx` | -| `sites/src/builder/pages/preview/` | `frontend/src/pages/Sites/Builder/Preview.tsx` | -| `sites/src/builder/pages/dashboard/` | `frontend/src/pages/Sites/Builder/Dashboard.tsx` | -| `sites/src/builder/components/layout/` | `frontend/src/components/sites/BuilderLayout.tsx` | -| `sites/src/pages/SiteRenderer.tsx` | `frontend/src/pages/Sites/PublicSiteRenderer.tsx` | -| `sites/src/loaders/loadSiteDefinition.ts` | `frontend/src/services/siteRenderer.api.ts` | -| `sites/src/utils/layoutRenderer.tsx` | `frontend/src/utils/siteRenderer/layoutRenderer.tsx` | -| `sites/src/utils/pageTypeRenderer.tsx` | `frontend/src/utils/siteRenderer/pageTypeRenderer.tsx` | -| `sites/src/utils/templateEngine.tsx` | `frontend/src/utils/siteRenderer/templateEngine.tsx` | - ---- - -## Route Changes - -### Site Builder Routes - -| Old Route (Port 8024) | New Route (Port 8021) | Auth | -|----------------------|----------------------|------| -| `/builder` | `/sites/builder` | Required ✅ | -| `/builder/preview` | `/sites/builder/preview` | Required ✅ | -| `/builder/dashboard` | `/sites/builder/dashboard` | Required ✅ | - -### Sites Renderer Routes - -| Old Route (Port 8024) | New Route (Port 8021) | Auth | -|----------------------|----------------------|------| -| `/:siteSlug` | `/sites/view/:siteSlug` | None ❌ | -| `/:siteSlug/:pageSlug` | `/sites/view/:siteSlug/:pageSlug` | None ❌ | - -**⚠️ Important:** Public site URLs will change! Update any hardcoded links. - ---- - -## Files to Update - -### Critical Files (Must Update) - -1. **`/frontend/src/App.tsx`** - - Add site builder routes - - Add public renderer routes - - Add lazy imports - -2. **`/docker-compose.app.yml`** - - Remove `igny8_sites` service - - Add `/sites-data` volume mount to `igny8_frontend` - - Add `SITES_DATA_PATH` env var to `igny8_frontend` - -3. **`/frontend/vite.config.ts`** - - Add `SITES_DATA_PATH` environment variable - -### Optional Files (Verify/Update) - -4. **`/backend/igny8_core/settings.py`** - - Verify CORS settings (remove port 8024) - -5. **`/frontend/src/components/sidebar/AppSidebar.tsx`** - - Add site builder menu items - ---- - -## Critical Commands - -### 1. Backup First -```bash -cd /data/app/igny8 -tar -czf backup-sites-$(date +%Y%m%d-%H%M%S).tar.gz sites/ site-builder/ -cp docker-compose.app.yml docker-compose.app.yml.backup -``` - -### 2. Copy Files -```bash -# Create directories -mkdir -p frontend/src/pages/Sites/Builder -mkdir -p frontend/src/components/sites/builder -mkdir -p frontend/src/utils/siteRenderer - -# Copy builder pages -cp sites/src/builder/pages/wizard/WizardPage.tsx frontend/src/pages/Sites/Builder/Wizard.tsx -cp sites/src/builder/pages/preview/PreviewCanvas.tsx frontend/src/pages/Sites/Builder/Preview.tsx -cp sites/src/builder/pages/dashboard/SiteDashboard.tsx frontend/src/pages/Sites/Builder/Dashboard.tsx - -# Copy renderer -cp sites/src/pages/SiteRenderer.tsx frontend/src/pages/Sites/PublicSiteRenderer.tsx -cp sites/src/loaders/loadSiteDefinition.ts frontend/src/services/siteRenderer.api.ts - -# Copy utilities -cp -r sites/src/utils/* frontend/src/utils/siteRenderer/ -``` - -### 3. Rebuild & Restart -```bash -# Rebuild frontend -cd /data/app/igny8/frontend -docker build -t igny8-frontend-dev:latest -f Dockerfile.dev . - -# Stop sites container -docker stop igny8_sites -docker rm igny8_sites - -# Start updated frontend -cd /data/app/igny8 -docker compose -f docker-compose.app.yml up -d igny8_frontend -``` - -### 4. Test -```bash -# Test site builder (requires auth) -curl -I http://localhost:8021/sites/builder - -# Test public renderer -curl -I http://localhost:8021/sites/view/test-site - -# Check logs -docker logs -f igny8_frontend -``` - -### 5. Cleanup (After Verification) -```bash -cd /data/app/igny8 -rm -rf sites/ -rm -rf site-builder/ -rm -rf frontend/src/modules/siteBuilder/ -docker rmi igny8-sites-dev:latest -``` - ---- - -## Impact on Other Features - -### ✅ No Impact (These Still Work) - -| Feature | Why No Impact | -|---------|---------------| -| Planner module | Uses only backend API | -| Writer module | Uses only backend API | -| Linker module | Uses only backend API | -| Optimizer module | Uses only backend API | -| WordPress Publishing | Talks to backend, not affected | -| Authentication | Frontend change only | -| Billing/Credits | Backend only | -| AI Content Generation | Backend/Celery only | - -### ⚠️ Needs Update (After Migration) - -| Feature | What to Update | Impact | -|---------|----------------|--------| -| Site Builder UI | Routes change to `/sites/builder/*` | Low - Internal only | -| Public Site Links | URLs change to `/sites/view/{slug}` | Medium - External links | -| Navigation Menu | Add site builder menu items | Low - UI only | -| Tests | Update route tests if any | Low - Dev only | - ---- - -## Docker Compose Changes - -### Before (3 Services) -```yaml -services: - igny8_frontend: # Port 8021 - igny8_sites: # Port 8024 ❌ REMOVE THIS - igny8_backend: # Port 8011 - igny8_celery_worker: - igny8_celery_beat: -``` - -### After (2 Services) -```yaml -services: - igny8_frontend: # Port 8021 (updated) - # igny8_sites removed - igny8_backend: # Port 8011 - igny8_celery_worker: - igny8_celery_beat: -``` - -### Frontend Service Update -```yaml -igny8_frontend: - image: igny8-frontend-dev:latest - container_name: igny8_frontend - ports: - - "0.0.0.0:8021:5173" - environment: - VITE_BACKEND_URL: "https://api.igny8.com/api" - SITES_DATA_PATH: "/sites" # ← ADD THIS - volumes: - - /data/app/igny8/frontend:/app:rw - - /data/app/sites-data:/sites:ro # ← ADD THIS -``` - ---- - -## Testing Checklist - -**Before Migration:** -- [ ] Site builder wizard works at `http://localhost:8024/builder` -- [ ] Site preview works at `http://localhost:8024/builder/preview` -- [ ] Public site loads at `http://localhost:8024/{siteSlug}` - -**After Migration:** -- [ ] Site builder wizard works at `http://localhost:8021/sites/builder` -- [ ] Site preview works at `http://localhost:8021/sites/builder/preview` -- [ ] Blueprint dashboard works at `http://localhost:8021/sites/builder/dashboard` -- [ ] Public site loads at `http://localhost:8021/sites/view/{siteSlug}` -- [ ] Auth required for builder routes -- [ ] No auth required for public renderer -- [ ] API calls succeed from frontend -- [ ] No console errors in browser -- [ ] Sites data loads from `/data/app/sites-data/` - ---- - -## Rollback (If Needed) - -```bash -# Quick rollback -cp docker-compose.app.yml.backup docker-compose.app.yml -docker compose -f docker-compose.app.yml up -d igny8_sites - -# Full rollback -cd /data/app/igny8 -tar -xzf backup-sites-YYYYMMDD-HHMMSS.tar.gz -cd sites -docker build -t igny8-sites-dev:latest -f Dockerfile.dev . -docker compose -f docker-compose.app.yml up -d igny8_sites -``` - ---- - -## Estimation - -| Task | Time | -|------|------| -| Backup & Prep | 30 min | -| Copy Files & Update Routes | 2 hours | -| Docker Config Updates | 30 min | -| Rebuild & Deploy | 30 min | -| Testing | 1-2 hours | -| Cleanup & Docs | 1 hour | -| **Total** | **5-6 hours** | - ---- - -## Decision Matrix - -### Keep Separate Sites Container If: -- ❌ You need different deployment schedules for sites vs main app -- ❌ You want sites on different infrastructure -- ❌ You have high traffic public sites (performance isolation) -- ❌ You want to scale sites independently - -### Merge Into Frontend If: -- ✅ You want simpler deployment (fewer containers) -- ✅ You want unified routing and authentication -- ✅ Sites are internal/low traffic -- ✅ You want easier development (single app) -- ✅ You want to reduce port complexity - ---- - -## Support - -**Full Documentation:** [SITES_REMOVAL_GUIDE.md](./SITES_REMOVAL_GUIDE.md) -**Architecture Reference:** [MASTER_REFERENCE.md](./MASTER_REFERENCE.md) -**API Reference:** [API-COMPLETE-REFERENCE.md](./API-COMPLETE-REFERENCE.md) - ---- - -**Last Updated:** November 29, 2025 -**Status:** Ready for implementation diff --git a/docs/03-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md b/docs/wordpress-integration-and-other/03-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md similarity index 100% rename from docs/03-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md rename to docs/wordpress-integration-and-other/03-WORDPRESS-PLUGIN-API-INTEGRATION-GUIDE.md diff --git a/docs/04-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md b/docs/wordpress-integration-and-other/04-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md similarity index 100% rename from docs/04-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md rename to docs/wordpress-integration-and-other/04-WORDPRESS-BIDIRECTIONAL-SYNC-REFERENCE.md diff --git a/docs/DEPLOYMENT-GUIDE-WP-FIXES.md b/docs/wordpress-integration-and-other/DEPLOYMENT-GUIDE-WP-FIXES.md similarity index 100% rename from docs/DEPLOYMENT-GUIDE-WP-FIXES.md rename to docs/wordpress-integration-and-other/DEPLOYMENT-GUIDE-WP-FIXES.md diff --git a/docs/WORDPRESS-INTEGRATION-FIXES-2025-11-30.md b/docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-FIXES-2025-11-30.md similarity index 100% rename from docs/WORDPRESS-INTEGRATION-FIXES-2025-11-30.md rename to docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-FIXES-2025-11-30.md diff --git a/docs/WORDPRESS-INTEGRATION-FIXES-IMPLEMENTATION-2025-12-01.md b/docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-FIXES-IMPLEMENTATION-2025-12-01.md similarity index 100% rename from docs/WORDPRESS-INTEGRATION-FIXES-IMPLEMENTATION-2025-12-01.md rename to docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-FIXES-IMPLEMENTATION-2025-12-01.md diff --git a/docs/WORDPRESS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md b/docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md similarity index 100% rename from docs/WORDPRESS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md rename to docs/wordpress-integration-and-other/WORDPRESS-INTEGRATION-REFACTOR-PLAN-2025-12-01.md diff --git a/docs/WORDPRESS-PUBLISHING-FIELD-MAPPING.md b/docs/wordpress-integration-and-other/WORDPRESS-PUBLISHING-FIELD-MAPPING.md similarity index 100% rename from docs/WORDPRESS-PUBLISHING-FIELD-MAPPING.md rename to docs/wordpress-integration-and-other/WORDPRESS-PUBLISHING-FIELD-MAPPING.md diff --git a/docs/WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md b/docs/wordpress-integration-and-other/WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md similarity index 100% rename from docs/WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md rename to docs/wordpress-integration-and-other/WP-CONTENT-TEMPLATE-IMPLEMENTATION-PLAN.md diff --git a/docs/WP-PLUGIN-REFACTOR-PLAN.md b/docs/wordpress-integration-and-other/WP-PLUGIN-REFACTOR-PLAN.md similarity index 100% rename from docs/WP-PLUGIN-REFACTOR-PLAN.md rename to docs/wordpress-integration-and-other/WP-PLUGIN-REFACTOR-PLAN.md diff --git a/docs/WP-PUBLISHING-SIMPLE-REFERENCE.md b/docs/wordpress-integration-and-other/WP-PUBLISHING-SIMPLE-REFERENCE.md similarity index 100% rename from docs/WP-PUBLISHING-SIMPLE-REFERENCE.md rename to docs/wordpress-integration-and-other/WP-PUBLISHING-SIMPLE-REFERENCE.md diff --git a/docs/WP-REFACTOR-IMPLEMENTATION-SUMMARY.md b/docs/wordpress-integration-and-other/WP-REFACTOR-IMPLEMENTATION-SUMMARY.md similarity index 100% rename from docs/WP-REFACTOR-IMPLEMENTATION-SUMMARY.md rename to docs/wordpress-integration-and-other/WP-REFACTOR-IMPLEMENTATION-SUMMARY.md