doc update

backend/SITES_INTEGRATION_PLAN.md

@@ -0,0 +1,705 @@
+# Sites Integration Plan - Content Types Structure
+
+**Date**: November 22, 2025  
+**Status**: 📋 **PLANNING**
+
+---
+
+## Overview
+
+Integrate the new unified content architecture (ContentTaxonomy, ContentAttribute, entity_type, content_format) with the Sites module and SiteIntegration model to enable WordPress content type discovery, configuration, and sync.
+
+---
+
+## Current State Analysis
+
+### ✅ What We Have
+
+**1. Unified Content Architecture (COMPLETE)**
+- `Content` model with `entity_type`, `content_format`, `cluster_role`
+- `ContentTaxonomy` model for categories, tags, product attributes
+- `ContentAttribute` model for product specs, service modifiers
+- WordPress sync fields (`external_id`, `external_taxonomy`, `sync_status`)
+
+**2. Site Model**
+- Basic site information (name, domain, industry)
+- `site_type` field (marketing, ecommerce, blog, portfolio, corporate)
+- `hosting_type` field (igny8_sites, wordpress, shopify, multi)
+- Legacy WP fields (`wp_url`, `wp_username`, `wp_api_key`)
+
+**3. SiteIntegration Model**
+- Platform-specific integrations (wordpress, shopify, custom)
+- `config_json` for configuration
+- `credentials_json` for API keys/tokens
+- `sync_enabled` flag for two-way sync
+
+**4. WordPress Plugin**
+- `/wp-json/igny8/v1/site-metadata/` endpoint
+- Returns post types, taxonomies, and counts
+- API key authentication support
+
+### ❌ What's Missing
+
+1. **Content Type Configuration Storage**
+   - No place to store which post types/taxonomies are enabled
+   - No fetch limits per content type
+   - No sync preferences per taxonomy
+
+2. **Site → Integration Connection**
+   - No clear link between Site.site_type and available content types
+   - No mapping of WP post types to IGNY8 entity types
+
+3. **Frontend UI**
+   - No "Content Types" tab in Site Settings
+   - No interface to enable/disable content types
+   - No way to set fetch limits
+
+4. **Backend Service Methods**
+   - No method to fetch WP structure and store in `config_json`
+   - No method to import taxonomies
+   - No method to import content titles
+
+---
+
+## Proposed Solution
+
+### Phase 1: Extend SiteIntegration.config_json Structure
+
+Store WordPress content type configuration in `SiteIntegration.config_json`:
+
+```json
+{
+  "url": "https://example.com",
+  "api_version": "v1",
+  "plugin_version": "1.0.0",
+  "content_types": {
+    "post_types": {
+      "post": {
+        "label": "Posts",
+        "count": 123,
+        "enabled": true,
+        "fetch_limit": 100,
+        "entity_type": "post",
+        "content_format": "article",
+        "last_synced": "2025-11-22T10:00:00Z"
+      },
+      "page": {
+        "label": "Pages",
+        "count": 12,
+        "enabled": true,
+        "fetch_limit": 50,
+        "entity_type": "page",
+        "content_format": null,
+        "last_synced": null
+      },
+      "product": {
+        "label": "Products",
+        "count": 456,
+        "enabled": true,
+        "fetch_limit": 200,
+        "entity_type": "product",
+        "content_format": null,
+        "last_synced": null
+      }
+    },
+    "taxonomies": {
+      "category": {
+        "label": "Categories",
+        "count": 25,
+        "enabled": true,
+        "fetch_limit": 100,
+        "taxonomy_type": "category",
+        "last_synced": "2025-11-22T10:00:00Z"
+      },
+      "post_tag": {
+        "label": "Tags",
+        "count": 102,
+        "enabled": true,
+        "fetch_limit": 200,
+        "taxonomy_type": "tag",
+        "last_synced": null
+      },
+      "product_cat": {
+        "label": "Product Categories",
+        "count": 15,
+        "enabled": true,
+        "fetch_limit": 50,
+        "taxonomy_type": "product_cat",
+        "last_synced": null
+      },
+      "pa_color": {
+        "label": "Color",
+        "count": 10,
+        "enabled": true,
+        "fetch_limit": 50,
+        "taxonomy_type": "product_attr",
+        "attribute_name": "Color",
+        "last_synced": null
+      }
+    }
+  },
+  "plugin_connection_enabled": true,
+  "two_way_sync_enabled": true,
+  "last_structure_fetch": "2025-11-22T10:00:00Z"
+}
+```
+
+---
+
+### Phase 2: Backend Service Methods
+
+#### 1. **IntegrationService.fetch_content_structure()**
+
+```python
+def fetch_content_structure(self, integration_id: int) -> Dict[str, Any]:
+    """
+    Fetch content structure from WordPress plugin and store in config_json.
+    
+    Steps:
+    1. GET /wp-json/igny8/v1/site-metadata/
+    2. Parse response
+    3. Update integration.config_json['content_types']
+    4. Return structure
+    """
+    integration = SiteIntegration.objects.get(id=integration_id)
+    
+    # Call WordPress plugin
+    wp_url = integration.config_json.get('url')
+    api_key = integration.credentials_json.get('api_key')
+    
+    response = requests.get(
+        f"{wp_url}/wp-json/igny8/v1/site-metadata/",
+        headers={'X-IGNY8-API-KEY': api_key}
+    )
+    
+    if response.status_code == 200:
+        data = response.json()['data']
+        
+        # Transform to our structure
+        content_types = {
+            'post_types': {},
+            'taxonomies': {}
+        }
+        
+        # Map post types
+        for wp_type, info in data['post_types'].items():
+            content_types['post_types'][wp_type] = {
+                'label': info['label'],
+                'count': info['count'],
+                'enabled': False,  # Default disabled
+                'fetch_limit': 100,  # Default limit
+                'entity_type': self._map_wp_type_to_entity(wp_type),
+                'content_format': None,
+                'last_synced': None
+            }
+        
+        # Map taxonomies
+        for wp_tax, info in data['taxonomies'].items():
+            content_types['taxonomies'][wp_tax] = {
+                'label': info['label'],
+                'count': info['count'],
+                'enabled': False,  # Default disabled
+                'fetch_limit': 100,  # Default limit
+                'taxonomy_type': self._map_wp_tax_to_type(wp_tax),
+                'last_synced': None
+            }
+        
+        # Update config
+        if 'content_types' not in integration.config_json:
+            integration.config_json['content_types'] = {}
+        
+        integration.config_json['content_types'] = content_types
+        integration.config_json['last_structure_fetch'] = timezone.now().isoformat()
+        integration.save()
+        
+        return content_types
+    else:
+        raise Exception(f"Failed to fetch structure: {response.status_code}")
+
+def _map_wp_type_to_entity(self, wp_type: str) -> str:
+    """Map WordPress post type to IGNY8 entity_type"""
+    mapping = {
+        'post': 'post',
+        'page': 'page',
+        'product': 'product',
+        'service': 'service',
+    }
+    return mapping.get(wp_type, 'post')
+
+def _map_wp_tax_to_type(self, wp_tax: str) -> str:
+    """Map WordPress taxonomy to ContentTaxonomy type"""
+    mapping = {
+        'category': 'category',
+        'post_tag': 'tag',
+        'product_cat': 'product_cat',
+        'product_tag': 'product_tag',
+    }
+    
+    # Product attributes start with pa_
+    if wp_tax.startswith('pa_'):
+        return 'product_attr'
+    
+    return mapping.get(wp_tax, 'category')
+```
+
+#### 2. **IntegrationService.import_taxonomies()**
+
+```python
+def import_taxonomies(
+    self, 
+    integration_id: int, 
+    taxonomy_type: str = None,
+    limit: int = None
+) -> int:
+    """
+    Import taxonomy terms from WordPress to ContentTaxonomy.
+    
+    Args:
+        integration_id: SiteIntegration ID
+        taxonomy_type: Specific taxonomy to import (e.g., 'category', 'post_tag')
+        limit: Max terms to import per taxonomy
+    
+    Returns:
+        Number of terms imported
+    """
+    integration = SiteIntegration.objects.get(id=integration_id)
+    site = integration.site
+    
+    # Get enabled taxonomies from config
+    content_types = integration.config_json.get('content_types', {})
+    taxonomies = content_types.get('taxonomies', {})
+    
+    imported_count = 0
+    
+    for wp_tax, config in taxonomies.items():
+        # Skip if not enabled or not requested
+        if not config.get('enabled'):
+            continue
+        if taxonomy_type and wp_tax != taxonomy_type:
+            continue
+        
+        # Fetch from WordPress
+        fetch_limit = limit or config.get('fetch_limit', 100)
+        wp_url = integration.config_json.get('url')
+        api_key = integration.credentials_json.get('api_key')
+        
+        # Map taxonomy endpoint
+        endpoint = self._get_wp_taxonomy_endpoint(wp_tax)
+        
+        response = requests.get(
+            f"{wp_url}/wp-json/wp/v2/{endpoint}?per_page={fetch_limit}",
+            headers={'X-IGNY8-API-KEY': api_key}
+        )
+        
+        if response.status_code == 200:
+            terms = response.json()
+            
+            for term in terms:
+                # Create or update ContentTaxonomy
+                taxonomy, created = ContentTaxonomy.objects.update_or_create(
+                    site=site,
+                    external_id=term['id'],
+                    external_taxonomy=wp_tax,
+                    defaults={
+                        'name': term['name'],
+                        'slug': term['slug'],
+                        'taxonomy_type': config['taxonomy_type'],
+                        'description': term.get('description', ''),
+                        'count': term.get('count', 0),
+                        'sync_status': 'imported',
+                        'account': site.account,
+                        'sector': site.sectors.first(),  # Default to first sector
+                    }
+                )
+                
+                if created:
+                    imported_count += 1
+            
+            # Update last_synced
+            config['last_synced'] = timezone.now().isoformat()
+            integration.save()
+    
+    return imported_count
+
+def _get_wp_taxonomy_endpoint(self, wp_tax: str) -> str:
+    """Get WordPress REST endpoint for taxonomy"""
+    mapping = {
+        'category': 'categories',
+        'post_tag': 'tags',
+        'product_cat': 'products/categories',
+        'product_tag': 'products/tags',
+    }
+    
+    # Product attributes
+    if wp_tax.startswith('pa_'):
+        attr_id = wp_tax.replace('pa_', '')
+        return f'products/attributes/{attr_id}/terms'
+    
+    return mapping.get(wp_tax, wp_tax)
+```
+
+#### 3. **IntegrationService.import_content_titles()**
+
+```python
+def import_content_titles(
+    self,
+    integration_id: int,
+    post_type: str = None,
+    limit: int = None
+) -> int:
+    """
+    Import content titles (not full content) from WordPress.
+    
+    Args:
+        integration_id: SiteIntegration ID
+        post_type: Specific post type to import (e.g., 'post', 'product')
+        limit: Max items to import per type
+    
+    Returns:
+        Number of content items imported
+    """
+    integration = SiteIntegration.objects.get(id=integration_id)
+    site = integration.site
+    
+    # Get enabled post types from config
+    content_types = integration.config_json.get('content_types', {})
+    post_types = content_types.get('post_types', {})
+    
+    imported_count = 0
+    
+    for wp_type, config in post_types.items():
+        # Skip if not enabled or not requested
+        if not config.get('enabled'):
+            continue
+        if post_type and wp_type != post_type:
+            continue
+        
+        # Fetch from WordPress
+        fetch_limit = limit or config.get('fetch_limit', 100)
+        wp_url = integration.config_json.get('url')
+        api_key = integration.credentials_json.get('api_key')
+        
+        # Determine endpoint
+        endpoint = 'products' if wp_type == 'product' else wp_type + 's'
+        
+        response = requests.get(
+            f"{wp_url}/wp-json/wp/v2/{endpoint}?per_page={fetch_limit}",
+            headers={'X-IGNY8-API-KEY': api_key}
+        )
+        
+        if response.status_code == 200:
+            items = response.json()
+            
+            for item in items:
+                # Create or update Content (title only, no html_content yet)
+                content, created = Content.objects.update_or_create(
+                    site=site,
+                    external_id=item['id'],
+                    external_type=wp_type,
+                    defaults={
+                        'title': item['title']['rendered'] if isinstance(item['title'], dict) else item['title'],
+                        'entity_type': config['entity_type'],
+                        'content_format': config.get('content_format'),
+                        'external_url': item.get('link', ''),
+                        'source': 'wordpress',
+                        'sync_status': 'imported',
+                        'account': site.account,
+                        'sector': site.sectors.first(),
+                    }
+                )
+                
+                # Map taxonomies
+                if 'categories' in item:
+                    for cat_id in item['categories']:
+                        try:
+                            taxonomy = ContentTaxonomy.objects.get(
+                                site=site,
+                                external_id=cat_id,
+                                taxonomy_type='category'
+                            )
+                            content.taxonomies.add(taxonomy)
+                        except ContentTaxonomy.DoesNotExist:
+                            pass
+                
+                if 'tags' in item:
+                    for tag_id in item['tags']:
+                        try:
+                            taxonomy = ContentTaxonomy.objects.get(
+                                site=site,
+                                external_id=tag_id,
+                                taxonomy_type='tag'
+                            )
+                            content.taxonomies.add(taxonomy)
+                        except ContentTaxonomy.DoesNotExist:
+                            pass
+                
+                if created:
+                    imported_count += 1
+            
+            # Update last_synced
+            config['last_synced'] = timezone.now().isoformat()
+            integration.save()
+    
+    return imported_count
+```
+
+---
+
+### Phase 3: Backend API Endpoints
+
+Add new actions to `IntegrationViewSet`:
+
+```python
+@action(detail=True, methods=['post'], url_path='fetch-structure')
+def fetch_structure(self, request, pk=None):
+    """
+    POST /api/v1/integration/integrations/{id}/fetch-structure/
+    
+    Fetch content type structure from WordPress and store in config.
+    """
+    integration = self.get_object()
+    service = IntegrationService()
+    
+    try:
+        structure = service.fetch_content_structure(integration.id)
+        
+        return success_response(
+            data=structure,
+            message="Content structure fetched successfully",
+            request=request
+        )
+    except Exception as e:
+        return error_response(
+            error=str(e),
+            status_code=status.HTTP_400_BAD_REQUEST,
+            request=request
+        )
+
+@action(detail=True, methods=['post'], url_path='import-taxonomies')
+def import_taxonomies(self, request, pk=None):
+    """
+    POST /api/v1/integration/integrations/{id}/import-taxonomies/
+    {
+      "taxonomy_type": "category",  // optional
+      "limit": 100  // optional
+    }
+    
+    Import taxonomy terms from WordPress.
+    """
+    integration = self.get_object()
+    service = IntegrationService()
+    
+    taxonomy_type = request.data.get('taxonomy_type')
+    limit = request.data.get('limit')
+    
+    try:
+        count = service.import_taxonomies(integration.id, taxonomy_type, limit)
+        
+        return success_response(
+            data={'imported_count': count},
+            message=f"Imported {count} taxonomy terms",
+            request=request
+        )
+    except Exception as e:
+        return error_response(
+            error=str(e),
+            status_code=status.HTTP_400_BAD_REQUEST,
+            request=request
+        )
+
+@action(detail=True, methods=['post'], url_path='import-content')
+def import_content(self, request, pk=None):
+    """
+    POST /api/v1/integration/integrations/{id}/import-content/
+    {
+      "post_type": "post",  // optional
+      "limit": 100  // optional
+    }
+    
+    Import content titles from WordPress.
+    """
+    integration = self.get_object()
+    service = IntegrationService()
+    
+    post_type = request.data.get('post_type')
+    limit = request.data.get('limit')
+    
+    try:
+        count = service.import_content_titles(integration.id, post_type, limit)
+        
+        return success_response(
+            data={'imported_count': count},
+            message=f"Imported {count} content items",
+            request=request
+        )
+    except Exception as e:
+        return error_response(
+            error=str(e),
+            status_code=status.HTTP_400_BAD_REQUEST,
+            request=request
+        )
+
+@action(detail=True, methods=['patch'], url_path='update-content-types')
+def update_content_types(self, request, pk=None):
+    """
+    PATCH /api/v1/integration/integrations/{id}/update-content-types/
+    {
+      "post_types": {
+        "post": {"enabled": true, "fetch_limit": 200}
+      },
+      "taxonomies": {
+        "category": {"enabled": true, "fetch_limit": 150}
+      }
+    }
+    
+    Update content type configuration.
+    """
+    integration = self.get_object()
+    
+    post_types = request.data.get('post_types', {})
+    taxonomies = request.data.get('taxonomies', {})
+    
+    # Update config
+    if 'content_types' not in integration.config_json:
+        integration.config_json['content_types'] = {'post_types': {}, 'taxonomies': {}}
+    
+    for wp_type, updates in post_types.items():
+        if wp_type in integration.config_json['content_types']['post_types']:
+            integration.config_json['content_types']['post_types'][wp_type].update(updates)
+    
+    for wp_tax, updates in taxonomies.items():
+        if wp_tax in integration.config_json['content_types']['taxonomies']:
+            integration.config_json['content_types']['taxonomies'][wp_tax].update(updates)
+    
+    integration.save()
+    
+    return success_response(
+        data=integration.config_json['content_types'],
+        message="Content types configuration updated",
+        request=request
+    )
+```
+
+---
+
+### Phase 4: Frontend UI - "Content Types" Tab
+
+**Location:** Site Settings → Content Types
+
+**Features:**
+1. Display fetched content types from `config_json`
+2. Enable/disable toggles per type
+3. Fetch limit inputs
+4. Last synced timestamps
+5. Sync buttons (Fetch Structure, Import Taxonomies, Import Content)
+
+**API Calls:**
+```javascript
+// Fetch structure
+POST /api/v1/integration/integrations/{id}/fetch-structure/
+
+// Update configuration
+PATCH /api/v1/integration/integrations/{id}/update-content-types/
+{
+  "post_types": {
+    "post": {"enabled": true, "fetch_limit": 200}
+  }
+}
+
+// Import taxonomies
+POST /api/v1/integration/integrations/{id}/import-taxonomies/
+
+// Import content
+POST /api/v1/integration/integrations/{id}/import-content/
+```
+
+---
+
+## Implementation Steps
+
+### Step 1: Backend Service Methods ✅ READY TO IMPLEMENT
+- [ ] Add `fetch_content_structure()` to IntegrationService
+- [ ] Add `import_taxonomies()` to IntegrationService
+- [ ] Add `import_content_titles()` to IntegrationService
+- [ ] Add helper methods for WP type mapping
+
+### Step 2: Backend API Endpoints ✅ READY TO IMPLEMENT
+- [ ] Add `fetch_structure` action to IntegrationViewSet
+- [ ] Add `import_taxonomies` action to IntegrationViewSet
+- [ ] Add `import_content` action to IntegrationViewSet
+- [ ] Add `update_content_types` action to IntegrationViewSet
+
+### Step 3: Frontend UI ⏳ PENDING
+- [ ] Create "Content Types" tab component
+- [ ] Add post types list with toggles
+- [ ] Add taxonomies list with toggles
+- [ ] Add fetch limit inputs
+- [ ] Add sync buttons
+- [ ] Add last synced timestamps
+
+### Step 4: Testing ⏳ PENDING
+- [ ] Test structure fetch from WP plugin
+- [ ] Test taxonomy import
+- [ ] Test content title import
+- [ ] Test configuration updates
+- [ ] Test UI interactions
+
+---
+
+## Migration Status
+
+### ✅ Database Ready
+- All tables exist
+- All fields exist
+- All migrations applied
+
+### ✅ Models Ready
+- ContentTaxonomy model complete
+- ContentAttribute model complete
+- Content model enhanced
+- SiteIntegration model ready
+
+### ✅ Admin Ready
+- All admin interfaces updated
+- All filters configured
+
+### ⏳ Services Pending
+- IntegrationService methods need implementation
+
+### ⏳ API Endpoints Pending
+- IntegrationViewSet actions need implementation
+
+### ⏳ Frontend Pending
+- Content Types tab needs creation
+
+---
+
+## Next Actions
+
+**IMMEDIATE:**
+1. Implement IntegrationService methods (fetch_structure, import_taxonomies, import_content_titles)
+2. Add API endpoints to IntegrationViewSet
+3. Test with WordPress plugin
+
+**SOON:**
+4. Create frontend "Content Types" tab
+5. Test end-to-end workflow
+6. Add AI semantic mapping endpoint
+
+---
+
+## Summary
+
+**We are going in the RIGHT direction!** ✅
+
+The unified content architecture is complete and production-ready. Now we need to:
+
+1. **Store WP structure** in `SiteIntegration.config_json`
+2. **Add service methods** to fetch and import from WP
+3. **Add API endpoints** for frontend to trigger imports
+4. **Build frontend UI** to manage content types
+
+The deleted migration file was incorrect (wrong location, wrong approach). The correct approach is to use `SiteIntegration.config_json` to store content type configuration, not database migrations.
+
+**Status: Ready to implement backend service methods!**
+

backend/migrations/0006_migrate_legacy_fields.py

@@ -1,48 +0,0 @@
-from django.db import migrations
-
-def migrate_legacy_fields(apps, schema_editor):
-    Content = apps.get_model('igny8_core', 'Content')
-    Task = apps.get_model('igny8_core', 'Task')
-
-    # Migrate legacy fields in Content model
-    for content in Content.objects.all():
-        if content.categories:
-            # Convert JSON categories to ContentTaxonomy
-            categories = content.categories
-            for category in categories:
-                taxonomy, created = ContentTaxonomy.objects.get_or_create(
-                    name=category['name'],
-                    slug=category['slug'],
-                    taxonomy_type='category'
-                )
-                content.taxonomies.add(taxonomy)
-
-        if content.tags:
-            # Convert JSON tags to ContentTaxonomy
-            tags = content.tags
-            for tag in tags:
-                taxonomy, created = ContentTaxonomy.objects.get_or_create(
-                    name=tag['name'],
-                    slug=tag['slug'],
-                    taxonomy_type='tag'
-                )
-                content.taxonomies.add(taxonomy)
-
-        content.save()
-
-    # Migrate legacy fields in Task model
-    for task in Task.objects.all():
-        task.entity_type = task.content.entity_type
-        task.cluster_role = task.content.cluster_role
-        task.cluster_id = task.content.cluster_id
-        task.save()
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('igny8_core', '0005_phase3_mark_deprecated_fields'),
-    ]
-
-    operations = [
-        migrations.RunPython(migrate_legacy_fields),
-    ]