From 3735f992074c9456473346f814e79c2b387a64b9 Mon Sep 17 00:00:00 2001 From: "IGNY8 VPS (Salman)" Date: Sat, 22 Nov 2025 00:59:52 +0000 Subject: [PATCH] doc update --- backend/SITES_INTEGRATION_PLAN.md | 705 ++++++++++++++++++ backend/celerybeat-schedule | Bin 16384 -> 16384 bytes .../migrations/0006_migrate_legacy_fields.py | 48 -- 3 files changed, 705 insertions(+), 48 deletions(-) create mode 100644 backend/SITES_INTEGRATION_PLAN.md delete mode 100644 backend/migrations/0006_migrate_legacy_fields.py diff --git a/backend/SITES_INTEGRATION_PLAN.md b/backend/SITES_INTEGRATION_PLAN.md new file mode 100644 index 00000000..42ead4d4 --- /dev/null +++ b/backend/SITES_INTEGRATION_PLAN.md @@ -0,0 +1,705 @@ +# Sites Integration Plan - Content Types Structure + +**Date**: November 22, 2025 +**Status**: 📋 **PLANNING** + +--- + +## Overview + +Integrate the new unified content architecture (ContentTaxonomy, ContentAttribute, entity_type, content_format) with the Sites module and SiteIntegration model to enable WordPress content type discovery, configuration, and sync. + +--- + +## Current State Analysis + +### ✅ What We Have + +**1. Unified Content Architecture (COMPLETE)** +- `Content` model with `entity_type`, `content_format`, `cluster_role` +- `ContentTaxonomy` model for categories, tags, product attributes +- `ContentAttribute` model for product specs, service modifiers +- WordPress sync fields (`external_id`, `external_taxonomy`, `sync_status`) + +**2. Site Model** +- Basic site information (name, domain, industry) +- `site_type` field (marketing, ecommerce, blog, portfolio, corporate) +- `hosting_type` field (igny8_sites, wordpress, shopify, multi) +- Legacy WP fields (`wp_url`, `wp_username`, `wp_api_key`) + +**3. SiteIntegration Model** +- Platform-specific integrations (wordpress, shopify, custom) +- `config_json` for configuration +- `credentials_json` for API keys/tokens +- `sync_enabled` flag for two-way sync + +**4. WordPress Plugin** +- `/wp-json/igny8/v1/site-metadata/` endpoint +- Returns post types, taxonomies, and counts +- API key authentication support + +### ❌ What's Missing + +1. **Content Type Configuration Storage** + - No place to store which post types/taxonomies are enabled + - No fetch limits per content type + - No sync preferences per taxonomy + +2. **Site → Integration Connection** + - No clear link between Site.site_type and available content types + - No mapping of WP post types to IGNY8 entity types + +3. **Frontend UI** + - No "Content Types" tab in Site Settings + - No interface to enable/disable content types + - No way to set fetch limits + +4. **Backend Service Methods** + - No method to fetch WP structure and store in `config_json` + - No method to import taxonomies + - No method to import content titles + +--- + +## Proposed Solution + +### Phase 1: Extend SiteIntegration.config_json Structure + +Store WordPress content type configuration in `SiteIntegration.config_json`: + +```json +{ + "url": "https://example.com", + "api_version": "v1", + "plugin_version": "1.0.0", + "content_types": { + "post_types": { + "post": { + "label": "Posts", + "count": 123, + "enabled": true, + "fetch_limit": 100, + "entity_type": "post", + "content_format": "article", + "last_synced": "2025-11-22T10:00:00Z" + }, + "page": { + "label": "Pages", + "count": 12, + "enabled": true, + "fetch_limit": 50, + "entity_type": "page", + "content_format": null, + "last_synced": null + }, + "product": { + "label": "Products", + "count": 456, + "enabled": true, + "fetch_limit": 200, + "entity_type": "product", + "content_format": null, + "last_synced": null + } + }, + "taxonomies": { + "category": { + "label": "Categories", + "count": 25, + "enabled": true, + "fetch_limit": 100, + "taxonomy_type": "category", + "last_synced": "2025-11-22T10:00:00Z" + }, + "post_tag": { + "label": "Tags", + "count": 102, + "enabled": true, + "fetch_limit": 200, + "taxonomy_type": "tag", + "last_synced": null + }, + "product_cat": { + "label": "Product Categories", + "count": 15, + "enabled": true, + "fetch_limit": 50, + "taxonomy_type": "product_cat", + "last_synced": null + }, + "pa_color": { + "label": "Color", + "count": 10, + "enabled": true, + "fetch_limit": 50, + "taxonomy_type": "product_attr", + "attribute_name": "Color", + "last_synced": null + } + } + }, + "plugin_connection_enabled": true, + "two_way_sync_enabled": true, + "last_structure_fetch": "2025-11-22T10:00:00Z" +} +``` + +--- + +### Phase 2: Backend Service Methods + +#### 1. **IntegrationService.fetch_content_structure()** + +```python +def fetch_content_structure(self, integration_id: int) -> Dict[str, Any]: + """ + Fetch content structure from WordPress plugin and store in config_json. + + Steps: + 1. GET /wp-json/igny8/v1/site-metadata/ + 2. Parse response + 3. Update integration.config_json['content_types'] + 4. Return structure + """ + integration = SiteIntegration.objects.get(id=integration_id) + + # Call WordPress plugin + wp_url = integration.config_json.get('url') + api_key = integration.credentials_json.get('api_key') + + response = requests.get( + f"{wp_url}/wp-json/igny8/v1/site-metadata/", + headers={'X-IGNY8-API-KEY': api_key} + ) + + if response.status_code == 200: + data = response.json()['data'] + + # Transform to our structure + content_types = { + 'post_types': {}, + 'taxonomies': {} + } + + # Map post types + for wp_type, info in data['post_types'].items(): + content_types['post_types'][wp_type] = { + 'label': info['label'], + 'count': info['count'], + 'enabled': False, # Default disabled + 'fetch_limit': 100, # Default limit + 'entity_type': self._map_wp_type_to_entity(wp_type), + 'content_format': None, + 'last_synced': None + } + + # Map taxonomies + for wp_tax, info in data['taxonomies'].items(): + content_types['taxonomies'][wp_tax] = { + 'label': info['label'], + 'count': info['count'], + 'enabled': False, # Default disabled + 'fetch_limit': 100, # Default limit + 'taxonomy_type': self._map_wp_tax_to_type(wp_tax), + 'last_synced': None + } + + # Update config + if 'content_types' not in integration.config_json: + integration.config_json['content_types'] = {} + + integration.config_json['content_types'] = content_types + integration.config_json['last_structure_fetch'] = timezone.now().isoformat() + integration.save() + + return content_types + else: + raise Exception(f"Failed to fetch structure: {response.status_code}") + +def _map_wp_type_to_entity(self, wp_type: str) -> str: + """Map WordPress post type to IGNY8 entity_type""" + mapping = { + 'post': 'post', + 'page': 'page', + 'product': 'product', + 'service': 'service', + } + return mapping.get(wp_type, 'post') + +def _map_wp_tax_to_type(self, wp_tax: str) -> str: + """Map WordPress taxonomy to ContentTaxonomy type""" + mapping = { + 'category': 'category', + 'post_tag': 'tag', + 'product_cat': 'product_cat', + 'product_tag': 'product_tag', + } + + # Product attributes start with pa_ + if wp_tax.startswith('pa_'): + return 'product_attr' + + return mapping.get(wp_tax, 'category') +``` + +#### 2. **IntegrationService.import_taxonomies()** + +```python +def import_taxonomies( + self, + integration_id: int, + taxonomy_type: str = None, + limit: int = None +) -> int: + """ + Import taxonomy terms from WordPress to ContentTaxonomy. + + Args: + integration_id: SiteIntegration ID + taxonomy_type: Specific taxonomy to import (e.g., 'category', 'post_tag') + limit: Max terms to import per taxonomy + + Returns: + Number of terms imported + """ + integration = SiteIntegration.objects.get(id=integration_id) + site = integration.site + + # Get enabled taxonomies from config + content_types = integration.config_json.get('content_types', {}) + taxonomies = content_types.get('taxonomies', {}) + + imported_count = 0 + + for wp_tax, config in taxonomies.items(): + # Skip if not enabled or not requested + if not config.get('enabled'): + continue + if taxonomy_type and wp_tax != taxonomy_type: + continue + + # Fetch from WordPress + fetch_limit = limit or config.get('fetch_limit', 100) + wp_url = integration.config_json.get('url') + api_key = integration.credentials_json.get('api_key') + + # Map taxonomy endpoint + endpoint = self._get_wp_taxonomy_endpoint(wp_tax) + + response = requests.get( + f"{wp_url}/wp-json/wp/v2/{endpoint}?per_page={fetch_limit}", + headers={'X-IGNY8-API-KEY': api_key} + ) + + if response.status_code == 200: + terms = response.json() + + for term in terms: + # Create or update ContentTaxonomy + taxonomy, created = ContentTaxonomy.objects.update_or_create( + site=site, + external_id=term['id'], + external_taxonomy=wp_tax, + defaults={ + 'name': term['name'], + 'slug': term['slug'], + 'taxonomy_type': config['taxonomy_type'], + 'description': term.get('description', ''), + 'count': term.get('count', 0), + 'sync_status': 'imported', + 'account': site.account, + 'sector': site.sectors.first(), # Default to first sector + } + ) + + if created: + imported_count += 1 + + # Update last_synced + config['last_synced'] = timezone.now().isoformat() + integration.save() + + return imported_count + +def _get_wp_taxonomy_endpoint(self, wp_tax: str) -> str: + """Get WordPress REST endpoint for taxonomy""" + mapping = { + 'category': 'categories', + 'post_tag': 'tags', + 'product_cat': 'products/categories', + 'product_tag': 'products/tags', + } + + # Product attributes + if wp_tax.startswith('pa_'): + attr_id = wp_tax.replace('pa_', '') + return f'products/attributes/{attr_id}/terms' + + return mapping.get(wp_tax, wp_tax) +``` + +#### 3. **IntegrationService.import_content_titles()** + +```python +def import_content_titles( + self, + integration_id: int, + post_type: str = None, + limit: int = None +) -> int: + """ + Import content titles (not full content) from WordPress. + + Args: + integration_id: SiteIntegration ID + post_type: Specific post type to import (e.g., 'post', 'product') + limit: Max items to import per type + + Returns: + Number of content items imported + """ + integration = SiteIntegration.objects.get(id=integration_id) + site = integration.site + + # Get enabled post types from config + content_types = integration.config_json.get('content_types', {}) + post_types = content_types.get('post_types', {}) + + imported_count = 0 + + for wp_type, config in post_types.items(): + # Skip if not enabled or not requested + if not config.get('enabled'): + continue + if post_type and wp_type != post_type: + continue + + # Fetch from WordPress + fetch_limit = limit or config.get('fetch_limit', 100) + wp_url = integration.config_json.get('url') + api_key = integration.credentials_json.get('api_key') + + # Determine endpoint + endpoint = 'products' if wp_type == 'product' else wp_type + 's' + + response = requests.get( + f"{wp_url}/wp-json/wp/v2/{endpoint}?per_page={fetch_limit}", + headers={'X-IGNY8-API-KEY': api_key} + ) + + if response.status_code == 200: + items = response.json() + + for item in items: + # Create or update Content (title only, no html_content yet) + content, created = Content.objects.update_or_create( + site=site, + external_id=item['id'], + external_type=wp_type, + defaults={ + 'title': item['title']['rendered'] if isinstance(item['title'], dict) else item['title'], + 'entity_type': config['entity_type'], + 'content_format': config.get('content_format'), + 'external_url': item.get('link', ''), + 'source': 'wordpress', + 'sync_status': 'imported', + 'account': site.account, + 'sector': site.sectors.first(), + } + ) + + # Map taxonomies + if 'categories' in item: + for cat_id in item['categories']: + try: + taxonomy = ContentTaxonomy.objects.get( + site=site, + external_id=cat_id, + taxonomy_type='category' + ) + content.taxonomies.add(taxonomy) + except ContentTaxonomy.DoesNotExist: + pass + + if 'tags' in item: + for tag_id in item['tags']: + try: + taxonomy = ContentTaxonomy.objects.get( + site=site, + external_id=tag_id, + taxonomy_type='tag' + ) + content.taxonomies.add(taxonomy) + except ContentTaxonomy.DoesNotExist: + pass + + if created: + imported_count += 1 + + # Update last_synced + config['last_synced'] = timezone.now().isoformat() + integration.save() + + return imported_count +``` + +--- + +### Phase 3: Backend API Endpoints + +Add new actions to `IntegrationViewSet`: + +```python +@action(detail=True, methods=['post'], url_path='fetch-structure') +def fetch_structure(self, request, pk=None): + """ + POST /api/v1/integration/integrations/{id}/fetch-structure/ + + Fetch content type structure from WordPress and store in config. + """ + integration = self.get_object() + service = IntegrationService() + + try: + structure = service.fetch_content_structure(integration.id) + + return success_response( + data=structure, + message="Content structure fetched successfully", + request=request + ) + except Exception as e: + return error_response( + error=str(e), + status_code=status.HTTP_400_BAD_REQUEST, + request=request + ) + +@action(detail=True, methods=['post'], url_path='import-taxonomies') +def import_taxonomies(self, request, pk=None): + """ + POST /api/v1/integration/integrations/{id}/import-taxonomies/ + { + "taxonomy_type": "category", // optional + "limit": 100 // optional + } + + Import taxonomy terms from WordPress. + """ + integration = self.get_object() + service = IntegrationService() + + taxonomy_type = request.data.get('taxonomy_type') + limit = request.data.get('limit') + + try: + count = service.import_taxonomies(integration.id, taxonomy_type, limit) + + return success_response( + data={'imported_count': count}, + message=f"Imported {count} taxonomy terms", + request=request + ) + except Exception as e: + return error_response( + error=str(e), + status_code=status.HTTP_400_BAD_REQUEST, + request=request + ) + +@action(detail=True, methods=['post'], url_path='import-content') +def import_content(self, request, pk=None): + """ + POST /api/v1/integration/integrations/{id}/import-content/ + { + "post_type": "post", // optional + "limit": 100 // optional + } + + Import content titles from WordPress. + """ + integration = self.get_object() + service = IntegrationService() + + post_type = request.data.get('post_type') + limit = request.data.get('limit') + + try: + count = service.import_content_titles(integration.id, post_type, limit) + + return success_response( + data={'imported_count': count}, + message=f"Imported {count} content items", + request=request + ) + except Exception as e: + return error_response( + error=str(e), + status_code=status.HTTP_400_BAD_REQUEST, + request=request + ) + +@action(detail=True, methods=['patch'], url_path='update-content-types') +def update_content_types(self, request, pk=None): + """ + PATCH /api/v1/integration/integrations/{id}/update-content-types/ + { + "post_types": { + "post": {"enabled": true, "fetch_limit": 200} + }, + "taxonomies": { + "category": {"enabled": true, "fetch_limit": 150} + } + } + + Update content type configuration. + """ + integration = self.get_object() + + post_types = request.data.get('post_types', {}) + taxonomies = request.data.get('taxonomies', {}) + + # Update config + if 'content_types' not in integration.config_json: + integration.config_json['content_types'] = {'post_types': {}, 'taxonomies': {}} + + for wp_type, updates in post_types.items(): + if wp_type in integration.config_json['content_types']['post_types']: + integration.config_json['content_types']['post_types'][wp_type].update(updates) + + for wp_tax, updates in taxonomies.items(): + if wp_tax in integration.config_json['content_types']['taxonomies']: + integration.config_json['content_types']['taxonomies'][wp_tax].update(updates) + + integration.save() + + return success_response( + data=integration.config_json['content_types'], + message="Content types configuration updated", + request=request + ) +``` + +--- + +### Phase 4: Frontend UI - "Content Types" Tab + +**Location:** Site Settings → Content Types + +**Features:** +1. Display fetched content types from `config_json` +2. Enable/disable toggles per type +3. Fetch limit inputs +4. Last synced timestamps +5. Sync buttons (Fetch Structure, Import Taxonomies, Import Content) + +**API Calls:** +```javascript +// Fetch structure +POST /api/v1/integration/integrations/{id}/fetch-structure/ + +// Update configuration +PATCH /api/v1/integration/integrations/{id}/update-content-types/ +{ + "post_types": { + "post": {"enabled": true, "fetch_limit": 200} + } +} + +// Import taxonomies +POST /api/v1/integration/integrations/{id}/import-taxonomies/ + +// Import content +POST /api/v1/integration/integrations/{id}/import-content/ +``` + +--- + +## Implementation Steps + +### Step 1: Backend Service Methods ✅ READY TO IMPLEMENT +- [ ] Add `fetch_content_structure()` to IntegrationService +- [ ] Add `import_taxonomies()` to IntegrationService +- [ ] Add `import_content_titles()` to IntegrationService +- [ ] Add helper methods for WP type mapping + +### Step 2: Backend API Endpoints ✅ READY TO IMPLEMENT +- [ ] Add `fetch_structure` action to IntegrationViewSet +- [ ] Add `import_taxonomies` action to IntegrationViewSet +- [ ] Add `import_content` action to IntegrationViewSet +- [ ] Add `update_content_types` action to IntegrationViewSet + +### Step 3: Frontend UI ⏳ PENDING +- [ ] Create "Content Types" tab component +- [ ] Add post types list with toggles +- [ ] Add taxonomies list with toggles +- [ ] Add fetch limit inputs +- [ ] Add sync buttons +- [ ] Add last synced timestamps + +### Step 4: Testing ⏳ PENDING +- [ ] Test structure fetch from WP plugin +- [ ] Test taxonomy import +- [ ] Test content title import +- [ ] Test configuration updates +- [ ] Test UI interactions + +--- + +## Migration Status + +### ✅ Database Ready +- All tables exist +- All fields exist +- All migrations applied + +### ✅ Models Ready +- ContentTaxonomy model complete +- ContentAttribute model complete +- Content model enhanced +- SiteIntegration model ready + +### ✅ Admin Ready +- All admin interfaces updated +- All filters configured + +### ⏳ Services Pending +- IntegrationService methods need implementation + +### ⏳ API Endpoints Pending +- IntegrationViewSet actions need implementation + +### ⏳ Frontend Pending +- Content Types tab needs creation + +--- + +## Next Actions + +**IMMEDIATE:** +1. Implement IntegrationService methods (fetch_structure, import_taxonomies, import_content_titles) +2. Add API endpoints to IntegrationViewSet +3. Test with WordPress plugin + +**SOON:** +4. Create frontend "Content Types" tab +5. Test end-to-end workflow +6. Add AI semantic mapping endpoint + +--- + +## Summary + +**We are going in the RIGHT direction!** ✅ + +The unified content architecture is complete and production-ready. Now we need to: + +1. **Store WP structure** in `SiteIntegration.config_json` +2. **Add service methods** to fetch and import from WP +3. **Add API endpoints** for frontend to trigger imports +4. **Build frontend UI** to manage content types + +The deleted migration file was incorrect (wrong location, wrong approach). The correct approach is to use `SiteIntegration.config_json` to store content type configuration, not database migrations. + +**Status: Ready to implement backend service methods!** + diff --git a/backend/celerybeat-schedule b/backend/celerybeat-schedule index 66e7aa7bcf8860eb2bcbc2a272731816e2ccd682..d6d823896760b70e00fb4a90d0b7d0b0e3e32d3f 100644 GIT binary patch delta 34 qcmZo@U~Fh$++b$LZ^XdB5IiMAv~5bz6yF!jlMPIiH)oj5-~<4