v2-exece-docs

This commit is contained in:
IGNY8 VPS (Salman)
2026-03-23 10:30:51 +00:00
parent b94d41b7f6
commit e78a41f11c
15 changed files with 2218 additions and 707 deletions

View File

@@ -1,4 +1,9 @@
# 01E: Blueprint-Aware Content Pipeline
> **Version:** 1.1 (codebase-verified)
> **Source of Truth:** Codebase at `/data/app/igny8/backend/`
> **Last Verified:** 2025-07-14
**IGNY8 Phase 1: Content Automation with SAG Blueprint Enhancement**
---
@@ -150,15 +155,15 @@ ELSE:
2. `blueprint_context` structure:
```json
{
"cluster_id": "uuid",
"cluster_id": "integer",
"cluster_name": "string",
"cluster_type": "string (topical|product|service)",
"cluster_sector": "string",
"hub_title": "string (cluster's main hub page title)",
"hub_url": "string (blueprint.site.domain/cluster_slug)",
"cluster_attributes": ["list of attribute terms"],
"related_clusters": ["list of related cluster ids"],
"cluster_products": ["list of product ids if product cluster"],
"related_clusters": ["list of related cluster integer ids"],
"cluster_products": ["list of product integer ids if product cluster"],
"content_structure": "string (guide_tutorial|comparison|review|how_to|question|listicle)",
"content_type": "string (cluster_hub|blog_post|product_page|term_page|service_page)",
"execution_phase": "integer (1-4)",
@@ -287,10 +292,13 @@ execution_priority = {
### Related Models (from 01A, 01C, 01D)
```python
# sag/models.py — SAG Blueprint Structure
# igny8_core/sag/models.py — SAG Blueprint Structure
# DEFAULT_AUTO_FIELD = BigAutoField (integer PKs)
class SAGBlueprint(models.Model):
site = ForeignKey(Site)
from igny8_core.auth.models import AccountBaseModel
class SAGBlueprint(AccountBaseModel):
site = ForeignKey('igny8_core_auth.Site', on_delete=models.CASCADE)
name = CharField(max_length=255)
status = CharField(choices=['draft', 'active', 'archived'])
created_at = DateTimeField(auto_now_add=True)
@@ -303,8 +311,8 @@ class SAGBlueprint(models.Model):
# Taxonomy mapping to WordPress custom taxonomies
wp_taxonomy_mapping = JSONField() # cluster_id → tax values
class SAGCluster(models.Model):
blueprint = ForeignKey(SAGBlueprint)
class SAGCluster(AccountBaseModel):
blueprint = ForeignKey('sag.SAGBlueprint', on_delete=models.CASCADE)
name = CharField(max_length=255)
cluster_type = CharField(choices=['topical', 'product', 'service'])
sector = CharField(max_length=255)
@@ -314,69 +322,135 @@ class SAGCluster(models.Model):
updated_at = DateTimeField(auto_now=True)
```
### Pipeline Models (existing)
### Pipeline Models (existing — names are PLURAL per codebase convention)
```python
# content/models.py — Content Pipeline
# igny8_core/business/planning/models.py — Planning Pipeline (app_label: planner)
# DEFAULT_AUTO_FIELD = BigAutoField (integer PKs, NOT UUIDs)
class Keyword(models.Model):
site = ForeignKey(Site)
term = CharField(max_length=255)
source = CharField(choices=['csv_import', 'seed_list', 'user', 'sag_blueprint'])
sag_cluster_id = UUIDField(null=True, blank=True) # NEW: links to blueprint cluster
class Keywords(SoftDeletableModel, SiteSectorBaseModel):
"""Site-specific keyword instances referencing global SeedKeywords."""
seed_keyword = ForeignKey(SeedKeyword, on_delete=models.CASCADE)
volume_override = IntegerField(null=True, blank=True)
difficulty_override = IntegerField(null=True, blank=True)
attribute_values = JSONField(default=list, blank=True)
cluster = ForeignKey('Clusters', on_delete=models.SET_NULL, null=True, blank=True)
status = CharField(max_length=50, choices=[('new','New'),('mapped','Mapped')], default='new')
disabled = BooleanField(default=False)
# NEW: optional SAG cluster link
sag_cluster_id = IntegerField(null=True, blank=True) # Links to sag.SAGCluster PK
created_at = DateTimeField(auto_now_add=True)
class Meta:
app_label = 'planner'
class Cluster(models.Model):
site = ForeignKey(Site)
name = CharField(max_length=255)
keywords = JSONField(default=list)
created_by = CharField(choices=['auto_cluster', 'sag_blueprint'])
class Clusters(SoftDeletableModel, SiteSectorBaseModel):
"""Keyword clusters — pure topic clusters."""
name = CharField(max_length=255, db_index=True)
description = TextField(blank=True, null=True)
keywords_count = IntegerField(default=0)
volume = IntegerField(default=0)
mapped_pages = IntegerField(default=0)
status = CharField(max_length=50, choices=[('new','New'),('mapped','Mapped')], default='new')
disabled = BooleanField(default=False)
created_at = DateTimeField(auto_now_add=True)
updated_at = DateTimeField(auto_now=True)
class Meta:
app_label = 'planner'
class Idea(models.Model):
site = ForeignKey(Site)
title = CharField(max_length=255)
keyword = ForeignKey(Keyword)
cluster = ForeignKey(Cluster, null=True)
sector = CharField(max_length=255) # NEW
structure = CharField(choices=['guide_tutorial', 'comparison', 'review', 'how_to', 'question', 'listicle']) # NEW
content_type = CharField(choices=['cluster_hub', 'blog_post', 'product_page', 'term_page', 'service_page', 'landing_page', 'business_page']) # NEW
sag_cluster_id = UUIDField(null=True, blank=True) # NEW
idea_source = CharField(choices=['auto_generate', 'sag_blueprint']) # NEW
class ContentIdeas(SoftDeletableModel, SiteSectorBaseModel):
"""Content ideas generated from keyword clusters."""
idea_title = CharField(max_length=255, db_index=True)
description = TextField(blank=True, null=True)
primary_focus_keywords = CharField(max_length=500, blank=True)
target_keywords = CharField(max_length=500, blank=True)
keyword_objects = ManyToManyField('Keywords', blank=True, related_name='content_ideas')
keyword_cluster = ForeignKey('Clusters', on_delete=models.SET_NULL, null=True, blank=True)
status = CharField(max_length=50, choices=[('new','New'),('queued','Queued'),('completed','Completed')], default='new')
disabled = BooleanField(default=False)
estimated_word_count = IntegerField(default=1000)
content_type = CharField(max_length=50, choices=[('post','Post'),('page','Page'),('product','Product'),('taxonomy','Taxonomy')], default='post')
content_structure = CharField(max_length=50, choices=[
('article','Article'),('guide','Guide'),('comparison','Comparison'),
('review','Review'),('listicle','Listicle'),('landing_page','Landing Page'),
('business_page','Business Page'),('service_page','Service Page'),
('general','General'),('cluster_hub','Cluster Hub'),('product_page','Product Page'),
('category_archive','Category Archive'),('tag_archive','Tag Archive'),
('attribute_archive','Attribute Archive'),
], default='article')
# NEW: SAG fields
sag_cluster_id = IntegerField(null=True, blank=True) # Links to sag.SAGCluster PK
idea_source = CharField(choices=['auto_generate', 'sag_blueprint'], null=True, blank=True) # NEW
execution_phase = IntegerField(null=True) # NEW: 1-4 from blueprint
created_at = DateTimeField(auto_now_add=True)
class Meta:
app_label = 'planner'
class Task(models.Model):
site = ForeignKey(Site)
title = CharField(max_length=255)
idea = ForeignKey(Idea)
status = CharField(choices=['pending', 'assigned', 'in_progress', 'review', 'completed'])
assigned_to = ForeignKey(User, null=True)
sag_cluster_id = UUIDField(null=True, blank=True) # NEW
# igny8_core/business/content/models.py — Content Pipeline (app_label: writer)
class Tasks(SoftDeletableModel, SiteSectorBaseModel):
"""Tasks model for content generation queue."""
title = CharField(max_length=255, db_index=True)
description = TextField(blank=True, null=True)
cluster = ForeignKey('planner.Clusters', on_delete=models.SET_NULL, null=True, blank=False)
idea = ForeignKey('planner.ContentIdeas', on_delete=models.SET_NULL, null=True, blank=True)
content_type = CharField(max_length=100, choices=[('post','Post'),('page','Page'),('product','Product'),('taxonomy','Taxonomy')], default='post')
content_structure = CharField(max_length=100, choices=[...same as ContentIdeas...], default='article')
taxonomy_term = ForeignKey('ContentTaxonomy', on_delete=models.SET_NULL, null=True, blank=True)
keywords = TextField(blank=True, null=True, help_text='Comma-separated keywords')
word_count = IntegerField(default=1000)
status = CharField(max_length=50, choices=[('queued','Queued'),('completed','Completed')], default='queued')
# NEW: SAG fields
sag_cluster_id = IntegerField(null=True, blank=True) # Links to sag.SAGCluster PK
blueprint_context = JSONField(null=True, blank=True) # NEW: execution context
created_at = DateTimeField(auto_now_add=True)
updated_at = DateTimeField(auto_now=True)
class Meta:
app_label = 'writer'
class Content(models.Model):
site = ForeignKey(Site)
title = CharField(max_length=255)
body = TextField()
task = ForeignKey(Task, null=True)
content_type = CharField(choices=['cluster_hub', 'blog_post', 'product_page', 'term_page', 'service_page', 'landing_page', 'business_page']) # NEW
content_structure = CharField(choices=['guide_tutorial', 'comparison', 'review', 'how_to', 'question', 'listicle']) # NEW
sag_cluster_id = UUIDField(null=True, blank=True) # NEW
taxonomies = JSONField(default=dict, null=True, blank=True) # NEW: custom WP taxonomies
status = CharField(choices=['draft', 'review', 'published'])
class Content(SoftDeletableModel, SiteSectorBaseModel):
"""Content model for AI-generated or WordPress-imported content."""
title = CharField(max_length=255, db_index=True)
content_html = TextField(help_text='Final HTML content') # NOTE: field is content_html, NOT body
word_count = IntegerField(default=0)
meta_title = CharField(max_length=255, blank=True, null=True)
meta_description = TextField(blank=True, null=True)
primary_keyword = CharField(max_length=255, blank=True, null=True)
secondary_keywords = JSONField(default=list, blank=True)
cluster = ForeignKey('planner.Clusters', on_delete=models.SET_NULL, null=True, blank=False)
content_type = CharField(max_length=50, choices=[('post','Post'),('page','Page'),('product','Product'),('taxonomy','Taxonomy')], default='post')
content_structure = CharField(max_length=50, choices=[...same as Tasks...], default='article')
taxonomy_terms = ManyToManyField('ContentTaxonomy', through='ContentTaxonomyRelation', blank=True)
external_id = CharField(max_length=255, blank=True, null=True)
external_url = URLField(blank=True, null=True)
source = CharField(max_length=50, choices=[('igny8','IGNY8 Generated'),('wordpress','WordPress Imported')], default='igny8')
status = CharField(max_length=50, choices=[('draft','Draft'),('review','Review'),('approved','Approved'),('published','Published')], default='draft')
# NEW: SAG fields
sag_cluster_id = IntegerField(null=True, blank=True) # Links to sag.SAGCluster PK
created_at = DateTimeField(auto_now_add=True)
updated_at = DateTimeField(auto_now=True)
class Meta:
app_label = 'writer'
class Image(models.Model):
content = ForeignKey(Content)
url = URLField()
alt_text = CharField(max_length=255)
style_type = CharField(choices=['hero', 'supporting', 'ecommerce', 'category', 'service', 'conversion']) # NEW
sag_cluster_id = UUIDField(null=True, blank=True) # NEW
class Images(SoftDeletableModel, SiteSectorBaseModel):
"""Images model — note: class is Images (plural)."""
content = ForeignKey(Content, on_delete=models.CASCADE, null=True, blank=True)
task = ForeignKey(Tasks, on_delete=models.CASCADE, null=True, blank=True)
image_type = CharField(max_length=50, choices=[('featured','Featured'),('desktop','Desktop'),('mobile','Mobile'),('in_article','In-Article')], default='featured')
image_url = CharField(max_length=500, blank=True, null=True) # NOTE: field is image_url, NOT url
image_path = CharField(max_length=500, blank=True, null=True)
prompt = TextField(blank=True, null=True) # Generation prompt
caption = TextField(blank=True, null=True) # NOTE: field is caption, NOT alt_text
status = CharField(max_length=50, default='pending')
position = IntegerField(default=0)
# NEW: SAG fields
sag_cluster_id = IntegerField(null=True, blank=True) # Links to sag.SAGCluster PK
style_type = CharField(max_length=50, choices=[('hero','Hero'),('supporting','Supporting'),('ecommerce','Ecommerce'),('category','Category'),('service','Service'),('conversion','Conversion')], null=True, blank=True) # NEW
created_at = DateTimeField(auto_now_add=True)
class Meta:
app_label = 'writer'
class Job(models.Model):
"""Pipeline execution tracking"""
site = ForeignKey(Site)
"""Pipeline execution tracking (NEW model — does not yet exist in codebase)."""
site = ForeignKey('igny8_core_auth.Site', on_delete=models.CASCADE)
status = CharField(choices=['pending', 'running', 'completed', 'failed'])
stage = IntegerField(choices=[(0, 'Blueprint Check'), (1, 'Keywords'), (2, 'Cluster'), (3, 'Ideas'), (4, 'Tasks'), (5, 'Content'), (6, 'Taxonomy'), (7, 'Images')])
blueprint_mode = CharField(choices=['legacy', 'blueprint_aware']) # NEW
@@ -389,24 +463,27 @@ class Job(models.Model):
#### Stage 0: Blueprint Check
```python
# celery_app/tasks.py
# igny8_core/tasks.py (Celery app: celery -A igny8_core)
@app.task(bind=True, max_retries=3)
def check_blueprint(self, site_id):
"""
Stage 0: Determine execution mode and load blueprint context.
Args:
site_id: integer PK (BigAutoField)
Returns:
{
'status': 'success',
'pipeline_mode': 'blueprint_aware' | 'legacy',
'blueprint_id': 'uuid' (if active),
'blueprint_id': integer (if active),
'execution_phases': list,
'next_stage': 1
}
"""
try:
site = Site.objects.get(id=site_id)
site = Site.objects.get(id=site_id) # integer PK lookup
job = Job.objects.create(site=site, stage=0, status='running')
blueprint = SAGBlueprint.objects.filter(
@@ -418,7 +495,7 @@ def check_blueprint(self, site_id):
result = {
'status': 'success',
'pipeline_mode': 'blueprint_aware',
'blueprint_id': str(blueprint.id),
'blueprint_id': blueprint.id,
'execution_phases': blueprint.execution_priority,
}
job.blueprint_mode = 'blueprint_aware'
@@ -464,7 +541,7 @@ def process_keywords(self, site_id, blueprint_context):
blueprint_mode=blueprint_context['pipeline_mode']
)
keywords = Keyword.objects.filter(site=site, sag_cluster_id__isnull=True)
keywords = Keywords.objects.filter(site=site, sag_cluster_id__isnull=True)
if blueprint_context['pipeline_mode'] == 'blueprint_aware':
blueprint = SAGBlueprint.objects.get(id=blueprint_context['blueprint_id'])
@@ -479,11 +556,11 @@ def process_keywords(self, site_id, blueprint_context):
if cluster:
keyword.sag_cluster_id = cluster.id
keyword.save()
cluster.keywords.append(keyword.term)
cluster.keywords.append(keyword.keyword)
cluster.save()
matched_count += 1
else:
unmatched_keywords.append(keyword.term)
unmatched_keywords.append(keyword.keyword)
job.log = f"Matched {matched_count} keywords. Unmatched: {unmatched_keywords}"
else:
@@ -615,15 +692,15 @@ def create_tasks(self, site_id, blueprint_context):
blueprint_mode=blueprint_context['pipeline_mode']
)
ideas = Idea.objects.filter(site=site, task__isnull=True)
ideas = ContentIdeas.objects.filter(site=site, task__isnull=True)
task_count = 0
for idea in ideas:
task = Task.objects.create(
task = Tasks.objects.create(
site=site,
title=idea.title,
title=idea.idea_title,
idea=idea,
status='pending'
status='queued' # Tasks.STATUS_CHOICES: queued/completed
)
if blueprint_context['pipeline_mode'] == 'blueprint_aware' and idea.sag_cluster_id:
@@ -632,14 +709,14 @@ def create_tasks(self, site_id, blueprint_context):
task.sag_cluster_id = idea.sag_cluster_id
task.blueprint_context = {
'cluster_id': str(cluster.id),
'cluster_id': cluster.id,
'cluster_name': cluster.name,
'cluster_type': cluster.cluster_type,
'cluster_sector': cluster.sector,
'hub_title': blueprint.content_plan.get(str(cluster.id), {}).get('hub_title'),
'hub_url': f"{site.domain}/hubs/{cluster.name.lower().replace(' ', '-')}",
'cluster_attributes': cluster.attributes,
'content_structure': idea.structure,
'content_structure': idea.content_structure,
'content_type': idea.content_type,
'execution_phase': idea.execution_phase,
}
@@ -683,7 +760,7 @@ def generate_content(self, site_id, blueprint_context):
blueprint_mode=blueprint_context['pipeline_mode']
)
tasks = Task.objects.filter(site=site, status='completed', content__isnull=True)
tasks = Tasks.objects.filter(site=site, status='completed', content__isnull=True)
content_count = 0
for task in tasks:
@@ -795,7 +872,7 @@ def assign_taxonomy(self, site_id, blueprint_context):
cluster = SAGCluster.objects.get(id=content.sag_cluster_id)
# Load taxonomy mapping from blueprint
tax_mapping = blueprint.wp_taxonomy_mapping.get(str(cluster.id), {})
tax_mapping = blueprint.wp_taxonomy_mapping.get(cluster.id, {})
# Assign taxonomies
content.taxonomies = tax_mapping
@@ -863,7 +940,7 @@ def generate_images(self, site_id, blueprint_context):
# Generate featured image
featured_image = GenerateImage(content.title, style)
image = Image.objects.create(
image = Images.objects.create(
content=content,
url=featured_image['url'],
alt_text=featured_image['alt_text'],
@@ -1019,7 +1096,7 @@ redis-server
# Create sample site and blueprint
python manage.py shell << EOF
from django.contrib.auth.models import User
from sites.models import Site
from igny8_core.auth.models import Site
from sag.models import SAGBlueprint, SAGCluster
site = Site.objects.create(name="Test Site", domain="test.local")
@@ -1052,27 +1129,25 @@ EOF
#### Execute Pipeline Stages
```bash
# Start Celery worker (in separate terminal)
celery -A igny8.celery_app worker --loglevel=info
celery -A igny8_core worker --loglevel=info
# Run Stage 0: Blueprint Check
python manage.py shell << EOF
from celery_app.tasks import check_blueprint
result = check_blueprint.delay(site_id="<site-uuid>")
from igny8_core.tasks import check_blueprint
result = check_blueprint.delay(site_id="<site-id>")
print(result.get())
EOF
# Run full pipeline
python manage.py shell << EOF
from celery_app.tasks import check_blueprint
from uuid import UUID
site_id = UUID("<site-uuid>")
from igny8_core.tasks import check_blueprint
site_id = 1 # integer PK (BigAutoField)
check_blueprint.delay(site_id)
# Each stage automatically chains to the next
EOF
# Monitor pipeline execution
celery -A igny8.celery_app events
celery -A igny8_core events
# or view logs: tail -f celery.log
```
@@ -1080,20 +1155,20 @@ celery -A igny8.celery_app events
#### Unit Tests
```bash
pytest content/tests/test_pipeline.py -v
pytest sag/tests/test_blueprint.py -v
pytest celery_app/tests/test_tasks.py -v
pytest igny8_core/business/content/tests/test_pipeline.py -v
pytest igny8_core/sag/tests/test_blueprint.py -v
pytest igny8_core/tests/test_tasks.py -v
```
#### Integration Test
```bash
pytest content/tests/test_pipeline_integration.py::test_full_blueprint_pipeline -v
pytest igny8_core/business/content/tests/test_pipeline_integration.py::test_full_blueprint_pipeline -v
# Test legacy mode
pytest content/tests/test_pipeline_integration.py::test_full_legacy_pipeline -v
pytest igny8_core/business/content/tests/test_pipeline_integration.py::test_full_legacy_pipeline -v
# Test mixed mode (some sites with blueprint, some without)
pytest content/tests/test_pipeline_integration.py::test_mixed_mode_execution -v
pytest igny8_core/business/content/tests/test_pipeline_integration.py::test_mixed_mode_execution -v
```
#### Manual Test Scenario
@@ -1103,37 +1178,37 @@ python manage.py shell < scripts/setup_test_data.py
# 2. Import sample keywords
python manage.py shell << EOF
from content.models import Keyword
from sites.models import Site
from igny8_core.business.content.models import Keyword
from igny8_core.auth.models import Site
site = Site.objects.get(name="Test Site")
keywords = ["python tutorial", "django rest", "web scraping"]
for kw in keywords:
Keyword.objects.create(site=site, term=kw, source='csv_import')
Keywords.objects.create(site=site, term=kw, source='csv_import')
EOF
# 3. Run pipeline
celery -A igny8.celery_app worker --loglevel=debug &
celery -A igny8_core worker --loglevel=debug &
python manage.py shell << EOF
from celery_app.tasks import check_blueprint
from sites.models import Site
from igny8_core.tasks import check_blueprint
from igny8_core.auth.models import Site
site = Site.objects.get(name="Test Site")
check_blueprint.delay(site.id)
EOF
# 4. Inspect results
python manage.py shell << EOF
from content.models import Keyword, Idea, Task, Content, Image
from sites.models import Site
from igny8_core.business.content.models import Keyword, Idea, Task, Content, Image
from igny8_core.auth.models import Site
site = Site.objects.get(name="Test Site")
print("Keywords:", Keyword.objects.filter(site=site).count())
print("Ideas:", Idea.objects.filter(site=site).count())
print("Tasks:", Task.objects.filter(site=site).count())
print("Keywords:", Keywords.objects.filter(site=site).count())
print("Ideas:", ContentIdeas.objects.filter(site=site).count())
print("Tasks:", Tasks.objects.filter(site=site).count())
print("Content:", Content.objects.filter(site=site).count())
print("Images:", Image.objects.filter(site=site).count())
print("Images:", Images.objects.filter(site=site).count())
# Check blueprint context
task = Task.objects.filter(site=site, blueprint_context__isnull=False).first()
task = Tasks.objects.filter(site=site, blueprint_context__isnull=False).first()
if task:
print("Blueprint context:", task.blueprint_context)
EOF
@@ -1146,7 +1221,7 @@ EOF
# Check if blueprint exists and is active
python manage.py shell << EOF
from sag.models import SAGBlueprint
from sites.models import Site
from igny8_core.auth.models import Site
site = Site.objects.get(id="<site-id>")
blueprint = SAGBlueprint.objects.filter(site=site, status='active').first()
print(f"Blueprint: {blueprint}")
@@ -1160,9 +1235,9 @@ EOF
```bash
# Check keyword-cluster mapping
python manage.py shell << EOF
from content.models import Keyword
from igny8_core.business.content.models import Keyword
from sag.models import SAGCluster
keywords = Keyword.objects.filter(sag_cluster_id__isnull=True)
keywords = Keywords.objects.filter(sag_cluster_id__isnull=True)
print(f"Unmatched keywords: {[kw.term for kw in keywords]}")
# Check available clusters
@@ -1176,16 +1251,16 @@ EOF
```bash
# Check task status
python manage.py shell << EOF
from content.models import Task
tasks = Task.objects.all()
from igny8_core.business.content.models import Task
tasks = Tasks.objects.all()
for task in tasks:
print(f"Task {task.id}: status={task.status}, blueprint_context={bool(task.blueprint_context)}")
EOF
# Check Celery task logs
celery -A igny8.celery_app inspect active
celery -A igny8.celery_app inspect reserved
celery -A igny8.celery_app purge # WARNING: clears queue
celery -A igny8_core inspect active
celery -A igny8_core inspect reserved
celery -A igny8_core purge # WARNING: clears queue
```
### Extending with Custom Prompt Templates
@@ -1225,7 +1300,7 @@ PROMPT_TEMPLATES = {
```bash
# View pipeline execution history
python manage.py shell << EOF
from content.models import Job
from igny8_core.business.content.models import Job
jobs = Job.objects.filter(stage=5).order_by('-created_at')[:10]
for job in jobs:
duration = (job.completed_at - job.created_at).total_seconds() if job.completed_at else None