From 38324aedcdd7d9f44c338e37c6257f385679c012 Mon Sep 17 00:00:00 2001
From: Gitea Deploy <deploy@gitea.local>
Date: Mon, 10 Nov 2025 06:28:34 +0000
Subject: [PATCH] Add function_id tracking and enable JSON mode for all AI
 functions

---
 backend/igny8_core/ai/ai_core.py              |  12 +-
 backend/igny8_core/ai/engine.py               |   5 +
 .../ai/functions/generate_content.py          |   6 +-
 .../igny8_core/ai/functions/generate_ideas.py |   4 +
 backend/igny8_core/ai/prompts.py              | 241 ++++++++++++++----
 backend/igny8_core/ai/settings.py             |  18 +-
 .../modules/system/management/__init__.py     |   2 +
 .../system/management/commands/__init__.py    |   2 +
 .../commands/update_default_prompts.py        |  57 +++++
 backend/igny8_core/modules/system/utils.py    | 241 ++++++++++++++----
 backend/igny8_core/utils/ai_processor.py      |  18 +-
 .../src/components/common/ProgressModal.tsx   |  31 ++-
 frontend/src/hooks/useProgressModal.ts        |   8 +-
 frontend/src/pages/Planner/Clusters.tsx       |   5 +-
 frontend/src/pages/Planner/Ideas.tsx          |   1 +
 frontend/src/pages/Planner/Keywords.tsx       |   3 +-
 frontend/src/pages/Writer/Tasks.tsx           |   3 +-
 17 files changed, 544 insertions(+), 113 deletions(-)
 create mode 100644 backend/igny8_core/modules/system/management/__init__.py
 create mode 100644 backend/igny8_core/modules/system/management/commands/__init__.py
 create mode 100644 backend/igny8_core/modules/system/management/commands/update_default_prompts.py

diff --git a/backend/igny8_core/ai/ai_core.py b/backend/igny8_core/ai/ai_core.py
index dc070856..f8f17e4f 100644
--- a/backend/igny8_core/ai/ai_core.py
+++ b/backend/igny8_core/ai/ai_core.py
@@ -103,6 +103,7 @@ class AICore:
         response_format: Optional[Dict] = None,
         api_key: Optional[str] = None,
         function_name: str = 'ai_request',
+        function_id: Optional[str] = None,
         tracker: Optional[ConsoleStepTracker] = None
     ) -> Dict[str, Any]:
         """
@@ -158,10 +159,17 @@ class AICore:
         else:
             tracker.ai_call("Using text response format")
         
-        # Step 4: Validate prompt length
+        # Step 4: Validate prompt length and add function_id
         prompt_length = len(prompt)
         tracker.ai_call(f"Prompt length: {prompt_length} characters")
         
+        # Add function_id to prompt if provided (for tracking)
+        final_prompt = prompt
+        if function_id:
+            function_id_prefix = f'function_id: "{function_id}"\n\n'
+            final_prompt = function_id_prefix + prompt
+            tracker.ai_call(f"Added function_id to prompt: {function_id}")
+        
         # Step 5: Build request payload
         url = 'https://api.openai.com/v1/chat/completions'
         headers = {
@@ -171,7 +179,7 @@ class AICore:
         
         body_data = {
             'model': active_model,
-            'messages': [{'role': 'user', 'content': prompt}],
+            'messages': [{'role': 'user', 'content': final_prompt}],
             'temperature': temperature,
         }
         
diff --git a/backend/igny8_core/ai/engine.py b/backend/igny8_core/ai/engine.py
index 42f1582d..11ed3925 100644
--- a/backend/igny8_core/ai/engine.py
+++ b/backend/igny8_core/ai/engine.py
@@ -82,11 +82,15 @@ class AIEngine:
             ai_core = AICore(account=self.account)
             function_name = fn.get_name()
             
+            # Generate function_id for tracking (ai-{function_name}-01)
+            function_id = f"ai-{function_name}-01"
+            
             # Get model config from settings (Stage 4 requirement)
             model_config = get_model_config(function_name)
             model = model_config.get('model')
             
             self.console_tracker.ai_call(f"Calling {model or 'default'} model with {len(prompt)} char prompt")
+            self.console_tracker.ai_call(f"Function ID: {function_id}")
             
             # Track AI call start
             self.step_tracker.add_response_step("AI_CALL", "success", f"Calling {model or 'default'} model...")
@@ -102,6 +106,7 @@ class AIEngine:
                     temperature=model_config.get('temperature'),
                     response_format=model_config.get('response_format'),
                     function_name=function_name,
+                    function_id=function_id,  # Pass function_id for tracking
                     tracker=self.console_tracker  # Pass console tracker for logging
                 )
             except Exception as e:
diff --git a/backend/igny8_core/ai/functions/generate_content.py b/backend/igny8_core/ai/functions/generate_content.py
index a2227429..ee260519 100644
--- a/backend/igny8_core/ai/functions/generate_content.py
+++ b/backend/igny8_core/ai/functions/generate_content.py
@@ -299,6 +299,9 @@ def generate_content_core(task_ids: List[int], account_id: int = None, progress_
             # Get model config from settings
             model_config = get_model_config('generate_content')
             
+            # Generate function_id for tracking (ai-generate-content-02 for legacy path)
+            function_id = "ai-generate-content-02"
+            
             # Call AI using centralized request handler
             ai_core = AICore(account=account)
             result = ai_core.run_ai_request(
@@ -307,7 +310,8 @@ def generate_content_core(task_ids: List[int], account_id: int = None, progress_
                 max_tokens=model_config.get('max_tokens'),
                 temperature=model_config.get('temperature'),
                 response_format=model_config.get('response_format'),
-                function_name='generate_content'
+                function_name='generate_content',
+                function_id=function_id  # Pass function_id for tracking
             )
             
             if result.get('error'):
diff --git a/backend/igny8_core/ai/functions/generate_ideas.py b/backend/igny8_core/ai/functions/generate_ideas.py
index 473fd7b1..43a24898 100644
--- a/backend/igny8_core/ai/functions/generate_ideas.py
+++ b/backend/igny8_core/ai/functions/generate_ideas.py
@@ -282,6 +282,9 @@ def generate_ideas_core(cluster_id: int, account_id: int = None, progress_callba
         # Get model config from settings
         model_config = get_model_config('generate_ideas')
         
+        # Generate function_id for tracking (ai-generate-ideas-02 for legacy path)
+        function_id = "ai-generate-ideas-02"
+        
         # Call AI using centralized request handler
         ai_core = AICore(account=account)
         result = ai_core.run_ai_request(
@@ -291,6 +294,7 @@ def generate_ideas_core(cluster_id: int, account_id: int = None, progress_callba
             temperature=model_config.get('temperature'),
             response_format=model_config.get('response_format'),
             function_name='generate_ideas',
+            function_id=function_id,  # Pass function_id for tracking
             tracker=tracker
         )
         
diff --git a/backend/igny8_core/ai/prompts.py b/backend/igny8_core/ai/prompts.py
index ba2f92d9..2df411eb 100644
--- a/backend/igny8_core/ai/prompts.py
+++ b/backend/igny8_core/ai/prompts.py
@@ -19,54 +19,209 @@ class PromptRegistry:
     
     # Default prompts stored in registry
     DEFAULT_PROMPTS = {
-        'clustering': """Analyze the following keywords and group them into topic clusters.
+        'clustering': """You are a semantic strategist and SEO architecture engine. Your task is to analyze the provided keyword list and group them into meaningful, intent-driven topic clusters that reflect how real users search, think, and act online.
 
-Each cluster should include:
-- "name": A clear, descriptive topic name
-- "description": A brief explanation of what the cluster covers
-- "keywords": A list of related keywords that belong to this cluster
+Return a single JSON object with a "clusters" array. Each cluster must follow this structure:
 
-Format the output as a JSON object with a "clusters" array.
+{
+  "name": "[Descriptive cluster name — natural, SEO-relevant, clearly expressing the topic]",
+  "description": "[1–2 concise sentences explaining what this cluster covers and why these keywords belong together]",
+  "keywords": ["keyword 1", "keyword 2", "keyword 3", "..."]
+}
 
-IMPORTANT: In the "keywords" array, you MUST use the EXACT keyword strings from the input list below. Do not modify, paraphrase, or create variations of the keywords. Only use the exact keywords as they appear in the input list.
+CLUSTERING STRATEGY:
 
-Clustering rules:
-- Group keywords based on strong semantic or topical relationships (intent, use-case, function, audience, etc.)
-- Clusters should reflect how people actually search — problem ➝ solution, general ➝ specific, product ➝ benefit, etc.
-- Avoid grouping keywords just because they share similar words — focus on meaning
-- Include 3–10 keywords per cluster where appropriate
-- Skip unrelated or outlier keywords that don't fit a clear theme
-- CRITICAL: Only return keywords that exactly match the input keywords (case-insensitive matching is acceptable)
+1. Keyword-first, structure-follows:
+   - Do NOT rely on assumed categories or existing content structures.
+   - Begin purely from the meaning, intent, and behavioral connection between keywords.
 
-Keywords to process:
-[IGNY8_KEYWORDS]""",
+2. Use multi-dimensional grouping logic:
+   - Group keywords by these behavioral dimensions:
+     • Search Intent → informational, commercial, transactional, navigational  
+     • Use-Case or Problem → what the user is trying to achieve or solve  
+     • Function or Feature → how something works or what it does  
+     • Persona or Audience → who the content or product serves  
+     • Context → location, time, season, platform, or device  
+   - Combine 2–3 dimensions naturally where they make sense.
+
+3. Model real search behavior:
+   - Favor clusters that form natural user journeys such as:
+     • Problem ➝ Solution
+     • General ➝ Specific
+     • Product ➝ Use-case
+     • Buyer ➝ Benefit
+     • Tool ➝ Function
+     • Task ➝ Method
+   - Each cluster should feel like a real topic hub users would explore in depth.
+
+4. Avoid superficial groupings:
+   - Do not cluster keywords just because they share words.
+   - Do not force-fit outliers or unrelated keywords.
+   - Exclude keywords that don't logically connect to any cluster.
+
+5. Quality rules:
+   - Each cluster should include between 3–10 strongly related keywords.
+   - Never duplicate a keyword across multiple clusters.
+   - Prioritize semantic strength, search intent, and usefulness for SEO-driven content structure.
+   - It's better to output fewer, high-quality clusters than many weak or shallow ones.
+
+INPUT FORMAT:
+{
+  "keywords": [IGNY8_KEYWORDS]
+}
+
+OUTPUT FORMAT:
+Return ONLY the final JSON object in this format:
+{
+  "clusters": [
+    {
+      "name": "...",
+      "description": "...",
+      "keywords": ["...", "...", "..."]
+    }
+  ]
+}
+
+Do not include any explanations, text, or commentary outside the JSON output.
+""",
         
-        'ideas': """Generate SEO-optimized, high-quality content ideas and detailed outlines for each of the following keyword clusters.
-
-Clusters to analyze:
-[IGNY8_CLUSTERS]
-
-Keywords in each cluster:
-[IGNY8_CLUSTER_KEYWORDS]
-
-Return your response as JSON with an "ideas" array.
-For each cluster, generate 1-3 content ideas.
+        'ideas': """Generate SEO-optimized, high-quality content ideas and outlines for each keyword cluster.
+Input:
+Clusters: [IGNY8_CLUSTERS]
+Keywords: [IGNY8_CLUSTER_KEYWORDS]
 
+Output: JSON with "ideas" array.
+Each cluster → 1 cluster_hub + 2–4 supporting ideas.
 Each idea must include:
-- "title": compelling blog/article title that naturally includes a primary keyword
-- "description": detailed content outline with H2/H3 structure (as plain text or structured JSON)
-- "content_type": the type of content (blog_post, article, guide, tutorial)
-- "content_structure": the editorial structure (cluster_hub, landing_page, pillar_page, supporting_page)
-- "estimated_word_count": estimated total word count (1500-2200 words)
-- "target_keywords": comma-separated list of keywords that will be covered (or "covered_keywords")
-- "cluster_name": name of the cluster this idea belongs to (REQUIRED)
-- "cluster_id": ID of the cluster this idea belongs to (REQUIRED - use the exact cluster ID from the input)
+title, description, content_type, content_structure, cluster_id, estimated_word_count (1500–2200), and covered_keywords.
 
-IMPORTANT: You MUST include the exact "cluster_id" from the cluster data provided. Match the cluster name to find the correct cluster_id.
+Outline Rules:
 
-Return only valid JSON with an "ideas" array.""",
+Intro: 1 hook (30–40 words) + 2 intro paragraphs (50–60 words each).
+
+5–8 H2 sections, each with 2–3 H3s.
+
+Each H2 ≈ 250–300 words, mixed content (paragraphs, lists, tables, blockquotes).
+
+Vary section format and tone; no bullets or lists at start.
+
+Tables have columns; blockquotes = expert POV or data insight.
+
+Use depth, examples, and real context.
+
+Avoid repetitive structure.
+
+Tone: Professional editorial flow. No generic phrasing. Use varied sentence openings and realistic examples.
+
+Output JSON Example:
+
+{
+  "ideas": [
+    {
+      "title": "Best Organic Cotton Duvet Covers for All Seasons",
+      "description": {
+        "introduction": {
+          "hook": "Transform your sleep with organic cotton that blends comfort and sustainability.",
+          "paragraphs": [
+            {"content_type": "paragraph", "details": "Overview of organic cotton's rise in bedding industry."},
+            {"content_type": "paragraph", "details": "Why consumers prefer organic bedding over synthetic alternatives."}
+          ]
+        },
+        "H2": [
+          {
+            "heading": "Why Choose Organic Cotton for Bedding?",
+            "subsections": [
+              {"subheading": "Health and Skin Benefits", "content_type": "paragraph", "details": "Discuss hypoallergenic and chemical-free aspects."},
+              {"subheading": "Environmental Sustainability", "content_type": "list", "details": "Eco benefits like low water use, no pesticides."},
+              {"subheading": "Long-Term Cost Savings", "content_type": "table", "details": "Compare durability and pricing over time."}
+            ]
+          }
+        ]
+      },
+      "content_type": "post",
+      "content_structure": "review",
+      "cluster_id": 12,
+      "estimated_word_count": 1800,
+      "covered_keywords": "organic duvet covers, eco-friendly bedding, sustainable sheets"
+    }
+  ]
+}""",
         
-        'content_generation': """You are an editorial content strategist. Generate a complete blog post/article based on the provided content idea.
+        'content_generation': """You are an editorial content strategist. Your task is to generate a complete JSON response object that includes all the fields listed below, based on the provided content idea, keyword cluster, and keyword list.
+
+Only the `content` field should contain HTML inside JSON object.
+
+==================
+Generate a complete JSON response object matching this structure:
+==================
+
+{
+  "title": "[Blog title using the primary keyword — full sentence case]",
+  "meta_title": "[Meta title under 60 characters — natural, optimized, and compelling]",
+  "meta_description": "[Meta description under 160 characters — clear and enticing summary]",
+  "content": "[HTML content — full editorial structure with <p>, <h2>, <h3>, <ul>, <ol>, <table>]",
+  "word_count": [Exact integer — word count of HTML body only],
+  "primary_keyword": "[Single primary keyword used in title and first paragraph]",
+  "secondary_keywords": [
+    "[Keyword 1]",
+    "[Keyword 2]",
+    "[Keyword 3]"
+  ],
+  "tags": [
+    "[2–4 word lowercase tag 1]",
+    "[2–4 word lowercase tag 2]",
+    "[2–4 word lowercase tag 3]",
+    "[2–4 word lowercase tag 4]",
+    "[2–4 word lowercase tag 5]"
+  ],
+  "categories": [
+    "[Parent Category > Child Category]",
+    "[Optional Second Category > Optional Subcategory]"
+  ]
+}
+
+===========================
+CONTENT FLOW RULES
+===========================
+
+**INTRODUCTION:**
+- Start with 1 italicized hook (30–40 words)
+- Follow with 2 narrative paragraphs (each 50–60 words; 2–3 sentences max)
+- No headings allowed in intro
+
+**H2 SECTIONS (5–8 total):**
+Each section should be 250–300 words and follow this format:
+1. Two narrative paragraphs (80–120 words each, 2–3 sentences)
+2. One list or table (must come *after* a paragraph)
+3. Optional closing paragraph (40–60 words)
+4. Insert 2–3  subsections naturally after main paragraphs
+
+**Formatting Rules:**
+- Vary use of unordered lists, ordered lists, and tables across sections
+- Never begin any section or sub-section with a list or table
+
+===========================
+KEYWORD & SEO RULES
+===========================
+
+- **Primary keyword** must appear in:
+  - The title
+  - First paragraph of the introduction
+  - At least 2 H2 headings
+
+- **Secondary keywords** must be used naturally, not forced
+
+- **Tone & style guidelines:**
+  - No robotic or passive voice
+  - Avoid generic intros like "In today's world…"
+  - Don't repeat heading in opening sentence
+  - Vary sentence structure and length
+
+
+
+===========================
+INPUT VARIABLES
+===========================
 
 CONTENT IDEA DETAILS:
 [IGNY8_IDEA]
@@ -77,14 +232,12 @@ KEYWORD CLUSTER:
 ASSOCIATED KEYWORDS:
 [IGNY8_KEYWORDS]
 
-Generate well-structured, SEO-optimized content with:
-- Engaging introduction
-- 5-8 H2 sections with H3 subsections
-- Natural keyword integration
-- 1500-2000 words total
-- Proper HTML formatting (h2, h3, p, ul, ol, table tags)
+===========================
+OUTPUT FORMAT
+===========================
 
-Return the content as plain text with HTML tags.""",
+Return ONLY the final JSON object.  
+Do NOT include any comments, formatting, or explanations.""",
         
         'image_prompt_extraction': """Extract image prompts from the following article content.
 
diff --git a/backend/igny8_core/ai/settings.py b/backend/igny8_core/ai/settings.py
index 6ba95e50..097a0fa2 100644
--- a/backend/igny8_core/ai/settings.py
+++ b/backend/igny8_core/ai/settings.py
@@ -11,18 +11,17 @@ MODEL_CONFIG = {
         "temperature": 0.7,
         "response_format": {"type": "json_object"},  # Auto-enabled for JSON mode models
     },
-    # REMOVED: generate_ideas function removed
-    # "generate_ideas": {
-    #     "model": "gpt-4.1",
-    #     "max_tokens": 4000,
-    #     "temperature": 0.7,
-    #     "response_format": {"type": "json_object"},
-    # },
+    "generate_ideas": {
+        "model": "gpt-4.1",
+        "max_tokens": 4000,
+        "temperature": 0.7,
+        "response_format": {"type": "json_object"},  # JSON output
+    },
     "generate_content": {
         "model": "gpt-4.1",
         "max_tokens": 8000,
         "temperature": 0.7,
-        "response_format": None,  # Text output
+        "response_format": {"type": "json_object"},  # JSON output
     },
     "generate_images": {
         "model": "dall-e-3",
@@ -41,8 +40,7 @@ MODEL_CONFIG = {
 FUNCTION_ALIASES = {
     "cluster_keywords": "auto_cluster",
     "auto_cluster_keywords": "auto_cluster",
-    # REMOVED: generate_ideas function removed
-    # "auto_generate_ideas": "generate_ideas",
+    "auto_generate_ideas": "generate_ideas",
     "auto_generate_content": "generate_content",
     "auto_generate_images": "generate_images",
 }
diff --git a/backend/igny8_core/modules/system/management/__init__.py b/backend/igny8_core/modules/system/management/__init__.py
new file mode 100644
index 00000000..b0314efa
--- /dev/null
+++ b/backend/igny8_core/modules/system/management/__init__.py
@@ -0,0 +1,2 @@
+# Management commands for system module
+
diff --git a/backend/igny8_core/modules/system/management/commands/__init__.py b/backend/igny8_core/modules/system/management/commands/__init__.py
new file mode 100644
index 00000000..16d82086
--- /dev/null
+++ b/backend/igny8_core/modules/system/management/commands/__init__.py
@@ -0,0 +1,2 @@
+# Management commands
+
diff --git a/backend/igny8_core/modules/system/management/commands/update_default_prompts.py b/backend/igny8_core/modules/system/management/commands/update_default_prompts.py
new file mode 100644
index 00000000..8e746aa0
--- /dev/null
+++ b/backend/igny8_core/modules/system/management/commands/update_default_prompts.py
@@ -0,0 +1,57 @@
+"""
+Management command to copy prompt values from database and update default prompts in code
+"""
+from django.core.management.base import BaseCommand
+from igny8_core.modules.system.models import AIPrompt
+
+
+class Command(BaseCommand):
+    help = 'Copy prompt values from database and update default prompts in prompts.py'
+
+    def handle(self, *args, **options):
+        prompt_types = ['clustering', 'ideas', 'content_generation']
+        
+        self.stdout.write("Fetching prompts from database...")
+        
+        prompts_data = {}
+        for prompt_type in prompt_types:
+            try:
+                # Get the first active prompt for this type (assuming there's one per account or we take the first)
+                prompt = AIPrompt.objects.filter(
+                    prompt_type=prompt_type,
+                    is_active=True
+                ).first()
+                
+                if prompt:
+                    prompts_data[prompt_type] = prompt.prompt_value
+                    self.stdout.write(
+                        self.style.SUCCESS(
+                            f"✓ Found {prompt_type} prompt (ID: {prompt.id}, Account: {prompt.account_id if hasattr(prompt, 'account') else 'N/A'})"
+                        )
+                    )
+                else:
+                    self.stdout.write(
+                        self.style.WARNING(
+                            f"⚠ No active prompt found for {prompt_type}"
+                        )
+                    )
+            except Exception as e:
+                self.stdout.write(
+                    self.style.ERROR(
+                        f"✗ Error fetching {prompt_type}: {str(e)}"
+                    )
+                )
+        
+        # Print the prompts for manual copying
+        self.stdout.write("\n" + "="*80)
+        self.stdout.write("PROMPT VALUES FROM DATABASE:")
+        self.stdout.write("="*80 + "\n")
+        
+        for prompt_type, prompt_value in prompts_data.items():
+            self.stdout.write(f"\n# {prompt_type.upper()}")
+            self.stdout.write(f"'{prompt_type}': \"\"\"{prompt_value}\"\"\",")
+        
+        self.stdout.write("\n" + "="*80)
+        self.stdout.write("Copy the above prompts and update prompts.py DEFAULT_PROMPTS")
+        self.stdout.write("="*80)
+
diff --git a/backend/igny8_core/modules/system/utils.py b/backend/igny8_core/modules/system/utils.py
index ea3aa8b5..19eb10cd 100644
--- a/backend/igny8_core/modules/system/utils.py
+++ b/backend/igny8_core/modules/system/utils.py
@@ -7,54 +7,209 @@ from typing import Optional
 def get_default_prompt(prompt_type: str) -> str:
     """Get default prompt value by type"""
     defaults = {
-        'clustering': """Analyze the following keywords and group them into topic clusters.
+        'clustering': """You are a semantic strategist and SEO architecture engine. Your task is to analyze the provided keyword list and group them into meaningful, intent-driven topic clusters that reflect how real users search, think, and act online.
 
-Each cluster should include:
-- "name": A clear, descriptive topic name
-- "description": A brief explanation of what the cluster covers
-- "keywords": A list of related keywords that belong to this cluster
+Return a single JSON object with a "clusters" array. Each cluster must follow this structure:
 
-Format the output as a JSON object with a "clusters" array.
+{
+  "name": "[Descriptive cluster name — natural, SEO-relevant, clearly expressing the topic]",
+  "description": "[1–2 concise sentences explaining what this cluster covers and why these keywords belong together]",
+  "keywords": ["keyword 1", "keyword 2", "keyword 3", "..."]
+}
 
-IMPORTANT: In the "keywords" array, you MUST use the EXACT keyword strings from the input list below. Do not modify, paraphrase, or create variations of the keywords. Only use the exact keywords as they appear in the input list.
+CLUSTERING STRATEGY:
 
-Clustering rules:
-- Group keywords based on strong semantic or topical relationships (intent, use-case, function, audience, etc.)
-- Clusters should reflect how people actually search — problem ➝ solution, general ➝ specific, product ➝ benefit, etc.
-- Avoid grouping keywords just because they share similar words — focus on meaning
-- Include 3–10 keywords per cluster where appropriate
-- Skip unrelated or outlier keywords that don't fit a clear theme
-- CRITICAL: Only return keywords that exactly match the input keywords (case-insensitive matching is acceptable)
+1. Keyword-first, structure-follows:
+   - Do NOT rely on assumed categories or existing content structures.
+   - Begin purely from the meaning, intent, and behavioral connection between keywords.
 
-Keywords to process:
-[IGNY8_KEYWORDS]""",
+2. Use multi-dimensional grouping logic:
+   - Group keywords by these behavioral dimensions:
+     • Search Intent → informational, commercial, transactional, navigational  
+     • Use-Case or Problem → what the user is trying to achieve or solve  
+     • Function or Feature → how something works or what it does  
+     • Persona or Audience → who the content or product serves  
+     • Context → location, time, season, platform, or device  
+   - Combine 2–3 dimensions naturally where they make sense.
+
+3. Model real search behavior:
+   - Favor clusters that form natural user journeys such as:
+     • Problem ➝ Solution
+     • General ➝ Specific
+     • Product ➝ Use-case
+     • Buyer ➝ Benefit
+     • Tool ➝ Function
+     • Task ➝ Method
+   - Each cluster should feel like a real topic hub users would explore in depth.
+
+4. Avoid superficial groupings:
+   - Do not cluster keywords just because they share words.
+   - Do not force-fit outliers or unrelated keywords.
+   - Exclude keywords that don't logically connect to any cluster.
+
+5. Quality rules:
+   - Each cluster should include between 3–10 strongly related keywords.
+   - Never duplicate a keyword across multiple clusters.
+   - Prioritize semantic strength, search intent, and usefulness for SEO-driven content structure.
+   - It's better to output fewer, high-quality clusters than many weak or shallow ones.
+
+INPUT FORMAT:
+{
+  "keywords": [IGNY8_KEYWORDS]
+}
+
+OUTPUT FORMAT:
+Return ONLY the final JSON object in this format:
+{
+  "clusters": [
+    {
+      "name": "...",
+      "description": "...",
+      "keywords": ["...", "...", "..."]
+    }
+  ]
+}
+
+Do not include any explanations, text, or commentary outside the JSON output.
+""",
         
-        'ideas': """Generate SEO-optimized, high-quality content ideas and detailed outlines for each of the following keyword clusters.
-
-Clusters to analyze:
-[IGNY8_CLUSTERS]
-
-Keywords in each cluster:
-[IGNY8_CLUSTER_KEYWORDS]
-
-Return your response as JSON with an "ideas" array.
-For each cluster, generate 1-3 content ideas.
+        'ideas': """Generate SEO-optimized, high-quality content ideas and outlines for each keyword cluster.
+Input:
+Clusters: [IGNY8_CLUSTERS]
+Keywords: [IGNY8_CLUSTER_KEYWORDS]
 
+Output: JSON with "ideas" array.
+Each cluster → 1 cluster_hub + 2–4 supporting ideas.
 Each idea must include:
-- "title": compelling blog/article title that naturally includes a primary keyword
-- "description": detailed content outline with H2/H3 structure (as plain text or structured JSON)
-- "content_type": the type of content (blog_post, article, guide, tutorial)
-- "content_structure": the editorial structure (cluster_hub, landing_page, pillar_page, supporting_page)
-- "estimated_word_count": estimated total word count (1500-2200 words)
-- "target_keywords": comma-separated list of keywords that will be covered (or "covered_keywords")
-- "cluster_name": name of the cluster this idea belongs to (REQUIRED)
-- "cluster_id": ID of the cluster this idea belongs to (REQUIRED - use the exact cluster ID from the input)
+title, description, content_type, content_structure, cluster_id, estimated_word_count (1500–2200), and covered_keywords.
 
-IMPORTANT: You MUST include the exact "cluster_id" from the cluster data provided. Match the cluster name to find the correct cluster_id.
+Outline Rules:
 
-Return only valid JSON with an "ideas" array.""",
+Intro: 1 hook (30–40 words) + 2 intro paragraphs (50–60 words each).
+
+5–8 H2 sections, each with 2–3 H3s.
+
+Each H2 ≈ 250–300 words, mixed content (paragraphs, lists, tables, blockquotes).
+
+Vary section format and tone; no bullets or lists at start.
+
+Tables have columns; blockquotes = expert POV or data insight.
+
+Use depth, examples, and real context.
+
+Avoid repetitive structure.
+
+Tone: Professional editorial flow. No generic phrasing. Use varied sentence openings and realistic examples.
+
+Output JSON Example:
+
+{
+  "ideas": [
+    {
+      "title": "Best Organic Cotton Duvet Covers for All Seasons",
+      "description": {
+        "introduction": {
+          "hook": "Transform your sleep with organic cotton that blends comfort and sustainability.",
+          "paragraphs": [
+            {"content_type": "paragraph", "details": "Overview of organic cotton's rise in bedding industry."},
+            {"content_type": "paragraph", "details": "Why consumers prefer organic bedding over synthetic alternatives."}
+          ]
+        },
+        "H2": [
+          {
+            "heading": "Why Choose Organic Cotton for Bedding?",
+            "subsections": [
+              {"subheading": "Health and Skin Benefits", "content_type": "paragraph", "details": "Discuss hypoallergenic and chemical-free aspects."},
+              {"subheading": "Environmental Sustainability", "content_type": "list", "details": "Eco benefits like low water use, no pesticides."},
+              {"subheading": "Long-Term Cost Savings", "content_type": "table", "details": "Compare durability and pricing over time."}
+            ]
+          }
+        ]
+      },
+      "content_type": "post",
+      "content_structure": "review",
+      "cluster_id": 12,
+      "estimated_word_count": 1800,
+      "covered_keywords": "organic duvet covers, eco-friendly bedding, sustainable sheets"
+    }
+  ]
+}""",
         
-        'content_generation': """You are an editorial content strategist. Generate a complete blog post/article based on the provided content idea.
+        'content_generation': """You are an editorial content strategist. Your task is to generate a complete JSON response object that includes all the fields listed below, based on the provided content idea, keyword cluster, and keyword list.
+
+Only the `content` field should contain HTML inside JSON object.
+
+==================
+Generate a complete JSON response object matching this structure:
+==================
+
+{
+  "title": "[Blog title using the primary keyword — full sentence case]",
+  "meta_title": "[Meta title under 60 characters — natural, optimized, and compelling]",
+  "meta_description": "[Meta description under 160 characters — clear and enticing summary]",
+  "content": "[HTML content — full editorial structure with <p>, <h2>, <h3>, <ul>, <ol>, <table>]",
+  "word_count": [Exact integer — word count of HTML body only],
+  "primary_keyword": "[Single primary keyword used in title and first paragraph]",
+  "secondary_keywords": [
+    "[Keyword 1]",
+    "[Keyword 2]",
+    "[Keyword 3]"
+  ],
+  "tags": [
+    "[2–4 word lowercase tag 1]",
+    "[2–4 word lowercase tag 2]",
+    "[2–4 word lowercase tag 3]",
+    "[2–4 word lowercase tag 4]",
+    "[2–4 word lowercase tag 5]"
+  ],
+  "categories": [
+    "[Parent Category > Child Category]",
+    "[Optional Second Category > Optional Subcategory]"
+  ]
+}
+
+===========================
+CONTENT FLOW RULES
+===========================
+
+**INTRODUCTION:**
+- Start with 1 italicized hook (30–40 words)
+- Follow with 2 narrative paragraphs (each 50–60 words; 2–3 sentences max)
+- No headings allowed in intro
+
+**H2 SECTIONS (5–8 total):**
+Each section should be 250–300 words and follow this format:
+1. Two narrative paragraphs (80–120 words each, 2–3 sentences)
+2. One list or table (must come *after* a paragraph)
+3. Optional closing paragraph (40–60 words)
+4. Insert 2–3  subsections naturally after main paragraphs
+
+**Formatting Rules:**
+- Vary use of unordered lists, ordered lists, and tables across sections
+- Never begin any section or sub-section with a list or table
+
+===========================
+KEYWORD & SEO RULES
+===========================
+
+- **Primary keyword** must appear in:
+  - The title
+  - First paragraph of the introduction
+  - At least 2 H2 headings
+
+- **Secondary keywords** must be used naturally, not forced
+
+- **Tone & style guidelines:**
+  - No robotic or passive voice
+  - Avoid generic intros like "In today's world…"
+  - Don't repeat heading in opening sentence
+  - Vary sentence structure and length
+
+
+
+===========================
+INPUT VARIABLES
+===========================
 
 CONTENT IDEA DETAILS:
 [IGNY8_IDEA]
@@ -65,14 +220,12 @@ KEYWORD CLUSTER:
 ASSOCIATED KEYWORDS:
 [IGNY8_KEYWORDS]
 
-Generate well-structured, SEO-optimized content with:
-- Engaging introduction
-- 5-8 H2 sections with H3 subsections
-- Natural keyword integration
-- 1500-2000 words total
-- Proper HTML formatting (h2, h3, p, ul, ol, table tags)
+===========================
+OUTPUT FORMAT
+===========================
 
-Return the content as plain text with HTML tags.""",
+Return ONLY the final JSON object.  
+Do NOT include any comments, formatting, or explanations.""",
         
         'image_prompt_extraction': """Extract image prompts from the following article content.
 
diff --git a/backend/igny8_core/utils/ai_processor.py b/backend/igny8_core/utils/ai_processor.py
index 4d01a2f0..582743c3 100644
--- a/backend/igny8_core/utils/ai_processor.py
+++ b/backend/igny8_core/utils/ai_processor.py
@@ -130,6 +130,7 @@ class AIProcessor:
         temperature: float = 0.7,
         response_format: Optional[Dict] = None,
         api_key: Optional[str] = None,
+        function_id: Optional[str] = None,
         response_steps=None
     ) -> Dict[str, Any]:
         """
@@ -167,10 +168,17 @@ class AIProcessor:
             'Content-Type': 'application/json',
         }
         
+        # Add function_id to prompt if provided (for tracking)
+        final_prompt = prompt
+        if function_id:
+            function_id_prefix = f'function_id: "{function_id}"\n\n'
+            final_prompt = function_id_prefix + prompt
+            logger.info(f"Added function_id to prompt: {function_id}")
+        
         # EXACT request format from reference plugin (openai-api.php line 402-404)
         body_data = {
             'model': model,
-            'messages': [{'role': 'user', 'content': prompt}],
+            'messages': [{'role': 'user', 'content': final_prompt}],
             'temperature': temperature,
         }
         
@@ -460,7 +468,13 @@ class AIProcessor:
         Returns:
             Dict with 'content', 'tokens_used', 'model', 'cost', 'error'
         """
-        result = self._call_openai(prompt, model, max_tokens, temperature)
+        # Generate function_id for tracking (ai-generate-content-03 for AIProcessor path)
+        function_id = "ai-generate-content-03"
+        # Get response_format from settings for generate_content
+        from igny8_core.ai.settings import get_model_config
+        model_config = get_model_config('generate_content')
+        response_format = model_config.get('response_format')
+        result = self._call_openai(prompt, model, max_tokens, temperature, response_format=response_format, function_id=function_id)
         
         return {
             'content': result.get('content', ''),
diff --git a/frontend/src/components/common/ProgressModal.tsx b/frontend/src/components/common/ProgressModal.tsx
index d45ebf31..0087cbea 100644
--- a/frontend/src/components/common/ProgressModal.tsx
+++ b/frontend/src/components/common/ProgressModal.tsx
@@ -19,8 +19,16 @@ export interface ProgressModalProps {
   onClose?: () => void;
   onCancel?: () => void;
   taskId?: string;
+  functionId?: string; // AI function ID for tracking (e.g., "ai-cluster-01")
 }
 
+// Generate modal instance ID (increments per modal instance)
+let modalInstanceCounter = 0;
+const getModalInstanceId = () => {
+  modalInstanceCounter++;
+  return `modal-${String(modalInstanceCounter).padStart(2, '0')}`;
+};
+
 export default function ProgressModal({
   isOpen,
   title,
@@ -31,7 +39,17 @@ export default function ProgressModal({
   onClose,
   onCancel,
   taskId,
+  functionId,
 }: ProgressModalProps) {
+  // Generate modal instance ID on first render
+  const modalInstanceIdRef = React.useRef<string | null>(null);
+  if (!modalInstanceIdRef.current) {
+    modalInstanceIdRef.current = getModalInstanceId();
+  }
+  const modalInstanceId = modalInstanceIdRef.current;
+  
+  // Build full function ID with modal instance
+  const fullFunctionId = functionId ? `${functionId}-${modalInstanceId}` : null;
   // Auto-close on completion after 2 seconds
   // Don't auto-close on error - let user manually close to see error details
   const hasAutoClosedRef = React.useRef(false);
@@ -175,10 +193,15 @@ export default function ProgressModal({
           </div>
         )}
 
-        {/* Task ID (for debugging) */}
-        {taskId && import.meta.env.DEV && (
-          <div className="mb-4 text-xs text-gray-400 dark:text-gray-600">
-            Task ID: {taskId}
+        {/* Function ID and Task ID (for debugging) */}
+        {(fullFunctionId || taskId) && import.meta.env.DEV && (
+          <div className="mb-4 space-y-1 text-xs text-gray-400 dark:text-gray-600">
+            {fullFunctionId && (
+              <div>Function ID: {fullFunctionId}</div>
+            )}
+            {taskId && (
+              <div>Task ID: {taskId}</div>
+            )}
           </div>
         )}
 
diff --git a/frontend/src/hooks/useProgressModal.ts b/frontend/src/hooks/useProgressModal.ts
index 9e757a76..02ba3bd7 100644
--- a/frontend/src/hooks/useProgressModal.ts
+++ b/frontend/src/hooks/useProgressModal.ts
@@ -19,17 +19,19 @@ export interface UseProgressModalReturn {
   isOpen: boolean;
   title: string;
   taskId: string | null;
-  openModal: (taskId: string, title: string) => void;
+  openModal: (taskId: string, title: string, functionId?: string) => void;
   updateTaskId: (taskId: string) => void;
   closeModal: () => void;
   setError: (errorMessage: string) => void;
   reset: () => void;
+  functionId?: string; // AI function ID for tracking
 }
 
 export function useProgressModal(): UseProgressModalReturn {
   const [isOpen, setIsOpen] = useState(false);
   const [taskId, setTaskId] = useState<string | null>(null);
   const [title, setTitle] = useState('');
+  const [functionId, setFunctionId] = useState<string | undefined>(undefined);
   const [progress, setProgress] = useState<ProgressState>({
     percentage: 0,
     message: 'Initializing...',
@@ -532,7 +534,7 @@ export function useProgressModal(): UseProgressModalReturn {
     };
   }, [taskId, isOpen]);
 
-  const openModal = useCallback((newTaskId: string, newTitle: string) => {
+  const openModal = useCallback((newTaskId: string, newTitle: string, newFunctionId?: string) => {
     // Clear any existing transition timeout
     if (stepTransitionTimeoutRef.current) {
       clearTimeout(stepTransitionTimeoutRef.current);
@@ -542,6 +544,7 @@ export function useProgressModal(): UseProgressModalReturn {
     currentStepRef.current = null;
     setTaskId(newTaskId);
     setTitle(newTitle);
+    setFunctionId(newFunctionId);
     setIsOpen(true);
     setProgress({
       percentage: 0,
@@ -610,6 +613,7 @@ export function useProgressModal(): UseProgressModalReturn {
     reset,
     title, // Expose title for use in component
     taskId, // Expose taskId for use in ProgressModal
+    functionId, // Expose functionId for use in ProgressModal
   };
 }
 
diff --git a/frontend/src/pages/Planner/Clusters.tsx b/frontend/src/pages/Planner/Clusters.tsx
index e865bec1..572e16b6 100644
--- a/frontend/src/pages/Planner/Clusters.tsx
+++ b/frontend/src/pages/Planner/Clusters.tsx
@@ -207,7 +207,7 @@ export default function Clusters() {
         
         if (result.success && result.task_id) {
           // Async task - show progress modal
-          progressModal.openModal(result.task_id, 'Generating Ideas');
+          progressModal.openModal(result.task_id, 'Generating Ideas', 'ai-generate-ideas-01');
         } else if (result.success && result.ideas_created) {
           // Synchronous completion
           toast.success(result.message || 'Ideas generated successfully');
@@ -249,7 +249,7 @@ export default function Clusters() {
           if (result.task_id) {
             // Async task - open progress modal
             hasReloadedRef.current = false;
-            progressModal.openModal(result.task_id, 'Generating Content Ideas');
+            progressModal.openModal(result.task_id, 'Generating Content Ideas', 'ai-generate-ideas-01');
             // Don't show toast - progress modal will show status
           } else {
             // Synchronous completion
@@ -481,6 +481,7 @@ export default function Clusters() {
         message={progressModal.progress.message}
         details={progressModal.progress.details}
         taskId={progressModal.taskId || undefined}
+        functionId={progressModal.functionId}
         onClose={() => {
           progressModal.closeModal();
           // Reload once when modal closes if task was completed
diff --git a/frontend/src/pages/Planner/Ideas.tsx b/frontend/src/pages/Planner/Ideas.tsx
index 2642b56b..2bdafd06 100644
--- a/frontend/src/pages/Planner/Ideas.tsx
+++ b/frontend/src/pages/Planner/Ideas.tsx
@@ -400,6 +400,7 @@ export default function Ideas() {
         message={progressModal.progress.message}
         details={progressModal.progress.details}
         taskId={progressModal.taskId || undefined}
+        functionId={progressModal.functionId}
         onClose={() => {
           const wasCompleted = progressModal.progress.status === 'completed';
           progressModal.closeModal();
diff --git a/frontend/src/pages/Planner/Keywords.tsx b/frontend/src/pages/Planner/Keywords.tsx
index bcc48e1b..43ab7bbe 100644
--- a/frontend/src/pages/Planner/Keywords.tsx
+++ b/frontend/src/pages/Planner/Keywords.tsx
@@ -395,7 +395,7 @@ export default function Keywords() {
             }]);
             // Async task - open progress modal
             hasReloadedRef.current = false;
-            progressModal.openModal(result.task_id, 'Auto-Clustering Keywords');
+            progressModal.openModal(result.task_id, 'Auto-Clustering Keywords', 'ai-auto-cluster-01');
             // Don't show toast - progress modal will show status
           } else {
             // Log success with results
@@ -870,6 +870,7 @@ export default function Keywords() {
         message={progressModal.progress.message}
         details={progressModal.progress.details}
         taskId={progressModal.taskId || undefined}
+        functionId={progressModal.functionId}
         onClose={() => {
           progressModal.closeModal();
           // Reload once when modal closes if task was completed
diff --git a/frontend/src/pages/Writer/Tasks.tsx b/frontend/src/pages/Writer/Tasks.tsx
index ec34327d..7f6a7d31 100644
--- a/frontend/src/pages/Writer/Tasks.tsx
+++ b/frontend/src/pages/Writer/Tasks.tsx
@@ -250,7 +250,7 @@ export default function Tasks() {
               data: { task_id: result.task_id, message: result.message },
             }]);
             // Async task - show progress modal
-            progressModal.openModal(result.task_id, 'Generating Content');
+            progressModal.openModal(result.task_id, 'Generating Content', 'ai-generate-content-03');
             toast.success('Content generation started');
           } else {
             // Log success with results
@@ -635,6 +635,7 @@ export default function Tasks() {
         message={progressModal.progress.message}
         details={progressModal.progress.details}
         taskId={progressModal.taskId || undefined}
+        functionId={progressModal.functionId}
         onClose={() => {
           const wasCompleted = progressModal.progress.status === 'completed';
           progressModal.closeModal();