From bb4fe9d6c11fa5af088246c8ba1a35bb6da6178b Mon Sep 17 00:00:00 2001 From: Desktop Date: Mon, 10 Nov 2025 23:36:14 +0500 Subject: [PATCH] Create IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md --- IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md | 649 +++++++++++++++++++++++++ 1 file changed, 649 insertions(+) create mode 100644 IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md diff --git a/IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md b/IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md new file mode 100644 index 00000000..e0fa6146 --- /dev/null +++ b/IGNY8_AI_UNIFICATION_MIGRATION_PLAN.md @@ -0,0 +1,649 @@ +# IGNY8 AI System Unification — Complete Migration Plan + +**Date:** 2024-12-19 +**Goal:** Unify all AI functions into single structure, remove all redundancy, implement checklist-style progress UI +**Estimated Time:** 5 stages, ~2-3 days total + +--- + +## Overview + +This migration plan unifies the IGNY8 AI system by: +1. Standardizing backend progress messages with input data +2. Implementing checklist-style progress UI with 3 states (pending/in-progress/completed) +3. Migrating all views to use unified `run_ai_task` entrypoint +4. Removing duplicate code and deprecated files +5. Final cleanup and verification + +--- + +## Stage 1: Backend — Standardize Progress Messages with Input Data + +**Goal:** Update `AIEngine` to send user-friendly messages with actual input data +**Files to Modify:** `backend/igny8_core/ai/engine.py` +**Estimated Time:** 1-2 hours + +### Step 1.1: Add Helper Methods to AIEngine + +**File:** `backend/igny8_core/ai/engine.py` + +Add these helper methods to the `AIEngine` class (after `__init__` method): + +```python +def _get_input_description(self, function_name: str, payload: dict, count: int) -> str: + """Get user-friendly input description""" + if function_name == 'auto_cluster': + return f"{count} keyword{'s' if count != 1 else ''}" + elif function_name == 'generate_ideas': + return f"{count} cluster{'s' if count != 1 else ''}" + elif function_name == 'generate_content': + return f"{count} task{'s' if count != 1 else ''}" + elif function_name == 'generate_images': + return f"{count} task{'s' if count != 1 else ''}" + return f"{count} item{'s' if count != 1 else ''}" + +def _get_prep_message(self, function_name: str, count: int, data: Any) -> str: + """Get user-friendly prep message""" + if function_name == 'auto_cluster': + return f"Loading {count} keyword{'s' if count != 1 else ''}" + elif function_name == 'generate_ideas': + return f"Loading {count} cluster{'s' if count != 1 else ''}" + elif function_name == 'generate_content': + return f"Preparing {count} content idea{'s' if count != 1 else ''}" + elif function_name == 'generate_images': + return f"Extracting image prompts from {count} task{'s' if count != 1 else ''}" + return f"Preparing {count} item{'s' if count != 1 else ''}" + +def _get_ai_call_message(self, function_name: str, count: int) -> str: + """Get user-friendly AI call message""" + if function_name == 'auto_cluster': + return f"Grouping {count} keyword{'s' if count != 1 else ''} into clusters" + elif function_name == 'generate_ideas': + return f"Generating content ideas for {count} cluster{'s' if count != 1 else ''}" + elif function_name == 'generate_content': + return f"Writing article{'s' if count != 1 else ''} with AI" + elif function_name == 'generate_images': + return f"Creating image{'s' if count != 1 else ''} with AI" + return f"Processing with AI" + +def _get_parse_message(self, function_name: str) -> str: + """Get user-friendly parse message""" + if function_name == 'auto_cluster': + return "Organizing clusters" + elif function_name == 'generate_ideas': + return "Structuring outlines" + elif function_name == 'generate_content': + return "Formatting content" + elif function_name == 'generate_images': + return "Processing images" + return "Processing results" + +def _get_save_message(self, function_name: str, count: int) -> str: + """Get user-friendly save message""" + if function_name == 'auto_cluster': + return f"Saving {count} cluster{'s' if count != 1 else ''}" + elif function_name == 'generate_ideas': + return f"Saving {count} idea{'s' if count != 1 else ''}" + elif function_name == 'generate_content': + return f"Saving {count} article{'s' if count != 1 else ''}" + elif function_name == 'generate_images': + return f"Saving {count} image{'s' if count != 1 else ''}" + return f"Saving {count} item{'s' if count != 1 else ''}" +``` + +### Step 1.2: Update execute() Method to Use Helper Methods + +**File:** `backend/igny8_core/ai/engine.py` + +In the `execute()` method, replace step tracking messages: + +**Replace lines 48-57 (INIT phase):** +```python +# OLD: +self.console_tracker.prep("Validating input payload") +validated = fn.validate(payload, self.account) +if not validated['valid']: + self.console_tracker.error('ValidationError', validated['error']) + return self._handle_error(validated['error'], fn) + +self.console_tracker.prep("Validation complete") +self.step_tracker.add_request_step("INIT", "success", "Validation complete") +self.tracker.update("INIT", 10, "Validation complete", meta=self.step_tracker.get_meta()) + +# NEW: +# Extract input data for user-friendly messages +ids = payload.get('ids', []) +input_count = len(ids) if ids else 0 +input_description = self._get_input_description(function_name, payload, input_count) + +self.console_tracker.prep(f"Validating {input_description}") +validated = fn.validate(payload, self.account) +if not validated['valid']: + self.console_tracker.error('ValidationError', validated['error']) + return self._handle_error(validated['error'], fn) + +validation_message = f"Validating {input_description}" +self.console_tracker.prep("Validation complete") +self.step_tracker.add_request_step("INIT", "success", validation_message) +self.tracker.update("INIT", 10, validation_message, meta=self.step_tracker.get_meta()) +``` + +**Replace lines 59-79 (PREP phase):** +```python +# OLD: +self.console_tracker.prep("Loading data from database") +data = fn.prepare(payload, self.account) +# ... existing data_count logic ... +self.console_tracker.prep(f"Building prompt from {data_count} items") +prompt = fn.build_prompt(data, self.account) +self.console_tracker.prep(f"Prompt built: {len(prompt)} characters") +self.step_tracker.add_request_step("PREP", "success", f"Loaded {data_count} items, built prompt ({len(prompt)} chars)") +self.tracker.update("PREP", 25, f"Data prepared: {data_count} items", meta=self.step_tracker.get_meta()) + +# NEW: +prep_message = self._get_prep_message(function_name, input_count, payload) +self.console_tracker.prep(prep_message) +data = fn.prepare(payload, self.account) +# ... existing data_count logic ... +prompt = fn.build_prompt(data, self.account) +self.console_tracker.prep(f"Prompt built: {len(prompt)} characters") +self.step_tracker.add_request_step("PREP", "success", prep_message) +self.tracker.update("PREP", 25, prep_message, meta=self.step_tracker.get_meta()) +``` + +**Replace lines 136-141 (AI_CALL phase):** +```python +# OLD: +self.step_tracker.add_response_step( + "AI_CALL", + "success", + f"Calling {model or 'default'} model..." +) +self.tracker.update("AI_CALL", 30, f"Sending to {model or 'default'}...", meta=self.step_tracker.get_meta()) + +# NEW: +ai_call_message = self._get_ai_call_message(function_name, data_count) +self.step_tracker.add_response_step("AI_CALL", "success", ai_call_message) +self.tracker.update("AI_CALL", 50, ai_call_message, meta=self.step_tracker.get_meta()) +``` + +**Find PARSE phase (around line 200-210) and replace:** +```python +# OLD: +self.step_tracker.add_response_step("PARSE", "success", "Parsing response...") +self.tracker.update("PARSE", 70, "Parsing response...", meta=self.step_tracker.get_meta()) + +# NEW: +parse_message = self._get_parse_message(function_name) +self.step_tracker.add_response_step("PARSE", "success", parse_message) +self.tracker.update("PARSE", 70, parse_message, meta=self.step_tracker.get_meta()) +``` + +**Find SAVE phase (around line 250-260) and replace:** +```python +# OLD: +self.step_tracker.add_response_step("SAVE", "success", "Saving results...") +self.tracker.update("SAVE", 85, "Saving results...", meta=self.step_tracker.get_meta()) + +# NEW: +save_message = self._get_save_message(function_name, data_count) +self.step_tracker.add_response_step("SAVE", "success", save_message) +self.tracker.update("SAVE", 85, save_message, meta=self.step_tracker.get_meta()) +``` + +### Step 1.3: Remove Technical Debug Messages + +**File:** `backend/igny8_core/ai/engine.py` + +Remove or comment out lines 115-124 (model configuration tracking in step tracker): +```python +# REMOVE these lines (keep console logging, but not step tracker): +# self.step_tracker.add_request_step( +# "PREP", +# "success", +# f"AI model in settings: {model_from_integration or 'Not set'}" +# ) +# self.step_tracker.add_request_step( +# "PREP", +# "success", +# f"AI model selected for request: {model or 'default'}" +# ) +``` + +### Verification Checklist for Stage 1: +- [ ] Helper methods added to `AIEngine` class +- [ ] All phase messages updated to use helper methods +- [ ] Messages include actual input counts (e.g., "Validating 5 keywords") +- [ ] No technical terms like "database" or "parsing" in user-facing messages +- [ ] Test: Run `auto_cluster` and verify messages in step logs + +--- + +## Stage 2: Frontend — Implement Checklist-Style Progress Modal + +**Goal:** Replace progress bar with checklist UI showing 3 states (pending/in-progress/completed) +**Files to Modify:** `frontend/src/components/common/ProgressModal.tsx` +**Files to Create:** None +**Estimated Time:** 2-3 hours + +### Step 2.1: Replace ProgressModal Component + +**File:** `frontend/src/components/common/ProgressModal.tsx` + +Replace the entire file with the new checklist-style implementation (see previous response for full code). + +Key changes: +- Remove progress bar component +- Add checklist-style step display +- Add 3-state logic (pending/in-progress/completed) +- Add success alert in same modal when completed +- Use step logs to determine current phase + +### Step 2.2: Update useProgressModal Hook (Simplify) + +**File:** `frontend/src/hooks/useProgressModal.ts` + +Remove all `aiRequestLogsStore` references: +- Remove lines 501-642 (all store-related code) +- Keep only polling logic and state management +- Simplify step mapping logic + +### Verification Checklist for Stage 2: +- [ ] ProgressModal shows checklist instead of progress bar +- [ ] Steps show as pending (gray/disabled) initially +- [ ] Steps show as in-progress (blue/spinner) when active +- [ ] Steps show as completed (green/checkmark) when done +- [ ] Success alert appears in same modal when completed +- [ ] No errors in browser console +- [ ] Test: Run `auto_cluster` and verify checklist UI + +--- + +## Stage 3: Migrate Views to Unified Entrypoint + +**Goal:** Update all views to use `run_ai_task` instead of legacy task functions +**Files to Modify:** +- `backend/igny8_core/modules/writer/views.py` +- `backend/igny8_core/modules/planner/views.py` (verify already migrated) + +**Estimated Time:** 2-3 hours + +### Step 3.1: Migrate auto_generate_content View + +**File:** `backend/igny8_core/modules/writer/views.py` + +**Replace lines 180-228** (the entire try/except block for Celery task): + +```python +# OLD: +from .tasks import auto_generate_content_task +if hasattr(auto_generate_content_task, 'delay'): + task = auto_generate_content_task.delay(ids, account_id=account_id) + # ... rest of old code + +# NEW: +from igny8_core.ai.tasks import run_ai_task +from kombu.exceptions import OperationalError as KombuOperationalError + +try: + if hasattr(run_ai_task, 'delay'): + task = run_ai_task.delay( + function_name='generate_content', + payload={'ids': ids}, + account_id=account_id + ) + logger.info(f"Task queued: {task.id}") + return Response({ + 'success': True, + 'task_id': str(task.id), + 'message': 'Content generation started' + }, status=status.HTTP_200_OK) + else: + # Celery not available - execute synchronously + logger.info("auto_generate_content: Executing synchronously (Celery not available)") + result = run_ai_task( + function_name='generate_content', + payload={'ids': ids}, + account_id=account_id + ) + if result.get('success'): + return Response({ + 'success': True, + 'tasks_updated': result.get('count', 0), + 'message': 'Content generated successfully' + }, status=status.HTTP_200_OK) + else: + return Response({ + 'error': result.get('error', 'Content generation failed'), + 'type': 'TaskExecutionError' + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) +except KombuOperationalError as e: + logger.error(f"Celery connection error: {str(e)}") + return Response({ + 'error': 'Task queue unavailable. Please try again.', + 'type': 'QueueError' + }, status=status.HTTP_503_SERVICE_UNAVAILABLE) +except Exception as e: + logger.error(f"Error queuing content generation task: {str(e)}", exc_info=True) + return Response({ + 'error': f'Failed to start content generation: {str(e)}', + 'type': 'TaskError' + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) +``` + +### Step 3.2: Migrate auto_generate_images View + +**File:** `backend/igny8_core/modules/writer/views.py` + +**Replace lines 358-377** (the entire try/except block): + +```python +# OLD: +from .tasks import auto_generate_images_task +if hasattr(auto_generate_images_task, 'delay'): + task = auto_generate_images_task.delay(task_ids, account_id=account_id) + # ... rest of old code + +# NEW: +from igny8_core.ai.tasks import run_ai_task +from kombu.exceptions import OperationalError as KombuOperationalError + +try: + if hasattr(run_ai_task, 'delay'): + task = run_ai_task.delay( + function_name='generate_images', + payload={'ids': task_ids}, + account_id=account_id + ) + return Response({ + 'success': True, + 'task_id': str(task.id), + 'message': 'Image generation started' + }, status=status.HTTP_200_OK) + else: + # Celery not available - execute synchronously + result = run_ai_task( + function_name='generate_images', + payload={'ids': task_ids}, + account_id=account_id + ) + if result.get('success'): + return Response({ + 'success': True, + 'images_created': result.get('count', 0), + 'message': result.get('message', 'Image generation completed') + }, status=status.HTTP_200_OK) + else: + return Response({ + 'error': result.get('error', 'Image generation failed'), + 'type': 'TaskExecutionError' + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) +except KombuOperationalError as e: + logger.error(f"Celery connection error: {str(e)}") + return Response({ + 'error': 'Task queue unavailable. Please try again.', + 'type': 'QueueError' + }, status=status.HTTP_503_SERVICE_UNAVAILABLE) +except Exception as e: + logger.error(f"Error queuing image generation task: {str(e)}", exc_info=True) + return Response({ + 'error': f'Failed to start image generation: {str(e)}', + 'type': 'TaskError' + }, status=status.HTTP_500_INTERNAL_SERVER_ERROR) +``` + +### Step 3.3: Verify Planner Views Already Migrated + +**File:** `backend/igny8_core/modules/planner/views.py` + +Verify that `auto_cluster` and `auto_generate_ideas` already use `run_ai_task`: +- [ ] `auto_cluster` uses `run_ai_task` (line 480) +- [ ] `auto_generate_ideas` uses `run_ai_task` (line 756) + +### Verification Checklist for Stage 3: +- [ ] `auto_generate_content` uses `run_ai_task` +- [ ] `auto_generate_images` uses `run_ai_task` +- [ ] All views return consistent response format +- [ ] Error handling is consistent +- [ ] Test: Generate content and verify it works +- [ ] Test: Generate images and verify it works + +--- + +## Stage 4: Remove Duplicate Code and Deprecated Files + +**Goal:** Clean up duplicate constants, remove deprecated files, simplify code +**Files to Delete:** 6 files +**Files to Modify:** 5 files +**Estimated Time:** 2-3 hours + +### Step 4.1: Remove Duplicate Constants + +**File:** `backend/igny8_core/utils/ai_processor.py` + +**Replace lines 18-44** (duplicate constants): + +```python +# OLD: +MODEL_RATES = { ... } +IMAGE_MODEL_RATES = { ... } +VALID_OPENAI_IMAGE_MODELS = { ... } +VALID_SIZES_BY_MODEL = { ... } + +# NEW: +from igny8_core.ai.constants import ( + MODEL_RATES, + IMAGE_MODEL_RATES, + VALID_OPENAI_IMAGE_MODELS, + VALID_SIZES_BY_MODEL, + DEFAULT_AI_MODEL, + JSON_MODE_MODELS, +) +``` + +### Step 4.2: Remove response_steps Parameter + +**File:** `backend/igny8_core/utils/ai_processor.py` + +Remove `response_steps` parameter from all methods: +- Find all method signatures with `response_steps=None` (lines 1064, 1135, etc.) +- Remove the parameter +- Remove all `response_steps.append()` calls (lines 1135-1299) +- Keep the file (still used by legacy code temporarily) + +### Step 4.3: Simplify task_progress Endpoint + +**File:** `backend/igny8_core/modules/system/integration_views.py` + +**Simplify `task_progress()` method** (lines 734-1163): + +Replace complex extraction logic with simple meta retrieval: +```python +# In task_progress method, replace lines 784-1100 with: +meta = {} +request_steps = [] +response_steps = [] + +try: + if hasattr(task, 'info') and task.info: + if isinstance(task.info, dict): + meta = task.info.get('meta', {}) + if isinstance(meta, dict): + request_steps = meta.get('request_steps', []) + response_steps = meta.get('response_steps', []) +except Exception as e: + logger.debug(f"Error extracting meta: {str(e)}") + +# Use request_steps and response_steps in response +``` + +### Step 4.4: Remove Deprecated Store References + +**File:** `frontend/src/services/api.ts` + +Remove all `aiRequestLogsStore` imports and references: +- Remove line 4: `import { useAIRequestLogsStore } from '../store/aiRequestLogsStore';` +- Remove lines 579, 601, 671-672, 730-731, 812-813, 1185-1186, 1265-1266 (all store references) + +**File:** `frontend/src/hooks/useProgressModal.ts` + +Remove all `aiRequestLogsStore` references (lines 501-642) + +**File:** `frontend/src/templates/TablePageTemplate.tsx` + +Remove commented import (line 44) + +### Step 4.5: Delete Deprecated Files + +**Delete these files:** + +1. `backend/igny8_core/modules/planner/tasks.py` + - Already deprecated, no longer used + +2. `backend/igny8_core/modules/writer/tasks.py` + - No longer used after Stage 3 migration + +3. `backend/igny8_core/ai/processor.py` + - Deprecated wrapper, redirects to AICore + +4. `frontend/src/store/aiRequestLogsStore.ts` + - Deprecated debug store + +5. `frontend/src/components/debug/ResourceDebugOverlay.tsx` + - Optional: Delete if not needed + +6. `frontend/src/components/debug/ResourceDebugToggle.tsx` + - Optional: Delete if not needed (or keep if still used) + +### Verification Checklist for Stage 4: +- [ ] Duplicate constants removed from `ai_processor.py` +- [ ] `response_steps` parameter removed from all methods +- [ ] `task_progress` endpoint simplified +- [ ] All deprecated store references removed +- [ ] Deprecated files deleted +- [ ] No import errors after deletions +- [ ] Test: Verify all AI functions still work + +--- + +## Stage 5: Final Cleanup and Verification + +**Goal:** Final testing, documentation, and cleanup +**Estimated Time:** 1-2 hours + +### Step 5.1: Remove Debug Overlay from Layout (if deleted) + +**File:** `frontend/src/layout/AppLayout.tsx` + +If you deleted debug components, remove: +- Line 12: `import ResourceDebugOverlay from "../components/debug/ResourceDebugOverlay";` +- Lines 166-180: Debug toggle listener +- Lines 197-198: `` + +### Step 5.2: Update Function Metadata + +**File:** `backend/igny8_core/ai/functions/auto_cluster.py` + +Verify `get_metadata()` returns correct phase messages (should already be correct) + +**File:** `backend/igny8_core/ai/functions/generate_ideas.py` + +Verify `get_metadata()` returns correct phase messages + +**File:** `backend/igny8_core/ai/functions/generate_content.py` + +Verify `get_metadata()` returns correct phase messages + +**File:** `backend/igny8_core/ai/functions/generate_images.py` + +Verify `get_metadata()` returns correct phase messages + +### Step 5.3: Comprehensive Testing + +Test each AI function end-to-end: + +**Test 1: Keyword Clustering** +- [ ] Select 5-10 keywords +- [ ] Click "Auto Cluster" +- [ ] Verify checklist shows: "Validating 5 keywords" → "Loading 5 keywords" → etc. +- [ ] Verify success message: "Clustering complete — keywords grouped into meaningful clusters." +- [ ] Verify clusters created in database + +**Test 2: Idea Generation** +- [ ] Select 1-2 clusters +- [ ] Click "Generate Ideas" +- [ ] Verify checklist shows correct steps +- [ ] Verify success message: "Content ideas and outlines created successfully." +- [ ] Verify ideas created in database + +**Test 3: Content Generation** +- [ ] Select 1-2 tasks +- [ ] Click "Generate Content" +- [ ] Verify checklist shows correct steps +- [ ] Verify success message: "Article drafted successfully." +- [ ] Verify content saved to tasks + +**Test 4: Image Generation** +- [ ] Select 1-2 tasks with content +- [ ] Click "Generate Images" +- [ ] Verify checklist shows correct steps +- [ ] Verify success message: "Images created and saved successfully." +- [ ] Verify images created in database + +### Step 5.4: Code Review Checklist + +- [ ] All AI functions use `run_ai_task` entrypoint +- [ ] All progress messages include input data +- [ ] No duplicate constants +- [ ] No deprecated code references +- [ ] Frontend shows checklist UI correctly +- [ ] Success messages appear in modal +- [ ] No console errors +- [ ] No TypeScript errors +- [ ] No Python linting errors + +### Step 5.5: Documentation Update + +Update any relevant documentation: +- [ ] Update API documentation if needed +- [ ] Update developer guide if needed +- [ ] Mark this migration as complete + +--- + +## Rollback Plan + +If issues occur during migration: + +1. **Stage 1-2 Issues:** Revert `engine.py` and `ProgressModal.tsx` changes +2. **Stage 3 Issues:** Revert view changes, keep using legacy tasks temporarily +3. **Stage 4 Issues:** Restore deleted files from git history +4. **Stage 5 Issues:** Fix specific issues without rolling back + +--- + +## Success Criteria + +Migration is complete when: +- ✅ All AI functions use unified `run_ai_task` entrypoint +- ✅ All progress messages are user-friendly with input data +- ✅ Frontend shows checklist-style progress UI +- ✅ Success messages appear in modal +- ✅ No duplicate code remains +- ✅ All deprecated files deleted +- ✅ All tests pass +- ✅ No console/terminal errors + +--- + +## Notes + +- **Backward Compatibility:** Legacy code in `utils/ai_processor.py` is kept temporarily for any remaining references +- **Debug Components:** ResourceDebugOverlay can be kept if still needed for other debugging +- **Testing:** Test each stage before moving to next stage +- **Git:** Commit after each stage for easy rollback + +--- + +**End of Migration Plan** +