automation and ai and some planning and fixes adn docs reorg

frontend/src/pages/Planner/Clusters.tsx

@@ -37,7 +37,11 @@ export default function Clusters() {
   // Data state
   const [clusters, setClusters] = useState<Cluster[]>([]);
   const [loading, setLoading] = useState(true);
-  
+
+  // Total counts for footer widget (not page-filtered)
+  const [totalWithIdeas, setTotalWithIdeas] = useState(0);
+  const [totalReady, setTotalReady] = useState(0);
+
   // Filter state
   const [searchTerm, setSearchTerm] = useState('');
   const [statusFilter, setStatusFilter] = useState('');
@@ -75,6 +79,34 @@ export default function Clusters() {
   const progressModal = useProgressModal();
   const hasReloadedRef = useRef(false);
 
+  // Load total metrics for footer widget (not affected by pagination)
+  const loadTotalMetrics = useCallback(async () => {
+    try {
+      // Get clusters with status='mapped' (those that have ideas)
+      const mappedRes = await fetchClusters({
+        page_size: 1,
+        ...(activeSector?.id && { sector_id: activeSector.id }),
+        status: 'mapped',
+      });
+      setTotalWithIdeas(mappedRes.count || 0);
+
+      // Get clusters with status='new' (those that are ready for ideas)
+      const newRes = await fetchClusters({
+        page_size: 1,
+        ...(activeSector?.id && { sector_id: activeSector.id }),
+        status: 'new',
+      });
+      setTotalReady(newRes.count || 0);
+    } catch (error) {
+      console.error('Error loading total metrics:', error);
+    }
+  }, [activeSector]);
+
+  // Load total metrics when sector changes
+  useEffect(() => {
+    loadTotalMetrics();
+  }, [loadTotalMetrics]);
+
   // Load clusters - wrapped in useCallback to prevent infinite loops
   const loadClusters = useCallback(async () => {
     setLoading(true);
@@ -494,17 +526,17 @@ export default function Clusters() {
           submoduleColor: 'green',
           metrics: [
             { label: 'Clusters', value: totalCount },
-            { label: 'With Ideas', value: clusters.filter(c => (c.ideas_count || 0) > 0).length, percentage: `${totalCount > 0 ? Math.round((clusters.filter(c => (c.ideas_count || 0) > 0).length / totalCount) * 100) : 0}%` },
+            { label: 'With Ideas', value: totalWithIdeas, percentage: `${totalCount > 0 ? Math.round((totalWithIdeas / totalCount) * 100) : 0}%` },
             { label: 'Keywords', value: clusters.reduce((sum, c) => sum + (c.keywords_count || 0), 0) },
-            { label: 'Ready', value: clusters.filter(c => (c.ideas_count || 0) === 0).length },
+            { label: 'Ready', value: totalReady },
           ],
           progress: {
-            value: totalCount > 0 ? Math.round((clusters.filter(c => (c.ideas_count || 0) > 0).length / totalCount) * 100) : 0,
+            value: totalCount > 0 ? Math.round((totalWithIdeas / totalCount) * 100) : 0,
             label: 'Have Ideas',
             color: 'green',
           },
-          hint: clusters.filter(c => (c.ideas_count || 0) === 0).length > 0
-            ? `${clusters.filter(c => (c.ideas_count || 0) === 0).length} clusters ready for idea generation`
+          hint: totalReady > 0
+            ? `${totalReady} clusters ready for idea generation`
             : 'All clusters have ideas!',
         }}
         moduleStats={{