automation and ai and some planning and fixes adn docs reorg

frontend/src/pages/Planner/Keywords.tsx

@@ -45,7 +45,12 @@ export default function Keywords() {
   const [keywords, setKeywords] = useState<Keyword[]>([]);
   const [clusters, setClusters] = useState<Cluster[]>([]);
   const [loading, setLoading] = useState(true);
-  
+
+  // Total counts for footer widget (not page-filtered)
+  const [totalClustered, setTotalClustered] = useState(0);
+  const [totalUnmapped, setTotalUnmapped] = useState(0);
+  const [totalVolume, setTotalVolume] = useState(0);
+
   // Filter state - match Keywords.tsx
   const [searchTerm, setSearchTerm] = useState('');
   const [statusFilter, setStatusFilter] = useState('');
@@ -108,6 +113,44 @@ export default function Keywords() {
     loadClusters();
   }, []);
 
+  // Load total metrics for footer widget (not affected by pagination)
+  const loadTotalMetrics = useCallback(async () => {
+    if (!activeSite) return;
+
+    try {
+      // Get all keywords (total count) - this is already in totalCount from main load
+      // Get keywords with status='mapped' (those that have been mapped to a cluster)
+      const mappedRes = await fetchKeywords({
+        page_size: 1,
+        site_id: activeSite.id,
+        ...(activeSector?.id && { sector_id: activeSector.id }),
+        status: 'mapped',
+      });
+      setTotalClustered(mappedRes.count || 0);
+
+      // Get keywords with status='new' (those that are ready to cluster but haven't been yet)
+      const newRes = await fetchKeywords({
+        page_size: 1,
+        site_id: activeSite.id,
+        ...(activeSector?.id && { sector_id: activeSector.id }),
+        status: 'new',
+      });
+      setTotalUnmapped(newRes.count || 0);
+
+      // Get total volume across all keywords (we need to fetch all or rely on backend aggregation)
+      // For now, we'll just calculate from current data or set to 0
+      // TODO: Backend should provide total volume as an aggregated metric
+      setTotalVolume(0);
+    } catch (error) {
+      console.error('Error loading total metrics:', error);
+    }
+  }, [activeSite, activeSector]);
+
+  // Load total metrics when site/sector changes
+  useEffect(() => {
+    loadTotalMetrics();
+  }, [loadTotalMetrics]);
+
   // Load keywords - wrapped in useCallback to prevent infinite loops
   const loadKeywords = useCallback(async () => {
     setLoading(true);
@@ -712,17 +755,17 @@ export default function Keywords() {
           submoduleColor: 'blue',
           metrics: [
             { label: 'Keywords', value: totalCount },
-            { label: 'Clustered', value: keywords.filter(k => k.cluster_id).length, percentage: `${totalCount > 0 ? Math.round((keywords.filter(k => k.cluster_id).length / totalCount) * 100) : 0}%` },
-            { label: 'Unmapped', value: keywords.filter(k => !k.cluster_id).length },
-            { label: 'Volume', value: `${(keywords.reduce((sum, k) => sum + (k.volume || 0), 0) / 1000).toFixed(1)}K` },
+            { label: 'Clustered', value: totalClustered, percentage: `${totalCount > 0 ? Math.round((totalClustered / totalCount) * 100) : 0}%` },
+            { label: 'Unmapped', value: totalUnmapped },
+            { label: 'Volume', value: totalVolume > 0 ? `${(totalVolume / 1000).toFixed(1)}K` : '-' },
           ],
           progress: {
-            value: totalCount > 0 ? Math.round((keywords.filter(k => k.cluster_id).length / totalCount) * 100) : 0,
+            value: totalCount > 0 ? Math.round((totalClustered / totalCount) * 100) : 0,
             label: 'Clustered',
             color: 'blue',
           },
-          hint: keywords.filter(k => !k.cluster_id).length > 0
-            ? `${keywords.filter(k => !k.cluster_id).length} keywords ready to cluster`
+          hint: totalUnmapped > 0
+            ? `${totalUnmapped} keywords ready to cluster`
             : 'All keywords clustered!',
         }}
         moduleStats={{