Enhance API response handling and implement unified API standard across multiple modules. Added feature flags for unified exception handling and debug throttling in settings. Updated pagination and response formats in various viewsets to align with the new standard. Improved error handling and response validation in frontend components for better user feedback.

frontend/src/components/dashboard/UsageChartWidget.tsx

@@ -65,13 +65,13 @@ export default function UsageChartWidget() {
         <div>
           <h4 className="text-sm font-semibold mb-2">By Operation</h4>
           <div className="space-y-2">
-            {Object.entries(usageSummary.by_operation).map(([op, stats]) => (
+            {usageSummary.by_operation && Object.entries(usageSummary.by_operation).map(([op, stats]) => (
               <div key={op} className="flex justify-between items-center text-sm">
                 <span className="capitalize">{op.replace('_', ' ')}</span>
                 <span className="font-medium">{stats.credits} credits (${(Number(stats.cost) || 0).toFixed(2)})</span>
               </div>
             ))}
-            {Object.keys(usageSummary.by_operation).length === 0 && (
+            {(!usageSummary.by_operation || Object.keys(usageSummary.by_operation || {}).length === 0) && (
               <div className="text-sm text-gray-500 dark:text-gray-400">No usage data available</div>
             )}
           </div>

frontend/src/pages/Settings/ApiMonitor.tsx

@@ -1,7 +1,7 @@
 import { useState, useEffect, useCallback } from "react";
 import PageMeta from "../../components/common/PageMeta";
 import ComponentCard from "../../components/common/ComponentCard";
-import { API_BASE_URL } from "../../services/api";
+import { API_BASE_URL, fetchContentImages, fetchUsageLimits, fetchAPI } from "../../services/api";
 
 interface EndpointStatus {
   endpoint: string;
@@ -10,6 +10,8 @@ interface EndpointStatus {
   responseTime?: number;
   lastChecked?: string;
   error?: string;
+  apiStatus?: 'healthy' | 'warning' | 'error'; // API endpoint status
+  dataStatus?: 'healthy' | 'warning' | 'error'; // Page data population status
 }
 
 interface EndpointGroup {
@@ -18,6 +20,8 @@ interface EndpointGroup {
     path: string;
     method: string;
     description: string;
+    pageFetchFunction?: () => Promise<any>; // Optional: function to test page data population
+    dataValidator?: (data: any) => boolean; // Optional: function to validate data is populated
   }[];
 }
 
@@ -64,6 +68,19 @@ const endpointGroups: EndpointGroup[] = [
       { path: "/v1/writer/content/", method: "GET", description: "List content" },
       { path: "/v1/writer/content/generate_image_prompts/", method: "POST", description: "Image prompts" },
       { path: "/v1/writer/images/", method: "GET", description: "List images" },
+      { 
+        path: "/v1/writer/images/content_images/", 
+        method: "GET", 
+        description: "Content images",
+        pageFetchFunction: async () => {
+          const data = await fetchContentImages({});
+          return data;
+        },
+        dataValidator: (data: any) => {
+          // Check if data has results array with content
+          return data && data.results && Array.isArray(data.results) && data.results.length > 0;
+        }
+      },
       { path: "/v1/writer/images/generate_images/", method: "POST", description: "AI images" },
     ],
   },
@@ -71,6 +88,21 @@ const endpointGroups: EndpointGroup[] = [
     name: "System & Billing",
     endpoints: [
       { path: "/v1/system/prompts/", method: "GET", description: "List prompts" },
+      { 
+        path: "/v1/system/prompts/by_type/clustering/", 
+        method: "GET", 
+        description: "Get prompt by type",
+        pageFetchFunction: async () => {
+          const response = await fetchAPI('/v1/system/prompts/by_type/clustering/');
+          const data = response?.data || response;
+          return data;
+        },
+        dataValidator: (data: any) => {
+          // Check if prompt data exists and has prompt_value
+          return data && data.prompt_type && (data.prompt_value !== null && data.prompt_value !== undefined);
+        }
+      },
+      { path: "/v1/system/prompts/save/", method: "POST", description: "Save prompt" },
       { path: "/v1/system/author-profiles/", method: "GET", description: "List author profiles" },
       { path: "/v1/system/strategies/", method: "GET", description: "List strategies" },
       { path: "/v1/system/settings/integrations/1/test/", method: "POST", description: "Test integration" },
@@ -78,6 +110,19 @@ const endpointGroups: EndpointGroup[] = [
       { path: "/v1/billing/credits/balance/balance/", method: "GET", description: "Credit balance" },
       { path: "/v1/billing/credits/usage/", method: "GET", description: "Usage logs" },
       { path: "/v1/billing/credits/usage/summary/", method: "GET", description: "Usage summary" },
+      { 
+        path: "/v1/billing/credits/usage/limits/", 
+        method: "GET", 
+        description: "Usage limits",
+        pageFetchFunction: async () => {
+          const data = await fetchUsageLimits();
+          return data;
+        },
+        dataValidator: (data: any) => {
+          // Check if limits array exists and has content
+          return data && data.limits && Array.isArray(data.limits) && data.limits.length > 0;
+        }
+      },
       { path: "/v1/billing/credits/transactions/", method: "GET", description: "Transactions" },
     ],
   },
@@ -127,7 +172,7 @@ export default function ApiMonitor() {
     return saved ? parseInt(saved, 10) : 30;
   });
 
-  const checkEndpoint = useCallback(async (path: string, method: string) => {
+  const checkEndpoint = useCallback(async (path: string, method: string, endpointConfig?: { pageFetchFunction?: () => Promise<any>; dataValidator?: (data: any) => boolean }) => {
     const key = `${method}:${path}`;
     
     // Set checking status
@@ -141,6 +186,8 @@ export default function ApiMonitor() {
     }));
 
     const startTime = Date.now();
+    let apiStatus: 'healthy' | 'warning' | 'error' = 'healthy';
+    let dataStatus: 'healthy' | 'warning' | 'error' = 'healthy';
     
     try {
       // Get token from auth store or localStorage
@@ -209,10 +256,19 @@ export default function ApiMonitor() {
       // Determine status based on response
       let status: 'healthy' | 'warning' | 'error' = 'healthy';
       let responseText = '';
+      let responseData: any = null;
       
-      // Read response body for debugging (but don't log errors for expected 400s)
+      // Read response body for debugging and content validation
       try {
         responseText = await response.text();
+        // Try to parse JSON to check unified API response format
+        if (responseText && responseText.trim().startsWith('{')) {
+          try {
+            responseData = JSON.parse(responseText);
+          } catch (e) {
+            // Not JSON, ignore
+          }
+        }
       } catch (e) {
         // Ignore body read errors
       }
@@ -236,9 +292,56 @@ export default function ApiMonitor() {
           status = 'warning';
         }
       } else if (method === 'GET') {
-        // GET: 2xx = healthy, 401/403 = warning (needs auth), 404 = error, 5xx = error
+        // GET: 2xx = healthy, 401/403 = warning (needs auth), 404 = error, 429 = warning (rate limit), 5xx = error
         if (response.status >= 200 && response.status < 300) {
-          status = 'healthy';
+          // Check unified API response format for errors or empty data
+          if (responseData) {
+            // Check if response has success: false (unified format error)
+            if (responseData.success === false) {
+              status = 'error'; // API returned an error in unified format
+            } else if (responseData.success === true) {
+              // Check if data is empty for endpoints that should return data
+              // These endpoints should have data: {count: X, results: [...]} or data: {...}
+              const shouldHaveData = 
+                path.includes('/content_images/') ||
+                path.includes('/prompts/by_type/') ||
+                path.includes('/usage/limits/') ||
+                path.includes('/prompts/') && !path.includes('/save/');
+              
+              if (shouldHaveData) {
+                // Check if data field exists and has content
+                if (responseData.data === null || responseData.data === undefined) {
+                  status = 'warning'; // Missing data field
+                } else if (Array.isArray(responseData.data) && responseData.data.length === 0) {
+                  // Empty array might be OK for some endpoints, but check if results should exist
+                  if (path.includes('/content_images/') || path.includes('/prompts/by_type/')) {
+                    // These endpoints should return data, empty might indicate a problem
+                    status = 'warning'; // Empty data - might indicate configuration issue
+                  }
+                } else if (typeof responseData.data === 'object' && responseData.data !== null) {
+                  // Check if it's a paginated response with empty results
+                  if (responseData.data.results && Array.isArray(responseData.data.results) && responseData.data.results.length === 0) {
+                    // Empty results might be OK, but for critical endpoints it's a warning
+                    if (path.includes('/content_images/') || path.includes('/prompts/by_type/')) {
+                      status = 'warning'; // Empty results - might indicate data issue
+                    }
+                  } else if (responseData.data.count !== undefined && responseData.data.count === 0) {
+                    // Paginated response with count: 0
+                    if (path.includes('/content_images/') || path.includes('/prompts/by_type/')) {
+                      status = 'warning'; // No data available - might indicate configuration issue
+                    }
+                  }
+                }
+              }
+            }
+          }
+          
+          // If status is still healthy after content checks, keep it healthy
+          if (status === 'healthy') {
+            status = 'healthy'; // HTTP 2xx and valid response = healthy
+          }
+        } else if (response.status === 429) {
+          status = 'warning'; // Rate limited - endpoint exists but temporarily throttled
         } else if (response.status === 401 || response.status === 403) {
           status = 'warning'; // Endpoint exists, needs authentication
         } else if (response.status === 404) {
@@ -249,12 +352,25 @@ export default function ApiMonitor() {
           status = 'warning';
         }
       } else if (method === 'POST') {
-        // POST: 400 = healthy (endpoint exists and validates), 401/403 = warning, 404 = error, 5xx = error
+        // POST: 400 = healthy (endpoint exists and validates), 401/403 = warning, 404 = error, 429 = warning (rate limit), 5xx = error
         if (response.status === 400) {
           // 400 means endpoint exists and validation works - this is healthy for monitoring
-          status = 'healthy';
+          // But check if it's a unified format error response
+          if (responseData && responseData.success === false) {
+            // This is expected for validation errors, so still healthy
+            status = 'healthy';
+          } else {
+            status = 'healthy';
+          }
         } else if (response.status >= 200 && response.status < 300) {
-          status = 'healthy';
+          // Check unified API response format for errors
+          if (responseData && responseData.success === false) {
+            status = 'error'; // API returned an error in unified format
+          } else {
+            status = 'healthy';
+          }
+        } else if (response.status === 429) {
+          status = 'warning'; // Rate limited - endpoint exists but temporarily throttled
         } else if (response.status === 401 || response.status === 403) {
           status = 'warning'; // Endpoint exists, needs authentication
         } else if (response.status === 404) {
@@ -266,13 +382,75 @@ export default function ApiMonitor() {
         }
       }
       
+      // Store API status
+      apiStatus = status;
+      
+      // Now check page data population if pageFetchFunction is configured
+      if (endpointConfig?.pageFetchFunction) {
+        try {
+          const pageData = await endpointConfig.pageFetchFunction();
+          
+          // Validate data using validator if provided
+          if (endpointConfig.dataValidator) {
+            const isValid = endpointConfig.dataValidator(pageData);
+            if (!isValid) {
+              dataStatus = 'warning'; // Data exists but doesn't pass validation (e.g., empty)
+            } else {
+              dataStatus = 'healthy'; // Data is valid and populated
+            }
+          } else {
+            // If no validator, just check if data exists
+            if (pageData === null || pageData === undefined) {
+              dataStatus = 'error';
+            } else {
+              dataStatus = 'healthy';
+            }
+          }
+        } catch (error: any) {
+          // Page fetch function failed
+          dataStatus = 'error';
+          console.warn(`[API Monitor] Page data fetch failed for ${path}:`, error.message);
+        }
+      } else {
+        // No page fetch function configured, data status matches API status
+        dataStatus = apiStatus;
+      }
+      
+      // Combine API and data statuses - both must be healthy for overall healthy
+      // If either is error, overall is error
+      // If either is warning, overall is warning
+      if (apiStatus === 'error' || dataStatus === 'error') {
+        status = 'error';
+      } else if (apiStatus === 'warning' || dataStatus === 'warning') {
+        status = 'warning';
+      } else {
+        status = 'healthy';
+      }
+      
+      // Log warnings/errors for issues detected in response content
+      if (status === 'warning' || status === 'error') {
+        if (responseData) {
+          if (responseData.success === false) {
+            console.warn(`[API Monitor] ${method} ${path}: Unified format error - ${responseData.error || 'Unknown error'}`);
+          } else if (responseData.data === null || responseData.data === undefined) {
+            console.warn(`[API Monitor] ${method} ${path}: Missing data field in response`);
+          } else if (Array.isArray(responseData.data) && responseData.data.length === 0) {
+            console.warn(`[API Monitor] ${method} ${path}: Empty data array returned`);
+          } else if (responseData.data?.results && Array.isArray(responseData.data.results) && responseData.data.results.length === 0) {
+            console.warn(`[API Monitor] ${method} ${path}: Empty results array returned`);
+          } else if (responseData.data?.count === 0) {
+            console.warn(`[API Monitor] ${method} ${path}: No data available (count: 0)`);
+          }
+        }
+      }
+      
       // Suppress console errors for expected monitoring responses
       // Only log real errors (5xx, network errors, or unexpected 4xx for GET endpoints)
       // Don't log expected 400s for POST endpoints (they indicate validation is working)
       const isExpectedResponse = 
         (method === 'POST' && response.status === 400) || // Expected validation error
         (actualMethod === 'OPTIONS' && response.status === 200) || // Expected OPTIONS success
-        (method === 'GET' && response.status >= 200 && response.status < 300); // Expected GET success
+        (method === 'GET' && response.status >= 200 && response.status < 300 && status === 'healthy'); // Expected GET success with valid data
       
       if (!isExpectedResponse && (response.status >= 500 || 
           (method === 'GET' && response.status === 404) ||
@@ -289,6 +467,8 @@ export default function ApiMonitor() {
           status,
           responseTime,
           lastChecked: new Date().toISOString(),
+          apiStatus,
+          dataStatus,
         },
       }));
     } catch (err: any) {
@@ -314,7 +494,10 @@ export default function ApiMonitor() {
     
     // Check all endpoints in parallel (but limit concurrency)
     const allChecks = endpointGroups.flatMap(group =>
-      group.endpoints.map(ep => checkEndpoint(ep.path, ep.method))
+      group.endpoints.map(ep => checkEndpoint(ep.path, ep.method, {
+        pageFetchFunction: ep.pageFetchFunction,
+        dataValidator: ep.dataValidator
+      }))
     );
 
     // Check in batches of 5 to avoid overwhelming the server
@@ -459,13 +642,25 @@ export default function ApiMonitor() {
                               </div>
                             </td>
                             <td className="px-3 py-2">
-                              <span
-                                className={`inline-flex items-center gap-1 text-xs px-2 py-1 rounded ${getStatusBadge(status.status)}`}
-                                title={status.error || status.status}
-                              >
-                                <span>{getStatusIcon(status.status)}</span>
-                                <span className="capitalize">{status.status}</span>
-                              </span>
+                              <div className="flex flex-col gap-1">
+                                <span
+                                  className={`inline-flex items-center gap-1 text-xs px-2 py-1 rounded ${getStatusBadge(status.status)}`}
+                                  title={status.error || status.status}
+                                >
+                                  <span>{getStatusIcon(status.status)}</span>
+                                  <span className="capitalize">{status.status}</span>
+                                </span>
+                                {status.apiStatus && status.dataStatus && endpoint.pageFetchFunction && status.apiStatus !== status.dataStatus && (
+                                  <div className="text-xs space-y-0.5 mt-1">
+                                    <div className={`${getStatusColor(status.apiStatus)}`}>
+                                      API: {status.apiStatus}
+                                    </div>
+                                    <div className={`${getStatusColor(status.dataStatus)}`}>
+                                      Data: {status.dataStatus}
+                                    </div>
+                                  </div>
+                                )}
+                              </div>
                             </td>
                             <td className="px-3 py-2">
                               {status.responseTime ? (

frontend/src/pages/Settings/Status.tsx

@@ -63,8 +63,10 @@ export default function Status() {
 
   const fetchStatus = async () => {
     try {
-      const data = await fetchAPI('/v1/system/status/');
-      setStatus(data);
+      const response = await fetchAPI('/v1/system/status/');
+      // Handle unified API response format: {success: true, data: {...}}
+      const statusData = response?.data || response;
+      setStatus(statusData);
       setError(null);
     } catch (err) {
       setError(err instanceof Error ? err.message : 'Unknown error');

frontend/src/pages/Thinker/Prompts.tsx

@@ -75,7 +75,10 @@ export default function Prompts() {
     try {
       const promises = PROMPT_TYPES.map(async (type) => {
         try {
-          const data = await fetchAPI(`/v1/system/prompts/by_type/${type.key}/`);
+          const response = await fetchAPI(`/v1/system/prompts/by_type/${type.key}/`);
+          // Extract data field from unified API response format
+          // Response format: { success: true, data: {...}, request_id: "..." }
+          const data = response?.data || response;
           return { key: type.key, data };
         } catch (error) {
           console.error(`Error loading prompt ${type.key}:`, error);
@@ -116,7 +119,7 @@ export default function Prompts() {
 
     setSaving({ ...saving, [promptType]: true });
     try {
-      const data = await fetchAPI('/v1/system/prompts/save/', {
+      const response = await fetchAPI('/v1/system/prompts/save/', {
         method: 'POST',
         body: JSON.stringify({
           prompt_type: promptType,
@@ -124,11 +127,15 @@ export default function Prompts() {
         }),
       });
 
-      if (data.success) {
-        toast.success(data.message || 'Prompt saved successfully');
+      // Extract data field from unified API response format
+      // Response format: { success: true, data: {...}, message: "...", request_id: "..." }
+      const data = response?.data || response;
+
+      if (response.success) {
+        toast.success(response.message || 'Prompt saved successfully');
         await loadPrompts(); // Reload to get updated data
       } else {
-        throw new Error(data.error || 'Failed to save prompt');
+        throw new Error(response.error || 'Failed to save prompt');
       }
     } catch (error: any) {
       console.error('Error saving prompt:', error);
@@ -145,18 +152,22 @@ export default function Prompts() {
 
     setSaving({ ...saving, [promptType]: true });
     try {
-      const data = await fetchAPI('/v1/system/prompts/reset/', {
+      const response = await fetchAPI('/v1/system/prompts/reset/', {
         method: 'POST',
         body: JSON.stringify({
           prompt_type: promptType,
         }),
       });
 
-      if (data.success) {
-        toast.success(data.message || 'Prompt reset to default');
+      // Extract data field from unified API response format
+      // Response format: { success: true, data: {...}, message: "...", request_id: "..." }
+      const data = response?.data || response;
+
+      if (response.success) {
+        toast.success(response.message || 'Prompt reset to default');
         await loadPrompts(); // Reload to get default value
       } else {
-        throw new Error(data.error || 'Failed to reset prompt');
+        throw new Error(response.error || 'Failed to reset prompt');
       }
     } catch (error: any) {
       console.error('Error resetting prompt:', error);

frontend/src/services/api.ts

@@ -1114,7 +1114,10 @@ export async function fetchContentImages(filters: ContentImagesFilters = {}): Pr
   if (filters.sector_id) params.append('sector_id', filters.sector_id.toString());
   
   const queryString = params.toString();
-  return fetchAPI(`/v1/writer/images/content_images/${queryString ? `?${queryString}` : ''}`);
+  const response = await fetchAPI(`/v1/writer/images/content_images/${queryString ? `?${queryString}` : ''}`);
+  // Extract data field from unified API response format
+  // Response format: { success: true, data: { count: ..., results: [...] }, request_id: "..." }
+  return response?.data || response;
 }
 
 export async function bulkUpdateImagesStatus(contentId: number, status: string): Promise<{ updated_count: number }> {
@@ -1442,7 +1445,11 @@ export async function fetchUsageSummary(startDate?: string, endDate?: string): P
   if (endDate) params.append('end_date', endDate);
   
   const queryString = params.toString();
-  return fetchAPI(`/v1/billing/credits/usage/summary/${queryString ? `?${queryString}` : ''}`);
+  const response = await fetchAPI(`/v1/billing/credits/usage/summary/${queryString ? `?${queryString}` : ''}`);
+  // Extract data field from unified API response format
+  // Response format: { success: true, data: {...}, request_id: "..." }
+  const summaryData = response?.data || response;
+  return summaryData;
 }
 
 export interface LimitCard {
@@ -1464,7 +1471,10 @@ export async function fetchUsageLimits(): Promise<UsageLimitsResponse> {
   try {
     const response = await fetchAPI('/v1/billing/credits/usage/limits/');
     console.log('Usage limits API response:', response);
-    return response;
+    // Extract data field from unified API response format
+    // Response format: { success: true, data: { limits: [...] }, request_id: "..." }
+    const limitsData = response?.data || response;
+    return limitsData;
   } catch (error) {
     console.error('Error fetching usage limits:', error);
     throw error;