fetching from all authors, instead of just the first1

src/pages/trend/index.tsx

@@ -79,12 +79,9 @@ const TrendPage: React.FC<TrendProps> = ({ monthlyData = [], totalData = [], det
   const providerData = Object.fromEntries(
     Object.keys(PROVIDERS_MAP).map(provider => {
       const providerMonthlyData = monthlyData.filter(d => {
-        const isDataset = d.name?.toLowerCase().includes('dataset') || 
-                         d.name?.toLowerCase().includes('corpus') ||
-                         d.name?.toLowerCase().includes('data');
         const matchesContentType = contentType === 'all' || 
-                                 (contentType === 'datasets' && isDataset) ||
-                                 (contentType === 'models' && !isDataset);
+                                 (contentType === 'datasets' && d.isDataset) ||
+                                 (contentType === 'models' && !d.isDataset);
         return d.provider === provider && matchesContentType;
       });
       return [provider, providerMonthlyData || []];
@@ -94,12 +91,9 @@ const TrendPage: React.FC<TrendProps> = ({ monthlyData = [], totalData = [], det
   // Filter and group detailed model data
   const filteredModels = (detailedData || [])
     .filter(model => {
-      const isDataset = model.name.toLowerCase().includes('dataset') || 
-                       model.name.toLowerCase().includes('corpus') ||
-                       model.name.toLowerCase().includes('data');
       const matchesContentType = contentType === 'all' || 
-                               (contentType === 'datasets' && isDataset) ||
-                               (contentType === 'models' && !isDataset);
+                               (contentType === 'datasets' && model.isDataset) ||
+                               (contentType === 'models' && !model.isDataset);
       
       return model.likes >= minLikes && 
         (selectedProviders.length === 0 || selectedProviders.includes(model.provider)) &&

src/utils/modelData.ts

@@ -4,6 +4,7 @@ export interface ModelData {
   createdAt: string;
   id: string;
   likes?: number;
+  isDataset?: boolean;
 }
 
 export interface Activity {
@@ -20,6 +21,8 @@ export interface MonthlyActivity {
   date: string; // YYYY-MM format
   count: number;
   provider: string;
+  isDataset?: boolean;
+  name?: string;
 }
 
 export interface DetailedModelData extends ModelData {
@@ -29,6 +32,7 @@ export interface DetailedModelData extends ModelData {
   monthKey: string; // YYYY-MM
   provider: string;
   sortKey: string; // YYYY-MM
+  isDataset: boolean;
 }
 
 // Generates calendar data from model data
@@ -112,53 +116,33 @@ export const aggregateCalendarData = (calendarData: CalendarData): Activity[] =>
 };
 
 export const generateMonthlyData = (modelData: ModelData[]): MonthlyActivity[] => {
-  if (!Array.isArray(modelData)) {
-    console.error('Model data is not an array:', modelData);
-    return [];
-  }
-
-  // Create a map to store counts for each provider and month
-  const monthlyMap: Record<string, Record<string, number>> = {};
+  const monthlyData: Record<string, Record<string, MonthlyActivity>> = {};
 
-  modelData.forEach(item => {
-    const [org] = item.id.split('/');
-    const provider = Object.entries(PROVIDERS_MAP).find(([_, info]) =>
-      info.authors.includes(org)
-    )?.[0];
+  modelData.forEach(model => {
+    const date = new Date(model.createdAt);
+    const monthKey = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`;
+    const provider = model.provider || 'unknown';
 
-    if (provider) {
-      const date = item.createdAt.substring(0, 7); // Get YYYY-MM
-      if (!monthlyMap[provider]) {
-        monthlyMap[provider] = {};
-      }
-      monthlyMap[provider][date] = (monthlyMap[provider][date] || 0) + 1;
+    if (!monthlyData[monthKey]) {
+      monthlyData[monthKey] = {};
     }
-  });
-
-  // Convert the map to an array of monthly activities
-  const monthlyActivities: MonthlyActivity[] = [];
-  
-  // Get all unique months across all providers
-  const allMonths = new Set<string>();
-  Object.values(monthlyMap).forEach(providerData => {
-    Object.keys(providerData).forEach(month => allMonths.add(month));
-  });
 
-  // Sort months chronologically
-  const sortedMonths = Array.from(allMonths).sort();
+    if (!monthlyData[monthKey][provider]) {
+      monthlyData[monthKey][provider] = {
+        date: monthKey,
+        count: 0,
+        provider,
+        isDataset: model.isDataset,
+        name: model.name
+      };
+    }
 
-  // Create entries for each provider and month
-  Object.entries(monthlyMap).forEach(([provider, data]) => {
-    sortedMonths.forEach(month => {
-      monthlyActivities.push({
-        date: month,
-        count: data[month] || 0,
-        provider
-      });
-    });
+    monthlyData[monthKey][provider].count++;
   });
 
-  return monthlyActivities;
+  return Object.values(monthlyData)
+    .flatMap(providerData => Object.values(providerData))
+    .sort((a, b) => a.date.localeCompare(b.date));
 };
 
 const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -182,104 +166,107 @@ async function fetchWithRetry(url: string, retries = 3, delayMs = 1000): Promise
   throw new Error('Max retries reached');
 }
 
-export async function fetchAllModelData(): Promise<ModelData[]> {
-  console.log('Starting to fetch model data...');
-  const allModels: ModelData[] = [];
-  const failedProviders: string[] = [];
+export const fetchAllModelData = async (): Promise<ModelData[]> => {
+  const allData: ModelData[] = [];
 
   for (const [provider, info] of Object.entries(PROVIDERS_MAP)) {
-    console.log(`Fetching models for provider: ${provider}`);
-    try {
-      const response = await fetchWithRetry(
-        `https://huggingface.co/api/models?author=${info.authors[0]}&sort=likes&direction=-1&limit=100`,
-        3,
-        2000 // 2 second delay between retries
-      );
-      
-      if (!response.ok) {
-        console.error(`Failed to fetch data for ${provider}:`, response.status, response.statusText);
-        failedProviders.push(provider);
-        continue;
+    console.log(`Fetching data for provider: ${provider}`);
+    
+    for (const author of info.authors) {
+      console.log(`  Fetching data for author: ${author}`);
+      try {
+        // Fetch models
+        const modelResponse = await fetchWithRetry(
+          `https://huggingface.co/api/models?author=${author}&sort=likes&direction=-1&limit=10000`,
+          3,
+          2000
+        );
+        const modelData = await modelResponse.json();
+        
+        // Fetch datasets
+        const datasetResponse = await fetchWithRetry(
+          `https://huggingface.co/api/datasets?author=${author}&sort=likes&direction=-1&limit=10000`,
+          3,
+          2000
+        );
+        const datasetData = await datasetResponse.json();
+
+        // Combine and process the data
+        const combinedData = [
+          ...modelData.map((item: any) => ({
+            id: item.id,
+            name: item.id,
+            createdAt: item.createdAt,
+            likes: item.likes,
+            downloads: item.downloads,
+            isDataset: false,
+            provider
+          })),
+          ...datasetData.map((item: any) => ({
+            id: item.id,
+            name: item.id,
+            createdAt: item.createdAt,
+            likes: item.likes,
+            downloads: item.downloads,
+            isDataset: true,
+            provider
+          }))
+        ];
+
+        allData.push(...combinedData);
+        console.log(`    Fetched ${combinedData.length} items (${modelData.length} models, ${datasetData.length} datasets) for ${author}`);
+      } catch (error) {
+        console.error(`Error fetching data for ${provider}/${author}:`, error);
       }
-
-      const models = await response.json();
-      console.log(`Received ${models.length} models for ${provider}`);
       
-      allModels.push(
-        ...models.map((model: any) => {
-          const date = new Date(model.createdAt);
-          return {
-            id: model.id,
-            name: model.modelId,
-            likes: model.likes || 0,
-            createdAt: model.createdAt,
-            provider,
-            monthKey: date.toLocaleDateString('en-US', {
-              year: 'numeric',
-              month: 'long'
-            }),
-            sortKey: `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`
-          };
-        })
-      );
-
-      // Add a small delay between requests to avoid rate limiting
-      await delay(500);
-    } catch (error) {
-      console.error(`Error fetching data for ${provider}:`, error);
-      failedProviders.push(provider);
+      // Add a delay between author requests to avoid rate limiting
+      await delay(1000);
     }
   }
 
-  if (failedProviders.length > 0) {
-    console.warn('Failed to fetch data for providers:', failedProviders);
-  }
-
-  console.log(`Total models fetched: ${allModels.length}`);
-  return allModels.sort((a, b) => {
-    // First sort by sortKey (year-month) in descending order
-    const dateCompare = b.sortKey.localeCompare(a.sortKey);
-    if (dateCompare !== 0) return dateCompare;
-    // Then by likes for models in the same month
-    return b.likes - a.likes;
-  });
-}
+  // Remove duplicates based on id
+  const uniqueData = Array.from(
+    new Map(allData.map(item => [item.id, item])).values()
+  );
+  
+  console.log(`Total unique items fetched: ${uniqueData.length}`);
+  return uniqueData;
+};
 
 export function processDetailedModelData(models: ModelData[]): DetailedModelData[] {
   if (!models || models.length === 0) {
-    console.log('No models to process');
     return [];
   }
 
-  console.log('Processing detailed model data...');
-  console.log('Input models:', models.length);
-  
-  // Group models by month
-  const groupedModels = models.reduce<Record<string, DetailedModelData[]>>((acc, model) => {
-    if (!acc[model.monthKey]) {
-      acc[model.monthKey] = [];
-    }
-    acc[model.monthKey].push(model as DetailedModelData);
-    return acc;
-  }, {});
+  return models.map(model => {
+    const date = new Date(model.createdAt);
+    const monthKey = date.toLocaleDateString('en-US', {
+      year: 'numeric',
+      month: 'long'
+    });
+    const sortKey = `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, '0')}`;
+    const [org] = model.id.split('/');
+    const provider = Object.entries(PROVIDERS_MAP).find(([_, info]) =>
+      info.authors.includes(org)
+    )?.[0] || 'unknown';
 
-  // Sort each month's models by likes
-  Object.values(groupedModels).forEach(monthModels => {
-    monthModels.sort((a, b) => b.likes - a.likes);
+    return {
+      ...model,
+      name: model.name || model.id,
+      likes: model.likes || 0,
+      downloads: model.downloads || 0,
+      monthKey,
+      provider,
+      sortKey,
+      isDataset: model.isDataset || false
+    };
+  }).sort((a, b) => {
+    // First sort by sortKey (year-month) in descending order
+    const dateCompare = b.sortKey.localeCompare(a.sortKey);
+    if (dateCompare !== 0) return dateCompare;
+    // Then by likes for items in the same month
+    return (b.likes || 0) - (a.likes || 0);
   });
-
-  const processed = Object.entries(groupedModels)
-    .sort((a, b) => {
-      const [monthKeyA, modelsA] = a;
-      const [monthKeyB, modelsB] = b;
-      return modelsB[0].sortKey.localeCompare(modelsA[0].sortKey);
-    })
-    .flatMap(([_, models]) => models);
-
-  console.log('Processed models:', processed.length);
-  console.log('Sample model:', processed[0]);
-  
-  return processed;
 }
 
 // Helper function to get total monthly data across all providers