Update app.py

app.py

@@ -999,29 +999,45 @@ target_datasets = {
 
 def get_korea_datasets():
     """Korea 관련 데이터셋 검색"""
-    params = {
-        "search": "korea",
-        "full": "True",
-        "limit": 1000
-    }
+    search_terms = ['korea', 'korean', 'kor']  # 검색어 확장
+    all_korea_datasets = []
     
-    try:
-        response = requests.get(
-            "https://huggingface.co/api/datasets",
-            headers={'Authorization': f'Bearer {HF_TOKEN}'},
-            params=params
-        )
+    for term in search_terms:
+        params = {
+            "search": term,
+            "full": "True",
+            "limit": 10000  # 검색 범위 확장
+        }
         
-        if response.status_code == 200:
-            return response.json()
-        else:
-            print(f"Failed to fetch Korea datasets: {response.status_code}")
-            return []
-    except Exception as e:
-        print(f"Error fetching Korea datasets: {str(e)}")
-        return []
+        try:
+            response = requests.get(
+                "https://huggingface.co/api/datasets",
+                headers={'Authorization': f'Bearer {HF_TOKEN}'},
+                params=params
+            )
+            
+            if response.status_code == 200:
+                datasets = response.json()
+                all_korea_datasets.extend(datasets)
+                print(f"Found {len(datasets)} datasets for search term '{term}'")
+            else:
+                print(f"Failed to fetch datasets for term '{term}': {response.status_code}")
+        except Exception as e:
+            print(f"Error fetching datasets for term '{term}': {str(e)}")
+    
+    # 중복 제거
+    seen_ids = set()
+    unique_datasets = []
+    for dataset in all_korea_datasets:
+        dataset_id = dataset.get('id', '')
+        if dataset_id and dataset_id not in seen_ids:
+            seen_ids.add(dataset_id)
+            unique_datasets.append(dataset)
+    
+    print(f"Total unique Korea-related datasets found: {len(unique_datasets)}")
+    return unique_datasets
 
-def get_all_datasets(limit=3000):
+def get_all_datasets(limit=10000):  # 기본 limit 증가
     """모든 데이터셋과 Korea 관련 데이터셋 가져오기"""
     all_datasets = []
     page_size = 1000
@@ -1033,27 +1049,37 @@ def get_all_datasets(limit=3000):
             'offset': offset
         }
         
-        response = requests.get(
-            "https://huggingface.co/api/datasets",
-            headers={'Authorization': f'Bearer {HF_TOKEN}'},
-            params=params
-        )
-        
-        if response.status_code == 200:
-            all_datasets.extend(response.json())
-            print(f"Fetched datasets {offset+1} to {offset+len(response.json())}")
-        else:
-            print(f"Failed to fetch datasets at offset {offset}: {response.status_code}")
+        try:
+            response = requests.get(
+                "https://huggingface.co/api/datasets",
+                headers={'Authorization': f'Bearer {HF_TOKEN}'},
+                params=params
+            )
+            
+            if response.status_code == 200:
+                datasets = response.json()
+                all_datasets.extend(datasets)
+                print(f"Fetched datasets {offset+1} to {offset+len(datasets)}")
+            else:
+                print(f"Failed to fetch datasets at offset {offset}: {response.status_code}")
+                break
+        except Exception as e:
+            print(f"Error fetching datasets at offset {offset}: {str(e)}")
             break
     
     # Korea 검색 결과 추가
     korea_datasets = get_korea_datasets()
     existing_ids = {dataset.get('id', '') for dataset in all_datasets}
     
+    added_count = 0
     for korea_dataset in korea_datasets:
         if korea_dataset.get('id', '') not in existing_ids:
             all_datasets.append(korea_dataset)
             existing_ids.add(korea_dataset.get('id', ''))
+            added_count += 1
+    
+    print(f"Added {added_count} additional Korea-related datasets")
+    print(f"Total datasets: {len(all_datasets)}")
     
     return all_datasets[:limit]
 
@@ -1115,7 +1141,7 @@ def get_datasets_data(progress=gr.Progress()):
                 else:
                     filtered_datasets.append({
                         'id': dataset_id,
-                        'global_rank': 'Not in top 3000',
+                        'global_rank': 'Not in top 10000',
                         'downloads': 0,
                         'likes': 0,
                         'title': 'No Title',
@@ -1154,9 +1180,9 @@ def get_datasets_data(progress=gr.Progress()):
                 xaxis_title="Dataset ID",
                 yaxis_title="Global Rank",
                 yaxis=dict(
-                    ticktext=[f"#{i}" for i in range(1, 3001, 100)],
-                    tickvals=[3001 - i for i in range(1, 3001, 100)],
-                    range=[0, 3000]
+                    ticktext=[f"#{i}" for i in range(1, 10001, 100)],
+                    tickvals=[10001 - i for i in range(1, 10001, 100)],
+                    range=[0, 10000]
                 ),
                 height=800,
                 showlegend=False,