Update app.py

app.py

@@ -989,49 +989,365 @@ def refresh_data():
     else:
         return create_error_plot(), "<div>API 인증이 필요합니다.</div>", pd.DataFrame()
 
+
+
+
+
+# 먼저 target_datasets 딕셔너리를 추가합니다
+target_datasets = {
+    "aiqtech/kolaw": "https://huggingface.co/datasets/aiqtech/kolaw",
+    # 추가 데이터셋은 여기에 계속 추가할 수 있습니다
+}
+
+def get_datasets_data(progress=gr.Progress()):
+    """데이터셋 데이터 가져오기"""
+    def calculate_rank(dataset_id, all_global_datasets, korea_datasets):
+        # 글로벌 순위 확인
+        global_rank = next((idx for idx, d in enumerate(all_global_datasets, 1) 
+                          if d.get('id', '').strip() == dataset_id.strip()), None)
+        
+        # Korea 데이터셋인 경우
+        is_korea = any(d.get('id', '').strip() == dataset_id.strip() for d in korea_datasets)
+        
+        if is_korea:
+            # Korea 데이터셋 중에서의 순위 확인
+            korea_rank = next((idx for idx, d in enumerate(korea_datasets, 1)
+                             if d.get('id', '').strip() == dataset_id.strip()), None)
+            
+            if korea_rank:
+                return min(global_rank or 3001, korea_rank + 1000), True
+        
+        return global_rank if global_rank else 'Not in top 3000', is_korea
+
+    try:
+        progress(0, desc="Fetching datasets...")
+        
+        if not HF_TOKEN:
+            fig = create_error_plot()
+            error_html = """
+            <div style='padding: 20px; background: #fee; border-radius: 10px; margin: 10px 0;'>
+                <h3 style='color: #c00;'>⚠️ API 인증이 필요합니다</h3>
+                <p>HuggingFace API 토큰이 설정되지 않았습니다.</p>
+            </div>
+            """
+            empty_df = pd.DataFrame(columns=['Global Rank', 'Dataset ID', 'Title', 'Downloads', 'Likes', 'Korea Search', 'URL'])
+            return fig, error_html, empty_df
+
+        # 일반 데이터셋과 Korea 관련 데이터셋 가져오기
+        params = {
+            'limit': 3000,
+            'full': 'true'
+        }
+        
+        all_datasets_response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Authorization': f'Bearer {HF_TOKEN}'},
+            params=params
+        )
+        
+        korea_params = {
+            'search': 'korea',
+            'limit': 3000,
+            'full': 'true'
+        }
+        
+        korea_datasets_response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Authorization': f'Bearer {HF_TOKEN}'},
+            params=korea_params
+        )
+        
+        all_global_datasets = all_datasets_response.json()
+        korea_datasets = korea_datasets_response.json()
+        
+        # 시각화를 위한 Figure 생성
+        fig = go.Figure()
+        
+        # 순위 정보 수집
+        filtered_datasets = []
+        for dataset_id in target_datasets.keys():
+            try:
+                normalized_id = dataset_id.strip('/')
+                dataset_url_api = f"https://huggingface.co/api/datasets/{normalized_id}"
+                response# 먼저 target_datasets 딕셔너리를 추가합니다
+
+
+
+target_datasets = {
+    "aiqtech/kolaw": "https://huggingface.co/datasets/aiqtech/kolaw",
+    "heegyu/korquad-v1-v2": "https://huggingface.co/datasets/heegyu/korquad-v1-v2",
+    "nlpai-lab/kowiki": "https://huggingface.co/datasets/nlpai-lab/kowiki",
+    "KETI-AIR/korpora": "https://huggingface.co/datasets/KETI-AIR/korpora",
+    "heegyu/korean-parallel-corpora": "https://huggingface.co/datasets/heegyu/korean-parallel-corpora",
+    "heegyu/korean-hate-speech": "https://huggingface.co/datasets/heegyu/korean-hate-speech",
+    "KETI-AIR/korean-parallel-corpora": "https://huggingface.co/datasets/KETI-AIR/korean-parallel-corpora",
+    "heegyu/korean-chatbot-data": "https://huggingface.co/datasets/heegyu/korean-chatbot-data",
+    "heegyu/korean-qa": "https://huggingface.co/datasets/heegyu/korean-qa",
+    "heegyu/korean-summarization": "https://huggingface.co/datasets/heegyu/korean-summarization",
+    "nlpai-lab/kullm-chat-v2": "https://huggingface.co/datasets/nlpai-lab/kullm-chat-v2",
+    "upstage/open-ko-llm-leaderboard": "https://huggingface.co/datasets/upstage/open-ko-llm-leaderboard"
+}
+
+def get_datasets_data(progress=gr.Progress()):
+    """데이터셋 데이터 가져오기"""
+    def calculate_rank(dataset_id, all_global_datasets, korea_datasets):
+        # 글로벌 순위 확인
+        global_rank = next((idx for idx, d in enumerate(all_global_datasets, 1) 
+                          if d.get('id', '').strip() == dataset_id.strip()), None)
+        
+        # Korea 데이터셋인 경우
+        is_korea = any(d.get('id', '').strip() == dataset_id.strip() for d in korea_datasets)
+        
+        if is_korea:
+            # Korea 데이터셋 중에서의 순위 확인
+            korea_rank = next((idx for idx, d in enumerate(korea_datasets, 1)
+                             if d.get('id', '').strip() == dataset_id.strip()), None)
+            
+            if korea_rank:
+                return min(global_rank or 3001, korea_rank + 1000), True
+        
+        return global_rank if global_rank else 'Not in top 3000', is_korea
+
+    try:
+        progress(0, desc="Fetching datasets...")
+        
+        if not HF_TOKEN:
+            fig = create_error_plot()
+            error_html = """
+            <div style='padding: 20px; background: #fee; border-radius: 10px; margin: 10px 0;'>
+                <h3 style='color: #c00;'>⚠️ API 인증이 필요합니다</h3>
+                <p>HuggingFace API 토큰이 설정되지 않았습니다.</p>
+            </div>
+            """
+            empty_df = pd.DataFrame(columns=['Global Rank', 'Dataset ID', 'Title', 'Downloads', 'Likes', 'Korea Search', 'URL'])
+            return fig, error_html, empty_df
+
+        # 일반 데이터셋과 Korea 관련 데이터셋 가져오기
+        params = {
+            'limit': 3000,
+            'full': 'true'
+        }
+        
+        all_datasets_response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Authorization': f'Bearer {HF_TOKEN}'},
+            params=params
+        )
+        
+        korea_params = {
+            'search': 'korea',
+            'limit': 3000,
+            'full': 'true'
+        }
+        
+        korea_datasets_response = requests.get(
+            "https://huggingface.co/api/datasets",
+            headers={'Authorization': f'Bearer {HF_TOKEN}'},
+            params=korea_params
+        )
+        
+        all_global_datasets = all_datasets_response.json()
+        korea_datasets = korea_datasets_response.json()
+        
+        # 시각화를 위한 Figure 생성
+        fig = go.Figure()
+        
+        # 순위 정보 수집
+        filtered_datasets = []
+        for dataset_id in target_datasets.keys():
+            try:
+                normalized_id = dataset_id.strip('/')
+                dataset_url_api = f"https://huggingface.co/api/datasets/{normalized_id}"
+                response = requests.get(
+                    dataset_url_api,
+                    headers={'Authorization': f'Bearer {HF_TOKEN}'}
+                )
+                
+                if response.status_code == 200:
+                    dataset_data = response.json()
+                    rank, is_korea = calculate_rank(dataset_id, all_global_datasets, korea_datasets)
+                    
+                    filtered_datasets.append({
+                        'id': dataset_id,
+                        'global_rank': rank,
+                        'downloads': dataset_data.get('downloads', 0),
+                        'likes': dataset_data.get('likes', 0),
+                        'title': dataset_data.get('title', 'No Title'),
+                        'is_korea': is_korea
+                    })
+                    
+                    print(f"Dataset {dataset_id}: Rank={rank}, Is Korea={is_korea}")
+                else:
+                    filtered_datasets.append({
+                        'id': dataset_id,
+                        'global_rank': 'Not in top 3000',
+                        'downloads': 0,
+                        'likes': 0,
+                        'title': 'No Title',
+                        'is_korea': False
+                    })
+            except Exception as e:
+                print(f"Error processing {dataset_id}: {str(e)}")
+                continue
+
+        # 순위로 정렬
+        filtered_datasets.sort(key=lambda x: float('inf') if isinstance(x['global_rank'], str) else x['global_rank'])
+        
+        # 시각화 데이터 준비
+        valid_datasets = [d for d in filtered_datasets if isinstance(d['global_rank'], (int, float))]
+        
+        if valid_datasets:
+            ids = [d['id'] for d in valid_datasets]
+            ranks = [d['global_rank'] for d in valid_datasets]
+            
+            fig.add_trace(go.Bar(
+                x=ids,
+                y=[3001 - r for r in ranks],
+                text=[f"Rank: #{r}<br>Downloads: {format(d['downloads'], ',')}<br>Likes: {format(d['likes'], ',')}"
+                      for r, d in zip(ranks, valid_datasets)],
+                textposition='auto',
+                marker_color=['rgba(255,0,0,0.6)' if d['is_korea'] else 'rgba(0,0,255,0.6)' 
+                            for d in valid_datasets],
+                opacity=0.8
+            ))
+
+            fig.update_layout(
+                title="HuggingFace Datasets Global Rankings (Up to #3000)",
+                xaxis_title="Dataset ID",
+                yaxis_title="Global Rank",
+                yaxis=dict(
+                    ticktext=[f"#{i}" for i in range(1, 3001, 100)],
+                    tickvals=[3001 - i for i in range(1, 3001, 100)],
+                    range=[0, 3000]
+                ),
+                height=800,
+                showlegend=False,
+                template='plotly_white',
+                xaxis_tickangle=-45
+            )
+
+        # HTML 카드 생성
+        html_content = """
+        <div style='padding: 20px; background: #f5f5f5;'>
+            <h2 style='color: #2c3e50;'>Datasets Rankings (Up to #3000)</h2>
+            <div style='display: grid; grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); gap: 20px;'>
+        """
+        
+        for dataset in filtered_datasets:
+            rank_display = f"Global Rank #{dataset['global_rank']}" if isinstance(dataset['global_rank'], (int, float)) else "Not in top 3000"
+            korea_badge = "🇰🇷 Korea Search Result" if dataset['is_korea'] else ""
+            
+            html_content += f"""
+            <div style='
+                background: white;
+                padding: 20px;
+                border-radius: 10px;
+                box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+                transition: transform 0.2s;
+                {f"border: 2px solid #e74c3c;" if dataset['is_korea'] else ""}
+            '>
+                <h3 style='color: #34495e;'>{rank_display}</h3>
+                <h4 style='color: #2c3e50;'>{dataset['id']}</h4>
+                <p style='color: #e74c3c; font-weight: bold;'>{korea_badge}</p>
+                <p style='color: #7f8c8d;'>⬇️ Downloads: {format(dataset['downloads'], ',')}</p>
+                <p style='color: #7f8c8d;'>👍 Likes: {format(dataset['likes'], ',')}</p>
+                <a href='{target_datasets[dataset['id']]}' 
+                   target='_blank' 
+                   style='
+                    display: inline-block;
+                    padding: 8px 16px;
+                    background: #3498db;
+                    color: white;
+                    text-decoration: none;
+                    border-radius: 5px;
+                    transition: background 0.3s;
+                   '>
+                   Visit Dataset 🔗
+                </a>
+            </div>
+            """
+        
+        html_content += "</div></div>"
+        
+        # 데이터프레임 생성
+        df = pd.DataFrame([{
+            'Global Rank': f"#{d['global_rank']}" if isinstance(d['global_rank'], (int, float)) else d['global_rank'],
+            'Dataset ID': d['id'],
+            'Title': d['title'],
+            'Downloads': format(d['downloads'], ','),
+            'Likes': format(d['likes'], ','),
+            'Korea Search': '🇰🇷' if d['is_korea'] else '',
+            'URL': target_datasets[d['id']]
+        } for d in filtered_datasets])
+        
+        progress(1.0, desc="Complete!")
+        return fig, html_content, df
+        
+    except Exception as e:
+        print(f"Error in get_datasets_data: {str(e)}")
+        error_fig = create_error_plot()
+        error_html = f"""
+        <div style='padding: 20px; background: #fee; border-radius: 10px; margin: 10px 0;'>
+            <h3 style='color: #c00;'>⚠️ 오류가 발생했습니다</h3>
+            <p>{str(e)}</p>
+        </div>
+        """
+        empty_df = pd.DataFrame(columns=['Global Rank', 'Dataset ID', 'Title', 'Downloads', 'Likes', 'Korea Search', 'URL'])
+        return error_fig, error_html, empty_df
+
+
+# Gradio UI 수정
 with gr.Blocks(theme=gr.themes.Soft()) as demo:
     gr.Markdown("""
     # 🤗 허깅페이스 '한국(언어) 리더보드'
-    HuggingFace가 제공하는 Spaces와 Models 실시간 인기 순위 반영하여 '한국안(기업/언어)'의 리스트(공개,검색,리더보드 등)만 분석. (c)'한국인공지능진흥협회' / 요청: [email protected]
+    HuggingFace가 제공하는 Spaces, Models, Datasets의 실시간 인기 순위를 반영하여 '한국안(기업/언어)'의 리스트(공개,검색,리더보드 등)만 분석. (c)'한국인공지능진흥협회' / 요청: [email protected]
     """)
     
-    # 새로 고침 버튼을 상단으로 이동하고 한글로 변경
     refresh_btn = gr.Button("🔄 새로 고침", variant="primary")
     
     with gr.Tab("Spaces Trending"):
         trending_plot = gr.Plot()
         trending_info = gr.HTML()
         trending_df = gr.DataFrame()
-    
+
     with gr.Tab("Models Trending"):
         models_plot = gr.Plot()
         models_info = gr.HTML()
         models_df = gr.DataFrame()
+        
+    with gr.Tab("Datasets Trending"):
+        datasets_plot = gr.Plot()
+        datasets_info = gr.HTML()
+        datasets_df = gr.DataFrame()
     
     def refresh_all_data():
         spaces_results = get_spaces_data("trending")
         models_results = get_models_data()
-        return [*spaces_results, *models_results]
+        datasets_results = get_datasets_data()
+        return [
+            *spaces_results,
+            *models_results,
+            *datasets_results
+        ]
     
     refresh_btn.click(
         refresh_all_data,
         outputs=[
             trending_plot, trending_info, trending_df,
-            models_plot, models_info, models_df
+            models_plot, models_info, models_df,
+            datasets_plot, datasets_info, datasets_df
         ]
     )
-    
-    # 초기 데이터 로드
-    spaces_results = get_spaces_data("trending")
-    models_results = get_models_data()
-    
-    trending_plot.value, trending_info.value, trending_df.value = spaces_results
-    models_plot.value, models_info.value, models_df.value = models_results
-    
-    
+
 # Gradio 앱 실행
 demo.launch(
     server_name="0.0.0.0",
     server_port=7860,
     share=False
-)
+)
+
+
+
+
+
+