feat: top k top p

app.py

@@ -1,69 +1,22 @@
 import streamlit as st
-from data_processor import load_data, process_data
-from visualizer import visualize_gmm, visualize_ratings
-from hypergraph_drawer import draw_hypergraph
+from pages import page1, page2, page3
 
 # 设置页面配置
 st.set_page_config(layout="wide")
 
 # 主应用
 def main():
-    st.title("高斯混合分布聚类可视化")
-
-    # 使用 sidebar 控制参数
-    with st.sidebar:
-        st.header("控制面板")
-        autoplay = st.button("自动播放")
-        if autoplay:
-            for i in range(1, 11):
-                with st.spinner(f"迭代 {i}"):
-                    time.sleep(1)
-                    st.session_state.iteration = i
-            st.session_state.autoplay = False
-            st.experimental_rerun()
-
-    # 主页面布局
-    if 'autoplay' not in st.session_state:
-        st.session_state.autoplay = True
-
-    if 'iteration' not in st.session_state:
-        st.session_state.iteration = 1
-
-    if st.session_state.autoplay:
-        # 隐藏迭代次数滑条
-        iteration = st.session_state.iteration
-    else:
-        # 显示迭代次数滑条
-        iteration = st.slider("选择迭代次数", min_value=1, max_value=10, value=st.session_state.iteration, step=1)
-
-    # 动态限制采样数量的最大值
-    df = load_data()
-    max_samples = len(df)
-    num_samples = st.slider("选择采样论文数量", min_value=1, max_value=min(100, max_samples), value=min(10, max_samples), step=1)
-
-    # 处理数据
-    sampled_df, probabilities, hyperedges = process_data(df, iteration, num_samples)
-
-    # 并排展示超图和高斯混合分布
-    col1, col2 = st.columns(2)
-    with col1:
-        st.header("超图可视化")
-        hypergraph_image = draw_hypergraph(hyperedges)
-        st.image(hypergraph_image, caption="超图可视化", use_container_width=True)
-
-    with col2:
-        st.header("高斯混合分布聚类结果")
-        fig_gmm = visualize_gmm(sampled_df, iteration)
-        st.plotly_chart(fig_gmm, use_container_width=True)
-
-    # 显示采样论文的详细信息
-    st.header("采样论文详细信息")
-    st.dataframe(sampled_df[["title", "keywords", "rating_avg", "confidence_avg", "site"]])
-
-    # 增加第二种可视化方式
-    st.header("论文评分分布")
-    fig_bar = visualize_ratings(sampled_df)
-    st.plotly_chart(fig_bar, use_container_width=True)
+    st.sidebar.title("导航")
+    pages = {
+        "NIPS 论文数据集高斯混合聚类分析": page1,
+        "第二个子应用": page2,
+        "第三个子应用": page3
+    }
+    
+    page = st.sidebar.radio("选择子应用", tuple(pages.keys()))
+    
+    # 根据选择的子应用加载相应的页面
+    pages[page].main()
 
 if __name__ == "__main__":
     main()

data_processor.py

@@ -1,25 +0,0 @@
-import pandas as pd
-import numpy as np
-
-def load_data():
-    return pd.read_csv("gmm_point_tracking_with_centroids.csv")
-
-def process_data(df, iteration, num_samples):
-    # 随机采样论文
-    sampled_df = df.sample(n=num_samples, random_state=iteration)
-
-    # 计算每个论文属于各个 cluster 的概率
-    probabilities = []
-    for idx, row in sampled_df.iterrows():
-        prob_str = row["probabilities"].strip("[]")
-        prob_list = list(map(float, prob_str.split(", ")))
-        probabilities.append(prob_list)
-
-    # 找到每个论文概率最高的 3 个 cluster
-    k = 3
-    hyperedges = {}
-    for idx, prob in enumerate(probabilities):
-        top_k = np.argsort(prob)[-k:][::-1]
-        hyperedges[idx] = [f"Cluster {c}" for c in top_k]
-
-    return sampled_df, probabilities, hyperedges

pages/page1.py

@@ -0,0 +1,92 @@
+from typing import Dict
+import streamlit as st
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import hypernetx as hnx
+import matplotlib.pyplot as plt
+from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
+from io import BytesIO
+import time
+from utils.data_processor import load_data, process_data, build_hyperedges
+from utils.visualizer import visualize_gmm, visualize_ratings
+from utils.hypergraph_drawer import draw_hypergraph
+
+def main():
+    st.title("NIPS 论文数据集高斯混合聚类分析")
+
+    # 使用 sidebar 控制参数
+    with st.sidebar:
+        st.header("控制面板")
+        autoplay = st.button("自动播放")
+        if autoplay:
+            for i in range(1, 11):
+                with st.spinner(f"迭代 {i}"):
+                    time.sleep(1)
+                    st.session_state.iteration = i
+            st.session_state.autoplay = False
+            st.experimental_rerun()
+
+        # 添加复选框选择显示 paper 的属性
+        display_attribute = st.selectbox(
+            "选择显示 paper 的属性",
+            ["index", "id", "title", "keywords", "author"]
+        )
+        # 选择是 top k 还是 top p
+        display_option = st.selectbox(
+            "选择显示的选项",
+            ["Top K Clusters", "Clusters Up To Probability P"]
+        )
+        # Top K Clusters
+        if display_option == "Top K Clusters":
+            top_k = st.slider("选择 K 值", min_value=1, max_value=10, value=3, step=1)
+            top_p = None
+        else:
+            top_k = None
+            top_p = st.slider("选择 P 值", min_value=0.0, max_value=1.0, value=0.5, step=0.01)
+
+    # 主页面布局
+    if 'autoplay' not in st.session_state:
+        st.session_state.autoplay = True
+
+    if 'iteration' not in st.session_state:
+        st.session_state.iteration = 1
+
+    if st.session_state.autoplay:
+        # 隐藏迭代次数滑条
+        iteration = st.session_state.iteration
+    else:
+        # 显示迭代次数滑条
+        iteration = st.slider("选择迭代次数", min_value=1, max_value=10, value=st.session_state.iteration, step=1)
+
+    # 动态限制采样数量的最大值
+    df = load_data()
+    max_samples = len(df)
+    num_samples = st.slider("选择采样论文数量", min_value=1, max_value=min(100, max_samples), value=min(10, max_samples), step=1)
+
+    # 处理数据
+    sampled_df, probabilities, paper_attributes = process_data(df, iteration, num_samples)
+    # print(display_attribute) # 字符串
+    hyperedges = build_hyperedges(probabilities, paper_attributes, display_attribute, top_k=top_k, top_p=top_p)
+    # print(hyperedges)
+
+    # 并排展示超图和高斯混合分布
+    col1, col2 = st.columns(2)
+    with col1:
+        st.header("超图可视化")
+        hypergraph_image = draw_hypergraph(hyperedges)
+        st.image(hypergraph_image, caption="超图可视化", use_container_width=True)
+
+    with col2:
+        st.header("高斯混合分布聚类结果")
+        fig_gmm = visualize_gmm(sampled_df, iteration)
+        st.plotly_chart(fig_gmm, use_container_width=True)
+
+    # 显示采样论文的详细信息
+    st.header("采样论文详细信息")
+    st.dataframe(sampled_df[["title", "keywords", "rating_avg", "confidence_avg", "site"]])
+
+    # 增加第二种可视化方式
+    st.header("论文评分分布")
+    fig_bar = visualize_ratings(sampled_df)
+    st.plotly_chart(fig_bar, use_container_width=True)

pages/page2.py

@@ -0,0 +1,5 @@
+import streamlit as st
+
+def main():
+    st.title("第二个子应用")
+    st.write("这里是第二个子应用的内容。")

pages/page3.py

@@ -0,0 +1,5 @@
+import streamlit as st
+
+def main():
+    st.title("第三个子应用")
+    st.write("这里是第三个子应用的内容。")

utils/data_processor.py

@@ -0,0 +1,65 @@
+from typing import Dict, List
+import pandas as pd
+import numpy as np
+
+
+def load_data():
+    return pd.read_csv("gmm_point_tracking_with_centroids.csv").reset_index()
+
+
+def process_data(df, iteration, num_samples):
+    # 随机采样论文
+    sampled_df = df.sample(n=num_samples, random_state=iteration)
+
+    # 计算每个论文属于各个 cluster 的概率
+    probabilities = []
+    paper_attributes = []
+    for idx, row in sampled_df.iterrows():
+        prob_str = row["probabilities"].strip("[]")
+        prob_list = list(map(float, prob_str.split(", ")))
+        probabilities.append(prob_list)
+        paper_attributes.append(
+            {
+                "id": row["id"],
+                "title": row["title"],
+                "keywords": row["keywords"],
+                "author": row["author"],
+            }
+        )
+
+    return sampled_df, probabilities, paper_attributes
+
+
+def build_hyperedges(
+    probabilities,
+    paper_attributes: List[Dict[str, str]],
+    display_attribute_name: str,
+    top_k: int = None,
+    top_p: float = None,
+) -> Dict[str, List[str]]:
+    # 构建超图边
+    hyperedges: Dict[str, List[str]] = {}
+    for idx, (prob, paper_attr) in enumerate(zip(probabilities, paper_attributes)):
+        if display_attribute_name == "index":
+            display_attribute = f"Paper {idx}"
+        display_attribute: str = paper_attr[display_attribute_name]
+        if top_k is not None:
+            selected_indices = np.argsort(prob)[-top_k:][::-1]
+        else:
+            # 累加起来，直到第一次大于等于 p
+            selected_indices = []
+            cumulative_prob = 0.0
+            for i, p in enumerate(prob):
+                selected_indices.append(i)
+                cumulative_prob += p
+                if cumulative_prob >= top_p:
+                    break
+
+            
+        for cluster in selected_indices:
+            cluster_name: str = f"Cluster {cluster}"
+            if cluster_name not in hyperedges:
+                hyperedges[cluster_name] = []
+            hyperedges[cluster_name].append(display_attribute)
+
+    return hyperedges

utils/gmm_dataset.py

@@ -0,0 +1,190 @@
+import numpy as np
+from pathlib import Path
+from scipy.special import gamma
+from typing import Optional, Tuple, Dict, List, Union
+import torch
+import os
+
+class GeneralizedGaussianMixture:
+    r"""广义高斯混合分布数据集生成器
+    P_{\theta_k}(x_i) = \eta_k \exp(-s_k d_k(x_i)) = \frac{p}{2\alpha_k \Gamma(1/p)}\exp(-|\frac{x_i-c_k}{\alpha_k}|^p)
+    """
+    
+    def __init__(self, 
+                 D: int = 2,           # 维度
+                 K: int = 3,           # 聚类数量
+                 p: float = 2.0,       # 幂次，p=2为标准高斯分布
+                 centers: Optional[np.ndarray] = None,  # 聚类中心
+                 scales: Optional[np.ndarray] = None,   # 尺度参数
+                 weights: Optional[np.ndarray] = None,  # 混合权重
+                 seed: int = 42):      # 随机种子
+        """初始化GMM数据集生成器
+        Args:
+            D: 数据维度
+            K: 聚类数量
+            p: 幂次参数，控制分布的形状
+            centers: 聚类中心，形状为(K, D)
+            scales: 尺度参数，形状为(K, D)
+            weights: 混合权重，形状为(K,)
+            seed: 随机种子
+        """
+        self.D = D
+        self.K = K
+        self.p = p
+        self.seed = seed
+        np.random.seed(seed)
+        
+        # 初始化分布参数
+        if centers is None:
+            self.centers = np.random.randn(K, D) * 2
+        else:
+            self.centers = centers
+            
+        if scales is None:
+            self.scales = np.random.uniform(0.1, 0.5, size=(K, D))
+        else:
+            self.scales = scales
+            
+        if weights is None:
+            self.weights = np.random.dirichlet(np.ones(K))
+        else:
+            self.weights = weights / weights.sum()  # 确保权重和为1
+        
+    def component_pdf(self, x: np.ndarray, k: int) -> np.ndarray:
+        """计算第k个分量的概率密度
+        Args:
+            x: 输入数据点，形状为(N, D)
+            k: 分量索引
+        Returns:
+            概率密度值，形状为(N,)
+        """
+        # 计算归一化常数
+        norm_const = self.p / (2 * self.scales[k] * gamma(1/self.p))
+        
+        # 计算|x_i - c_k|^p / α_k^p
+        z = np.abs(x - self.centers[k]) / self.scales[k]
+        exp_term = np.exp(-np.sum(z**self.p, axis=1))
+        
+        return np.prod(norm_const) * exp_term
+    
+    def pdf(self, x: np.ndarray) -> np.ndarray:
+        """计算混合分布的概率密度
+        Args:
+            x: 输入数据点，形状为(N, D)
+        Returns:
+            概率密度值，形状为(N,)
+        """
+        density = np.zeros(len(x))
+        for k in range(self.K):
+            density += self.weights[k] * self.component_pdf(x, k)
+        return density
+    
+    def generate_component_samples(self, n: int, k: int) -> np.ndarray:
+        """从第k个分量生成样本
+        Args:
+            n: 样本数量
+            k: 分量索引
+        Returns:
+            样本点，形状为(n, D)
+        """
+        # 使用幂指数分布的反变换采样
+        u = np.random.uniform(-1, 1, size=(n, self.D))
+        r = np.abs(u) ** (1/self.p)
+        samples = self.centers[k] + self.scales[k] * np.sign(u) * r
+        return samples
+    
+    def generate_samples(self, N: int) -> Tuple[np.ndarray, np.ndarray]:
+        """生成混合分布的样本
+        Args:
+            N: 总样本数量
+        Returns:
+            X: 生成的数据点，形状为(N, D)
+            y: 对应的概率密度值，形状为(N,)
+        """
+        # 根据混合权重确定每个分量的样本数量
+        n_samples = np.random.multinomial(N, self.weights)
+        
+        # 从每个分量生成样本
+        samples = []
+        for k in range(self.K):
+            x = self.generate_component_samples(n_samples[k], k)
+            samples.append(x)
+        
+        # 合并并打乱样本
+        X = np.vstack(samples)
+        idx = np.random.permutation(N)
+        X = X[idx]
+        
+        # 计算概率密度
+        y = self.pdf(X)
+        
+        return X, y
+    
+    def save_dataset(self, save_dir: Union[str, Path], name: str = 'gmm_dataset') -> None:
+        """保存数据集到文件
+        Args:
+            save_dir: 保存目录
+            name: 数据集名称
+        """
+        save_path = Path(save_dir)
+        save_path.mkdir(parents=True, exist_ok=True)
+        
+        # 生成并保存数据
+        X, y = self.generate_samples(N=1000)
+        np.savez(str(save_path / f'{name}.npz'),
+                 X=X, y=y,
+                 centers=self.centers,
+                 scales=self.scales,
+                 weights=self.weights,
+                 D=self.D,
+                 K=self.K,
+                 p=self.p)
+    
+    @classmethod
+    def load_dataset(cls, file_path: Union[str, Path]) -> "GeneralizedGaussianMixture":
+        """从文件加载数据集
+        Args:
+            file_path: 数据文件路径
+        Returns:
+            加载的GMM对象
+        """
+        data = np.load(str(file_path))
+        return cls(
+            D=int(data['D']),
+            K=int(data['K']),
+            p=float(data['p']),
+            centers=data['centers'],
+            scales=data['scales'],
+            weights=data['weights']
+        )
+
+def test_gmm_dataset():
+    """测试GMM数据集生成器"""
+    # 创建2D的GMM数据集
+    gmm = GeneralizedGaussianMixture(
+        D=2, 
+        K=3, 
+        p=2.0,
+        centers=np.array([[-2, -2], [0, 0], [2, 2]]),
+        scales=np.array([[0.3, 0.3], [0.2, 0.2], [0.4, 0.4]]),
+        weights=np.array([0.3, 0.4, 0.3])
+    )
+    
+    # 生成样本
+    X, y = gmm.generate_samples(1000)
+    
+    # 保存数据集
+    gmm.save_dataset('test_data')
+    
+    # 加载数据集
+    loaded_gmm = GeneralizedGaussianMixture.load_dataset('test_data/gmm_dataset.npz')
+    
+    # 验证保存和加载的参数是否一致
+    assert np.allclose(gmm.centers, loaded_gmm.centers)
+    assert np.allclose(gmm.scales, loaded_gmm.scales)
+    assert np.allclose(gmm.weights, loaded_gmm.weights)
+    
+    print("GMM数据集测试通过！")
+
+if __name__ == '__main__':
+    test_gmm_dataset()

utils/gmm_vis.py

@@ -0,0 +1,89 @@
+def create_gmm_plot(dataset, centers, K, samples=None):
+    """创建GMM分布的可视化图形"""
+    # 生成网格数据
+    x = np.linspace(-5, 5, 100)
+    y = np.linspace(-5, 5, 100)
+    X, Y = np.meshgrid(x, y)
+    xy = np.column_stack((X.ravel(), Y.ravel()))
+
+    # 计算概率密度
+    Z = dataset.pdf(xy).reshape(X.shape)
+
+    # 创建2D和3D可视化
+    fig = make_subplots(
+        rows=1, cols=2,
+        specs=[[{'type': 'surface'}, {'type': 'contour'}]],
+        subplot_titles=('3D概率密度曲面', '等高线图与分量中心')
+    )
+
+    # 3D Surface
+    surface = go.Surface(
+        x=X, y=Y, z=Z,
+        colorscale='viridis',
+        showscale=True,
+        colorbar=dict(x=0.45)
+    )
+    fig.add_trace(surface, row=1, col=1)
+
+    # Contour Plot
+    contour = go.Contour(
+        x=x, y=y, z=Z,
+        colorscale='viridis',
+        showscale=True,
+        colorbar=dict(x=1.0),
+        contours=dict(
+            showlabels=True,
+            labelfont=dict(size=12)
+        )
+    )
+    fig.add_trace(contour, row=1, col=2)
+
+    # 添加分量中心点
+    fig.add_trace(
+        go.Scatter(
+            x=centers[:K, 0], y=centers[:K, 1],
+            mode='markers+text',
+            marker=dict(size=10, color='red'),
+            text=[f'C{i+1}' for i in range(K)],
+            textposition="top center",
+            name='分量中心'
+        ),
+        row=1, col=2
+    )
+
+    # 添加采样点（如果有）
+    if samples is not None:
+        fig.add_trace(
+            go.Scatter(
+                x=samples[:, 0], y=samples[:, 1],
+                mode='markers+text',
+                marker=dict(
+                    size=8,
+                    color='yellow',
+                    line=dict(color='black', width=1)
+                ),
+                text=[f'S{i+1}' for i in range(len(samples))],
+                textposition="bottom center",
+                name='采样点'
+            ),
+            row=1, col=2
+        )
+
+    # 更新布局
+    fig.update_layout(
+        title='广义高斯混合分布',
+        showlegend=True,
+        width=1200,
+        height=600,
+        scene=dict(
+            xaxis_title='X',
+            yaxis_title='Y',
+            zaxis_title='密度'
+        )
+    )
+
+    # 更新2D图的坐标轴
+    fig.update_xaxes(title_text='X', row=1, col=2)
+    fig.update_yaxes(title_text='Y', row=1, col=2)
+
+    return fig