add analysis and vis support

app.py

@@ -1,103 +1,37 @@
 import gradio as gr
 import pandas as pd
-import requests
-import os
-import shutil
-import json
 import pandas as pd
-import subprocess
+import json
 import plotly.express as px
-def on_confirm(dataset_radio, num_parts_dropdown, token_counts_radio, line_counts_radio, cyclomatic_complexity_radio, problem_type_checkbox):
+
+def on_confirm(dataset_radio, num_parts_dropdown, perspective_radio, division_method_radio):
     # 根据用户选择的参数构建文件路径
     num_parts = num_parts_dropdown
-    # token_counts_split = token_counts_radio
-    # line_counts_split = line_counts_radio
-    # cyclomatic_complexity_split = cyclomatic_complexity_radio
-
-
-    # 读取数据
-    dataframes = []
+    
     if dataset_radio == "HumanEval":
-        if token_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/token_counts_QS.csv")
-            dataframes.append(token_counts_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/token_counts_EI.csv")
-            dataframes.append(token_counts_df)
-        if line_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/line_counts_QS.csv")
-            dataframes.append(line_counts_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/line_counts_EI.csv")
-            dataframes.append(line_counts_df)
-        if cyclomatic_complexity_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/CC_QS.csv")
-            dataframes.append(CC_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/CC_EI.csv")
-            dataframes.append(CC_df)
-
-
-
-    #以下改为直接从一个划分文件中读取即可
-        if problem_type_checkbox:
-            problem_type_df = pd.read_csv("/home/user/app/dividing_into_different_subsets/cata_result.csv")
-            dataframes.append(problem_type_df)
-    if dataset_radio == "MBPP":
-        if token_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/token_counts_QS.csv")
-            dataframes.append(token_counts_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/token_counts_EI.csv")
-            dataframes.append(token_counts_df)
-        if line_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/line_counts_QS.csv")
-            dataframes.append(line_counts_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/line_counts_EI.csv")
-            dataframes.append(line_counts_df)
-        if cyclomatic_complexity_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
-            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/CC_QS.csv")
-            dataframes.append(CC_df)
-        if token_counts_radio=="Equal Interval Partitioning":
-            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/CC_EI.csv")
-            dataframes.append(CC_df)
-
-
-
-    #以下改为直接从一个划分文件中读取即可
-        if problem_type_checkbox:
-            problem_type_df = pd.read_csv("/home/user/app/dividing_into_different_subsets_mbpp/cata_result.csv")
-            dataframes.append(problem_type_df)
-
-    # 如果所有三个radio都有value，将三个文件中的所有行拼接
-    if len(dataframes) > 0:
-        combined_df = dataframes[0]
-        for df in dataframes[1:]:
-            combined_df = pd.merge(combined_df, df, left_index=True, right_index=True, suffixes=('', '_y'))
-            combined_df = combined_df.loc[:, ~combined_df.columns.str.endswith('_y')]  # 去除重复的列
-        return combined_df
-    else:
-        return pd.DataFrame()
-
-
-
-
-def execute_specified_python_files(directory_list, file_list):
-    for directory in directory_list:
-        for py_file in file_list:
-            file_path = os.path.join(directory, py_file)
-            if os.path.isfile(file_path) and py_file.endswith('.py'):
-                print(f"Executing {file_path}...")
-                try:
-                    # 使用subprocess执行Python文件
-                    subprocess.run(['python', file_path], check=True)
-                    print(f"{file_path} executed successfully.")
-                except subprocess.CalledProcessError as e:
-                    print(f"Error executing {file_path}: {e}")
-            else:
-                print(f"File {file_path} does not exist or is not a Python file.")
-# 定义一个函数来生成 CSS 样式
+        base_path = "./dividing_into_different_subsets"
+    else:  # MBPP
+        base_path = "./dividing_into_different_subsets_mbpp"
+    
+    method = "QS" if division_method_radio == "Equal Frequency Partitioning" else "EI"
+    
+    # 根据perspective选择读取对应的文件
+    if "Tokens" in perspective_radio:
+        df = pd.read_csv(f"{base_path}/{num_parts}/{method}/token_counts_{method}.csv")
+    elif "Lines" in perspective_radio:
+        df = pd.read_csv(f"{base_path}/{num_parts}/{method}/line_counts_{method}.csv")
+    elif "Complexity" in perspective_radio:
+        df = pd.read_csv(f"{base_path}/{num_parts}/{method}/CC_{method}.csv")
+    elif "Problem Types" in perspective_radio:
+        df = pd.read_csv(f"{base_path}/cata_result.csv")
+    
+    # 加载分析报告
+    analysis_result,_ = load_analysis_report(dataset_radio, num_parts_dropdown, perspective_radio, division_method_radio)
+    # AI分析列
+    df["Analysis"] = df["Model"].map(lambda m: analysis_result.get(m, "No analysis provided."))
+    return df
+
+# 生成 CSS 样式
 def generate_css(line_counts, token_counts, cyclomatic_complexity, problem_type, show_high, show_medium, show_low):
     css = """
     #dataframe th {
@@ -139,225 +73,423 @@ def generate_css(line_counts, token_counts, cyclomatic_complexity, problem_type,
 
     return css
 
+# AI分析
+def load_analysis_report(dataset_radio, num_parts_dropdown, perspective_radio, division_method_radio):
 
+    num_parts = num_parts_dropdown
+    method = "QS" if division_method_radio == "Equal Frequency Partitioning" else "EI"
+
+    # 根据perspective确定文件路径
+    if "Tokens" in perspective_radio:
+        perspective = "token_counts"
+    elif "Lines" in perspective_radio:
+        perspective = "line_counts"
+    elif "Complexity" in perspective_radio:
+        perspective = "CC"
+    else:
+        perspective = "problem_type"
 
+    base_path = "./llm_insight"
+    if perspective == "problem_type":
+        report_file = f"{base_path}/{dataset_radio}/{perspective}_report.json"
+        recommendation_file = f"{base_path}/{dataset_radio}/{perspective}_recommendation.json"
+    else:
+        report_file = f"{base_path}/{dataset_radio}/{num_parts}/{method}/{perspective}_report.json"
+        recommendation_file = f"{base_path}/{dataset_radio}/{num_parts}/{method}/{perspective}_recommendation.json"
 
-def update_radio_options(token_counts, line_counts, cyclomatic_complexity, problem_type):
-    options = []
-    if token_counts:
-        options.append("The Number of Tokens in Problem Descriptions")
-    if line_counts:
-        options.append("The Number of Lines in Problem Descriptions")
-    if cyclomatic_complexity:
-        options.append("The Complexity of Reference Code")
-    if problem_type:
-        options.append("Problem Type")
+    try:
+        with open(report_file, 'r', encoding='utf-8') as f:
+            analysis_result = json.load(f)
+    except Exception as e:
+        analysis_result = f"[Error] error load analysis report: {e}"
 
-    return gr.update(choices=options)
+    try:
+        with open(recommendation_file, 'r', encoding='utf-8') as f:
+            recommendation_result = json.load(f)
+    except Exception as e:
+        recommendation_result = f"[Error] error load model recommendation: {e}"
+
+    return (analysis_result,recommendation_result)
+
+# 可视化
+def plot_visualization(dataset_radio, perspective_radio, num_parts, plot_type):
+    if dataset_radio == "HumanEval":
+        base_path = "./dividing_into_different_subsets"
+    else:  # MBPP
+        base_path = "./dividing_into_different_subsets_mbpp"
+
+    if "Tokens" in perspective_radio:
+        file_path = f'{base_path}/{num_parts}/QS/token_counts_QS.csv'
+    elif "Lines" in perspective_radio:
+        file_path = f'{base_path}/{num_parts}/QS/line_counts_QS.csv'
+    elif "Complexity" in perspective_radio:
+        file_path = f'{base_path}/{num_parts}/QS/CC_QS.csv'
+    else:  # Problem Types
+        file_path = f'{base_path}/cata_result.csv'
 
-def plot_csv(dataset_radio,radio,num):
-    print(dataset_radio,radio)
-    if dataset_radio=="HumanEval":
-    
-        if radio=="The Number of Tokens in Problem Descriptions":
-            radio_choice="token_counts"
-            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="The Number of Lines in Problem Descriptions":
-            radio_choice="line_counts"
-            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="The Complexity of Reference Code":
-            radio_choice="CC"
-            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="Problem Type":
-            radio_choice="problem_type"
-            file_path = f'/home/user/app/dividing_into_different_subsets/cata_result.csv'
-        print("test!")
-    elif dataset_radio=="MBPP":
-        if radio=="The Number of Tokens in Problem Descriptions":
-            radio_choice="token_counts"
-            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="The Number of Lines in Problem Descriptions":
-            radio_choice="line_counts"
-            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="The Complexity of Reference Code":
-            radio_choice="CC"
-            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
-        elif radio=="Problem Type":
-            radio_choice="problem_type"
-            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/cata_result.csv'
-        print("test!")
-
-    # file_path="E:/python-testn/pythonProject3/hh_1/dividing_into_different_subsets/3/QS/CC_QS.csv"
     df = pd.read_csv(file_path)
-    # 将第一列作为索引
     df.set_index('Model', inplace=True)
-
-    # 转置数据框，使得模型作为列，横轴作为行
     df_transposed = df.T
 
-    # 使用plotly绘制折线图
-    fig = px.line(df_transposed, x=df_transposed.index, y=df_transposed.columns,
-                  title='Model Evaluation Results',
-                  labels={'value': 'Evaluation Score', 'index': 'Evaluation Metric'},
-                  color_discrete_sequence=px.colors.qualitative.Plotly)
-
-    # 设置悬停效果
-    fig.update_traces(hovertemplate='%{y}')
+    if plot_type == "Line Chart":
+        fig = px.line(df_transposed, 
+                     x=df_transposed.index, 
+                     y=df_transposed.columns,
+                     title='Model Performance Across Different Subsets',
+                     labels={'value': 'Evaluation Score', 'index': 'Subsets'},
+                     color_discrete_sequence=px.colors.qualitative.Plotly)
+        fig.update_traces(hovertemplate='%{y}')
+    elif plot_type == "Radar Chart":  # Radar Chart
+        # 重新组织数据为雷达图所需格式
+        radar_data = []
+        for model in df.index:
+            for subset, score in df.loc[model].items():
+                radar_data.append({
+                    'Model': model,
+                    'Subset': subset,
+                    'Score': score
+                })
+        
+        radar_df = pd.DataFrame(radar_data)
+        
+        colors = ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']
+        
+        # 创建雷达图
+        fig = px.line_polar(radar_df, 
+                           r='Score',
+                           theta='Subset',
+                           color='Model',
+                           line_close=True,
+                           color_discrete_sequence=colors,
+                           title='Model Performance Radar Chart')
+        
+        # 自定义每个模型的线条样式
+        for i, trace in enumerate(fig.data):
+            trace.update(
+                fill=None,  # 移除填充
+                line=dict(
+                    width=2,
+                    dash='solid' if i % 2 == 0 else 'dash',  # 交替使用实线和虚线
+                )
+            )
+        
+        # 优化雷达图的显示
+        fig.update_layout(
+            polar=dict(
+                radialaxis=dict(
+                    visible=True,
+                    range=[0, 100],
+                    showline=True,
+                    linewidth=1,
+                    gridcolor='lightgrey'
+                ),
+                angularaxis=dict(
+                    showline=True,
+                    linewidth=1,
+                    gridcolor='lightgrey'
+                )
+            ),
+            showlegend=True,
+            legend=dict(
+                yanchor="middle",  # 垂直居中
+                y=0.5,
+                xanchor="left",
+                x=1.2,  # 将图例移到雷达图右侧
+                bgcolor="rgba(255, 255, 255, 0.8)",  # 半透明白色背景
+                bordercolor="lightgrey",  # 添加边框
+                borderwidth=1
+            ),
+            margin=dict(r=150),  # 增加右侧边距，为图例留出空间
+            paper_bgcolor='white'
+        )
+    else:  # Heatmap
+        # 创建热力图
+        fig = px.imshow(df_transposed,
+                       labels=dict(x="Model", y="Subset", color="Score"),
+                       color_continuous_scale="RdYlBu_r",  # 使用科研风格配色：红-黄-蓝
+                       aspect="auto",  # 自动调整宽高比
+                       title="Model Performance Heatmap")
+        
+        # 优化热力图显示
+        fig.update_layout(
+            title=dict(
+                text='Model Performance Distribution Across Subsets',
+                x=0.5,
+                y=0.95,
+                xanchor='center',
+                yanchor='top',
+                font=dict(size=14)
+            ),
+            xaxis=dict(
+                title="Model",
+                tickangle=45,  # 斜着显示模型名称
+                tickfont=dict(size=10),
+                side="bottom"
+            ),
+            yaxis=dict(
+                title="Subset",
+                tickfont=dict(size=10)
+            ),
+            coloraxis=dict(
+                colorbar=dict(
+                    title="Score",
+                    titleside="right",
+                    tickfont=dict(size=10),
+                    titlefont=dict(size=12),
+                    len=0.9,  # 色条长度
+                )
+            ),
+            margin=dict(t=80, r=100, b=80, l=80),  # 调整边距
+            paper_bgcolor='white',
+            plot_bgcolor='white'
+        )
+
+        # 添加具体数值标注
+        annotations = []
+        for i in range(len(df_transposed.index)):
+            for j in range(len(df_transposed.columns)):
+                annotations.append(
+                    dict(
+                        x=j,
+                        y=i,
+                        text=f"{df_transposed.iloc[i, j]:.1f}",
+                        showarrow=False,
+                        font=dict(size=9, color='black')
+                    )
+                )
+        fig.update_layout(annotations=annotations)
 
     return fig
 
-def toggle_radio(checkbox, radio):
-    return gr.update(visible=checkbox)
+# 旭日图
+def plot_recommendation_sunburst(dataset_radio, num_parts_dropdown, perspective_radio, division_method_radio):
+    import plotly.graph_objects as go
+    _, recommendation_result = load_analysis_report(dataset_radio, num_parts_dropdown, perspective_radio, division_method_radio)
+    labels = ['Model Recommendation']  # 根节点
+    parents = ['']
+    values = []
+    customdata = ['Choose your preference model']
+
+    # 统计每个场景下模型数量
+    scenario_model_count = {}
+    total_model_count = 0
+
+    for scenario, model_list in recommendation_result.items():
+        # 处理模型
+        model_items = []
+        if isinstance(model_list, dict):
+            model_items = model_list.items()
+        elif isinstance(model_list, list):
+            for d in model_list:
+                if isinstance(d, dict):
+                    for k, v in d.items():
+                        model_items.append((k, v))
+
+        scenario_model_count[scenario] = len(model_items)
+        total_model_count += len(model_items)
+
+    # 根节点 value
+    values.append(total_model_count)
+
+    # 再次遍历，填充 labels/parents/values/customdata
+    for scenario, model_list in recommendation_result.items():
+        scenario_words = scenario.split()
+        short_label = " ".join(scenario_words[:3]) + "..." if len(scenario_words) > 3 else scenario
+        labels.append(short_label)
+        parents.append('Model Recommendation')
+        values.append(scenario_model_count[scenario])
+        customdata.append(scenario)
+
+        # 处理模型
+        model_items = []
+        if isinstance(model_list, dict):
+            model_items = model_list.items()
+        elif isinstance(model_list, list):
+            for d in model_list:
+                if isinstance(d, dict):
+                    for k, v in d.items():
+                        model_items.append((k, v))
+
+        for model, reason in model_items:
+            labels.append(model)
+            parents.append(short_label)
+            values.append(1)
+            customdata.append(reason)
+
+    fig = go.Figure(go.Sunburst(
+        labels=labels,
+        parents=parents,
+        values=values,
+        branchvalues="total",
+        hovertemplate='%{customdata}<extra></extra>',
+        customdata=customdata
+    ))
+    fig.update_layout(margin=dict(t=10, l=10, r=10, b=10), height=500)
+    return fig
 
-def toggle_line_counts_visibility(dataset):
-    if dataset == "MBPP":
-        return gr.update(visible=False)
-    else:
-        return gr.update(visible=True)
+### Gradio代码部分 ### 
 
-    # 创建 Gradio 界面
-import gradio as gr
+# 自定义 CSS 样式
+custom_css = """
+<style>
+    body {
+        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+        background-color: #f9f9f9;
+    }
+    .gr-label {
+        font-size: 15px;
+    }
+    .gr-button-primary {
+        background-color: #4CAF50;
+        color: white;
+        border-radius: 8px;
+    }
+    .gr-tabs > .tab-nav {
+        background-color: #e0e0e0;
+        border-bottom: 2px solid #ccc;
+    }
+    .gr-tabs > .tab-nav button.selected {
+        background-color: #ffffff !important;
+        border-bottom: 2px solid #4CAF50;
+    }
+    .gr-panel {
+        padding: 20px;
+        border-radius: 10px;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        background-color: #fff;
+    }
+    .markdown-title {
+        font-size: 1.5em;
+        font-weight: bold;
+        margin-bottom: 10px;
+    }
+    .analysis-box {
+        background-color: #f1f8ff;
+        padding: 20px;
+        border-left: 5px solid #4CAF50;
+        border-radius: 6px;
+        margin-top: 10px;
+    }
+    .recommendation-box {
+        background-color: #fff3cd;
+        padding: 20px;
+        border-left: 5px solid #ff9800;
+        border-radius: 6px;
+        margin-top: 10px;
+    }
+</style>
+"""
 
-with gr.Blocks() as iface:
+# 构建界面
+with gr.Blocks(css=custom_css) as iface:
     gr.HTML("""
-               <style>
-                   # body {
-                   #     max-width: 50%; /* 设置最大宽度为50% */
-                   #     margin: 0 auto; /* 将内容居中 */
-                   # }
-                   .title {
-                       text-align: center;
-                       font-size: 3em;
-                       font-weight: bold;
-                       margin-bottom: 0.5em;
-                   }
-                   .subtitle {
-                       text-align: center;
-                       font-size: 2em;
-                       margin-bottom: 1em;
-                   }
-               </style>
-
-           """)
-
-    with gr.Tabs() as tabs:
-        with gr.TabItem("Evaluation Result"):
-            with gr.Row():
-                with gr.Column(scale=2):
-                    with gr.Row():
-                        with gr.Column():
-                            dataset_radio = gr.Radio(["HumanEval", "MBPP"], label="Select Dataset ")
-
-
-            with gr.Row():
-                custom_css = """  
-                    <style>  
-                        .markdown-class {  
-                            font-family: 'Helvetica', sans-serif;
-                            font-size: 20px; 
-                            font-weight: bold;  
-                            color: #333; 
-                        }  
-                    </style>  
-                    """
-
-                with gr.Column():
-                    gr.Markdown(
-                        f"{custom_css}<div class='markdown-class'> Choose Division Perspective </div>")
-
-                    token_counts_checkbox = gr.Checkbox(label="I-The Number of Tokens in Problem Descriptions")
-                    line_counts_checkbox = gr.Checkbox(label="II-The Number of Lines in Problem Descriptions")
-                    dataset_radio.change(fn=toggle_line_counts_visibility, inputs=dataset_radio,
-                                         outputs=line_counts_checkbox)
-                    cyclomatic_complexity_checkbox = gr.Checkbox(label="III-The Complexity of Reference Code")
-                    problem_type_checkbox = gr.Checkbox(label="IV-Problem Types ")
-                css_code = """  
-                    .dropdown-container {  
-                        display: none;  
-                    }  
-                """
-
-                with gr.Column():
-                    # gr.Markdown("<div class='markdown-class'>Choose Subsets </div>")
-                    num_parts_dropdown = gr.Dropdown(choices=[0,3, 4, 5, 6, 7, 8], label="Choose the Number of Subsets",value="")
-
-            with gr.Row():
-                with gr.Column():
-                    token_counts_radio = gr.Radio(
-                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
-                        label="Choose the Division Method for Perspective-I",
-                        visible=False)
-                with gr.Column():
-                    line_counts_radio = gr.Radio(
-                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
-                        label="Choose the Division Method for Perspective-II",
-                        visible=False)
-                with gr.Column():
-                    cyclomatic_complexity_radio = gr.Radio(
-                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
-                        label="Choose the Division Method for Perspective-III",
-                        visible=False)
-
-            token_counts_checkbox.change(fn=lambda x: toggle_radio(x, token_counts_radio),
-                                         inputs=token_counts_checkbox, outputs=token_counts_radio)
-            line_counts_checkbox.change(fn=lambda x: toggle_radio(x, line_counts_radio),
-                                        inputs=line_counts_checkbox, outputs=line_counts_radio)
-            cyclomatic_complexity_checkbox.change(fn=lambda x: toggle_radio(x, cyclomatic_complexity_radio),
-                                                  inputs=cyclomatic_complexity_checkbox,
-                                                  outputs=cyclomatic_complexity_radio)
-
-            with gr.Tabs() as inner_tabs:
+    <div style='text-align:center; padding:15px;'>
+        <h1>Multi-view Code LLM Leaderboard</h1>
+        <p>Multi-view Leaderboard: Evaluating Large  Language Models From Multiple Views</p>
+    </div>
+    """)
+
+    with gr.Row():
+        # 选择配置
+        with gr.Column(scale=1):
+            dataset_radio = gr.Radio(
+                ["HumanEval", "MBPP"],
+                label="Select a dataset",
+                value="HumanEval"
+            )
+            num_parts_slider = gr.Slider(
+                minimum=3,
+                maximum=8,
+                step=1,
+                label="Choose the Number of Subsets",
+                value=3
+            )
+            
+            # 将多个checkbox改为一个radio
+            perspective_radio = gr.Radio(
+                ["I - Num of Tokens in Problem Desc",
+                 "II - Num of Lines in Problem Desc",
+                 "III - Complexity of Reference Code",
+                 "IV - Problem Types"],
+                label="Choose Perspective",
+                value="I - Num of Tokens in Problem Desc"
+            )
+
+            # 统一的division method radio
+            division_method_radio = gr.Radio(
+                ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
+                label="Choose the Division Method",
+                visible=True
+            )
+
+            confirm_btn = gr.Button("Confirm", variant="primary")
+
+        # 核心展示
+        with gr.Column(scale=2):
+            with gr.Tabs():
+                # 表格
                 with gr.TabItem("Ranking Table"):
-                    dataframe_output = gr.Dataframe(elem_id="dataframe")
-                    css_output = gr.HTML()
-                    confirm_button = gr.Button("Confirm ")
-                    confirm_button.click(fn=on_confirm, inputs=[dataset_radio, num_parts_dropdown, token_counts_radio,
-                                                                line_counts_radio, cyclomatic_complexity_radio,
-                                                                problem_type_checkbox],
-                                         outputs=dataframe_output)
-
-                with gr.TabItem("Line chart"):
-                    select_radio = gr.Radio(choices=[], label="Select One Perpective")
-                    checkboxes = [token_counts_checkbox, line_counts_checkbox, cyclomatic_complexity_checkbox,
-                                  problem_type_checkbox]
-                    for checkbox in checkboxes:
-                        checkbox.change(fn=update_radio_options, inputs=checkboxes, outputs=select_radio)
-                    select_radio.change(fn=plot_csv, inputs=[dataset_radio, select_radio, num_parts_dropdown],
-                                        outputs=gr.Plot(label="Line Plot "))
-
-        # with gr.TabItem("Upload Inference File"):
-        #     gr.Markdown("Upload a JSON file")
-        #     with gr.Row():
-        #         with gr.Column():
-        #             string_input = gr.Textbox(label="Enter the Model Name")
-        #             number_input = gr.Number(label="Select the Number of Samples")
-        #             dataset_choice = gr.Dropdown(label="Select Dataset", choices=["HumanEval", "MBPP"])
-        #         with gr.Column():
-        #             file_input = gr.File(label="Upload Generation Result in JSON file")
-        #             upload_button = gr.Button("Confirm and Upload")
-
-        #     json_output = gr.JSON(label="")
-
-        #     upload_button.click(fn=generate_file, inputs=[file_input, string_input, number_input, dataset_choice],
-        #                         outputs=json_output)
-
-    css = """  
-        #scale1 {  
-    border: 1px solid rgba(0, 0, 0, 0.2); 
-    padding: 10px;  
-    border-radius: 8px;
-    background-color: #f9f9f9; 
-    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); 
-}  
-        }  
-        """
-    gr.HTML(f"<style>{css}</style>")
-
-    # 初始化数据表格
-    # initial_df = show_data(False, False, False, False, False, False, False)
-    # initial_css = generate_css(False, False, False, False, True, False, False)
-    # dataframe_output.value = initial_df
-    # css_output.value = f"<style>{initial_css}</style>"
-
+                    data_table = gr.Dataframe(headers=["Model", "Score","Analysis"],interactive=True)
+                # 可视化
+                with gr.TabItem("Visualization"):
+                    plot_type = gr.Radio(
+                        choices=["Line Chart", "Radar Chart","Heatmap"],
+                        label="Select Plot Type",
+                        value="Line Chart"
+                    )
+                    chart = gr.Plot()
+                # AI分析
+                with gr.TabItem("Model selection suggestions"):
+                    with gr.Column():
+                        gr.Markdown("<h2 class='markdown-title'>🎯 Model Recommendation</h2>")
+                        recommendation_plot = gr.Plot()
+                        scenario_legend = gr.Markdown(value="")  # 新增图例
+    
+    def update_perspective_options(dataset):
+        if dataset == "MBPP":
+            return gr.update(choices=[
+                "I - Num of Tokens in Problem Desc",
+                "III - Complexity of Reference Code",
+                "IV - Problem Types"
+            ])
+        else:
+            return gr.update(choices=[
+                "I - Num of Tokens in Problem Desc",
+                "II - Num of Lines in Problem Desc",
+                "III - Complexity of Reference Code",
+                "IV - Problem Types"
+            ])
+
+    dataset_radio.change(
+        fn=update_perspective_options,
+        inputs=dataset_radio,
+        outputs=perspective_radio
+    )
+
+
+    # 绑定事件
+    confirm_btn.click(
+        fn=on_confirm,
+        inputs=[dataset_radio, num_parts_slider, perspective_radio, division_method_radio],
+        outputs=data_table
+    ).then(
+        fn=load_analysis_report,
+        inputs=[dataset_radio, num_parts_slider, perspective_radio, division_method_radio],
+        outputs=[gr.State()]
+    ).then(
+        fn=plot_visualization,
+        inputs=[dataset_radio, perspective_radio, num_parts_slider, plot_type],
+        outputs=chart
+    ).then(
+        fn=plot_recommendation_sunburst,
+        inputs=[dataset_radio, num_parts_slider, perspective_radio, division_method_radio],
+        outputs=[recommendation_plot]  # 注意这里是列表
+    )
+
+    plot_type.change(
+        fn=plot_visualization,
+        inputs=[dataset_radio, perspective_radio, num_parts_slider, plot_type],
+        outputs=chart
+    )
 # 启动界面
 iface.launch()

app_back.py

@@ -0,0 +1,363 @@
+import gradio as gr
+import pandas as pd
+import requests
+import os
+import shutil
+import json
+import pandas as pd
+import subprocess
+import plotly.express as px
+def on_confirm(dataset_radio, num_parts_dropdown, token_counts_radio, line_counts_radio, cyclomatic_complexity_radio, problem_type_checkbox):
+    # 根据用户选择的参数构建文件路径
+    num_parts = num_parts_dropdown
+    # token_counts_split = token_counts_radio
+    # line_counts_split = line_counts_radio
+    # cyclomatic_complexity_split = cyclomatic_complexity_radio
+
+
+    # 读取数据
+    dataframes = []
+    if dataset_radio == "HumanEval":
+        if token_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/token_counts_QS.csv")
+            dataframes.append(token_counts_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/token_counts_EI.csv")
+            dataframes.append(token_counts_df)
+        if line_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/line_counts_QS.csv")
+            dataframes.append(line_counts_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/line_counts_EI.csv")
+            dataframes.append(line_counts_df)
+        if cyclomatic_complexity_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/QS/CC_QS.csv")
+            dataframes.append(CC_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets/{num_parts}/EI/CC_EI.csv")
+            dataframes.append(CC_df)
+
+
+
+    #以下改为直接从一个划分文件中读取即可
+        if problem_type_checkbox:
+            problem_type_df = pd.read_csv("/home/user/app/dividing_into_different_subsets/cata_result.csv")
+            dataframes.append(problem_type_df)
+    if dataset_radio == "MBPP":
+        if token_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/token_counts_QS.csv")
+            dataframes.append(token_counts_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            token_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/token_counts_EI.csv")
+            dataframes.append(token_counts_df)
+        if line_counts_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/line_counts_QS.csv")
+            dataframes.append(line_counts_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            line_counts_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/line_counts_EI.csv")
+            dataframes.append(line_counts_df)
+        if cyclomatic_complexity_radio=="Equal Frequency Partitioning":#等频划分，每个子集数据点的数量基本一致
+            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/QS/CC_QS.csv")
+            dataframes.append(CC_df)
+        if token_counts_radio=="Equal Interval Partitioning":
+            CC_df = pd.read_csv(f"/home/user/app/dividing_into_different_subsets_mbpp/{num_parts}/EI/CC_EI.csv")
+            dataframes.append(CC_df)
+
+
+
+    #以下改为直接从一个划分文件中读取即可
+        if problem_type_checkbox:
+            problem_type_df = pd.read_csv("/home/user/app/dividing_into_different_subsets_mbpp/cata_result.csv")
+            dataframes.append(problem_type_df)
+
+    # 如果所有三个radio都有value，将三个文件中的所有行拼接
+    if len(dataframes) > 0:
+        combined_df = dataframes[0]
+        for df in dataframes[1:]:
+            combined_df = pd.merge(combined_df, df, left_index=True, right_index=True, suffixes=('', '_y'))
+            combined_df = combined_df.loc[:, ~combined_df.columns.str.endswith('_y')]  # 去除重复的列
+        return combined_df
+    else:
+        return pd.DataFrame()
+
+
+
+
+def execute_specified_python_files(directory_list, file_list):
+    for directory in directory_list:
+        for py_file in file_list:
+            file_path = os.path.join(directory, py_file)
+            if os.path.isfile(file_path) and py_file.endswith('.py'):
+                print(f"Executing {file_path}...")
+                try:
+                    # 使用subprocess执行Python文件
+                    subprocess.run(['python', file_path], check=True)
+                    print(f"{file_path} executed successfully.")
+                except subprocess.CalledProcessError as e:
+                    print(f"Error executing {file_path}: {e}")
+            else:
+                print(f"File {file_path} does not exist or is not a Python file.")
+# 定义一个函数来生成 CSS 样式
+def generate_css(line_counts, token_counts, cyclomatic_complexity, problem_type, show_high, show_medium, show_low):
+    css = """
+    #dataframe th {
+        background-color: #f2f2f2
+        
+    }
+    """
+    colors = ["#e6f7ff", "#ffeecc", "#e6ffe6", "#ffe6e6"]
+    categories = [line_counts, token_counts, cyclomatic_complexity]
+    category_index = 0
+    column_index = 1
+
+    for category in categories:
+        if category:
+            if show_high:
+                css += f"#dataframe td:nth-child({column_index + 1}) {{ background-color: {colors[category_index]}; }}\n"
+                column_index += 1
+            if show_medium:
+                css += f"#dataframe td:nth-child({column_index + 1}) {{ background-color: {colors[category_index]}; }}\n"
+                column_index += 1
+            if show_low:
+                css += f"#dataframe td:nth-child({column_index + 1}) {{ background-color: {colors[category_index]}; }}\n"
+                column_index += 1
+        category_index += 1
+
+    # 为 Problem Type 相关的三个子列设置固定颜色
+    if problem_type:
+        problem_type_color = "#d4f0fc"  # 你可以选择任何你喜欢的颜色
+        css += f"#dataframe td:nth-child({column_index + 1}) {{ background-color: {problem_type_color}; }}\n"
+        css += f"#dataframe td:nth-child({column_index + 2}) {{ background-color: {problem_type_color}; }}\n"
+        css += f"#dataframe td:nth-child({column_index + 3}) {{ background-color: {problem_type_color}; }}\n"
+
+    # 隐藏 "data" 标识
+    css += """
+    .gradio-container .dataframe-container::before {
+        content: none !important;
+    }
+    """
+
+    return css
+
+
+
+
+def update_radio_options(token_counts, line_counts, cyclomatic_complexity, problem_type):
+    options = []
+    if token_counts:
+        options.append("The Number of Tokens in Problem Descriptions")
+    if line_counts:
+        options.append("The Number of Lines in Problem Descriptions")
+    if cyclomatic_complexity:
+        options.append("The Complexity of Reference Code")
+    if problem_type:
+        options.append("Problem Type")
+
+    return gr.update(choices=options)
+
+def plot_csv(dataset_radio,radio,num):
+    print(dataset_radio,radio)
+    if dataset_radio=="HumanEval":
+    
+        if radio=="The Number of Tokens in Problem Descriptions":
+            radio_choice="token_counts"
+            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="The Number of Lines in Problem Descriptions":
+            radio_choice="line_counts"
+            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="The Complexity of Reference Code":
+            radio_choice="CC"
+            file_path = f'/home/user/app/dividing_into_different_subsets/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="Problem Type":
+            radio_choice="problem_type"
+            file_path = f'/home/user/app/dividing_into_different_subsets/cata_result.csv'
+        print("test!")
+    elif dataset_radio=="MBPP":
+        if radio=="The Number of Tokens in Problem Descriptions":
+            radio_choice="token_counts"
+            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="The Number of Lines in Problem Descriptions":
+            radio_choice="line_counts"
+            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="The Complexity of Reference Code":
+            radio_choice="CC"
+            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/{num}/QS/{radio_choice}_QS.csv'
+        elif radio=="Problem Type":
+            radio_choice="problem_type"
+            file_path = f'/home/user/app/dividing_into_different_subsets_mbpp/cata_result.csv'
+        print("test!")
+
+    # file_path="E:/python-testn/pythonProject3/hh_1/dividing_into_different_subsets/3/QS/CC_QS.csv"
+    df = pd.read_csv(file_path)
+    # 将第一列作为索引
+    df.set_index('Model', inplace=True)
+
+    # 转置数据框，使得模型作为列，横轴作为行
+    df_transposed = df.T
+
+    # 使用plotly绘制折线图
+    fig = px.line(df_transposed, x=df_transposed.index, y=df_transposed.columns,
+                  title='Model Evaluation Results',
+                  labels={'value': 'Evaluation Score', 'index': 'Evaluation Metric'},
+                  color_discrete_sequence=px.colors.qualitative.Plotly)
+
+    # 设置悬停效果
+    fig.update_traces(hovertemplate='%{y}')
+
+    return fig
+
+def toggle_radio(checkbox, radio):
+    return gr.update(visible=checkbox)
+
+def toggle_line_counts_visibility(dataset):
+    if dataset == "MBPP":
+        return gr.update(visible=False)
+    else:
+        return gr.update(visible=True)
+
+    # 创建 Gradio 界面
+import gradio as gr
+
+with gr.Blocks() as iface:
+    gr.HTML("""
+               <style>
+                   # body {
+                   #     max-width: 50%; /* 设置最大宽度为50% */
+                   #     margin: 0 auto; /* 将内容居中 */
+                   # }
+                   .title {
+                       text-align: center;
+                       font-size: 3em;
+                       font-weight: bold;
+                       margin-bottom: 0.5em;
+                   }
+                   .subtitle {
+                       text-align: center;
+                       font-size: 2em;
+                       margin-bottom: 1em;
+                   }
+               </style>
+
+           """)
+
+    with gr.Tabs() as tabs:
+        with gr.TabItem("Evaluation Result"):
+            with gr.Row():
+                with gr.Column(scale=2):
+                    with gr.Row():
+                        with gr.Column():
+                            dataset_radio = gr.Radio(["HumanEval", "MBPP"], label="Select Dataset ")
+
+
+            with gr.Row():
+                custom_css = """  
+                    <style>  
+                        .markdown-class {  
+                            font-family: 'Helvetica', sans-serif;
+                            font-size: 20px; 
+                            font-weight: bold;  
+                            color: #333; 
+                        }  
+                    </style>  
+                    """
+
+                with gr.Column():
+                    gr.Markdown(
+                        f"{custom_css}<div class='markdown-class'> Choose Division Perspective </div>")
+
+                    token_counts_checkbox = gr.Checkbox(label="I-The Number of Tokens in Problem Descriptions")
+                    line_counts_checkbox = gr.Checkbox(label="II-The Number of Lines in Problem Descriptions")
+                    dataset_radio.change(fn=toggle_line_counts_visibility, inputs=dataset_radio,
+                                         outputs=line_counts_checkbox)
+                    cyclomatic_complexity_checkbox = gr.Checkbox(label="III-The Complexity of Reference Code")
+                    problem_type_checkbox = gr.Checkbox(label="IV-Problem Types ")
+                css_code = """  
+                    .dropdown-container {  
+                        display: none;  
+                    }  
+                """
+
+                with gr.Column():
+                    # gr.Markdown("<div class='markdown-class'>Choose Subsets </div>")
+                    num_parts_dropdown = gr.Dropdown(choices=[0,3, 4, 5, 6, 7, 8], label="Choose the Number of Subsets",value="")
+
+            with gr.Row():
+                with gr.Column():
+                    token_counts_radio = gr.Radio(
+                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
+                        label="Choose the Division Method for Perspective-I",
+                        visible=False)
+                with gr.Column():
+                    line_counts_radio = gr.Radio(
+                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
+                        label="Choose the Division Method for Perspective-II",
+                        visible=False)
+                with gr.Column():
+                    cyclomatic_complexity_radio = gr.Radio(
+                        ["Equal Frequency Partitioning", "Equal Interval Partitioning"],
+                        label="Choose the Division Method for Perspective-III",
+                        visible=False)
+
+            token_counts_checkbox.change(fn=lambda x: toggle_radio(x, token_counts_radio),
+                                         inputs=token_counts_checkbox, outputs=token_counts_radio)
+            line_counts_checkbox.change(fn=lambda x: toggle_radio(x, line_counts_radio),
+                                        inputs=line_counts_checkbox, outputs=line_counts_radio)
+            cyclomatic_complexity_checkbox.change(fn=lambda x: toggle_radio(x, cyclomatic_complexity_radio),
+                                                  inputs=cyclomatic_complexity_checkbox,
+                                                  outputs=cyclomatic_complexity_radio)
+
+            with gr.Tabs() as inner_tabs:
+                with gr.TabItem("Ranking Table"):
+                    dataframe_output = gr.Dataframe(elem_id="dataframe")
+                    css_output = gr.HTML()
+                    confirm_button = gr.Button("Confirm ")
+                    confirm_button.click(fn=on_confirm, inputs=[dataset_radio, num_parts_dropdown, token_counts_radio,
+                                                                line_counts_radio, cyclomatic_complexity_radio,
+                                                                problem_type_checkbox],
+                                         outputs=dataframe_output)
+
+                with gr.TabItem("Line chart"):
+                    select_radio = gr.Radio(choices=[], label="Select One Perpective")
+                    checkboxes = [token_counts_checkbox, line_counts_checkbox, cyclomatic_complexity_checkbox,
+                                  problem_type_checkbox]
+                    for checkbox in checkboxes:
+                        checkbox.change(fn=update_radio_options, inputs=checkboxes, outputs=select_radio)
+                    select_radio.change(fn=plot_csv, inputs=[dataset_radio, select_radio, num_parts_dropdown],
+                                        outputs=gr.Plot(label="Line Plot "))
+
+        # with gr.TabItem("Upload Inference File"):
+        #     gr.Markdown("Upload a JSON file")
+        #     with gr.Row():
+        #         with gr.Column():
+        #             string_input = gr.Textbox(label="Enter the Model Name")
+        #             number_input = gr.Number(label="Select the Number of Samples")
+        #             dataset_choice = gr.Dropdown(label="Select Dataset", choices=["HumanEval", "MBPP"])
+        #         with gr.Column():
+        #             file_input = gr.File(label="Upload Generation Result in JSON file")
+        #             upload_button = gr.Button("Confirm and Upload")
+
+        #     json_output = gr.JSON(label="")
+
+        #     upload_button.click(fn=generate_file, inputs=[file_input, string_input, number_input, dataset_choice],
+        #                         outputs=json_output)
+
+    css = """  
+        #scale1 {  
+    border: 1px solid rgba(0, 0, 0, 0.2); 
+    padding: 10px;  
+    border-radius: 8px;
+    background-color: #f9f9f9; 
+    box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); 
+}  
+        }  
+        """
+    gr.HTML(f"<style>{css}</style>")
+
+    # 初始化数据表格
+    # initial_df = show_data(False, False, False, False, False, False, False)
+    # initial_css = generate_css(False, False, False, False, True, False, False)
+    # dataframe_output.value = initial_df
+    # css_output.value = f"<style>{initial_css}</style>"
+
+# 启动界面
+iface.launch()

llm_insight/HumanEval/3/EI/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high scores across all subsets, making it the most reliable choice."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance in two subsets, though with a drop in one, indicating good but not perfect robustness."
+        }
+    ],
+    "Moderate performance with cost-effectiveness": [
+        {
+            "deepseek_coder_33b-instruct": "Decent performance across all subsets, offering a balance between cost and effectiveness."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Good performance in two subsets, suitable for scenarios where some variability is acceptable."
+        }
+    ],
+    "Not recommended": [
+        {
+            "codegemma-2b": "Extremely low performance across all subsets."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Poor performance, especially in one subset with 0.0 score."
+        }
+    ]
+}

llm_insight/HumanEval/3/EI/CC_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in CC_subset_1 (79.86) and CC_subset_3 (75.0), but a significant drop in CC_subset_2 (50.0). This suggests it may struggle with certain types of data in the CC perspective.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets (88.65, 75.0, 87.5), indicating robustness in handling diverse data splits under the CC perspective.",
+    "codegemma-2b": "Poor performance across all subsets (30.56, 2.5, 2.5), suggesting it is not suitable for tasks under the CC perspective.",
+    "codegemma-7b": "Moderate performance in CC_subset_1 (43.4) but very low in CC_subset_2 (12.5) and CC_subset_3 (18.75), indicating limited applicability.",
+    "codegemma-7b-it": "Better than codegemma-7b but still inconsistent (56.32, 32.5, 21.25), with performance dropping in later subsets.",
+    "deepseek-coder-1.3b-base": "Very low performance, especially in CC_subset_3 (0.0), making it unsuitable for this perspective.",
+    "deepseek-coder-6.7b-base": "Moderate in CC_subset_1 (50.31) but poor in others (14.06, 5.0), indicating limited use.",
+    "deepseek_coder-6.7b-instruct": "Strong in CC_subset_1 (74.62) and decent in CC_subset_2 (53.12), but drops in CC_subset_3 (36.25).",
+    "deepseek_coder_33b-base": "Moderate performance across subsets (55.9, 31.87, 22.5), with a downward trend.",
+    "deepseek_coder_33b-instruct": "Consistently decent performance (68.65, 49.69, 41.25), though not as high as Nxcode-CQ-7B.",
+    "codeqwen1.5-7b": "Inconsistent performance (55.0, 17.19, 57.5), with a notable drop in CC_subset_2.",
+    "new": "Similar to codegemma-7b-it (56.32, 32.5, 21.25), indicating no significant improvement.",
+    "global_insights": "Nxcode-CQ-7B is the top performer across all subsets, showing robustness. CodeFuse-DeepSeek-33b and deepseek_coder_33b-instruct also perform well but with some variability. Smaller models like codegemma-2b and deepseek-coder-1.3b-base are not recommended for this perspective."
+}

llm_insight/HumanEval/3/EI/line_counts_recommendation.json

@@ -0,0 +1,23 @@
+{
+    "Small to medium line counts with high accuracy": [
+        {
+            "Nxcode-CQ-7B": "Excels in line_subset_1 (88.82) and line_subset_2 (87.27), making it ideal for small to medium line counts."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in line_subset_1 (75.65) and line_subset_2 (67.42), suitable for similar scenarios."
+        }
+    ],
+    "Large line counts with acceptable performance": [
+        {
+            "CodeFuse-DeepSeek-33b": "Best performance in line_subset_3 (87.5), making it the top choice for large line counts."
+        }
+    ],
+    "Cost-effective for small line counts": [
+        {
+            "codegemma-7b-it": "Moderate performance in line_subset_1 (57.32) at a potentially lower cost than larger models."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Balanced performance in line_subset_1 (70.85) and line_subset_2 (60.0) for its size."
+        }
+    ]
+}

llm_insight/HumanEval/3/EI/line_counts_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in line_subset_3 (87.5) but drops significantly in line_subset_2 (72.73). This suggests it handles larger line counts better but may struggle with medium-sized inputs.",
+    "Nxcode-CQ-7B": "This model excels in line_subset_1 (88.82) and line_subset_2 (87.27) but performs poorly in line_subset_3 (63.75), indicating it's optimized for small to medium line counts but not for larger ones.",
+    "codegemma-2b": "Consistently poor performance across all subsets, with the worst in line_subset_3 (1.88). Not suitable for any line count scenario.",
+    "codegemma-7b": "Better than codegemma-2b but still underperforms, especially in line_subset_3 (11.88). Limited utility across line counts.",
+    "codegemma-7b-it": "Shows improvement over other codegemma models, particularly in line_subset_1 (57.32), but still struggles with larger line counts (26.88 in line_subset_3).",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, with line_subset_1 being the best (35.69). Not recommended for any scenario.",
+    "deepseek-coder-6.7b-base": "Moderate performance in line_subset_1 (49.47) and line_subset_2 (40.61), but very poor in line_subset_3 (8.13). Avoid for large line counts.",
+    "deepseek_coder-6.7b-instruct": "Strong in line_subset_1 (75.65) and line_subset_2 (67.42), but drops in line_subset_3 (26.25). Suitable for small to medium line counts.",
+    "deepseek_coder_33b-base": "Decent in line_subset_1 (57.4) and line_subset_2 (44.55), but poor in line_subset_3 (15.0). Limited to small-medium line counts.",
+    "deepseek_coder_33b-instruct": "Good performance in line_subset_1 (70.85) and line_subset_2 (60.0), but struggles with line_subset_3 (18.75). Best for small-medium line counts.",
+    "codeqwen1.5-7b": "Moderate across all subsets, with line_subset_1 (53.7) being the best. Not outstanding in any scenario.",
+    "new": "Similar to codegemma-7b-it, with identical scores. Shows potential for small-medium line counts but not for large ones.",
+    "global_insights": "Models generally perform best in line_subset_1 and worst in line_subset_3, indicating a trend of decreasing performance with increasing line counts. Nxcode-CQ-7B and CodeFuse-DeepSeek-33b are exceptions, excelling in different subsets. Smaller models (e.g., codegemma-2b) are consistently poor, while larger models (e.g., deepseek_coder-6.7b-instruct) show better adaptability."
+}

llm_insight/HumanEval/3/EI/token_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance with cost-effectiveness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance in shorter and medium token subsets, making it suitable for most tasks."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance in shorter and longer token subsets, ideal for tasks with varying token lengths."
+        }
+    ],
+    "Moderate performance with lower cost": [
+        {
+            "deepseek_coder-6.7b-instruct": "Good performance in shorter and medium token subsets, suitable for budget-conscious users."
+        },
+        {
+            "codegemma-7b-it": "Better performance than base models, offering a balance between cost and capability."
+        }
+    ],
+    "Not recommended for complex tasks": [
+        {
+            "codegemma-2b": "Poor performance across all subsets, not suitable for any serious tasks."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Low scores across all subsets, indicating limited utility."
+        }
+    ]
+}

llm_insight/HumanEval/3/EI/token_counts_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in token_subset_1 and token_subset_3 but a significant drop in token_subset_2, indicating potential sensitivity to medium-length tokens.",
+    "Nxcode-CQ-7B": "Consistently high performance across token_subset_1 and token_subset_2, but a notable decline in token_subset_3, suggesting challenges with longer token sequences.",
+    "codegemma-2b": "Poor performance across all subsets, with particularly low scores in token_subset_3, indicating it may not be suitable for tasks involving varying token lengths.",
+    "codegemma-7b": "Moderate performance with a steady decline as token length increases, suggesting limited scalability with longer inputs.",
+    "codegemma-7b-it": "Better performance than codegemma-7b but still struggles with longer tokens, indicating some improvement but not enough for complex tasks.",
+    "deepseek-coder-1.3b-base": "Low scores across all subsets, with a slight improvement in token_subset_1, indicating limited capability.",
+    "deepseek-coder-6.7b-base": "Moderate performance in token_subset_1 but significant drops in longer tokens, suggesting scalability issues.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in token_subset_1 and token_subset_2 but a sharp decline in token_subset_3, indicating challenges with very long tokens.",
+    "deepseek_coder_33b-base": "Moderate performance with a steady decline as token length increases, similar to codegemma-7b.",
+    "deepseek_coder_33b-instruct": "Good performance in token_subset_1 and token_subset_3 but a drop in token_subset_2, indicating inconsistency with medium-length tokens.",
+    "codeqwen1.5-7b": "Moderate performance across all subsets, with a steady decline as token length increases, suggesting limited scalability.",
+    "new": "Identical performance to codegemma-7b-it, indicating no additional improvements.",
+    "global_insights": "Models generally perform best on shorter token sequences (token_subset_1) and struggle with longer ones (token_subset_3). Nxcode-CQ-7B and CodeFuse-DeepSeek-33b are the top performers but show inconsistency across subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base are not recommended for tasks involving varying token lengths."
+}

llm_insight/HumanEval/3/QS/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across all subsets, making it ideal for scenarios requiring reliable and stable outputs."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Good balance of performance and adaptability, suitable for diverse data conditions."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "deepseek_coder_33b-instruct": "Offers decent performance at a potentially lower cost compared to larger models like Nxcode-CQ-7B."
+        },
+        {
+            "codeqwen1.5-7b": "Provides moderate performance at a lower computational cost, suitable for budget-constrained scenarios."
+        }
+    ],
+    "Low-cost with acceptable performance": [
+        {
+            "codegemma-7b-it": "While not the best performer, it offers a cost-effective solution for less critical applications."
+        },
+        {
+            "deepseek-coder-6.7b-base": "A budget-friendly option with moderate performance, suitable for non-critical tasks."
+        }
+    ]
+}

llm_insight/HumanEval/3/QS/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows a consistent decline in performance across subsets, indicating potential sensitivity to dataset variations. It performs best in CC_subset_1 but drops significantly in CC_subset_3.",
+    "Nxcode-CQ-7B": "This model demonstrates robust performance across all subsets, with minimal variation. It maintains high accuracy, suggesting strong generalization capabilities.",
+    "codegemma-2b": "Performance degrades sharply across subsets, indicating poor adaptability to varying data conditions. The model struggles significantly in CC_subset_3.",
+    "codegemma-7b": "Similar to codegemma-2b, this model shows a steady decline in performance, though it starts from a higher baseline. It may not be suitable for diverse data conditions.",
+    "codegemma-7b-it": "While performance declines across subsets, the drop is less severe compared to other codegemma variants. It maintains moderate accuracy in CC_subset_3.",
+    "deepseek-coder-1.3b-base": "This model exhibits a steep performance drop, particularly in CC_subset_3, suggesting limited robustness to dataset changes.",
+    "deepseek-coder-6.7b-base": "Performance declines steadily but remains above smaller models like codegemma-2b. It shows moderate adaptability.",
+    "deepseek_coder-6.7b-instruct": "This model maintains relatively high performance across subsets, with a slight decline in CC_subset_3. It shows good generalization.",
+    "deepseek_coder_33b-base": "Performance drops steadily but remains above smaller models. It shows moderate robustness to dataset variations.",
+    "deepseek_coder_33b-instruct": "The model maintains decent performance across subsets, with a gradual decline. It shows better adaptability than its base counterpart.",
+    "codeqwen1.5-7b": "Performance declines across subsets but remains above smaller models. It shows moderate adaptability to varying data conditions.",
+    "global_insights": "Larger models generally perform better across subsets, with Nxcode-CQ-7B being the most consistent. Performance degradation is observed in all models as dataset complexity increases (from CC_subset_1 to CC_subset_3). Instruct-tuned models tend to outperform their base counterparts, suggesting the value of instruction fine-tuning."
+}

llm_insight/HumanEval/3/QS/line_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy and consistency": [
+        {
+            "Nxcode-CQ-7B": "Consistently outperforms all other models across all subsets, making it the best choice for high accuracy tasks."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance and robustness, though slightly less consistent than Nxcode-CQ-7B."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "Good performance at a potentially lower cost compared to larger models."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Better performance than its base version, suitable for tasks needing moderate accuracy."
+        }
+    ],
+    "Low-cost for simpler tasks": [
+        {
+            "codegemma-7b-it": "Decent performance for simpler tasks, though not suitable for high accuracy needs."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance at a lower cost, suitable for less critical tasks."
+        }
+    ]
+}

llm_insight/HumanEval/3/QS/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance across all subsets, with the highest score in subset_1 (81.82) and a slight drop in subset_2 (72.22) and subset_3 (76.36). This indicates robustness but some variability based on line counts.",
+    "Nxcode-CQ-7B": "This model outperforms all others, especially in subset_1 (92.09) and maintains high scores in subset_2 (88.33) and subset_3 (81.45). Its consistency and high performance make it a top contender.",
+    "codegemma-2b": "The model performs poorly across all subsets, with the lowest scores (44.09, 17.5, 19.64). It is not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Performance is mediocre, with scores (52.45, 35.19, 31.64) showing a significant drop as line counts increase. Not recommended for complex tasks.",
+    "codegemma-7b-it": "Better than its non-IT counterpart, but still shows a declining trend (66.36, 49.26, 43.73). Suitable for simpler tasks but not for high accuracy needs.",
+    "deepseek-coder-1.3b-base": "Low scores (47.45, 26.39, 23.0) indicate this model is not suitable for tasks requiring precision or handling larger line counts.",
+    "deepseek-coder-6.7b-base": "Moderate performance (63.36, 39.35, 34.18) but declines with larger subsets. Limited utility for complex tasks.",
+    "deepseek_coder-6.7b-instruct": "Strong performance (85.0, 66.85, 62.82) but not as consistent as Nxcode-CQ-7B. Good for tasks needing moderate accuracy.",
+    "deepseek_coder_33b-base": "Decent performance (68.0, 48.89, 41.27) but declines with larger subsets. Not the best choice for high accuracy.",
+    "deepseek_coder_33b-instruct": "Good performance (82.09, 62.31, 53.91) but still not as consistent as Nxcode-CQ-7B. Suitable for tasks needing moderate accuracy.",
+    "codeqwen1.5-7b": "Moderate scores (59.73, 48.7, 45.64) with a declining trend. Limited utility for high accuracy tasks.",
+    "global_insights": "Nxcode-CQ-7B is the top performer across all subsets, showing both high accuracy and consistency. Smaller models like codegemma-2b and deepseek-coder-1.3b-base perform poorly, especially with larger line counts. Instruct versions of models generally perform better than their base counterparts. Line count significantly impacts performance for most models, with larger subsets showing lower scores."
+}

llm_insight/HumanEval/3/QS/token_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high scores across all token subsets, making it reliable for varied token counts."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in all subsets, suitable for tasks requiring stability across token counts."
+        }
+    ],
+    "Balanced performance and cost-effectiveness": [
+        {
+            "deepseek_coder_33b-instruct": "Good performance with a reasonable drop in higher token counts, offering a balance between cost and capability."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance with a stable decline, suitable for tasks where cost is a concern but performance cannot be compromised too much."
+        }
+    ],
+    "Low-cost for low token count tasks": [
+        {
+            "codegemma-7b-it": "Performs well in low to medium token counts, offering a cost-effective solution for smaller tasks."
+        },
+        {
+            "deepseek-coder-6.7b-base": "Better than smaller models and cost-effective for tasks with moderate token counts."
+        }
+    ]
+}

llm_insight/HumanEval/3/QS/token_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in token_subset_2 (88.89) but drops significantly in token_subset_3 (69.09), indicating potential sensitivity to token count variations.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, with the highest score in token_subset_1 (90.73). This suggests robustness to token count changes.",
+    "codegemma-2b": "Performance declines sharply as token count increases, with the lowest score in token_subset_3 (11.82). Not suitable for high token count tasks.",
+    "codegemma-7b": "Similar to codegemma-2b but with better overall performance, though still struggles with higher token counts.",
+    "codegemma-7b-it": "Shows a steady decline with increasing token counts but maintains relatively better performance than other codegemma variants.",
+    "deepseek-coder-1.3b-base": "Performance drops significantly with higher token counts, similar to codegemma models.",
+    "deepseek-coder-6.7b-base": "Better than 1.3b-base but still shows a notable decline in token_subset_3 (25.09).",
+    "deepseek_coder-6.7b-instruct": "Strong performance across all subsets, with the highest score in token_subset_1 (83.64). Robust to token count variations.",
+    "deepseek_coder_33b-base": "Moderate performance with a steady decline in higher token counts.",
+    "deepseek_coder_33b-instruct": "Good performance overall but shows a noticeable drop in token_subset_3 (50.36).",
+    "codeqwen1.5-7b": "Performance declines with higher token counts but remains relatively stable compared to other models.",
+    "global_insights": "Models like Nxcode-CQ-7B and deepseek_coder-6.7b-instruct show robustness across token count variations, while smaller models (e.g., codegemma-2b, deepseek-coder-1.3b-base) struggle with higher token counts. Larger models generally perform better but may still show declines in the highest token subset."
+}

llm_insight/HumanEval/4/EI/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance with no cost constraints": [
+        {
+            "Nxcode-CQ-7B": "Consistently top performer across most subsets, making it the best choice for high-stakes CC tasks."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance in multiple subsets, though slightly less consistent than Nxcode-CQ-7B."
+        }
+    ],
+    "Balanced performance and cost": [
+        {
+            "deepseek_coder_33b-instruct": "Reasonable performance across subsets, offering a good balance between cost and capability."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance in some subsets, potentially more cost-effective than larger models."
+        }
+    ],
+    "Low cost with acceptable performance": [
+        {
+            "codegemma-7b-it": "Better than other small models, though still inconsistent. Suitable for less critical tasks."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Better among smaller deepseek models, but still has significant performance drops."
+        }
+    ]
+}

llm_insight/HumanEval/4/EI/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in CC_subset_1 and CC_subset_4 but drops significantly in CC_subset_2 and CC_subset_3. This suggests it may struggle with certain types of CC-related tasks.",
+    "Nxcode-CQ-7B": "This model performs exceptionally well in CC_subset_1, CC_subset_2, and CC_subset_4, but has a notable drop in CC_subset_3. It's consistently one of the top performers across most subsets.",
+    "codegemma-2b": "The model performs poorly across all subsets, particularly in CC_subset_3 and CC_subset_4 where it scores 0.0 and 2.5 respectively. It's not suitable for CC-related tasks.",
+    "codegemma-7b": "While better than its 2b counterpart, this model still struggles, especially in CC_subset_3 where it scores 0.0. It shows some improvement in other subsets but remains weak overall.",
+    "codegemma-7b-it": "This model shows moderate performance in CC_subset_1 and CC_subset_2 but drops in CC_subset_3 and CC_subset_4. It's better than other codegemma versions but still inconsistent.",
+    "deepseek-coder-1.3b-base": "The model performs poorly across all subsets, particularly in CC_subset_3 and CC_subset_4 where it scores 0.0. It's not recommended for CC tasks.",
+    "deepseek-coder-6.7b-base": "This model shows some capability in CC_subset_1 and CC_subset_2 but fails completely in CC_subset_3 and CC_subset_4. It's inconsistent and not reliable.",
+    "deepseek_coder-6.7b-instruct": "The model performs well in CC_subset_1 and CC_subset_2 but drops significantly in CC_subset_3 and CC_subset_4. It's a better option among deepseek models but still inconsistent.",
+    "deepseek_coder_33b-base": "This model shows moderate performance in CC_subset_1 and CC_subset_2 but struggles in CC_subset_3 and CC_subset_4. It's better than smaller deepseek models but not top-tier.",
+    "deepseek_coder_33b-instruct": "The model performs reasonably well across all subsets, though it drops in CC_subset_3 and CC_subset_4. It's one of the better deepseek models but still not the best.",
+    "codeqwen1.5-7b": "This model shows moderate performance in CC_subset_1 and CC_subset_2, fails in CC_subset_3, but recovers somewhat in CC_subset_4. It's inconsistent but has some strengths.",
+    "global_insights": "Nxcode-CQ-7B is the top performer overall, excelling in most subsets. CodeFuse-DeepSeek-33b also shows strong performance in some subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base perform poorly across the board. The CC_subset_3 appears to be the most challenging for all models, with many scoring 0.0. Models generally perform better in CC_subset_1 and CC_subset_2, suggesting these subsets may contain less complex tasks."
+}

llm_insight/HumanEval/4/EI/line_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy for small to medium tasks": [
+        {
+            "Nxcode-CQ-7B": "Best performance in subsets 1 and 2, suitable for tasks with smaller line counts."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Consistent performance across all subsets, reliable for varied tasks."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "Good balance of performance and cost for smaller tasks."
+        },
+        {
+            "codegemma-7b-it": "Cost-effective for moderate line counts with decent performance."
+        }
+    ],
+    "Not recommended": [
+        {
+            "codegemma-2b": "Poor performance across all subsets."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Inconsistent and low performance."
+        }
+    ]
+}

llm_insight/HumanEval/4/EI/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "Consistent performance across subsets, with a slight dip in line_subset_2. Demonstrates robustness in handling varying line counts.",
+    "Nxcode-CQ-7B": "Highest performance in line_subset_1 and line_subset_2, but significant drop in line_subset_3. May struggle with larger line counts.",
+    "codegemma-2b": "Poor performance across all subsets, especially in line_subset_3 and line_subset_4. Not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Better than codegemma-2b but still underperforms in larger line counts. Limited scalability.",
+    "codegemma-7b-it": "Improved over codegemma-7b, especially in line_subset_4. Shows potential for tasks with moderate line counts.",
+    "deepseek-coder-1.3b-base": "Low performance overall, with a surprising spike in line_subset_4. Inconsistent behavior.",
+    "deepseek-coder-6.7b-base": "Moderate performance, but drops significantly in line_subset_3 and line_subset_4. Limited reliability.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in line_subset_1 and line_subset_2, but drops in larger subsets. Best for smaller tasks.",
+    "deepseek_coder_33b-base": "Decent performance but inconsistent across subsets. May require fine-tuning.",
+    "deepseek_coder_33b-instruct": "Good performance in line_subset_1 and line_subset_2, but drops sharply in line_subset_4. Unpredictable for larger tasks.",
+    "codeqwen1.5-7b": "Moderate performance across subsets, with a dip in line_subset_3. Balanced but not outstanding.",
+    "global_insights": "Models generally perform better in smaller line counts (subsets 1 and 2). Larger subsets (3 and 4) show significant performance drops, indicating scalability issues. Nxcode-CQ-7B and CodeFuse-DeepSeek-33b are top performers but have trade-offs. Smaller models like codegemma-2b and deepseek-coder-1.3b-base are not recommended for any serious tasks."
+}

llm_insight/HumanEval/4/EI/token_counts_recommendation.json

@@ -0,0 +1,23 @@
+{
+    "High performance with long inputs": [
+        {
+            "CodeFuse-DeepSeek-33b": "This model is the only one that handles the longest token subset perfectly, making it ideal for tasks requiring processing of very long inputs."
+        }
+    ],
+    "Balanced performance across most inputs": [
+        {
+            "Nxcode-CQ-7B": "This model performs consistently well across most token subsets, making it a reliable choice for general tasks with varying input lengths."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "This model shows strong performance in shorter to medium token subsets, making it suitable for tasks where inputs are not extremely long."
+        }
+    ],
+    "Cost-effective for shorter inputs": [
+        {
+            "codegemma-7b-it": "This model offers reasonable performance for shorter inputs at a lower computational cost compared to larger models."
+        },
+        {
+            "codeqwen1.5-7b": "This model provides a balance between performance and cost for tasks with shorter to medium input lengths."
+        }
+    ]
+}

llm_insight/HumanEval/4/EI/token_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in token_subset_4 with a perfect score of 100.0, but its performance degrades as token counts decrease, indicating potential sensitivity to input length.",
+    "Nxcode-CQ-7B": "This model performs consistently well across token_subset_1 to token_subset_3 but shows a significant drop in token_subset_4, suggesting it may struggle with very long inputs.",
+    "codegemma-2b": "The model's performance is poor across all subsets, with particularly low scores in token_subset_3 and token_subset_4, indicating it may not be suitable for tasks requiring handling of varying token lengths.",
+    "codegemma-7b": "While better than its 2b counterpart, this model still struggles with longer token subsets, showing a clear decline in performance as token counts increase.",
+    "codegemma-7b-it": "This model shows a moderate improvement over the base 7b model, particularly in token_subset_3, but still fails to handle token_subset_4.",
+    "deepseek-coder-1.3b-base": "The model's performance is weak across all subsets, with scores dropping sharply as token counts increase, making it unsuitable for tasks with longer inputs.",
+    "deepseek-coder-6.7b-base": "This model shows better performance than the 1.3b version but still struggles with longer token subsets.",
+    "deepseek_coder-6.7b-instruct": "The model performs well in token_subset_1 and token_subset_2 but shows a significant drop in token_subset_3 and fails in token_subset_4.",
+    "deepseek_coder_33b-base": "This model shows decent performance in token_subset_1 but declines as token counts increase, indicating limitations with longer inputs.",
+    "deepseek_coder_33b-instruct": "The model performs reasonably well in token_subset_1 and token_subset_2 but struggles with longer token subsets.",
+    "codeqwen1.5-7b": "This model shows moderate performance across subsets but fails to handle token_subset_4, similar to other models.",
+    "global_insights": "Most models struggle with longer token subsets, with only CodeFuse-DeepSeek-33b showing perfect performance in token_subset_4. Nxcode-CQ-7B performs consistently well across most subsets but fails in the longest subset. Smaller models generally perform worse, especially with longer inputs. The instruct versions of models tend to perform better than their base counterparts."
+}

llm_insight/HumanEval/4/QS/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance with robust generalization": [
+        {
+            "Nxcode-CQ-7B": "Consistently high scores across all subsets, indicating strong generalization and reliability."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Despite the decline, it maintains competitive performance, suitable for tasks where initial high accuracy is critical."
+        }
+    ],
+    "Cost-effective with moderate performance": [
+        {
+            "deepseek_coder-6.7b-instruct": "Balances performance and cost, especially beneficial with instruction tuning."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Larger but still cost-effective for tasks needing stable performance."
+        }
+    ],
+    "Budget-constrained scenarios": [
+        {
+            "codegemma-7b-it": "Reasonable performance for its size, suitable when resources are limited."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance at a lower computational cost."
+        }
+    ]
+}

llm_insight/HumanEval/4/QS/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows a consistent decline in performance across subsets, indicating potential overfitting or lack of generalization. The drop from 85.37 to 68.29 suggests sensitivity to data distribution shifts.",
+    "Nxcode-CQ-7B": "This model demonstrates strong performance across all subsets, with a slight dip in CC_subset_2. Its high scores in CC_subset_3 and CC_subset_4 indicate robustness to varying data conditions.",
+    "codegemma-2b": "Poor performance across all subsets, with particularly low scores in CC_subset_4. This suggests the model may lack the capacity for the task or requires significant fine-tuning.",
+    "codegemma-7b": "Better than its 2b counterpart but still underperforms compared to larger models. The performance drop in CC_subset_4 is significant, indicating limitations in handling more complex data.",
+    "codegemma-7b-it": "Improved over the base 7b model, especially in CC_subset_2 and CC_subset_4, suggesting that instruction tuning helps but may not be sufficient for top-tier performance.",
+    "deepseek-coder-1.3b-base": "Low scores across all subsets, similar to codegemma-2b, indicating that small models struggle with this task regardless of architecture.",
+    "deepseek-coder-6.7b-base": "Moderate performance with a notable drop in CC_subset_4. The model shows potential but may need further optimization or scaling.",
+    "deepseek_coder-6.7b-instruct": "Strong performance, especially in CC_subset_1 and CC_subset_3, indicating that instruction tuning significantly benefits this model size.",
+    "deepseek_coder_33b-base": "Decent performance but inconsistent, with a significant drop in CC_subset_4. The base model may need fine-tuning to stabilize performance.",
+    "deepseek_coder_33b-instruct": "Consistently good performance across subsets, though not the best. Instruction tuning helps, but the model may still lag behind the top performers.",
+    "codeqwen1.5-7b": "Moderate performance with a steady decline across subsets. The model shows promise but may need further scaling or tuning.",
+    "global_insights": "Larger models generally perform better, with Nxcode-CQ-7B and CodeFuse-DeepSeek-33b leading. Instruction tuning (e.g., deepseek_coder-6.7b-instruct) significantly boosts performance. Smaller models (codegemma-2b, deepseek-coder-1.3b-base) struggle, indicating a minimum model size is required for this task. Performance drops in later subsets suggest increasing complexity or distribution shifts that challenge generalization."
+}

llm_insight/HumanEval/4/QS/line_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance with moderate cost": [
+        {
+            "Nxcode-CQ-7B": "Maintains high performance across all subsets, making it suitable for tasks requiring consistent accuracy."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Offers robust performance across subsets, ideal for tasks needing reliable results with varying line counts."
+        }
+    ],
+    "Balanced performance and cost": [
+        {
+            "CodeFuse-DeepSeek-33b": "Good performance in smaller subsets, suitable for tasks where line counts are generally low."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Performs well in smaller subsets and moderately in larger ones, offering a balance between cost and performance."
+        }
+    ],
+    "Low cost with acceptable performance for small tasks": [
+        {
+            "codegemma-7b-it": "Performs adequately in smaller subsets, suitable for budget-conscious projects with limited line counts."
+        },
+        {
+            "codeqwen1.5-7b": "Offers moderate performance at a lower cost, ideal for smaller-scale tasks."
+        }
+    ]
+}

llm_insight/HumanEval/4/QS/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows consistent performance across subsets 1 and 2 but drops significantly in subsets 3 and 4, indicating potential sensitivity to line count variations.",
+    "Nxcode-CQ-7B": "This model maintains relatively high performance across all subsets, though there is a gradual decline as line counts increase, suggesting robustness but with some sensitivity to complexity.",
+    "codegemma-2b": "Performance drops sharply from subset 1 to subset 2 and remains low, indicating poor scalability with increasing line counts.",
+    "codegemma-7b": "Similar to codegemma-2b, this model shows a significant drop in performance after subset 1, though it performs slightly better overall.",
+    "codegemma-7b-it": "This model performs better than its non-IT counterpart but still shows a decline as line counts increase, suggesting limited scalability.",
+    "deepseek-coder-1.3b-base": "Performance declines sharply after subset 1, indicating poor handling of larger code segments.",
+    "deepseek-coder-6.7b-base": "Shows a moderate drop in performance across subsets, with a significant decline in subset 3, suggesting limitations with more complex code.",
+    "deepseek_coder-6.7b-instruct": "Maintains high performance in subsets 1 and 2 but drops in subsets 3 and 4, indicating sensitivity to line count increases.",
+    "deepseek_coder_33b-base": "Performance declines steadily across subsets, suggesting a linear relationship between line count and performance drop.",
+    "deepseek_coder_33b-instruct": "Shows a gradual decline in performance across subsets, but maintains relatively higher scores compared to other models.",
+    "codeqwen1.5-7b": "Performance drops gradually across subsets, indicating moderate sensitivity to line count increases.",
+    "global_insights": "Models generally perform better on smaller line counts (subsets 1 and 2) and decline as line counts increase (subsets 3 and 4). Nxcode-CQ-7B and deepseek_coder_33b-instruct show the most robustness across all subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base struggle significantly with larger line counts."
+}

llm_insight/HumanEval/4/QS/token_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance and robustness across all token counts": [
+        {
+            "Nxcode-CQ-7B": "Consistently high scores across all subsets, making it reliable for varied token counts."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in most subsets, though slightly drops in the highest token count."
+        }
+    ],
+    "Moderate performance with cost-effectiveness": [
+        {
+            "codegemma-7b-it": "Better performance than codegemma-7b and codegemma-2b, but still affordable."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance with a balance of cost and capability."
+        }
+    ],
+    "Not recommended for high token count tasks": [
+        {
+            "codegemma-2b": "Poor performance across all subsets, especially in higher token counts."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Sharp decline in performance with higher token counts."
+        }
+    ]
+}

llm_insight/HumanEval/4/QS/token_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in token_subset_2 (85.37) but drops significantly in token_subset_4 (63.41). This suggests it handles moderate token counts well but struggles with higher token counts.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, especially in token_subset_1 (93.9). This indicates robustness across varying token counts.",
+    "codegemma-2b": "Poor performance across all subsets, with a steep decline in token_subset_4 (8.05). Not suitable for tasks with high token counts.",
+    "codegemma-7b": "Moderate performance, but drops significantly in token_subset_3 (27.56) and token_subset_4 (26.34). Limited capability with higher token counts.",
+    "codegemma-7b-it": "Better than codegemma-7b, but still shows a decline in token_subset_3 (37.07) and token_subset_4 (41.83).",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, with a sharp drop in token_subset_4 (12.32). Not recommended for high token count tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance, but declines in token_subset_3 (36.34) and token_subset_4 (23.9).",
+    "deepseek_coder-6.7b-instruct": "Strong performance in token_subset_1 (85.98) and token_subset_3 (72.8), but drops in token_subset_4 (56.34).",
+    "deepseek_coder_33b-base": "Moderate performance, with a decline in token_subset_4 (32.8).",
+    "deepseek_coder_33b-instruct": "Strong performance across all subsets, though it drops in token_subset_4 (46.1).",
+    "codeqwen1.5-7b": "Moderate performance, with a decline in token_subset_3 (39.88) and token_subset_4 (40.37).",
+    "global_insights": "Nxcode-CQ-7B and deepseek_coder-6.7b-instruct show the most consistent performance across all token subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base struggle with higher token counts. Larger models generally perform better but still show declines in the highest token subset."
+}

llm_insight/HumanEval/5/EI/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across most subsets, making it reliable for diverse tasks."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance in specific subsets, suitable for tasks similar to CC_subset_1 and CC_subset_4."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "Decent performance at a potentially lower computational cost compared to larger models."
+        },
+        {
+            "codeqwen1.5-7b": "Balanced performance and may offer a good trade-off between cost and accuracy."
+        }
+    ],
+    "Not recommended": [
+        {
+            "codegemma-2b": "Poor performance across all subsets."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Very low scores, not suitable for any serious tasks."
+        }
+    ]
+}

llm_insight/HumanEval/5/EI/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in CC_subset_4 (100%) and CC_subset_1 (81.63%), but struggles in CC_subset_3 (33.33%). This indicates variability in handling different subsets, possibly due to the nature of the tasks in each subset.",
+    "Nxcode-CQ-7B": "Consistently high performance across most subsets (CC_subset_1: 87.35%, CC_subset_2: 88.73%, CC_subset_3: 84.17%, CC_subset_5: 98.33%), except for CC_subset_4 (37.5%). This suggests robustness but a potential weakness in tasks similar to CC_subset_4.",
+    "codegemma-2b": "Poor performance across all subsets, with the highest score being 31.63% in CC_subset_1. This model is not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Moderate performance, with the best score in CC_subset_1 (46.84%). Performance drops significantly in other subsets, indicating limited generalization.",
+    "codegemma-7b-it": "Better than codegemma-7b, with the highest score in CC_subset_1 (57.35%). However, performance is inconsistent, especially in CC_subset_4 (2.5%).",
+    "deepseek-coder-1.3b-base": "Very poor performance, with scores below 40% in all subsets. Not recommended for any serious tasks.",
+    "deepseek-coder-6.7b-base": "Slightly better than the 1.3b version, but still underperforms with a maximum score of 51.68% in CC_subset_1.",
+    "deepseek_coder-6.7b-instruct": "Decent performance in CC_subset_1 (73.16%) and CC_subset_2 (76.36%), but struggles in CC_subset_4 (0%).",
+    "deepseek_coder_33b-base": "Inconsistent performance, with highs in CC_subset_4 (40%) and lows in CC_subset_5 (3.33%).",
+    "deepseek_coder_33b-instruct": "Good performance in CC_subset_1 (67.86%), CC_subset_2 (67.09%), and CC_subset_3 (56.67%), but fails in CC_subset_4 (5%).",
+    "codeqwen1.5-7b": "Moderate performance across subsets, with the best score in CC_subset_4 (47.5%). Shows some consistency but lacks top-tier performance.",
+    "global_insights": "Nxcode-CQ-7B and CodeFuse-DeepSeek-33b are the top performers, with Nxcode-CQ-7B being more consistent. Smaller models like codegemma-2b and deepseek-coder-1.3b-base are not suitable for high-accuracy tasks. The instruct versions of models generally perform better than their base counterparts. CC_subset_4 appears to be the most challenging subset for most models."
+}

llm_insight/HumanEval/5/EI/line_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High Performance, High Cost": [
+        {
+            "CodeFuse-DeepSeek-33b": "Consistently high performance across most subsets, though with some variability."
+        },
+        {
+            "Nxcode-CQ-7B": "Strong performance in most scenarios, except for very long lines of code."
+        }
+    ],
+    "Balanced Performance and Cost": [
+        {
+            "deepseek_coder-6.7b-instruct": "Decent performance with instruction tuning, suitable for tasks not involving very long lines of code."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance across most subsets, a good balance for general use."
+        }
+    ],
+    "Low Cost, Limited Performance": [
+        {
+            "codegemma-7b-it": "Better than its base version but still limited, suitable for low-budget scenarios."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Instruction-tuned but inconsistent, only recommended for low-priority tasks."
+        }
+    ]
+}

llm_insight/HumanEval/5/EI/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance across most subsets, particularly excelling in line_subset_4 with a perfect score. However, there is variability in performance, indicating potential sensitivity to line count differences.",
+    "Nxcode-CQ-7B": "This model demonstrates consistent performance in line_subset_1, line_subset_2, and line_subset_3 but shows a significant drop in line_subset_4. This suggests potential limitations with very short or very long lines of code.",
+    "codegemma-2b": "The model performs poorly across all subsets, with particularly low scores in line_subset_4 and line_subset_5. This indicates a general lack of capability in handling varying line counts.",
+    "codegemma-7b": "While better than codegemma-2b, this model still struggles, especially with longer lines of code. Performance is inconsistent and generally low.",
+    "codegemma-7b-it": "This model shows improvement over its non-instruction-tuned counterpart but still has significant room for improvement, particularly in handling longer lines of code.",
+    "deepseek-coder-1.3b-base": "The model performs poorly, with very low scores in line_subset_4 and inconsistent performance elsewhere. Not suitable for tasks requiring robustness across line counts.",
+    "deepseek-coder-6.7b-base": "Performance is inconsistent, with a complete failure in line_subset_4. This model is not reliable for tasks involving varying line counts.",
+    "deepseek_coder-6.7b-instruct": "The model shows decent performance in line_subset_1 and line_subset_2 but struggles with longer lines. The instruction tuning helps but is not sufficient for all scenarios.",
+    "deepseek_coder_33b-base": "Performance is middling, with no standout strengths. The model is inconsistent across subsets, indicating limited robustness.",
+    "deepseek_coder_33b-instruct": "This model performs well in line_subset_1 and line_subset_3 but shows significant drops in line_subset_4 and line_subset_5. Instruction tuning helps but does not fully address the limitations.",
+    "codeqwen1.5-7b": "The model shows moderate performance across most subsets but struggles with longer lines. It is a middle-of-the-road option with no extreme weaknesses or strengths.",
+    "global_insights": "Models generally perform better with shorter lines of code (line_subset_1 and line_subset_2) and struggle with longer lines (line_subset_4 and line_subset_5). Instruction-tuned models tend to perform better than their base counterparts, but the improvement is not always significant. The best-performing models are CodeFuse-DeepSeek-33b and Nxcode-CQ-7B, but even they have notable weaknesses."
+}

llm_insight/HumanEval/5/EI/token_counts_recommendation.json

@@ -0,0 +1,29 @@
+{
+    "High performance with balanced token handling": [
+        {
+            "CodeFuse-DeepSeek-33b": "Consistently high performance across most token subsets, making it reliable for diverse token ranges."
+        },
+        {
+            "Nxcode-CQ-7B": "Strong performance in lower to mid token subsets, ideal for tasks not requiring very high token counts."
+        }
+    ],
+    "Cost-effective for lower token tasks": [
+        {
+            "deepseek_coder-6.7b-instruct": "Good performance in lower to mid token subsets at a potentially lower cost."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Balanced performance in lower subsets, suitable for tasks with moderate token counts."
+        }
+    ],
+    "Avoid for high token tasks": [
+        {
+            "codegemma-2b": "Poor performance across all subsets, especially in higher token counts."
+        },
+        {
+            "codegemma-7b": "Fails in higher token ranges, not recommended for robust tasks."
+        },
+        {
+            "codegemma-7b-it": "Better than other codegemma variants but still inadequate for high token tasks."
+        }
+    ]
+}

llm_insight/HumanEval/5/EI/token_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance across most subsets, particularly excelling in token_subset_5 with a perfect score. However, there is a noticeable dip in token_subset_3, suggesting potential variability in handling certain token ranges.",
+    "Nxcode-CQ-7B": "This model performs consistently well in subsets 1-4 but has a significant drop in token_subset_5, indicating a potential limitation with very high token counts.",
+    "codegemma-2b": "Poor performance across all subsets, especially in higher token counts, making it unsuitable for tasks requiring robust token handling.",
+    "codegemma-7b": "Moderate performance in lower token subsets but fails completely in higher token ranges, similar to codegemma-2b.",
+    "codegemma-7b-it": "Better than other codegemma variants but still struggles with higher token counts, showing a steep decline in performance.",
+    "deepseek-coder-1.3b-base": "Low performance across the board, with particularly poor results in higher token subsets.",
+    "deepseek-coder-6.7b-base": "Moderate performance in lower token subsets but fails to maintain consistency in higher ranges.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in lower to mid token subsets but drops significantly in token_subset_5, similar to Nxcode-CQ-7B.",
+    "deepseek_coder_33b-base": "Decent performance in lower token subsets but struggles as token counts increase.",
+    "deepseek_coder_33b-instruct": "Good performance in lower subsets but inconsistent in higher token ranges, with a notable drop in token_subset_3.",
+    "codeqwen1.5-7b": "Moderate performance across subsets but fails in higher token counts, similar to other models.",
+    "global_insights": "Models generally perform better in lower token subsets, with performance degrading as token counts increase. CodeFuse-DeepSeek-33b and Nxcode-CQ-7B are exceptions, showing strong performance in most subsets except the highest token range. This suggests that token count is a critical factor in model performance, and models vary significantly in their ability to handle different token ranges."
+}

llm_insight/HumanEval/5/QS/CC_recommendation.json

@@ -0,0 +1,29 @@
+{
+    "High accuracy and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across all subsets, making it the most reliable choice for high-accuracy tasks."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance across all subsets, particularly in subsets 1 and 4, indicating robustness."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Reliable performance across all subsets, with high scores in subsets 1 and 4, making it a strong contender for high-accuracy tasks."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "codegemma-7b-it": "Better performance than codegemma-7b and more cost-effective than larger models, suitable for tasks where moderate accuracy is acceptable."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance in subsets 1-4, making it a cost-effective choice for tasks where high accuracy is not critical."
+        }
+    ],
+    "Low accuracy for lightweight tasks": [
+        {
+            "codegemma-2b": "Suitable for lightweight tasks where accuracy is not a priority."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Low performance but may be suitable for very basic tasks where cost is a major concern."
+        }
+    ]
+}

llm_insight/HumanEval/5/QS/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in subsets 1-4 but drops significantly in subset 5, indicating potential issues with specific types of data in subset 5.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, with the highest scores in subsets 1 and 3. This model is robust and reliable across different data types.",
+    "codegemma-2b": "Poor performance across all subsets, particularly in subset 5. This model is not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Moderate performance with a noticeable drop in subset 5. The model struggles with certain data types but performs reasonably well in others.",
+    "codegemma-7b-it": "Better performance than codegemma-7b, especially in subsets 2-5, but still not as strong as the top-performing models.",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, similar to codegemma-2b. Not recommended for high-accuracy tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance with a significant drop in subset 5. The model is inconsistent across different data types.",
+    "deepseek_coder-6.7b-instruct": "Strong performance across all subsets, particularly in subsets 1 and 4. This model is reliable and performs well under various conditions.",
+    "deepseek_coder_33b-base": "Good performance in subsets 1-4 but drops in subset 5. The model is generally reliable but has some weaknesses.",
+    "deepseek_coder_33b-instruct": "Strong performance across all subsets, with particularly high scores in subsets 1 and 4. This model is robust and performs well under various conditions.",
+    "codeqwen1.5-7b": "Moderate performance with a noticeable drop in subset 5. The model is inconsistent but performs reasonably well in subsets 1-4.",
+    "global_insights": "The models Nxcode-CQ-7B, deepseek_coder-6.7b-instruct, and deepseek_coder_33b-instruct consistently perform well across all subsets, indicating robustness. Subset 5 appears to be challenging for most models, suggesting it contains more complex or diverse data. The codegemma and deepseek-coder-1.3b-base models generally underperform, making them less suitable for high-accuracy tasks."
+}

llm_insight/HumanEval/5/QS/line_counts_recommendation.json

@@ -0,0 +1,23 @@
+{
+    "High performance and robustness across varying line counts": [
+        {
+            "Nxcode-CQ-7B": "This model consistently performs well across all subsets, making it ideal for tasks requiring stability and high accuracy regardless of line count."
+        }
+    ],
+    "Balanced performance and cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "This model offers strong performance in lower line counts and is more cost-effective than larger models, suitable for tasks where line counts are moderate."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Provides decent performance across subsets and is a good balance between cost and performance for tasks with varying line counts."
+        }
+    ],
+    "Budget-conscious scenarios with lower line counts": [
+        {
+            "codeqwen1.5-7b": "A cost-effective option for tasks with lower line counts, though performance drops as line counts increase."
+        },
+        {
+            "codegemma-7b-it": "Suitable for budget-conscious scenarios where line counts are low to moderate, though performance is not as robust as larger models."
+        }
+    ]
+}

llm_insight/HumanEval/5/QS/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows consistent performance across subsets, with a peak in line_subset_2 (84.85). However, it has a noticeable dip in line_subset_4 (66.67), indicating potential sensitivity to certain line count distributions.",
+    "Nxcode-CQ-7B": "This model demonstrates strong and stable performance across all subsets, with the highest score in line_subset_1 (92.27) and maintaining above 87 in most subsets. It is robust to varying line counts.",
+    "codegemma-2b": "Performance is consistently low across all subsets, with the highest score in line_subset_1 (51.67) and dropping significantly in other subsets. This suggests the model struggles with tasks involving varying line counts.",
+    "codegemma-7b": "While better than codegemma-2b, this model still underperforms, with scores ranging from 28.44 to 60.45. It shows a declining trend as line counts increase.",
+    "codegemma-7b-it": "This model performs better than its non-IT counterpart, with scores ranging from 39.69 to 73.33. However, it still shows a decline as line counts increase.",
+    "deepseek-coder-1.3b-base": "The model's performance is poor, with scores ranging from 21.72 to 52.73. It shows a clear downward trend as line counts increase.",
+    "deepseek-coder-6.7b-base": "Performance is moderate, with scores ranging from 29.53 to 74.09. Like other models, it shows a decline with increasing line counts.",
+    "deepseek_coder-6.7b-instruct": "This model shows strong performance in line_subset_1 (79.55) and line_subset_2 (84.39), but drops significantly in line_subset_5 (52.03). It is sensitive to higher line counts.",
+    "deepseek_coder_33b-base": "Performance is moderate, with scores ranging from 38.59 to 75.0. It shows a consistent decline as line counts increase.",
+    "deepseek_coder_33b-instruct": "This model performs well in line_subset_1 (82.27) but shows a steady decline to 50.16 in line_subset_5. It is less robust to higher line counts.",
+    "codeqwen1.5-7b": "Performance is moderate, with scores ranging from 41.72 to 63.18. It shows a steady decline as line counts increase.",
+    "global_insights": "Most models show a decline in performance as line counts increase, indicating a general sensitivity to larger code blocks. Nxcode-CQ-7B stands out as the most robust model across all subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base struggle significantly, while larger models like Nxcode-CQ-7B and deepseek_coder-6.7b-instruct show better but varying performance."
+}

llm_insight/HumanEval/5/QS/token_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance and robustness": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across all token subsets, making it reliable for varying input sizes."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in most subsets, though it shows some inconsistency in the largest token subset."
+        }
+    ],
+    "Moderate performance with cost-effectiveness": [
+        {
+            "codegemma-7b-it": "Balances performance and cost, though it declines with larger token counts."
+        },
+        {
+            "deepseek_coder-33b-instruct": "Good performance in smaller token subsets and maintains a baseline in larger ones."
+        }
+    ],
+    "Low-cost with basic capabilities": [
+        {
+            "codeqwen1.5-7b": "Moderate performance at a lower cost, suitable for less demanding tasks."
+        },
+        {
+            "deepseek-coder-6.7b-base": "Basic performance at a lower cost, though it declines with larger token counts."
+        }
+    ]
+}

llm_insight/HumanEval/5/QS/token_counts_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in token_subset_3 (90.91) but has inconsistent results across other subsets, indicating potential instability with varying token counts.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, especially in token_subset_1 (95.45), suggesting robustness to token count variations.",
+    "codegemma-2b": "Poor performance across all subsets, with a significant drop in token_subset_5 (5.31), indicating it struggles with larger token counts.",
+    "codegemma-7b": "Moderate performance with a steady decline as token counts increase, suggesting limitations in handling larger inputs.",
+    "codegemma-7b-it": "Better than codegemma-7b but still shows a decline in performance with increasing token counts, though it maintains a baseline in token_subset_5 (39.22).",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, with a sharp drop in token_subset_4 (15.45) and token_subset_5 (10.94), indicating it is not suitable for larger token counts.",
+    "deepseek-coder-6.7b-base": "Moderate performance with a steady decline as token counts increase, similar to codegemma-7b.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in token_subset_1 (87.88) and token_subset_4 (76.36), but a significant drop in token_subset_5 (51.72), indicating inconsistency.",
+    "deepseek_coder_33b-base": "Moderate performance with a steady decline as token counts increase, similar to other base models.",
+    "deepseek_coder_33b-instruct": "Strong performance in token_subset_1 (87.12) but declines steadily, though it maintains a baseline in token_subset_4 (64.7).",
+    "codeqwen1.5-7b": "Moderate performance with a steady decline as token counts increase, similar to other 7b models.",
+    "new_model": "Performance mirrors codegemma-7b-it, suggesting similar capabilities and limitations.",
+    "global_insights": "Nxcode-CQ-7B and deepseek_coder-6.7b-instruct show the highest and most consistent performance across subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base struggle significantly with larger token counts. There is a general trend of declining performance as token counts increase, with some models like Nxcode-CQ-7B being exceptions."
+}

llm_insight/HumanEval/6/EI/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy tasks with tolerance for variability": [
+        {
+            "CodeFuse-DeepSeek-33b": "Achieves perfect scores in subsets 4 and 5, making it ideal for tasks where these subsets are representative."
+        },
+        {
+            "Nxcode-CQ-7B": "Excels in subsets 2 and 6, suitable for tasks prioritizing these data types."
+        }
+    ],
+    "Balanced performance and cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "Offers strong performance in multiple subsets without the extreme variability of larger models."
+        },
+        {
+            "codeqwen1.5-7b": "Provides good performance in subset 5 and moderate performance elsewhere, suitable for balanced tasks."
+        }
+    ],
+    "Low-resource environments": [
+        {
+            "codegemma-7b-it": "Better performance than its non-IT counterpart, suitable for environments with limited computational resources."
+        },
+        {
+            "deepseek-coder-6.7b-base": "Moderate performance at a lower computational cost compared to larger models."
+        }
+    ]
+}

llm_insight/HumanEval/6/EI/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in subsets 4 and 5 with perfect scores, but has a significant drop in subset 3 (46.67). This indicates potential variability in handling different types of data within the CC perspective.",
+    "Nxcode-CQ-7B": "This model excels in subsets 2 and 6 (91.41 and 98.33) but performs poorly in subset 4 (20.0). The wide range of scores suggests sensitivity to specific data characteristics.",
+    "codegemma-2b": "Consistently low performance across all subsets, with the highest score being 31.63 in subset 1. This model may not be suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Moderate performance with peaks in subset 1 (46.84) and subset 5 (40.0). The model struggles in subset 4 (0.0), indicating a potential weakness in certain data types.",
+    "codegemma-7b-it": "Better than its non-IT counterpart, with a high of 57.35 in subset 1. However, it still shows significant drops in subsets 4 and 5 (0.0 and 5.0).",
+    "deepseek-coder-1.3b-base": "Very low performance across all subsets, with no scores above 38.57. Not recommended for high-accuracy tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance with a high of 51.68 in subset 1. Similar to other models, it fails in subset 4 (0.0).",
+    "deepseek_coder-6.7b-instruct": "Strong performance in subsets 1, 2, and 3 (73.16, 77.72, 56.67), but fails in subset 4 (0.0). This suggests good generalizability except for specific data types.",
+    "deepseek_coder_33b-base": "Variable performance with a high of 80.0 in subset 5 but 0.0 in subset 4. This inconsistency may limit its reliability.",
+    "deepseek_coder_33b-instruct": "Consistently good performance in subsets 1, 2, and 3 (67.86, 70.33, 53.0), but struggles in subset 4 (0.0). The model shows promise but has clear limitations.",
+    "codeqwen1.5-7b": "Strong performance in subset 5 (95.0) but fails in subset 4 (0.0). The model's performance is otherwise moderate, indicating potential for specific use cases.",
+    "global_insights": "1. Most models struggle with subset 4, indicating a potential outlier or particularly challenging data type within the CC perspective. 2. Larger models (e.g., 33b variants) generally perform better but are not immune to significant drops in specific subsets. 3. The Nxcode-CQ-7B and CodeFuse-DeepSeek-33b models show the highest peaks but also the most variability, suggesting a trade-off between peak performance and consistency."
+}

llm_insight/HumanEval/6/EI/line_counts_recommendation.json

@@ -0,0 +1,31 @@
+{
+    "Small to Medium Line Counts (Cost-Effective)": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance in subsets 1-4, making it reliable for smaller to medium line counts."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in subsets 1-4, offering a good balance of cost and effectiveness."
+        }
+    ],
+    "Large Line Counts (High Performance)": [
+        {
+            "CodeFuse-DeepSeek-33b": "Exceptional performance in subset 6, though inconsistent in others. Best for tasks focusing on large line counts."
+        }
+    ],
+    "General Purpose (Balanced)": [
+        {
+            "codeqwen1.5-7b": "Moderate performance across most subsets, suitable for general use where line counts vary."
+        },
+        {
+            "deepseek_coder_33b-instruct": "More consistent than its base version, offering a balance for varied line counts."
+        }
+    ],
+    "Avoid": [
+        {
+            "codegemma-2b": "Poor performance across all subsets."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Consistently low scores, not suitable for any scenario."
+        }
+    ]
+}

llm_insight/HumanEval/6/EI/line_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in subsets 1-3 and subset 6, but a significant drop in subset 4. This suggests it handles larger line counts well but struggles with medium-sized subsets.",
+    "Nxcode-CQ-7B": "Consistently high performance across subsets 1-4, with a notable drop in subsets 5-6. This indicates robustness for smaller to medium line counts but less effectiveness for larger ones.",
+    "codegemma-2b": "Poor performance across all subsets, with particularly low scores in subsets 4-6. Not suitable for tasks requiring handling of larger line counts.",
+    "codegemma-7b": "Moderate performance in subsets 1-3, but sharply declines in subsets 4-6. Better than codegemma-2b but still not reliable for larger line counts.",
+    "codegemma-7b-it": "Improved over codegemma-7b, especially in subsets 1-3, but still struggles with larger line counts. A middle-ground option among codegemma models.",
+    "deepseek-coder-1.3b-base": "Low performance across the board, with slight improvements in subset 6. Not recommended for any line count range.",
+    "deepseek-coder-6.7b-base": "Better than 1.3b but still inconsistent, with a sharp drop in subsets 4-6. Limited utility for larger line counts.",
+    "deepseek_coder-6.7b-instruct": "Strong in subsets 1-4, but performance halves in subsets 5-6. Good for smaller to medium line counts but not larger ones.",
+    "deepseek_coder_33b-base": "Variable performance, with high scores in subsets 1 and 6 but poor in subset 5. Unpredictable for consistent use.",
+    "deepseek_coder_33b-instruct": "Similar to 33b-base but slightly more consistent in subsets 1-4. Still not reliable for larger line counts.",
+    "codeqwen1.5-7b": "Moderate in subsets 1-3, but drops significantly in subsets 4-6. A balanced option but not for larger line counts.",
+    "global_insights": "Models generally perform better on smaller to medium line counts (subsets 1-3) and struggle with larger ones (subsets 4-6). Nxcode-CQ-7B and CodeFuse-DeepSeek-33b are top performers but have specific weaknesses. Codegemma models are consistently weak, while deepseek models show variability. Instruct versions of models often outperform their base counterparts."
+}

llm_insight/HumanEval/6/EI/token_counts_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High performance with moderate token counts": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across most subsets except the largest."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Robust performance for most token counts, though not the largest."
+        }
+    ],
+    "Cost-effective for small to medium tasks": [
+        {
+            "codegemma-7b-it": "Better performance than smaller codegemma models for mid-range token counts."
+        },
+        {
+            "deepseek-coder-6.7b-base": "Balanced performance and resource usage for smaller tasks."
+        }
+    ],
+    "Large and complex tasks": [
+        {
+            "CodeFuse-DeepSeek-33b": "Handles larger token counts better than most, though with some variability."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Good for large inputs but inconsistent in some subsets."
+        }
+    ]
+}

llm_insight/HumanEval/6/EI/token_counts_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in subsets with higher token counts (token_subset_2 and token_subset_6), but struggles in token_subset_3 and token_subset_4. This suggests it may be better suited for larger code snippets or more complex tasks.",
+    "Nxcode-CQ-7B": "Consistently performs well across most subsets except token_subset_6, where it drops significantly. This indicates robustness in handling varying token counts but may have limitations with very large inputs.",
+    "codegemma-2b": "Performance degrades sharply as token count increases, making it unsuitable for larger or more complex code tasks.",
+    "codegemma-7b": "Similar to codegemma-2b but with slightly better performance across subsets. Still not ideal for larger inputs.",
+    "codegemma-7b-it": "Shows improvement over codegemma-7b, particularly in mid-range token subsets, but still fails in token_subset_6.",
+    "deepseek-coder-1.3b-base": "Struggles with larger token counts, performing poorly in subsets 3 through 6.",
+    "deepseek-coder-6.7b-base": "Better than the 1.3b version but still not suitable for larger inputs.",
+    "deepseek_coder-6.7b-instruct": "Performs well in subsets 1-5 but fails in token_subset_6. This suggests it is robust for most tasks but may not handle very large inputs well.",
+    "deepseek_coder_33b-base": "Shows moderate performance across subsets but struggles with larger inputs.",
+    "deepseek_coder_33b-instruct": "Performs well in subsets 1-2 and subset 5, but drops in subsets 3-4 and fails in subset 6. This indicates variability in handling different token counts.",
+    "codeqwen1.5-7b": "Moderate performance across subsets but struggles with larger inputs.",
+    "global_insights": "Models generally perform better with smaller token counts, with performance degrading as token count increases. Larger models (e.g., 33b variants) tend to handle larger inputs better but still have limitations. The Nxcode-CQ-7B and deepseek_coder-6.7b-instruct models show the most consistent performance across varying token counts, though they still struggle with the largest inputs."
+}

llm_insight/HumanEval/6/QS/CC_recommendation.json

@@ -0,0 +1,26 @@
+{
+    "High accuracy and reliability": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across all subsets, making it the most reliable choice for critical applications."
+        },
+        {
+            "CodeFuse-DeepSeek-33b": "Strong performance in most subsets, though with some instability in subset 6."
+        }
+    ],
+    "Moderate accuracy with cost-effectiveness": [
+        {
+            "deepseek_coder-6.7b-instruct": "Good performance in most subsets, offering a balance between cost and accuracy."
+        },
+        {
+            "deepseek_coder_33b-instruct": "Solid performance across most subsets, suitable for less critical tasks."
+        }
+    ],
+    "Low-cost with basic performance": [
+        {
+            "codegemma-7b-it": "Better than its non-IT counterpart, suitable for basic tasks where high accuracy is not required."
+        },
+        {
+            "deepseek-coder-6.7b-base": "Moderate performance, suitable for non-critical applications."
+        }
+    ]
+}

llm_insight/HumanEval/6/QS/CC_report.json

@@ -0,0 +1,14 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows strong performance in subsets 2, 3, and 5, but struggles in subset 6 with a significant drop to 50.0. This indicates potential instability in handling certain types of data within the CC perspective.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, with the lowest score being 80.18 in subset 3. This model demonstrates robustness and reliability across different data splits.",
+    "codegemma-2b": "Poor performance across all subsets, particularly in subset 6 with a score of 2.29. This model is not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Moderate performance with scores ranging from 14.37 to 61.61. The model shows some capability but is inconsistent, especially in subset 6.",
+    "codegemma-7b-it": "Better than its non-IT counterpart, with scores ranging from 33.33 to 65.36. Still, it lacks the consistency needed for critical applications.",
+    "deepseek-coder-1.3b-base": "Low performance across the board, with a significant drop in subset 6 to 6.67. Not recommended for any serious tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance, with scores between 13.75 and 69.64. Shows some promise but is inconsistent.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in subsets 1, 4, and 5, with a notable drop in subset 6 to 54.58. This model is reliable but has some limitations.",
+    "deepseek_coder_33b-base": "Moderate to good performance, with scores ranging from 28.96 to 70.71. Shows potential but is not the top performer.",
+    "deepseek_coder_33b-instruct": "Good performance across most subsets, with scores between 45.42 and 80.18. A solid choice but not the best.",
+    "codeqwen1.5-7b": "Moderate performance, with scores ranging from 29.58 to 64.64. Inconsistent and not recommended for high-stakes tasks.",
+    "global_insights": "Nxcode-CQ-7B is the top performer, showing consistent high scores across all subsets. CodeFuse-DeepSeek-33b and deepseek_coder-6.7b-instruct also perform well but have notable drops in subset 6. Smaller models like codegemma-2b and deepseek-coder-1.3b-base perform poorly, indicating that model size and architecture play a significant role in performance. The CC perspective seems to favor larger, more robust models."
+}

llm_insight/HumanEval/6/QS/line_counts_recommendation.json

@@ -0,0 +1,34 @@
+{
+    "High accuracy tasks with variable line counts": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across most subsets."
+        },
+        {
+            "deepseek_coder-6.7b-instruct": "Strong in subsets with higher line counts."
+        }
+    ],
+    "Cost-effective for moderate line counts": [
+        {
+            "CodeFuse-DeepSeek-33b": "Balanced performance and cost for moderate line counts."
+        },
+        {
+            "codeqwen1.5-7b": "Moderate performance at a lower cost."
+        }
+    ],
+    "Tasks with smaller line counts": [
+        {
+            "deepseek-coder-6.7b-base": "Suitable for smaller line counts at a lower cost."
+        },
+        {
+            "deepseek_coder_33b-base": "Better performance for small to medium line counts."
+        }
+    ],
+    "Not recommended for any tasks": [
+        {
+            "codegemma-2b": "Poor performance across all subsets."
+        },
+        {
+            "deepseek-coder-1.3b-base": "Low accuracy in all scenarios."
+        }
+    ]
+}

llm_insight/HumanEval/6/QS/line_counts_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows consistent performance across subsets, with a slight dip in subset_4. It performs well in subsets with moderate line counts.",
+    "Nxcode-CQ-7B": "This model excels in most subsets, particularly in subset_1 and subset_2, but shows a noticeable drop in subset_6. It is robust for larger line counts.",
+    "codegemma-2b": "Performance is poor across all subsets, with the lowest scores in subset_4 and subset_6. Not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Better than codegemma-2b but still underperforms in subsets with higher line counts. Moderate performance overall.",
+    "codegemma-7b-it": "Improved over codegemma-7b, especially in subsets_1 and subset_5. Still struggles with higher line counts.",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, particularly in subset_4 and subset_6. Not recommended for complex tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance in subsets_1 and subset_2, but drops significantly in subset_6. Suitable for smaller line counts.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in subset_2, but inconsistent in others. Good for tasks with variable line counts.",
+    "deepseek_coder_33b-base": "Decent performance in subset_1, but declines as line counts increase. Best for smaller to medium line counts.",
+    "deepseek_coder_33b-instruct": "High performance in subset_1 and subset_2, but drops in subset_6. Suitable for tasks with moderate line counts.",
+    "codeqwen1.5-7b": "Moderate performance across subsets, with a dip in subset_4. Best for tasks with consistent line counts.",
+    "new": "Similar to codegemma-7b-it, with better performance in subset_1 and subset_5. Still struggles with higher line counts.",
+    "global_insights": "Models like Nxcode-CQ-7B and deepseek_coder-6.7b-instruct perform well in subsets with higher line counts, while smaller models like codegemma-2b struggle. Larger models generally handle variability better, but cost-effectiveness must be considered."
+}

llm_insight/HumanEval/6/QS/token_counts_recommendation.json

@@ -0,0 +1,23 @@
+{
+    "High accuracy and robustness across token counts": [
+        {
+            "Nxcode-CQ-7B": "Consistently high performance across all token subsets, making it reliable for varied token counts."
+        }
+    ],
+    "Balanced performance and cost-effectiveness for low to medium token tasks": [
+        {
+            "deepseek_coder-6.7b-instruct": "Strong performance in lower token subsets and moderate in medium, suitable for cost-effective solutions."
+        },
+        {
+            "deepseek_coder_33b-instruct": "High performance in low token subsets, good for tasks where token count is controlled."
+        }
+    ],
+    "Budget-friendly for low token tasks": [
+        {
+            "codegemma-7b-it": "Moderate performance in low token subsets at a lower cost compared to larger models."
+        },
+        {
+            "codeqwen1.5-7b": "Decent performance in low token subsets, suitable for budget constraints."
+        }
+    ]
+}

llm_insight/HumanEval/6/QS/token_counts_report.json

@@ -0,0 +1,15 @@
+{
+    "CodeFuse-DeepSeek-33b": "The model shows high variability in performance across token subsets, peaking at 96.43% in token_subset_3 but dropping to 62.5% in token_subset_6. This suggests sensitivity to token count variations.",
+    "Nxcode-CQ-7B": "Consistently high performance across all subsets, with the lowest score at 77.08% in token_subset_6. Demonstrates robustness to token count changes.",
+    "codegemma-2b": "Poor performance across all subsets, especially in token_subset_6 with only 1.25%. Not suitable for tasks requiring high accuracy.",
+    "codegemma-7b": "Moderate performance with significant drops in higher token subsets. Performance degrades as token count increases.",
+    "codegemma-7b-it": "Better than codegemma-7b but still shows a declining trend with increasing token counts. Best performance in token_subset_1 at 75.89%.",
+    "deepseek-coder-1.3b-base": "Low performance across all subsets, with a sharp decline in token_subset_6 to 5.62%. Not recommended for high-token tasks.",
+    "deepseek-coder-6.7b-base": "Moderate performance with a steady decline as token count increases. Best in token_subset_1 at 73.04%.",
+    "deepseek_coder-6.7b-instruct": "Strong performance in lower token subsets (85.71% in token_subset_1) but drops to 46.67% in token_subset_6. Good for low to medium token tasks.",
+    "deepseek_coder_33b-base": "Variable performance, peaking at 73.57% in token_subset_1 and dropping to 25.42% in token_subset_6. Sensitive to token count.",
+    "deepseek_coder_33b-instruct": "High performance in lower token subsets (86.25% in token_subset_1) but declines to 39.58% in token_subset_6. Suitable for low to medium token tasks.",
+    "codeqwen1.5-7b": "Moderate performance with a steady decline as token count increases. Best in token_subset_1 at 69.11%.",
+    "new": "Similar to codegemma-7b-it, with best performance in token_subset_1 at 75.89% and declining to 36.25% in token_subset_6.",
+    "global_insights": "Models generally perform better in lower token subsets, with performance degrading as token count increases. Nxcode-CQ-7B is the most robust across all subsets. Smaller models like codegemma-2b and deepseek-coder-1.3b-base perform poorly, especially in high-token subsets. Instruct models (e.g., deepseek_coder-6.7b-instruct) show better performance than their base counterparts but still decline with higher token counts."
+}