Spaces:
Sleeping
Sleeping
File size: 3,181 Bytes
9fcf6ea |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
import os
import pandas as pd
import pingouin as pg
import seaborn as sns
import matplotlib.pyplot as plt
# Set up output directories
HEATMAPS_FOLDER = "icc_heatmaps/"
os.makedirs(HEATMAPS_FOLDER, exist_ok=True)
def preprocess_data(df, selected_assessors, selected_respondents, selected_criteria):
"""
Filters the dataset based on user-selected assessors, respondents, and criteria.
Ensures data is properly formatted for ICC computation.
"""
df = df[df["assessor"].isin(selected_assessors) & df["respondent"].isin(selected_respondents)]
df = df[["assessor", "respondent"] + selected_criteria]
# Convert all columns to numeric (handling comma decimals)
for col in selected_criteria:
df[col] = df[col].str.replace(",", ".").astype(float)
# Ensure 'assessor' and 'respondent' are treated as categorical
df["assessor"] = df["assessor"].astype(str)
df["respondent"] = df["respondent"].astype(str)
return df
def compute_icc(df):
"""
Computes the overall ICC (Intraclass Correlation Coefficient).
"""
melted_df = df.melt(id_vars=["assessor", "respondent"], var_name="Criterion", value_name="Score")
if melted_df["respondent"].nunique() >= 5:
icc_results = pg.intraclass_corr(data=melted_df, targets="respondent", raters="assessor", ratings="Score").round(3)
return icc_results
else:
return None
def compute_assessor_icc(df):
"""
Computes ICC matrices between assessors and generates heatmaps.
"""
melted_df = df.melt(id_vars=["assessor", "respondent"], var_name="Criterion", value_name="Score")
assessors = df["assessor"].unique()
icc_matrix_types = {icc_type: pd.DataFrame(index=assessors, columns=assessors, dtype=float) for icc_type in ["ICC1", "ICC2", "ICC3"]}
for assessor1 in assessors:
for assessor2 in assessors:
if assessor1 != assessor2:
subset = melted_df[melted_df["assessor"].isin([assessor1, assessor2])]
if subset["respondent"].nunique() >= 5:
icc_results = pg.intraclass_corr(
data=subset, targets="respondent", raters="assessor", ratings="Score"
).round(3)
for icc_type in ["ICC1", "ICC2", "ICC3"]:
icc_matrix_types[icc_type].loc[assessor1, assessor2] = icc_results.set_index("Type").loc[icc_type]["ICC"]
return icc_matrix_types
def generate_heatmaps(icc_matrix_types):
"""
Generates and saves heatmaps for ICC matrices.
"""
heatmap_files = {}
for icc_type, icc_matrix in icc_matrix_types.items():
plt.figure(figsize=(8, 6))
sns.heatmap(icc_matrix.astype(float), annot=True, cmap="coolwarm", linewidths=0.5, fmt=".2f")
plt.title(f"Assessor ICC Matrix ({icc_type})")
plt.xlabel("Assessor (LLM)")
plt.ylabel("Assessor (LLM)")
plt.xticks(rotation=45)
plt.yticks(rotation=0)
heatmap_file = os.path.join(HEATMAPS_FOLDER, f"icc_matrix_{icc_type}.png")
plt.savefig(heatmap_file)
plt.close()
heatmap_files[icc_type] = heatmap_file
return heatmap_files
|