Update app.py

app.py

@@ -179,90 +179,269 @@ def build_demo():
 # print(os.system('cd src/gen/ && python show_result.py --output'))
 
 
-def update_board():
-    need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
-    logging.info("Updating the judgement: %s", need_reset)
-    if need_reset != "1":
-        # return
-        pass
-    os.environ[RESET_JUDGEMENT_ENV] = "0"
-
-    # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
-    # recursively. In this specific context, it is used to delete the directory named "m_data" along
-    # with all its files and subdirectories. This command helps in cleaning up the existing data in
-    # the "m_data" directory before downloading new dataset files into it.
-    # shutil.rmtree("./m_data")
-    # shutil.rmtree("./data")
+# def update_board():
+#     need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
+#     logging.info("Updating the judgement: %s", need_reset)
+#     if need_reset != "1":
+#         # return
+#         pass
+#     os.environ[RESET_JUDGEMENT_ENV] = "0"
+
+#     # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
+#     # recursively. In this specific context, it is used to delete the directory named "m_data" along
+#     # with all its files and subdirectories. This command helps in cleaning up the existing data in
+#     # the "m_data" directory before downloading new dataset files into it.
+#     # shutil.rmtree("./m_data")
+#     # shutil.rmtree("./data")
+#     download_dataset("kz-transformers/s-openbench-eval", "m_data")
+#     data_list = [{"model_dtype": "torch.float16", "model": "dummy-random-baseline", "ppl": 0, "mmlu_translated_kk": 0.22991508817766165, "kk_constitution_mc": 0.25120772946859904, "kk_dastur_mc": 0.24477611940298508, "kazakh_and_literature_unt_mc": 0.2090443686006826, "kk_geography_unt_mc": 0.2019790454016298, "kk_world_history_unt_mc": 0.1986970684039088, "kk_history_of_kazakhstan_unt_mc": 0.19417177914110428, "kk_english_unt_mc": 0.189804278561675, "kk_biology_unt_mc": 0.22330729166666666, "kk_human_society_rights_unt_mc": 0.242152466367713}, {"model_dtype": "torch.float16", "model": "gpt-4o-mini", "ppl": 0, "mmlu_translated_kk": 0.5623775310254735, "kk_constitution_mc": 0.79, "kk_dastur_mc": 0.755, "kazakh_and_literature_unt_mc": 0.4953071672354949, "kk_geography_unt_mc": 0.5675203725261933, "kk_world_history_unt_mc": 0.6091205211726385, "kk_history_of_kazakhstan_unt_mc": 0.47883435582822087, "kk_english_unt_mc": 0.6763768775603095, "kk_biology_unt_mc": 0.607421875, "kk_human_society_rights_unt_mc": 0.7309417040358744}, {"model_dtype": "api", "model": "gpt-4o", "ppl": 0, "mmlu_translated_kk": 0.7419986936642717, "kk_constitution_mc": 0.841, "kk_dastur_mc": 0.798, "kazakh_and_literature_unt_mc": 0.6785409556313993, "kk_geography_unt_mc": 0.629802095459837, "kk_world_history_unt_mc": 0.6783387622149837, "kk_history_of_kazakhstan_unt_mc": 0.6785276073619632, "kk_english_unt_mc": 0.7410104688211198, "kk_biology_unt_mc": 0.6979166666666666, "kk_human_society_rights_unt_mc": 0.7937219730941704}, {"model_dtype": "torch.float16", "model": "nova-pro-v1", "ppl": 0, "mmlu_translated_kk": 0.6792945787067276, "kk_constitution_mc": 0.7753623188405797, "kk_dastur_mc": 0.718407960199005, "kazakh_and_literature_unt_mc": 0.4656569965870307, "kk_geography_unt_mc": 0.5541327124563445, "kk_world_history_unt_mc": 0.6425081433224755, "kk_history_of_kazakhstan_unt_mc": 0.5, "kk_english_unt_mc": 0.6845698680018206, "kk_biology_unt_mc": 0.6197916666666666, "kk_human_society_rights_unt_mc": 0.7713004484304933}, {"model_dtype": "torch.float16", "model": "gemini-1.5-pro", "ppl": 0, "mmlu_translated_kk": 0.7380796864794252, "kk_constitution_mc": 0.8164251207729468, "kk_dastur_mc": 0.7383084577114428, "kazakh_and_literature_unt_mc": 0.5565273037542662, "kk_geography_unt_mc": 0.6065192083818394, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.5791411042944785, "kk_english_unt_mc": 0.7114246700045517, "kk_biology_unt_mc": 0.6673177083333334, "kk_human_society_rights_unt_mc": 0.7623318385650224}, {"model_dtype": "torch.float16", "model": "gemini-1.5-flash", "ppl": 0, "mmlu_translated_kk": 0.6335728282168517, "kk_constitution_mc": 0.748792270531401, "kk_dastur_mc": 0.7054726368159204, "kazakh_and_literature_unt_mc": 0.4761092150170648, "kk_geography_unt_mc": 0.5640279394644936, "kk_world_history_unt_mc": 0.5838762214983714, "kk_history_of_kazakhstan_unt_mc": 0.43374233128834355, "kk_english_unt_mc": 0.6681838871187984, "kk_biology_unt_mc": 0.6217447916666666, "kk_human_society_rights_unt_mc": 0.7040358744394619}, {"model_dtype": "torch.float16", "model": "claude-3-5-sonnet", "ppl": 0, "mmlu_translated_kk": 0.7335075114304376, "kk_constitution_mc": 0.8623188405797102, "kk_dastur_mc": 0.7950248756218905, "kazakh_and_literature_unt_mc": 0.6548634812286689, "kk_geography_unt_mc": 0.6431897555296857, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.6251533742331289, "kk_english_unt_mc": 0.7291761492944925, "kk_biology_unt_mc": 0.6686197916666666, "kk_human_society_rights_unt_mc": 0.8026905829596412}, {"model_dtype": "torch.float16", "model": "yandex-gpt", "ppl": 0, "mmlu_translated_kk": 0.39777922926192033, "kk_constitution_mc": 0.7028985507246377, "kk_dastur_mc": 0.6159203980099502, "kazakh_and_literature_unt_mc": 0.3914249146757679, "kk_geography_unt_mc": 0.4912689173457509, "kk_world_history_unt_mc": 0.5244299674267101, "kk_history_of_kazakhstan_unt_mc": 0.4030674846625767, "kk_english_unt_mc": 0.5844333181611289, "kk_biology_unt_mc": 0.4368489583333333, "kk_human_society_rights_unt_mc": 0.6995515695067265}]
+#     files_list = glob.glob("./m_data/model_data/external/*.json")
+#     print(f'FILES LIST: {files_list}')
+#     for file in files_list:
+#         with open(file) as f:
+#             print(f'trying to read external submit file: {file}')
+#             try:
+#                 data = json.load(f)
+#                 print(f'succeed to read: {file}, got {len(data)}')
+#                 data_list.append(data)
+#             except Exception as e:
+#                 pass # data was badly formatted, should not fail
+#     print("DATALIST: ", data_list)
+
+#     with open("genned.json", "w") as f:
+#         json.dump(data_list, f)
+
+
+#     API.upload_file(
+#             path_or_fileobj="genned.json",
+#             path_in_repo="leaderboard.json",
+#             repo_id="kz-transformers/kaz-llm-lb-metainfo",
+#             repo_type="dataset",
+#     )
+#     restart_space()
+
+
+#     # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py")
+#     # subprocess.run(["python3", gen_judgement_file], check=True)
+
+# def update_board_():
+#     need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
+#     logging.info("Updating the judgement: %s", need_reset)
+#     if need_reset != "1":
+#         # return
+#         pass
+#     os.environ[RESET_JUDGEMENT_ENV] = "0"
+
+#     # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
+#     # recursively. In this specific context, it is used to delete the directory named "m_data" along
+#     # with all its files and subdirectories. This command helps in cleaning up the existing data in
+#     # the "m_data" directory before downloading new dataset files into it.
+#     # shutil.rmtree("./m_data")
+#     # shutil.rmtree("./data")
+#     download_dataset("kz-transformers/s-openbench-eval", "m_data")
+#     data_list = [{"model_dtype": "torch.float16", "model": "dummy-random-baseline", "ppl": 0, "mmlu_translated_kk": 0.22991508817766165, "kk_constitution_mc": 0.25120772946859904, "kk_dastur_mc": 0.24477611940298508, "kazakh_and_literature_unt_mc": 0.2090443686006826, "kk_geography_unt_mc": 0.2019790454016298, "kk_world_history_unt_mc": 0.1986970684039088, "kk_history_of_kazakhstan_unt_mc": 0.19417177914110428, "kk_english_unt_mc": 0.189804278561675, "kk_biology_unt_mc": 0.22330729166666666, "kk_human_society_rights_unt_mc": 0.242152466367713}, {"model_dtype": "torch.float16", "model": "gpt-4o-mini", "ppl": 0, "mmlu_translated_kk": 0.5623775310254735, "kk_constitution_mc": 0.79, "kk_dastur_mc": 0.755, "kazakh_and_literature_unt_mc": 0.4953071672354949, "kk_geography_unt_mc": 0.5675203725261933, "kk_world_history_unt_mc": 0.6091205211726385, "kk_history_of_kazakhstan_unt_mc": 0.47883435582822087, "kk_english_unt_mc": 0.6763768775603095, "kk_biology_unt_mc": 0.607421875, "kk_human_society_rights_unt_mc": 0.7309417040358744}, {"model_dtype": "api", "model": "gpt-4o", "ppl": 0, "mmlu_translated_kk": 0.7419986936642717, "kk_constitution_mc": 0.841, "kk_dastur_mc": 0.798, "kazakh_and_literature_unt_mc": 0.6785409556313993, "kk_geography_unt_mc": 0.629802095459837, "kk_world_history_unt_mc": 0.6783387622149837, "kk_history_of_kazakhstan_unt_mc": 0.6785276073619632, "kk_english_unt_mc": 0.7410104688211198, "kk_biology_unt_mc": 0.6979166666666666, "kk_human_society_rights_unt_mc": 0.7937219730941704}, {"model_dtype": "torch.float16", "model": "nova-pro-v1", "ppl": 0, "mmlu_translated_kk": 0.6792945787067276, "kk_constitution_mc": 0.7753623188405797, "kk_dastur_mc": 0.718407960199005, "kazakh_and_literature_unt_mc": 0.4656569965870307, "kk_geography_unt_mc": 0.5541327124563445, "kk_world_history_unt_mc": 0.6425081433224755, "kk_history_of_kazakhstan_unt_mc": 0.5, "kk_english_unt_mc": 0.6845698680018206, "kk_biology_unt_mc": 0.6197916666666666, "kk_human_society_rights_unt_mc": 0.7713004484304933}, {"model_dtype": "torch.float16", "model": "gemini-1.5-pro", "ppl": 0, "mmlu_translated_kk": 0.7380796864794252, "kk_constitution_mc": 0.8164251207729468, "kk_dastur_mc": 0.7383084577114428, "kazakh_and_literature_unt_mc": 0.5565273037542662, "kk_geography_unt_mc": 0.6065192083818394, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.5791411042944785, "kk_english_unt_mc": 0.7114246700045517, "kk_biology_unt_mc": 0.6673177083333334, "kk_human_society_rights_unt_mc": 0.7623318385650224}, {"model_dtype": "torch.float16", "model": "gemini-1.5-flash", "ppl": 0, "mmlu_translated_kk": 0.6335728282168517, "kk_constitution_mc": 0.748792270531401, "kk_dastur_mc": 0.7054726368159204, "kazakh_and_literature_unt_mc": 0.4761092150170648, "kk_geography_unt_mc": 0.5640279394644936, "kk_world_history_unt_mc": 0.5838762214983714, "kk_history_of_kazakhstan_unt_mc": 0.43374233128834355, "kk_english_unt_mc": 0.6681838871187984, "kk_biology_unt_mc": 0.6217447916666666, "kk_human_society_rights_unt_mc": 0.7040358744394619}, {"model_dtype": "torch.float16", "model": "claude-3-5-sonnet", "ppl": 0, "mmlu_translated_kk": 0.7335075114304376, "kk_constitution_mc": 0.8623188405797102, "kk_dastur_mc": 0.7950248756218905, "kazakh_and_literature_unt_mc": 0.6548634812286689, "kk_geography_unt_mc": 0.6431897555296857, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.6251533742331289, "kk_english_unt_mc": 0.7291761492944925, "kk_biology_unt_mc": 0.6686197916666666, "kk_human_society_rights_unt_mc": 0.8026905829596412}, {"model_dtype": "torch.float16", "model": "yandex-gpt", "ppl": 0, "mmlu_translated_kk": 0.39777922926192033, "kk_constitution_mc": 0.7028985507246377, "kk_dastur_mc": 0.6159203980099502, "kazakh_and_literature_unt_mc": 0.3914249146757679, "kk_geography_unt_mc": 0.4912689173457509, "kk_world_history_unt_mc": 0.5244299674267101, "kk_history_of_kazakhstan_unt_mc": 0.4030674846625767, "kk_english_unt_mc": 0.5844333181611289, "kk_biology_unt_mc": 0.4368489583333333, "kk_human_society_rights_unt_mc": 0.6995515695067265}]
+#     files_list = glob.glob("./m_data/model_data/external/*.json")
+#     print(f'FILES LIST: {files_list}')
+#     for file in files_list:
+#         with open(file) as f:
+#             print(f'trying to read external submit file: {file}')
+#             try:
+#                 data = json.load(f)
+#                 print(f'succeed to read: {file}, got {len(data)}')
+#                 data_list.append(data)
+#             except Exception as e:
+#                 pass # data was badly formatted, should not fail
+#     print("DATALIST: ", data_list)
+
+#     with open("genned.json", "w") as f:
+#         json.dump(data_list, f)
+
+#     API.upload_file(
+#             path_or_fileobj="genned.json",
+#             path_in_repo="leaderboard.json",
+#             repo_id="kz-transformers/kaz-llm-lb-metainfo",
+#             repo_type="dataset",
+#     )
+
+def aggregate_leaderboard_data():
     download_dataset("kz-transformers/s-openbench-eval", "m_data")
-    data_list = [{"model_dtype": "torch.float16", "model": "dummy-random-baseline", "ppl": 0, "mmlu_translated_kk": 0.22991508817766165, "kk_constitution_mc": 0.25120772946859904, "kk_dastur_mc": 0.24477611940298508, "kazakh_and_literature_unt_mc": 0.2090443686006826, "kk_geography_unt_mc": 0.2019790454016298, "kk_world_history_unt_mc": 0.1986970684039088, "kk_history_of_kazakhstan_unt_mc": 0.19417177914110428, "kk_english_unt_mc": 0.189804278561675, "kk_biology_unt_mc": 0.22330729166666666, "kk_human_society_rights_unt_mc": 0.242152466367713}, {"model_dtype": "torch.float16", "model": "gpt-4o-mini", "ppl": 0, "mmlu_translated_kk": 0.5623775310254735, "kk_constitution_mc": 0.79, "kk_dastur_mc": 0.755, "kazakh_and_literature_unt_mc": 0.4953071672354949, "kk_geography_unt_mc": 0.5675203725261933, "kk_world_history_unt_mc": 0.6091205211726385, "kk_history_of_kazakhstan_unt_mc": 0.47883435582822087, "kk_english_unt_mc": 0.6763768775603095, "kk_biology_unt_mc": 0.607421875, "kk_human_society_rights_unt_mc": 0.7309417040358744}, {"model_dtype": "api", "model": "gpt-4o", "ppl": 0, "mmlu_translated_kk": 0.7419986936642717, "kk_constitution_mc": 0.841, "kk_dastur_mc": 0.798, "kazakh_and_literature_unt_mc": 0.6785409556313993, "kk_geography_unt_mc": 0.629802095459837, "kk_world_history_unt_mc": 0.6783387622149837, "kk_history_of_kazakhstan_unt_mc": 0.6785276073619632, "kk_english_unt_mc": 0.7410104688211198, "kk_biology_unt_mc": 0.6979166666666666, "kk_human_society_rights_unt_mc": 0.7937219730941704}, {"model_dtype": "torch.float16", "model": "nova-pro-v1", "ppl": 0, "mmlu_translated_kk": 0.6792945787067276, "kk_constitution_mc": 0.7753623188405797, "kk_dastur_mc": 0.718407960199005, "kazakh_and_literature_unt_mc": 0.4656569965870307, "kk_geography_unt_mc": 0.5541327124563445, "kk_world_history_unt_mc": 0.6425081433224755, "kk_history_of_kazakhstan_unt_mc": 0.5, "kk_english_unt_mc": 0.6845698680018206, "kk_biology_unt_mc": 0.6197916666666666, "kk_human_society_rights_unt_mc": 0.7713004484304933}, {"model_dtype": "torch.float16", "model": "gemini-1.5-pro", "ppl": 0, "mmlu_translated_kk": 0.7380796864794252, "kk_constitution_mc": 0.8164251207729468, "kk_dastur_mc": 0.7383084577114428, "kazakh_and_literature_unt_mc": 0.5565273037542662, "kk_geography_unt_mc": 0.6065192083818394, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.5791411042944785, "kk_english_unt_mc": 0.7114246700045517, "kk_biology_unt_mc": 0.6673177083333334, "kk_human_society_rights_unt_mc": 0.7623318385650224}, {"model_dtype": "torch.float16", "model": "gemini-1.5-flash", "ppl": 0, "mmlu_translated_kk": 0.6335728282168517, "kk_constitution_mc": 0.748792270531401, "kk_dastur_mc": 0.7054726368159204, "kazakh_and_literature_unt_mc": 0.4761092150170648, "kk_geography_unt_mc": 0.5640279394644936, "kk_world_history_unt_mc": 0.5838762214983714, "kk_history_of_kazakhstan_unt_mc": 0.43374233128834355, "kk_english_unt_mc": 0.6681838871187984, "kk_biology_unt_mc": 0.6217447916666666, "kk_human_society_rights_unt_mc": 0.7040358744394619}, {"model_dtype": "torch.float16", "model": "claude-3-5-sonnet", "ppl": 0, "mmlu_translated_kk": 0.7335075114304376, "kk_constitution_mc": 0.8623188405797102, "kk_dastur_mc": 0.7950248756218905, "kazakh_and_literature_unt_mc": 0.6548634812286689, "kk_geography_unt_mc": 0.6431897555296857, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.6251533742331289, "kk_english_unt_mc": 0.7291761492944925, "kk_biology_unt_mc": 0.6686197916666666, "kk_human_society_rights_unt_mc": 0.8026905829596412}, {"model_dtype": "torch.float16", "model": "yandex-gpt", "ppl": 0, "mmlu_translated_kk": 0.39777922926192033, "kk_constitution_mc": 0.7028985507246377, "kk_dastur_mc": 0.6159203980099502, "kazakh_and_literature_unt_mc": 0.3914249146757679, "kk_geography_unt_mc": 0.4912689173457509, "kk_world_history_unt_mc": 0.5244299674267101, "kk_history_of_kazakhstan_unt_mc": 0.4030674846625767, "kk_english_unt_mc": 0.5844333181611289, "kk_biology_unt_mc": 0.4368489583333333, "kk_human_society_rights_unt_mc": 0.6995515695067265}]
+    
+    # Start with your baseline data
+    data_list = [
+        {
+            "model_dtype": "torch.float16",
+            "model": "dummy-random-baseline",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.22991508817766165,
+            "kk_constitution_mc": 0.25120772946859904,
+            "kk_dastur_mc": 0.24477611940298508,
+            "kazakh_and_literature_unt_mc": 0.2090443686006826,
+            "kk_geography_unt_mc": 0.2019790454016298,
+            "kk_world_history_unt_mc": 0.1986970684039088,
+            "kk_history_of_kazakhstan_unt_mc": 0.19417177914110428,
+            "kk_english_unt_mc": 0.189804278561675,
+            "kk_biology_unt_mc": 0.22330729166666666,
+            "kk_human_society_rights_unt_mc": 0.242152466367713,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "gpt-4o-mini",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.5623775310254735,
+            "kk_constitution_mc": 0.79,
+            "kk_dastur_mc": 0.755,
+            "kazakh_and_literature_unt_mc": 0.4953071672354949,
+            "kk_geography_unt_mc": 0.5675203725261933,
+            "kk_world_history_unt_mc": 0.6091205211726385,
+            "kk_history_of_kazakhstan_unt_mc": 0.47883435582822087,
+            "kk_english_unt_mc": 0.6763768775603095,
+            "kk_biology_unt_mc": 0.607421875,
+            "kk_human_society_rights_unt_mc": 0.7309417040358744,
+        },
+        {
+            "model_dtype": "api",
+            "model": "gpt-4o",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.7419986936642717,
+            "kk_constitution_mc": 0.841,
+            "kk_dastur_mc": 0.798,
+            "kazakh_and_literature_unt_mc": 0.6785409556313993,
+            "kk_geography_unt_mc": 0.629802095459837,
+            "kk_world_history_unt_mc": 0.6783387622149837,
+            "kk_history_of_kazakhstan_unt_mc": 0.6785276073619632,
+            "kk_english_unt_mc": 0.7410104688211198,
+            "kk_biology_unt_mc": 0.6979166666666666,
+            "kk_human_society_rights_unt_mc": 0.7937219730941704,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "nova-pro-v1",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.6792945787067276,
+            "kk_constitution_mc": 0.7753623188405797,
+            "kk_dastur_mc": 0.718407960199005,
+            "kazakh_and_literature_unt_mc": 0.4656569965870307,
+            "kk_geography_unt_mc": 0.5541327124563445,
+            "kk_world_history_unt_mc": 0.6425081433224755,
+            "kk_history_of_kazakhstan_unt_mc": 0.5,
+            "kk_english_unt_mc": 0.6845698680018206,
+            "kk_biology_unt_mc": 0.6197916666666666,
+            "kk_human_society_rights_unt_mc": 0.7713004484304933,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "gemini-1.5-pro",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.7380796864794252,
+            "kk_constitution_mc": 0.8164251207729468,
+            "kk_dastur_mc": 0.7383084577114428,
+            "kazakh_and_literature_unt_mc": 0.5565273037542662,
+            "kk_geography_unt_mc": 0.6065192083818394,
+            "kk_world_history_unt_mc": 0.6669381107491856,
+            "kk_history_of_kazakhstan_unt_mc": 0.5791411042944785,
+            "kk_english_unt_mc": 0.7114246700045517,
+            "kk_biology_unt_mc": 0.6673177083333334,
+            "kk_human_society_rights_unt_mc": 0.7623318385650224,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "gemini-1.5-flash",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.6335728282168517,
+            "kk_constitution_mc": 0.748792270531401,
+            "kk_dastur_mc": 0.7054726368159204,
+            "kazakh_and_literature_unt_mc": 0.4761092150170648,
+            "kk_geography_unt_mc": 0.5640279394644936,
+            "kk_world_history_unt_mc": 0.5838762214983714,
+            "kk_history_of_kazakhstan_unt_mc": 0.43374233128834355,
+            "kk_english_unt_mc": 0.6681838871187984,
+            "kk_biology_unt_mc": 0.6217447916666666,
+            "kk_human_society_rights_unt_mc": 0.7040358744394619,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "claude-3-5-sonnet",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.7335075114304376,
+            "kk_constitution_mc": 0.8623188405797102,
+            "kk_dastur_mc": 0.7950248756218905,
+            "kazakh_and_literature_unt_mc": 0.6548634812286689,
+            "kk_geography_unt_mc": 0.6431897555296857,
+            "kk_world_history_unt_mc": 0.6669381107491856,
+            "kk_history_of_kazakhstan_unt_mc": 0.6251533742331289,
+            "kk_english_unt_mc": 0.7291761492944925,
+            "kk_biology_unt_mc": 0.6686197916666666,
+            "kk_human_society_rights_unt_mc": 0.8026905829596412,
+        },
+        {
+            "model_dtype": "torch.float16",
+            "model": "yandex-gpt",
+            "ppl": 0,
+            "mmlu_translated_kk": 0.39777922926192033,
+            "kk_constitution_mc": 0.7028985507246377,
+            "kk_dastur_mc": 0.6159203980099502,
+            "kazakh_and_literature_unt_mc": 0.3914249146757679,
+            "kk_geography_unt_mc": 0.4912689173457509,
+            "kk_world_history_unt_mc": 0.5244299674267101,
+            "kk_history_of_kazakhstan_unt_mc": 0.4030674846625767,
+            "kk_english_unt_mc": 0.5844333181611289,
+            "kk_biology_unt_mc": 0.4368489583333333,
+            "kk_human_society_rights_unt_mc": 0.6995515695067265,
+        },
+    ]
+    
     files_list = glob.glob("./m_data/model_data/external/*.json")
-    print(f'FILES LIST: {files_list}')
+    logging.info(f'FILES LIST: {files_list}')
+    
     for file in files_list:
         with open(file) as f:
-            print(f'trying to read external submit file: {file}')
+            logging.info(f'Trying to read external submit file: {file}')
             try:
                 data = json.load(f)
-                print(f'succeed to read: {file}, got {len(data)}')
+                # Validate that data is a dict and has some required keys:
+                if not isinstance(data, dict):
+                    logging.warning(f"File {file} is not a dict, skipping")
+                    continue
+                required_keys = {'model_dtype', 'model', 'ppl', 'mmlu_translated_kk'}
+                if not required_keys.issubset(data.keys()):
+                    logging.warning(f"File {file} missing required keys, skipping")
+                    continue
+
+                logging.info(f'Successfully read: {file}, got {len(data)} keys')
                 data_list.append(data)
             except Exception as e:
-                pass # data was badly formatted, should not fail
-    print("DATALIST: ", data_list)
+                logging.error(f"Error reading file {file}: {e}")
+                continue
 
+    logging.info("Combined data_list length: %d", len(data_list))
+    
     with open("genned.json", "w") as f:
         json.dump(data_list, f)
-
-
+    
     API.upload_file(
-            path_or_fileobj="genned.json",
-            path_in_repo="leaderboard.json",
-            repo_id="kz-transformers/kaz-llm-lb-metainfo",
-            repo_type="dataset",
+        path_or_fileobj="genned.json",
+        path_in_repo="leaderboard.json",
+        repo_id="kz-transformers/kaz-llm-lb-metainfo",
+        repo_type="dataset",
     )
-    restart_space()
-
-
-    # gen_judgement_file = os.path.join(HF_HOME, "src/gen/gen_judgement.py")
-    # subprocess.run(["python3", gen_judgement_file], check=True)
 
-def update_board_():
+def update_board():
     need_reset = os.environ.get(RESET_JUDGEMENT_ENV)
-    logging.info("Updating the judgement: %s", need_reset)
+    logging.info("Updating the judgement (scheduled update): %s", need_reset)
     if need_reset != "1":
-        # return
+        # Optionally return early if no update is needed
         pass
     os.environ[RESET_JUDGEMENT_ENV] = "0"
 
-    # `shutil.rmtree("./m_data")` is a Python command that removes a directory and all its contents
-    # recursively. In this specific context, it is used to delete the directory named "m_data" along
-    # with all its files and subdirectories. This command helps in cleaning up the existing data in
-    # the "m_data" directory before downloading new dataset files into it.
-    # shutil.rmtree("./m_data")
-    # shutil.rmtree("./data")
-    download_dataset("kz-transformers/s-openbench-eval", "m_data")
-    data_list = [{"model_dtype": "torch.float16", "model": "dummy-random-baseline", "ppl": 0, "mmlu_translated_kk": 0.22991508817766165, "kk_constitution_mc": 0.25120772946859904, "kk_dastur_mc": 0.24477611940298508, "kazakh_and_literature_unt_mc": 0.2090443686006826, "kk_geography_unt_mc": 0.2019790454016298, "kk_world_history_unt_mc": 0.1986970684039088, "kk_history_of_kazakhstan_unt_mc": 0.19417177914110428, "kk_english_unt_mc": 0.189804278561675, "kk_biology_unt_mc": 0.22330729166666666, "kk_human_society_rights_unt_mc": 0.242152466367713}, {"model_dtype": "torch.float16", "model": "gpt-4o-mini", "ppl": 0, "mmlu_translated_kk": 0.5623775310254735, "kk_constitution_mc": 0.79, "kk_dastur_mc": 0.755, "kazakh_and_literature_unt_mc": 0.4953071672354949, "kk_geography_unt_mc": 0.5675203725261933, "kk_world_history_unt_mc": 0.6091205211726385, "kk_history_of_kazakhstan_unt_mc": 0.47883435582822087, "kk_english_unt_mc": 0.6763768775603095, "kk_biology_unt_mc": 0.607421875, "kk_human_society_rights_unt_mc": 0.7309417040358744}, {"model_dtype": "api", "model": "gpt-4o", "ppl": 0, "mmlu_translated_kk": 0.7419986936642717, "kk_constitution_mc": 0.841, "kk_dastur_mc": 0.798, "kazakh_and_literature_unt_mc": 0.6785409556313993, "kk_geography_unt_mc": 0.629802095459837, "kk_world_history_unt_mc": 0.6783387622149837, "kk_history_of_kazakhstan_unt_mc": 0.6785276073619632, "kk_english_unt_mc": 0.7410104688211198, "kk_biology_unt_mc": 0.6979166666666666, "kk_human_society_rights_unt_mc": 0.7937219730941704}, {"model_dtype": "torch.float16", "model": "nova-pro-v1", "ppl": 0, "mmlu_translated_kk": 0.6792945787067276, "kk_constitution_mc": 0.7753623188405797, "kk_dastur_mc": 0.718407960199005, "kazakh_and_literature_unt_mc": 0.4656569965870307, "kk_geography_unt_mc": 0.5541327124563445, "kk_world_history_unt_mc": 0.6425081433224755, "kk_history_of_kazakhstan_unt_mc": 0.5, "kk_english_unt_mc": 0.6845698680018206, "kk_biology_unt_mc": 0.6197916666666666, "kk_human_society_rights_unt_mc": 0.7713004484304933}, {"model_dtype": "torch.float16", "model": "gemini-1.5-pro", "ppl": 0, "mmlu_translated_kk": 0.7380796864794252, "kk_constitution_mc": 0.8164251207729468, "kk_dastur_mc": 0.7383084577114428, "kazakh_and_literature_unt_mc": 0.5565273037542662, "kk_geography_unt_mc": 0.6065192083818394, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.5791411042944785, "kk_english_unt_mc": 0.7114246700045517, "kk_biology_unt_mc": 0.6673177083333334, "kk_human_society_rights_unt_mc": 0.7623318385650224}, {"model_dtype": "torch.float16", "model": "gemini-1.5-flash", "ppl": 0, "mmlu_translated_kk": 0.6335728282168517, "kk_constitution_mc": 0.748792270531401, "kk_dastur_mc": 0.7054726368159204, "kazakh_and_literature_unt_mc": 0.4761092150170648, "kk_geography_unt_mc": 0.5640279394644936, "kk_world_history_unt_mc": 0.5838762214983714, "kk_history_of_kazakhstan_unt_mc": 0.43374233128834355, "kk_english_unt_mc": 0.6681838871187984, "kk_biology_unt_mc": 0.6217447916666666, "kk_human_society_rights_unt_mc": 0.7040358744394619}, {"model_dtype": "torch.float16", "model": "claude-3-5-sonnet", "ppl": 0, "mmlu_translated_kk": 0.7335075114304376, "kk_constitution_mc": 0.8623188405797102, "kk_dastur_mc": 0.7950248756218905, "kazakh_and_literature_unt_mc": 0.6548634812286689, "kk_geography_unt_mc": 0.6431897555296857, "kk_world_history_unt_mc": 0.6669381107491856, "kk_history_of_kazakhstan_unt_mc": 0.6251533742331289, "kk_english_unt_mc": 0.7291761492944925, "kk_biology_unt_mc": 0.6686197916666666, "kk_human_society_rights_unt_mc": 0.8026905829596412}, {"model_dtype": "torch.float16", "model": "yandex-gpt", "ppl": 0, "mmlu_translated_kk": 0.39777922926192033, "kk_constitution_mc": 0.7028985507246377, "kk_dastur_mc": 0.6159203980099502, "kazakh_and_literature_unt_mc": 0.3914249146757679, "kk_geography_unt_mc": 0.4912689173457509, "kk_world_history_unt_mc": 0.5244299674267101, "kk_history_of_kazakhstan_unt_mc": 0.4030674846625767, "kk_english_unt_mc": 0.5844333181611289, "kk_biology_unt_mc": 0.4368489583333333, "kk_human_society_rights_unt_mc": 0.6995515695067265}]
-    files_list = glob.glob("./m_data/model_data/external/*.json")
-    print(f'FILES LIST: {files_list}')
-    for file in files_list:
-        with open(file) as f:
-            print(f'trying to read external submit file: {file}')
-            try:
-                data = json.load(f)
-                print(f'succeed to read: {file}, got {len(data)}')
-                data_list.append(data)
-            except Exception as e:
-                pass # data was badly formatted, should not fail
-    print("DATALIST: ", data_list)
-
-    with open("genned.json", "w") as f:
-        json.dump(data_list, f)
+    # Use the common aggregation function
+    aggregate_leaderboard_data()
+    restart_space()  # Scheduled update restarts the space
 
+def update_board_():
+    # Startup update (without restart)
+    logging.info("Updating the judgement at startup")
+    aggregate_leaderboard_data()
 
-    API.upload_file(
-            path_or_fileobj="genned.json",
-            path_in_repo="leaderboard.json",
-            repo_id="kz-transformers/kaz-llm-lb-metainfo",
-            repo_type="dataset",
-    )
 
 if __name__ == "__main__":
     os.environ[RESET_JUDGEMENT_ENV] = "1"