Upload 12 files

evaluate_result/10_CodeFuse-DeepSeek-33b-result.json

@@ -0,0 +1,703 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        1.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        1.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        1.0
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        0.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        1.0
+      ],
+      [
+        27,
+        0.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        1.0
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        0.0
+      ],
+      [
+        54,
+        1.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        0.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        1.0
+      ],
+      [
+        65,
+        1.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        1.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        1.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        1.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        1.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        0.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        1.0
+      ],
+      [
+        91,
+        1.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        1.0
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        1.0
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        1.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        1.0
+      ],
+      [
+        106,
+        1.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        1.0
+      ],
+      [
+        109,
+        1.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        1.0
+      ],
+      [
+        116,
+        0.0
+      ],
+      [
+        117,
+        0.0
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        1.0
+      ],
+      [
+        120,
+        1.0
+      ],
+      [
+        121,
+        0.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.0
+      ],
+      [
+        125,
+        1.0
+      ],
+      [
+        126,
+        1.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.0
+      ],
+      [
+        129,
+        1.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        1.0
+      ],
+      [
+        133,
+        1.0
+      ],
+      [
+        134,
+        1.0
+      ],
+      [
+        135,
+        1.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        1.0
+      ],
+      [
+        138,
+        0.0
+      ],
+      [
+        139,
+        1.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        0.0
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        1.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        0.0
+      ],
+      [
+        148,
+        0.0
+      ],
+      [
+        149,
+        0.0
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        1.0
+      ],
+      [
+        155,
+        1.0
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        0.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.0
+      ],
+      [
+        160,
+        1.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "CodeFuse-DeepSeek-33b",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "10_CodeFuse-DeepSeek-33b.json",
+    "load_data_path": null,
+    "metric_output_path": "10_CodeFuse-DeepSeek-33b.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/11_Nxcode-CQ-7B-orpo_result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        1.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        1.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.95
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.4999999999999999
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.3500000000000001
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.85
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.85
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        1.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        1.0
+      ],
+      [
+        65,
+        0.15000000000000002
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        1.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        1.0
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        1.0
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        0.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        1.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        1.0
+      ],
+      [
+        84,
+        1.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        1.0
+      ],
+      [
+        91,
+        0.5499999999999998
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.09999999999999998
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        0.5499999999999998
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        1.0
+      ],
+      [
+        100,
+        1.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        1.0
+      ],
+      [
+        106,
+        1.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        0.8999999999999999
+      ],
+      [
+        109,
+        0.95
+      ],
+      [
+        110,
+        1.0
+      ],
+      [
+        111,
+        1.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        1.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.95
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        1.0
+      ],
+      [
+        119,
+        1.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.050000000000000044
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        1.0
+      ],
+      [
+        124,
+        0.95
+      ],
+      [
+        125,
+        0.75
+      ],
+      [
+        126,
+        0.15000000000000002
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        1.0
+      ],
+      [
+        129,
+        0.20000000000000007
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        1.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.8999999999999999
+      ],
+      [
+        135,
+        0.85
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        1.0
+      ],
+      [
+        138,
+        1.0
+      ],
+      [
+        139,
+        1.0
+      ],
+      [
+        140,
+        1.0
+      ],
+      [
+        141,
+        0.8999999999999999
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        1.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        1.0
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        1.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        0.09999999999999998
+      ],
+      [
+        155,
+        0.65
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        1.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        1.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.9999945874558878
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.9984520123839009
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        1.0
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        1.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        1.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        1.0
+      ],
+      [
+        65,
+        0.8947368421052632
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        1.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        1.0
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        1.0
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        0.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        1.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        1.0
+      ],
+      [
+        84,
+        1.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        1.0
+      ],
+      [
+        91,
+        1.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.763157894736842
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        1.0
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        1.0
+      ],
+      [
+        100,
+        1.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        1.0
+      ],
+      [
+        106,
+        1.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        1.0
+      ],
+      [
+        109,
+        1.0
+      ],
+      [
+        110,
+        1.0
+      ],
+      [
+        111,
+        1.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        1.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        1.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        1.0
+      ],
+      [
+        119,
+        1.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.5
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        1.0
+      ],
+      [
+        124,
+        1.0
+      ],
+      [
+        125,
+        1.0
+      ],
+      [
+        126,
+        0.8947368421052632
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        1.0
+      ],
+      [
+        129,
+        0.956656346749226
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        1.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        1.0
+      ],
+      [
+        135,
+        1.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        1.0
+      ],
+      [
+        138,
+        1.0
+      ],
+      [
+        139,
+        1.0
+      ],
+      [
+        140,
+        1.0
+      ],
+      [
+        141,
+        1.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        1.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        1.0
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        1.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        0.763157894736842
+      ],
+      [
+        155,
+        1.0
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "Nxcode-CQ-7B-orpo",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "11_Nxcode-CQ-7B-orpo.json",
+    "load_data_path": null,
+    "metric_output_path": "11_Nxcode-CQ-7B-orpo_result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/1_codegemma-2b-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        0.85
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        0.3500000000000001
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        0.75
+      ],
+      [
+        5,
+        0.25
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.95
+      ],
+      [
+        9,
+        0.09999999999999998
+      ],
+      [
+        10,
+        0.4
+      ],
+      [
+        11,
+        0.95
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        0.95
+      ],
+      [
+        14,
+        0.5499999999999998
+      ],
+      [
+        15,
+        0.15000000000000002
+      ],
+      [
+        16,
+        0.09999999999999998
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.4
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.0
+      ],
+      [
+        21,
+        0.7999999999999999
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        0.30000000000000004
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        0.75
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        0.95
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.0
+      ],
+      [
+        37,
+        0.09999999999999998
+      ],
+      [
+        38,
+        0.95
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        0.95
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.7999999999999999
+      ],
+      [
+        44,
+        0.8999999999999999
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.95
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        0.30000000000000004
+      ],
+      [
+        49,
+        0.09999999999999998
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        0.4999999999999999
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        0.85
+      ],
+      [
+        56,
+        0.4
+      ],
+      [
+        57,
+        0.09999999999999998
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.6000000000000001
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.7
+      ],
+      [
+        62,
+        0.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.09999999999999998
+      ],
+      [
+        66,
+        0.85
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.050000000000000044
+      ],
+      [
+        69,
+        0.050000000000000044
+      ],
+      [
+        70,
+        0.0
+      ],
+      [
+        71,
+        0.25
+      ],
+      [
+        72,
+        0.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.0
+      ],
+      [
+        79,
+        0.0
+      ],
+      [
+        80,
+        0.050000000000000044
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.050000000000000044
+      ],
+      [
+        85,
+        0.0
+      ],
+      [
+        86,
+        0.09999999999999998
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        0.0
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.15000000000000002
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.30000000000000004
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.050000000000000044
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.09999999999999998
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.09999999999999998
+      ],
+      [
+        99,
+        0.050000000000000044
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.050000000000000044
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.3500000000000001
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        0.050000000000000044
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        0.85
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        0.8999999999999999
+      ],
+      [
+        117,
+        0.050000000000000044
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.4999999999999999
+      ],
+      [
+        122,
+        0.30000000000000004
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.09999999999999998
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.050000000000000044
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.65
+      ],
+      [
+        143,
+        0.0
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.0
+      ],
+      [
+        148,
+        0.0
+      ],
+      [
+        149,
+        0.0
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.09999999999999998
+      ],
+      [
+        152,
+        0.09999999999999998
+      ],
+      [
+        153,
+        0.0
+      ],
+      [
+        154,
+        0.050000000000000044
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.050000000000000044
+      ],
+      [
+        157,
+        0.09999999999999998
+      ],
+      [
+        158,
+        0.15000000000000002
+      ],
+      [
+        159,
+        0.09999999999999998
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        0.9984520123839009
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.9837461300309598
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.763157894736842
+      ],
+      [
+        10,
+        0.9996427720885925
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        0.8947368421052632
+      ],
+      [
+        16,
+        0.763157894736842
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.9996427720885925
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        0.9945820433436533
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.0
+      ],
+      [
+        37,
+        0.763157894736842
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        0.9945820433436533
+      ],
+      [
+        49,
+        0.763157894736842
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        0.9999945874558878
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        0.9996427720885925
+      ],
+      [
+        57,
+        0.763157894736842
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        0.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.763157894736842
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.5
+      ],
+      [
+        69,
+        0.5
+      ],
+      [
+        70,
+        0.0
+      ],
+      [
+        71,
+        0.9837461300309598
+      ],
+      [
+        72,
+        0.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.0
+      ],
+      [
+        79,
+        0.0
+      ],
+      [
+        80,
+        0.5
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.5
+      ],
+      [
+        85,
+        0.0
+      ],
+      [
+        86,
+        0.763157894736842
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        0.0
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.8947368421052632
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.9945820433436533
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.5
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.763157894736842
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.763157894736842
+      ],
+      [
+        99,
+        0.5
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.5
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.9984520123839009
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        0.5
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.5
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.9999945874558878
+      ],
+      [
+        122,
+        0.9945820433436533
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.763157894736842
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.5
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        0.0
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.0
+      ],
+      [
+        148,
+        0.0
+      ],
+      [
+        149,
+        0.0
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.763157894736842
+      ],
+      [
+        152,
+        0.763157894736842
+      ],
+      [
+        153,
+        0.0
+      ],
+      [
+        154,
+        0.5
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.5
+      ],
+      [
+        157,
+        0.763157894736842
+      ],
+      [
+        158,
+        0.8947368421052632
+      ],
+      [
+        159,
+        0.763157894736842
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "codegemma-2b",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "2_codegemma-2b_generations_humaneval_codegemma-2b_humaneval.json",
+    "load_data_path": null,
+    "metric_output_path": "2codegemma-2b-result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/2_codegemma-7b-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.4999999999999999
+      ],
+      [
+        6,
+        0.050000000000000044
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.5499999999999998
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        0.4999999999999999
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        0.5499999999999998
+      ],
+      [
+        17,
+        0.30000000000000004
+      ],
+      [
+        18,
+        0.09999999999999998
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.09999999999999998
+      ],
+      [
+        21,
+        0.3500000000000001
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.65
+      ],
+      [
+        25,
+        0.4
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        0.85
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        0.95
+      ],
+      [
+        31,
+        0.95
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.15000000000000002
+      ],
+      [
+        37,
+        0.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        0.95
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.8999999999999999
+      ],
+      [
+        44,
+        0.75
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.7
+      ],
+      [
+        47,
+        0.75
+      ],
+      [
+        48,
+        0.8999999999999999
+      ],
+      [
+        49,
+        0.8999999999999999
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        0.5499999999999998
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        0.75
+      ],
+      [
+        56,
+        0.44999999999999984
+      ],
+      [
+        57,
+        0.5499999999999998
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.15000000000000002
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.6000000000000001
+      ],
+      [
+        62,
+        0.65
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.25
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.6000000000000001
+      ],
+      [
+        69,
+        0.09999999999999998
+      ],
+      [
+        70,
+        0.20000000000000007
+      ],
+      [
+        71,
+        0.7999999999999999
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.20000000000000007
+      ],
+      [
+        74,
+        0.44999999999999984
+      ],
+      [
+        75,
+        0.09999999999999998
+      ],
+      [
+        76,
+        0.7
+      ],
+      [
+        77,
+        0.050000000000000044
+      ],
+      [
+        78,
+        0.4999999999999999
+      ],
+      [
+        79,
+        0.5499999999999998
+      ],
+      [
+        80,
+        0.44999999999999984
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.050000000000000044
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.25
+      ],
+      [
+        86,
+        0.15000000000000002
+      ],
+      [
+        87,
+        0.3500000000000001
+      ],
+      [
+        88,
+        0.20000000000000007
+      ],
+      [
+        89,
+        0.25
+      ],
+      [
+        90,
+        0.30000000000000004
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.6000000000000001
+      ],
+      [
+        93,
+        0.050000000000000044
+      ],
+      [
+        94,
+        0.0
+      ],
+      [
+        95,
+        0.4
+      ],
+      [
+        96,
+        0.4
+      ],
+      [
+        97,
+        0.75
+      ],
+      [
+        98,
+        0.7
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.050000000000000044
+      ],
+      [
+        101,
+        0.30000000000000004
+      ],
+      [
+        102,
+        0.15000000000000002
+      ],
+      [
+        103,
+        0.65
+      ],
+      [
+        104,
+        0.20000000000000007
+      ],
+      [
+        105,
+        0.050000000000000044
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.6000000000000001
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.050000000000000044
+      ],
+      [
+        111,
+        0.15000000000000002
+      ],
+      [
+        112,
+        0.25
+      ],
+      [
+        113,
+        0.44999999999999984
+      ],
+      [
+        114,
+        0.4999999999999999
+      ],
+      [
+        115,
+        0.09999999999999998
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.25
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.15000000000000002
+      ],
+      [
+        121,
+        0.4
+      ],
+      [
+        122,
+        0.75
+      ],
+      [
+        123,
+        0.15000000000000002
+      ],
+      [
+        124,
+        0.20000000000000007
+      ],
+      [
+        125,
+        0.09999999999999998
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.050000000000000044
+      ],
+      [
+        128,
+        0.20000000000000007
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.75
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.09999999999999998
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.20000000000000007
+      ],
+      [
+        143,
+        0.15000000000000002
+      ],
+      [
+        144,
+        0.3500000000000001
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.3500000000000001
+      ],
+      [
+        148,
+        0.30000000000000004
+      ],
+      [
+        149,
+        0.25
+      ],
+      [
+        150,
+        0.050000000000000044
+      ],
+      [
+        151,
+        0.3500000000000001
+      ],
+      [
+        152,
+        0.75
+      ],
+      [
+        153,
+        0.15000000000000002
+      ],
+      [
+        154,
+        0.20000000000000007
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.30000000000000004
+      ],
+      [
+        157,
+        0.3500000000000001
+      ],
+      [
+        158,
+        0.25
+      ],
+      [
+        159,
+        0.09999999999999998
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.09999999999999998
+      ],
+      [
+        162,
+        0.5499999999999998
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.9999945874558878
+      ],
+      [
+        6,
+        0.5
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        0.9999945874558878
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.9945820433436533
+      ],
+      [
+        18,
+        0.763157894736842
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.763157894736842
+      ],
+      [
+        21,
+        0.9984520123839009
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        0.9996427720885925
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.8947368421052632
+      ],
+      [
+        37,
+        0.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        0.9999404620147654
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.8947368421052632
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.9837461300309598
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        0.763157894736842
+      ],
+      [
+        70,
+        0.956656346749226
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.956656346749226
+      ],
+      [
+        74,
+        0.9999404620147654
+      ],
+      [
+        75,
+        0.763157894736842
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.5
+      ],
+      [
+        78,
+        0.9999945874558878
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        0.9999404620147654
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.5
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.9837461300309598
+      ],
+      [
+        86,
+        0.8947368421052632
+      ],
+      [
+        87,
+        0.9984520123839009
+      ],
+      [
+        88,
+        0.956656346749226
+      ],
+      [
+        89,
+        0.9837461300309598
+      ],
+      [
+        90,
+        0.9945820433436533
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.5
+      ],
+      [
+        94,
+        0.0
+      ],
+      [
+        95,
+        0.9996427720885925
+      ],
+      [
+        96,
+        0.9996427720885925
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.5
+      ],
+      [
+        101,
+        0.9945820433436533
+      ],
+      [
+        102,
+        0.8947368421052632
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        0.956656346749226
+      ],
+      [
+        105,
+        0.5
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.5
+      ],
+      [
+        111,
+        0.8947368421052632
+      ],
+      [
+        112,
+        0.9837461300309598
+      ],
+      [
+        113,
+        0.9999404620147654
+      ],
+      [
+        114,
+        0.9999945874558878
+      ],
+      [
+        115,
+        0.763157894736842
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.9837461300309598
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.8947368421052632
+      ],
+      [
+        121,
+        0.9996427720885925
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.8947368421052632
+      ],
+      [
+        124,
+        0.956656346749226
+      ],
+      [
+        125,
+        0.763157894736842
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.5
+      ],
+      [
+        128,
+        0.956656346749226
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.763157894736842
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.956656346749226
+      ],
+      [
+        143,
+        0.8947368421052632
+      ],
+      [
+        144,
+        0.9984520123839009
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.9984520123839009
+      ],
+      [
+        148,
+        0.9945820433436533
+      ],
+      [
+        149,
+        0.9837461300309598
+      ],
+      [
+        150,
+        0.5
+      ],
+      [
+        151,
+        0.9984520123839009
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.8947368421052632
+      ],
+      [
+        154,
+        0.956656346749226
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.9945820433436533
+      ],
+      [
+        157,
+        0.9984520123839009
+      ],
+      [
+        158,
+        0.9837461300309598
+      ],
+      [
+        159,
+        0.763157894736842
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.763157894736842
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "codegemma-7b",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "3_codegemma-7b_generations_humaneval_codegemma-7b_humaneval.json",
+    "load_data_path": null,
+    "metric_output_path": "3_codegemma-7b-result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/3_codegemma-7b-it-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        0.95
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.8999999999999999
+      ],
+      [
+        6,
+        0.20000000000000007
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.4999999999999999
+      ],
+      [
+        10,
+        0.050000000000000044
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        0.8999999999999999
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        0.7999999999999999
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.7
+      ],
+      [
+        18,
+        0.7
+      ],
+      [
+        19,
+        0.3500000000000001
+      ],
+      [
+        20,
+        0.4
+      ],
+      [
+        21,
+        0.85
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.8999999999999999
+      ],
+      [
+        25,
+        0.85
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.050000000000000044
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.5499999999999998
+      ],
+      [
+        37,
+        0.25
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.050000000000000044
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.4999999999999999
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        0.6000000000000001
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        0.8999999999999999
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        0.85
+      ],
+      [
+        56,
+        0.95
+      ],
+      [
+        57,
+        0.4
+      ],
+      [
+        58,
+        0.7999999999999999
+      ],
+      [
+        59,
+        0.44999999999999984
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.95
+      ],
+      [
+        62,
+        0.7
+      ],
+      [
+        63,
+        0.4
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.4
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.25
+      ],
+      [
+        68,
+        0.7999999999999999
+      ],
+      [
+        69,
+        0.7
+      ],
+      [
+        70,
+        0.25
+      ],
+      [
+        71,
+        0.85
+      ],
+      [
+        72,
+        0.5499999999999998
+      ],
+      [
+        73,
+        0.09999999999999998
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.15000000000000002
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.4999999999999999
+      ],
+      [
+        79,
+        0.050000000000000044
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.5499999999999998
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.050000000000000044
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        0.7999999999999999
+      ],
+      [
+        87,
+        0.5499999999999998
+      ],
+      [
+        88,
+        0.20000000000000007
+      ],
+      [
+        89,
+        0.8999999999999999
+      ],
+      [
+        90,
+        0.09999999999999998
+      ],
+      [
+        91,
+        0.15000000000000002
+      ],
+      [
+        92,
+        0.7999999999999999
+      ],
+      [
+        93,
+        0.050000000000000044
+      ],
+      [
+        94,
+        0.30000000000000004
+      ],
+      [
+        95,
+        0.050000000000000044
+      ],
+      [
+        96,
+        0.8999999999999999
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.8999999999999999
+      ],
+      [
+        99,
+        0.3500000000000001
+      ],
+      [
+        100,
+        0.15000000000000002
+      ],
+      [
+        101,
+        0.09999999999999998
+      ],
+      [
+        102,
+        0.7
+      ],
+      [
+        103,
+        0.5499999999999998
+      ],
+      [
+        104,
+        0.3500000000000001
+      ],
+      [
+        105,
+        0.20000000000000007
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.8999999999999999
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.15000000000000002
+      ],
+      [
+        110,
+        0.15000000000000002
+      ],
+      [
+        111,
+        0.6000000000000001
+      ],
+      [
+        112,
+        0.7999999999999999
+      ],
+      [
+        113,
+        0.7
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.050000000000000044
+      ],
+      [
+        116,
+        0.15000000000000002
+      ],
+      [
+        117,
+        0.65
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.09999999999999998
+      ],
+      [
+        120,
+        0.09999999999999998
+      ],
+      [
+        121,
+        0.5499999999999998
+      ],
+      [
+        122,
+        0.85
+      ],
+      [
+        123,
+        0.4999999999999999
+      ],
+      [
+        124,
+        0.050000000000000044
+      ],
+      [
+        125,
+        0.050000000000000044
+      ],
+      [
+        126,
+        0.050000000000000044
+      ],
+      [
+        127,
+        0.050000000000000044
+      ],
+      [
+        128,
+        0.4999999999999999
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.8999999999999999
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.0
+      ],
+      [
+        139,
+        0.09999999999999998
+      ],
+      [
+        140,
+        0.050000000000000044
+      ],
+      [
+        141,
+        0.65
+      ],
+      [
+        142,
+        0.25
+      ],
+      [
+        143,
+        0.15000000000000002
+      ],
+      [
+        144,
+        0.44999999999999984
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.20000000000000007
+      ],
+      [
+        147,
+        0.8999999999999999
+      ],
+      [
+        148,
+        0.7
+      ],
+      [
+        149,
+        0.5499999999999998
+      ],
+      [
+        150,
+        0.3500000000000001
+      ],
+      [
+        151,
+        1.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.75
+      ],
+      [
+        154,
+        0.5499999999999998
+      ],
+      [
+        155,
+        0.25
+      ],
+      [
+        156,
+        0.44999999999999984
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        0.8999999999999999
+      ],
+      [
+        159,
+        0.8999999999999999
+      ],
+      [
+        160,
+        0.15000000000000002
+      ],
+      [
+        161,
+        0.25
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        0.956656346749226
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.9999945874558878
+      ],
+      [
+        10,
+        0.5
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.9984520123839009
+      ],
+      [
+        20,
+        0.9996427720885925
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.5
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        0.9837461300309598
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.5
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.9999945874558878
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        0.9996427720885925
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.9999404620147654
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        0.9996427720885925
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.9996427720885925
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.9837461300309598
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        0.9837461300309598
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.763157894736842
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.8947368421052632
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.9999945874558878
+      ],
+      [
+        79,
+        0.5
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.5
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        0.956656346749226
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        0.763157894736842
+      ],
+      [
+        91,
+        0.8947368421052632
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.5
+      ],
+      [
+        94,
+        0.9945820433436533
+      ],
+      [
+        95,
+        0.5
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.9984520123839009
+      ],
+      [
+        100,
+        0.8947368421052632
+      ],
+      [
+        101,
+        0.763157894736842
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        0.9984520123839009
+      ],
+      [
+        105,
+        0.956656346749226
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.8947368421052632
+      ],
+      [
+        110,
+        0.8947368421052632
+      ],
+      [
+        111,
+        1.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        1.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.5
+      ],
+      [
+        116,
+        0.8947368421052632
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.763157894736842
+      ],
+      [
+        120,
+        0.763157894736842
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.9999945874558878
+      ],
+      [
+        124,
+        0.5
+      ],
+      [
+        125,
+        0.5
+      ],
+      [
+        126,
+        0.5
+      ],
+      [
+        127,
+        0.5
+      ],
+      [
+        128,
+        0.9999945874558878
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.0
+      ],
+      [
+        139,
+        0.763157894736842
+      ],
+      [
+        140,
+        0.5
+      ],
+      [
+        141,
+        1.0
+      ],
+      [
+        142,
+        0.9837461300309598
+      ],
+      [
+        143,
+        0.8947368421052632
+      ],
+      [
+        144,
+        0.9999404620147654
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.956656346749226
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        1.0
+      ],
+      [
+        150,
+        0.9984520123839009
+      ],
+      [
+        151,
+        1.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        1.0
+      ],
+      [
+        155,
+        0.9837461300309598
+      ],
+      [
+        156,
+        0.9999404620147654
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.8947368421052632
+      ],
+      [
+        161,
+        0.9837461300309598
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "codegemma-7b-it",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "1_codegemma-7b-it_generations_humaneval_codegemma-7b-it_humaneval.json",
+    "load_data_path": null,
+    "metric_output_path": "1_codegemma-7b-it-result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/4_deepseek-coder-1.3b-base-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        0.8999999999999999
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        0.85
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.0
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.95
+      ],
+      [
+        9,
+        0.0
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        0.85
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        0.95
+      ],
+      [
+        14,
+        0.25
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.050000000000000044
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.050000000000000044
+      ],
+      [
+        21,
+        0.7999999999999999
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        0.7999999999999999
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.0
+      ],
+      [
+        37,
+        0.4
+      ],
+      [
+        38,
+        0.15000000000000002
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        0.8999999999999999
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.25
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.7
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        0.75
+      ],
+      [
+        52,
+        0.09999999999999998
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.44999999999999984
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        0.7999999999999999
+      ],
+      [
+        57,
+        0.6000000000000001
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.65
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.75
+      ],
+      [
+        62,
+        0.25
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.8999999999999999
+      ],
+      [
+        69,
+        0.050000000000000044
+      ],
+      [
+        70,
+        0.0
+      ],
+      [
+        71,
+        0.20000000000000007
+      ],
+      [
+        72,
+        0.75
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.0
+      ],
+      [
+        79,
+        0.6000000000000001
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.4999999999999999
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.0
+      ],
+      [
+        86,
+        0.0
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        0.0
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.3500000000000001
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.4999999999999999
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.15000000000000002
+      ],
+      [
+        97,
+        0.0
+      ],
+      [
+        98,
+        0.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.0
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.25
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        0.5499999999999998
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        0.6000000000000001
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.0
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.85
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.0
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.050000000000000044
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.050000000000000044
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.0
+      ],
+      [
+        143,
+        0.20000000000000007
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.0
+      ],
+      [
+        148,
+        0.0
+      ],
+      [
+        149,
+        0.0
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.30000000000000004
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.0
+      ],
+      [
+        154,
+        0.0
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.5499999999999998
+      ],
+      [
+        157,
+        0.7999999999999999
+      ],
+      [
+        158,
+        0.0
+      ],
+      [
+        159,
+        0.44999999999999984
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.15000000000000002
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.0
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.0
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        0.9837461300309598
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.5
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.5
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.0
+      ],
+      [
+        37,
+        0.9996427720885925
+      ],
+      [
+        38,
+        0.8947368421052632
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.9837461300309598
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        0.763157894736842
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.9999404620147654
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        0.9837461300309598
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        0.5
+      ],
+      [
+        70,
+        0.0
+      ],
+      [
+        71,
+        0.956656346749226
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.9999945874558878
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.0
+      ],
+      [
+        86,
+        0.0
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        0.0
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.9984520123839009
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.9999945874558878
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.8947368421052632
+      ],
+      [
+        97,
+        0.0
+      ],
+      [
+        98,
+        0.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.0
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.9837461300309598
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.0
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.0
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.5
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.5
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.0
+      ],
+      [
+        143,
+        0.956656346749226
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.0
+      ],
+      [
+        148,
+        0.0
+      ],
+      [
+        149,
+        0.0
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.9945820433436533
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.0
+      ],
+      [
+        154,
+        0.0
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        0.0
+      ],
+      [
+        159,
+        0.9999404620147654
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.8947368421052632
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "deepseek-coder-1.3b-base",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "4_deepseek-coder-1.3b-base_generations_humaneval_deepseek-coder-1.3b-base.json",
+    "load_data_path": null,
+    "metric_output_path": "4_deepseek-coder-1.3b-base-result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/5_deepseek-coder-6.7b-base-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.15000000000000002
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.25
+      ],
+      [
+        6,
+        0.25
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.85
+      ],
+      [
+        9,
+        0.5499999999999998
+      ],
+      [
+        10,
+        0.3500000000000001
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.09999999999999998
+      ],
+      [
+        18,
+        0.4999999999999999
+      ],
+      [
+        19,
+        0.95
+      ],
+      [
+        20,
+        0.75
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.75
+      ],
+      [
+        37,
+        0.050000000000000044
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.6000000000000001
+      ],
+      [
+        40,
+        0.95
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.7999999999999999
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.85
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        0.8999999999999999
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        0.85
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.85
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.7999999999999999
+      ],
+      [
+        62,
+        0.85
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        0.5499999999999998
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.050000000000000044
+      ],
+      [
+        68,
+        0.3500000000000001
+      ],
+      [
+        69,
+        0.4
+      ],
+      [
+        70,
+        0.44999999999999984
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        0.95
+      ],
+      [
+        73,
+        0.050000000000000044
+      ],
+      [
+        74,
+        0.30000000000000004
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.95
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.30000000000000004
+      ],
+      [
+        79,
+        0.5499999999999998
+      ],
+      [
+        80,
+        0.25
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.30000000000000004
+      ],
+      [
+        86,
+        0.050000000000000044
+      ],
+      [
+        87,
+        0.6000000000000001
+      ],
+      [
+        88,
+        0.5499999999999998
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.050000000000000044
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.8999999999999999
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.4999999999999999
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.4
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.0
+      ],
+      [
+        99,
+        0.050000000000000044
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.95
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        0.95
+      ],
+      [
+        113,
+        0.050000000000000044
+      ],
+      [
+        114,
+        0.95
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.20000000000000007
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.15000000000000002
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        0.050000000000000044
+      ],
+      [
+        124,
+        0.15000000000000002
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.75
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.4
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.4999999999999999
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.25
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.7999999999999999
+      ],
+      [
+        143,
+        0.20000000000000007
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.09999999999999998
+      ],
+      [
+        148,
+        0.6000000000000001
+      ],
+      [
+        149,
+        0.3500000000000001
+      ],
+      [
+        150,
+        0.4
+      ],
+      [
+        151,
+        0.09999999999999998
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.050000000000000044
+      ],
+      [
+        154,
+        0.0
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.0
+      ],
+      [
+        157,
+        0.3500000000000001
+      ],
+      [
+        158,
+        0.85
+      ],
+      [
+        159,
+        0.30000000000000004
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.95
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.8947368421052632
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.9837461300309598
+      ],
+      [
+        6,
+        0.9837461300309598
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        0.9984520123839009
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.763157894736842
+      ],
+      [
+        18,
+        0.9999945874558878
+      ],
+      [
+        19,
+        1.0
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        0.5
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        1.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        1.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.5
+      ],
+      [
+        68,
+        0.9984520123839009
+      ],
+      [
+        69,
+        0.9996427720885925
+      ],
+      [
+        70,
+        0.9999404620147654
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.5
+      ],
+      [
+        74,
+        0.9945820433436533
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.0
+      ],
+      [
+        78,
+        0.9945820433436533
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        0.9837461300309598
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        0.9945820433436533
+      ],
+      [
+        86,
+        0.5
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        0.0
+      ],
+      [
+        90,
+        0.5
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.9999945874558878
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        0.9996427720885925
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.0
+      ],
+      [
+        99,
+        0.5
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        0.0
+      ],
+      [
+        105,
+        0.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.5
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.956656346749226
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.8947368421052632
+      ],
+      [
+        122,
+        0.0
+      ],
+      [
+        123,
+        0.5
+      ],
+      [
+        124,
+        0.8947368421052632
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        1.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.9996427720885925
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.9999945874558878
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.9837461300309598
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        0.956656346749226
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.0
+      ],
+      [
+        147,
+        0.763157894736842
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        0.9984520123839009
+      ],
+      [
+        150,
+        0.9996427720885925
+      ],
+      [
+        151,
+        0.763157894736842
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.5
+      ],
+      [
+        154,
+        0.0
+      ],
+      [
+        155,
+        0.0
+      ],
+      [
+        156,
+        0.0
+      ],
+      [
+        157,
+        0.9984520123839009
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.9945820433436533
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "deepseek-coder-6.7b-base",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "5_deepseek-coder-6.7b-base_generations_humaneval_deepseek-coder-6.7b-base.json",
+    "load_data_path": null,
+    "metric_output_path": "deepseek-coder-6.7b-base-result.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/6_deepseek_coder-6.7b-instruct-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.4999999999999999
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        1.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.0
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        0.95
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.25
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.050000000000000044
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        1.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.09999999999999998
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.65
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.95
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        0.0
+      ],
+      [
+        63,
+        0.95
+      ],
+      [
+        64,
+        0.20000000000000007
+      ],
+      [
+        65,
+        0.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.09999999999999998
+      ],
+      [
+        68,
+        0.7
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        1.0
+      ],
+      [
+        74,
+        0.7999999999999999
+      ],
+      [
+        75,
+        0.30000000000000004
+      ],
+      [
+        76,
+        0.15000000000000002
+      ],
+      [
+        77,
+        0.20000000000000007
+      ],
+      [
+        78,
+        0.85
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        0.95
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        0.7999999999999999
+      ],
+      [
+        89,
+        0.7
+      ],
+      [
+        90,
+        0.15000000000000002
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.75
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.4
+      ],
+      [
+        100,
+        1.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        0.8999999999999999
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        0.4
+      ],
+      [
+        106,
+        1.0
+      ],
+      [
+        107,
+        0.7999999999999999
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.09999999999999998
+      ],
+      [
+        110,
+        1.0
+      ],
+      [
+        111,
+        0.09999999999999998
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.44999999999999984
+      ],
+      [
+        114,
+        0.95
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        1.0
+      ],
+      [
+        119,
+        0.25
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.8999999999999999
+      ],
+      [
+        124,
+        0.44999999999999984
+      ],
+      [
+        125,
+        0.8999999999999999
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.15000000000000002
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        1.0
+      ],
+      [
+        134,
+        0.09999999999999998
+      ],
+      [
+        135,
+        0.5499999999999998
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.0
+      ],
+      [
+        139,
+        1.0
+      ],
+      [
+        140,
+        0.050000000000000044
+      ],
+      [
+        141,
+        0.3500000000000001
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        0.15000000000000002
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        0.95
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        0.7
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        0.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        0.8999999999999999
+      ],
+      [
+        155,
+        0.85
+      ],
+      [
+        156,
+        0.7
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.30000000000000004
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.9999945874558878
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        1.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.0
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.9837461300309598
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.5
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        1.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        0.763157894736842
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        0.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.956656346749226
+      ],
+      [
+        65,
+        0.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.763157894736842
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        1.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        1.0
+      ],
+      [
+        74,
+        1.0
+      ],
+      [
+        75,
+        0.9945820433436533
+      ],
+      [
+        76,
+        0.8947368421052632
+      ],
+      [
+        77,
+        0.956656346749226
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        0.8947368421052632
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        0.0
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.9996427720885925
+      ],
+      [
+        100,
+        1.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        0.9996427720885925
+      ],
+      [
+        106,
+        1.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.763157894736842
+      ],
+      [
+        110,
+        1.0
+      ],
+      [
+        111,
+        0.763157894736842
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.9999404620147654
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.0
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        1.0
+      ],
+      [
+        119,
+        0.9837461300309598
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        1.0
+      ],
+      [
+        124,
+        0.9999404620147654
+      ],
+      [
+        125,
+        1.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.0
+      ],
+      [
+        128,
+        0.8947368421052632
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        1.0
+      ],
+      [
+        134,
+        0.763157894736842
+      ],
+      [
+        135,
+        1.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.0
+      ],
+      [
+        139,
+        1.0
+      ],
+      [
+        140,
+        0.5
+      ],
+      [
+        141,
+        0.9984520123839009
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        0.8947368421052632
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        1.0
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        0.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        1.0
+      ],
+      [
+        155,
+        1.0
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.9945820433436533
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        1.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "deepseek-coder-6.7b-instruct",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "generations_humaneval_deepseek-coder-6.7b-instruct.json",
+    "load_data_path": null,
+    "metric_output_path": "evaluation_results.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/7_deepseek_coder_33b-base-results.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.15000000000000002
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.7999999999999999
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.050000000000000044
+      ],
+      [
+        9,
+        0.8999999999999999
+      ],
+      [
+        10,
+        0.95
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        0.15000000000000002
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.3500000000000001
+      ],
+      [
+        19,
+        0.20000000000000007
+      ],
+      [
+        20,
+        0.7999999999999999
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.30000000000000004
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        0.5499999999999998
+      ],
+      [
+        37,
+        0.6000000000000001
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        0.95
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        0.6000000000000001
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.7999999999999999
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        0.85
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        0.5499999999999998
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        1.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        0.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        0.95
+      ],
+      [
+        72,
+        0.95
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.20000000000000007
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.95
+      ],
+      [
+        77,
+        0.050000000000000044
+      ],
+      [
+        78,
+        0.3500000000000001
+      ],
+      [
+        79,
+        0.85
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.3500000000000001
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.050000000000000044
+      ],
+      [
+        85,
+        0.95
+      ],
+      [
+        86,
+        0.85
+      ],
+      [
+        87,
+        0.95
+      ],
+      [
+        88,
+        0.85
+      ],
+      [
+        89,
+        0.6000000000000001
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        0.95
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.3500000000000001
+      ],
+      [
+        95,
+        0.7999999999999999
+      ],
+      [
+        96,
+        0.4999999999999999
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.7
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.95
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.4999999999999999
+      ],
+      [
+        104,
+        0.75
+      ],
+      [
+        105,
+        0.20000000000000007
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.44999999999999984
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.15000000000000002
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.30000000000000004
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        0.85
+      ],
+      [
+        115,
+        0.050000000000000044
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.3500000000000001
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.20000000000000007
+      ],
+      [
+        122,
+        0.8999999999999999
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.0
+      ],
+      [
+        125,
+        0.25
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.050000000000000044
+      ],
+      [
+        128,
+        0.4999999999999999
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.44999999999999984
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.09999999999999998
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        0.6000000000000001
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.3500000000000001
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.30000000000000004
+      ],
+      [
+        142,
+        0.6000000000000001
+      ],
+      [
+        143,
+        0.65
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.95
+      ],
+      [
+        147,
+        0.7999999999999999
+      ],
+      [
+        148,
+        0.6000000000000001
+      ],
+      [
+        149,
+        0.15000000000000002
+      ],
+      [
+        150,
+        0.95
+      ],
+      [
+        151,
+        0.20000000000000007
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.09999999999999998
+      ],
+      [
+        154,
+        0.20000000000000007
+      ],
+      [
+        155,
+        0.44999999999999984
+      ],
+      [
+        156,
+        0.15000000000000002
+      ],
+      [
+        157,
+        0.15000000000000002
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.0
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.5499999999999998
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.8947368421052632
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.5
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        0.8947368421052632
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.9984520123839009
+      ],
+      [
+        19,
+        0.956656346749226
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.9945820433436533
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.0
+      ],
+      [
+        65,
+        1.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        0.0
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.956656346749226
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.5
+      ],
+      [
+        78,
+        0.9984520123839009
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        0.9984520123839009
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.5
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        1.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        0.9984520123839009
+      ],
+      [
+        95,
+        1.0
+      ],
+      [
+        96,
+        0.9999945874558878
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        0.0
+      ],
+      [
+        103,
+        0.9999945874558878
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        0.956656346749226
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.9999404620147654
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.8947368421052632
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.9945820433436533
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.5
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.9984520123839009
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        0.956656346749226
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.0
+      ],
+      [
+        124,
+        0.0
+      ],
+      [
+        125,
+        0.9837461300309598
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.5
+      ],
+      [
+        128,
+        0.9999945874558878
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.9999404620147654
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.763157894736842
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.9984520123839009
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.9945820433436533
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        0.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        0.8947368421052632
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        0.956656346749226
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.763157894736842
+      ],
+      [
+        154,
+        0.956656346749226
+      ],
+      [
+        155,
+        0.9999404620147654
+      ],
+      [
+        156,
+        0.8947368421052632
+      ],
+      [
+        157,
+        0.8947368421052632
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.0
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "deepseek-coder-33b-base",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "generations_humaneval_deepseek-coder-33b-base.json",
+    "load_data_path": null,
+    "metric_output_path": "deepseek_coder_33b-base_evaluation_results.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/8_deepseek_coder_33b-instruct-results.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.6000000000000001
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.8999999999999999
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        0.75
+      ],
+      [
+        9,
+        0.65
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        0.75
+      ],
+      [
+        19,
+        0.09999999999999998
+      ],
+      [
+        20,
+        0.75
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.4999999999999999
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.7999999999999999
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.6000000000000001
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.5499999999999998
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.30000000000000004
+      ],
+      [
+        65,
+        0.95
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.4999999999999999
+      ],
+      [
+        69,
+        0.4
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        0.95
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.20000000000000007
+      ],
+      [
+        76,
+        0.7
+      ],
+      [
+        77,
+        0.20000000000000007
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.20000000000000007
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        0.8999999999999999
+      ],
+      [
+        87,
+        0.44999999999999984
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        0.09999999999999998
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.20000000000000007
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        0.4
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        0.95
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.7999999999999999
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.75
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.4
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.15000000000000002
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.7999999999999999
+      ],
+      [
+        118,
+        0.44999999999999984
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.3500000000000001
+      ],
+      [
+        124,
+        0.5499999999999998
+      ],
+      [
+        125,
+        0.5499999999999998
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.7999999999999999
+      ],
+      [
+        128,
+        0.6000000000000001
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.30000000000000004
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.5499999999999998
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.65
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        0.7
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.85
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        0.15000000000000002
+      ],
+      [
+        149,
+        0.20000000000000007
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        0.20000000000000007
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        0.8999999999999999
+      ],
+      [
+        155,
+        1.0
+      ],
+      [
+        156,
+        0.8999999999999999
+      ],
+      [
+        157,
+        0.050000000000000044
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.050000000000000044
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        0.4
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        1.0
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        1.0
+      ],
+      [
+        6,
+        0.0
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        1.0
+      ],
+      [
+        11,
+        1.0
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.763157894736842
+      ],
+      [
+        20,
+        1.0
+      ],
+      [
+        21,
+        1.0
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        1.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.9999945874558878
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        1.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        1.0
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        1.0
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        1.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.9945820433436533
+      ],
+      [
+        65,
+        1.0
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.0
+      ],
+      [
+        68,
+        0.9999945874558878
+      ],
+      [
+        69,
+        0.9996427720885925
+      ],
+      [
+        70,
+        1.0
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.0
+      ],
+      [
+        74,
+        0.0
+      ],
+      [
+        75,
+        0.956656346749226
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.956656346749226
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        1.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.956656346749226
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        1.0
+      ],
+      [
+        87,
+        0.9999404620147654
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        1.0
+      ],
+      [
+        90,
+        0.0
+      ],
+      [
+        91,
+        0.0
+      ],
+      [
+        92,
+        1.0
+      ],
+      [
+        93,
+        0.0
+      ],
+      [
+        94,
+        1.0
+      ],
+      [
+        95,
+        0.763157894736842
+      ],
+      [
+        96,
+        1.0
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        1.0
+      ],
+      [
+        99,
+        0.0
+      ],
+      [
+        100,
+        0.956656346749226
+      ],
+      [
+        101,
+        1.0
+      ],
+      [
+        102,
+        0.9996427720885925
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        1.0
+      ],
+      [
+        105,
+        1.0
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        1.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        1.0
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.9996427720885925
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.8947368421052632
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        1.0
+      ],
+      [
+        118,
+        0.9999404620147654
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        0.9984520123839009
+      ],
+      [
+        124,
+        1.0
+      ],
+      [
+        125,
+        1.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        1.0
+      ],
+      [
+        128,
+        1.0
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.9945820433436533
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        1.0
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        1.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        1.0
+      ],
+      [
+        144,
+        1.0
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        1.0
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        0.8947368421052632
+      ],
+      [
+        149,
+        0.956656346749226
+      ],
+      [
+        150,
+        1.0
+      ],
+      [
+        151,
+        0.956656346749226
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        1.0
+      ],
+      [
+        154,
+        1.0
+      ],
+      [
+        155,
+        1.0
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        0.5
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        0.5
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        1.0
+      ],
+      [
+        162,
+        0.9996427720885925
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "deepseek-coder-33b-instruct",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "generations_humaneval_deepseek-coder-33b-instruct.json",
+    "load_data_path": null,
+    "metric_output_path": "deepseek_coder_33b-instruct_evaluation_results.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/9_codeqwen1.5-7b-result.json

@@ -0,0 +1,1361 @@
+{
+  "humaneval": {
+    "pass@1": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.09999999999999998
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.30000000000000004
+      ],
+      [
+        6,
+        0.15000000000000002
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        0.95
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        0.15000000000000002
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        0.65
+      ],
+      [
+        18,
+        0.7
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.050000000000000044
+      ],
+      [
+        21,
+        0.25
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        0.65
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        0.050000000000000044
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.15000000000000002
+      ],
+      [
+        40,
+        0.7999999999999999
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        0.85
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        0.7
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        0.5499999999999998
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.15000000000000002
+      ],
+      [
+        65,
+        0.050000000000000044
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.15000000000000002
+      ],
+      [
+        68,
+        0.6000000000000001
+      ],
+      [
+        69,
+        0.0
+      ],
+      [
+        70,
+        0.4
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        0.7
+      ],
+      [
+        73,
+        0.4
+      ],
+      [
+        74,
+        0.4
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        0.75
+      ],
+      [
+        77,
+        0.20000000000000007
+      ],
+      [
+        78,
+        0.75
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        0.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        0.5499999999999998
+      ],
+      [
+        85,
+        0.8999999999999999
+      ],
+      [
+        86,
+        0.3500000000000001
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        0.050000000000000044
+      ],
+      [
+        90,
+        0.95
+      ],
+      [
+        91,
+        0.050000000000000044
+      ],
+      [
+        92,
+        0.44999999999999984
+      ],
+      [
+        93,
+        0.050000000000000044
+      ],
+      [
+        94,
+        0.0
+      ],
+      [
+        95,
+        0.95
+      ],
+      [
+        96,
+        0.30000000000000004
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.30000000000000004
+      ],
+      [
+        99,
+        0.85
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        0.6000000000000001
+      ],
+      [
+        103,
+        0.8999999999999999
+      ],
+      [
+        104,
+        0.09999999999999998
+      ],
+      [
+        105,
+        0.09999999999999998
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        0.75
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.44999999999999984
+      ],
+      [
+        112,
+        0.95
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.30000000000000004
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.4
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        0.6000000000000001
+      ],
+      [
+        123,
+        1.0
+      ],
+      [
+        124,
+        0.85
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.20000000000000007
+      ],
+      [
+        128,
+        0.050000000000000044
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.15000000000000002
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        0.7999999999999999
+      ],
+      [
+        143,
+        0.09999999999999998
+      ],
+      [
+        144,
+        0.09999999999999998
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.30000000000000004
+      ],
+      [
+        147,
+        0.65
+      ],
+      [
+        148,
+        0.65
+      ],
+      [
+        149,
+        0.050000000000000044
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        0.75
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.4999999999999999
+      ],
+      [
+        154,
+        0.8999999999999999
+      ],
+      [
+        155,
+        0.30000000000000004
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        0.7999999999999999
+      ],
+      [
+        159,
+        0.6000000000000001
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ],
+    "pass@10": [
+      [
+        0,
+        1.0
+      ],
+      [
+        1,
+        0.763157894736842
+      ],
+      [
+        2,
+        1.0
+      ],
+      [
+        3,
+        1.0
+      ],
+      [
+        4,
+        1.0
+      ],
+      [
+        5,
+        0.9945820433436533
+      ],
+      [
+        6,
+        0.8947368421052632
+      ],
+      [
+        7,
+        1.0
+      ],
+      [
+        8,
+        1.0
+      ],
+      [
+        9,
+        1.0
+      ],
+      [
+        10,
+        0.0
+      ],
+      [
+        11,
+        0.8947368421052632
+      ],
+      [
+        12,
+        1.0
+      ],
+      [
+        13,
+        1.0
+      ],
+      [
+        14,
+        1.0
+      ],
+      [
+        15,
+        1.0
+      ],
+      [
+        16,
+        1.0
+      ],
+      [
+        17,
+        1.0
+      ],
+      [
+        18,
+        1.0
+      ],
+      [
+        19,
+        0.0
+      ],
+      [
+        20,
+        0.5
+      ],
+      [
+        21,
+        0.9837461300309598
+      ],
+      [
+        22,
+        1.0
+      ],
+      [
+        23,
+        1.0
+      ],
+      [
+        24,
+        0.0
+      ],
+      [
+        25,
+        1.0
+      ],
+      [
+        26,
+        0.0
+      ],
+      [
+        27,
+        1.0
+      ],
+      [
+        28,
+        1.0
+      ],
+      [
+        29,
+        1.0
+      ],
+      [
+        30,
+        1.0
+      ],
+      [
+        31,
+        1.0
+      ],
+      [
+        32,
+        0.0
+      ],
+      [
+        33,
+        0.0
+      ],
+      [
+        34,
+        1.0
+      ],
+      [
+        35,
+        1.0
+      ],
+      [
+        36,
+        1.0
+      ],
+      [
+        37,
+        0.5
+      ],
+      [
+        38,
+        1.0
+      ],
+      [
+        39,
+        0.8947368421052632
+      ],
+      [
+        40,
+        1.0
+      ],
+      [
+        41,
+        0.0
+      ],
+      [
+        42,
+        1.0
+      ],
+      [
+        43,
+        1.0
+      ],
+      [
+        44,
+        1.0
+      ],
+      [
+        45,
+        1.0
+      ],
+      [
+        46,
+        1.0
+      ],
+      [
+        47,
+        1.0
+      ],
+      [
+        48,
+        1.0
+      ],
+      [
+        49,
+        1.0
+      ],
+      [
+        50,
+        1.0
+      ],
+      [
+        51,
+        1.0
+      ],
+      [
+        52,
+        1.0
+      ],
+      [
+        53,
+        1.0
+      ],
+      [
+        54,
+        0.0
+      ],
+      [
+        55,
+        1.0
+      ],
+      [
+        56,
+        1.0
+      ],
+      [
+        57,
+        1.0
+      ],
+      [
+        58,
+        1.0
+      ],
+      [
+        59,
+        1.0
+      ],
+      [
+        60,
+        1.0
+      ],
+      [
+        61,
+        1.0
+      ],
+      [
+        62,
+        1.0
+      ],
+      [
+        63,
+        1.0
+      ],
+      [
+        64,
+        0.8947368421052632
+      ],
+      [
+        65,
+        0.5
+      ],
+      [
+        66,
+        1.0
+      ],
+      [
+        67,
+        0.8947368421052632
+      ],
+      [
+        68,
+        1.0
+      ],
+      [
+        69,
+        0.0
+      ],
+      [
+        70,
+        0.9996427720885925
+      ],
+      [
+        71,
+        1.0
+      ],
+      [
+        72,
+        1.0
+      ],
+      [
+        73,
+        0.9996427720885925
+      ],
+      [
+        74,
+        0.9996427720885925
+      ],
+      [
+        75,
+        0.0
+      ],
+      [
+        76,
+        1.0
+      ],
+      [
+        77,
+        0.956656346749226
+      ],
+      [
+        78,
+        1.0
+      ],
+      [
+        79,
+        1.0
+      ],
+      [
+        80,
+        0.0
+      ],
+      [
+        81,
+        0.0
+      ],
+      [
+        82,
+        1.0
+      ],
+      [
+        83,
+        0.0
+      ],
+      [
+        84,
+        1.0
+      ],
+      [
+        85,
+        1.0
+      ],
+      [
+        86,
+        0.9984520123839009
+      ],
+      [
+        87,
+        0.0
+      ],
+      [
+        88,
+        1.0
+      ],
+      [
+        89,
+        0.5
+      ],
+      [
+        90,
+        1.0
+      ],
+      [
+        91,
+        0.5
+      ],
+      [
+        92,
+        0.9999404620147654
+      ],
+      [
+        93,
+        0.5
+      ],
+      [
+        94,
+        0.0
+      ],
+      [
+        95,
+        1.0
+      ],
+      [
+        96,
+        0.9945820433436533
+      ],
+      [
+        97,
+        1.0
+      ],
+      [
+        98,
+        0.9945820433436533
+      ],
+      [
+        99,
+        1.0
+      ],
+      [
+        100,
+        0.0
+      ],
+      [
+        101,
+        0.0
+      ],
+      [
+        102,
+        1.0
+      ],
+      [
+        103,
+        1.0
+      ],
+      [
+        104,
+        0.763157894736842
+      ],
+      [
+        105,
+        0.763157894736842
+      ],
+      [
+        106,
+        0.0
+      ],
+      [
+        107,
+        1.0
+      ],
+      [
+        108,
+        0.0
+      ],
+      [
+        109,
+        0.0
+      ],
+      [
+        110,
+        0.0
+      ],
+      [
+        111,
+        0.9999404620147654
+      ],
+      [
+        112,
+        1.0
+      ],
+      [
+        113,
+        0.0
+      ],
+      [
+        114,
+        1.0
+      ],
+      [
+        115,
+        0.9945820433436533
+      ],
+      [
+        116,
+        1.0
+      ],
+      [
+        117,
+        0.9996427720885925
+      ],
+      [
+        118,
+        0.0
+      ],
+      [
+        119,
+        0.0
+      ],
+      [
+        120,
+        0.0
+      ],
+      [
+        121,
+        1.0
+      ],
+      [
+        122,
+        1.0
+      ],
+      [
+        123,
+        1.0
+      ],
+      [
+        124,
+        1.0
+      ],
+      [
+        125,
+        0.0
+      ],
+      [
+        126,
+        0.0
+      ],
+      [
+        127,
+        0.956656346749226
+      ],
+      [
+        128,
+        0.5
+      ],
+      [
+        129,
+        0.0
+      ],
+      [
+        130,
+        0.0
+      ],
+      [
+        131,
+        0.0
+      ],
+      [
+        132,
+        0.0
+      ],
+      [
+        133,
+        0.0
+      ],
+      [
+        134,
+        0.0
+      ],
+      [
+        135,
+        0.0
+      ],
+      [
+        136,
+        1.0
+      ],
+      [
+        137,
+        0.0
+      ],
+      [
+        138,
+        0.8947368421052632
+      ],
+      [
+        139,
+        0.0
+      ],
+      [
+        140,
+        0.0
+      ],
+      [
+        141,
+        0.0
+      ],
+      [
+        142,
+        1.0
+      ],
+      [
+        143,
+        0.763157894736842
+      ],
+      [
+        144,
+        0.763157894736842
+      ],
+      [
+        145,
+        0.0
+      ],
+      [
+        146,
+        0.9945820433436533
+      ],
+      [
+        147,
+        1.0
+      ],
+      [
+        148,
+        1.0
+      ],
+      [
+        149,
+        0.5
+      ],
+      [
+        150,
+        0.0
+      ],
+      [
+        151,
+        1.0
+      ],
+      [
+        152,
+        1.0
+      ],
+      [
+        153,
+        0.9999945874558878
+      ],
+      [
+        154,
+        1.0
+      ],
+      [
+        155,
+        0.9945820433436533
+      ],
+      [
+        156,
+        1.0
+      ],
+      [
+        157,
+        1.0
+      ],
+      [
+        158,
+        1.0
+      ],
+      [
+        159,
+        1.0
+      ],
+      [
+        160,
+        0.0
+      ],
+      [
+        161,
+        0.0
+      ],
+      [
+        162,
+        0.0
+      ],
+      [
+        163,
+        0.0
+      ]
+    ]
+  },
+  "config": {
+    "prefix": "",
+    "do_sample": true,
+    "temperature": 0.2,
+    "top_k": 0,
+    "top_p": 0.95,
+    "n_samples": 20,
+    "eos": "<|endoftext|>",
+    "seed": 0,
+    "model": "CodeQwen1.5-7B",
+    "modeltype": "causal",
+    "peft_model": null,
+    "revision": null,
+    "use_auth_token": false,
+    "trust_remote_code": false,
+    "tasks": "humaneval",
+    "instruction_tokens": null,
+    "batch_size": 1,
+    "max_length_generation": 512,
+    "precision": "fp32",
+    "load_in_8bit": false,
+    "load_in_4bit": false,
+    "left_padding": false,
+    "limit": null,
+    "limit_start": 0,
+    "save_every_k_tasks": -1,
+    "postprocess": true,
+    "allow_code_execution": true,
+    "generation_only": false,
+    "load_generations_path": "generations_humaneval_CodeQwen1.5-7B_humaneval.json",
+    "load_data_path": null,
+    "metric_output_path": "codeqwen1.5_7b-humaneval.json",
+    "save_generations": false,
+    "load_generations_intermediate_paths": null,
+    "save_generations_path": "generations.json",
+    "save_references": false,
+    "save_references_path": "references.json",
+    "prompt": "prompt",
+    "max_memory_per_gpu": null,
+    "check_references": false
+  }
+}

evaluate_result/new-model.json

@@ -0,0 +1,1361 @@
+{
+    "humaneval": {
+        "pass@1": [
+            [
+                0,
+                1.0
+            ],
+            [
+                1,
+                0.0
+            ],
+            [
+                2,
+                0.95
+            ],
+            [
+                3,
+                1.0
+            ],
+            [
+                4,
+                1.0
+            ],
+            [
+                5,
+                0.8999999999999999
+            ],
+            [
+                6,
+                0.20000000000000007
+            ],
+            [
+                7,
+                1.0
+            ],
+            [
+                8,
+                1.0
+            ],
+            [
+                9,
+                0.4999999999999999
+            ],
+            [
+                10,
+                0.050000000000000044
+            ],
+            [
+                11,
+                1.0
+            ],
+            [
+                12,
+                0.8999999999999999
+            ],
+            [
+                13,
+                1.0
+            ],
+            [
+                14,
+                0.7999999999999999
+            ],
+            [
+                15,
+                1.0
+            ],
+            [
+                16,
+                1.0
+            ],
+            [
+                17,
+                0.7
+            ],
+            [
+                18,
+                0.7
+            ],
+            [
+                19,
+                0.3500000000000001
+            ],
+            [
+                20,
+                0.4
+            ],
+            [
+                21,
+                0.85
+            ],
+            [
+                22,
+                1.0
+            ],
+            [
+                23,
+                1.0
+            ],
+            [
+                24,
+                0.8999999999999999
+            ],
+            [
+                25,
+                0.85
+            ],
+            [
+                26,
+                0.0
+            ],
+            [
+                27,
+                1.0
+            ],
+            [
+                28,
+                1.0
+            ],
+            [
+                29,
+                1.0
+            ],
+            [
+                30,
+                1.0
+            ],
+            [
+                31,
+                1.0
+            ],
+            [
+                32,
+                0.0
+            ],
+            [
+                33,
+                0.050000000000000044
+            ],
+            [
+                34,
+                1.0
+            ],
+            [
+                35,
+                1.0
+            ],
+            [
+                36,
+                0.5499999999999998
+            ],
+            [
+                37,
+                0.25
+            ],
+            [
+                38,
+                1.0
+            ],
+            [
+                39,
+                0.0
+            ],
+            [
+                40,
+                1.0
+            ],
+            [
+                41,
+                0.050000000000000044
+            ],
+            [
+                42,
+                1.0
+            ],
+            [
+                43,
+                1.0
+            ],
+            [
+                44,
+                1.0
+            ],
+            [
+                45,
+                1.0
+            ],
+            [
+                46,
+                0.4999999999999999
+            ],
+            [
+                47,
+                1.0
+            ],
+            [
+                48,
+                1.0
+            ],
+            [
+                49,
+                0.6000000000000001
+            ],
+            [
+                50,
+                1.0
+            ],
+            [
+                51,
+                0.8999999999999999
+            ],
+            [
+                52,
+                1.0
+            ],
+            [
+                53,
+                1.0
+            ],
+            [
+                54,
+                0.0
+            ],
+            [
+                55,
+                0.85
+            ],
+            [
+                56,
+                0.95
+            ],
+            [
+                57,
+                0.4
+            ],
+            [
+                58,
+                0.7999999999999999
+            ],
+            [
+                59,
+                0.44999999999999984
+            ],
+            [
+                60,
+                1.0
+            ],
+            [
+                61,
+                0.95
+            ],
+            [
+                62,
+                0.7
+            ],
+            [
+                63,
+                0.4
+            ],
+            [
+                64,
+                0.0
+            ],
+            [
+                65,
+                0.4
+            ],
+            [
+                66,
+                1.0
+            ],
+            [
+                67,
+                0.25
+            ],
+            [
+                68,
+                0.7999999999999999
+            ],
+            [
+                69,
+                0.7
+            ],
+            [
+                70,
+                0.25
+            ],
+            [
+                71,
+                0.85
+            ],
+            [
+                72,
+                0.5499999999999998
+            ],
+            [
+                73,
+                0.09999999999999998
+            ],
+            [
+                74,
+                1.0
+            ],
+            [
+                75,
+                0.0
+            ],
+            [
+                76,
+                0.15000000000000002
+            ],
+            [
+                77,
+                0.0
+            ],
+            [
+                78,
+                0.4999999999999999
+            ],
+            [
+                79,
+                0.050000000000000044
+            ],
+            [
+                80,
+                1.0
+            ],
+            [
+                81,
+                0.0
+            ],
+            [
+                82,
+                0.5499999999999998
+            ],
+            [
+                83,
+                0.0
+            ],
+            [
+                84,
+                0.050000000000000044
+            ],
+            [
+                85,
+                1.0
+            ],
+            [
+                86,
+                0.7999999999999999
+            ],
+            [
+                87,
+                0.5499999999999998
+            ],
+            [
+                88,
+                0.20000000000000007
+            ],
+            [
+                89,
+                0.8999999999999999
+            ],
+            [
+                90,
+                0.09999999999999998
+            ],
+            [
+                91,
+                0.15000000000000002
+            ],
+            [
+                92,
+                0.7999999999999999
+            ],
+            [
+                93,
+                0.050000000000000044
+            ],
+            [
+                94,
+                0.30000000000000004
+            ],
+            [
+                95,
+                0.050000000000000044
+            ],
+            [
+                96,
+                0.8999999999999999
+            ],
+            [
+                97,
+                1.0
+            ],
+            [
+                98,
+                0.8999999999999999
+            ],
+            [
+                99,
+                0.3500000000000001
+            ],
+            [
+                100,
+                0.15000000000000002
+            ],
+            [
+                101,
+                0.09999999999999998
+            ],
+            [
+                102,
+                0.7
+            ],
+            [
+                103,
+                0.5499999999999998
+            ],
+            [
+                104,
+                0.3500000000000001
+            ],
+            [
+                105,
+                0.20000000000000007
+            ],
+            [
+                106,
+                0.0
+            ],
+            [
+                107,
+                0.8999999999999999
+            ],
+            [
+                108,
+                0.0
+            ],
+            [
+                109,
+                0.15000000000000002
+            ],
+            [
+                110,
+                0.15000000000000002
+            ],
+            [
+                111,
+                0.6000000000000001
+            ],
+            [
+                112,
+                0.7999999999999999
+            ],
+            [
+                113,
+                0.7
+            ],
+            [
+                114,
+                1.0
+            ],
+            [
+                115,
+                0.050000000000000044
+            ],
+            [
+                116,
+                0.15000000000000002
+            ],
+            [
+                117,
+                0.65
+            ],
+            [
+                118,
+                0.0
+            ],
+            [
+                119,
+                0.09999999999999998
+            ],
+            [
+                120,
+                0.09999999999999998
+            ],
+            [
+                121,
+                0.5499999999999998
+            ],
+            [
+                122,
+                0.85
+            ],
+            [
+                123,
+                0.4999999999999999
+            ],
+            [
+                124,
+                0.050000000000000044
+            ],
+            [
+                125,
+                0.050000000000000044
+            ],
+            [
+                126,
+                0.050000000000000044
+            ],
+            [
+                127,
+                0.050000000000000044
+            ],
+            [
+                128,
+                0.4999999999999999
+            ],
+            [
+                129,
+                0.0
+            ],
+            [
+                130,
+                0.0
+            ],
+            [
+                131,
+                0.0
+            ],
+            [
+                132,
+                0.0
+            ],
+            [
+                133,
+                0.0
+            ],
+            [
+                134,
+                0.0
+            ],
+            [
+                135,
+                0.0
+            ],
+            [
+                136,
+                0.8999999999999999
+            ],
+            [
+                137,
+                0.0
+            ],
+            [
+                138,
+                0.0
+            ],
+            [
+                139,
+                0.09999999999999998
+            ],
+            [
+                140,
+                0.050000000000000044
+            ],
+            [
+                141,
+                0.65
+            ],
+            [
+                142,
+                0.25
+            ],
+            [
+                143,
+                0.15000000000000002
+            ],
+            [
+                144,
+                0.44999999999999984
+            ],
+            [
+                145,
+                0.0
+            ],
+            [
+                146,
+                0.20000000000000007
+            ],
+            [
+                147,
+                0.8999999999999999
+            ],
+            [
+                148,
+                0.7
+            ],
+            [
+                149,
+                0.5499999999999998
+            ],
+            [
+                150,
+                0.3500000000000001
+            ],
+            [
+                151,
+                1.0
+            ],
+            [
+                152,
+                1.0
+            ],
+            [
+                153,
+                0.75
+            ],
+            [
+                154,
+                0.5499999999999998
+            ],
+            [
+                155,
+                0.25
+            ],
+            [
+                156,
+                0.44999999999999984
+            ],
+            [
+                157,
+                1.0
+            ],
+            [
+                158,
+                0.8999999999999999
+            ],
+            [
+                159,
+                0.8999999999999999
+            ],
+            [
+                160,
+                0.15000000000000002
+            ],
+            [
+                161,
+                0.25
+            ],
+            [
+                162,
+                1.0
+            ],
+            [
+                163,
+                0.0
+            ]
+        ],
+        "pass@10": [
+            [
+                0,
+                1.0
+            ],
+            [
+                1,
+                0.0
+            ],
+            [
+                2,
+                1.0
+            ],
+            [
+                3,
+                1.0
+            ],
+            [
+                4,
+                1.0
+            ],
+            [
+                5,
+                1.0
+            ],
+            [
+                6,
+                0.956656346749226
+            ],
+            [
+                7,
+                1.0
+            ],
+            [
+                8,
+                1.0
+            ],
+            [
+                9,
+                0.9999945874558878
+            ],
+            [
+                10,
+                0.5
+            ],
+            [
+                11,
+                1.0
+            ],
+            [
+                12,
+                1.0
+            ],
+            [
+                13,
+                1.0
+            ],
+            [
+                14,
+                1.0
+            ],
+            [
+                15,
+                1.0
+            ],
+            [
+                16,
+                1.0
+            ],
+            [
+                17,
+                1.0
+            ],
+            [
+                18,
+                1.0
+            ],
+            [
+                19,
+                0.9984520123839009
+            ],
+            [
+                20,
+                0.9996427720885925
+            ],
+            [
+                21,
+                1.0
+            ],
+            [
+                22,
+                1.0
+            ],
+            [
+                23,
+                1.0
+            ],
+            [
+                24,
+                1.0
+            ],
+            [
+                25,
+                1.0
+            ],
+            [
+                26,
+                0.0
+            ],
+            [
+                27,
+                1.0
+            ],
+            [
+                28,
+                1.0
+            ],
+            [
+                29,
+                1.0
+            ],
+            [
+                30,
+                1.0
+            ],
+            [
+                31,
+                1.0
+            ],
+            [
+                32,
+                0.0
+            ],
+            [
+                33,
+                0.5
+            ],
+            [
+                34,
+                1.0
+            ],
+            [
+                35,
+                1.0
+            ],
+            [
+                36,
+                1.0
+            ],
+            [
+                37,
+                0.9837461300309598
+            ],
+            [
+                38,
+                1.0
+            ],
+            [
+                39,
+                0.0
+            ],
+            [
+                40,
+                1.0
+            ],
+            [
+                41,
+                0.5
+            ],
+            [
+                42,
+                1.0
+            ],
+            [
+                43,
+                1.0
+            ],
+            [
+                44,
+                1.0
+            ],
+            [
+                45,
+                1.0
+            ],
+            [
+                46,
+                0.9999945874558878
+            ],
+            [
+                47,
+                1.0
+            ],
+            [
+                48,
+                1.0
+            ],
+            [
+                49,
+                1.0
+            ],
+            [
+                50,
+                1.0
+            ],
+            [
+                51,
+                1.0
+            ],
+            [
+                52,
+                1.0
+            ],
+            [
+                53,
+                1.0
+            ],
+            [
+                54,
+                0.0
+            ],
+            [
+                55,
+                1.0
+            ],
+            [
+                56,
+                1.0
+            ],
+            [
+                57,
+                0.9996427720885925
+            ],
+            [
+                58,
+                1.0
+            ],
+            [
+                59,
+                0.9999404620147654
+            ],
+            [
+                60,
+                1.0
+            ],
+            [
+                61,
+                1.0
+            ],
+            [
+                62,
+                1.0
+            ],
+            [
+                63,
+                0.9996427720885925
+            ],
+            [
+                64,
+                0.0
+            ],
+            [
+                65,
+                0.9996427720885925
+            ],
+            [
+                66,
+                1.0
+            ],
+            [
+                67,
+                0.9837461300309598
+            ],
+            [
+                68,
+                1.0
+            ],
+            [
+                69,
+                1.0
+            ],
+            [
+                70,
+                0.9837461300309598
+            ],
+            [
+                71,
+                1.0
+            ],
+            [
+                72,
+                1.0
+            ],
+            [
+                73,
+                0.763157894736842
+            ],
+            [
+                74,
+                1.0
+            ],
+            [
+                75,
+                0.0
+            ],
+            [
+                76,
+                0.8947368421052632
+            ],
+            [
+                77,
+                0.0
+            ],
+            [
+                78,
+                0.9999945874558878
+            ],
+            [
+                79,
+                0.5
+            ],
+            [
+                80,
+                1.0
+            ],
+            [
+                81,
+                0.0
+            ],
+            [
+                82,
+                1.0
+            ],
+            [
+                83,
+                0.0
+            ],
+            [
+                84,
+                0.5
+            ],
+            [
+                85,
+                1.0
+            ],
+            [
+                86,
+                1.0
+            ],
+            [
+                87,
+                1.0
+            ],
+            [
+                88,
+                0.956656346749226
+            ],
+            [
+                89,
+                1.0
+            ],
+            [
+                90,
+                0.763157894736842
+            ],
+            [
+                91,
+                0.8947368421052632
+            ],
+            [
+                92,
+                1.0
+            ],
+            [
+                93,
+                0.5
+            ],
+            [
+                94,
+                0.9945820433436533
+            ],
+            [
+                95,
+                0.5
+            ],
+            [
+                96,
+                1.0
+            ],
+            [
+                97,
+                1.0
+            ],
+            [
+                98,
+                1.0
+            ],
+            [
+                99,
+                0.9984520123839009
+            ],
+            [
+                100,
+                0.8947368421052632
+            ],
+            [
+                101,
+                0.763157894736842
+            ],
+            [
+                102,
+                1.0
+            ],
+            [
+                103,
+                1.0
+            ],
+            [
+                104,
+                0.9984520123839009
+            ],
+            [
+                105,
+                0.956656346749226
+            ],
+            [
+                106,
+                0.0
+            ],
+            [
+                107,
+                1.0
+            ],
+            [
+                108,
+                0.0
+            ],
+            [
+                109,
+                0.8947368421052632
+            ],
+            [
+                110,
+                0.8947368421052632
+            ],
+            [
+                111,
+                1.0
+            ],
+            [
+                112,
+                1.0
+            ],
+            [
+                113,
+                1.0
+            ],
+            [
+                114,
+                1.0
+            ],
+            [
+                115,
+                0.5
+            ],
+            [
+                116,
+                0.8947368421052632
+            ],
+            [
+                117,
+                1.0
+            ],
+            [
+                118,
+                0.0
+            ],
+            [
+                119,
+                0.763157894736842
+            ],
+            [
+                120,
+                0.763157894736842
+            ],
+            [
+                121,
+                1.0
+            ],
+            [
+                122,
+                1.0
+            ],
+            [
+                123,
+                0.9999945874558878
+            ],
+            [
+                124,
+                0.5
+            ],
+            [
+                125,
+                0.5
+            ],
+            [
+                126,
+                0.5
+            ],
+            [
+                127,
+                0.5
+            ],
+            [
+                128,
+                0.9999945874558878
+            ],
+            [
+                129,
+                0.0
+            ],
+            [
+                130,
+                0.0
+            ],
+            [
+                131,
+                0.0
+            ],
+            [
+                132,
+                0.0
+            ],
+            [
+                133,
+                0.0
+            ],
+            [
+                134,
+                0.0
+            ],
+            [
+                135,
+                0.0
+            ],
+            [
+                136,
+                1.0
+            ],
+            [
+                137,
+                0.0
+            ],
+            [
+                138,
+                0.0
+            ],
+            [
+                139,
+                0.763157894736842
+            ],
+            [
+                140,
+                0.5
+            ],
+            [
+                141,
+                1.0
+            ],
+            [
+                142,
+                0.9837461300309598
+            ],
+            [
+                143,
+                0.8947368421052632
+            ],
+            [
+                144,
+                0.9999404620147654
+            ],
+            [
+                145,
+                0.0
+            ],
+            [
+                146,
+                0.956656346749226
+            ],
+            [
+                147,
+                1.0
+            ],
+            [
+                148,
+                1.0
+            ],
+            [
+                149,
+                1.0
+            ],
+            [
+                150,
+                0.9984520123839009
+            ],
+            [
+                151,
+                1.0
+            ],
+            [
+                152,
+                1.0
+            ],
+            [
+                153,
+                1.0
+            ],
+            [
+                154,
+                1.0
+            ],
+            [
+                155,
+                0.9837461300309598
+            ],
+            [
+                156,
+                0.9999404620147654
+            ],
+            [
+                157,
+                1.0
+            ],
+            [
+                158,
+                1.0
+            ],
+            [
+                159,
+                1.0
+            ],
+            [
+                160,
+                0.8947368421052632
+            ],
+            [
+                161,
+                0.9837461300309598
+            ],
+            [
+                162,
+                1.0
+            ],
+            [
+                163,
+                0.0
+            ]
+        ]
+    },
+    "config": {
+        "prefix": "",
+        "do_sample": true,
+        "temperature": 0.2,
+        "top_k": 0,
+        "top_p": 0.95,
+        "n_samples": 20,
+        "eos": "<|endoftext|>",
+        "seed": 0,
+        "model": "1_java_codegemma-2b_multiple-java",
+        "modeltype": "causal",
+        "peft_model": null,
+        "revision": null,
+        "use_auth_token": false,
+        "trust_remote_code": false,
+        "tasks": "humaneval",
+        "instruction_tokens": null,
+        "batch_size": 1,
+        "max_length_generation": 512,
+        "precision": "fp32",
+        "load_in_8bit": false,
+        "load_in_4bit": false,
+        "left_padding": false,
+        "limit": null,
+        "limit_start": 0,
+        "save_every_k_tasks": -1,
+        "postprocess": true,
+        "allow_code_execution": true,
+        "generation_only": false,
+        "load_generations_path": "1_codegemma-7b-it_generations_humaneval_codegemma-7b-it_humaneval.json",
+        "load_data_path": null,
+        "metric_output_path": "hhhhh.json",
+        "save_generations": false,
+        "load_generations_intermediate_paths": null,
+        "save_generations_path": "generations.json",
+        "save_references": false,
+        "save_references_path": "references.json",
+        "prompt": "prompt",
+        "max_memory_per_gpu": null,
+        "check_references": false
+    }
+}