all working blocks

app.py

@@ -2,8 +2,9 @@ import pandas as pd
 import gradio as gr
 import os
 from gradio_rangeslider import RangeSlider
+import math
 
-from utils.filter_utils import filter
+from utils.filter_utils import filter, filter_cols
 
 # MAPS = filter_utils.LANG_MAPPING
 
@@ -12,45 +13,56 @@ text_leaderboard = pd.read_csv(os.path.join('src', 'main_df.csv'))
 text = "## The range is: {min} to {max}"
 
 # Short leaderboard containing fixed columns
-short_leaderboard = text_leaderboard[[
-    'model_name', 
-    'input_price', 
-    'output_price',
-    'release_date', 
-    'context_size', 
-    'average_clemscore',
-    'average_latency',
-    'parameter_size',
-    ]]
-
-## Get Languages
+short_leaderboard = filter_cols(text_leaderboard)
+
+
+## Extract data
 langs = []
+licenses = []
+ip_prices = []
+op_prices = []
+latencies = []
+parameters = []
+contexts = []
+dates = []
+
 for i in range(len(text_leaderboard)):
-    lang_splits = text_leaderboard.iloc[i]['languages'].split(',')
+    lang_splits = text_leaderboard.iloc[i]['Languages'].split(',')
     lang_splits = [s.strip() for s in lang_splits]
     langs += lang_splits
+    license_name = text_leaderboard.iloc[i]['License Name']
+
+    licenses.append(license_name)
+    ip_prices.append(text_leaderboard.iloc[i]['Input $/1M'])
+    op_prices.append(text_leaderboard.iloc[i]['Output $/1M'])
+    latencies.append(text_leaderboard.iloc[i]['Average Latency (s)'])
+    parameters.append(text_leaderboard.iloc[i]['Parameter Size (B)'])
+    contexts.append(text_leaderboard.iloc[i]['Context Size'])
+    dates.append(text_leaderboard.iloc[i]['Release Date'])
+
+
 langs = list(set(langs))
 langs.sort()
 
-## Get input prices
-ip_prices = []
-op_prices = []
-for i in range(len(text_leaderboard)):
-    ip_prices.append(text_leaderboard.iloc[i]['input_price'])
-    op_prices.append(text_leaderboard.iloc[i]['output_price'])
+licenses = list(set(licenses))
+licenses.sort()
 
 max_input_price = max(ip_prices)
 max_output_price = max(op_prices)
+max_latency = max(latencies)
 
+max_parameter = max(parameters)
+max_parameter = math.ceil(math.log2(max_parameter))
 
-llm_calc_app = gr.Blocks()
-with llm_calc_app:
+max_context = max(contexts)/1024
+max_context = math.ceil(math.log2(max_context))
 
-    with gr.Row():
-        """
-        Main Filters Row
-        """
-        
+min_date = min(dates)
+max_date = max(dates)
+
+
+llm_calc_app = gr.Blocks()
+with llm_calc_app:        
     ### Language filter
     with gr.Row():
         lang_dropdown = gr.Dropdown(
@@ -60,13 +72,46 @@ with llm_calc_app:
             label="Select Languages 🕹️"
         )
 
-        clemscore_slider = RangeSlider(
-            minimum=0, 
-            maximum=100, 
-            value=(0, 100), 
-            label="Select Clemscore range"
-        )
+    with gr.Row():
+        with gr.Column(scale=3):
+            parameter_slider = RangeSlider(
+                minimum=0, 
+                maximum=max_parameter, 
+                label="Select Parameters. Range -> 2^x - 2^y"
+            )
+        with gr.Column(scale=1):
+            range_ = gr.Markdown(value=text.format(min=0, max=math.pow(2, max_parameter)))
+            parameter_slider.change(lambda s: text.format(min=int(pow(2,s[0])), max=int(pow(2,s[1]))), parameter_slider, range_,
+                                    show_progress="hide", trigger_mode="always_last")
+            
+    with gr.Row():
+        with gr.Column(scale=3):
+            context_slider = RangeSlider(
+                minimum=0, 
+                maximum=max_context, 
+                label="Select Context length range. Range -> 2^x k - 2^y k"
+            )
+        with gr.Column(scale=1):
+            context_range_ = gr.Markdown(value=text.format(min=0, max=math.pow(2, max_context)))
+            context_slider.change(lambda s: text.format(min=int(pow(2,s[0])), max=int(pow(2,s[1]))), context_slider, context_range_,
+                                    show_progress="hide", trigger_mode="always_last")
+            
+    with gr.Row():
+        with gr.Column():
+            start_date = gr.DateTime(
+                value=min_date,
+                type="string",
+                label="Select start date"
+            )
+
+        with gr.Column():
+            end_date = gr.DateTime(
+                value=max_date,
+                type="string",
+                label="Select end date"
+            )
 
+    with gr.Row():
         input_pricing_slider = RangeSlider(
             minimum=0, 
             maximum=max_input_price, 
@@ -81,6 +126,28 @@ with llm_calc_app:
             label="Select Price range /1M output tokens"
         )
        
+    with gr.Row():
+        with gr.Column():
+            multimodal_checkbox = gr.CheckboxGroup(
+                choices=['Image', 'Multi-Image', 'Audio', 'Video'],
+                value=[],
+                label="Select additional Modalities",
+            )
+
+        with gr.Column():
+            open_weight_checkbox = gr.CheckboxGroup(
+                choices=['Open', 'Commercial'],
+                value=['Open', 'Commercial'],
+                label="Filter Open-weight model or commercial model",
+            )    
+
+        with gr.Column():
+            license_checkbox = gr.CheckboxGroup(
+                choices=licenses,
+                value=licenses,
+                label="Filter based on the type of License",
+            )             
+
 
 
     with gr.Row():
@@ -93,7 +160,8 @@ with llm_calc_app:
                                 elem_id="text-leaderboard-table",
                                 interactive=False,
                                 visible=True,
-                                height=800
+                                height=800,
+                                datatype=['html', 'number', 'number', 'date', 'number', 'number', 'number', 'number', 'html']
                             )
 
         
@@ -106,32 +174,90 @@ with llm_calc_app:
         
         lang_dropdown.change(
             filter,
-            [dummy_leaderboard_table, lang_dropdown, clemscore_slider,
-             input_pricing_slider, output_pricing_slider],
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
             [leaderboard_table],   
             queue=True
         )
 
-        clemscore_slider.change(
+        parameter_slider.change(
             filter,
-            [dummy_leaderboard_table, lang_dropdown, clemscore_slider,
-             input_pricing_slider, output_pricing_slider],
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
             [leaderboard_table],
             queue=True
         )
 
         input_pricing_slider.change(
             filter,
-            [dummy_leaderboard_table, lang_dropdown, clemscore_slider,
-             input_pricing_slider, output_pricing_slider],
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
             [leaderboard_table],
             queue=True
         )
 
         output_pricing_slider.change(
             filter,
-            [dummy_leaderboard_table, lang_dropdown, clemscore_slider,
-             input_pricing_slider, output_pricing_slider],
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        multimodal_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        open_weight_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        context_slider.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        start_date.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        end_date.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
+            [leaderboard_table],
+            queue=True
+        )
+
+        license_checkbox.change(
+            filter,
+            [dummy_leaderboard_table, lang_dropdown, parameter_slider,
+             input_pricing_slider, output_pricing_slider, multimodal_checkbox,
+             context_slider, open_weight_checkbox, start_date, end_date, license_checkbox],
             [leaderboard_table],
             queue=True
         )

src/main_df.csv

@@ -1,24 +1,24 @@
-model_name,input_price,output_price,multimodality_image,multimodality_multiple_image,multimodality_audio,multimodality_video,source,license_name,license_url,languages,release_date,open_weight,context_size,average_clemscore,average_latency,parameter_size,estimated
-Meta-Llama-3-70B-Instruct-hf,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,English,2024-04-18,True,8192,11.703333333333333,1.1160853862207483,70.0,False
-Meta-Llama-3-8B-Instruct-hf,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct,Meta Llama 3 License,https://www.llama.com/llama3/license/,English,2024-04-18,True,8192,6.663333333333333,0.7054825144189354,8.0,False
-Meta-Llama-3.1-405B-Instruct-Turbo,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,17.37,0.2628701315515277,405.0,False
-Meta-Llama-3.1-70B-Instruct,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,12.943333333333333,0.27016850919817575,70.0,False
-Meta-Llama-3.1-8B-Instruct,0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct,Llama 3.1 Community License,https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE,"English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,6.12,0.06876858280202812,8.0,False
-InternVL2-40B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-40B,MIT,https://choosealicense.com/licenses/mit/,"Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,21.810000000000002,2.609271782765464,40.0,False
-InternVL2-8B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-8B,MIT,https://choosealicense.com/licenses/mit/,"Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,19.74,0.8367998047485775,8.0,False
-InternVL2-Llama3-76B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B,MIT,https://choosealicense.com/licenses/mit/,"Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,25.709999999999997,4.591395944741546,76.0,False
-InternVL2-26B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,23.24,1.7593004986949285,26.0,False
-InternVL2-26B,0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,https://choosealicense.com/licenses/mit/,"Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,23.24,1.7593004986949285,26.0,False
-Mistral-Large-Instruct-2407,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-Large-Instruct-2407,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"English, French, Spanish, German, Italian, Russian, Chinese, Japanese, Korean",2024-06-12,True,8192,15.13,0.41482225628780656,70.0,False
-Mixtral-8x22B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"English, French, Spanish, German, Italian, Russian",2024-04-17,True,8192,4.2299999999999995,0.3586451521191292,141.0,False
-Mistral-7B-Instruct-v0.2,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"English, French, Spanish, German, Italian, Russian, Chinese",2024-01-15,True,8192,3.25,0.25450503989030154,7.0,False
-Mistral-7B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"English, French, Spanish, German, Italian, Russian, Chinese",2023-12-11,True,8192,2.67,0.09428825169239076,7.0,False
-Mixtral-8x7B-Instruct-v0.1,0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,"English, French, Spanish, German, Italian, Russian",2023-12-11,True,8192,2.723333333333333,0.31309892202121054,46.7,False
-openchat-3.5-0106,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-0106,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,English,2024-01-06,True,8192,5.7,0.09736504835188847,7.0,False
-openchat-3.5-1210,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-1210,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,English,2023-12-10,True,8192,6.073333333333333,0.09349942563676568,7.0,False
-openchat_3.5,0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat_3.5,Apache 2.0,https://www.apache.org/licenses/LICENSE-2.0,English,2023-10-30,True,8192,7.88,0.10576256228206875,7.0,False
-gpt-4o-mini-2024-07-18,0.15,0.6,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-07-18,False,131072,52.32333333333333,1.619222935116773,8.0,True
-gpt-4o-2024-08-06,2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-08-06,False,131072,69.57000000000001,1.5771123003908176,200.0,True
-gpt-4o-2024-05-13,2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-05-13,False,131072,66.87333333333333,3.704921340164487,200.0,True
-gpt-4-1106-vision-preview,10.0,30.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,https://openai.com/policies/terms-of-use,"English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2023-11-06,False,131072,47.23,2.217200177676117,1760.0,True
-gemini-1.5-flash-latest,0.075,0.3,True,True,True,True,https://cloud.google.com/vertex-ai/generative-ai/pricing,Commercial License,,"Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovenian, Spanish, Swahili, Swedish, Thai, Turkish, Ukrainian, Vietnamese, Chinese, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Hebrew, Hindi, Hungarian, Indonesian, Italian, Japanese, Korean, Latvian, Arabic, Bengali, Bulgarian",2024-05-24,False,131072,42.53666666666667,26.268280234692302,1760.0,True
+Model Name,Input $/1M,Output $/1M,Multimodality Image,Multimodality Multiple Image,Multimodality Audio,Multimodality Video,Source,License Name,License,Languages,Release Date,Open Weight,Context Size,Average Clemscore,Average Latency (s),Parameter Size (B),Estimated
+"<a href=""https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct"" style=""color: blue;"">Meta-Llama-3-70B-Instruct-hf</a>",0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct,Meta Llama 3 License,"<a href=""https://www.llama.com/llama3/license/"" style=""color: blue;"">Meta Llama 3 License</a>",English,2024-04-18,True,8192,11.703,1.116,70.0,False
+"<a href=""https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct"" style=""color: blue;"">Meta-Llama-3-8B-Instruct-hf</a>",0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct,Meta Llama 3 License,"<a href=""https://www.llama.com/llama3/license/"" style=""color: blue;"">Meta Llama 3 License</a>",English,2024-04-18,True,8192,6.663,0.705,8.0,False
+"<a href=""https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct"" style=""color: blue;"">Meta-Llama-3.1-405B-Instruct-Turbo</a>",0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct,Llama 3.1 Community License,"<a href=""https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"" style=""color: blue;"">Llama 3.1 Community License</a>","English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,17.37,0.263,405.0,False
+"<a href=""https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct"" style=""color: blue;"">Meta-Llama-3.1-70B-Instruct</a>",0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct,Llama 3.1 Community License,"<a href=""https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"" style=""color: blue;"">Llama 3.1 Community License</a>","English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,12.943,0.27,70.0,False
+"<a href=""https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct"" style=""color: blue;"">Meta-Llama-3.1-8B-Instruct</a>",0.0,0.0,False,False,False,False,https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct,Llama 3.1 Community License,"<a href=""https://github.com/meta-llama/llama-models/blob/main/models/llama3_1/LICENSE"" style=""color: blue;"">Llama 3.1 Community License</a>","English, German, French, Italian, Hindi, Portuguese, Spanish, Thai",2024-07-23,True,131072,6.12,0.069,8.0,False
+"<a href=""https://huggingface.co/OpenGVLab/InternVL2-40B"" style=""color: blue;"">InternVL2-40B</a>",0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-40B,MIT,"<a href=""https://choosealicense.com/licenses/mit/"" style=""color: blue;"">MIT</a>","Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,21.81,2.609,40.0,False
+"<a href=""https://huggingface.co/OpenGVLab/InternVL2-8B"" style=""color: blue;"">InternVL2-8B</a>",0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-8B,MIT,"<a href=""https://choosealicense.com/licenses/mit/"" style=""color: blue;"">MIT</a>","Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,19.74,0.837,8.0,False
+"<a href=""https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B"" style=""color: blue;"">InternVL2-Llama3-76B</a>",0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B,MIT,"<a href=""https://choosealicense.com/licenses/mit/"" style=""color: blue;"">MIT</a>","Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,25.71,4.591,76.0,False
+"<a href=""https://huggingface.co/OpenGVLab/InternVL2-26B"" style=""color: blue;"">InternVL2-26B</a>",0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,"<a href=""https://choosealicense.com/licenses/mit/"" style=""color: blue;"">MIT</a>","Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,23.24,1.759,26.0,False
+"<a href=""https://huggingface.co/OpenGVLab/InternVL2-26B"" style=""color: blue;"">InternVL2-26B</a>",0.0,0.0,True,True,False,False,https://huggingface.co/OpenGVLab/InternVL2-26B,MIT,"<a href=""https://choosealicense.com/licenses/mit/"" style=""color: blue;"">MIT</a>","Chinese, English, French, Spanish, Portuguese, German, Italian, Russian, Japanese, Korean, Vietnamese, Thai, Arabic",2024-07-15,True,8192,23.24,1.759,26.0,False
+"<a href=""https://huggingface.co/mistralai/Mistral-Large-Instruct-2407"" style=""color: blue;"">Mistral-Large-Instruct-2407</a>",0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-Large-Instruct-2407,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>","English, French, Spanish, German, Italian, Russian, Chinese, Japanese, Korean",2024-06-12,True,8192,15.13,0.415,70.0,False
+"<a href=""https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1"" style=""color: blue;"">Mixtral-8x22B-Instruct-v0.1</a>",0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>","English, French, Spanish, German, Italian, Russian",2024-04-17,True,8192,4.23,0.359,141.0,False
+"<a href=""https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2"" style=""color: blue;"">Mistral-7B-Instruct-v0.2</a>",0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>","English, French, Spanish, German, Italian, Russian, Chinese",2024-01-15,True,8192,3.25,0.255,7.0,False
+"<a href=""https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1"" style=""color: blue;"">Mistral-7B-Instruct-v0.1</a>",0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>","English, French, Spanish, German, Italian, Russian, Chinese",2023-12-11,True,8192,2.67,0.094,7.0,False
+"<a href=""https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1"" style=""color: blue;"">Mixtral-8x7B-Instruct-v0.1</a>",0.0,0.0,False,False,False,False,https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>","English, French, Spanish, German, Italian, Russian",2023-12-11,True,8192,2.723,0.313,46.7,False
+"<a href=""https://huggingface.co/openchat/openchat-3.5-0106"" style=""color: blue;"">openchat-3.5-0106</a>",0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-0106,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",English,2024-01-06,True,8192,5.7,0.097,7.0,False
+"<a href=""https://huggingface.co/openchat/openchat-3.5-1210"" style=""color: blue;"">openchat-3.5-1210</a>",0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat-3.5-1210,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",English,2023-12-10,True,8192,6.073,0.093,7.0,False
+"<a href=""https://huggingface.co/openchat/openchat_3.5"" style=""color: blue;"">openchat_3.5</a>",0.0,0.0,False,False,False,False,https://huggingface.co/openchat/openchat_3.5,Apache 2.0,"<a href=""https://www.apache.org/licenses/LICENSE-2.0"" style=""color: blue;"">Apache 2.0</a>",English,2023-10-30,True,8192,7.88,0.106,7.0,False
+"<a href=""https://openai.com/api/pricing/"" style=""color: blue;"">gpt-4o-mini-2024-07-18</a>",0.15,0.6,True,True,False,False,https://openai.com/api/pricing/,Commercial License,"<a href=""https://openai.com/policies/terms-of-use"" style=""color: blue;"">Commercial License</a>","English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-07-18,False,131072,52.323,1.619,8.0,True
+"<a href=""https://openai.com/api/pricing/"" style=""color: blue;"">gpt-4o-2024-08-06</a>",2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,"<a href=""https://openai.com/policies/terms-of-use"" style=""color: blue;"">Commercial License</a>","English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-08-06,False,131072,69.57,1.577,200.0,True
+"<a href=""https://openai.com/api/pricing/"" style=""color: blue;"">gpt-4o-2024-05-13</a>",2.5,10.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,"<a href=""https://openai.com/policies/terms-of-use"" style=""color: blue;"">Commercial License</a>","English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2024-05-13,False,131072,66.873,3.705,200.0,True
+"<a href=""https://openai.com/api/pricing/"" style=""color: blue;"">gpt-4-1106-vision-preview</a>",10.0,30.0,True,True,False,False,https://openai.com/api/pricing/,Commercial License,"<a href=""https://openai.com/policies/terms-of-use"" style=""color: blue;"">Commercial License</a>","English, Spanish, French, German, Chinese, Japanese, Korean, Italian, Portuguese, Dutch, Russian, Arabic, Hindi, Turkish, Vietnamese, Polish, Thai, Swedish, Danish, Norwegian, Finnish, Hungarian, Czech, Slovak, Romanian, Bulgarian, Ukrainian, Lithuanian, Latvian, Estonian, Slovenian, Malay, Indonesian, Tagalog, Swahili, Amharic",2023-11-06,False,131072,47.23,2.217,1760.0,True
+"<a href=""https://cloud.google.com/vertex-ai/generative-ai/pricing"" style=""color: blue;"">gemini-1.5-flash-latest</a>",0.075,0.3,True,True,True,True,https://cloud.google.com/vertex-ai/generative-ai/pricing,Commercial License,"<a href="""" style=""color: blue;"">Commercial License</a>","Lithuanian, Norwegian, Polish, Portuguese, Romanian, Russian, Serbian, Slovak, Slovenian, Spanish, Swahili, Swedish, Thai, Turkish, Ukrainian, Vietnamese, Chinese, Croatian, Czech, Danish, Dutch, English, Estonian, Finnish, French, German, Greek, Hebrew, Hindi, Hungarian, Indonesian, Italian, Japanese, Korean, Latvian, Arabic, Bengali, Bulgarian",2024-05-24,False,131072,42.537,26.268,1760.0,True

src/process_data.py

@@ -81,14 +81,14 @@ df['latency_v1.6.5_ascii'] = df['model_name'].map(latency_map_1_6_5_ascii).filln
 
 
 # Calculate average latency and clemscore
-df['average_clemscore'] = df[['clemscore_v1.6.5_multimodal', 'clemscore_v1.6.5_ascii', 'clemscore_v1.6']].mean(axis=1)
-df['average_latency'] = df[['latency_v1.6', 'latency_v1.6.5_multimodal', 'latency_v1.6.5_ascii']].mean(axis=1)
+df['average_clemscore'] = df[['clemscore_v1.6.5_multimodal', 'clemscore_v1.6.5_ascii', 'clemscore_v1.6']].mean(axis=1).round(3)
+df['average_latency'] = df[['latency_v1.6', 'latency_v1.6.5_multimodal', 'latency_v1.6.5_ascii']].mean(axis=1).round(3)
 
 
 # More clean up
 # Clean and convert prices to float
-df['input_price'] = df['input_price'].replace({'\$': '', '': None}, regex=True).astype(float)
-df['output_price'] = df['output_price'].replace({'\$': '', '': None}, regex=True).astype(float)
+df['input_price'] = df['input_price'].replace({'\$': '', '': None}, regex=True).astype(float).round(3)
+df['output_price'] = df['output_price'].replace({'\$': '', '': None}, regex=True).astype(float).round(3)
 
 # Clean and convert additional prices to float
 additional_price_columns = [
@@ -103,7 +103,7 @@ additional_price_columns = [
 ]
 
 for col in additional_price_columns:
-    df[col] = df[col].replace({'\$': '', '': None}, regex=True).astype(float)
+    df[col] = df[col].replace({'\$': '', '': None}, regex=True).astype(float).round(3)
 
 # Clean and convert context to integer
 df['context_size'] = df['context_size'].replace({'k': ''}, regex=True).astype(int)
@@ -180,6 +180,29 @@ df = df[[
     'estimated'
 ]]
 
-
+df = df.rename(columns={
+    'model_name': 'Model Name',
+    'input_price': 'Input $/1M',
+    'output_price': 'Output $/1M',
+    'multimodality_image': 'Multimodality Image',
+    'multimodality_multiple_image': 'Multimodality Multiple Image',
+    'multimodality_audio': 'Multimodality Audio',
+    'multimodality_video': 'Multimodality Video',
+    'source': 'Source',
+    'license_name': 'License Name',
+    'license_url': 'License',
+    'languages': 'Languages',
+    'release_date': 'Release Date', 
+    'open_weight': 'Open Weight',
+    'context_size': 'Context Size', 
+    'average_clemscore': 'Average Clemscore',
+    'average_latency': 'Average Latency (s)',
+    'parameter_size': 'Parameter Size (B)',
+    'estimated': 'Estimated'
+})
+
+df['License'] = df.apply(lambda row: f'<a href="{row["License"]}" style="color: blue;">{row["License Name"]}</a>', axis=1)
+df['Model Name'] = df.apply(lambda row: f'<a href="{row["Source"]}" style="color: blue;">{row["Model Name"]}</a>', axis=1)
+print(df)
 # Save to CSV
 df.to_csv('src/main_df.csv', index=False)

utils/filter_utils.py

@@ -1,29 +1,60 @@
 # Utility functions for filtering the dataframe
 
+import pandas as pd
+
 def filter_cols(df):
 
     df = df[[
-    'model_name', 
-    'input_price', 
-    'output_price',
-    'release_date', 
-    'context_size', 
-    'average_clemscore',
-    'average_latency',
-    'parameter_size',
+    'Model Name', 
+    'Input $/1M', 
+    'Output $/1M',
+    'Release Date', 
+    'Context Size', 
+    'Average Clemscore',
+    'Average Latency (s)',
+    'Parameter Size (B)',
+    'License'
     ]]
 
     return df
 
 
-def filter(df, language_list, clemscore, input_price, output_price):
-    df = df[df['languages'].apply(lambda x: all(lang in x for lang in language_list))]
-    df = df[(df['average_clemscore'] >= clemscore[0]) & (df['average_clemscore'] <= clemscore[1])]
-    df = df[(df['input_price'] >= input_price[0]) & (df['input_price'] <= input_price[1])]
-    df = df[(df['output_price'] >= output_price[0]) & (df['output_price'] <= output_price[1])]
+def filter(df, language_list, parameters, input_price, output_price, multimodal,
+           context, open_weight, start, end, license ):
+    
+    df = df[df['Languages'].apply(lambda x: all(lang in x for lang in language_list))]
+    df = df[(df['Parameter Size (B)'] >= pow(2, parameters[0])) & (df['Parameter Size (B)'] <= pow(2, parameters[1]))]
+    df = df[(df['Input $/1M'] >= input_price[0]) & (df['Input $/1M'] <= input_price[1])]
+    df = df[(df['Output $/1M'] >= output_price[0]) & (df['Output $/1M'] <= output_price[1])]
+
+    if "Image" in multimodal:
+        df = df[df['Multimodality Image'] == True]
+    if "Multi-Image" in multimodal:
+        df = df[df['Multimodality Multiple Image'] == True]
+    if "Audio" in multimodal:
+        df = df[df['Multimodality Audio'] == True]
+    if "Video" in multimodal:
+        df = df[df['Multimodality Video'] == True]
+
+    df = df[(df['Context Size'] >= (2**context[0])*1024) & (df['Context Size'] <= (2**context[1])*1024)]
+
+    if "Open" in open_weight and "Commercial" not in open_weight:
+        df = df[df['Open Weight'] == True]
+    elif "Commercial" in open_weight and "Open" not in open_weight:
+        df = df[df['Open Weight'] == False]
+        
+    df = df[df['License Name'].apply(lambda x: any(lic in x for lic in license))]
+
+    # Convert 'Release Date' to int temporarily
+    df['Temp Date'] = pd.to_datetime(df['Release Date']).astype(int) // 10**9  # Convert to seconds since epoch
+
+    # Convert start and end to int (seconds since epoch)
+    start = int(pd.to_datetime(start).timestamp())  
+    end = int(pd.to_datetime(end).timestamp())    
+
+    # Filter based on the converted 'Release Date'
+    df = df[(df['Temp Date'] >= start) & (df['Temp Date'] <= end)]
 
     df = filter_cols(df)
     return df  # Return the filtered dataframe
     
-    
-