Spaces:
Running
Running
| import pandas as pd | |
| import streamlit as st | |
| from app_utils import filter_dataframe, calculate_height_to_display | |
| from contants import WELCOME_TEXT, CITATION_TEXT | |
| from utils import BASE_SUMMARY_METRICS | |
| from utils import load_catalog, load_taxonomy | |
| from utils import datasets_count_and_size, datasets_count_and_size_standard, metadata_coverage, catalog_summary_statistics | |
| import matplotlib.pyplot as plt | |
| import seaborn as sns | |
| st.set_page_config(layout="wide") | |
| st.title("Polish Speech Datasets Catalog and Survey analysis") | |
| st.write(WELCOME_TEXT) | |
| st.write(CITATION_TEXT) | |
| # Cache the dataframe so it's only loaded once | |
| df_cat = load_catalog() | |
| df_tax = load_taxonomy() | |
| # Filter out non available datasets | |
| df_cat_available = df_cat[df_cat['Available online'] == 'yes'] | |
| # Available and free | |
| df_cat_available_free = df_cat[(df_cat['Available online'] == 'yes') & (df_cat['Price - non-commercial usage'] == 'free')] | |
| # Available and paid | |
| df_cat_available_paid = df_cat[(df_cat['Available online'] == 'yes') & (df_cat['Price - non-commercial usage'] != 'free')] | |
| # Display catalog contents | |
| st.dataframe(filter_dataframe(df_cat), hide_index=True, use_container_width=True) | |
| # Display taxonomy contents | |
| # Display summary statistics | |
| st.header("Polish ASR speech datasets summary statistics") | |
| df_summary_metrics = catalog_summary_statistics(df_cat) | |
| df_basic_stats = df_summary_metrics.loc[BASE_SUMMARY_METRICS[0:5]] | |
| st.dataframe(df_basic_stats, use_container_width=False) | |
| st.header("Speech data available across Polish ASR speech datasets") | |
| df_stats_audio_available = df_summary_metrics.loc[BASE_SUMMARY_METRICS[5:10]] | |
| st.dataframe(df_stats_audio_available, use_container_width=False) | |
| st.header("Transcribed data available across Polish ASR speech datasets") | |
| df_stats_transcribed_available = df_summary_metrics.loc[BASE_SUMMARY_METRICS[10:15]] | |
| st.dataframe(df_stats_transcribed_available, use_container_width=False) | |
| # Display distribution of datasets created per year | |
| st.header("Polish ASR speech datasets created in 1997-2023") | |
| col_groupby = ['Creation year'] | |
| df_datasets_per_speech_type = datasets_count_and_size(df_cat, col_groupby, col_sort=col_groupby, col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_speech_type, use_container_width=False) | |
| st.header("Institutions contributing Polish ASR speech dataset") | |
| col_groupby = ['Publisher'] | |
| df_datasets_per_publisher = datasets_count_and_size(df_cat, col_groupby, col_sort='Count Dataset ID', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_publisher, use_container_width=False) | |
| st.header("Repositories hosting Polish ASR speech datasets") | |
| col_groupby = ['Repository'] | |
| df_datasets_per_repo = datasets_count_and_size(df_cat, col_groupby, col_sort='Count Dataset ID', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_repo, use_container_width=False) | |
| st.header("Public domain Polish ASR speech datasets") | |
| col_groupby = ['License', "Dataset ID"] | |
| df_datasets_public = datasets_count_and_size(df_cat_available_free, col_groupby, col_sort='License', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = []) | |
| st.dataframe(df_datasets_public, use_container_width=False) | |
| st.header("Commercialy available Polish ASR speech datasets") | |
| col_groupby = ['License', "Dataset ID"] | |
| df_datasets_paid = datasets_count_and_size(df_cat_available_paid, col_groupby, col_sort='License', col_percent=None, col_sum=['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = []) | |
| st.dataframe(df_datasets_paid, use_container_width=False) | |
| st.header("Coverage of metadata across Polish ASR speech datasets") | |
| df_meta_all_flat, df_meta_all_pivot = metadata_coverage(df_cat, df_cat_available_free, df_cat_available_paid) | |
| st.dataframe(df_meta_all_pivot, use_container_width=False) | |
| # Display distribution of datasets for various speech types | |
| st.header("Datasets per speech type") | |
| col_groupby = ['Speech type'] | |
| df_datasets_per_speech_type = datasets_count_and_size(df_cat, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_speech_type, use_container_width=False) | |
| # Display distribution of datasets for various speech types | |
| st.header("Distribution of available speech data per audio device - Public domain datasets") | |
| col_groupby = ['Audio device'] | |
| df_datasets_per_device = datasets_count_and_size(df_cat_available_free, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_device, use_container_width=False) | |
| # Display distribution of datasets for various speech types | |
| st.header("Distribution of available speech data per audio device - Commercial datasets") | |
| col_groupby = ['Audio device'] | |
| df_datasets_per_device = datasets_count_and_size(df_cat_available_paid, col_groupby, col_sort=col_groupby, col_percent = ['Size audio transcribed [hours]'], col_sum = ['Size audio transcribed [hours]','Audio recordings', 'Speakers'], col_count = ['Dataset ID']) | |
| st.dataframe(df_datasets_per_device, use_container_width=False) |