Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import requests | |
| import pandas as pd | |
| import re | |
| from appStore.prep_data import process_giz_worldwide, remove_duplicates, get_max_end_year, extract_year | |
| from appStore.prep_utils import create_documents, get_client | |
| from appStore.embed import hybrid_embed_chunks | |
| from appStore.search import hybrid_search | |
| from appStore.region_utils import load_region_data, get_country_name, get_regions | |
| from appStore.tfidf_extraction import extract_top_keywords | |
| from torch import cuda | |
| import json | |
| from datetime import datetime | |
| #model_config = getconfig("model_params.cfg") | |
| ########### | |
| # ToDo move functions to utils and model specifications to config file! | |
| # Configuration for the dedicated model | |
| DEDICATED_MODEL = "meta-llama/Llama-3.1-8B-Instruct" | |
| DEDICATED_ENDPOINT = "https://qu2d8m6dmsollhly.us-east-1.aws.endpoints.huggingface.cloud" | |
| # Write access token from the settings | |
| WRITE_ACCESS_TOKEN = st.secrets["Llama_3_1"] | |
| def get_rag_answer(query, top_results): | |
| context = "\n\n".join([res.payload["page_content"] for res in top_results]) | |
| max_context_chars = 13500 | |
| if len(context) > max_context_chars: | |
| context = context[:max_context_chars] | |
| prompt = ( | |
| "You are a project portfolio adviser at the development cooperation GIZ. " | |
| "Using the following context, answer the question precisely. " | |
| "Only output the final answer below, without repeating the context or question.\n\n" | |
| f"Context:\n{context}\n\n" | |
| f"Question: {query}\n\n" | |
| "Answer:" | |
| ) | |
| headers = {"Authorization": f"Bearer {WRITE_ACCESS_TOKEN}"} | |
| payload = { | |
| "inputs": prompt, | |
| "parameters": {"max_new_tokens": 220} | |
| } | |
| response = requests.post(DEDICATED_ENDPOINT, headers=headers, json=payload) | |
| if response.status_code == 200: | |
| result = response.json() | |
| answer = result[0]["generated_text"] | |
| if "Answer:" in answer: | |
| answer = answer.split("Answer:")[-1].strip() | |
| return answer | |
| else: | |
| return f"Error in generating answer: {response.text}" | |
| # Helper: Format project id (e.g., "201940485" -> "2019.4048.5") | |
| def format_project_id(pid): | |
| s = str(pid) | |
| if len(s) > 5: | |
| return s[:4] + "." + s[4:-1] + "." + s[-1] | |
| return s | |
| # Helper: Compute title from metadata using name.en (or name.de if empty) | |
| def compute_title(metadata): | |
| name_en = metadata.get("name.en", "").strip() | |
| name_de = metadata.get("name.de", "").strip() | |
| base = name_en if name_en else name_de | |
| pid = metadata.get("id", "") | |
| if base and pid: | |
| return f"{base} [{format_project_id(pid)}]" | |
| return base or "No Title" | |
| # Helper: Get CRS filter options from all documents | |
| def get_crs_options(_client, collection_name): | |
| results = hybrid_search(_client, "", collection_name) | |
| all_results = results[0] + results[1] | |
| crs_set = set() | |
| for res in all_results: | |
| metadata = res.payload.get('metadata', {}) | |
| crs_key = metadata.get("crs_key", "").strip() | |
| crs_value = metadata.get("crs_value", "").strip() | |
| if crs_key or crs_value: | |
| # Convert crs_value to integer if possible: | |
| try: | |
| crs_int = int(float(crs_value)) | |
| except: | |
| crs_int = crs_value | |
| crs_combined = f"{crs_key}: {crs_int}" | |
| crs_set.add(crs_combined) | |
| return sorted(crs_set) | |
| # Revised filter_results: Allow missing end_year or CRS; enforce CRS only when present. | |
| def filter_results(results, country_filter, region_filter, end_year_range, crs_filter): | |
| filtered = [] | |
| for r in results: | |
| metadata = r.payload.get('metadata', {}) | |
| countries = metadata.get('countries', "[]") | |
| year_str = metadata.get('end_year') | |
| if year_str: | |
| extracted = extract_year(year_str) | |
| try: | |
| end_year_val = int(extracted) if extracted != "Unknown" else 0 | |
| except ValueError: | |
| end_year_val = 0 | |
| else: | |
| end_year_val = 0 | |
| try: | |
| c_list = json.loads(countries.replace("'", '"')) | |
| c_list = [code.upper() for code in c_list if len(code) == 2] | |
| except json.JSONDecodeError: | |
| c_list = [] | |
| selected_iso_code = country_name_mapping.get(country_filter, None) | |
| if region_filter != "All/Not allocated": | |
| countries_in_region = [code for code in c_list if iso_code_to_sub_region.get(code) == region_filter] | |
| else: | |
| countries_in_region = c_list | |
| crs_key = metadata.get("crs_key", "").strip() | |
| crs_value = metadata.get("crs_value", "").strip() | |
| try: | |
| crs_int = int(float(crs_value)) | |
| except: | |
| crs_int = crs_value | |
| crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else "" | |
| # Only enforce CRS filter if result has a CRS value. | |
| if crs_filter != "All/Not allocated" and crs_combined: | |
| if crs_filter != crs_combined: | |
| continue | |
| # Allow projects with no valid end_year to pass (if end_year_val is 0) | |
| year_ok = True if end_year_val == 0 else (end_year_range[0] <= end_year_val <= end_year_range[1]) | |
| if ((country_filter == "All/Not allocated" or (selected_iso_code and selected_iso_code in c_list)) | |
| and (region_filter == "All/Not allocated" or countries_in_region) | |
| and year_ok): | |
| filtered.append(r) | |
| return filtered | |
| # Get the device to be used (GPU or CPU) | |
| device = 'cuda' if cuda.is_available() else 'cpu' | |
| st.set_page_config(page_title="SEARCH IATI", layout='wide') | |
| st.title("GIZ Project Database (PROTOTYPE)") | |
| var = st.text_input("Enter Search Question") | |
| # Load the region lookup CSV | |
| region_lookup_path = "docStore/regions_lookup.csv" | |
| region_df = load_region_data(region_lookup_path) | |
| #################### Create the embeddings collection and save ###################### | |
| # the steps below need to be performed only once and then commented out any unnecssary compute over-run | |
| ##### First we process and create the chunks for relvant data source | |
| #chunks = process_giz_worldwide() | |
| ##### Convert to langchain documents | |
| #temp_doc = create_documents(chunks,'chunks') | |
| ##### Embed and store docs, check if collection exist then you need to update the collection | |
| collection_name = "giz_worldwide" | |
| #hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True) | |
| ################### Hybrid Search ##################################################### | |
| client = get_client() | |
| print(client.get_collections()) | |
| max_end_year = get_max_end_year(client, collection_name) | |
| _, unique_sub_regions = get_regions(region_df) | |
| def get_country_name_and_region_mapping(_client, collection_name, region_df): | |
| results = hybrid_search(_client, "", collection_name) | |
| country_set = set() | |
| for res in results[0] + results[1]: | |
| countries = res.payload.get('metadata', {}).get('countries', "[]") | |
| try: | |
| country_list = json.loads(countries.replace("'", '"')) | |
| two_digit_codes = [code.upper() for code in country_list if len(code) == 2] | |
| country_set.update(two_digit_codes) | |
| except json.JSONDecodeError: | |
| pass | |
| country_name_to_code = {} | |
| iso_code_to_sub_region = {} | |
| for code in country_set: | |
| name = get_country_name(code, region_df) | |
| sub_region_row = region_df[region_df['alpha-2'] == code] | |
| sub_region = sub_region_row['sub-region'].values[0] if not sub_region_row.empty else "Not allocated" | |
| country_name_to_code[name] = code | |
| iso_code_to_sub_region[code] = sub_region | |
| return country_name_to_code, iso_code_to_sub_region | |
| client = get_client() | |
| country_name_mapping, iso_code_to_sub_region = get_country_name_and_region_mapping(client, collection_name, region_df) | |
| unique_country_names = sorted(country_name_mapping.keys()) | |
| # Layout filters in columns | |
| col1, col2, col3, col4 = st.columns([1, 1, 1, 1]) | |
| with col1: | |
| region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions)) | |
| # Compute filtered_country_names based on region_filter: | |
| if region_filter == "All/Not allocated": | |
| filtered_country_names = unique_country_names | |
| else: | |
| filtered_country_names = [name for name, code in country_name_mapping.items() if iso_code_to_sub_region.get(code) == region_filter] | |
| with col2: | |
| country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names) | |
| with col3: | |
| current_year = datetime.now().year | |
| default_start_year = current_year - 4 | |
| end_year_range = st.slider("Project End Year", min_value=2010, max_value=max_end_year, value=(default_start_year, max_end_year)) | |
| with col4: | |
| crs_options = ["All/Not allocated"] + get_crs_options(client, collection_name) | |
| crs_filter = st.selectbox("CRS", crs_options) | |
| # Checkbox for exact matches | |
| show_exact_matches = st.checkbox("Show only exact matches", value=False) | |
| # Run the search | |
| results = hybrid_search(client, var, collection_name, limit=500) | |
| semantic_all = results[0] | |
| lexical_all = results[1] | |
| semantic_all = [r for r in semantic_all if len(r.payload["page_content"]) >= 5] | |
| lexical_all = [r for r in lexical_all if len(r.payload["page_content"]) >= 5] | |
| semantic_thresholded = [r for r in semantic_all if r.score >= 0.0] | |
| filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter, end_year_range, crs_filter) | |
| filtered_lexical = filter_results(lexical_all, country_filter, region_filter, end_year_range, crs_filter) | |
| filtered_semantic_no_dupe = remove_duplicates(filtered_semantic) | |
| filtered_lexical_no_dupe = remove_duplicates(filtered_lexical) | |
| def format_currency(value): | |
| try: | |
| return f"€{int(float(value)):,}" | |
| except (ValueError, TypeError): | |
| return value | |
| # Helper to highlight query matches (case-insensitive) | |
| def highlight_query(text, query): | |
| pattern = re.compile(re.escape(query), re.IGNORECASE) | |
| return pattern.sub(lambda m: f"**{m.group(0)}**", text) | |
| ############################### | |
| # Display Lexical Results Branch | |
| ############################### | |
| if show_exact_matches: | |
| st.write(f"Showing **Top 15 Lexical Search results** for query: {var}") | |
| query_substring = var.strip().lower() | |
| lexical_substring_filtered = [r for r in lexical_all if query_substring in r.payload["page_content"].lower()] | |
| filtered_lexical = filter_results(lexical_substring_filtered, country_filter, region_filter, end_year_range, crs_filter) | |
| filtered_lexical_no_dupe = remove_duplicates(filtered_lexical) | |
| if not filtered_lexical_no_dupe: | |
| st.write('No exact matches, consider unchecking "Show only exact matches"') | |
| else: | |
| top_results = filtered_lexical_no_dupe[:10] | |
| rag_answer = get_rag_answer(var, top_results) | |
| st.markdown(f"**{var}**") | |
| st.write(rag_answer) | |
| st.divider() | |
| for res in top_results: | |
| metadata = res.payload.get('metadata', {}) | |
| if "title" not in metadata: | |
| metadata["title"] = compute_title(metadata) | |
| # Highlight query in red and bold (allow HTML) | |
| display_title = ( | |
| st.markdown(highlight_query(metadata["title"], var), unsafe_allow_html=True) | |
| if var.strip() else metadata["title"] | |
| ) | |
| proj_id = metadata.get('id', 'Unknown') | |
| st.markdown(f"#### {metadata['title']}") | |
| countries = metadata.get('countries') | |
| client_name = metadata.get('client', 'Unknown Client') | |
| start_year = metadata.get('start_year', None) | |
| end_year = metadata.get('end_year', None) | |
| total_volume = metadata.get('total_volume', "Unknown") | |
| total_project = metadata.get('total_project', "Unknown") | |
| objectives = metadata.get("objectives", "") | |
| desc_de = metadata.get("description.de", "") | |
| desc_en = metadata.get("description.en", "") | |
| description = desc_de if desc_de else desc_en | |
| full_snippet = f"{objectives} {description}" | |
| words = full_snippet.split() | |
| preview_word_count = 90 | |
| preview_text = " ".join(words[:preview_word_count]) | |
| remainder_text = " ".join(words[preview_word_count:]) | |
| preview_text = highlight_query(preview_text, var) if var.strip() else preview_text | |
| st.write(preview_text) | |
| if remainder_text: | |
| with st.expander("Show more"): | |
| st.write(remainder_text) | |
| full_text = res.payload['page_content'] | |
| top_keywords = extract_top_keywords(full_text, top_n=5) | |
| if top_keywords: | |
| st.markdown(f"_{' · '.join(top_keywords)}_") | |
| # Format year range and budget info | |
| start_year_str = extract_year(start_year) if start_year else "Unknown" | |
| end_year_str = extract_year(end_year) if end_year else "Unknown" | |
| formatted_project_budget = format_currency(total_project) | |
| formatted_total_volume = format_currency(total_volume) | |
| # Compute matched country names (as before) | |
| try: | |
| c_list = json.loads(metadata.get('countries', "[]").replace("'", '"')) | |
| except json.JSONDecodeError: | |
| c_list = [] | |
| matched_countries = [] | |
| for code in c_list: | |
| if len(code) == 2: | |
| resolved_name = get_country_name(code.upper(), region_df) | |
| if resolved_name.upper() != code.upper(): | |
| matched_countries.append(resolved_name) | |
| # Compute CRS combined value as integer | |
| crs_key = metadata.get("crs_key", "").strip() | |
| crs_value = metadata.get("crs_value", "").strip() | |
| try: | |
| crs_int = int(float(crs_value)) | |
| except: | |
| crs_int = crs_value | |
| crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else "Unknown" | |
| # Build the additional text with original details, then add Sector and contact. | |
| additional_text = ( | |
| f"Commissioned by **{client_name}**\n" | |
| f"Projekt duration **{start_year_str}-{end_year_str}**\n" | |
| f"Budget: Project: **{formatted_project_budget}**, Total volume: **{formatted_total_volume}**\n" | |
| f"Country: **{', '.join(matched_countries)}**\n" | |
| f"Sector: **{crs_combined}**" | |
| ) | |
| #contact = metadata.get("contact", "").strip() | |
| #if contact and contact.lower() != "[email protected]": | |
| # additional_text += f" | Contact: **{contact}**" | |
| st.markdown(additional_text) | |
| st.divider() | |
| ############################### | |
| # Display Semantic Results Branch | |
| ############################### | |
| else: | |
| if not filtered_semantic_no_dupe: | |
| st.write("No relevant results found.") | |
| else: | |
| top_results = filtered_semantic_no_dupe[:10] | |
| rag_answer = get_rag_answer(var, top_results) | |
| st.markdown(f"**{var}**") | |
| st.write(rag_answer) | |
| st.divider() | |
| st.write(f"Showing **Top 15 Semantic Search results** for query: {var}") | |
| for res in top_results: | |
| metadata = res.payload.get('metadata', {}) | |
| if "title" not in metadata: | |
| metadata["title"] = compute_title(metadata) | |
| display_title = metadata["title"] | |
| st.markdown(f"#### {display_title}") | |
| countries = metadata.get('countries') | |
| client_name = metadata.get('client', 'Unknown Client') | |
| start_year = metadata.get('start_year', None) | |
| end_year = metadata.get('end_year', None) | |
| total_volume = metadata.get('total_volume', "Unknown") | |
| total_project = metadata.get('total_project', "Unknown") | |
| objectives = metadata.get("objectives", "") | |
| desc_de = metadata.get("description.de", "") | |
| desc_en = metadata.get("description.en", "") | |
| description = desc_de if desc_de else desc_en | |
| full_snippet = f"{objectives} {description}" | |
| words = full_snippet.split() | |
| preview_word_count = 90 | |
| preview_text = " ".join(words[:preview_word_count]) | |
| remainder_text = " ".join(words[preview_word_count:]) | |
| st.write(preview_text) | |
| if remainder_text: | |
| with st.expander("Show more"): | |
| st.write(remainder_text) | |
| top_keywords = extract_top_keywords(res.payload['page_content'], top_n=5) | |
| if top_keywords: | |
| st.markdown(f"_{' · '.join(top_keywords)}_") | |
| # Format year range and budget info | |
| start_year_str = extract_year(start_year) if start_year else "Unknown" | |
| end_year_str = extract_year(end_year) if end_year else "Unknown" | |
| formatted_project_budget = format_currency(total_project) | |
| formatted_total_volume = format_currency(total_volume) | |
| # Compute matched country names (as before) | |
| try: | |
| c_list = json.loads(metadata.get('countries', "[]").replace("'", '"')) | |
| except json.JSONDecodeError: | |
| c_list = [] | |
| matched_countries = [] | |
| for code in c_list: | |
| if len(code) == 2: | |
| resolved_name = get_country_name(code.upper(), region_df) | |
| if resolved_name.upper() != code.upper(): | |
| matched_countries.append(resolved_name) | |
| # Compute CRS combined value | |
| crs_key = metadata.get("crs_key", "").strip() | |
| crs_value = metadata.get("crs_value", "").strip() | |
| try: | |
| crs_int = int(float(crs_value)) | |
| except: | |
| crs_int = crs_value | |
| crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else "Unknown" | |
| # Build the additional text with original details, then add Sector and contact. | |
| additional_text = ( | |
| f"Commissioned by **{client_name}**\n" | |
| f"Projekt duration **{start_year_str}-{end_year_str}**\n" | |
| f"Budget: Project: **{formatted_project_budget}**, Total volume: **{formatted_total_volume}**\n" | |
| f"Country: **{', '.join(matched_countries)}**\n" | |
| f"Sector: **{crs_combined}**" | |
| ) | |
| #contact = metadata.get("contact", "").strip() | |
| #if contact and contact.lower() != "[email protected]": | |
| # additional_text += f" | Contact: **{contact}**" | |
| st.markdown(additional_text) | |
| st.divider() | |
| # for i in results: | |
| # st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main'])) | |
| # st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}") | |
| # st.write(i.page_content) | |
| # st.divider() |