Update app.py

app.py

@@ -10,11 +10,10 @@ from torch import cuda
 import json
 from datetime import datetime
 
-# get the device to be used eithe gpu or cpu
+# get the device to be used either gpu or cpu
 device = 'cuda' if cuda.is_available() else 'cpu'
 
-
-st.set_page_config(page_title="SEARCH IATI",layout='wide')
+st.set_page_config(page_title="SEARCH IATI", layout='wide')
 st.title("GIZ Project Database (PROTOTYPE)")
 var = st.text_input("Enter Search Query")
 
@@ -23,14 +22,11 @@ region_lookup_path = "docStore/regions_lookup.csv"
 region_df = load_region_data(region_lookup_path)
 
 ####################  Create the embeddings collection and save ######################
-# the steps below need to be performed only once and then commented out any unnecssary compute over-run
-##### First we process and create the chunks for relvant data source
-#chunks = process_giz_worldwide()
-##### Convert to langchain documents
-#temp_doc = create_documents(chunks,'chunks')
-##### Embed and store docs, check if collection exist then you need to update the collection
+# Uncomment these lines to process and embed your data only once.
+# chunks = process_giz_worldwide()
+# temp_doc = create_documents(chunks, 'chunks')
 collection_name = "giz_worldwide"
-#hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
+# hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
 
 ################### Hybrid Search ######################################################
 client = get_client()
@@ -81,11 +77,11 @@ col1, col2, col3, col4 = st.columns([1, 1, 1, 4])
 
 # Region filter
 with col1:
-    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions))  # Display region names
+    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions))
 
 # Dynamically filter countries based on selected region
 if region_filter == "All/Not allocated":
-    filtered_country_names = unique_country_names  # Show all countries if no region is selected
+    filtered_country_names = unique_country_names
 else:
     filtered_country_names = [
         name for name, code in country_name_mapping.items() if iso_code_to_sub_region.get(code) == region_filter
@@ -93,14 +89,12 @@ else:
 
 # Country filter
 with col2:
-    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)  # Display filtered country names
+    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)
 
-# # Year range slider # ToDo add end_year filter again
+# ToDo: Add year filter later if needed (currently commented out)
 # with col3:
 #     current_year = datetime.now().year
 #     default_start_year = current_year - 5
-
-#     # 3) The max_value is now the actual max end_year from collection
 #     end_year_range = st.slider(
 #         "Project End Year",
 #         min_value=2010,
@@ -111,7 +105,7 @@ with col2:
 # Checkbox to control whether to show only exact matches
 show_exact_matches = st.checkbox("Show only exact matches", value=False)
 
-def filter_results(results, country_filter, region_filter): ## , end_year_range ToDo add end_year filter again
+def filter_results(results, country_filter, region_filter):
     filtered = []
     for r in results:
         metadata = r.payload.get('metadata', {})
@@ -142,135 +136,97 @@ def filter_results(results, country_filter, region_filter): ## , end_year_range
         else:
             countries_in_region = c_list
 
-        # Filtering
         if (
             (country_filter == "All/Not allocated" or selected_iso_code in c_list)
             and (region_filter == "All/Not allocated" or countries_in_region)
-#            and (end_year_range[0] <= end_year_val <= end_year_range[1]) # ToDo add end_year filter again
         ):
             filtered.append(r)
     return filtered
 
 # Run the search 
-
-# 1) Adjust limit so we get more than 15 results
-results = hybrid_search(client, var, collection_name, limit=500)  # e.g., 100 or 200
-
-# results is a tuple: (semantic_results, lexical_results)
+results = hybrid_search(client, var, collection_name, limit=500)
 semantic_all = results[0]
 lexical_all = results[1]
 
-# 2) Filter out content < 20 chars (as intermediate fix to problem that e.g. super short paragraphs with few chars get high similarity score)
-semantic_all = [
-    r for r in semantic_all if len(r.payload["page_content"]) >= 5
-]
-lexical_all = [
-    r for r in lexical_all if len(r.payload["page_content"]) >= 5
-]
+# Filter out very short content
+semantic_all = [r for r in semantic_all if len(r.payload["page_content"]) >= 5]
+lexical_all = [r for r in lexical_all if len(r.payload["page_content"]) >= 5]
 
-# 2) Apply a threshold to SEMANTIC results (score >= 0.4)
+# Apply a threshold to SEMANTIC results (score >= 0.0)
 semantic_thresholded = [r for r in semantic_all if r.score >= 0.0]
 
-# 2) Filter the entire sets
-filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter) ## , end_year_range ToDo add end_year filter again
-filtered_lexical = filter_results(lexical_all, country_filter, region_filter)## , end_year_range ToDo add end_year filter again
+filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter)
+filtered_lexical = filter_results(lexical_all, country_filter, region_filter)
 
-filtered_semantic_no_dupe = remove_duplicates(filtered_semantic) # ToDo remove duplicates again?
+filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
 filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
+# Define a helper function to format currency values
+def format_currency(value):
+    try:
+        # Convert to float then int for formatting (assumes whole numbers)
+        return f"€{int(float(value)):,}"
+    except (ValueError, TypeError):
+        return value
 
-# 3) Retrieve top 15 *after* filtering
-# Check user preference
+#  Display Results
 if show_exact_matches:
-    # 1) Display heading
     st.write(f"Showing **Top 15 Lexical Search results** for query: {var}")
 
-    # 2) Do a simple substring check (case-insensitive)
-    #    We'll create a new list lexical_substring_filtered
     query_substring = var.strip().lower()
     lexical_substring_filtered = []
     for r in lexical_all:
-        # page_content in lowercase
-        page_text_lower = r.payload["page_content"].lower()
-        # Keep this result only if the query substring is found
-        if query_substring in page_text_lower:
+        if query_substring in r.payload["page_content"].lower():
             lexical_substring_filtered.append(r)
 
-    # 3) Now apply your region/country/year filter on that new list
-    filtered_lexical = filter_results(
-        lexical_substring_filtered, country_filter, region_filter
-    ) ## , end_year_range ToDo add end_year filter again
-
-    # 4) Remove duplicates
+    filtered_lexical = filter_results(lexical_substring_filtered, country_filter, region_filter)
     filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
-    # 5) If empty after substring + filters + dedupe, show a custom message
     if not filtered_lexical_no_dupe:
         st.write('No exact matches, consider unchecking "Show only exact matches"')
     else:
-        # 6) Display the first 15 matching results
         for res in filtered_lexical_no_dupe[:15]:
-            project_name = res.payload['metadata'].get('project_name', 'Project Link')
-            url = res.payload['metadata'].get('url', '#')
-            st.markdown(f"#### [{project_name}]({url})")
-
-            # Snippet logic (80 words)
+            metadata = res.payload.get('metadata', {})
+            # Get title and id; do not format as a link.
+            project_name = metadata.get('project_name', 'Project Link')
+            proj_id = metadata.get('id', 'Unknown')
+            st.markdown(f"#### {project_name} [{proj_id}]")
+
+            # Build snippet from objectives and descriptions.
+            objectives = metadata.get("objectives", "")
+            desc_de = metadata.get("description.de", "")
+            desc_en = metadata.get("description.en", "")
+            description = desc_de if desc_de else desc_en
+            full_snippet = f"Objective: {objectives} Description: {description}"
+            preview_limit = 400  # preview limit in characters
+            preview_snippet = full_snippet if len(full_snippet) <= preview_limit else full_snippet[:preview_limit] + "..."
+            # Using HTML to add a tooltip with the full snippet text.
+            st.markdown(f'<span title="{full_snippet}">{preview_snippet}</span>', unsafe_allow_html=True)
+
+            # Keywords remain the same.
             full_text = res.payload['page_content']
-            words = full_text.split()
-            preview_word_count = 200
-            preview_text = " ".join(words[:preview_word_count])
-            remainder_text = " ".join(words[preview_word_count:])
-            st.write(preview_text + ("..." if remainder_text else ""))
-
-            # Keywords
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
                 st.markdown(f"_{' · '.join(top_keywords)}_")
 
-            # Metadata
-            metadata = res.payload.get('metadata', {})
-            countries = metadata.get('countries', "[]")
+            # Metadata: get client, duration and budget details.
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
             total_volume = metadata.get('total_volume', "Unknown")
             total_project = metadata.get('total_project', "Unknown")
-            id = metadata.get('id', "Unknown")
 
-            
-            try:
-                c_list = json.loads(countries.replace("'", '"'))
-            except json.JSONDecodeError:
-                c_list = []
-            
-            # Only keep country names if the region lookup (get_country_name) 
-            # returns something different than the raw code.
-            matched_countries = []
-            for code in c_list:
-                if len(code) == 2:
-                    resolved_name = get_country_name(code.upper(), region_df)
-                    # If get_country_name didn't find a match, 
-                    # it typically just returns the same code (like "XX"). 
-                    # We'll consider "successfully looked up" if 
-                    # resolved_name != code.upper().
-                    if resolved_name.upper() != code.upper():
-                        matched_countries.append(resolved_name)
-            
-            # Format the year range
             start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
             end_year_str = f"{int(round(float(end_year)))}" if end_year else "Unknown"
-            
-            if matched_countries:
-                additional_text = (
-                    f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
-                    f"**{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
-                )
-            else:
-                additional_text = (
-                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
-                )
-
-            
+
+            formatted_project_budget = format_currency(total_project)
+            formatted_total_volume = format_currency(total_volume)
+
+            additional_text = (
+                f"Commissioned by **{client_name}**\n"
+                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
+                f"Budget: Project: **{formatted_project_budget}**, total volume: **{formatted_total_volume}**"
+            )
             st.markdown(additional_text)
             st.divider()
 
@@ -280,75 +236,51 @@ else:
     if not filtered_semantic_no_dupe:
         st.write("No relevant results found.")
     else:
-        # Show the top 15 from filtered_semantic
         for res in filtered_semantic_no_dupe[:15]:
-            project_name = res.payload['metadata'].get('project_name', 'Project Link')
-            url = res.payload['metadata'].get('url', '#')
-            st.markdown(f"#### [{project_name}]({url})")
-
-            # Snippet logic
-            full_text = res.payload['page_content']
-            words = full_text.split()
-            preview_word_count = 10
-            preview_text = " ".join(words[:preview_word_count])
-            remainder_text = " ".join(words[preview_word_count:])
-            st.write(preview_text + ("..." if remainder_text else ""))
+            metadata = res.payload.get('metadata', {})
+            project_name = metadata.get('project_name', 'Project Link')
+            proj_id = metadata.get('id', 'Unknown')
+            st.markdown(f"#### {project_name} [{proj_id}]")
+
+            # Build snippet from objectives and descriptions.
+            objectives = metadata.get("objectives", "")
+            desc_de = metadata.get("description.de", "")
+            desc_en = metadata.get("description.en", "")
+            description = desc_de if desc_de else desc_en
+            full_snippet = f"Objective: {objectives} Description: {description}"
+            preview_limit = 400
+            preview_snippet = full_snippet if len(full_snippet) <= preview_limit else full_snippet[:preview_limit] + "..."
+            st.markdown(f'<span title="{full_snippet}">{preview_snippet}</span>', unsafe_allow_html=True)
 
             # Keywords
+            full_text = res.payload['page_content']
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
                 st.markdown(f"_{' · '.join(top_keywords)}_")
 
-            # Metadata
-            metadata = res.payload.get('metadata', {})
-            countries = metadata.get('countries', "[]")
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
             total_volume = metadata.get('total_volume', "Unknown")
             total_project = metadata.get('total_project', "Unknown")
-            id = metadata.get('id', "Unknown")
-            
-            try:
-                c_list = json.loads(countries.replace("'", '"'))
-            except json.JSONDecodeError:
-                c_list = []
-            
-            # Only keep country names if the region lookup (get_country_name) 
-            # returns something different than the raw code.
-            matched_countries = []
-            for code in c_list:
-                if len(code) == 2:
-                    resolved_name = get_country_name(code.upper(), region_df)
-                    # If get_country_name didn't find a match, 
-                    # it typically just returns the same code (like "XX"). 
-                    # We'll consider "successfully looked up" if 
-                    # resolved_name != code.upper().
-                    if resolved_name.upper() != code.upper():
-                        matched_countries.append(resolved_name)
-            
-            # Format the year range
+
             start_year_str = extract_year(start_year) if start_year else "Unknown"
             end_year_str = extract_year(end_year) if end_year else "Unknown"
-            
-            # Build the final string
-            if matched_countries:
-                additional_text = (
-                    f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
-                    f"**{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
-                )
-            else:
-                additional_text = (
-                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**, project ID: {id}, project budget: {total_project}, total volumne: {total_volume}"
-                )
-
-            
+
+            formatted_project_budget = format_currency(total_project)
+            formatted_total_volume = format_currency(total_volume)
+
+            additional_text = (
+                f"Commissioned by **{client_name}**\n"
+                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
+                f"Budget: Project: **{formatted_project_budget}**, total volume: **{formatted_total_volume}**"
+            )
             st.markdown(additional_text)
             st.divider()
 
-
-    #  for i in results: 
-    #      st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main']))
-    #      st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}")
-    #      st.write(i.page_content)
-    #      st.divider()
+# Uncomment the following lines if you need to debug by listing raw results.
+# for i in results: 
+#     st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main']))
+#     st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}")
+#     st.write(i.page_content)
+#     st.divider()