Update app.py

app.py

@@ -10,10 +10,11 @@ from torch import cuda
 import json
 from datetime import datetime
 
-# get the device to be used either gpu or cpu
+# get the device to be used eithe gpu or cpu
 device = 'cuda' if cuda.is_available() else 'cpu'
 
-st.set_page_config(page_title="SEARCH IATI", layout='wide')
+
+st.set_page_config(page_title="SEARCH IATI",layout='wide')
 st.title("GIZ Project Database (PROTOTYPE)")
 var = st.text_input("Enter Search Query")
 
@@ -22,11 +23,14 @@ region_lookup_path = "docStore/regions_lookup.csv"
 region_df = load_region_data(region_lookup_path)
 
 ####################  Create the embeddings collection and save ######################
-# Uncomment these lines to process and embed your data only once.
-# chunks = process_giz_worldwide()
-# temp_doc = create_documents(chunks, 'chunks')
+# the steps below need to be performed only once and then commented out any unnecssary compute over-run
+##### First we process and create the chunks for relvant data source
+chunks = process_giz_worldwide()
+##### Convert to langchain documents
+temp_doc = create_documents(chunks,'chunks')
+##### Embed and store docs, check if collection exist then you need to update the collection
 collection_name = "giz_worldwide"
-# hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
+hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)
 
 ################### Hybrid Search ######################################################
 client = get_client()
@@ -43,7 +47,6 @@ _, unique_sub_regions = get_regions(region_df)
 def get_country_name_and_region_mapping(_client, collection_name, region_df):
     results = hybrid_search(_client, "", collection_name)
     country_set = set()
-    
     for res in results[0] + results[1]:
         countries = res.payload.get('metadata', {}).get('countries', "[]")
         try:
@@ -77,11 +80,11 @@ col1, col2, col3, col4 = st.columns([1, 1, 1, 4])
 
 # Region filter
 with col1:
-    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions))
+    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions))  # Display region names
 
 # Dynamically filter countries based on selected region
 if region_filter == "All/Not allocated":
-    filtered_country_names = unique_country_names
+    filtered_country_names = unique_country_names  # Show all countries if no region is selected
 else:
     filtered_country_names = [
         name for name, code in country_name_mapping.items() if iso_code_to_sub_region.get(code) == region_filter
@@ -89,23 +92,25 @@ else:
 
 # Country filter
 with col2:
-    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)
-
-# ToDo: Add year filter later if needed (currently commented out)
-# with col3:
-#     current_year = datetime.now().year
-#     default_start_year = current_year - 5
-#     end_year_range = st.slider(
-#         "Project End Year",
-#         min_value=2010,
-#         max_value=max_end_year,
-#         value=(default_start_year, max_end_year),
-#     )
+    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)  # Display filtered country names
+
+# Year range slider
+with col3:
+    current_year = datetime.now().year
+    default_start_year = current_year - 5
+
+    # 3) The max_value is now the actual max end_year from collection
+    end_year_range = st.slider(
+        "Project End Year",
+        min_value=2010,
+        max_value=max_end_year,
+        value=(default_start_year, max_end_year),
+    )
 
 # Checkbox to control whether to show only exact matches
 show_exact_matches = st.checkbox("Show only exact matches", value=False)
 
-def filter_results(results, country_filter, region_filter):
+def filter_results(results, country_filter, region_filter, end_year_range):
     filtered = []
     for r in results:
         metadata = r.payload.get('metadata', {})
@@ -136,97 +141,133 @@ def filter_results(results, country_filter, region_filter):
         else:
             countries_in_region = c_list
 
+        # Filtering
         if (
             (country_filter == "All/Not allocated" or selected_iso_code in c_list)
             and (region_filter == "All/Not allocated" or countries_in_region)
+            and (end_year_range[0] <= end_year_val <= end_year_range[1])
         ):
             filtered.append(r)
     return filtered
 
 # Run the search 
-results = hybrid_search(client, var, collection_name, limit=500)
+
+# 1) Adjust limit so we get more than 15 results
+results = hybrid_search(client, var, collection_name, limit=500)  # e.g., 100 or 200
+
+# results is a tuple: (semantic_results, lexical_results)
 semantic_all = results[0]
 lexical_all = results[1]
 
-# Filter out very short content
-semantic_all = [r for r in semantic_all if len(r.payload["page_content"]) >= 5]
-lexical_all = [r for r in lexical_all if len(r.payload["page_content"]) >= 5]
+# 2) Filter out content < 20 chars (as intermediate fix to problem that e.g. super short paragraphs with few chars get high similarity score)
+semantic_all = [
+    r for r in semantic_all if len(r.payload["page_content"]) >= 20
+]
+lexical_all = [
+    r for r in lexical_all if len(r.payload["page_content"]) >= 20
+]
 
-# Apply a threshold to SEMANTIC results (score >= 0.0)
-semantic_thresholded = [r for r in semantic_all if r.score >= 0.0]
+# 2) Apply a threshold to SEMANTIC results (score >= 0.4)
+semantic_thresholded = [r for r in semantic_all if r.score >= 0.4]
 
-filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter)
-filtered_lexical = filter_results(lexical_all, country_filter, region_filter)
+# 2) Filter the entire sets
+filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter, end_year_range)
+filtered_lexical = filter_results(lexical_all, country_filter, region_filter, end_year_range)
 
 filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
 filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
-# Define a helper function to format currency values
-def format_currency(value):
-    try:
-        # Convert to float then int for formatting (assumes whole numbers)
-        return f"€{int(float(value)):,}"
-    except (ValueError, TypeError):
-        return value
 
-#  Display Results
+# 3) Retrieve top 15 *after* filtering
+# Check user preference
 if show_exact_matches:
+    # 1) Display heading
     st.write(f"Showing **Top 15 Lexical Search results** for query: {var}")
 
+    # 2) Do a simple substring check (case-insensitive)
+    #    We'll create a new list lexical_substring_filtered
     query_substring = var.strip().lower()
     lexical_substring_filtered = []
     for r in lexical_all:
-        if query_substring in r.payload["page_content"].lower():
+        # page_content in lowercase
+        page_text_lower = r.payload["page_content"].lower()
+        # Keep this result only if the query substring is found
+        if query_substring in page_text_lower:
             lexical_substring_filtered.append(r)
 
-    filtered_lexical = filter_results(lexical_substring_filtered, country_filter, region_filter)
+    # 3) Now apply your region/country/year filter on that new list
+    filtered_lexical = filter_results(
+        lexical_substring_filtered, country_filter, region_filter, end_year_range
+    )
+
+    # 4) Remove duplicates
     filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
 
+    # 5) If empty after substring + filters + dedupe, show a custom message
     if not filtered_lexical_no_dupe:
         st.write('No exact matches, consider unchecking "Show only exact matches"')
     else:
+        # 6) Display the first 15 matching results
         for res in filtered_lexical_no_dupe[:15]:
-            metadata = res.payload.get('metadata', {})
-            # Get title and id; do not format as a link.
-            project_name = metadata.get('project_name', 'Project Link')
-            proj_id = metadata.get('id', 'Unknown')
-            st.markdown(f"#### {project_name} [{proj_id}]")
-
-            # Build snippet from objectives and descriptions.
-            objectives = metadata.get("objectives", "")
-            desc_de = metadata.get("description.de", "")
-            desc_en = metadata.get("description.en", "")
-            description = desc_de if desc_de else desc_en
-            full_snippet = f"Objective: {objectives} Description: {description}"
-            preview_limit = 400  # preview limit in characters
-            preview_snippet = full_snippet if len(full_snippet) <= preview_limit else full_snippet[:preview_limit] + "..."
-            # Using HTML to add a tooltip with the full snippet text.
-            st.markdown(f'<span title="{full_snippet}">{preview_snippet}</span>', unsafe_allow_html=True)
-
-            # Keywords remain the same.
+            project_name = res.payload['metadata'].get('project_name', 'Project Link')
+            url = res.payload['metadata'].get('url', '#')
+            st.markdown(f"#### [{project_name}]({url})")
+
+            # Snippet logic (80 words)
             full_text = res.payload['page_content']
+            words = full_text.split()
+            preview_word_count = 80
+            preview_text = " ".join(words[:preview_word_count])
+            remainder_text = " ".join(words[preview_word_count:])
+            st.write(preview_text + ("..." if remainder_text else ""))
+
+            # Keywords
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
                 st.markdown(f"_{' · '.join(top_keywords)}_")
 
-            # Metadata: get client, duration and budget details.
+            # Metadata
+            metadata = res.payload.get('metadata', {})
+            countries = metadata.get('countries', "[]")
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
-            total_volume = metadata.get('total_volume', "Unknown")
-            total_project = metadata.get('total_project', "Unknown")
-
+            
+            try:
+                c_list = json.loads(countries.replace("'", '"'))
+            except json.JSONDecodeError:
+                c_list = []
+            
+            # Only keep country names if the region lookup (get_country_name) 
+            # returns something different than the raw code.
+            matched_countries = []
+            for code in c_list:
+                if len(code) == 2:
+                    resolved_name = get_country_name(code.upper(), region_df)
+                    # If get_country_name didn't find a match, 
+                    # it typically just returns the same code (like "XX"). 
+                    # We'll consider "successfully looked up" if 
+                    # resolved_name != code.upper().
+                    if resolved_name.upper() != code.upper():
+                        matched_countries.append(resolved_name)
+            
+            # Format the year range
             start_year_str = f"{int(round(float(start_year)))}" if start_year else "Unknown"
             end_year_str = f"{int(round(float(end_year)))}" if end_year else "Unknown"
-
-            formatted_project_budget = format_currency(total_project)
-            formatted_total_volume = format_currency(total_volume)
-
-            additional_text = (
-                f"Commissioned by **{client_name}**\n"
-                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
-                f"Budget: Project: **{formatted_project_budget}**, total volume: **{formatted_total_volume}**"
-            )
+            
+            # Build the final string
+            if matched_countries:
+                # We have at least 1 valid country name
+                additional_text = (
+                    f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
+                    f"**{start_year_str}-{end_year_str}**"
+                )
+            else:
+                # No valid countries found
+                additional_text = (
+                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+                )
+            
             st.markdown(additional_text)
             st.divider()
 
@@ -236,51 +277,73 @@ else:
     if not filtered_semantic_no_dupe:
         st.write("No relevant results found.")
     else:
+        # Show the top 15 from filtered_semantic
         for res in filtered_semantic_no_dupe[:15]:
-            metadata = res.payload.get('metadata', {})
-            project_name = metadata.get('project_name', 'Project Link')
-            proj_id = metadata.get('id', 'Unknown')
-            st.markdown(f"#### {project_name} [{proj_id}]")
-
-            # Build snippet from objectives and descriptions.
-            objectives = metadata.get("objectives", "")
-            desc_de = metadata.get("description.de", "")
-            desc_en = metadata.get("description.en", "")
-            description = desc_de if desc_de else desc_en
-            full_snippet = f"Objective: {objectives} Description: {description}"
-            preview_limit = 400
-            preview_snippet = full_snippet if len(full_snippet) <= preview_limit else full_snippet[:preview_limit] + "..."
-            st.markdown(f'<span title="{full_snippet}">{preview_snippet}</span>', unsafe_allow_html=True)
+            project_name = res.payload['metadata'].get('project_name', 'Project Link')
+            url = res.payload['metadata'].get('url', '#')
+            st.markdown(f"#### [{project_name}]({url})")
 
-            # Keywords
+            # Snippet logic
             full_text = res.payload['page_content']
+            words = full_text.split()
+            preview_word_count = 10
+            preview_text = " ".join(words[:preview_word_count])
+            remainder_text = " ".join(words[preview_word_count:])
+            st.write(preview_text + ("..." if remainder_text else ""))
+
+            # Keywords
             top_keywords = extract_top_keywords(full_text, top_n=5)
             if top_keywords:
                 st.markdown(f"_{' · '.join(top_keywords)}_")
 
+            # Metadata
+            metadata = res.payload.get('metadata', {})
+            countries = metadata.get('countries', "[]")
             client_name = metadata.get('client', 'Unknown Client')
             start_year = metadata.get('start_year', None)
             end_year = metadata.get('end_year', None)
-            total_volume = metadata.get('total_volume', "Unknown")
-            total_project = metadata.get('total_project', "Unknown")
-
+            
+            try:
+                c_list = json.loads(countries.replace("'", '"'))
+            except json.JSONDecodeError:
+                c_list = []
+            
+            # Only keep country names if the region lookup (get_country_name) 
+            # returns something different than the raw code.
+            matched_countries = []
+            for code in c_list:
+                if len(code) == 2:
+                    resolved_name = get_country_name(code.upper(), region_df)
+                    # If get_country_name didn't find a match, 
+                    # it typically just returns the same code (like "XX"). 
+                    # We'll consider "successfully looked up" if 
+                    # resolved_name != code.upper().
+                    if resolved_name.upper() != code.upper():
+                        matched_countries.append(resolved_name)
+            
+            # Format the year range
             start_year_str = extract_year(start_year) if start_year else "Unknown"
             end_year_str = extract_year(end_year) if end_year else "Unknown"
-
-            formatted_project_budget = format_currency(total_project)
-            formatted_total_volume = format_currency(total_volume)
-
-            additional_text = (
-                f"Commissioned by **{client_name}**\n"
-                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
-                f"Budget: Project: **{formatted_project_budget}**, total volume: **{formatted_total_volume}**"
-            )
+            
+            # Build the final string
+            if matched_countries:
+                # We have at least 1 valid country name
+                additional_text = (
+                    f"**{', '.join(matched_countries)}**, commissioned by **{client_name}**, "
+                    f"**{start_year_str}-{end_year_str}**"
+                )
+            else:
+                # No valid countries found
+                additional_text = (
+                    f"Commissioned by **{client_name}**, **{start_year_str}-{end_year_str}**"
+                )
+            
             st.markdown(additional_text)
             st.divider()
 
-# Uncomment the following lines if you need to debug by listing raw results.
-# for i in results: 
-#     st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main']))
-#     st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}")
-#     st.write(i.page_content)
-#     st.divider()
+
+    #  for i in results: 
+    #      st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main']))
+    #      st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}")
+    #      st.write(i.page_content)
+    #      st.divider()