app.py

import streamlit as st
import requests
import pandas as pd
import re
from appStore.prep_data import process_giz_worldwide, remove_duplicates, get_max_end_year, extract_year
from appStore.prep_utils import create_documents, get_client
from appStore.embed import hybrid_embed_chunks
from appStore.search import hybrid_search
from appStore.region_utils import load_region_data, get_country_name, get_regions
from appStore.tfidf_extraction import extract_top_keywords 
from torch import cuda
import json
from datetime import datetime



#model_config = getconfig("model_params.cfg")
###########
# ToDo move functions to utils and model specifications to config file!
# Configuration for the dedicated model
DEDICATED_MODEL = "meta-llama/Llama-3.1-8B-Instruct"
DEDICATED_ENDPOINT = "https://qu2d8m6dmsollhly.us-east-1.aws.endpoints.huggingface.cloud"
# Write access token from the settings
WRITE_ACCESS_TOKEN = st.secrets["Llama_3_1"]

def get_rag_answer(query, top_results):
    context = "\n\n".join([res.payload["page_content"] for res in top_results])
    max_context_chars = 13500  
    if len(context) > max_context_chars:
        context = context[:max_context_chars]
    prompt = (
        "You are a project portfolio adviser at the development cooperation GIZ. "
        "Using the following context, answer the question precisely. "
        "Only output the final answer below, without repeating the context or question.\n\n"
        f"Context:\n{context}\n\n"
        f"Question: {query}\n\n"
        "Answer:"
    )
    headers = {"Authorization": f"Bearer {WRITE_ACCESS_TOKEN}"}
    payload = {
        "inputs": prompt,
        "parameters": {"max_new_tokens": 220}
    }
    response = requests.post(DEDICATED_ENDPOINT, headers=headers, json=payload)
    if response.status_code == 200:
        result = response.json()
        answer = result[0]["generated_text"]
        if "Answer:" in answer:
            answer = answer.split("Answer:")[-1].strip()
        return answer
    else:
        return f"Error in generating answer: {response.text}"

# Helper: Format project id (e.g., "201940485" -> "2019.4048.5")
def format_project_id(pid):
    s = str(pid)
    if len(s) > 5:
        return s[:4] + "." + s[4:-1] + "." + s[-1]
    return s

# Helper: Compute title from metadata using name.en (or name.de if empty)
def compute_title(metadata):
    name_en = metadata.get("name.en", "").strip()
    name_de = metadata.get("name.de", "").strip()
    base = name_en if name_en else name_de
    pid = metadata.get("id", "")
    if base and pid:
        return f"{base} [{format_project_id(pid)}]"
    return base or "No Title"

# Helper: Get CRS filter options from all documents
@st.cache_data
def get_crs_options(_client, collection_name):
    results = hybrid_search(_client, "", collection_name)
    all_results = results[0] + results[1]
    crs_set = set()
    for res in all_results:
        metadata = res.payload.get('metadata', {})
        crs_key = metadata.get("crs_key", "").strip()
        crs_value = metadata.get("crs_value", "").strip()
        if crs_key or crs_value:
            # Convert crs_value to integer if possible:
            try:
                crs_int = int(float(crs_value))
            except:
                crs_int = crs_value
            crs_combined = f"{crs_key}: {crs_int}"
            crs_set.add(crs_combined)
    return sorted(crs_set)

# Revised filter_results: Allow missing end_year or CRS; enforce CRS only when present.
def filter_results(results, country_filter, region_filter, end_year_range, crs_filter):
    filtered = []
    for r in results:
        metadata = r.payload.get('metadata', {})
        countries = metadata.get('countries', "[]")
        year_str = metadata.get('end_year')
        if year_str:
            extracted = extract_year(year_str)
            try:
                end_year_val = int(extracted) if extracted != "Unknown" else 0
            except ValueError:
                end_year_val = 0
        else:
            end_year_val = 0

        try:
            c_list = json.loads(countries.replace("'", '"'))
            c_list = [code.upper() for code in c_list if len(code) == 2]
        except json.JSONDecodeError:
            c_list = []

        selected_iso_code = country_name_mapping.get(country_filter, None)
        if region_filter != "All/Not allocated":
            countries_in_region = [code for code in c_list if iso_code_to_sub_region.get(code) == region_filter]
        else:
            countries_in_region = c_list

        crs_key = metadata.get("crs_key", "").strip()
        crs_value = metadata.get("crs_value", "").strip()
        try:
            crs_int = int(float(crs_value))
        except:
            crs_int = crs_value
        crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else ""
        # Only enforce CRS filter if result has a CRS value.
        if crs_filter != "All/Not allocated" and crs_combined:
            if crs_filter != crs_combined:
                continue

        # Allow projects with no valid end_year to pass (if end_year_val is 0)
        year_ok = True if end_year_val == 0 else (end_year_range[0] <= end_year_val <= end_year_range[1])

        if ((country_filter == "All/Not allocated" or (selected_iso_code and selected_iso_code in c_list))
            and (region_filter == "All/Not allocated" or countries_in_region)
            and year_ok):
            filtered.append(r)
    return filtered

# Get the device to be used (GPU or CPU)
device = 'cuda' if cuda.is_available() else 'cpu'

st.set_page_config(page_title="SEARCH IATI", layout='wide')
st.title("GIZ Project Database (PROTOTYPE)")
var = st.text_input("Enter Search Question")


# Load the region lookup CSV
region_lookup_path = "docStore/regions_lookup.csv"
region_df = load_region_data(region_lookup_path)

####################  Create the embeddings collection and save ######################
# the steps below need to be performed only once and then commented out any unnecssary compute over-run
##### First we process and create the chunks for relvant data source
#chunks = process_giz_worldwide()
##### Convert to langchain documents
#temp_doc = create_documents(chunks,'chunks')
##### Embed and store docs, check if collection exist then you need to update the collection
collection_name = "giz_worldwide"
#hybrid_embed_chunks(docs=temp_doc, collection_name=collection_name, del_if_exists=True)

################### Hybrid Search #####################################################
client = get_client()
print(client.get_collections())
max_end_year = get_max_end_year(client, collection_name)
_, unique_sub_regions = get_regions(region_df)

@st.cache_data
def get_country_name_and_region_mapping(_client, collection_name, region_df):
    results = hybrid_search(_client, "", collection_name)
    country_set = set()
    for res in results[0] + results[1]:
        countries = res.payload.get('metadata', {}).get('countries', "[]")
        try:
            country_list = json.loads(countries.replace("'", '"'))
            two_digit_codes = [code.upper() for code in country_list if len(code) == 2]
            country_set.update(two_digit_codes)
        except json.JSONDecodeError:
            pass
    country_name_to_code = {}
    iso_code_to_sub_region = {}
    for code in country_set:
        name = get_country_name(code, region_df)
        sub_region_row = region_df[region_df['alpha-2'] == code]
        sub_region = sub_region_row['sub-region'].values[0] if not sub_region_row.empty else "Not allocated"
        country_name_to_code[name] = code
        iso_code_to_sub_region[code] = sub_region
    return country_name_to_code, iso_code_to_sub_region

client = get_client()
country_name_mapping, iso_code_to_sub_region = get_country_name_and_region_mapping(client, collection_name, region_df)
unique_country_names = sorted(country_name_mapping.keys())

# Layout filters in columns
col1, col2, col3, col4 = st.columns([1, 1, 1, 1])
with col1:
    region_filter = st.selectbox("Region", ["All/Not allocated"] + sorted(unique_sub_regions))
# Compute filtered_country_names based on region_filter:
if region_filter == "All/Not allocated":
    filtered_country_names = unique_country_names
else:
    filtered_country_names = [name for name, code in country_name_mapping.items() if iso_code_to_sub_region.get(code) == region_filter]
with col2:
    country_filter = st.selectbox("Country", ["All/Not allocated"] + filtered_country_names)
with col3:
    current_year = datetime.now().year
    default_start_year = current_year - 4
    end_year_range = st.slider("Project End Year", min_value=2010, max_value=max_end_year, value=(default_start_year, max_end_year))
with col4:
    crs_options = ["All/Not allocated"] + get_crs_options(client, collection_name)
    crs_filter = st.selectbox("CRS", crs_options)

# Checkbox for exact matches
show_exact_matches = st.checkbox("Show only exact matches", value=False)

# Run the search
results = hybrid_search(client, var, collection_name, limit=500)
semantic_all = results[0]
lexical_all = results[1]

semantic_all = [r for r in semantic_all if len(r.payload["page_content"]) >= 5]
lexical_all = [r for r in lexical_all if len(r.payload["page_content"]) >= 5]

semantic_thresholded = [r for r in semantic_all if r.score >= 0.0]

filtered_semantic = filter_results(semantic_thresholded, country_filter, region_filter, end_year_range, crs_filter)
filtered_lexical = filter_results(lexical_all, country_filter, region_filter, end_year_range, crs_filter)

filtered_semantic_no_dupe = remove_duplicates(filtered_semantic)
filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)

def format_currency(value):
    try:
        return f"€{int(float(value)):,}"
    except (ValueError, TypeError):
        return value

# Helper to highlight query matches (case-insensitive)
def highlight_query(text, query):
    pattern = re.compile(re.escape(query), re.IGNORECASE)
    return pattern.sub(lambda m: f"**{m.group(0)}**", text)

###############################
# Display Lexical Results Branch
###############################
if show_exact_matches:
    st.write(f"Showing **Top 15 Lexical Search results** for query: {var}")
    query_substring = var.strip().lower()
    lexical_substring_filtered = [r for r in lexical_all if query_substring in r.payload["page_content"].lower()]
    filtered_lexical = filter_results(lexical_substring_filtered, country_filter, region_filter, end_year_range, crs_filter)
    filtered_lexical_no_dupe = remove_duplicates(filtered_lexical)
    if not filtered_lexical_no_dupe:
        st.write('No exact matches, consider unchecking "Show only exact matches"')
    else:
        top_results = filtered_lexical_no_dupe[:10]
        rag_answer = get_rag_answer(var, top_results)
        st.markdown(f"**{var}**")
        st.write(rag_answer)
        st.divider()
        for res in top_results:
            metadata = res.payload.get('metadata', {})
            if "title" not in metadata:
                metadata["title"] = compute_title(metadata)
            # Highlight query in red and bold (allow HTML)
            display_title = (
                st.markdown(highlight_query(metadata["title"], var), unsafe_allow_html=True)
                if var.strip() else metadata["title"]
            )
            proj_id = metadata.get('id', 'Unknown')
            st.markdown(f"#### {metadata['title']}")
            countries = metadata.get('countries')
            client_name = metadata.get('client', 'Unknown Client')
            start_year = metadata.get('start_year', None)
            end_year = metadata.get('end_year', None)
            total_volume = metadata.get('total_volume', "Unknown")
            total_project = metadata.get('total_project', "Unknown")
            objectives = metadata.get("objectives", "")
            desc_de = metadata.get("description.de", "")
            desc_en = metadata.get("description.en", "")
            description = desc_de if desc_de else desc_en
            full_snippet = f"{objectives} {description}"
            words = full_snippet.split()
            preview_word_count = 90
            preview_text = " ".join(words[:preview_word_count])
            remainder_text = " ".join(words[preview_word_count:])
            preview_text = highlight_query(preview_text, var) if var.strip() else preview_text
            st.write(preview_text)
            if remainder_text:
                with st.expander("Show more"):
                    st.write(remainder_text)
            
            full_text = res.payload['page_content']
            top_keywords = extract_top_keywords(full_text, top_n=5)
            if top_keywords:
                st.markdown(f"_{' · '.join(top_keywords)}_")
            
            # Format year range and budget info
            start_year_str = extract_year(start_year) if start_year else "Unknown"
            end_year_str = extract_year(end_year) if end_year else "Unknown"
            formatted_project_budget = format_currency(total_project)
            formatted_total_volume = format_currency(total_volume)
            
            # Compute matched country names (as before)
            try:
                c_list = json.loads(metadata.get('countries', "[]").replace("'", '"'))
            except json.JSONDecodeError:
                c_list = []
            matched_countries = []
            for code in c_list:
                if len(code) == 2:
                    resolved_name = get_country_name(code.upper(), region_df)
                    if resolved_name.upper() != code.upper():
                        matched_countries.append(resolved_name)
            
            # Compute CRS combined value as integer
            crs_key = metadata.get("crs_key", "").strip()
            crs_value = metadata.get("crs_value", "").strip()
            try:
                crs_int = int(float(crs_value))
            except:
                crs_int = crs_value
            crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else "Unknown"
            
            # Build the additional text with original details, then add Sector and contact.
            additional_text = (
                f"Commissioned by **{client_name}**\n"
                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
                f"Budget: Project: **{formatted_project_budget}**, Total volume: **{formatted_total_volume}**\n"
                f"Country: **{', '.join(matched_countries)}**\n"
                f"Sector: **{crs_combined}**"
            )
            #contact = metadata.get("contact", "").strip()
            #if contact and contact.lower() != "[email protected]":
            #    additional_text += f" | Contact: **{contact}**"
            
            st.markdown(additional_text)
            st.divider()

###############################
# Display Semantic Results Branch
###############################
else:
    if not filtered_semantic_no_dupe:
        st.write("No relevant results found.")
    else:
        top_results = filtered_semantic_no_dupe[:10]
        rag_answer = get_rag_answer(var, top_results)
        st.markdown(f"**{var}**")
        st.write(rag_answer)
        st.divider()
        st.write(f"Showing **Top 15 Semantic Search results** for query: {var}")
        for res in top_results:
            metadata = res.payload.get('metadata', {})
            if "title" not in metadata:
                metadata["title"] = compute_title(metadata)
            display_title = metadata["title"]
            st.markdown(f"#### {display_title}")
            countries = metadata.get('countries')
            client_name = metadata.get('client', 'Unknown Client')
            start_year = metadata.get('start_year', None)
            end_year = metadata.get('end_year', None)
            total_volume = metadata.get('total_volume', "Unknown")
            total_project = metadata.get('total_project', "Unknown")
            objectives = metadata.get("objectives", "")
            desc_de = metadata.get("description.de", "")
            desc_en = metadata.get("description.en", "")
            description = desc_de if desc_de else desc_en
            full_snippet = f"{objectives} {description}"
            words = full_snippet.split()
            preview_word_count = 90
            preview_text = " ".join(words[:preview_word_count])
            remainder_text = " ".join(words[preview_word_count:])
            st.write(preview_text)
            if remainder_text:
                with st.expander("Show more"):
                    st.write(remainder_text)
            
            top_keywords = extract_top_keywords(res.payload['page_content'], top_n=5)
            if top_keywords:
                st.markdown(f"_{' · '.join(top_keywords)}_")
            
            # Format year range and budget info
            start_year_str = extract_year(start_year) if start_year else "Unknown"
            end_year_str = extract_year(end_year) if end_year else "Unknown"
            formatted_project_budget = format_currency(total_project)
            formatted_total_volume = format_currency(total_volume)
            
            # Compute matched country names (as before)
            try:
                c_list = json.loads(metadata.get('countries', "[]").replace("'", '"'))
            except json.JSONDecodeError:
                c_list = []
            matched_countries = []
            for code in c_list:
                if len(code) == 2:
                    resolved_name = get_country_name(code.upper(), region_df)
                    if resolved_name.upper() != code.upper():
                        matched_countries.append(resolved_name)
            
            # Compute CRS combined value
            crs_key = metadata.get("crs_key", "").strip()
            crs_value = metadata.get("crs_value", "").strip()
            try:
                crs_int = int(float(crs_value))
            except:
                crs_int = crs_value
            crs_combined = f"{crs_key}: {crs_int}" if (crs_key or crs_value) else "Unknown"
            
            # Build the additional text with original details, then add Sector and contact.
            additional_text = (
                f"Commissioned by **{client_name}**\n"
                f"Projekt duration **{start_year_str}-{end_year_str}**\n"
                f"Budget: Project: **{formatted_project_budget}**, Total volume: **{formatted_total_volume}**\n"
                f"Country: **{', '.join(matched_countries)}**\n"
                f"Sector: **{crs_combined}**"
            )
            #contact = metadata.get("contact", "").strip()
            #if contact and contact.lower() != "[email protected]":
            #    additional_text += f" | Contact: **{contact}**"
            
            st.markdown(additional_text)
            st.divider()                
    #  for i in results: 
    #      st.subheader(str(i.metadata['id'])+":"+str(i.metadata['title_main']))
    #      st.caption(f"Status:{str(i.metadata['status'])}, Country:{str(i.metadata['country_name'])}")
    #      st.write(i.page_content)
    #      st.divider()