Update appStore/prep_data.py

appStore/prep_data.py

@@ -33,63 +33,64 @@ def process_iati():
     return projects_df
 
 
-# def process_giz_worldwide():
-#     """
-#     This will read the new giz_worldwide file and create the chunks.
-#     The following adjustments have been made:
-#       - Reads the file 'giz_worldwide_api_download_23_02_2025.json'
-#       - Renames 'name.en' to 'project_name'
-#       - Uses the 'merged_text' column for creating chunks and computing text size
-#       - Creates an empty 'url' column (since the new dataset has an empty URL)
-#       - Renames 'country' to 'countries'
-#       - Renames 'duration.project.start' to 'start_year' and 'duration.project.end' to 'end_year'
-#     """
-#     # Read the new JSON file
-#     giz_df = pd.read_json(f'{path_to_data}giz_worldwide/giz_worldwide_api_download_23_02_2025.json')
+def process_giz_worldwide():
+    """
+    This will read the new giz_worldwide file and create the chunks.
+    The following adjustments have been made:
+      - Reads the file 'giz_worldwide_api_download_23_02_2025.json'
+      - Renames 'name.en' to 'project_name'
+      - Uses the 'merged_text' column for creating chunks and computing text size
+      - Creates an empty 'url' column (since the new dataset has an empty URL)
+      - Renames 'country' to 'countries'
+      - Renames 'duration.project.start' to 'start_year' and 'duration.project.end' to 'end_year'
+    """
+    # Read the new JSON file
+    giz_df = pd.read_json(f'{path_to_data}giz_worldwide/giz_worldwide_api_download_23_02_2025.json')
+    # Sample random rows for quick embeddings (seed set for reproducibility)
+    giz_df = giz_df.sample(n=5, random_state=42)  
+    # Rename columns per new dataset requirements
+    giz_df = giz_df.rename(columns={
+        'name.en': 'project_name',
+        'country': 'countries',
+        'duration.project.start': 'start_year',
+        'duration.project.end': 'end_year'
+    })
     
-#     # Rename columns per new dataset requirements
-#     giz_df = giz_df.rename(columns={
-#         'name.en': 'project_name',
-#         'country': 'countries',
-#         'duration.project.start': 'start_year',
-#         'duration.project.end': 'end_year'
-#     })
+    # Create an empty 'url' column as the new dataset has an empty URL
+    giz_df['url'] = ''
     
-#     # Create an empty 'url' column as the new dataset has an empty URL
-#     giz_df['url'] = ''
+    # Create text_size based on merged_text and create chunks from merged_text
+    giz_df['text_size'] = giz_df['merged_text'].apply(lambda text: len(text.split()) if isinstance(text, str) else 0)
+    giz_df['chunks'] = giz_df['merged_text'].apply(lambda text: create_chunks(text) if isinstance(text, str) else [])
     
-#     # Create text_size based on merged_text and create chunks from merged_text
-#     giz_df['text_size'] = giz_df['merged_text'].apply(lambda text: len(text.split()) if isinstance(text, str) else 0)
-#     giz_df['chunks'] = giz_df['merged_text'].apply(lambda text: create_chunks(text) if isinstance(text, str) else [])
+    print("initial df length:", len(giz_df))
+    giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
+    print("new df length:", len(giz_df))
+    print(giz_df.columns)
     
-#     print("initial df length:", len(giz_df))
+    giz_df['source'] = 'GIZ_WORLDWIDE'
+    return giz_df
+
+
+# def process_giz_worldwide():
+#     """
+#     this will read the giz_worldwide files and create the chunks
+#     """
+#     giz_df = pd.read_json(f'{path_to_data}giz_worldwide/data_giz_website.json')
+#     giz_df = giz_df.rename(columns={'content':'project_description'})
+#     # Sample random rows for quick embeddings (seed set for reproducibility)
+#     giz_df = giz_df.sample(n=5, random_state=42)    
+#     giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
+#     giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
+#     print("initial df length:",len(giz_df))
 #     giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
-#     print("new df length:", len(giz_df))
+#     print("new df length:",len(giz_df))
 #     print(giz_df.columns)
-    
+#     #giz_df.drop(columns = ['filename', 'url', 'name', 'mail', 
+#     #                    'language', 'start_year', 'end_year','poli_trager'], inplace=True)
 #     giz_df['source'] = 'GIZ_WORLDWIDE'
 #     return giz_df
 
-
-def process_giz_worldwide():
-    """
-    this will read the giz_worldwide files and create the chunks
-    """
-    giz_df = pd.read_json(f'{path_to_data}giz_worldwide/data_giz_website.json')
-    giz_df = giz_df.rename(columns={'content':'project_description'})
-    # Sample 10 random rows for quick embeddings (seed set for reproducibility)
-    giz_df = giz_df.sample(n=5, random_state=42)    
-    giz_df['text_size'] = giz_df.apply(lambda x: len((x['project_name'] + x['project_description']).split()), axis=1)
-    giz_df['chunks'] = giz_df.apply(lambda x:create_chunks(x['project_name'] + x['project_description']),axis=1)
-    print("initial df length:",len(giz_df))
-    giz_df = giz_df.explode(column=['chunks'], ignore_index=True)
-    print("new df length:",len(giz_df))
-    print(giz_df.columns)
-    #giz_df.drop(columns = ['filename', 'url', 'name', 'mail', 
-    #                    'language', 'start_year', 'end_year','poli_trager'], inplace=True)
-    giz_df['source'] = 'GIZ_WORLDWIDE'
-    return giz_df
-
 def remove_duplicates(results_list):
     """
     Return a new list of results with duplicates removed,