1.12

app.py

@@ -321,7 +321,7 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
     return f'<a href="data:application/vnd.openxmlformats-officedocument.spreadsheetml.sheet;base64,{b64}" download="{filename}">Download {filename}</a>'
 
 def main():
-    st.title("кластеризуем новости v.1.11")
+    st.title("кластеризуем новости v.1.12")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])
@@ -356,20 +356,21 @@ def main():
                 try:
                     progress_bar = st.progress(0)
                     
+                    # Step 1: Deduplicate
                     deduplicator = NewsDeduplicator(fuzzy_threshold)
                     dedup_df = deduplicator.deduplicate(df, progress_bar)
                     st.success(f"Removed {len(df) - len(dedup_df)} duplicates")
                     
-                    st.write("Sample of deduplicated data:")
-                    st.dataframe(dedup_df[['datetime', 'company', 'text', 'company_count', 'duplicate_count']].head())
+                    # Preserve all columns from original DataFrame in dedup_df
+                    dedup_df = df_original.loc[dedup_df.index].copy()
                     
-                    processor = NewsProcessor(similarity_threshold, time_threshold)
-                    result_df = processor.process_news(dedup_df, progress_bar)
+                    # Create working copy of dedup_df with required columns for clustering
+                    working_df = dedup_df[[company_column, datetime_column, title_column, text_column]].copy()
+                    working_df.columns = ['company', 'datetime', 'title', 'text']
                     
-                    # Create a mapping between original indices and cluster information
-                    index_to_cluster = pd.Series(0, index=df_original.index)  # Default cluster 0 for non-clustered rows
-                    for idx, row in result_df.iterrows():
-                        index_to_cluster[idx] = row['cluster_id']
+                    # Step 2: Cluster deduplicated news
+                    processor = NewsProcessor(similarity_threshold, time_threshold)
+                    result_df = processor.process_news(working_df, progress_bar)
                     
                     # Initialize list of indices to keep
                     indices_to_keep = []
@@ -382,26 +383,27 @@ def main():
                             # For clusters with multiple items, keep only the one with longest text
                             cluster_rows = result_df[result_df['cluster_id'] == cluster_id]
                             cluster_indices = cluster_rows.index
-                            text_lengths = df_original.iloc[cluster_indices][text_column].str.len()
+                            text_lengths = dedup_df.iloc[cluster_indices][text_column].str.len()
                             longest_text_idx = cluster_indices[text_lengths.argmax()]
                             indices_to_keep.append(longest_text_idx)
                         else:
                             # For single-item clusters, keep the item
                             indices_to_keep.extend(result_df[result_df['cluster_id'] == cluster_id].index)
                     
-                    # Add all non-clustered rows (cluster_id = 0)
-                    non_clustered_indices = df_original.index[~df_original.index.isin(result_df.index)]
+                    # Add all non-clustered rows from dedup_df
+                    non_clustered_indices = dedup_df.index[~dedup_df.index.isin(result_df.index)]
                     indices_to_keep.extend(non_clustered_indices)
                     
-                    # Create final declustered DataFrame
-                    declustered_df = df_original.iloc[sorted(indices_to_keep)].copy()
+                    # Create final declustered DataFrame from dedup_df
+                    declustered_df = dedup_df.iloc[sorted(indices_to_keep)].copy()
                     
                     st.success(f"""
                         Processing results:
                         - Original rows: {len(df_original)}
+                        - After deduplication: {len(dedup_df)}
                         - Rows in clusters: {len(result_df)}
                         - Multi-item clusters: {len(result_df[result_df['cluster_size'] > 1]['cluster_id'].unique())}
-                        - Rows kept after declustering: {len(declustered_df)}
+                        - Final rows after declustering: {len(declustered_df)}
                     """)
                     
                     # Download buttons for all results