1.16

app.py

@@ -322,7 +322,7 @@ def create_download_link(df: pd.DataFrame, filename: str) -> str:
 
 
 def main():
-    st.title("кластеризуем новости v.1.15")
+    st.title("кластеризуем новости v.1.16")
     st.write("Upload Excel file with columns: company, datetime, text")
     
     uploaded_file = st.file_uploader("Choose Excel file", type=['xlsx'])
@@ -362,69 +362,58 @@ def main():
                     dedup_df = deduplicator.deduplicate(df, progress_bar)
                     st.success(f"Removed {len(df) - len(dedup_df)} duplicates")
                     
-                    # Get working copy of deduplicated data with all columns
+                    # Preserve all columns from original DataFrame in dedup_df
                     dedup_df_full = df_original.loc[dedup_df.index].copy()
                     
-                    # Create working copy for clustering with required columns
-                    working_df = dedup_df_full[[company_column, datetime_column, title_column, text_column]].copy()
-                    working_df.columns = ['company', 'datetime', 'title', 'text']
-                    
                     # Step 2: Cluster deduplicated news
                     processor = NewsProcessor(similarity_threshold, time_threshold)
-                    result_df = processor.process_news(working_df, progress_bar)
+                    result_df = processor.process_news(dedup_df, progress_bar)
                     
-                    # Create a dictionary to store indices to delete for each cluster
+                    # Initialize set of indices to delete
                     indices_to_delete = set()
                     
-                    # Process clusters with more than 1 member
-                    if not result_df.empty:
-                        for cluster_id in result_df['cluster_id'].unique():
-                            cluster_rows = result_df[result_df['cluster_id'] == cluster_id]
+                    # Find rows to delete from multi-item clusters
+                    if len(result_df) > 0:
+                        # Get all multi-item clusters
+                        multi_clusters = result_df[result_df['cluster_size'] > 1]['cluster_id'].unique()
+                        
+                        # For each multi-item cluster
+                        for cluster_id in multi_clusters:
+                            # Get indices of all rows in this cluster
+                            cluster_indices = result_df[result_df['cluster_id'] == cluster_id].index.tolist()
+                            
+                            # Get their text lengths
+                            text_lengths = dedup_df_full.loc[cluster_indices, text_column].fillna('').str.len()
                             
-                            if len(cluster_rows) > 1:  # Only process multi-member clusters
-                                # Get indices of all rows in this cluster
-                                cluster_indices = cluster_rows.index
-                                
-                                # Find the row with the longest text
-                                text_lengths = working_df.loc[cluster_indices, 'text'].str.len()
-                                longest_text_idx = text_lengths.idxmax()
-                                
-                                # Add all other indices from this cluster to deletion set
-                                cluster_indices_to_delete = set(cluster_indices) - {longest_text_idx}
-                                indices_to_delete.update(cluster_indices_to_delete)
+                            # Find index with longest text
+                            longest_text_idx = text_lengths.idxmax()
+                            
+                            # Add all other indices to delete set
+                            indices_to_delete.update(set(cluster_indices) - {longest_text_idx})
                     
-                    # Create final declustered DataFrame by dropping the identified rows
+                    # Create final declustered DataFrame by removing identified rows
                     declustered_df = dedup_df_full.copy()
                     if indices_to_delete:
                         declustered_df = declustered_df.drop(index=list(indices_to_delete))
                     
-                    # Print statistics for verification
+                    # Print statistics
                     st.success(f"""
                         Processing results:
                         - Original rows: {len(df_original)}
                         - After deduplication: {len(dedup_df_full)}
-                        - Rows in clusters (any size): {len(result_df) if not result_df.empty else 0}
-                        - Rows removed from multi-member clusters: {len(indices_to_delete)}
-                        - Final rows after declustering: {len(declustered_df)}
+                        - Multi-item clusters found: {len(multi_clusters) if len(result_df) > 0 else 0}
+                        - Rows removed from clusters: {len(indices_to_delete)}
+                        - Final rows kept: {len(declustered_df)}
                     """)
                     
-                    # Add debugging information
-                    if not result_df.empty:
-                        multi_clusters = len(result_df[result_df['cluster_size'] > 1]['cluster_id'].unique())
-                        st.write(f"Number of multi-member clusters found: {multi_clusters}")
-                        
-                        # Show cluster sizes
-                        cluster_sizes = result_df['cluster_size'].value_counts().sort_index()
-                        st.write("Cluster size distribution:")
-                        st.write(cluster_sizes)
-                    
                     # Download buttons for all results
                     st.subheader("Download Results")
                     st.markdown(create_download_link(dedup_df_full, "deduplicated_news.xlsx"), unsafe_allow_html=True)
+                    st.markdown(create_download_link(result_df, "clustered_news.xlsx"), unsafe_allow_html=True)
                     st.markdown(create_download_link(declustered_df, "declustered_news.xlsx"), unsafe_allow_html=True)
                     
-                    # Show cluster statistics if clusters were found
-                    if not result_df.empty:
+                    # Show clusters info
+                    if len(result_df) > 0:
                         st.subheader("Largest Clusters")
                         largest_clusters = result_df[result_df['cluster_size'] > 1].sort_values(
                             ['cluster_size', 'cluster_id', 'datetime'],