v.1.20

app.py

@@ -399,7 +399,7 @@ def create_interface():
     control = ProcessControl()
     
     with gr.Blocks(theme=gr.themes.Soft()) as app:
-        gr.Markdown("# AI-анализ мониторинга новостей v.1.17")
+        gr.Markdown("# AI-анализ мониторинга новостей v.1.20")
         
         with gr.Row():
             file_input = gr.File(
@@ -445,7 +445,8 @@ def create_interface():
         def stop_processing():
             control.request_stop()
             return "Остановка обработки..."
-                
+        
+        @spaces.GPU(duration=300)  # 5 minutes duration for the entire analysis
         def analyze(file_bytes):
             if file_bytes is None:
                 gr.Warning("Пожалуйста, загрузите файл")
@@ -458,75 +459,87 @@ def create_interface():
                 file_obj = io.BytesIO(file_bytes)
                 logger.info("File loaded into BytesIO successfully")
                 
-                progress_status = "Начинаем обработку файла..."
-                yield None, None, None, progress_status
+                detector = EventDetector()
                 
-                # Process file
-                df = pd.read_excel(file_obj, sheet_name='Публикации')
-                logger.info(f"Successfully read Excel file. Shape: {df.shape}")
+                # Initialize models with GPU
+                @spaces.GPU(duration=30)
+                def init_models():
+                    return detector.initialize_models()
+                
+                if not init_models():
+                    raise Exception("Failed to initialize models")
+                
+                # Process in batches with GPU allocation
+                @spaces.GPU(duration=20)
+                def process_batch(batch, entity):
+                    event_type, event_summary = detector.detect_events(batch, entity)
+                    time.sleep(1)  # Wait between GPU operations
+                    sentiment = detector.analyze_sentiment(batch)
+                    return event_type, event_summary, sentiment
                 
-                # Deduplication
+                # Read and deduplicate data
+                df = pd.read_excel(file_obj, sheet_name='Публикации')
                 original_count = len(df)
                 df = fuzzy_deduplicate(df, 'Выдержки из текста', threshold=55)
                 logger.info(f"Removed {original_count - len(df)} duplicate entries")
                 
-                detector = EventDetector()
-                detector.control = control  # Pass control object
                 processed_rows = []
                 total = len(df)
+                batch_size = 3
                 
-                # Initialize models
-                if not detector.initialize_models():
-                    raise Exception("Failed to initialize models")
-                
-                for idx, row in df.iterrows():
+                for batch_start in range(0, total, batch_size):
                     if control.should_stop():
-                        yield (
-                            pd.DataFrame(processed_rows) if processed_rows else None,
-                            None, None,
-                            f"Обработка остановлена. Обработано {idx} из {total} строк"
-                        )
-                        return
+                        break
+                        
+                    batch_end = min(batch_start + batch_size, total)
+                    batch = df.iloc[batch_start:batch_end]
                     
-                    try:
-                        text = str(row.get('Выдержки из текста', ''))
-                        if not text.strip():
-                            continue
+                    for idx, row in batch.iterrows():
+                        try:
+                            text = str(row.get('Выдержки из текста', '')).strip()
+                            entity = str(row.get('Объект', '')).strip()
                             
-                        entity = str(row.get('Объект', ''))
-                        if not entity.strip():
-                            continue
-                        
-                        event_type, event_summary = detector.detect_events(text, entity)
-                        sentiment = detector.analyze_sentiment(text)
-                        
-                        processed_rows.append({
-                            'Объект': entity,
-                            'Заголовок': str(row.get('Заголовок', '')),
-                            'Sentiment': sentiment,
-                            'Event_Type': event_type,
-                            'Event_Summary': event_summary,
-                            'Текст': text[:1000]
-                        })
-                        
-                        if idx % 5 == 0:
-                            progress_status = f"Обработано {idx + 1}/{total} строк"
-                            yield None, None, None, progress_status
+                            if not text or not entity:
+                                continue
                             
-                    except Exception as e:
-                        logger.error(f"Error processing row {idx}: {str(e)}")
-                        continue
-                
-                result_df = pd.DataFrame(processed_rows)
-                fig_sentiment, fig_events = create_visualizations(result_df)
-                
-                return (
-                    result_df,
-                    fig_sentiment,
-                    fig_events,
-                    f"Обработка завершена успешно! Обработано {len(result_df)} строк"
-                )
+                            # Process with GPU
+                            event_type, event_summary, sentiment = process_batch(text, entity)
+                            
+                            processed_rows.append({
+                                'Объект': entity,
+                                'Заголовок': str(row.get('Заголовок', '')),
+                                'Sentiment': sentiment,
+                                'Event_Type': event_type,
+                                'Event_Summary': event_summary,
+                                'Текст': text[:1000]
+                            })
+                            
+                        except Exception as e:
+                            logger.error(f"Error processing row {idx}: {str(e)}")
+                            continue
+                    
+                    # Create intermediate results
+                    if processed_rows:
+                        result_df = pd.DataFrame(processed_rows)
+                        fig_sentiment, fig_events = create_visualizations(result_df)
+                        yield (
+                            result_df,
+                            fig_sentiment,
+                            fig_events,
+                            f"Обработано {len(processed_rows)}/{total} строк"
+                        )
+                    
+                    # Cleanup GPU resources after batch
+                    torch.cuda.empty_cache()
+                    time.sleep(2)
                 
+                if processed_rows:
+                    final_df = pd.DataFrame(processed_rows)
+                    fig_sentiment, fig_events = create_visualizations(final_df)
+                    return final_df, fig_sentiment, fig_events, "Обработка завершена!"
+                else:
+                    return None, None, None, "Нет обработанных данных"
+                    
             except Exception as e:
                 error_msg = f"Ошибка анализа: {str(e)}"
                 logger.error(error_msg)
@@ -544,4 +557,4 @@ def create_interface():
 
 if __name__ == "__main__":
     app = create_interface()
-    app.launch(share=True)
+    app.launch(share=True)