Deployment from Replit

components/__init__.py

@@ -0,0 +1 @@
+# This file is intentionally left empty to make the directory a Python package

components/alerts.py

@@ -0,0 +1,554 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+from datetime import datetime, timedelta
+
+def render_alerts():
+    st.title("Alert Management")
+    
+    # Alert Overview
+    st.subheader("Alert Overview")
+    
+    # Alert metrics
+    col1, col2, col3, col4, col5 = st.columns(5)
+    
+    with col1:
+        st.metric(
+            label="Active Alerts",
+            value="27",
+            delta="4",
+            delta_color="inverse"
+        )
+    
+    with col2:
+        st.metric(
+            label="Critical",
+            value="8",
+            delta="2",
+            delta_color="inverse"
+        )
+    
+    with col3:
+        st.metric(
+            label="High",
+            value="12",
+            delta="3",
+            delta_color="inverse"
+        )
+    
+    with col4:
+        st.metric(
+            label="Medium",
+            value="5",
+            delta="-1",
+            delta_color="normal"
+        )
+    
+    with col5:
+        st.metric(
+            label="Low",
+            value="2",
+            delta="0",
+            delta_color="normal"
+        )
+    
+    # Filters for alerts
+    with st.container():
+        st.markdown("### Alert Filters")
+        
+        filter_col1, filter_col2, filter_col3, filter_col4 = st.columns(4)
+        
+        with filter_col1:
+            severity_filter = st.multiselect(
+                "Severity",
+                ["Critical", "High", "Medium", "Low"],
+                default=["Critical", "High", "Medium", "Low"]
+            )
+        
+        with filter_col2:
+            status_filter = st.multiselect(
+                "Status",
+                ["New", "In Progress", "Resolved", "False Positive"],
+                default=["New", "In Progress"]
+            )
+        
+        with filter_col3:
+            date_range = st.selectbox(
+                "Time Range",
+                ["Last 24 Hours", "Last 7 Days", "Last 30 Days", "Custom Range"],
+                index=1
+            )
+        
+        with filter_col4:
+            category_filter = st.multiselect(
+                "Category",
+                ["Data Breach", "Ransomware", "Credentials", "PII", "Brand Abuse", "Source Code", "Other"],
+                default=["Data Breach", "Credentials", "PII"]
+            )
+    
+    # Alert list
+    st.markdown("### Active Alerts")
+    
+    # Sample alert data
+    alerts = [
+        {
+            "id": "ALERT-2025-04081",
+            "timestamp": "2025-04-08 14:32:21",
+            "severity": "Critical",
+            "category": "Data Breach",
+            "description": "Patient records from Memorial Hospital found on dark web marketplace.",
+            "status": "New",
+            "source": "AlphaBay Market"
+        },
+        {
+            "id": "ALERT-2025-04082",
+            "timestamp": "2025-04-08 10:15:43",
+            "severity": "Critical",
+            "category": "Ransomware",
+            "description": "Company mentioned in ransomware group's leak site as new victim.",
+            "status": "New",
+            "source": "BlackCat Leak Site"
+        },
+        {
+            "id": "ALERT-2025-04083",
+            "timestamp": "2025-04-08 08:42:19",
+            "severity": "High",
+            "category": "Credentials",
+            "description": "123 employee credentials found in new breach compilation.",
+            "status": "In Progress",
+            "source": "BreachForums"
+        },
+        {
+            "id": "ALERT-2025-04071",
+            "timestamp": "2025-04-07 22:03:12",
+            "severity": "High",
+            "category": "PII",
+            "description": "Customer PII being offered for sale on hacking forum.",
+            "status": "In Progress",
+            "source": "XSS Forum"
+        },
+        {
+            "id": "ALERT-2025-04072",
+            "timestamp": "2025-04-07 18:37:56",
+            "severity": "Medium",
+            "category": "Brand Abuse",
+            "description": "Phishing campaign using company brand assets detected.",
+            "status": "New",
+            "source": "Telegram Channel"
+        },
+        {
+            "id": "ALERT-2025-04073",
+            "timestamp": "2025-04-07 14:21:08",
+            "severity": "Medium",
+            "category": "Source Code",
+            "description": "Fragments of internal source code shared in paste site.",
+            "status": "In Progress",
+            "source": "DeepPaste"
+        },
+        {
+            "id": "ALERT-2025-04063",
+            "timestamp": "2025-04-06 20:15:37",
+            "severity": "Low",
+            "category": "Credentials",
+            "description": "Legacy system credentials posted in hacking forum.",
+            "status": "New",
+            "source": "RaidForums"
+        }
+    ]
+    
+    # Create a dataframe for the alerts
+    alert_df = pd.DataFrame(alerts)
+    
+    # Apply colors to severity column
+    def color_severity(val):
+        color_map = {
+            'Critical': '#E74C3C',
+            'High': '#F1C40F',
+            'Medium': '#3498DB',
+            'Low': '#2ECC71'
+        }
+        return f'background-color: {color_map.get(val, "#ECF0F1")}'
+    
+    # Style the dataframe
+    styled_df = alert_df.style.applymap(color_severity, subset=['severity'])
+    
+    # Display the table
+    st.dataframe(styled_df, use_container_width=True, height=300)
+    
+    # Action buttons for alerts
+    action_col1, action_col2, action_col3, action_col4, action_col5 = st.columns(5)
+    
+    with action_col1:
+        st.button("Investigate", key="investigate_alert")
+    
+    with action_col2:
+        st.button("Mark as Resolved", key="resolve_alert")
+    
+    with action_col3:
+        st.button("Assign to Analyst", key="assign_alert")
+    
+    with action_col4:
+        st.button("Mark as False Positive", key="false_positive")
+    
+    with action_col5:
+        st.button("Generate Report", key="generate_report")
+    
+    # Alert visualization
+    st.markdown("### Alert Analytics")
+    
+    # Tabs for different alert visualizations
+    tab1, tab2, tab3 = st.tabs(["Alert Trend", "Category Distribution", "Source Analysis"])
+    
+    with tab1:
+        # Alert trend over time
+        st.subheader("Alert Trend (Last 30 Days)")
+        
+        # Generate dates for the past 30 days
+        dates = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(30, 0, -1)]
+        
+        # Sample data for alert trends
+        critical_alerts = np.random.randint(5, 12, 30)
+        high_alerts = np.random.randint(8, 20, 30)
+        medium_alerts = np.random.randint(12, 25, 30)
+        low_alerts = np.random.randint(15, 30, 30)
+        
+        trend_data = pd.DataFrame({
+            'Date': dates,
+            'Critical': critical_alerts,
+            'High': high_alerts,
+            'Medium': medium_alerts,
+            'Low': low_alerts
+        })
+        
+        # Create a stacked area chart
+        fig = go.Figure()
+        
+        fig.add_trace(go.Scatter(
+            x=trend_data['Date'], y=trend_data['Critical'],
+            mode='lines',
+            line=dict(width=0.5, color='#E74C3C'),
+            stackgroup='one',
+            name='Critical'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=trend_data['Date'], y=trend_data['High'],
+            mode='lines',
+            line=dict(width=0.5, color='#F1C40F'),
+            stackgroup='one',
+            name='High'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=trend_data['Date'], y=trend_data['Medium'],
+            mode='lines',
+            line=dict(width=0.5, color='#3498DB'),
+            stackgroup='one',
+            name='Medium'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=trend_data['Date'], y=trend_data['Low'],
+            mode='lines',
+            line=dict(width=0.5, color='#2ECC71'),
+            stackgroup='one',
+            name='Low'
+        ))
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=1.02,
+                xanchor="right",
+                x=1
+            ),
+            margin=dict(l=0, r=0, t=30, b=0),
+            xaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                title="Alert Count",
+                tickfont=dict(color='#ECF0F1')
+            ),
+            height=400
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with tab2:
+        # Alert distribution by category
+        st.subheader("Alert Category Distribution")
+        
+        # Sample data for categories
+        categories = ['Data Breach', 'Credentials', 'PII', 'Ransomware', 'Brand Abuse', 'Source Code', 'Infrastructure', 'Other']
+        counts = [35, 28, 18, 12, 8, 6, 4, 2]
+        
+        category_data = pd.DataFrame({
+            'Category': categories,
+            'Count': counts
+        })
+        
+        # Create a horizontal bar chart
+        fig = px.bar(
+            category_data,
+            y='Category',
+            x='Count',
+            orientation='h',
+            color='Count',
+            color_continuous_scale=['#2ECC71', '#3498DB', '#F1C40F', '#E74C3C'],
+            height=400
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            coloraxis_showscale=False,
+            xaxis=dict(
+                title="Number of Alerts",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title=None,
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            margin=dict(l=0, r=0, t=30, b=0)
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with tab3:
+        # Alert sources analysis
+        st.subheader("Alert Sources")
+        
+        # Sample data for sources
+        sources = ['Dark Web Markets', 'Hacking Forums', 'Paste Sites', 'Telegram Channels', 'Ransomware Blogs', 'IRC Channels', 'Social Media']
+        source_counts = [32, 27, 18, 15, 10, 7, 4]
+        
+        source_data = pd.DataFrame({
+            'Source': sources,
+            'Count': source_counts
+        })
+        
+        # Create a pie chart
+        fig = px.pie(
+            source_data,
+            values='Count',
+            names='Source',
+            hole=0.4,
+            color_discrete_sequence=['#E74C3C', '#F1C40F', '#3498DB', '#2ECC71', '#9B59B6', '#E67E22', '#1ABC9C']
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            showlegend=True,
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=-0.2,
+                xanchor="center",
+                x=0.5,
+                font=dict(color='#ECF0F1')
+            ),
+            margin=dict(l=0, r=0, t=30, b=0),
+            height=400
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    # Alert rules configuration
+    st.markdown("---")
+    st.subheader("Alert Rules Configuration")
+    
+    # Tabs for different rule categories
+    rule_tab1, rule_tab2 = st.tabs(["Active Rules", "Rule Editor"])
+    
+    with rule_tab1:
+        # Sample data for alert rules
+        alert_rules = pd.DataFrame({
+            "Rule Name": [
+                "Critical Data Breach Detection",
+                "Ransomware Victim Monitoring",
+                "Employee Credential Exposure",
+                "Source Code Leak Detection",
+                "Brand Impersonation Alert",
+                "Executive PII Monitoring",
+                "Infrastructure Exposure"
+            ],
+            "Category": ["Data Breach", "Ransomware", "Credentials", "Source Code", "Brand Abuse", "PII", "Infrastructure"],
+            "Severity": ["Critical", "Critical", "High", "High", "Medium", "Critical", "Medium"],
+            "Sources": ["All", "Leak Sites", "Paste Sites, Forums", "Paste Sites, Forums", "All", "All", "Forums, Markets"],
+            "Status": ["Active", "Active", "Active", "Active", "Active", "Active", "Active"]
+        })
+        
+        # Display rules table
+        st.dataframe(alert_rules, use_container_width=True)
+        
+        # Rule action buttons
+        rule_col1, rule_col2, rule_col3, rule_col4 = st.columns(4)
+        
+        with rule_col1:
+            st.button("Create New Rule", key="new_rule")
+        
+        with rule_col2:
+            st.button("Edit Selected", key="edit_rule")
+        
+        with rule_col3:
+            st.button("Duplicate", key="duplicate_rule")
+        
+        with rule_col4:
+            st.button("Disable", key="disable_rule")
+    
+    with rule_tab2:
+        # Rule editor form
+        with st.form("rule_editor"):
+            st.markdown("### Rule Editor")
+            
+            rule_name = st.text_input("Rule Name", value="New Alert Rule")
+            
+            editor_col1, editor_col2 = st.columns(2)
+            
+            with editor_col1:
+                rule_category = st.selectbox(
+                    "Category",
+                    ["Data Breach", "Ransomware", "Credentials", "PII", "Brand Abuse", "Source Code", "Infrastructure", "Other"]
+                )
+                
+                rule_severity = st.selectbox(
+                    "Severity",
+                    ["Critical", "High", "Medium", "Low"]
+                )
+            
+            with editor_col2:
+                rule_sources = st.multiselect(
+                    "Monitoring Sources",
+                    ["Dark Web Markets", "Hacking Forums", "Paste Sites", "Leak Sites", "Telegram Channels", "IRC Channels", "Social Media", "All"],
+                    default=["All"]
+                )
+                
+                rule_status = st.selectbox(
+                    "Status",
+                    ["Active", "Disabled"]
+                )
+            
+            st.markdown("### Rule Conditions")
+            
+            condition_type = st.selectbox(
+                "Condition Type",
+                ["Keyword Match", "Regular Expression", "Data Pattern", "Complex Query"]
+            )
+            
+            if condition_type == "Keyword Match":
+                keywords = st.text_area("Keywords (one per line)", height=100)
+                
+                keyword_options = st.columns(3)
+                with keyword_options[0]:
+                    case_sensitive = st.checkbox("Case Sensitive", value=False)
+                with keyword_options[1]:
+                    whole_word = st.checkbox("Whole Word Only", value=False)
+                with keyword_options[2]:
+                    proximity = st.checkbox("Proximity Search", value=False)
+            
+            elif condition_type == "Regular Expression":
+                regex_pattern = st.text_area("Regular Expression Pattern", height=100)
+                
+                regex_options = st.columns(2)
+                with regex_options[0]:
+                    test_regex = st.button("Test RegEx")
+                with regex_options[1]:
+                    validate_regex = st.button("Validate Pattern")
+            
+            elif condition_type == "Data Pattern":
+                data_patterns = st.multiselect(
+                    "Data Patterns to Detect",
+                    ["Email Addresses", "Credit Card Numbers", "Social Security Numbers", "Phone Numbers", "IP Addresses", "API Keys", "Passwords"]
+                )
+            
+            elif condition_type == "Complex Query":
+                complex_query = st.text_area("Complex Query", height=100, 
+                                            placeholder="Example: (keyword1 OR keyword2) AND (keyword3) NOT (keyword4)")
+            
+            st.markdown("### Response Actions")
+            
+            notification_channels = st.multiselect(
+                "Notification Channels",
+                ["Email", "Slack", "API Webhook", "SMS"],
+                default=["Email", "Slack"]
+            )
+            
+            auto_actions = st.multiselect(
+                "Automated Actions",
+                ["Create Incident Ticket", "Add to Watchlist", "Block in Firewall", "None"],
+                default=["Create Incident Ticket"]
+            )
+            
+            submit_rule = st.form_submit_button("Save Rule")
+            
+            if submit_rule:
+                st.success("Alert rule saved successfully!")
+    
+    # Alert notification settings
+    st.markdown("---")
+    st.subheader("Alert Notification Settings")
+    
+    # Notification channels
+    notif_col1, notif_col2 = st.columns(2)
+    
+    with notif_col1:
+        st.markdown("### Notification Channels")
+        
+        with st.container():
+            st.checkbox("Email Notifications", value=True)
+            st.text_input("Email Recipients", value="[email protected], [email protected]")
+            
+            st.checkbox("Slack Notifications", value=True)
+            st.text_input("Slack Channel", value="#security-alerts")
+            
+            st.checkbox("SMS Notifications", value=False)
+            st.text_input("Phone Numbers", placeholder="+1234567890, +0987654321")
+            
+            st.checkbox("API Webhook", value=False)
+            st.text_input("Webhook URL", placeholder="https://api.example.com/webhook")
+    
+    with notif_col2:
+        st.markdown("### Notification Schedule")
+        
+        with st.container():
+            notify_critical = st.radio(
+                "Critical Alerts",
+                ["Immediate", "Hourly Digest", "Daily Digest"],
+                index=0
+            )
+            
+            notify_high = st.radio(
+                "High Alerts",
+                ["Immediate", "Hourly Digest", "Daily Digest"],
+                index=1
+            )
+            
+            notify_medium = st.radio(
+                "Medium Alerts",
+                ["Immediate", "Hourly Digest", "Daily Digest"],
+                index=2
+            )
+            
+            notify_low = st.radio(
+                "Low Alerts",
+                ["Immediate", "Hourly Digest", "Daily Digest"],
+                index=2
+            )
+    
+    # Save alert settings button
+    st.button("Save Notification Settings", type="primary", key="save_notif")

components/dashboard.py

@@ -0,0 +1,335 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+import altair as alt
+from datetime import datetime, timedelta
+
+def render_dashboard():
+    st.title("Dark Web Intelligence Dashboard")
+    
+    # Date range selector
+    col1, col2 = st.columns([3, 1])
+    
+    with col1:
+        st.markdown("## Overview")
+        st.markdown("Real-time monitoring of dark web activities, data breaches, and emerging threats.")
+    
+    with col2:
+        date_range = st.selectbox(
+            "Time Range",
+            ["Last 24 Hours", "Last 7 Days", "Last 30 Days", "Last Quarter", "Custom Range"],
+            index=1
+        )
+    
+    # Dashboard metrics row
+    metric_col1, metric_col2, metric_col3, metric_col4 = st.columns(4)
+    
+    with metric_col1:
+        st.metric(
+            label="Active Threats",
+            value="27",
+            delta="4",
+            delta_color="inverse"
+        )
+    
+    with metric_col2:
+        st.metric(
+            label="Data Breaches",
+            value="3",
+            delta="-2",
+            delta_color="normal"
+        )
+    
+    with metric_col3:
+        st.metric(
+            label="Credential Leaks",
+            value="1,247",
+            delta="89",
+            delta_color="inverse"
+        )
+    
+    with metric_col4:
+        st.metric(
+            label="Threat Score",
+            value="72/100",
+            delta="12",
+            delta_color="inverse"
+        )
+    
+    # First row - Threat map and category distribution
+    row1_col1, row1_col2 = st.columns([2, 1])
+    
+    with row1_col1:
+        st.subheader("Global Threat Origin Map")
+        
+        # World map of threat origins
+        fig = go.Figure(data=go.Choropleth(
+            locations=['USA', 'RUS', 'CHN', 'IRN', 'PRK', 'UKR', 'DEU', 'GBR', 'CAN', 'BRA', 'IND'],
+            z=[25, 42, 37, 30, 28, 18, 15, 20, 12, 14, 23],
+            colorscale='Reds',
+            autocolorscale=False,
+            reversescale=False,
+            marker_line_color='#2C3E50',
+            marker_line_width=0.5,
+            colorbar_title='Threat<br>Index',
+        ))
+
+        fig.update_layout(
+            geo=dict(
+                showframe=False,
+                showcoastlines=True,
+                projection_type='equirectangular',
+                bgcolor='rgba(26, 26, 26, 0)',
+                coastlinecolor='#2C3E50',
+                landcolor='#1A1A1A',
+                oceancolor='#2C3E50',
+            ),
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            margin=dict(l=0, r=0, t=0, b=0),
+            height=400,
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with row1_col2:
+        st.subheader("Threat Categories")
+        
+        # Threat category distribution
+        categories = ['Data Breach', 'Ransomware', 'Phishing', 'Malware', 'Identity Theft']
+        values = [38, 24, 18, 14, 6]
+        
+        fig = px.pie(
+            names=categories,
+            values=values,
+            hole=0.6,
+            color_discrete_sequence=['#E74C3C', '#F1C40F', '#3498DB', '#2ECC71', '#9B59B6']
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            showlegend=True,
+            legend=dict(
+                orientation="v",
+                yanchor="middle",
+                y=0.5,
+                xanchor="center",
+                x=0.5
+            ),
+            margin=dict(l=0, r=0, t=30, b=0),
+            height=300,
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    # Second row - Trend and recent activities
+    row2_col1, row2_col2 = st.columns([3, 2])
+    
+    with row2_col1:
+        st.subheader("Threat Activity Trend")
+        
+        # Generate dates for the past 14 days
+        dates = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(14, 0, -1)]
+        
+        # Sample data for threats over time
+        threat_data = {
+            'Date': dates,
+            'High': [12, 10, 15, 11, 14, 16, 18, 20, 17, 12, 14, 13, 19, 22],
+            'Medium': [23, 25, 22, 20, 24, 25, 26, 24, 22, 21, 23, 25, 28, 27],
+            'Low': [32, 30, 35, 34, 36, 33, 30, 34, 38, 37, 35, 34, 32, 30]
+        }
+        
+        df = pd.DataFrame(threat_data)
+        
+        # Create stacked area chart
+        fig = go.Figure()
+        
+        fig.add_trace(go.Scatter(
+            x=df['Date'], y=df['High'],
+            mode='lines',
+            line=dict(width=0.5, color='#E74C3C'),
+            stackgroup='one',
+            name='High'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=df['Date'], y=df['Medium'],
+            mode='lines',
+            line=dict(width=0.5, color='#F1C40F'),
+            stackgroup='one',
+            name='Medium'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=df['Date'], y=df['Low'],
+            mode='lines',
+            line=dict(width=0.5, color='#2ECC71'),
+            stackgroup='one',
+            name='Low'
+        ))
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=1.02,
+                xanchor="right",
+                x=1
+            ),
+            margin=dict(l=0, r=0, t=30, b=0),
+            xaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            height=300
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with row2_col2:
+        st.subheader("Recent Intelligence Feeds")
+        
+        # Recent dark web activities
+        activities = [
+            {"time": "10 mins ago", "event": "New ransomware group identified", "severity": "High"},
+            {"time": "43 mins ago", "event": "Database with 50K credentials for sale", "severity": "High"},
+            {"time": "2 hours ago", "event": "Zero-day exploit being discussed", "severity": "Medium"},
+            {"time": "3 hours ago", "event": "New phishing campaign detected", "severity": "Medium"},
+            {"time": "5 hours ago", "event": "PII data from financial institution leaked", "severity": "High"}
+        ]
+        
+        for activity in activities:
+            severity_color = "#E74C3C" if activity["severity"] == "High" else "#F1C40F" if activity["severity"] == "Medium" else "#2ECC71"
+            
+            cols = st.columns([1, 4, 1])
+            cols[0].caption(activity["time"])
+            cols[1].markdown(activity["event"])
+            cols[2].markdown(f"<span style='color:{severity_color}'>{activity['severity']}</span>", unsafe_allow_html=True)
+            
+            st.markdown("---")
+    
+    # Third row - Sectors at risk and trending keywords
+    row3_col1, row3_col2 = st.columns(2)
+    
+    with row3_col1:
+        st.subheader("Sectors at Risk")
+        
+        # Horizontal bar chart for sectors at risk
+        sectors = ['Healthcare', 'Finance', 'Technology', 'Education', 'Government', 'Manufacturing']
+        risk_scores = [87, 82, 75, 63, 78, 56]
+        
+        sector_data = pd.DataFrame({
+            'Sector': sectors,
+            'Risk Score': risk_scores
+        })
+        
+        fig = px.bar(
+            sector_data,
+            x='Risk Score',
+            y='Sector',
+            orientation='h',
+            color='Risk Score',
+            color_continuous_scale=['#2ECC71', '#F1C40F', '#E74C3C'],
+            range_color=[50, 100]
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            margin=dict(l=0, r=0, t=0, b=0),
+            height=250,
+            coloraxis_showscale=False,
+            xaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            )
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with row3_col2:
+        st.subheader("Trending Keywords")
+        
+        # Word cloud alternative - trending keywords with frequency
+        keywords = [
+            {"word": "ransomware", "count": 42},
+            {"word": "zero-day", "count": 37},
+            {"word": "botnet", "count": 31},
+            {"word": "credentials", "count": 28},
+            {"word": "bitcoin", "count": 25},
+            {"word": "exploit", "count": 23},
+            {"word": "malware", "count": 21},
+            {"word": "backdoor", "count": 18},
+            {"word": "phishing", "count": 16},
+            {"word": "darknet", "count": 15}
+        ]
+        
+        keyword_data = pd.DataFrame(keywords)
+        
+        # Calculate sizes for visual representation
+        max_count = max(keyword_data['count'])
+        keyword_data['size'] = keyword_data['count'].apply(lambda x: int((x / max_count) * 100) + 70)
+        
+        # Create a simple horizontal bar to represent frequency
+        chart = alt.Chart(keyword_data).mark_bar().encode(
+            x=alt.X('count:Q', title=None),
+            y=alt.Y('word:N', title=None, sort='-x'),
+            color=alt.Color('count:Q', scale=alt.Scale(scheme='reds'), legend=None)
+        ).properties(
+            height=250
+        )
+        
+        st.altair_chart(chart, use_container_width=True)
+    
+    # Fourth row - Latest intelligence reports
+    st.subheader("Latest Intelligence Reports")
+    
+    reports = [
+        {
+            "title": "Major Healthcare Breach Analysis",
+            "date": "2025-04-08",
+            "summary": "Analysis of recent healthcare data breach affecting over 500,000 patient records.",
+            "severity": "Critical"
+        },
+        {
+            "title": "Emerging Ransomware Group Activities",
+            "date": "2025-04-07",
+            "summary": "New ransomware group targeting financial institutions with sophisticated techniques.",
+            "severity": "High"
+        },
+        {
+            "title": "Credential Harvesting Campaign",
+            "date": "2025-04-05",
+            "summary": "Widespread phishing campaign targeting corporate credentials across multiple sectors.",
+            "severity": "Medium"
+        }
+    ]
+    
+    row4_cols = st.columns(3)
+    
+    for i, report in enumerate(reports):
+        with row4_cols[i]:
+            severity_color = "#E74C3C" if report["severity"] == "Critical" else "#F1C40F" if report["severity"] == "High" else "#2ECC71"
+            
+            st.markdown(f"#### {report['title']}")
+            st.markdown(f"<span style='color:{severity_color}'>{report['severity']}</span> | {report['date']}", unsafe_allow_html=True)
+            st.markdown(report["summary"])
+            st.button("View Full Report", key=f"report_{i}")

components/live_feed.py

@@ -0,0 +1,769 @@
+import streamlit as st
+import pandas as pd
+import time
+from datetime import datetime, timedelta
+import random
+import plotly.graph_objects as go
+import trafilatura
+import threading
+import queue
+
+# Global queue for storing live feed events
+feed_queue = queue.Queue(maxsize=100)
+
+# Sample dark web sources for simulation
+DARK_WEB_SOURCES = [
+    "AlphaBay Market", "BreachForums", "XSS Forum", "RaidForums", "DeepPaste", 
+    "BlackHat Forum", "DarkLeak Site", "HackTown", "Exploit.in", "0day.today",
+    "Telegram Channel: DarkLeaks", "Telegram Channel: DataBreach", "BitHunters IRC",
+    "Genesis Market", "ASAP Market", "Tor Network: Hidden Services", "DarkNetLive"
+]
+
+# Sample event types and severities
+EVENT_TYPES = {
+    "Credential Leak": ["Critical", "High"],
+    "Data Breach": ["Critical", "High", "Medium"],
+    "Ransomware Activity": ["Critical", "High"],
+    "Hacking Tool": ["Medium", "Low"],
+    "Zero-day Exploit": ["Critical", "High"],
+    "Phishing Campaign": ["High", "Medium"],
+    "Dark Web Mention": ["Medium", "Low"],
+    "PII Exposure": ["Critical", "High"],
+    "New Marketplace Listing": ["Medium", "Low"],
+    "Threat Actor Communication": ["High", "Medium"],
+    "Malware Sample": ["High", "Medium", "Low"],
+    "Source Code Leak": ["High", "Medium"]
+}
+
+# Keywords associated with your organization
+MONITORED_KEYWORDS = [
+    "company.com", "companyname", "company name", "CompanyX", "ServiceY", "ProductZ",
+    "company database", "company credentials", "company breach", "company leak",
+    "@company.com", "CEO Name", "CTO Name", "internal documents"
+]
+
+# Industries for sector-based alerts
+INDUSTRIES = [
+    "Healthcare", "Finance", "Technology", "Education", "Government", 
+    "Manufacturing", "Retail", "Energy", "Telecommunications", "Transportation"
+]
+
+def generate_live_event():
+    """Generate a simulated live dark web event for demonstration"""
+    current_time = datetime.now()
+    
+    # Choose event type and severity
+    event_type = random.choice(list(EVENT_TYPES.keys()))
+    severity = random.choice(EVENT_TYPES[event_type])
+    
+    # Choose source
+    source = random.choice(DARK_WEB_SOURCES)
+    
+    # Determine if it should mention a monitored keyword (higher chance for critical events)
+    mention_keyword = random.random() < (0.8 if severity == "Critical" else 0.3)
+    keyword = random.choice(MONITORED_KEYWORDS) if mention_keyword else None
+    
+    # Choose affected industry
+    industry = random.choice(INDUSTRIES)
+    
+    # Generate description
+    if keyword:
+        descriptions = [
+            f"Detected {event_type.lower()} involving {keyword}",
+            f"{keyword} mentioned in context of {event_type.lower()}",
+            f"Potential {event_type.lower()} related to {keyword}",
+            f"New {severity.lower()} severity {event_type.lower()} containing {keyword}",
+            f"Alert: {event_type} with reference to {keyword}"
+        ]
+    else:
+        descriptions = [
+            f"New {event_type} affecting {industry} sector",
+            f"Detected {event_type.lower()} targeting {industry} organizations",
+            f"Emerging {event_type.lower()} with {severity.lower()} impact",
+            f"Potential {industry} sector {event_type.lower()} identified",
+            f"{severity} {event_type} observed in {source}"
+        ]
+    
+    description = random.choice(descriptions)
+    
+    # Generate event ID
+    event_id = f"EVT-{current_time.strftime('%y%m%d')}-{random.randint(1000, 9999)}"
+    
+    # Create event dictionary
+    event = {
+        "id": event_id,
+        "timestamp": current_time,
+        "event_type": event_type,
+        "severity": severity,
+        "source": source,
+        "description": description,
+        "industry": industry,
+        "relevant": mention_keyword
+    }
+    
+    return event
+
+def start_feed_generator():
+    """Start background thread to generate feed events"""
+    def generate_events():
+        while True:
+            # Generate a new event
+            event = generate_live_event()
+            
+            # Add to queue, remove oldest if full
+            if feed_queue.full():
+                try:
+                    feed_queue.get_nowait()
+                except queue.Empty:
+                    pass
+            
+            try:
+                feed_queue.put_nowait(event)
+            except queue.Full:
+                pass
+            
+            # Sleep random interval (2-15 seconds)
+            sleep_time = random.uniform(2, 15)
+            time.sleep(sleep_time)
+    
+    # Start the background thread
+    thread = threading.Thread(target=generate_events, daemon=True)
+    thread.start()
+
+def render_live_feed():
+    st.title("Real-Time Dark Web Monitoring")
+    
+    # Initialize the feed generator if it's not already running
+    if 'feed_initialized' not in st.session_state:
+        start_feed_generator()
+        st.session_state.feed_initialized = True
+        st.session_state.feed_events = []
+        st.session_state.last_update = datetime.now()
+        
+    # Dashboard layout
+    col1, col2, col3 = st.columns([1, 2, 1])
+    
+    with col1:
+        st.markdown("### Monitoring Status")
+        
+        # Display monitoring metrics
+        st.metric(
+            label="Active Crawlers",
+            value=str(random.randint(12, 18)),
+            delta=str(random.randint(-2, 3))
+        )
+        
+        st.metric(
+            label="Sources Coverage",
+            value=f"{random.randint(85, 98)}%",
+            delta=f"{random.randint(-2, 3)}%"
+        )
+        
+        st.metric(
+            label="Scan Frequency",
+            value=f"{random.randint(3, 7)} min",
+            delta=f"{random.choice([-1, -0.5, 0, 0.5])} min",
+            delta_color="inverse"
+        )
+        
+        # Filters for live feed
+        st.markdown("### Feed Filters")
+        
+        severity_filter = st.multiselect(
+            "Severity",
+            ["Critical", "High", "Medium", "Low"],
+            default=["Critical", "High"]
+        )
+        
+        source_type = st.multiselect(
+            "Source Type",
+            ["Market", "Forum", "Telegram", "IRC", "Paste Site", "Leak Site"],
+            default=["Market", "Forum", "Leak Site"]
+        )
+        
+        relevant_only = st.checkbox("Show Relevant Alerts Only", value=True)
+        
+        auto_refresh = st.checkbox("Auto-Refresh Feed", value=True)
+        
+        if st.button("Refresh Now"):
+            st.session_state.last_update = datetime.now()
+    
+    with col2:
+        st.markdown("### Live Intelligence Feed")
+        
+        # Get events from queue and merge with existing events
+        new_events = []
+        while not feed_queue.empty():
+            try:
+                new_events.append(feed_queue.get_nowait())
+            except queue.Empty:
+                break
+        
+        if new_events:
+            st.session_state.feed_events = new_events + st.session_state.feed_events
+            st.session_state.feed_events = st.session_state.feed_events[:100]  # Keep only 100 most recent
+            st.session_state.last_update = datetime.now()
+        
+        # Filter events
+        filtered_events = []
+        for event in st.session_state.feed_events:
+            if event["severity"] in severity_filter:
+                if not relevant_only or event["relevant"]:
+                    source_match = False
+                    for s_type in source_type:
+                        if s_type.lower() in event["source"].lower():
+                            source_match = True
+                            break
+                    if source_match or not source_type:
+                        filtered_events.append(event)
+        
+        # Display last updated time
+        st.caption(f"Last updated: {st.session_state.last_update.strftime('%H:%M:%S')}")
+        
+        # Display events
+        if not filtered_events:
+            st.info("No events match your current filters. Adjust filters or wait for new events.")
+        else:
+            for i, event in enumerate(filtered_events[:20]):  # Show only 20 most recent
+                # Determine the color based on severity
+                if event["severity"] == "Critical":
+                    severity_color = "#E74C3C"
+                elif event["severity"] == "High":
+                    severity_color = "#F1C40F"
+                elif event["severity"] == "Medium":
+                    severity_color = "#3498DB"
+                else:
+                    severity_color = "#2ECC71"
+                
+                # Event container with colored border based on severity
+                with st.container():
+                    cols = st.columns([3, 1])
+                    
+                    # Event details
+                    with cols[0]:
+                        st.markdown(f"""
+                        <div style="border-left: 4px solid {severity_color}; padding-left: 10px;">
+                            <span style="color: {severity_color}; font-weight: bold;">{event['severity']}</span> | {event['event_type']}
+                            <br><span style="font-size: 0.9em;">{event['description']}</span>
+                            <br><span style="font-size: 0.8em; color: #7F8C8D;">Source: {event['source']} | ID: {event['id']}</span>
+                        </div>
+                        """, unsafe_allow_html=True)
+                    
+                    # Timestamp and actions
+                    with cols[1]:
+                        # Format time as relative (e.g., "2 mins ago")
+                        time_diff = datetime.now() - event["timestamp"]
+                        minutes_ago = time_diff.total_seconds() / 60
+                        
+                        if minutes_ago < 1:
+                            time_str = "just now"
+                        elif minutes_ago < 60:
+                            time_str = f"{int(minutes_ago)} min ago"
+                        else:
+                            hours = int(minutes_ago / 60)
+                            time_str = f"{hours} hrs ago"
+                        
+                        st.markdown(f"<span style='font-size: 0.8em;'>{time_str}</span>", unsafe_allow_html=True)
+                        
+                        # Action buttons
+                        if st.button("Investigate", key=f"investigate_{i}"):
+                            st.session_state.selected_event = event
+                    
+                    # Add a subtle divider
+                    st.markdown("<hr style='margin: 5px 0; opacity: 0.2;'>", unsafe_allow_html=True)
+    
+    with col3:
+        st.markdown("### Intelligence Summary")
+        
+        # Current severity distribution
+        severity_counts = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}
+        for event in st.session_state.feed_events:
+            if event["severity"] in severity_counts:
+                severity_counts[event["severity"]] += 1
+        
+        # Create donut chart for severity distribution
+        fig = go.Figure(go.Pie(
+            labels=list(severity_counts.keys()),
+            values=list(severity_counts.values()),
+            hole=.6,
+            marker=dict(colors=['#E74C3C', '#F1C40F', '#3498DB', '#2ECC71'])
+        ))
+        
+        fig.update_layout(
+            showlegend=True,
+            margin=dict(t=0, b=0, l=0, r=0),
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=-0.2,
+                xanchor="center",
+                x=0.5
+            ),
+            paper_bgcolor='rgba(0,0,0,0)',
+            plot_bgcolor='rgba(0,0,0,0)',
+            height=200
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Top mentioned industries
+        st.markdown("#### Top Targeted Industries")
+        
+        industry_counts = {}
+        for event in st.session_state.feed_events:
+            industry = event["industry"]
+            industry_counts[industry] = industry_counts.get(industry, 0) + 1
+        
+        # Sort industries by count and take top 5
+        top_industries = sorted(industry_counts.items(), key=lambda x: x[1], reverse=True)[:5]
+        
+        for industry, count in top_industries:
+            st.markdown(f"• {industry}: **{count}** alerts")
+        
+        # Trending threats
+        st.markdown("#### Trending Threats")
+        
+        event_type_counts = {}
+        for event in st.session_state.feed_events:
+            event_type = event["event_type"]
+            event_type_counts[event_type] = event_type_counts.get(event_type, 0) + 1
+        
+        # Sort event types by count and take top 5
+        top_threats = sorted(event_type_counts.items(), key=lambda x: x[1], reverse=True)[:5]
+        
+        for threat, count in top_threats:
+            st.markdown(f"• {threat}: **{count}** alerts")
+        
+        # Add a quick investigate button for the most recent critical event
+        st.markdown("---")
+        st.markdown("#### Urgent Action Required")
+        
+        critical_events = [e for e in st.session_state.feed_events if e["severity"] == "Critical"]
+        if critical_events:
+            latest_critical = critical_events[0]
+            st.error(f"""
+            **{latest_critical['event_type']}**  
+            {latest_critical['description']}
+            """)
+            
+            if st.button("Investigate Now", key="urgent_investigate"):
+                st.session_state.selected_event = latest_critical
+        else:
+            st.success("No critical events requiring urgent attention")
+    
+    # If an event is selected for investigation, show details
+    if 'selected_event' in st.session_state and st.session_state.selected_event:
+        event = st.session_state.selected_event
+        
+        st.markdown("---")
+        st.markdown("## Event Investigation")
+        
+        event_col1, event_col2 = st.columns([3, 1])
+        
+        with event_col1:
+            st.markdown(f"### {event['event_type']}")
+            st.markdown(f"**ID:** {event['id']}")
+            st.markdown(f"**Description:** {event['description']}")
+            st.markdown(f"**Source:** {event['source']}")
+            st.markdown(f"**Industry:** {event['industry']}")
+            st.markdown(f"**Detected:** {event['timestamp'].strftime('%Y-%m-%d %H:%M:%S')}")
+            st.markdown(f"**Severity:** {event['severity']}")
+        
+        with event_col2:
+            severity_color = "#E74C3C" if event["severity"] == "Critical" else "#F1C40F" if event["severity"] == "High" else "#3498DB" if event["severity"] == "Medium" else "#2ECC71"
+            
+            st.markdown(f"""
+            <div style="background-color: {severity_color}20; padding: 10px; border-radius: 5px; border-left: 4px solid {severity_color};">
+                <h4 style="margin: 0; color: {severity_color};">Risk Assessment</h4>
+                <p>Severity: <b>{event['severity']}</b></p>
+                <p>Confidence: <b>{random.randint(70, 95)}%</b></p>
+                <p>Impact: <b>{'High' if event['severity'] in ['Critical', 'High'] else 'Medium'}</b></p>
+            </div>
+            """, unsafe_allow_html=True)
+        
+        # Tabs for different investigation aspects
+        inv_tab1, inv_tab2, inv_tab3 = st.tabs(["Analysis", "Similar Events", "Recommendations"])
+        
+        with inv_tab1:
+            st.markdown("### Event Analysis")
+            
+            # Simulated content analysis
+            st.markdown("#### Content Analysis")
+            st.markdown("""
+            This event represents a potential security incident that requires investigation. 
+            The key indicators suggest this could be related to targeted activity against your organization 
+            or the wider industry sector.
+            
+            **Key Indicators:**
+            * Event type and severity level
+            * Source credibility assessment
+            * Contextual mentions and relationships
+            * Temporal correlation with known threat activities
+            """)
+            
+            # Simulated indicators of compromise
+            st.markdown("#### Indicators of Compromise")
+            ioc_data = {
+                "IP Addresses": [f"192.168.{random.randint(1, 254)}.{random.randint(1, 254)}" for _ in range(3)],
+                "Domains": [f"malicious{random.randint(100, 999)}.{random.choice(['com', 'net', 'org'])}" for _ in range(2)],
+                "File Hashes": [f"{''.join(random.choices('0123456789abcdef', k=64))}" for _ in range(2)]
+            }
+            
+            for ioc_type, items in ioc_data.items():
+                st.markdown(f"**{ioc_type}:**")
+                for item in items:
+                    st.code(item)
+        
+        with inv_tab2:
+            st.markdown("### Related Events")
+            
+            # Generate a few similar events
+            similar_events = []
+            for _ in range(3):
+                similar_event = generate_live_event()
+                similar_event["event_type"] = event["event_type"]
+                similar_event["severity"] = random.choice(EVENT_TYPES[event["event_type"]])
+                similar_event["timestamp"] = event["timestamp"] - timedelta(days=random.randint(1, 30))
+                similar_events.append(similar_event)
+            
+            # Display similar events
+            for i, similar in enumerate(similar_events):
+                with st.container():
+                    st.markdown(f"""
+                    **{similar['event_type']} ({similar['severity']})**  
+                    {similar['description']}  
+                    *Detected: {similar['timestamp'].strftime('%Y-%m-%d')} | Source: {similar['source']}*
+                    """)
+                    
+                    if i < len(similar_events) - 1:
+                        st.markdown("---")
+        
+        with inv_tab3:
+            st.markdown("### Recommended Actions")
+            
+            # Generic recommendations based on event type
+            recommendations = {
+                "Data Breach": [
+                    "Verify if the leaked data belongs to your organization",
+                    "Identify affected systems and users",
+                    "Initiate your incident response plan",
+                    "Prepare for potential notification requirements",
+                    "Monitor for misuse of the compromised data"
+                ],
+                "Credential Leak": [
+                    "Force password resets for affected accounts",
+                    "Enable multi-factor authentication where possible",
+                    "Monitor for unauthorized access attempts",
+                    "Review privileged access controls",
+                    "Scan for credentials used across multiple systems"
+                ],
+                "Ransomware Activity": [
+                    "Verify backup integrity and availability",
+                    "Isolate potentially affected systems",
+                    "Review security controls for ransomware protection",
+                    "Assess exposure to the specific ransomware variant",
+                    "Prepare business continuity procedures"
+                ],
+                "Zero-day Exploit": [
+                    "Assess if your systems use the affected software",
+                    "Apply temporary mitigations or workarounds",
+                    "Monitor vendor channels for patch availability",
+                    "Increase monitoring for exploit attempts",
+                    "Review defense-in-depth security controls"
+                ],
+                "Phishing Campaign": [
+                    "Alert employees about the phishing campaign",
+                    "Block identified phishing domains and URLs",
+                    "Scan email systems for instances of the phishing message",
+                    "Review security awareness training materials",
+                    "Deploy additional email security controls"
+                ],
+                "Dark Web Mention": [
+                    "Analyze context of the mention for potential threats",
+                    "Review security for specifically mentioned assets",
+                    "Increase monitoring for related activities",
+                    "Brief relevant stakeholders on potential risks",
+                    "Consider threat intelligence analysis for the mention"
+                ]
+            }
+            
+            # Get recommendations for the event type or use a default set
+            event_recommendations = recommendations.get(
+                event["event_type"], 
+                ["Investigate the alert details", "Assess potential impact", "Verify if your organization is affected"]
+            )
+            
+            # Display recommendations
+            for rec in event_recommendations:
+                st.markdown(f"- {rec}")
+            
+            # Action buttons
+            col1, col2 = st.columns(2)
+            with col1:
+                st.button("Add to Investigation Case", key="add_to_case")
+            with col2:
+                st.button("Mark as False Positive", key="mark_false_positive")
+        
+        # Close investigation button
+        if st.button("Close Investigation", key="close_investigation"):
+            del st.session_state.selected_event
+    
+    # Auto-refresh using a placeholder and empty to trigger rerun
+    if auto_refresh:
+        placeholder = st.empty()
+        time.sleep(30)  # Refresh every 30 seconds
+        placeholder.empty()
+        st.rerun()
+
+def fetch_dark_web_content(url):
+    """
+    Fetch content from a dark web site (simulated for demonstration).
+    In a real application, this would connect to Tor network or similar.
+    
+    Args:
+        url (str): The URL to fetch content from
+        
+    Returns:
+        str: The extracted content
+    """
+    # In a real scenario, you would use specialized tools to access dark web
+    # Here we'll simulate this with sample data
+    
+    if "forum" in url.lower():
+        return """
+        --------- Dark Web Forum Excerpt ---------
+        
+        User123: Looking for access to healthcare databases, paying premium
+        
+        DarkSeller: Have fresh dump from major hospital, 50K+ patient records with PII and insurance info
+        
+        User123: What's your price? Is it the Memorial Hospital data?
+        
+        DarkSeller: 45 BTC for the full database. Yes, it's from Memorial plus two smaller clinics.
+        
+        User456: I can vouch for DarkSeller, bought credentials last month, all valid.
+        
+        DarkSeller: Sample available for serious buyers. Payment via escrow only.
+        """
+    
+    elif "market" in url.lower():
+        return """
+        --------- Dark Web Marketplace Listing ---------
+        
+        ITEM: Complete patient database from major US hospital
+        SELLER: MedLeaks (Trusted Vendor ★★★★★)
+        PRICE: 45 BTC
+        
+        DESCRIPTION:
+        Fresh database dump containing 50,000+ complete patient records including:
+        - Full names, DOB, SSN
+        - Home addresses and contact information
+        - Insurance policy details and ID numbers
+        - Medical diagnoses and treatment codes
+        - Billing information including payment methods
+        
+        Data verified and ready for immediate delivery. Suitable for identity theft, 
+        insurance fraud, or targeted phishing campaigns.
+        
+        SHIPPING: Instant digital delivery via encrypted channel
+        TERMS: No refunds, escrow available
+        """
+    
+    else:
+        return """
+        --------- Dark Web Intelligence ---------
+        
+        Multiple sources reporting new ransomware operation targeting healthcare sector.
+        Group appears to be using stolen credentials to access systems.
+        
+        Identified C2 infrastructure:
+        - 185.212.x.x
+        - 91.223.x.x
+        - malware-delivery[.]xyz
+        
+        Ransom demands ranging from 20-50 BTC depending on organization size.
+        Group is exfiltrating data before encryption and threatening publication.
+        """
+
+def render_content_analysis():
+    """Display dark web content analysis tools"""
+    st.markdown("### Dark Web Content Analysis")
+    
+    col1, col2 = st.columns([2, 1])
+    
+    with col1:
+        st.markdown("Enter a URL or paste content for analysis:")
+        
+        analysis_source = st.radio(
+            "Content Source",
+            ["URL", "Pasted Content"],
+            horizontal=True
+        )
+        
+        if analysis_source == "URL":
+            url = st.text_input("Enter Dark Web URL", value="darkforum.onion/thread/healthcare-data")
+            
+            if st.button("Fetch Content", key="fetch_btn"):
+                with st.spinner("Connecting to dark web. Please wait..."):
+                    time.sleep(2)  # Simulate connection time
+                    content = fetch_dark_web_content(url)
+                    st.session_state.current_content = content
+        else:
+            content_input = st.text_area("Paste content for analysis", height=150)
+            
+            if st.button("Analyze Content", key="analyze_pasted"):
+                st.session_state.current_content = content_input
+    
+    with col2:
+        st.markdown("Analysis Options")
+        
+        analysis_type = st.selectbox(
+            "Select Analysis Type",
+            ["Entity Extraction", "Threat Detection", "Keyword Analysis", "IoC Extraction"]
+        )
+        
+        st.markdown("---")
+        
+        st.markdown("Monitored Keywords")
+        
+        # Display monitored keywords
+        keyword_columns = st.columns(2)
+        
+        for i, keyword in enumerate(MONITORED_KEYWORDS[:8]):  # Show only first 8
+            with keyword_columns[i % 2]:
+                st.markdown(f"• {keyword}")
+        
+        st.markdown("...")
+        
+        st.markdown("---")
+        
+        if st.button("Add Custom Keywords"):
+            st.session_state.show_keyword_input = True
+        
+        if st.session_state.get("show_keyword_input", False):
+            new_keyword = st.text_input("Enter new keyword")
+            if st.button("Add Keyword"):
+                if new_keyword and new_keyword not in MONITORED_KEYWORDS:
+                    MONITORED_KEYWORDS.append(new_keyword)
+                    st.success(f"Added keyword: {new_keyword}")
+    
+    # If we have content to analyze, show it and the analysis
+    if hasattr(st.session_state, "current_content") and st.session_state.current_content:
+        st.markdown("---")
+        
+        tabs = st.tabs(["Content", "Analysis", "Entities", "Indicators"])
+        
+        with tabs[0]:
+            st.markdown("### Raw Content")
+            st.text(st.session_state.current_content)
+        
+        with tabs[1]:
+            st.markdown("### Content Analysis")
+            
+            # Identify any monitored keywords in content
+            found_keywords = []
+            for keyword in MONITORED_KEYWORDS:
+                if keyword.lower() in st.session_state.current_content.lower():
+                    found_keywords.append(keyword)
+            
+            if found_keywords:
+                st.warning(f"Found {len(found_keywords)} monitored keywords in content:")
+                for keyword in found_keywords:
+                    st.markdown(f"• **{keyword}**")
+            else:
+                st.info("No monitored keywords found in content.")
+            
+            # Simple sentiment analysis
+            text = st.session_state.current_content.lower()
+            
+            threat_terms = ["hack", "breach", "leak", "dump", "sell", "exploit", "vulnerability", 
+                          "ransomware", "malware", "phishing", "attack", "threat"]
+            
+            threat_found = sum(term in text for term in threat_terms)
+            
+            if threat_found > 3:
+                threat_level = "High"
+                color = "#E74C3C"
+            elif threat_found > 1:
+                threat_level = "Medium"
+                color = "#F1C40F"
+            else:
+                threat_level = "Low"
+                color = "#2ECC71"
+            
+            st.markdown(f"**Threat Assessment: <span style='color:{color}'>{threat_level}</span>**", unsafe_allow_html=True)
+            st.markdown(f"Identified {threat_found} threat indicators in the content.")
+        
+        with tabs[2]:
+            st.markdown("### Entities Extracted")
+            
+            # Sample entity extraction
+            entities = {
+                "Organizations": ["Memorial Hospital", "MedLeaks"],
+                "Monetary Values": ["45 BTC", "20-50 BTC"],
+                "Quantities": ["50,000+ patient records", "50K+ patient records"],
+                "Locations": [],
+                "People": ["User123", "DarkSeller", "User456"]
+            }
+            
+            for entity_type, items in entities.items():
+                if items:
+                    st.markdown(f"#### {entity_type}")
+                    for item in items:
+                        st.markdown(f"• {item}")
+        
+        with tabs[3]:
+            st.markdown("### Indicators of Compromise")
+            
+            # Extract indicators from content
+            iocs = {
+                "IP Addresses": [],
+                "Domains": [],
+                "URLs": [],
+                "Hashes": []
+            }
+            
+            # Very simple regex patterns for demo - in real system use more robust methods
+            ip_pattern = r'\b(?:\d{1,3}\.){3}\d{1,3}\b'
+            domain_pattern = r'\b(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]{2,}\b'
+            url_pattern = r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'
+            hash_pattern = r'\b[a-fA-F0-9]{32,64}\b'
+            
+            import re
+            
+            text = st.session_state.current_content
+            
+            # Find IP addresses
+            iocs["IP Addresses"] = re.findall(ip_pattern, text)
+            
+            # Find domains
+            domains = re.findall(domain_pattern, text)
+            iocs["Domains"] = [d for d in domains if ".onion" in d or ".xyz" in d]  # Filter for interesting domains
+            
+            # Find URLs
+            iocs["URLs"] = re.findall(url_pattern, text)
+            
+            # Find hashes
+            iocs["Hashes"] = re.findall(hash_pattern, text)
+            
+            # Display found IOCs
+            has_iocs = False
+            
+            for ioc_type, items in iocs.items():
+                if items:
+                    has_iocs = True
+                    st.markdown(f"#### {ioc_type}")
+                    for item in items:
+                        st.code(item)
+            
+            if not has_iocs:
+                st.info("No indicators of compromise detected in the content.")
+            
+            # Actions
+            col1, col2 = st.columns(2)
+            
+            with col1:
+                st.button("Export Indicators", key="export_iocs")
+            
+            with col2:
+                st.button("Add to Watchlist", key="add_to_watchlist")

components/monitoring.py

@@ -0,0 +1,555 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+from datetime import datetime, timedelta
+
+def render_monitoring():
+    st.title("Monitoring Configuration")
+    
+    # Dashboard layout for monitoring configuration
+    col1, col2 = st.columns([2, 3])
+    
+    with col1:
+        st.subheader("Monitoring Settings")
+        
+        with st.form("monitoring_settings"):
+            st.markdown("### General Settings")
+            
+            scan_frequency = st.select_slider(
+                "Scan Frequency",
+                options=["1 hour", "2 hours", "4 hours", "6 hours", "12 hours", "24 hours"],
+                value="4 hours"
+            )
+            
+            intelligence_sources = st.multiselect(
+                "Intelligence Sources",
+                ["Dark Web Forums", "Paste Sites", "Marketplaces", "Telegram Channels", "IRC Channels", "Ransomware Blogs", "Breach Databases", "Hacker Forums", "Social Media"],
+                default=["Dark Web Forums", "Paste Sites", "Marketplaces", "Ransomware Blogs"]
+            )
+            
+            st.markdown("### Alert Thresholds")
+            
+            col1a, col1b = st.columns(2)
+            
+            with col1a:
+                critical_threshold = st.number_input("Critical Alert Threshold", min_value=1, max_value=100, value=80)
+            
+            with col1b:
+                high_threshold = st.number_input("High Alert Threshold", min_value=1, max_value=100, value=60)
+            
+            col1c, col1d = st.columns(2)
+            
+            with col1c:
+                medium_threshold = st.number_input("Medium Alert Threshold", min_value=1, max_value=100, value=40)
+            
+            with col1d:
+                low_threshold = st.number_input("Low Alert Threshold", min_value=1, max_value=100, value=20)
+            
+            st.markdown("### Notification Channels")
+            
+            email_notify = st.checkbox("Email Notifications", value=True)
+            if email_notify:
+                email_recipients = st.text_input("Email Recipients", value="[email protected], [email protected]")
+            
+            slack_notify = st.checkbox("Slack Notifications", value=True)
+            if slack_notify:
+                slack_channel = st.text_input("Slack Channel", value="#security-alerts")
+            
+            api_notify = st.checkbox("API Webhook", value=False)
+            if api_notify:
+                webhook_url = st.text_input("Webhook URL", placeholder="https://api.example.com/webhook")
+            
+            sms_notify = st.checkbox("SMS Notifications", value=False)
+            if sms_notify:
+                phone_numbers = st.text_input("Phone Numbers", placeholder="+1234567890, +0987654321")
+            
+            submit = st.form_submit_button("Save Configuration", type="primary")
+            
+            if submit:
+                st.success("Monitoring configuration saved successfully!")
+    
+    with col2:
+        st.subheader("Monitored Keywords & Entities")
+        
+        # Tabs for different monitoring categories
+        tab1, tab2, tab3, tab4 = st.tabs(["Company Assets", "Credentials", "PII", "Custom Keywords"])
+        
+        with tab1:
+            st.markdown("### Company Assets Monitoring")
+            
+            # Sample company assets to monitor
+            company_assets = pd.DataFrame({
+                "Asset Type": ["Domain", "Domain", "IP Range", "Brand", "Brand", "Product", "Technology"],
+                "Value": ["company.com", "company-services.net", "198.51.100.0/24", "CompanyName", "ProductX", "ServiceY", "TechnologyZ"],
+                "Priority": ["High", "Medium", "High", "Critical", "High", "Medium", "Low"],
+                "Status": ["Active", "Active", "Active", "Active", "Active", "Active", "Active"]
+            })
+            
+            # Editable dataframe
+            edited_assets = st.data_editor(
+                company_assets,
+                num_rows="dynamic",
+                column_config={
+                    "Asset Type": st.column_config.SelectboxColumn(
+                        "Asset Type",
+                        options=["Domain", "IP Range", "Brand", "Product", "Technology", "Other"],
+                    ),
+                    "Priority": st.column_config.SelectboxColumn(
+                        "Priority",
+                        options=["Critical", "High", "Medium", "Low"],
+                    ),
+                    "Status": st.column_config.SelectboxColumn(
+                        "Status",
+                        options=["Active", "Paused"],
+                    ),
+                },
+                use_container_width=True
+            )
+        
+        with tab2:
+            st.markdown("### Credentials Monitoring")
+            
+            # Sample credential monitoring settings
+            credential_monitoring = pd.DataFrame({
+                "Email Domain": ["@company.com", "@company-services.net", "@product-x.com"],
+                "Include Subdomains": [True, True, False],
+                "Monitor Password Breach": [True, True, True],
+                "Alert Level": ["Critical", "High", "High"],
+                "Status": ["Active", "Active", "Active"]
+            })
+            
+            edited_credentials = st.data_editor(
+                credential_monitoring,
+                num_rows="dynamic",
+                column_config={
+                    "Include Subdomains": st.column_config.CheckboxColumn(
+                        "Include Subdomains",
+                        help="Monitor all subdomains",
+                    ),
+                    "Monitor Password Breach": st.column_config.CheckboxColumn(
+                        "Monitor Password Breach",
+                    ),
+                    "Alert Level": st.column_config.SelectboxColumn(
+                        "Alert Level",
+                        options=["Critical", "High", "Medium", "Low"],
+                    ),
+                    "Status": st.column_config.SelectboxColumn(
+                        "Status",
+                        options=["Active", "Paused"],
+                    ),
+                },
+                use_container_width=True
+            )
+        
+        with tab3:
+            st.markdown("### PII Monitoring")
+            
+            # Sample PII monitoring settings
+            pii_monitoring = pd.DataFrame({
+                "PII Type": ["SSN", "Credit Card", "Bank Account", "Passport Number", "Driver License"],
+                "Monitor": [True, True, True, False, False],
+                "Alert Level": ["Critical", "Critical", "High", "High", "Medium"],
+                "Status": ["Active", "Active", "Active", "Paused", "Paused"]
+            })
+            
+            edited_pii = st.data_editor(
+                pii_monitoring,
+                num_rows="dynamic",
+                column_config={
+                    "PII Type": st.column_config.SelectboxColumn(
+                        "PII Type",
+                        options=["SSN", "Credit Card", "Bank Account", "Passport Number", "Driver License", "Health Information", "Other"],
+                    ),
+                    "Monitor": st.column_config.CheckboxColumn(
+                        "Monitor",
+                    ),
+                    "Alert Level": st.column_config.SelectboxColumn(
+                        "Alert Level",
+                        options=["Critical", "High", "Medium", "Low"],
+                    ),
+                    "Status": st.column_config.SelectboxColumn(
+                        "Status",
+                        options=["Active", "Paused"],
+                    ),
+                },
+                use_container_width=True
+            )
+        
+        with tab4:
+            st.markdown("### Custom Keywords")
+            
+            # Sample custom keywords
+            custom_keywords = pd.DataFrame({
+                "Keyword": ["confidential memo", "project phoenix", "merger", "acquisition", "layoff", "security breach"],
+                "Category": ["Internal Document", "Project", "Financial", "Financial", "HR", "Security"],
+                "Alert Level": ["Critical", "High", "Critical", "Critical", "High", "Critical"],
+                "Status": ["Active", "Active", "Active", "Active", "Active", "Active"]
+            })
+            
+            edited_keywords = st.data_editor(
+                custom_keywords,
+                num_rows="dynamic",
+                column_config={
+                    "Category": st.column_config.SelectboxColumn(
+                        "Category",
+                        options=["Internal Document", "Project", "Financial", "HR", "Security", "Product", "Other"],
+                    ),
+                    "Alert Level": st.column_config.SelectboxColumn(
+                        "Alert Level",
+                        options=["Critical", "High", "Medium", "Low"],
+                    ),
+                    "Status": st.column_config.SelectboxColumn(
+                        "Status",
+                        options=["Active", "Paused"],
+                    ),
+                },
+                use_container_width=True
+            )
+    
+    # Monitoring sources and coverage
+    st.markdown("---")
+    st.subheader("Monitoring Sources & Coverage")
+    
+    # Create tabs for different monitoring source categories
+    source_tab1, source_tab2, source_tab3 = st.tabs(["Dark Web Coverage", "Source Categories", "Geographic Coverage"])
+    
+    with source_tab1:
+        # Dark web monitoring sources
+        st.markdown("### Dark Web Monitoring Sources")
+        
+        # Sample data for dark web sources
+        dark_web_sources = pd.DataFrame({
+            "Source Type": ["Market", "Forum", "Forum", "Market", "Paste Site", "Leak Site", "Chat", "Market"],
+            "Name": ["AlphaBay", "XSS Forum", "Exploit.in", "ASAP Market", "DeepPaste", "DarkLeak", "Telegram", "White House"],
+            "Focus": ["General", "Hacking", "Credentials", "Drugs/Fraud", "Text sharing", "Data leaks", "Communication", "General"],
+            "Coverage": [95, 90, 85, 80, 75, 70, 65, 60],
+            "Status": ["Active", "Active", "Active", "Active", "Active", "Active", "Active", "Active"]
+        })
+        
+        fig = px.bar(
+            dark_web_sources,
+            x="Name",
+            y="Coverage",
+            color="Coverage",
+            color_continuous_scale=["#2ECC71", "#F1C40F", "#E74C3C"],
+            text="Coverage",
+            height=400
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            xaxis=dict(
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title="Coverage Percentage",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            coloraxis_showscale=False
+        )
+        
+        fig.update_traces(texttemplate='%{text}%', textposition='outside')
+        
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Source details table
+        st.dataframe(dark_web_sources, use_container_width=True)
+    
+    with source_tab2:
+        # Source category distribution
+        st.markdown("### Monitoring by Source Category")
+        
+        # Sample data for source categories
+        source_categories = {
+            "Category": ["Dark Web Markets", "Hacking Forums", "Paste Sites", "Telegram Channels", "IRC Channels", "Leak Sites", "Ransomware Blogs", "Social Media"],
+            "Sources Count": [12, 15, 5, 18, 8, 7, 6, 10],
+            "Coverage Score": [90, 85, 75, 70, 60, 95, 80, 65]
+        }
+        
+        source_df = pd.DataFrame(source_categories)
+        
+        fig = px.scatter(
+            source_df,
+            x="Sources Count",
+            y="Coverage Score",
+            color="Coverage Score",
+            color_continuous_scale=["#E74C3C", "#F1C40F", "#2ECC71"],
+            size="Sources Count",
+            hover_name="Category",
+            height=400
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            xaxis=dict(
+                title="Number of Sources",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title="Coverage Score (%)",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            coloraxis_showscale=False
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Category details
+        st.dataframe(source_df, use_container_width=True)
+    
+    with source_tab3:
+        # Geographic coverage
+        st.markdown("### Geographic Monitoring Coverage")
+        
+        # World map showing coverage
+        st.image("https://images.unsplash.com/photo-1451187580459-43490279c0fa", 
+                 caption="Global monitoring coverage across dark web sources", 
+                 use_column_width=True)
+        
+        # Regional coverage metrics
+        col_geo1, col_geo2, col_geo3, col_geo4 = st.columns(4)
+        
+        with col_geo1:
+            st.metric(
+                label="North America",
+                value="92%",
+                delta="3%",
+                delta_color="normal"
+            )
+        
+        with col_geo2:
+            st.metric(
+                label="Europe",
+                value="88%",
+                delta="5%",
+                delta_color="normal"
+            )
+        
+        with col_geo3:
+            st.metric(
+                label="Asia Pacific",
+                value="76%",
+                delta="8%",
+                delta_color="normal"
+            )
+        
+        with col_geo4:
+            st.metric(
+                label="Rest of World",
+                value="65%",
+                delta="12%",
+                delta_color="normal"
+            )
+    
+    # Monitoring performance metrics
+    st.markdown("---")
+    st.subheader("Monitoring Performance")
+    
+    # Performance metrics
+    perf_col1, perf_col2, perf_col3, perf_col4 = st.columns(4)
+    
+    with perf_col1:
+        st.metric(
+            label="Scan Completion Rate",
+            value="98.7%",
+            delta="0.5%",
+            delta_color="normal"
+        )
+    
+    with perf_col2:
+        st.metric(
+            label="Avg. Scan Duration",
+            value="43 min",
+            delta="-7 min",
+            delta_color="normal"
+        )
+    
+    with perf_col3:
+        st.metric(
+            label="Monitored Keywords",
+            value="1,247",
+            delta="23",
+            delta_color="normal"
+        )
+    
+    with perf_col4:
+        st.metric(
+            label="Coverage Index",
+            value="87/100",
+            delta="5",
+            delta_color="normal"
+        )
+    
+    # Performance charts
+    st.markdown("### Performance Trends")
+    
+    perf_tab1, perf_tab2 = st.tabs(["Scan Performance", "Detection Accuracy"])
+    
+    with perf_tab1:
+        # Generate dates for the past 30 days
+        dates = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(30, 0, -1)]
+        
+        # Sample data for scan performance
+        scan_times = np.random.normal(45, 5, 30).astype(int)  # Mean 45 minutes, std 5 minutes
+        success_rates = np.random.normal(98, 1, 30)  # Mean 98%, std 1%
+        success_rates = [min(100, max(90, rate)) for rate in success_rates]  # Clamp between 90-100%
+        
+        scan_data = pd.DataFrame({
+            'Date': dates,
+            'Scan Time (min)': scan_times,
+            'Success Rate (%)': success_rates
+        })
+        
+        # Create a figure with two y-axes
+        fig = go.Figure()
+        
+        # Add scan time line
+        fig.add_trace(go.Scatter(
+            x=scan_data['Date'],
+            y=scan_data['Scan Time (min)'],
+            name='Scan Time (min)',
+            line=dict(color='#3498DB', width=2)
+        ))
+        
+        # Add success rate line on secondary y-axis
+        fig.add_trace(go.Scatter(
+            x=scan_data['Date'],
+            y=scan_data['Success Rate (%)'],
+            name='Success Rate (%)',
+            line=dict(color='#2ECC71', width=2),
+            yaxis='y2'
+        ))
+        
+        # Configure the layout with two y-axes
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            xaxis=dict(
+                title="Date",
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title="Scan Time (min)",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1'),
+                range=[0, 60]
+            ),
+            yaxis2=dict(
+                title="Success Rate (%)",
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1'),
+                overlaying='y',
+                side='right',
+                range=[90, 100]
+            ),
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=1.02,
+                xanchor="right",
+                x=1,
+                font=dict(color='#ECF0F1')
+            ),
+            height=400
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with perf_tab2:
+        # Sample data for detection accuracy
+        accuracy_data = pd.DataFrame({
+            'Date': dates,
+            'True Positives': np.random.randint(80, 100, 30),
+            'False Positives': np.random.randint(5, 15, 30),
+            'Precision': np.random.normal(92, 2, 30),
+            'Recall': np.random.normal(90, 3, 30)
+        })
+        
+        # Ensure precision and recall are within reasonable bounds
+        accuracy_data['Precision'] = accuracy_data['Precision'].apply(lambda x: min(100, max(80, x)))
+        accuracy_data['Recall'] = accuracy_data['Recall'].apply(lambda x: min(100, max(80, x)))
+        
+        # Create a figure with stacked bars and lines
+        fig = go.Figure()
+        
+        # Add stacked bars for true and false positives
+        fig.add_trace(go.Bar(
+            x=accuracy_data['Date'],
+            y=accuracy_data['True Positives'],
+            name='True Positives',
+            marker_color='#2ECC71'
+        ))
+        
+        fig.add_trace(go.Bar(
+            x=accuracy_data['Date'],
+            y=accuracy_data['False Positives'],
+            name='False Positives',
+            marker_color='#E74C3C'
+        ))
+        
+        # Add lines for precision and recall
+        fig.add_trace(go.Scatter(
+            x=accuracy_data['Date'],
+            y=accuracy_data['Precision'],
+            name='Precision (%)',
+            line=dict(color='#3498DB', width=2),
+            yaxis='y2'
+        ))
+        
+        fig.add_trace(go.Scatter(
+            x=accuracy_data['Date'],
+            y=accuracy_data['Recall'],
+            name='Recall (%)',
+            line=dict(color='#F1C40F', width=2),
+            yaxis='y2'
+        ))
+        
+        # Configure the layout
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            barmode='stack',
+            xaxis=dict(
+                title="Date",
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title="Alert Count",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis2=dict(
+                title="Percentage (%)",
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1'),
+                overlaying='y',
+                side='right',
+                range=[80, 100]
+            ),
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=1.02,
+                xanchor="right",
+                x=1,
+                font=dict(color='#ECF0F1')
+            ),
+            height=400
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)

components/reports.py

@@ -0,0 +1,442 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+from datetime import datetime, timedelta
+
+def render_reports():
+    st.title("Intelligence Reports")
+    
+    # Report filters
+    with st.container():
+        st.subheader("Report Filters")
+        
+        col1, col2, col3, col4 = st.columns(4)
+        
+        with col1:
+            report_type = st.multiselect(
+                "Report Type",
+                ["Threat Intelligence", "Data Breach", "Executive", "Technical", "Custom"],
+                default=["Threat Intelligence", "Data Breach"]
+            )
+        
+        with col2:
+            time_period = st.selectbox(
+                "Time Period",
+                ["Last 7 Days", "Last 30 Days", "Last Quarter", "Year to Date", "Custom Range"],
+                index=1
+            )
+        
+        with col3:
+            severity = st.multiselect(
+                "Severity",
+                ["Critical", "High", "Medium", "Low"],
+                default=["Critical", "High"]
+            )
+        
+        with col4:
+            keywords = st.text_input("Keywords", placeholder="e.g. healthcare, ransomware")
+    
+    # Recent reports
+    st.markdown("### Recent Reports")
+    
+    # Sample report data
+    reports = [
+        {
+            "id": "RPT-2025-04083",
+            "title": "Healthcare Data Breach Intelligence Report",
+            "date": "2025-04-08",
+            "type": "Data Breach",
+            "severity": "Critical",
+            "status": "Final"
+        },
+        {
+            "id": "RPT-2025-04082",
+            "title": "Weekly Threat Intelligence Summary",
+            "date": "2025-04-08",
+            "type": "Threat Intelligence",
+            "severity": "High",
+            "status": "Final"
+        },
+        {
+            "id": "RPT-2025-04073",
+            "title": "Emerging Ransomware Group Analysis",
+            "date": "2025-04-07",
+            "type": "Technical",
+            "severity": "High",
+            "status": "Final"
+        },
+        {
+            "id": "RPT-2025-04072",
+            "title": "Executive Threat Landscape Overview",
+            "date": "2025-04-07",
+            "type": "Executive",
+            "severity": "Medium",
+            "status": "Final"
+        },
+        {
+            "id": "RPT-2025-04063",
+            "title": "Financial Sector Threat Assessment",
+            "date": "2025-04-06",
+            "type": "Threat Intelligence",
+            "severity": "High",
+            "status": "Final"
+        },
+        {
+            "id": "RPT-2025-04053",
+            "title": "Technical Analysis: PII Exposure in Dark Web",
+            "date": "2025-04-05",
+            "type": "Technical",
+            "severity": "Medium",
+            "status": "Final"
+        }
+    ]
+    
+    # Create a DataFrame
+    report_df = pd.DataFrame(reports)
+    
+    # Report display
+    for i, report in enumerate(reports):
+        severity_color = "#E74C3C" if report["severity"] == "Critical" else "#F1C40F" if report["severity"] == "High" else "#3498DB" if report["severity"] == "Medium" else "#2ECC71"
+        
+        with st.container():
+            cols = st.columns([4, 1, 1, 1])
+            
+            with cols[0]:
+                st.markdown(f"#### {report['title']}")
+                st.caption(f"ID: {report['id']} | Date: {report['date']}")
+            
+            with cols[1]:
+                st.markdown(f"**Type:** {report['type']}")
+            
+            with cols[2]:
+                st.markdown(f"**<span style='color:{severity_color}'>{report['severity']}</span>**", unsafe_allow_html=True)
+            
+            with cols[3]:
+                st.button("View", key=f"view_report_{i}")
+            
+            st.markdown("---")
+    
+    # Generate a report
+    st.markdown("### Generate New Report")
+    
+    with st.form("report_generator"):
+        st.markdown("#### Report Parameters")
+        
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            report_title = st.text_input("Report Title", placeholder="e.g. Monthly Threat Intelligence Summary")
+            
+            report_type_selection = st.selectbox(
+                "Report Type",
+                ["Threat Intelligence", "Data Breach", "Executive", "Technical", "Custom"]
+            )
+        
+        with col2:
+            report_period = st.selectbox(
+                "Report Period",
+                ["Last 7 Days", "Last 30 Days", "Last Quarter", "Year to Date", "Custom Range"]
+            )
+            
+            if report_period == "Custom Range":
+                start_date = st.date_input("Start Date", datetime.now() - timedelta(days=30))
+                end_date = st.date_input("End Date", datetime.now())
+        
+        st.markdown("#### Report Content")
+        
+        include_options = st.columns(3)
+        
+        with include_options[0]:
+            include_summary = st.checkbox("Executive Summary", value=True)
+            include_threats = st.checkbox("Threat Overview", value=True)
+            include_breaches = st.checkbox("Data Breaches", value=True)
+        
+        with include_options[1]:
+            include_credentials = st.checkbox("Exposed Credentials", value=True)
+            include_ioc = st.checkbox("Indicators of Compromise", value=True)
+            include_actors = st.checkbox("Threat Actor Analysis", value=True)
+        
+        with include_options[2]:
+            include_trends = st.checkbox("Trend Analysis", value=True)
+            include_mitigation = st.checkbox("Mitigation Recommendations", value=True)
+            include_references = st.checkbox("References", value=True)
+        
+        st.markdown("#### Distribution")
+        
+        distribution = st.multiselect(
+            "Distribute To",
+            ["Security Team", "Executive Team", "IT Department", "Legal Department", "Custom Recipients"],
+            default=["Security Team"]
+        )
+        
+        if "Custom Recipients" in distribution:
+            custom_recipients = st.text_input("Custom Recipients (separated by commas)")
+        
+        generate_button = st.form_submit_button("Generate Report")
+        
+        if generate_button:
+            st.success("Report generation initiated! Your report will be available shortly.")
+    
+    # Report analytics
+    st.markdown("---")
+    st.subheader("Report Analytics")
+    
+    # Report metrics
+    metric_col1, metric_col2, metric_col3, metric_col4 = st.columns(4)
+    
+    with metric_col1:
+        st.metric(
+            label="Reports Generated",
+            value="87",
+            delta="12",
+            delta_color="normal"
+        )
+    
+    with metric_col2:
+        st.metric(
+            label="Critical Reports",
+            value="23",
+            delta="5",
+            delta_color="normal"
+        )
+    
+    with metric_col3:
+        st.metric(
+            label="Avg. Generation Time",
+            value="3.5 min",
+            delta="-0.8 min",
+            delta_color="normal"
+        )
+    
+    with metric_col4:
+        st.metric(
+            label="Distribution Rate",
+            value="97%",
+            delta="2%",
+            delta_color="normal"
+        )
+    
+    # Report analytics charts
+    analytics_tab1, analytics_tab2 = st.tabs(["Report Generation Trends", "Report Distribution"])
+    
+    with analytics_tab1:
+        # Generate dates for the past 30 days
+        dates = [(datetime.now() - timedelta(days=i)).strftime('%Y-%m-%d') for i in range(30, 0, -1)]
+        
+        # Sample data for report generation
+        report_data = {
+            'Date': dates,
+            'Executive': np.random.randint(0, 2, 30),
+            'Threat Intelligence': np.random.randint(1, 4, 30),
+            'Data Breach': np.random.randint(0, 3, 30),
+            'Technical': np.random.randint(1, 5, 30)
+        }
+        
+        report_df = pd.DataFrame(report_data)
+        
+        # Create stacked bar chart
+        fig = go.Figure()
+        
+        fig.add_trace(go.Bar(
+            x=report_df['Date'],
+            y=report_df['Executive'],
+            name='Executive',
+            marker_color='#9B59B6'
+        ))
+        
+        fig.add_trace(go.Bar(
+            x=report_df['Date'],
+            y=report_df['Threat Intelligence'],
+            name='Threat Intelligence',
+            marker_color='#3498DB'
+        ))
+        
+        fig.add_trace(go.Bar(
+            x=report_df['Date'],
+            y=report_df['Data Breach'],
+            name='Data Breach',
+            marker_color='#E74C3C'
+        ))
+        
+        fig.add_trace(go.Bar(
+            x=report_df['Date'],
+            y=report_df['Technical'],
+            name='Technical',
+            marker_color='#2ECC71'
+        ))
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            barmode='stack',
+            xaxis=dict(
+                title="Date",
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title="Number of Reports",
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=1.02,
+                xanchor="right",
+                x=1,
+                font=dict(color='#ECF0F1')
+            ),
+            height=400
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with analytics_tab2:
+        # Report distribution pie chart
+        st.subheader("Report Distribution by Recipient")
+        
+        distribution_data = {
+            'Recipient': ['Security Team', 'Executive Team', 'IT Department', 'Legal Department', 'Other'],
+            'Count': [45, 23, 31, 15, 8]
+        }
+        
+        dist_df = pd.DataFrame(distribution_data)
+        
+        fig = px.pie(
+            dist_df,
+            values='Count',
+            names='Recipient',
+            hole=0.4,
+            color_discrete_sequence=['#3498DB', '#9B59B6', '#2ECC71', '#F1C40F', '#E74C3C']
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            showlegend=True,
+            legend=dict(
+                orientation="h",
+                yanchor="bottom",
+                y=-0.2,
+                xanchor="center",
+                x=0.5,
+                font=dict(color='#ECF0F1')
+            ),
+            margin=dict(l=0, r=0, t=0, b=10),
+            height=350
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    # Sample report view
+    st.markdown("---")
+    st.subheader("Sample Report Preview")
+    
+    # Report header
+    st.markdown("# Healthcare Data Breach Intelligence Report")
+    st.markdown("**Report ID:** RPT-2025-04083")
+    st.markdown("**Date:** April 8, 2025")
+    st.markdown("**Classification:** Confidential")
+    st.markdown("**Severity:** Critical")
+    
+    # Table of contents
+    st.markdown("## Table of Contents")
+    st.markdown("""
+    1. Executive Summary
+    2. Breach Details
+    3. Affected Data
+    4. Threat Actor Analysis
+    5. Timeline of Events
+    6. Technical Indicators
+    7. Recommendations
+    8. References
+    """)
+    
+    # Executive Summary
+    st.markdown("## 1. Executive Summary")
+    st.markdown("""
+    On April 7, 2025, CyberForge OSINT Platform detected evidence of a significant data breach affecting Memorial Hospital. 
+    Patient records containing personally identifiable information (PII) and protected health information (PHI) were 
+    discovered for sale on a prominent dark web marketplace. Initial analysis indicates approximately 50,000 patient 
+    records may be affected. This report provides detailed analysis of the breach, indicators of compromise, and 
+    recommended actions.
+    """)
+    
+    # Key findings
+    st.info("""
+    **Key Findings:**
+    
+    * Patient data including names, addresses, social security numbers, and medical records are being offered for sale
+    * The threat actor appears to be affiliated with the BlackCat ransomware group
+    * Initial access likely occurred between March 15-20, 2025
+    * The breach has not yet been publicly disclosed by the healthcare provider
+    * Similar tactics have been observed in other healthcare breaches in the past 60 days
+    """)
+    
+    # Breach details
+    st.markdown("## 2. Breach Details")
+    st.markdown("""
+    The data breach was detected on April 7, 2025, at 22:03 UTC when our monitoring system identified a new listing 
+    on AlphaBay marketplace offering "Complete patient database from major US hospital" for sale. The listing specifically 
+    mentioned Memorial Hospital by name and included sample data as proof of the breach. The seller, operating under the 
+    username "MedLeaks", is requesting 45 BTC (approximately $1.8 million USD) for the complete dataset.
+    """)
+    
+    # Sample chart
+    affected_data = {
+        'Data Type': ['Medical Records', 'Personally Identifiable Information', 'Insurance Information', 'Billing Information', 'Staff Credentials'],
+        'Records': [42000, 50000, 38000, 35000, 1200]
+    }
+    
+    affected_df = pd.DataFrame(affected_data)
+    
+    fig = px.bar(
+        affected_df,
+        x='Records',
+        y='Data Type',
+        orientation='h',
+        color='Records',
+        color_continuous_scale=['#3498DB', '#F1C40F', '#E74C3C'],
+        height=300
+    )
+    
+    fig.update_layout(
+        paper_bgcolor='rgba(26, 26, 26, 0)',
+        plot_bgcolor='rgba(26, 26, 26, 0)',
+        coloraxis_showscale=False,
+        xaxis=dict(
+            title="Number of Records",
+            showgrid=True,
+            gridcolor='rgba(44, 62, 80, 0.3)',
+            tickfont=dict(color='#ECF0F1')
+        ),
+        yaxis=dict(
+            title=None,
+            showgrid=False,
+            tickfont=dict(color='#ECF0F1')
+        ),
+        margin=dict(l=0, r=0, t=10, b=0)
+    )
+    
+    st.plotly_chart(fig, use_container_width=True)
+    
+    # Report actions
+    action_col1, action_col2, action_col3 = st.columns(3)
+    
+    with action_col1:
+        st.download_button(
+            label="Download Full Report",
+            data="This is a placeholder for the full report download",
+            file_name="Healthcare_Data_Breach_Report.pdf",
+            mime="application/pdf"
+        )
+    
+    with action_col2:
+        st.button("Share Report", key="share_report")
+    
+    with action_col3:
+        st.button("Print Report", key="print_report")

components/search_trends.py

@@ -0,0 +1,684 @@
+"""
+Search History and Trends Component
+
+This component provides UI for displaying and analyzing search history and trends.
+"""
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+from datetime import datetime, timedelta
+import asyncio
+import json
+from typing import Dict, List, Any, Optional
+import random
+
+from src.api.services.search_history_service import (
+    get_search_history,
+    get_trending_topics,
+    get_search_trend_analysis,
+    get_popular_searches,
+    add_search_history,
+    save_search,
+    create_saved_search,
+    get_saved_searches
+)
+
+# For demo/placeholder data when database is not populated
+def generate_demo_trends():
+    """Generate demo trend data"""
+    topics = [
+        "ransomware", "databreach", "malware", "phishing", "zeroday",
+        "darkmarket", "cryptolocker", "anonymity", "botnet", "exploit",
+        "vulnerability", "trojan", "blackmarket", "identity", "creditcard",
+        "hacking", "ddos", "credentials", "bitcoin", "monero"
+    ]
+    
+    return [
+        {
+            "topic": topic,
+            "mentions": random.randint(5, 100),
+            "growth_rate": random.uniform(0.5, 25.0)
+        }
+        for topic in random.sample(topics, min(len(topics), 10))
+    ]
+
+def generate_demo_search_data(days=30):
+    """Generate demo search frequency data"""
+    base_date = datetime.now() - timedelta(days=days)
+    dates = [base_date + timedelta(days=i) for i in range(days)]
+    
+    base_count = 10
+    trend = [random.randint(max(0, base_count-5), base_count+15) for _ in range(days)]
+    # Add a spike for visual interest
+    spike_day = random.randint(5, days-5)
+    trend[spike_day] = trend[spike_day] * 3
+    
+    return [
+        {"interval": date, "count": count}
+        for date, count in zip(dates, trend)
+    ]
+
+def generate_demo_search_categories():
+    """Generate demo search categories data"""
+    categories = [
+        "Marketplace", "Forum", "Data Breach", "Hacking Tools", 
+        "Credential Dumps", "Crypto", "Scam", "Uncategorized"
+    ]
+    return [
+        {"category": cat, "count": random.randint(10, 100)}
+        for cat in categories
+    ]
+
+def generate_demo_popular_searches():
+    """Generate demo popular searches data"""
+    searches = [
+        "ransomware as a service", "credit card dumps", "personal data breach",
+        "hacking tools", "bank account access", "identity documents", "covid vaccine cards",
+        "social security numbers", "corporate credentials", "zero day exploits"
+    ]
+    return [
+        {"query": query, "count": random.randint(5, 50)}
+        for query in searches
+    ]
+
+async def get_trend_data(days=90, trend_days=7, limit=10):
+    """Get trend data from the database"""
+    try:
+        # Create a session without context manager
+        from src.streamlit_database import async_session
+        session = async_session()
+        
+        try:
+            data = await get_search_trend_analysis(
+                db=session,
+                days=days,
+                trend_days=trend_days,
+                limit=limit
+            )
+            await session.commit()
+            return data
+        except Exception as e:
+            await session.rollback()
+            raise e
+        finally:
+            await session.close()
+    except Exception as e:
+        st.error(f"Error fetching trend data: {e}")
+        # Use demo data as fallback
+        return {
+            "frequency": generate_demo_search_data(days),
+            "popular_searches": generate_demo_popular_searches(),
+            "trending_topics": generate_demo_trends(),
+            "categories": generate_demo_search_categories(),
+            "recent_popular": generate_demo_popular_searches(),
+            "velocity": random.uniform(-10, 30),
+            "total_searches": {
+                "total": 1000,
+                "recent": 400,
+                "previous": 600
+            }
+        }
+
+async def save_search_query(query, user_id=None, category=None, tags=None):
+    """Save a search query to the database"""
+    try:
+        # Create a session without context manager
+        from src.streamlit_database import async_session
+        session = async_session()
+        
+        try:
+            search = await add_search_history(
+                db=session,
+                query=query,
+                user_id=user_id,
+                category=category,
+                tags=tags,
+                result_count=random.randint(5, 100)  # Placeholder
+            )
+            await session.commit()
+            return search
+        except Exception as e:
+            await session.rollback()
+            raise e
+        finally:
+            await session.close()
+    except Exception as e:
+        st.error(f"Error saving search: {e}")
+        return None
+
+async def get_user_searches(user_id=None, limit=50):
+    """Get search history for a user"""
+    try:
+        # Create a session without context manager
+        from src.streamlit_database import async_session
+        session = async_session()
+        
+        try:
+            searches = await get_search_history(
+                db=session,
+                user_id=user_id,
+                limit=limit
+            )
+            await session.commit()
+            return searches
+        except Exception as e:
+            await session.rollback()
+            raise e
+        finally:
+            await session.close()
+    except Exception as e:
+        st.error(f"Error fetching search history: {e}")
+        return []
+
+async def get_user_saved_searches(user_id=None):
+    """Get saved searches for a user"""
+    try:
+        # Create a session without context manager
+        from src.streamlit_database import async_session
+        session = async_session()
+        
+        try:
+            searches = await get_saved_searches(
+                db=session,
+                user_id=user_id
+            )
+            await session.commit()
+            return searches
+        except Exception as e:
+            await session.rollback()
+            raise e
+        finally:
+            await session.close()
+    except Exception as e:
+        st.error(f"Error fetching saved searches: {e}")
+        return []
+
+async def create_new_saved_search(name, query, user_id=None, frequency=24, category=None):
+    """Create a new saved search"""
+    try:
+        # Create a session without context manager
+        from src.streamlit_database import async_session
+        session = async_session()
+        
+        try:
+            saved_search = await create_saved_search(
+                db=session,
+                name=name,
+                query=query,
+                user_id=user_id or 1,  # Default user ID
+                frequency=frequency,
+                category=category
+            )
+            await session.commit()
+            return saved_search
+        except Exception as e:
+            await session.rollback()
+            raise e
+        finally:
+            await session.close()
+    except Exception as e:
+        st.error(f"Error creating saved search: {e}")
+        return None
+
+def plot_search_trends(frequency_data):
+    """Create a plot of search frequency over time"""
+    if not frequency_data:
+        return None
+    
+    df = pd.DataFrame(frequency_data)
+    if 'interval' in df.columns:
+        df['interval'] = pd.to_datetime(df['interval'])
+        
+        fig = px.line(
+            df, 
+            x='interval', 
+            y='count',
+            title='Search Frequency Over Time',
+            labels={'interval': 'Date', 'count': 'Number of Searches'},
+            template='plotly_dark'
+        )
+        
+        fig.update_layout(
+            xaxis_title="Date",
+            yaxis_title="Number of Searches",
+            plot_bgcolor='rgba(17, 17, 17, 0.8)',
+            paper_bgcolor='rgba(17, 17, 17, 0)',
+            font=dict(color='white')
+        )
+        
+        return fig
+    
+    return None
+
+def plot_category_distribution(category_data):
+    """Create a plot of search categories distribution"""
+    if not category_data:
+        return None
+    
+    df = pd.DataFrame(category_data)
+    
+    fig = px.pie(
+        df, 
+        values='count', 
+        names='category',
+        title='Search Categories Distribution',
+        template='plotly_dark',
+        hole=0.4
+    )
+    
+    fig.update_layout(
+        plot_bgcolor='rgba(17, 17, 17, 0.8)',
+        paper_bgcolor='rgba(17, 17, 17, 0)',
+        font=dict(color='white')
+    )
+    
+    return fig
+
+def plot_trending_topics(trending_data):
+    """Create a bar chart of trending topics"""
+    if not trending_data:
+        return None
+    
+    df = pd.DataFrame(trending_data)
+    if len(df) == 0:
+        return None
+    
+    # Sort by mentions or growth rate
+    df = df.sort_values('growth_rate', ascending=False)
+    
+    fig = px.bar(
+        df, 
+        y='topic', 
+        x='growth_rate',
+        title='Trending Topics by Growth Rate',
+        labels={'topic': 'Topic', 'growth_rate': 'Growth Rate (%)'},
+        orientation='h',
+        template='plotly_dark',
+        color='growth_rate',
+        color_continuous_scale='Viridis'
+    )
+    
+    fig.update_layout(
+        xaxis_title="Growth Rate (%)",
+        yaxis_title="Topic",
+        plot_bgcolor='rgba(17, 17, 17, 0.8)',
+        paper_bgcolor='rgba(17, 17, 17, 0)',
+        font=dict(color='white'),
+        yaxis={'categoryorder': 'total ascending'}
+    )
+    
+    return fig
+
+def plot_popular_searches(popular_data):
+    """Create a bar chart of popular searches"""
+    if not popular_data:
+        return None
+    
+    df = pd.DataFrame(popular_data)
+    if len(df) == 0:
+        return None
+    
+    df = df.sort_values('count', ascending=True)
+    
+    fig = px.bar(
+        df, 
+        y='query', 
+        x='count',
+        title='Most Popular Search Terms',
+        labels={'query': 'Search Term', 'count': 'Number of Searches'},
+        orientation='h',
+        template='plotly_dark'
+    )
+    
+    fig.update_layout(
+        xaxis_title="Number of Searches",
+        yaxis_title="Search Term",
+        plot_bgcolor='rgba(17, 17, 17, 0.8)',
+        paper_bgcolor='rgba(17, 17, 17, 0)',
+        font=dict(color='white'),
+        yaxis={'categoryorder': 'total ascending'}
+    )
+    
+    return fig
+
+def render_search_box():
+    """Render the search box component"""
+    st.markdown("### Search Dark Web Content")
+    
+    col1, col2 = st.columns([3, 1])
+    
+    with col1:
+        search_query = st.text_input("Enter search terms", placeholder="Enter keywords to search dark web content...")
+    
+    with col2:
+        categories = ["All Categories", "Marketplace", "Forum", "Paste Site", "Data Breach", "Hacking", "Cryptocurrency"]
+        selected_category = st.selectbox("Category", categories, index=0)
+        
+        if selected_category == "All Categories":
+            selected_category = None
+    
+    advanced_options = st.expander("Advanced Search Options", expanded=False)
+    with advanced_options:
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            date_range = st.selectbox(
+                "Date Range",
+                ["All Time", "Last 24 Hours", "Last 7 Days", "Last 30 Days", "Last 90 Days", "Custom Range"]
+            )
+            
+            include_images = st.checkbox("Include Images", value=False)
+            include_code = st.checkbox("Include Code Snippets", value=True)
+        
+        with col2:
+            sources = st.multiselect(
+                "Sources",
+                ["Dark Forums", "Marketplaces", "Paste Sites", "Leak Sites", "Chat Channels"],
+                default=["Dark Forums", "Marketplaces", "Leak Sites"]
+            )
+            
+            sort_by = st.selectbox(
+                "Sort Results By",
+                ["Relevance", "Date (Newest First)", "Date (Oldest First)"]
+            )
+    
+    tags_input = st.text_input("Tags (comma-separated)", placeholder="Add tags to organize your search...")
+    
+    search_button = st.button("Search Dark Web")
+    
+    if search_button and search_query:
+        # Save search to history
+        user_id = getattr(st.session_state, "user_id", None)
+        
+        # Process tags
+        tags = tags_input.strip() if tags_input else None
+        
+        # Run the search
+        with st.spinner("Searching dark web..."):
+            search = asyncio.run(save_search_query(
+                query=search_query,
+                user_id=user_id,
+                category=selected_category,
+                tags=tags
+            ))
+            
+            if search:
+                st.success(f"Search completed: Found {search.result_count} results for '{search_query}'")
+                # In a real application, we would display results here
+                
+                # Offer to save as a monitored search
+                save_col1, save_col2 = st.columns([3, 1])
+                with save_col1:
+                    search_name = st.text_input(
+                        "Save this search for monitoring (enter a name)",
+                        placeholder="My saved search"
+                    )
+                with save_col2:
+                    frequency = st.selectbox(
+                        "Check frequency",
+                        ["Manual only", "Daily", "Every 12 hours", "Every 6 hours", "Hourly"],
+                        index=1
+                    )
+                    
+                    # Map to hours
+                    freq_mapping = {
+                        "Manual only": 0,
+                        "Daily": 24,
+                        "Every 12 hours": 12,
+                        "Every 6 hours": 6,
+                        "Hourly": 1
+                    }
+                    freq_hours = freq_mapping.get(frequency, 24)
+                
+                if st.button("Save for Monitoring"):
+                    if search_name:
+                        saved = asyncio.run(create_new_saved_search(
+                            name=search_name,
+                            query=search_query,
+                            user_id=user_id,
+                            frequency=freq_hours,
+                            category=selected_category
+                        ))
+                        
+                        if saved:
+                            st.success(f"Saved search '{search_name}' created successfully!")
+                    else:
+                        st.error("Please enter a name for your saved search")
+            else:
+                st.error("Failed to perform search. Please try again.")
+
+def render_search_history():
+    """Render the search history component"""
+    st.markdown("### Your Search History")
+    
+    user_id = getattr(st.session_state, "user_id", None)
+    
+    # Fetch search history
+    searches = asyncio.run(get_user_searches(user_id))
+    
+    if not searches:
+        st.info("No search history found. Try searching for dark web content.")
+        return
+    
+    # Convert to DataFrame for display
+    search_data = []
+    for search in searches:
+        search_data.append({
+            "ID": search.id,
+            "Query": search.query,
+            "Date": search.timestamp.strftime("%Y-%m-%d %H:%M"),
+            "Results": search.result_count,
+            "Category": search.category or "All",
+            "Saved": "✓" if search.is_saved else ""
+        })
+    
+    df = pd.DataFrame(search_data)
+    
+    # Display as table
+    st.dataframe(
+        df,
+        use_container_width=True,
+        column_config={
+            "ID": st.column_config.NumberColumn(format="%d"),
+            "Query": st.column_config.TextColumn(),
+            "Date": st.column_config.DatetimeColumn(),
+            "Results": st.column_config.NumberColumn(),
+            "Category": st.column_config.TextColumn(),
+            "Saved": st.column_config.TextColumn()
+        }
+    )
+
+def render_saved_searches():
+    """Render the saved searches component"""
+    st.markdown("### Saved Searches")
+    
+    user_id = getattr(st.session_state, "user_id", None)
+    
+    # Fetch saved searches
+    saved_searches = asyncio.run(get_user_saved_searches(user_id))
+    
+    if not saved_searches:
+        st.info("No saved searches found. Save a search to monitor for new results.")
+        return
+    
+    # Convert to DataFrame for display
+    search_data = []
+    for search in saved_searches:
+        # Calculate next run time
+        if search.last_run_at and search.frequency > 0:
+            next_run = search.last_run_at + timedelta(hours=search.frequency)
+        else:
+            next_run = "Manual only"
+        
+        search_data.append({
+            "ID": search.id,
+            "Name": search.name,
+            "Query": search.query,
+            "Category": search.category or "All",
+            "Frequency": f"{search.frequency}h" if search.frequency > 0 else "Manual",
+            "Last Run": search.last_run_at.strftime("%Y-%m-%d %H:%M") if search.last_run_at else "Never",
+            "Next Run": next_run if isinstance(next_run, str) else next_run.strftime("%Y-%m-%d %H:%M"),
+            "Status": "Active" if search.is_active else "Paused"
+        })
+    
+    df = pd.DataFrame(search_data)
+    
+    # Display as table
+    st.dataframe(
+        df,
+        use_container_width=True
+    )
+    
+    # Action buttons
+    col1, col2, col3 = st.columns(3)
+    
+    with col1:
+        if st.button("Run Selected Searches Now"):
+            st.info("This would trigger manual execution of selected searches")
+    
+    with col2:
+        if st.button("Pause Selected"):
+            st.info("This would pause the selected searches")
+    
+    with col3:
+        if st.button("Delete Selected"):
+            st.info("This would delete the selected searches")
+
+def render_trend_dashboard():
+    """Render the trend dashboard component"""
+    st.markdown("## Search Trends Analysis")
+    
+    # Time period selector
+    col1, col2 = st.columns([1, 3])
+    with col1:
+        time_period = st.selectbox(
+            "Time Period",
+            ["Last 7 Days", "Last 30 Days", "Last 90 Days", "Last Year"],
+            index=1
+        )
+        
+        # Map to days
+        period_mapping = {
+            "Last 7 Days": 7,
+            "Last 30 Days": 30,
+            "Last 90 Days": 90,
+            "Last Year": 365
+        }
+        days = period_mapping.get(time_period, 30)
+    
+    with col2:
+        st.markdown("")  # Spacing
+    
+    # Fetch trend data
+    with st.spinner("Loading trend data..."):
+        trend_data = asyncio.run(get_trend_data(days=days))
+    
+    # Create layout for visualizations
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        search_trend_fig = plot_search_trends(trend_data.get("frequency", []))
+        if search_trend_fig:
+            st.plotly_chart(search_trend_fig, use_container_width=True)
+        else:
+            st.error("Failed to load search trend data")
+        
+        popular_searches_fig = plot_popular_searches(trend_data.get("popular_searches", []))
+        if popular_searches_fig:
+            st.plotly_chart(popular_searches_fig, use_container_width=True)
+        else:
+            st.error("Failed to load popular searches data")
+    
+    with col2:
+        trending_topics_fig = plot_trending_topics(trend_data.get("trending_topics", []))
+        if trending_topics_fig:
+            st.plotly_chart(trending_topics_fig, use_container_width=True)
+        else:
+            st.error("Failed to load trending topics data")
+        
+        category_fig = plot_category_distribution(trend_data.get("categories", []))
+        if category_fig:
+            st.plotly_chart(category_fig, use_container_width=True)
+        else:
+            st.error("Failed to load category distribution data")
+    
+    # Display trend insights
+    st.markdown("### Trend Insights")
+    
+    col1, col2, col3 = st.columns(3)
+    
+    with col1:
+        velocity = trend_data.get("velocity", 0)
+        velocity_color = "green" if velocity > 0 else "red"
+        velocity_icon = "↗️" if velocity > 0 else "↘️"
+        st.markdown(f"""
+        ### Search Velocity
+        <h2 style="color:{velocity_color}">{velocity_icon} {abs(velocity):.1f}%</h2>
+        <p>Change in search volume compared to previous period</p>
+        """, unsafe_allow_html=True)
+    
+    with col2:
+        total_searches = trend_data.get("total_searches", {}).get("total", 0)
+        st.markdown(f"""
+        ### Total Searches
+        <h2>{total_searches:,}</h2>
+        <p>Total searches in the selected period</p>
+        """, unsafe_allow_html=True)
+    
+    with col3:
+        top_topic = "None"
+        top_growth = 0
+        if trend_data.get("trending_topics"):
+            top_item = max(trend_data["trending_topics"], key=lambda x: x.get("growth_rate", 0))
+            top_topic = top_item.get("topic", "None")
+            top_growth = top_item.get("growth_rate", 0)
+        
+        st.markdown(f"""
+        ### Fastest Growing Topic
+        <h2>{top_topic}</h2>
+        <p>Growth rate: {top_growth:.1f}%</p>
+        """, unsafe_allow_html=True)
+    
+    # Display emerging themes (if available)
+    if trend_data.get("trending_topics"):
+        st.markdown("### Emerging Dark Web Themes")
+        
+        # Group topics by similar growth rates
+        topics = trend_data["trending_topics"]
+        
+        # Display as topic clusters with common themes
+        theme_groups = {
+            "High Growth": [t for t in topics if t.get("growth_rate", 0) > 15],
+            "Moderate Growth": [t for t in topics if 5 <= t.get("growth_rate", 0) <= 15],
+            "Stable": [t for t in topics if t.get("growth_rate", 0) < 5]
+        }
+        
+        for theme, items in theme_groups.items():
+            if items:
+                st.markdown(f"#### {theme}")
+                themes_text = ", ".join([f"{t.get('topic')} ({t.get('growth_rate', 0):.1f}%)" for t in items])
+                st.markdown(f"<p>{themes_text}</p>", unsafe_allow_html=True)
+
+def render_search_trends():
+    """Main function to render the search trends component"""
+    st.title("Dark Web Search & Trends")
+    
+    tabs = st.tabs([
+        "Search Dark Web", 
+        "Search History", 
+        "Saved Searches",
+        "Trend Analysis"
+    ])
+    
+    with tabs[0]:
+        render_search_box()
+    
+    with tabs[1]:
+        render_search_history()
+    
+    with tabs[2]:
+        render_saved_searches()
+    
+    with tabs[3]:
+        render_trend_dashboard()

components/subscriptions.py

@@ -0,0 +1,478 @@
+"""
+Subscription management component.
+
+This component provides UI for managing subscription plans.
+"""
+import os
+import streamlit as st
+import pandas as pd
+from datetime import datetime
+import json
+
+import stripe
+from streamlit_extras.colored_header import colored_header
+from streamlit_extras.metric_cards import style_metric_cards
+
+from src.streamlit_subscription_services import (
+    get_subscription_plans_df,
+    get_subscription_plan,
+    get_user_current_subscription,
+    subscribe_user_to_plan,
+    cancel_subscription,
+    initialize_default_plans
+)
+
+# Set up Stripe publishable key for client-side usage
+STRIPE_PUBLISHABLE_KEY = os.environ.get("STRIPE_PUBLISHABLE_KEY")
+
+
+def format_price(price):
+    """Format price display."""
+    if price == 0:
+        return "Free"
+    return f"${price:.2f}"
+
+
+def render_pricing_card(plan, selected_period="monthly"):
+    """Render a pricing card for a subscription plan."""
+    plan_id = plan["id"]
+    plan_name = plan["name"]
+    plan_tier = plan["tier"]
+    description = plan["description"]
+    
+    # Determine price based on selected period
+    if selected_period == "monthly":
+        price = plan["price_monthly"]
+        period_text = "per month"
+        billing_term = "monthly"
+    else:
+        price = plan["price_annually"]
+        period_text = "per year"
+        billing_term = "annually"
+    
+    # Format price for display
+    price_display = format_price(price)
+    
+    # Feature list
+    features = [
+        f"✓ {plan['max_alerts'] if plan['max_alerts'] > 0 else 'Unlimited'} alerts",
+        f"✓ {plan['max_reports'] if plan['max_reports'] > 0 else 'Unlimited'} reports",
+        f"✓ {plan['max_searches_per_day'] if plan['max_searches_per_day'] > 0 else 'Unlimited'} searches per day",
+        f"✓ {plan['max_monitoring_keywords'] if plan['max_monitoring_keywords'] > 0 else 'Unlimited'} monitoring keywords",
+        f"✓ {plan['max_data_retention_days']} days data retention"
+    ]
+    
+    if plan["supports_api_access"]:
+        features.append("✓ API access")
+    
+    if plan["supports_live_feed"]:
+        features.append("✓ Live feed")
+    
+    if plan["supports_dark_web_monitoring"]:
+        features.append("✓ Dark web monitoring")
+    
+    if plan["supports_export"]:
+        features.append("✓ Data export")
+    
+    if plan["supports_advanced_analytics"]:
+        features.append("✓ Advanced analytics")
+    
+    # Card style based on tier
+    if plan_tier == "free":
+        border_color = "#3498db"  # Blue
+        header_color = "#3498db"
+    elif plan_tier == "basic":
+        border_color = "#2ecc71"  # Green
+        header_color = "#2ecc71"
+    elif plan_tier == "professional":
+        border_color = "#f39c12"  # Orange
+        header_color = "#f39c12"
+    else:  # Enterprise
+        border_color = "#9b59b6"  # Purple
+        header_color = "#9b59b6"
+    
+    # Render card
+    st.markdown(f"""
+    <div style="border: 2px solid {border_color}; border-radius: 10px; padding: 20px; height: 100%;">
+        <h3 style="color: {header_color}; text-align: center;">{plan_name}</h3>
+        <h2 style="text-align: center; margin-top: 10px; margin-bottom: 5px;">{price_display}</h2>
+        <p style="text-align: center; color: #999; margin-bottom: 20px;">{period_text}</p>
+        <p style="text-align: center; margin-bottom: 20px;">{description}</p>
+        <div style="margin-bottom: 20px;">
+            {"<br>".join([f'<div style="margin-bottom: 8px;">{feature}</div>' for feature in features])}
+        </div>
+    </div>
+    """, unsafe_allow_html=True)
+    
+    # Subscribe button
+    if st.button(f"Choose {plan_name}", key=f"choose_{plan_id}_{selected_period}"):
+        if not plan_tier == "free" and STRIPE_PUBLISHABLE_KEY:
+            st.session_state.show_payment_form = True
+            st.session_state.selected_plan = plan
+            st.session_state.selected_billing_period = billing_term
+        else:
+            # Free plan - no payment needed
+            # Assume user ID 1 for demonstration
+            user_id = 1
+            subscription = subscribe_user_to_plan(
+                user_id=user_id,
+                plan_id=plan_id,
+                billing_period=billing_term,
+                create_stripe_subscription=False
+            )
+            
+            if subscription:
+                st.success(f"You're now subscribed to the {plan_name} plan!")
+                st.session_state.current_user_subscription = subscription
+            else:
+                st.error("Failed to subscribe. Please try again.")
+            
+            st.rerun()
+
+
+def render_payment_form():
+    """Render the payment form for subscription."""
+    if not STRIPE_PUBLISHABLE_KEY:
+        st.error("Stripe API key is not configured. Payment processing is unavailable.")
+        return
+    
+    plan = st.session_state.selected_plan
+    billing_period = st.session_state.selected_billing_period
+    
+    st.markdown("### Payment Information")
+    
+    # Calculate amount based on billing period
+    if billing_period == "monthly":
+        amount = plan["price_monthly"]
+    else:
+        amount = plan["price_annually"]
+    
+    st.markdown(f"You're subscribing to the **{plan['name']} plan** ({billing_period}) for **{format_price(amount)}**.")
+    
+    # Name and email inputs
+    col1, col2 = st.columns(2)
+    
+    with col1:
+        name = st.text_input("Full Name")
+    
+    with col2:
+        email = st.text_input("Email Address")
+    
+    # HTML/JS for Stripe Elements
+    st.markdown("""
+    <div id="card-element" style="padding: 10px; border: 1px solid #ccc; border-radius: 4px; margin-bottom: 20px;"></div>
+    <div id="card-errors" style="color: #e74c3c; margin-bottom: 20px;"></div>
+    
+    <script src="https://js.stripe.com/v3/"></script>
+    <script type="text/javascript">
+        // Initialize Stripe with publishable key
+        var stripe = Stripe('%s');
+        var elements = stripe.elements();
+        
+        // Create card element
+        var card = elements.create('card');
+        card.mount('#card-element');
+        
+        // Handle real-time validation errors
+        card.addEventListener('change', function(event) {
+            var displayError = document.getElementById('card-errors');
+            if (event.error) {
+                displayError.textContent = event.error.message;
+            } else {
+                displayError.textContent = '';
+            }
+        });
+        
+        // Set up payment method creation
+        window.createPaymentMethod = function() {
+            stripe.createPaymentMethod({
+                type: 'card',
+                card: card,
+                billing_details: {
+                    name: '%s',
+                    email: '%s'
+                }
+            }).then(function(result) {
+                if (result.error) {
+                    var errorElement = document.getElementById('card-errors');
+                    errorElement.textContent = result.error.message;
+                } else {
+                    // Send payment method ID to Streamlit
+                    window.parent.postMessage({
+                        type: 'payment-method-created',
+                        paymentMethodId: result.paymentMethod.id
+                    }, '*');
+                }
+            });
+        }
+    </script>
+    """ % (STRIPE_PUBLISHABLE_KEY, name, email), unsafe_allow_html=True)
+    
+    # Submit button
+    if st.button("Subscribe Now", key="subscribe_button"):
+        # Call JavaScript function to create payment method
+        st.markdown("""
+        <script>
+            window.createPaymentMethod();
+        </script>
+        """, unsafe_allow_html=True)
+        
+        # In a real implementation, we would need to handle the callback from Stripe
+        # For now, simulate success for demonstration
+        payment_method_id = "pm_" + "".join([str(i) for i in range(24)])  # Fake payment method ID
+        
+        # Subscribe user with payment method
+        # Assume user ID 1 for demonstration
+        user_id = 1
+        subscription = subscribe_user_to_plan(
+            user_id=user_id,
+            plan_id=plan["id"],
+            billing_period=billing_period,
+            create_stripe_subscription=True,
+            payment_method_id=payment_method_id
+        )
+        
+        if subscription:
+            st.success(f"You're now subscribed to the {plan['name']} plan!")
+            st.session_state.show_payment_form = False
+            st.session_state.current_user_subscription = subscription
+        else:
+            st.error("Failed to subscribe. Please try again.")
+        
+        st.rerun()
+    
+    # Cancel button
+    if st.button("Cancel", key="cancel_payment"):
+        st.session_state.show_payment_form = False
+        st.rerun()
+
+
+def render_subscription_dashboard(user_id=1):
+    """Render the subscription dashboard for the current user."""
+    # Get current subscription
+    subscription = get_user_current_subscription(user_id)
+    st.session_state.current_user_subscription = subscription
+    
+    if subscription:
+        plan_tier = subscription.get("plan_tier", "").capitalize()
+        plan_name = subscription.get("plan_name", "Unknown Plan")
+        status = subscription.get("status", "").capitalize()
+        billing_period = subscription.get("billing_period", "").capitalize()
+        
+        period_start = subscription.get("current_period_start")
+        period_end = subscription.get("current_period_end")
+        
+        # Format dates
+        start_date = period_start.strftime("%B %d, %Y") if period_start else "N/A"
+        end_date = period_end.strftime("%B %d, %Y") if period_end else "N/A"
+        
+        st.markdown(f"### Current Subscription: {plan_name}")
+        
+        col1, col2, col3 = st.columns(3)
+        
+        with col1:
+            st.metric("Status", status)
+        
+        with col2:
+            st.metric("Billing Period", billing_period)
+        
+        with col3:
+            days_left = (period_end - datetime.now()).days if period_end else 0
+            days_left = max(0, days_left)
+            st.metric("Days Remaining", days_left)
+        
+        style_metric_cards()
+        
+        st.markdown(f"""
+        <div style="margin-top: 20px; padding: 15px; background-color: rgba(44, 62, 80, 0.2); border-radius: 6px;">
+            <p><strong>Billing Period:</strong> {start_date} to {end_date}</p>
+        </div>
+        """, unsafe_allow_html=True)
+        
+        # Cancel subscription button
+        if status.lower() != "canceled":
+            if st.button("Cancel Subscription", key="cancel_subscription"):
+                if cancel_subscription(subscription["id"]):
+                    st.success("Your subscription has been canceled. You'll still have access until the end of your billing period.")
+                    st.rerun()
+                else:
+                    st.error("Failed to cancel subscription. Please try again.")
+    
+    # View other plans button
+    if st.button("View Available Plans", key="view_plans"):
+        st.session_state.show_pricing_table = True
+        st.rerun()
+
+
+def render_subscription_metrics(user_id=1):
+    """Render subscription usage metrics for the current user."""
+    # Get current subscription
+    subscription = st.session_state.get("current_user_subscription") or get_user_current_subscription(user_id)
+    
+    if not subscription:
+        return
+    
+    # Get subscription plan
+    plan = get_subscription_plan(subscription["plan_id"])
+    
+    if not plan:
+        return
+    
+    st.markdown("### Usage Metrics")
+    
+    # Create metrics
+    col1, col2 = st.columns(2)
+    col3, col4 = st.columns(2)
+    
+    with col1:
+        max_alerts = plan["max_alerts"]
+        alerts_used = 3  # Placeholder value
+        alerts_percent = (alerts_used / max_alerts * 100) if max_alerts > 0 else 0
+        st.metric("Alerts", f"{alerts_used}/{max_alerts if max_alerts > 0 else '∞'}")
+        st.progress(min(alerts_percent, 100) / 100)
+    
+    with col2:
+        max_reports = plan["max_reports"]
+        reports_used = 1  # Placeholder value
+        reports_percent = (reports_used / max_reports * 100) if max_reports > 0 else 0
+        st.metric("Reports", f"{reports_used}/{max_reports if max_reports > 0 else '∞'}")
+        st.progress(min(reports_percent, 100) / 100)
+    
+    with col3:
+        max_searches = plan["max_searches_per_day"]
+        searches_used = 8  # Placeholder value
+        searches_percent = (searches_used / max_searches * 100) if max_searches > 0 else 0
+        st.metric("Daily Searches", f"{searches_used}/{max_searches if max_searches > 0 else '∞'}")
+        st.progress(min(searches_percent, 100) / 100)
+    
+    with col4:
+        max_keywords = plan["max_monitoring_keywords"]
+        keywords_used = 4  # Placeholder value
+        keywords_percent = (keywords_used / max_keywords * 100) if max_keywords > 0 else 0
+        st.metric("Monitoring Keywords", f"{keywords_used}/{max_keywords if max_keywords > 0 else '∞'}")
+        st.progress(min(keywords_percent, 100) / 100)
+    
+    # List other features
+    st.markdown("### Features")
+    
+    features = []
+    
+    if plan["supports_api_access"]:
+        features.append("✓ API Access")
+    else:
+        features.append("✗ API Access")
+    
+    if plan["supports_live_feed"]:
+        features.append("✓ Live Feed")
+    else:
+        features.append("✗ Live Feed")
+    
+    if plan["supports_dark_web_monitoring"]:
+        features.append("✓ Dark Web Monitoring")
+    else:
+        features.append("✗ Dark Web Monitoring")
+    
+    if plan["supports_export"]:
+        features.append("✓ Data Export")
+    else:
+        features.append("✗ Data Export")
+    
+    if plan["supports_advanced_analytics"]:
+        features.append("✓ Advanced Analytics")
+    else:
+        features.append("✗ Advanced Analytics")
+    
+    # Display features
+    cols = st.columns(len(features))
+    for i, feature in enumerate(features):
+        with cols[i]:
+            if feature.startswith("✓"):
+                st.markdown(f'<div style="text-align: center; color: #2ecc71; font-weight: bold;">{feature}</div>', unsafe_allow_html=True)
+            else:
+                st.markdown(f'<div style="text-align: center; color: #e74c3c; font-weight: bold;">{feature}</div>', unsafe_allow_html=True)
+
+
+def render_pricing_table():
+    """Render a pricing table with all subscription plans."""
+    st.markdown("## Subscription Plans")
+    
+    # Billing period toggle
+    selected_period = st.radio(
+        "Billing Period",
+        ["monthly", "annually"],
+        format_func=lambda x: x.capitalize(),
+        horizontal=True
+    )
+    
+    # Note about annual savings
+    if selected_period == "annually":
+        st.info("Save up to 20% with annual billing")
+    
+    # Get subscription plans
+    plans_df = get_subscription_plans_df()
+    
+    if plans_df.empty:
+        st.warning("No subscription plans available.")
+        return
+    
+    # Convert DataFrame to list of dictionaries
+    plans = plans_df.to_dict("records")
+    
+    # Create a column for each plan
+    cols = st.columns(len(plans))
+    
+    # Render pricing cards
+    for i, plan in enumerate(plans):
+        with cols[i]:
+            render_pricing_card(plan, selected_period)
+    
+    # Close button
+    if st.button("Back to Dashboard", key="close_pricing"):
+        st.session_state.show_pricing_table = False
+        st.rerun()
+
+
+def render_subscriptions():
+    """
+    Main function to render the subscription management component.
+    """
+    colored_header(
+        label="Subscription Management",
+        description="Manage your subscription and billing",
+        color_name="violet-70"
+    )
+    
+    # Initialize default plans if needed
+    initialize_default_plans()
+    
+    # Initialize session state
+    if "show_pricing_table" not in st.session_state:
+        st.session_state.show_pricing_table = False
+    
+    if "show_payment_form" not in st.session_state:
+        st.session_state.show_payment_form = False
+    
+    if "selected_plan" not in st.session_state:
+        st.session_state.selected_plan = None
+    
+    if "selected_billing_period" not in st.session_state:
+        st.session_state.selected_billing_period = "monthly"
+    
+    if "current_user_subscription" not in st.session_state:
+        st.session_state.current_user_subscription = None
+    
+    # Check if we need to show payment form
+    if st.session_state.show_payment_form:
+        render_payment_form()
+        return
+    
+    # Check if we need to show pricing table
+    if st.session_state.show_pricing_table:
+        render_pricing_table()
+        return
+    
+    # Render current subscription status and dashboard
+    render_subscription_dashboard()
+    
+    # Render subscription metrics
+    render_subscription_metrics()

components/threats.py

@@ -0,0 +1,543 @@
+import streamlit as st
+import pandas as pd
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+from datetime import datetime, timedelta
+import altair as alt
+
+def render_threats():
+    st.title("Threat Detection & Analysis")
+    
+    # Filters section
+    with st.container():
+        st.subheader("Threat Filters")
+        
+        col1, col2, col3, col4 = st.columns(4)
+        
+        with col1:
+            severity_filter = st.multiselect(
+                "Severity Level",
+                ["Critical", "High", "Medium", "Low"],
+                default=["Critical", "High"]
+            )
+        
+        with col2:
+            threat_type = st.multiselect(
+                "Threat Type",
+                ["Data Breach", "Ransomware", "Phishing", "Malware", "Identity Theft", "Zero-day Exploit"],
+                default=["Data Breach", "Ransomware"]
+            )
+        
+        with col3:
+            date_range = st.selectbox(
+                "Time Range",
+                ["Last 24 Hours", "Last 7 Days", "Last 30 Days", "Last Quarter", "Custom Range"],
+                index=1
+            )
+        
+        with col4:
+            st.text_input("Search Keywords", placeholder="e.g. healthcare, banking")
+            
+        st.button("Apply Filters", type="primary")
+    
+    # Threat overview metrics
+    st.markdown("### Threat Overview")
+    
+    metric_col1, metric_col2, metric_col3, metric_col4, metric_col5 = st.columns(5)
+    
+    with metric_col1:
+        st.metric(
+            label="Critical Threats",
+            value="8",
+            delta="2",
+            delta_color="inverse"
+        )
+    
+    with metric_col2:
+        st.metric(
+            label="High Threats",
+            value="19",
+            delta="4",
+            delta_color="inverse"
+        )
+    
+    with metric_col3:
+        st.metric(
+            label="Medium Threats",
+            value="35",
+            delta="0",
+            delta_color="normal"
+        )
+    
+    with metric_col4:
+        st.metric(
+            label="Low Threats",
+            value="52",
+            delta="-5",
+            delta_color="normal"
+        )
+    
+    with metric_col5:
+        st.metric(
+            label="Avg. Response Time",
+            value="47m",
+            delta="-13m",
+            delta_color="normal"
+        )
+    
+    # Threat detection visualization
+    tab1, tab2, tab3 = st.tabs(["Threat Timeline", "Category Analysis", "Threat Details"])
+    
+    with tab1:
+        st.subheader("Threat Detection Timeline")
+        
+        # Generate dates and times for the past 14 days with hourly granularity
+        now = datetime.now()
+        timeline_data = []
+        
+        for day in range(14, 0, -1):
+            base_date = now - timedelta(days=day)
+            for hour in range(0, 24, 2):  # Every 2 hours
+                timestamp = base_date + timedelta(hours=hour)
+                
+                # Random threat count for each severity level
+                if np.random.random() > 0.7:  # 30% chance of critical
+                    severity = "Critical"
+                    count = np.random.randint(1, 4)
+                elif np.random.random() > 0.5:  # 20% chance of high
+                    severity = "High"
+                    count = np.random.randint(1, 6)
+                elif np.random.random() > 0.3:  # 20% chance of medium
+                    severity = "Medium"
+                    count = np.random.randint(1, 8)
+                else:  # 30% chance of low
+                    severity = "Low"
+                    count = np.random.randint(1, 10)
+                
+                timeline_data.append({
+                    "timestamp": timestamp,
+                    "severity": severity,
+                    "count": count
+                })
+        
+        timeline_df = pd.DataFrame(timeline_data)
+        
+        # Convert to a format suitable for visualization
+        # Group by date and severity to get counts
+        timeline_df['date'] = timeline_df['timestamp'].dt.strftime('%Y-%m-%d')
+        
+        # Create a scatter plot for the timeline with varying dot sizes based on count
+        fig = px.scatter(
+            timeline_df,
+            x='timestamp',
+            y='severity',
+            size='count',
+            color='severity',
+            color_discrete_map={
+                'Critical': '#E74C3C',
+                'High': '#F1C40F',
+                'Medium': '#3498DB',
+                'Low': '#2ECC71'
+            },
+            hover_data=['count'],
+            height=400
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            xaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                showgrid=False,
+                title=None,
+                tickfont=dict(color='#ECF0F1'),
+                categoryorder='array',
+                categoryarray=['Low', 'Medium', 'High', 'Critical']
+            ),
+            margin=dict(l=10, r=10, t=10, b=10)
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+    
+    with tab2:
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            st.subheader("Threat Categories")
+            
+            # Threat category distribution
+            categories = ['Data Breach', 'Ransomware', 'Phishing', 'Malware', 'Identity Theft', 'Zero-day Exploit']
+            values = [38, 24, 18, 14, 6, 8]
+            
+            category_data = pd.DataFrame({
+                'Category': categories,
+                'Count': values
+            })
+            
+            fig = px.bar(
+                category_data,
+                x='Category',
+                y='Count',
+                color='Count',
+                color_continuous_scale=['#2ECC71', '#3498DB', '#F1C40F', '#E74C3C'],
+                height=350
+            )
+            
+            fig.update_layout(
+                paper_bgcolor='rgba(26, 26, 26, 0)',
+                plot_bgcolor='rgba(26, 26, 26, 0)',
+                coloraxis_showscale=False,
+                xaxis=dict(
+                    title=None,
+                    tickfont=dict(color='#ECF0F1')
+                ),
+                yaxis=dict(
+                    title=None,
+                    showgrid=True,
+                    gridcolor='rgba(44, 62, 80, 0.3)',
+                    tickfont=dict(color='#ECF0F1')
+                ),
+                margin=dict(l=10, r=10, t=10, b=10)
+            )
+            
+            st.plotly_chart(fig, use_container_width=True)
+        
+        with col2:
+            st.subheader("Threat Severity Distribution")
+            
+            # Severity distribution
+            severity_labels = ['Critical', 'High', 'Medium', 'Low']
+            severity_values = [8, 19, 35, 52]
+            
+            fig = px.pie(
+                names=severity_labels,
+                values=severity_values,
+                color=severity_labels,
+                color_discrete_map={
+                    'Critical': '#E74C3C',
+                    'High': '#F1C40F',
+                    'Medium': '#3498DB',
+                    'Low': '#2ECC71'
+                },
+                hole=0.4,
+                height=350
+            )
+            
+            fig.update_layout(
+                paper_bgcolor='rgba(26, 26, 26, 0)',
+                plot_bgcolor='rgba(26, 26, 26, 0)',
+                showlegend=True,
+                legend=dict(
+                    orientation="h",
+                    yanchor="bottom",
+                    y=-0.2,
+                    xanchor="center",
+                    x=0.5
+                ),
+                margin=dict(l=10, r=10, t=10, b=10),
+            )
+            
+            st.plotly_chart(fig, use_container_width=True)
+    
+    with tab3:
+        st.subheader("Active Threat Details")
+        
+        # Create data for the threat details table
+        threat_details = [
+            {
+                "id": "T-2025-0428",
+                "detected": "2025-04-08 14:32:21",
+                "type": "Data Breach",
+                "target": "Healthcare",
+                "severity": "Critical",
+                "status": "Active",
+                "details": "Patient data exposed on dark web marketplace."
+            },
+            {
+                "id": "T-2025-0427",
+                "detected": "2025-04-08 09:17:45",
+                "type": "Ransomware",
+                "target": "Finance",
+                "severity": "Critical",
+                "status": "Active",
+                "details": "New ransomware variant targeting financial institutions."
+            },
+            {
+                "id": "T-2025-0426",
+                "detected": "2025-04-07 22:03:12",
+                "type": "Zero-day Exploit",
+                "target": "Technology",
+                "severity": "High",
+                "status": "Active",
+                "details": "Critical vulnerability in enterprise software being exploited."
+            },
+            {
+                "id": "T-2025-0425",
+                "detected": "2025-04-07 15:45:39",
+                "type": "Phishing",
+                "target": "Government",
+                "severity": "High",
+                "status": "Active",
+                "details": "Sophisticated phishing campaign targeting government employees."
+            },
+            {
+                "id": "T-2025-0424",
+                "detected": "2025-04-07 11:27:03",
+                "type": "Malware",
+                "target": "Multiple",
+                "severity": "Medium",
+                "status": "Active",
+                "details": "New strain of data-stealing malware distributed via email attachments."
+            }
+        ]
+        
+        # Create a dataframe for the table
+        threat_df = pd.DataFrame(threat_details)
+        
+        # Apply colors to severity column
+        def color_severity(val):
+            color_map = {
+                'Critical': '#E74C3C',
+                'High': '#F1C40F',
+                'Medium': '#3498DB',
+                'Low': '#2ECC71'
+            }
+            return f'background-color: {color_map.get(val, "#ECF0F1")}'
+        
+        # Style the dataframe
+        styled_df = threat_df.style.applymap(color_severity, subset=['severity'])
+        
+        # Display the table
+        st.dataframe(styled_df, use_container_width=True, height=300)
+        
+        # Add action buttons below the table
+        col1, col2, col3, col4 = st.columns(4)
+        
+        with col1:
+            st.button("Investigate Selected", key="investigate_btn")
+        
+        with col2:
+            st.button("Mark as Resolved", key="resolve_btn")
+        
+        with col3:
+            st.button("Export Report", key="export_btn")
+        
+        with col4:
+            st.button("Assign to Analyst", key="assign_btn")
+    
+    # Threat intelligence section
+    st.markdown("### Threat Intelligence Analysis")
+    
+    # Tabs for different intelligence views
+    intel_tab1, intel_tab2, intel_tab3 = st.tabs(["Actor Analysis", "Attack Vectors", "Indicators of Compromise"])
+    
+    with intel_tab1:
+        st.subheader("Threat Actor Analysis")
+        
+        # Actor table
+        actor_data = [
+            {
+                "actor": "BlackCat Group",
+                "type": "Ransomware",
+                "activity": "High",
+                "targets": "Healthcare, Finance",
+                "ttps": "Double extortion, DDoS threats",
+                "attribution": "Likely Eastern Europe"
+            },
+            {
+                "actor": "CryptoLock",
+                "type": "Ransomware",
+                "activity": "Medium",
+                "targets": "Manufacturing, Energy",
+                "ttps": "Supply chain attacks",
+                "attribution": "Unknown"
+            },
+            {
+                "actor": "DarkLeaks",
+                "type": "Data Broker",
+                "activity": "High",
+                "targets": "All sectors",
+                "ttps": "Data aggregation, auction site",
+                "attribution": "Multiple affiliates"
+            }
+        ]
+        
+        actor_df = pd.DataFrame(actor_data)
+        st.dataframe(actor_df, use_container_width=True)
+        
+        # Relationship graph placeholder
+        st.subheader("Threat Actor Relationships")
+        st.image("https://images.unsplash.com/photo-1510987836583-e3fb9586c7b3", 
+                 caption="Network analysis of threat actor relationships and infrastructure", 
+                 use_column_width=True)
+        
+    with intel_tab2:
+        st.subheader("Common Attack Vectors")
+        
+        # Attack vector distribution
+        vectors = ['Phishing Email', 'Compromised Credentials', 'Malware Infection', 
+                   'Supply Chain', 'Unpatched Vulnerability', 'Social Engineering']
+        percentages = [35, 28, 15, 10, 8, 4]
+        
+        vector_data = pd.DataFrame({
+            'Vector': vectors,
+            'Percentage': percentages
+        })
+        
+        # Horizontal bar chart for attack vectors
+        fig = px.bar(
+            vector_data,
+            x='Percentage',
+            y='Vector',
+            orientation='h',
+            color='Percentage',
+            color_continuous_scale=['#2ECC71', '#3498DB', '#F1C40F', '#E74C3C'],
+            height=300
+        )
+        
+        fig.update_layout(
+            paper_bgcolor='rgba(26, 26, 26, 0)',
+            plot_bgcolor='rgba(26, 26, 26, 0)',
+            coloraxis_showscale=False,
+            xaxis=dict(
+                title='Percentage of Attacks',
+                showgrid=True,
+                gridcolor='rgba(44, 62, 80, 0.3)',
+                tickfont=dict(color='#ECF0F1')
+            ),
+            yaxis=dict(
+                title=None,
+                showgrid=False,
+                tickfont=dict(color='#ECF0F1')
+            ),
+            margin=dict(l=10, r=10, t=10, b=10)
+        )
+        
+        st.plotly_chart(fig, use_container_width=True)
+        
+        # Technical details section
+        st.subheader("Technical Analysis")
+        
+        vector_tabs = st.tabs(["Phishing", "Malware", "Vulnerabilities"])
+        
+        with vector_tabs[0]:
+            st.markdown("#### Phishing Campaign Analysis")
+            st.markdown("""
+            Recent phishing campaigns observed in dark web forums targeting:
+            - Financial institutions (spoofed login pages)
+            - Healthcare providers (fake patient portals)
+            - Government employees (document sharing lures)
+            
+            **Tactics include:**
+            - Lookalike domains with valid SSL certificates
+            - Evasion of email security through legitimate hosting services
+            - Use of shortened URLs to disguise destinations
+            """)
+            
+        with vector_tabs[1]:
+            st.markdown("#### Malware Analysis")
+            st.markdown("""
+            Prevalent malware families being distributed:
+            - TrickBot (banking trojan with evolving capabilities)
+            - Emotet (modular malware with spam capabilities)
+            - Conti (ransomware with data exfiltration)
+            
+            **Distribution channels:**
+            - Malicious email attachments (Excel files with macros)
+            - Compromised software updates
+            - Drive-by downloads from compromised websites
+            """)
+            
+        with vector_tabs[2]:
+            st.markdown("#### Vulnerability Exploitation")
+            st.markdown("""
+            Critical vulnerabilities being actively exploited:
+            - CVE-2024-1234: Remote code execution in web servers
+            - CVE-2024-5678: Authentication bypass in VPN appliances
+            - CVE-2024-9101: Privilege escalation in enterprise software
+            
+            **Exploitation timeline:**
+            - Average time from disclosure to exploitation: 72 hours
+            - Peak exploitation activity occurs within 2 weeks
+            - Persistence mechanisms often installed for long-term access
+            """)
+    
+    with intel_tab3:
+        st.subheader("Indicators of Compromise (IoCs)")
+        
+        # IoC tabs
+        ioc_tabs = st.tabs(["IP Addresses", "Domains", "File Hashes", "URLs"])
+        
+        with ioc_tabs[0]:
+            ip_data = pd.DataFrame({
+                'IP Address': ['198.51.100.123', '203.0.113.45', '198.51.100.67', '203.0.113.89', '198.51.100.213'],
+                'ASN': ['AS12345', 'AS67890', 'AS12345', 'AS23456', 'AS34567'],
+                'Country': ['Russia', 'China', 'Russia', 'Ukraine', 'Brazil'],
+                'First Seen': ['2025-04-01', '2025-04-03', '2025-04-04', '2025-04-05', '2025-04-07'],
+                'Last Seen': ['2025-04-08', '2025-04-08', '2025-04-08', '2025-04-07', '2025-04-08'],
+                'Associated Malware': ['TrickBot', 'Emotet', 'TrickBot', 'BlackCat', 'Conti']
+            })
+            
+            st.dataframe(ip_data, use_container_width=True)
+            
+        with ioc_tabs[1]:
+            domain_data = pd.DataFrame({
+                'Domain': ['secure-banklogin.com', 'microsoft-update.xyz', 'docusign-view.net', 'healthcare-portal.org', 'service-login.co'],
+                'IP Address': ['198.51.100.123', '203.0.113.45', '198.51.100.67', '203.0.113.89', '198.51.100.213'],
+                'Registrar': ['NameCheap', 'GoDaddy', 'Namecheap', 'Hostinger', 'GoDaddy'],
+                'Created Date': ['2025-03-30', '2025-04-01', '2025-04-02', '2025-04-03', '2025-04-05'],
+                'Classification': ['Phishing', 'Malware C2', 'Phishing', 'Phishing', 'Phishing']
+            })
+            
+            st.dataframe(domain_data, use_container_width=True)
+            
+        with ioc_tabs[2]:
+            hash_data = pd.DataFrame({
+                'File Hash (SHA-256)': [
+                    'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+                    'a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a',
+                    '3f39d5c348e5b79d06e842c114e6cc571583bbf44e4b0ebfda1a01ec05745d43',
+                    'ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb',
+                    '2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae'
+                ],
+                'File Name': ['invoice.doc', 'setup.exe', 'update.exe', 'report.xlsx', 'attachment.pdf'],
+                'File Type': ['DOC', 'EXE', 'EXE', 'XLSX', 'PDF'],
+                'Detection Ratio': ['37/58', '42/58', '29/58', '35/58', '23/58'],
+                'Malware Family': ['Emotet', 'TrickBot', 'Conti', 'Emotet', 'AgentTesla']
+            })
+            
+            st.dataframe(hash_data, use_container_width=True)
+            
+        with ioc_tabs[3]:
+            url_data = pd.DataFrame({
+                'URL': [
+                    'https://secure-banklogin.com/auth/login.php',
+                    'https://microsoft-update.xyz/download/patch.exe',
+                    'https://docusign-view.net/document/invoice.doc',
+                    'https://healthcare-portal.org/patient/login',
+                    'https://service-login.co/auth/reset'
+                ],
+                'Status': ['Active', 'Active', 'Inactive', 'Active', 'Active'],
+                'Classification': ['Phishing', 'Malware Distribution', 'Phishing', 'Phishing', 'Phishing'],
+                'Target': ['Banking Customers', 'General', 'Business', 'Healthcare', 'General'],
+                'First Reported': ['2025-04-02', '2025-04-03', '2025-04-04', '2025-04-06', '2025-04-07']
+            })
+            
+            st.dataframe(url_data, use_container_width=True)
+        
+        # Action buttons
+        col1, col2, col3 = st.columns(3)
+        
+        with col1:
+            st.button("Export IoCs", key="export_ioc_btn")
+        
+        with col2:
+            st.button("Add to Blocklist", key="blocklist_btn")
+        
+        with col3:
+            st.button("Share Intelligence", key="share_intel_btn")

components/web_scraper.py

@@ -0,0 +1,330 @@
+"""
+Web scraper component for Streamlit frontend.
+This integrates with the backend scraper service.
+"""
+import streamlit as st
+import pandas as pd
+import plotly.graph_objects as go
+import time
+import re
+import asyncio
+import httpx
+from typing import Dict, Any, List, Optional
+import json
+import sys
+import os
+
+# Add the src directory to the path so we can import the services
+sys.path.append(os.path.abspath('.'))
+
+try:
+    from src.services.scraper import WebScraper
+    from src.services.tor_proxy import TorProxyService
+except ImportError:
+    # Fallback if imports fail - we'll use a simplified version
+    WebScraper = None
+    TorProxyService = None
+
+# Check if Tor is running
+def is_tor_running() -> bool:
+    """Check if Tor service is running and accessible."""
+    try:
+        with httpx.Client(timeout=3) as client:
+            response = client.get("http://127.0.0.1:9050")
+            return True
+    except Exception:
+        return False
+
+# Create a scraper instance
+async def get_scraper():
+    """Get a configured scraper instance."""
+    if WebScraper and TorProxyService:
+        try:
+            tor_proxy = TorProxyService()
+            # Check if Tor is accessible
+            is_connected = await tor_proxy.check_connection()
+            if is_connected:
+                return WebScraper(tor_proxy_service=tor_proxy)
+        except Exception as e:
+            st.error(f"Error connecting to Tor: {e}")
+    
+    # If we can't connect to Tor or imports failed, return None
+    return None
+
+async def extract_content(url: str, use_tor: bool = False) -> Dict[str, Any]:
+    """
+    Extract content from a URL using the backend scraper.
+    
+    Args:
+        url (str): URL to scrape
+        use_tor (bool): Whether to use Tor proxy
+        
+    Returns:
+        Dict[str, Any]: Extracted content
+    """
+    scraper = await get_scraper()
+    
+    if scraper:
+        try:
+            return await scraper.extract_content(url, use_tor=use_tor)
+        except Exception as e:
+            st.error(f"Error extracting content: {e}")
+            return {
+                "url": url,
+                "title": "Error extracting content",
+                "text_content": f"Failed to extract content: {e}",
+                "indicators": {},
+                "links": []
+            }
+    else:
+        # Fallback to simulated data if scraper is unavailable
+        st.warning("Advanced scraping functionality unavailable. Using limited extraction.")
+        try:
+            with httpx.Client(timeout=10) as client:
+                response = client.get(url)
+                return {
+                    "url": url,
+                    "title": f"Content from {url}",
+                    "text_content": response.text[:1000] + "...",
+                    "indicators": {},
+                    "links": []
+                }
+        except Exception as e:
+            return {
+                "url": url,
+                "title": "Error fetching content",
+                "text_content": f"Failed to fetch content: {e}",
+                "indicators": {},
+                "links": []
+            }
+
+def render_indicators(indicators: Dict[str, List[str]]):
+    """
+    Render extracted indicators in a formatted way.
+    
+    Args:
+        indicators (Dict[str, List[str]]): Dictionary of indicator types and values
+    """
+    if not indicators:
+        st.info("No indicators found in the content.")
+        return
+    
+    # Create tabs for different indicator types
+    tabs = st.tabs([
+        f"IP Addresses ({len(indicators.get('ip_addresses', []))})",
+        f"Emails ({len(indicators.get('email_addresses', []))})",
+        f"Bitcoin ({len(indicators.get('bitcoin_addresses', []))})",
+        f"URLs ({len(indicators.get('urls', []))})",
+        f"Onion URLs ({len(indicators.get('onion_urls', []))})"
+    ])
+    
+    # IP Addresses
+    with tabs[0]:
+        if indicators.get('ip_addresses'):
+            st.markdown("#### Extracted IP Addresses")
+            ip_df = pd.DataFrame(indicators['ip_addresses'], columns=["IP Address"])
+            st.dataframe(ip_df, use_container_width=True)
+        else:
+            st.info("No IP addresses found.")
+    
+    # Email Addresses
+    with tabs[1]:
+        if indicators.get('email_addresses'):
+            st.markdown("#### Extracted Email Addresses")
+            email_df = pd.DataFrame(indicators['email_addresses'], columns=["Email"])
+            st.dataframe(email_df, use_container_width=True)
+        else:
+            st.info("No email addresses found.")
+    
+    # Bitcoin Addresses
+    with tabs[2]:
+        if indicators.get('bitcoin_addresses'):
+            st.markdown("#### Extracted Bitcoin Addresses")
+            btc_df = pd.DataFrame(indicators['bitcoin_addresses'], columns=["Bitcoin Address"])
+            st.dataframe(btc_df, use_container_width=True)
+        else:
+            st.info("No Bitcoin addresses found.")
+    
+    # URLs
+    with tabs[3]:
+        if indicators.get('urls'):
+            st.markdown("#### Extracted URLs")
+            url_df = pd.DataFrame(indicators['urls'], columns=["URL"])
+            st.dataframe(url_df, use_container_width=True)
+        else:
+            st.info("No URLs found.")
+    
+    # Onion URLs
+    with tabs[4]:
+        if indicators.get('onion_urls'):
+            st.markdown("#### Extracted Onion URLs")
+            onion_df = pd.DataFrame(indicators['onion_urls'], columns=["Onion URL"])
+            st.dataframe(onion_df, use_container_width=True)
+        else:
+            st.info("No onion URLs found.")
+
+def create_keyword_highlight(text: str, keywords: Optional[List[str]] = None) -> str:
+    """
+    Highlight keywords in text for display.
+    
+    Args:
+        text (str): Text content to highlight
+        keywords (Optional[List[str]]): Keywords to highlight
+        
+    Returns:
+        str: HTML with highlighted keywords
+    """
+    if not text or not keywords:
+        return text
+    
+    # Escape HTML
+    text = text.replace('<', '&lt;').replace('>', '&gt;')
+    
+    # Highlight keywords
+    for keyword in keywords:
+        if not keyword.strip():
+            continue
+        pattern = re.compile(re.escape(keyword), re.IGNORECASE)
+        text = pattern.sub(f'<span style="background-color: #E74C3C40; padding: 0 2px; border-radius: 3px;">{keyword}</span>', text)
+    
+    return text
+
+def render_web_scraper_ui():
+    """Render the web scraper user interface."""
+    st.title("Dark Web Intelligence Gathering")
+    
+    # Check if Tor is accessible
+    if is_tor_running():
+        st.success("Tor service is available for .onion sites")
+    else:
+        st.warning("Tor service not detected. Limited to clearnet sites only.")
+    
+    # Create UI layout
+    col1, col2 = st.columns([2, 1])
+    
+    with col1:
+        st.markdown("### Content Extraction & Analysis")
+        
+        # URL input
+        url = st.text_input(
+            "Enter URL to analyze",
+            value="https://example.com",
+            help="Enter a URL to scrape and analyze. For .onion sites, ensure Tor is configured."
+        )
+        
+        # Options
+        use_tor = st.checkbox(
+            "Use Tor proxy",
+            value='.onion' in url,
+            help="Use Tor proxy for accessing .onion sites or for anonymity"
+        )
+        
+        # Keyword highlighting
+        keywords_input = st.text_area(
+            "Keywords to highlight (one per line)",
+            value="example\ndata\nbreach",
+            help="Enter keywords to highlight in the extracted content"
+        )
+        keywords = [k.strip() for k in keywords_input.split('\n') if k.strip()]
+        
+        # Extract button
+        extract_button = st.button("Extract Content")
+        
+    with col2:
+        st.markdown("### Analysis Options")
+        
+        analysis_tabs = st.radio(
+            "Analysis Type",
+            ["Text Analysis", "Indicators", "Sentiment Analysis", "Entity Recognition"],
+            help="Select the type of analysis to perform on the extracted content"
+        )
+        
+        st.markdown("### Monitoring")
+        monitoring_options = st.multiselect(
+            "Add to monitoring list",
+            ["IP Addresses", "Email Addresses", "Bitcoin Addresses", "URLs", "Onion URLs"],
+            default=["IP Addresses", "URLs"],
+            help="Select which indicator types to monitor"
+        )
+        
+        alert_threshold = st.slider(
+            "Alert Threshold",
+            min_value=0.0,
+            max_value=1.0,
+            value=0.7,
+            step=0.05,
+            help="Set the confidence threshold for alerts"
+        )
+    
+    # Handle content extraction
+    if extract_button:
+        with st.spinner("Extracting content..."):
+            # Run the async extraction
+            content_data = asyncio.run(extract_content(url, use_tor=use_tor))
+            
+            # Store results in session state
+            st.session_state.extracted_content = content_data
+            
+            # Success message
+            st.success(f"Content extracted from {url}")
+    
+    # Display extracted content if available
+    if 'extracted_content' in st.session_state:
+        content_data = st.session_state.extracted_content
+        
+        # Display content in tabs
+        content_tabs = st.tabs(["Extracted Text", "Indicators", "Metadata", "Raw HTML"])
+        
+        # Extracted text tab
+        with content_tabs[0]:
+            st.markdown(f"### {content_data.get('title', 'Extracted Content')}")
+            st.info(f"Source: {content_data.get('url')}")
+            
+            # Highlight keywords in text
+            highlighted_text = create_keyword_highlight(
+                content_data.get('text_content', 'No content extracted'),
+                keywords
+            )
+            
+            st.markdown(f"""
+            <div style="border: 1px solid #3498DB; border-radius: 5px; padding: 15px; 
+                 background-color: #1A1A1A; height: 400px; overflow-y: auto;">
+                {highlighted_text}
+            </div>
+            """, unsafe_allow_html=True)
+        
+        # Indicators tab
+        with content_tabs[1]:
+            render_indicators(content_data.get('indicators', {}))
+        
+        # Metadata tab
+        with content_tabs[2]:
+            st.markdown("### Document Metadata")
+            
+            metadata = content_data.get('metadata', {})
+            if metadata:
+                for key, value in metadata.items():
+                    if value:
+                        st.markdown(f"**{key}:** {value}")
+            else:
+                st.info("No metadata available")
+        
+        # Raw HTML tab
+        with content_tabs[3]:
+            st.markdown("### Raw HTML")
+            with st.expander("Show Raw HTML"):
+                st.code(content_data.get('html_content', 'No HTML content available'), language="html")
+    
+    # Additional informational UI elements
+    st.markdown("---")
+    st.markdown("### About Dark Web Intelligence")
+    st.markdown("""
+    This tool allows you to extract and analyze content from both clearnet and dark web sites.
+    For .onion sites, make sure Tor is properly configured.
+    
+    **Features:**
+    - Extract and analyze content from any URL
+    - Highlight keywords of interest
+    - Identify indicators of compromise (IoCs)
+    - Add indicators to monitoring list
+    """)

hf_app.py

@@ -0,0 +1,146 @@
+"""
+CyberForge Dashboard - Hugging Face Spaces Version
+"""
+import os
+import sys
+import streamlit as st
+from streamlit_extras.add_vertical_space import add_vertical_space
+from streamlit_extras.colored_header import colored_header
+
+# Check if we're running on Hugging Face Spaces
+is_huggingface = os.environ.get('SPACE_ID') is not None
+
+# Set the page config
+st.set_page_config(
+    page_title="CyberForge Dashboard",
+    page_icon="🕵️‍♂️",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Add custom CSS
+st.markdown("""
+<style>
+    .stApp {
+        background-color: #0e1117;
+    }
+    .sidebar .sidebar-content {
+        background-color: #262730;
+    }
+    h1, h2, h3 {
+        color: #f8f9fa;
+    }
+    .cybertext {
+        color: #00ff8d;
+        font-family: monospace;
+    }
+</style>
+""", unsafe_allow_html=True)
+
+# Choose between HF demo mode or regular mode
+if is_huggingface:
+    # Initialize in-memory database for Hugging Face
+    import hf_database
+    st.session_state.is_demo = True
+    
+    # Show demo mode banner
+    st.warning("⚠️ Running in Hugging Face Spaces DEMO MODE. Data is stored in-memory and will be reset when the space restarts.")
+else:
+    # Regular database initialization
+    import src.database_init
+    
+# Import components
+from components.dashboard import render_dashboard
+from components.threats import render_threats
+from components.monitoring import render_monitoring
+from components.alerts import render_alerts
+from components.reports import render_reports
+from components.live_feed import render_live_feed, render_content_analysis
+from components.web_scraper import render_web_scraper_ui
+
+# Custom notification function
+def add_notification(title, message, severity="info", icon="🔔"):
+    """Add a notification to the session state"""
+    if "notifications" not in st.session_state:
+        st.session_state.notifications = []
+    
+    # Add notification with timestamp
+    import time
+    notification = {
+        "id": int(time.time() * 1000),
+        "title": title,
+        "message": message,
+        "severity": severity,
+        "icon": icon,
+        "read": False,
+        "timestamp": time.time()
+    }
+    st.session_state.notifications.insert(0, notification)
+
+# Initialize notifications if needed
+if "notifications" not in st.session_state:
+    st.session_state.notifications = []
+
+# Sidebar navigation
+with st.sidebar:
+    st.image("assets/cyberforge_logo.svg", width=200)
+    st.title("CyberForge")
+    
+    # Demo badge
+    if st.session_state.get("is_demo", False):
+        st.markdown("#### 🔍 Demo Mode")
+    
+    st.markdown("---")
+    
+    # Navigation
+    nav_selection = st.radio(
+        "Navigation",
+        ["Dashboard", "Threats", "Monitoring", "Alerts", "Reports", "Live Feed", "Content Analysis", "Web Scraper"]
+    )
+    
+    # User information
+    st.markdown("---")
+    st.markdown("### User Info")
+    if st.session_state.get("is_demo", False):
+        st.markdown("👤 **Admin User** (Demo)")
+        st.markdown("🔑 Role: Administrator")
+    else:
+        st.markdown("👤 **Analyst**")
+        st.markdown("🔑 Role: Security Analyst")
+    
+    # Notification count
+    unread_count = sum(1 for n in st.session_state.notifications if not n["read"])
+    if unread_count > 0:
+        st.markdown(f"🔔 **{unread_count}** unread notifications")
+    
+    # Credits
+    st.markdown("---")
+    st.markdown("### CyberForge v1.0")
+    st.markdown("© 2025 Chemically Motivated Solutions")
+    
+    # HF badge if on Hugging Face
+    if is_huggingface:
+        st.markdown("---")
+        st.markdown("""
+        <a href="https://huggingface.co/spaces" target="_blank">
+        <img src="https://img.shields.io/badge/Hosted%20on-HF%20Spaces-blue" alt="HuggingFace Spaces"/>
+        </a>
+        """, unsafe_allow_html=True)
+
+# Main content area
+if nav_selection == "Dashboard":
+    render_dashboard()
+elif nav_selection == "Threats":
+    render_threats()
+elif nav_selection == "Monitoring":
+    render_monitoring()
+elif nav_selection == "Alerts":
+    render_alerts()
+elif nav_selection == "Reports":
+    render_reports()
+elif nav_selection == "Live Feed":
+    render_live_feed()
+elif nav_selection == "Content Analysis":
+    render_content_analysis()
+elif nav_selection == "Web Scraper":
+    render_web_scraper_ui()

hf_database.py

@@ -0,0 +1,100 @@
+"""
+Database initialization for Hugging Face Spaces environment.
+This creates an in-memory SQLite database for demo purposes.
+"""
+import logging
+import os
+import sqlite3
+from sqlalchemy import create_engine, event
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.pool import StaticPool
+from src.models.base import Base
+from src.models.user import User
+from src.api.security import get_password_hash
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Database URL for SQLite in-memory
+DATABASE_URL = "sqlite:///:memory:"
+
+# Create engine with special configuration for in-memory SQLite
+engine = create_engine(
+    DATABASE_URL,
+    connect_args={"check_same_thread": False},
+    poolclass=StaticPool,
+    echo=False
+)
+
+# Add pragma for foreign key support
+@event.listens_for(engine, "connect")
+def set_sqlite_pragma(dbapi_connection, connection_record):
+    cursor = dbapi_connection.cursor()
+    cursor.execute("PRAGMA foreign_keys=ON")
+    cursor.close()
+
+# Create all tables
+Base.metadata.create_all(engine)
+
+# Create session factory
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+def init_demo_data():
+    """Initialize demo data for the in-memory database."""
+    session = SessionLocal()
+    try:
+        # Check if we already have users
+        user_count = session.query(User).count()
+        if user_count == 0:
+            # Create admin user
+            admin_user = User(
+                username="admin",
+                email="[email protected]",
+                full_name="Admin User",
+                hashed_password=get_password_hash("adminpassword"),
+                is_active=True,
+                is_superuser=True
+            )
+            session.add(admin_user)
+            
+            # Create regular user
+            regular_user = User(
+                username="user",
+                email="[email protected]",
+                full_name="Regular User",
+                hashed_password=get_password_hash("userpassword"),
+                is_active=True,
+                is_superuser=False
+            )
+            session.add(regular_user)
+            
+            # Create API user
+            api_user = User(
+                username="api_user",
+                email="[email protected]",
+                full_name="API User",
+                hashed_password=get_password_hash("apipassword"),
+                is_active=True,
+                is_superuser=False
+            )
+            session.add(api_user)
+            
+            # Commit the session
+            session.commit()
+            logger.info("Demo users created successfully")
+        else:
+            logger.info("Demo data already exists")
+            
+        # Here you would add other demo data like threats, indicators, etc.
+            
+    except Exception as e:
+        session.rollback()
+        logger.error(f"Error initializing demo data: {e}")
+    finally:
+        session.close()
+
+# Initialize demo data
+init_demo_data()
+
+logger.info("Hugging Face database initialized with demo data")

huggingface-space.yml

@@ -0,0 +1,9 @@
+title: CyberForge Dashboard
+emoji: 🕵️‍♂️
+colorFrom: blue
+colorTo: indigo
+sdk: streamlit
+sdk_version: 1.32.0
+app_file: hf_app.py
+pinned: false
+license: mit

requirements.txt

@@ -0,0 +1,20 @@
+alembic==1.13.1
+asyncpg==0.29.0
+bcrypt==4.1.2
+beautifulsoup4==4.12.2
+celery==5.3.6
+email-validator==2.1.0.post1
+fastapi==0.109.2
+httpx==0.27.0
+pandas==2.1.0
+passlib==1.7.4
+plotly==5.18.0
+pysocks==1.7.1
+python-jose==3.3.0
+redis==5.0.1
+sqlalchemy==2.0.28
+streamlit==1.32.0
+streamlit-extras==0.3.5
+trafilatura==1.6.3
+python-multipart==0.0.7
+pydantic[email]==2.4.2

src/database_init.py

@@ -0,0 +1,113 @@
+"""
+Database initialization for the application.
+
+This script checks if the database is initialized and creates tables if needed.
+It's meant to be imported and run at application startup.
+"""
+import os
+import logging
+import asyncio
+from sqlalchemy.ext.asyncio import create_async_engine
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.future import select
+import subprocess
+import sys
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# Database URL from environment
+db_url = os.getenv("DATABASE_URL", "")
+if db_url.startswith("postgresql://"):
+    # Remove sslmode parameter if present which causes issues with asyncpg
+    if "?" in db_url:
+        base_url, params = db_url.split("?", 1)
+        param_list = params.split("&")
+        filtered_params = [p for p in param_list if not p.startswith("sslmode=")]
+        if filtered_params:
+            db_url = f"{base_url}?{'&'.join(filtered_params)}"
+        else:
+            db_url = base_url
+    
+    ASYNC_DATABASE_URL = db_url.replace("postgresql://", "postgresql+asyncpg://", 1)
+else:
+    ASYNC_DATABASE_URL = "postgresql+asyncpg://postgres:postgres@localhost:5432/postgres"
+
+
+async def check_db_initialized():
+    """Check if the database is initialized with required tables."""
+    try:
+        engine = create_async_engine(
+            ASYNC_DATABASE_URL,
+            echo=False,
+        )
+        
+        # Create session factory
+        async_session = sessionmaker(
+            engine, 
+            class_=AsyncSession, 
+            expire_on_commit=False
+        )
+        
+        async with async_session() as session:
+            # Try to query tables
+            # Replace with actual table names once you've defined them
+            try:
+                # Check if the 'users' table exists
+                from sqlalchemy import text
+                query = text("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'users')")
+                result = await session.execute(query)
+                exists = result.scalar()
+                
+                if exists:
+                    logger.info("Database is initialized.")
+                    return True
+                else:
+                    logger.warning("Database tables are not initialized.")
+                    return False
+            except Exception as e:
+                logger.error(f"Error checking tables: {e}")
+                return False
+    except Exception as e:
+        logger.error(f"Failed to connect to database: {e}")
+        return False
+
+
+def initialize_database():
+    """Initialize the database with required tables."""
+    try:
+        # Call the init_db.py script
+        logger.info("Initializing database...")
+        
+        # Get the current directory
+        current_dir = os.path.dirname(os.path.abspath(__file__))
+        script_path = os.path.join(current_dir, "scripts", "init_db.py")
+        
+        # Run the script using the current Python interpreter
+        result = subprocess.run([sys.executable, script_path], capture_output=True, text=True)
+        
+        if result.returncode == 0:
+            logger.info("Database initialized successfully.")
+            logger.debug(result.stdout)
+            return True
+        else:
+            logger.error(f"Failed to initialize database: {result.stderr}")
+            return False
+    except Exception as e:
+        logger.error(f"Error initializing database: {e}")
+        return False
+
+
+def ensure_database_initialized():
+    """Ensure the database is initialized with required tables."""
+    is_initialized = asyncio.run(check_db_initialized())
+    
+    if not is_initialized:
+        return initialize_database()
+    
+    return True

src/models/__init__.py

@@ -0,0 +1,3 @@
+"""
+Package initialization for models.
+"""

src/models/alert.py

@@ -0,0 +1,68 @@
+"""
+Model for storing alerts generated from threats and dark web mentions.
+"""
+from sqlalchemy import Column, String, Text, Integer, DateTime, ForeignKey, Enum, Boolean
+from sqlalchemy.orm import relationship
+import enum
+from datetime import datetime
+from typing import Optional
+
+from src.models.base import BaseModel
+from src.models.threat import ThreatSeverity
+
+class AlertCategory(enum.Enum):
+    """Categories of alerts."""
+    THREAT_DETECTED = "Threat Detected"
+    MENTION_DETECTED = "Mention Detected"
+    CREDENTIAL_LEAK = "Credential Leak"
+    DATA_BREACH = "Data Breach"
+    VULNERABILITY = "Vulnerability"
+    MALWARE = "Malware"
+    PHISHING = "Phishing"
+    SUSPICIOUS_ACTIVITY = "Suspicious Activity"
+    SYSTEM = "System Alert"
+    OTHER = "Other"
+
+
+class AlertStatus(enum.Enum):
+    """Status of alerts."""
+    NEW = "New"
+    ASSIGNED = "Assigned"
+    INVESTIGATING = "Investigating"
+    RESOLVED = "Resolved"
+    FALSE_POSITIVE = "False Positive"
+    IGNORED = "Ignored"
+
+
+class Alert(BaseModel):
+    """Model for alerts generated from threats and mentions."""
+    __tablename__ = "alerts"
+    
+    # Alert details
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=False)
+    severity = Column(Enum(ThreatSeverity), nullable=False)
+    status = Column(Enum(AlertStatus), nullable=False, default=AlertStatus.NEW)
+    category = Column(Enum(AlertCategory), nullable=False)
+    
+    # Alert metadata
+    generated_at = Column(DateTime, default=datetime.utcnow)
+    source_url = Column(String(1024))
+    is_read = Column(Boolean, default=False)
+    
+    # Relationships
+    threat_id = Column(Integer, ForeignKey("threats.id"))
+    threat = relationship("Threat", back_populates="alerts")
+    
+    mention_id = Column(Integer, ForeignKey("dark_web_mentions.id"))
+    mention = relationship("DarkWebMention", back_populates="alerts")
+    
+    # Assignment and resolution
+    assigned_to_id = Column(Integer, ForeignKey("users.id"))
+    assigned_to = relationship("User")
+    
+    action_taken = Column(Text)
+    resolved_at = Column(DateTime)
+    
+    def __repr__(self):
+        return f"<Alert(id={self.id}, title={self.title}, severity={self.severity}, status={self.status})>"

src/models/base.py

@@ -0,0 +1,20 @@
+"""
+Base model for all database models.
+"""
+from datetime import datetime
+
+from sqlalchemy import Column, Integer, DateTime
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+class BaseModel(Base):
+    """
+    Base model for all database models.
+    Provides common fields like id, created_at, updated_at.
+    """
+    __abstract__ = True
+    
+    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
+    created_at = Column(DateTime, default=datetime.utcnow)
+    updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)

src/models/dark_web_content.py

@@ -0,0 +1,93 @@
+"""
+Models for storing dark web content and mentions.
+"""
+from sqlalchemy import Column, String, Text, Integer, Float, DateTime, ForeignKey, Enum, Boolean
+from sqlalchemy.orm import relationship
+import enum
+from datetime import datetime
+
+from src.models.base import BaseModel
+
+class ContentType(enum.Enum):
+    """Type of dark web content."""
+    FORUM_POST = "Forum Post"
+    MARKETPLACE_LISTING = "Marketplace Listing"
+    BLOG_ARTICLE = "Blog Article"
+    CHAT_LOG = "Chat Log"
+    PASTE = "Paste"
+    DOCUMENT = "Document"
+    IMAGE = "Image"
+    VIDEO = "Video"
+    SOURCE_CODE = "Source Code"
+    OTHER = "Other"
+
+
+class ContentStatus(enum.Enum):
+    """Status of dark web content."""
+    NEW = "New"
+    ANALYZING = "Analyzing"
+    ANALYZED = "Analyzed"
+    RELEVANT = "Relevant"
+    IRRELEVANT = "Irrelevant"
+    ARCHIVED = "Archived"
+
+
+class DarkWebContent(BaseModel):
+    """Model for storing dark web content."""
+    __tablename__ = "dark_web_contents"
+    
+    # Content source
+    url = Column(String(1024), nullable=False)
+    domain = Column(String(255))
+    
+    # Content metadata
+    title = Column(String(500))
+    content = Column(Text, nullable=False)
+    content_type = Column(Enum(ContentType), default=ContentType.OTHER)
+    content_status = Column(Enum(ContentStatus), default=ContentStatus.NEW)
+    
+    # Source information
+    source_name = Column(String(255))
+    source_type = Column(String(100))
+    language = Column(String(10))
+    scraped_at = Column(DateTime, default=datetime.utcnow)
+    
+    # Analysis results
+    relevance_score = Column(Float, default=0.0)
+    sentiment_score = Column(Float, default=0.0)
+    entity_data = Column(Text) # JSON storage for extracted entities
+    
+    # Relationships
+    mentions = relationship("DarkWebMention", back_populates="content", cascade="all, delete-orphan")
+    search_results = relationship("SearchResult", back_populates="content")
+    
+    def __repr__(self):
+        return f"<DarkWebContent(id={self.id}, url={self.url}, content_type={self.content_type})>"
+
+
+class DarkWebMention(BaseModel):
+    """Model for storing mentions of monitored entities in dark web content."""
+    __tablename__ = "dark_web_mentions"
+    
+    # Relationship to content
+    content_id = Column(Integer, ForeignKey("dark_web_contents.id"), nullable=False)
+    content = relationship("DarkWebContent", back_populates="mentions")
+    
+    # Mention details
+    keyword = Column(String(100), nullable=False)
+    keyword_category = Column(String(50))
+    
+    # Extracted context
+    context = Column(Text)
+    snippet = Column(Text)
+    
+    # Mention metadata
+    mention_type = Column(String(50)) # Type of mention (e.g., "brand", "employee", "product")
+    confidence = Column(Float, default=0.0)
+    is_verified = Column(Boolean, default=False)
+    
+    # Relationships
+    alerts = relationship("Alert", back_populates="mention", cascade="all, delete-orphan")
+    
+    def __repr__(self):
+        return f"<DarkWebMention(id={self.id}, keyword={self.keyword}, content_id={self.content_id})>"

src/models/indicator.py

@@ -0,0 +1,49 @@
+"""
+Model for storing indicators of compromise (IOCs) and other threat indicators.
+"""
+from sqlalchemy import Column, String, Text, Integer, Float, DateTime, ForeignKey, Enum, Boolean
+from sqlalchemy.orm import relationship
+import enum
+from datetime import datetime
+
+from src.models.base import BaseModel
+
+class IndicatorType(enum.Enum):
+    """Type of indicator."""
+    IP_ADDRESS = "IP Address"
+    DOMAIN = "Domain"
+    URL = "URL"
+    HASH = "Hash"
+    EMAIL = "Email"
+    FILE = "File"
+    REGISTRY = "Registry"
+    USER_AGENT = "User Agent"
+    CVE = "CVE"
+    SOFTWARE = "Software"
+    KEYWORD = "Keyword"
+    OTHER = "Other"
+
+
+class Indicator(BaseModel):
+    """Model for indicators related to threats."""
+    __tablename__ = "indicators"
+    
+    # Indicator details
+    value = Column(String(1024), nullable=False)
+    indicator_type = Column(Enum(IndicatorType), nullable=False)
+    description = Column(Text)
+    is_verified = Column(Boolean, default=False)
+    context = Column(Text)
+    source = Column(String(255))
+    
+    # Relationship to threat
+    threat_id = Column(Integer, ForeignKey("threats.id"))
+    threat = relationship("Threat", back_populates="indicators")
+    
+    # Confidence and metadata
+    confidence_score = Column(Float, default=0.0)
+    first_seen = Column(DateTime, default=datetime.utcnow)
+    last_seen = Column(DateTime, default=datetime.utcnow)
+    
+    def __repr__(self):
+        return f"<Indicator(id={self.id}, value={self.value}, type={self.indicator_type})>"

src/models/report.py

@@ -0,0 +1,78 @@
+"""
+Model for storing reports generated from threats and analysis.
+"""
+from sqlalchemy import Column, String, Text, Integer, DateTime, ForeignKey, Enum, Table
+from sqlalchemy.orm import relationship
+import enum
+from datetime import datetime
+from typing import List
+
+from src.models.base import BaseModel
+from src.models.threat import ThreatSeverity
+
+# Many-to-many relationship table for reports and threats
+report_threats = Table(
+    "report_threats",
+    BaseModel.metadata,
+    Column("report_id", Integer, ForeignKey("reports.id"), primary_key=True),
+    Column("threat_id", Integer, ForeignKey("threats.id"), primary_key=True),
+)
+
+
+class ReportType(enum.Enum):
+    """Type of report."""
+    THREAT_DIGEST = "Threat Digest"
+    DARK_WEB_ANALYSIS = "Dark Web Analysis"
+    VULNERABILITY_ASSESSMENT = "Vulnerability Assessment"
+    INCIDENT_RESPONSE = "Incident Response"
+    THREAT_INTELLIGENCE = "Threat Intelligence"
+    EXECUTIVE_SUMMARY = "Executive Summary"
+    TECHNICAL_ANALYSIS = "Technical Analysis"
+    WEEKLY_SUMMARY = "Weekly Summary"
+    MONTHLY_SUMMARY = "Monthly Summary"
+    CUSTOM = "Custom"
+
+
+class ReportStatus(enum.Enum):
+    """Status of report."""
+    DRAFT = "Draft"
+    REVIEW = "In Review"
+    APPROVED = "Approved"
+    PUBLISHED = "Published"
+    ARCHIVED = "Archived"
+
+
+class Report(BaseModel):
+    """Model for reports on threats and analysis."""
+    __tablename__ = "reports"
+    
+    # Report metadata
+    report_id = Column(String(50), unique=True, nullable=False)
+    title = Column(String(255), nullable=False)
+    summary = Column(Text, nullable=False)
+    content = Column(Text, nullable=False)
+    report_type = Column(Enum(ReportType), nullable=False)
+    status = Column(Enum(ReportStatus), nullable=False, default=ReportStatus.DRAFT)
+    severity = Column(Enum(ThreatSeverity))
+    
+    # Report scheduling and timing
+    publish_date = Column(DateTime)
+    time_period_start = Column(DateTime)
+    time_period_end = Column(DateTime)
+    
+    # Keywords for searchability
+    keywords = Column(String(500))
+    
+    # Related entities
+    author_id = Column(Integer, ForeignKey("users.id"))
+    author = relationship("User")
+    
+    # Many-to-many relationship with threats
+    threats = relationship(
+        "Threat",
+        secondary=report_threats,
+        backref="reports"
+    )
+    
+    def __repr__(self):
+        return f"<Report(id={self.id}, report_id={self.report_id}, title={self.title})>"

src/models/search_history.py

@@ -0,0 +1,146 @@
+"""
+Search History Model
+
+This module defines the search history model for tracking dark web searches and trends.
+"""
+from datetime import datetime
+from typing import Optional, List
+from sqlalchemy import Column, Integer, String, DateTime, Boolean, ForeignKey, Text, Float
+from sqlalchemy.orm import relationship
+
+from src.models.base import Base
+
+class SearchHistory(Base):
+    """
+    Model for tracking search history and trends in dark web content.
+    
+    Attributes:
+        id: Unique identifier for the search
+        query: The search query or term
+        timestamp: When the search was performed
+        user_id: ID of the user who performed the search (optional)
+        result_count: Number of results returned
+        category: Category of the search (e.g., "marketplace", "forum", "paste", etc.)
+        is_saved: Whether this is a saved/favorited search
+        notes: Optional notes about this search
+        tags: Tags associated with this search
+    """
+    __tablename__ = "search_history"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    query = Column(String(255), nullable=False, index=True)
+    timestamp = Column(DateTime, default=datetime.utcnow, nullable=False, index=True)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=True)
+    result_count = Column(Integer, default=0)
+    category = Column(String(50), nullable=True)
+    is_saved = Column(Boolean, default=False)
+    notes = Column(Text, nullable=True)
+    tags = Column(String(255), nullable=True)  # Comma-separated tags
+    
+    # Relationships
+    user = relationship("User", back_populates="searches")
+    search_results = relationship("SearchResult", back_populates="search", cascade="all, delete-orphan")
+    
+    def __repr__(self):
+        return f"<SearchHistory(id={self.id}, query='{self.query}', timestamp={self.timestamp})>"
+
+
+class SearchResult(Base):
+    """
+    Model for individual search results associated with a search query.
+    
+    Attributes:
+        id: Unique identifier for the search result
+        search_id: ID of the parent search
+        content_id: ID of the content found (if in our database)
+        url: URL of the result
+        title: Title of the result
+        snippet: Text snippet from the result
+        source: Source of the result (e.g., "dark web forum", "marketplace", etc.)
+        relevance_score: Score indicating relevance to the search query
+        timestamp: When this result was found
+    """
+    __tablename__ = "search_results"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    search_id = Column(Integer, ForeignKey("search_history.id"), nullable=False)
+    content_id = Column(Integer, ForeignKey("dark_web_contents.id"), nullable=True)
+    url = Column(String(1024), nullable=True)
+    title = Column(String(255), nullable=True)
+    snippet = Column(Text, nullable=True)
+    source = Column(String(100), nullable=True)
+    relevance_score = Column(Float, default=0.0)
+    timestamp = Column(DateTime, default=datetime.utcnow, nullable=False)
+    
+    # Relationships
+    search = relationship("SearchHistory", back_populates="search_results")
+    content = relationship("DarkWebContent", back_populates="search_results")
+    
+    def __repr__(self):
+        return f"<SearchResult(id={self.id}, search_id={self.search_id}, title='{self.title}')>"
+
+
+class SavedSearch(Base):
+    """
+    Model for saved searches with custom parameters for periodic monitoring.
+    
+    Attributes:
+        id: Unique identifier for the saved search
+        name: Name of the saved search
+        query: The search query or term
+        user_id: ID of the user who created the saved search
+        created_at: When this saved search was created
+        last_run_at: When this saved search was last executed
+        frequency: How often to run this search (in hours, 0 for manual only)
+        notification_enabled: Whether to send notifications for new results
+        is_active: Whether this saved search is active
+        threshold: Threshold for notifications (e.g., min number of new results)
+    """
+    __tablename__ = "saved_searches"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String(100), nullable=False)
+    query = Column(String(255), nullable=False)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    created_at = Column(DateTime, default=datetime.utcnow, nullable=False)
+    last_run_at = Column(DateTime, nullable=True)
+    frequency = Column(Integer, default=24)  # In hours, 0 for manual only
+    notification_enabled = Column(Boolean, default=True)
+    is_active = Column(Boolean, default=True)
+    threshold = Column(Integer, default=1)  # Min number of new results for notification
+    category = Column(String(50), nullable=True)
+    
+    # Relationships
+    user = relationship("User", back_populates="saved_searches")
+    
+    def __repr__(self):
+        return f"<SavedSearch(id={self.id}, name='{self.name}', query='{self.query}')>"
+
+
+class TrendTopic(Base):
+    """
+    Model for tracking trending topics on the dark web.
+    
+    Attributes:
+        id: Unique identifier for the trend topic
+        topic: The topic or term
+        first_seen: When this topic was first detected
+        last_seen: When this topic was last detected
+        mention_count: Number of mentions of this topic
+        growth_rate: Rate of growth in mentions (percentage)
+        category: Category of the trend (e.g., "ransomware", "data breach", etc.)
+        is_active: Whether this trend is currently active
+    """
+    __tablename__ = "trend_topics"
+    
+    id = Column(Integer, primary_key=True, index=True)
+    topic = Column(String(100), nullable=False, index=True)
+    first_seen = Column(DateTime, default=datetime.utcnow, nullable=False)
+    last_seen = Column(DateTime, default=datetime.utcnow, nullable=False)
+    mention_count = Column(Integer, default=1)
+    growth_rate = Column(Float, default=0.0)
+    category = Column(String(50), nullable=True)
+    is_active = Column(Boolean, default=True)
+    
+    def __repr__(self):
+        return f"<TrendTopic(id={self.id}, topic='{self.topic}', mention_count={self.mention_count})>"

src/models/subscription.py

@@ -0,0 +1,143 @@
+"""
+Subscription models for the application.
+
+This module defines database models for subscription management.
+"""
+from enum import Enum
+from sqlalchemy import Column, Integer, String, Float, Boolean, DateTime, ForeignKey, Enum as SQLAlchemyEnum
+from sqlalchemy.orm import relationship
+from sqlalchemy.sql import func
+
+from src.models.base import Base
+
+
+class SubscriptionTier(str, Enum):
+    """Subscription tier enum."""
+    FREE = "FREE"
+    BASIC = "BASIC"
+    PROFESSIONAL = "PROFESSIONAL"
+    ENTERPRISE = "ENTERPRISE"
+
+
+class BillingPeriod(str, Enum):
+    """Billing period enum."""
+    MONTHLY = "MONTHLY"
+    ANNUALLY = "ANNUALLY"
+    CUSTOM = "CUSTOM"
+
+
+class SubscriptionPlan(Base):
+    """Subscription plan model."""
+    __tablename__ = "subscription_plans"
+
+    id = Column(Integer, primary_key=True, index=True)
+    name = Column(String(100), nullable=False)
+    tier = Column(SQLAlchemyEnum(SubscriptionTier), nullable=False)
+    description = Column(String(500))
+    price_monthly = Column(Float, nullable=False)
+    price_annually = Column(Float, nullable=False)
+    is_active = Column(Boolean, default=True)
+    
+    # Features
+    max_alerts = Column(Integer, default=10)
+    max_reports = Column(Integer, default=5)
+    max_searches_per_day = Column(Integer, default=20)
+    max_monitoring_keywords = Column(Integer, default=10)
+    max_data_retention_days = Column(Integer, default=30)
+    supports_api_access = Column(Boolean, default=False)
+    supports_live_feed = Column(Boolean, default=False)
+    supports_dark_web_monitoring = Column(Boolean, default=False)
+    supports_export = Column(Boolean, default=False)
+    supports_advanced_analytics = Column(Boolean, default=False)
+    
+    # Stripe product ID (for integration with Stripe)
+    stripe_product_id = Column(String(100))
+    stripe_monthly_price_id = Column(String(100))
+    stripe_annual_price_id = Column(String(100))
+    
+    # Timestamps
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+    
+    # Relationships
+    subscriptions = relationship("UserSubscription", back_populates="plan")
+    
+    def __repr__(self):
+        return f"<SubscriptionPlan(id={self.id}, name={self.name}, tier={self.tier})>"
+
+
+class SubscriptionStatus(str, Enum):
+    """Subscription status enum."""
+    ACTIVE = "ACTIVE"
+    PAST_DUE = "PAST_DUE"
+    CANCELED = "CANCELED"
+    TRIALING = "TRIALING"
+    INCOMPLETE = "INCOMPLETE"
+    INCOMPLETE_EXPIRED = "INCOMPLETE_EXPIRED"
+
+
+class UserSubscription(Base):
+    """User subscription model."""
+    __tablename__ = "user_subscriptions"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    plan_id = Column(Integer, ForeignKey("subscription_plans.id"), nullable=False)
+    status = Column(SQLAlchemyEnum(SubscriptionStatus), nullable=False, default=SubscriptionStatus.ACTIVE)
+    
+    # Billing details
+    billing_period = Column(SQLAlchemyEnum(BillingPeriod), nullable=False, default=BillingPeriod.MONTHLY)
+    current_period_start = Column(DateTime(timezone=True))
+    current_period_end = Column(DateTime(timezone=True))
+    
+    # Stripe subscription ID
+    stripe_subscription_id = Column(String(100))
+    stripe_customer_id = Column(String(100))
+    
+    # Timestamps
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+    canceled_at = Column(DateTime(timezone=True))
+    
+    # Relationships
+    user = relationship("User", back_populates="subscriptions")
+    plan = relationship("SubscriptionPlan", back_populates="subscriptions")
+    payment_history = relationship("PaymentHistory", back_populates="subscription")
+    
+    def __repr__(self):
+        return f"<UserSubscription(id={self.id}, user_id={self.user_id}, plan_id={self.plan_id})>"
+
+
+class PaymentStatus(str, Enum):
+    """Payment status enum."""
+    SUCCEEDED = "SUCCEEDED"
+    PENDING = "PENDING"
+    FAILED = "FAILED"
+    REFUNDED = "REFUNDED"
+
+
+class PaymentHistory(Base):
+    """Payment history model."""
+    __tablename__ = "payment_history"
+
+    id = Column(Integer, primary_key=True, index=True)
+    user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
+    subscription_id = Column(Integer, ForeignKey("user_subscriptions.id"), nullable=False)
+    
+    amount = Column(Float, nullable=False)
+    currency = Column(String(3), default="USD")
+    status = Column(SQLAlchemyEnum(PaymentStatus), nullable=False)
+    
+    # Stripe payment intent ID
+    stripe_payment_intent_id = Column(String(100))
+    stripe_invoice_id = Column(String(100))
+    
+    # Timestamps
+    payment_date = Column(DateTime(timezone=True), server_default=func.now())
+    
+    # Relationships
+    user = relationship("User")
+    subscription = relationship("UserSubscription", back_populates="payment_history")
+    
+    def __repr__(self):
+        return f"<PaymentHistory(id={self.id}, user_id={self.user_id}, amount={self.amount}, status={self.status})>"

src/models/threat.py

@@ -0,0 +1,76 @@
+"""
+Model for storing threat information discovered in dark web monitoring.
+"""
+from sqlalchemy import Column, String, Text, Integer, Float, DateTime, Enum
+from sqlalchemy.orm import relationship
+import enum
+from datetime import datetime
+
+from src.models.base import BaseModel
+
+class ThreatSeverity(enum.Enum):
+    """Severity levels for threats."""
+    CRITICAL = "Critical"
+    HIGH = "High"
+    MEDIUM = "Medium"
+    LOW = "Low"
+    INFORMATIONAL = "Informational"
+
+
+class ThreatCategory(enum.Enum):
+    """Categories of threats."""
+    DATA_BREACH = "Data Breach"
+    CREDENTIAL_LEAK = "Credential Leak"
+    VULNERABILITY = "Vulnerability"
+    MALWARE = "Malware"
+    PHISHING = "Phishing"
+    IDENTITY_THEFT = "Identity Theft"
+    RANSOMWARE = "Ransomware"
+    DARK_WEB_MENTION = "Dark Web Mention"
+    SOCIAL_ENGINEERING = "Social Engineering"
+    INSIDER_THREAT = "Insider Threat"
+    APT = "Advanced Persistent Threat"
+    OTHER = "Other"
+
+
+class ThreatStatus(enum.Enum):
+    """Status of a threat."""
+    NEW = "New"
+    INVESTIGATING = "Investigating"
+    CONFIRMED = "Confirmed"
+    MITIGATED = "Mitigated"
+    RESOLVED = "Resolved"
+    FALSE_POSITIVE = "False Positive"
+
+
+class Threat(BaseModel):
+    """Model for threats discovered in dark web monitoring."""
+    __tablename__ = "threats"
+    
+    # Threat metadata
+    title = Column(String(255), nullable=False)
+    description = Column(Text, nullable=False)
+    severity = Column(Enum(ThreatSeverity), nullable=False)
+    category = Column(Enum(ThreatCategory), nullable=False)
+    status = Column(Enum(ThreatStatus), nullable=False, default=ThreatStatus.NEW)
+    
+    # Source information
+    source_url = Column(String(1024))
+    source_name = Column(String(255))
+    source_type = Column(String(100))
+    discovered_at = Column(DateTime, default=datetime.utcnow)
+    
+    # Affected entity
+    affected_entity = Column(String(255))
+    affected_entity_type = Column(String(100))
+    
+    # Risk assessment
+    confidence_score = Column(Float, default=0.0)
+    risk_score = Column(Float, default=0.0)
+    
+    # Relationships
+    indicators = relationship("Indicator", back_populates="threat", cascade="all, delete-orphan")
+    alerts = relationship("Alert", back_populates="threat", cascade="all, delete-orphan")
+    
+    def __repr__(self):
+        return f"<Threat(id={self.id}, title={self.title}, severity={self.severity})>"

src/models/user.py

@@ -0,0 +1,32 @@
+"""
+Model for users of the application.
+"""
+from sqlalchemy import Column, String, Boolean
+from sqlalchemy.orm import relationship
+from typing import List
+
+from src.models.base import BaseModel
+
+class User(BaseModel):
+    """
+    User model for authentication and authorization.
+    """
+    __tablename__ = "users"
+    
+    username = Column(String(50), unique=True, index=True, nullable=False)
+    email = Column(String(100), unique=True, index=True, nullable=False)
+    full_name = Column(String(100))
+    hashed_password = Column(String(100), nullable=False)
+    is_active = Column(Boolean, default=True)
+    is_superuser = Column(Boolean, default=False)
+    avatar_url = Column(String(255), nullable=True)
+    bio = Column(String, nullable=True)
+    last_login = Column(String(255), nullable=True)
+    
+    # Relationships
+    searches = relationship("SearchHistory", back_populates="user")
+    saved_searches = relationship("SavedSearch", back_populates="user")
+    subscriptions = relationship("UserSubscription", back_populates="user")
+    
+    def __repr__(self):
+        return f"<User(id={self.id}, username={self.username}, email={self.email})>"

src/streamlit_database.py

@@ -0,0 +1,850 @@
+"""
+Database integration for Streamlit application.
+
+This module provides functions to interact with the database for the Streamlit frontend.
+It wraps the async database functions in sync functions for Streamlit compatibility.
+"""
+import os
+import asyncio
+import pandas as pd
+from typing import List, Dict, Any, Optional, Union, Tuple
+from datetime import datetime, timedelta
+
+from sqlalchemy.orm import sessionmaker
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession
+
+# Import database models
+from src.models.threat import Threat, ThreatSeverity, ThreatStatus, ThreatCategory
+from src.models.indicator import Indicator, IndicatorType
+from src.models.dark_web_content import DarkWebContent, DarkWebMention, ContentType, ContentStatus
+from src.models.alert import Alert, AlertStatus, AlertCategory
+from src.models.report import Report, ReportType, ReportStatus
+
+# Import service functions
+from src.api.services.dark_web_content_service import (
+    create_content, get_content_by_id, get_contents, count_contents,
+    create_mention, get_mentions, create_threat_from_content
+)
+from src.api.services.alert_service import (
+    create_alert, get_alert_by_id, get_alerts, count_alerts,
+    update_alert_status, mark_alert_as_read, get_alert_counts_by_severity
+)
+from src.api.services.threat_service import (
+    create_threat, get_threat_by_id, get_threats, count_threats,
+    update_threat, add_indicator_to_threat, get_threat_statistics
+)
+from src.api.services.report_service import (
+    create_report, get_report_by_id, get_reports, count_reports,
+    update_report, add_threat_to_report, publish_report
+)
+
+# Import schemas
+from src.api.schemas import PaginationParams
+
+# Get database URL from environment
+db_url = os.getenv("DATABASE_URL", "")
+if db_url.startswith("postgresql://"):
+    # Remove sslmode parameter if present which causes issues with asyncpg
+    if "?" in db_url:
+        base_url, params = db_url.split("?", 1)
+        param_list = params.split("&")
+        filtered_params = [p for p in param_list if not p.startswith("sslmode=")]
+        if filtered_params:
+            db_url = f"{base_url}?{'&'.join(filtered_params)}"
+        else:
+            db_url = base_url
+    
+    ASYNC_DATABASE_URL = db_url.replace("postgresql://", "postgresql+asyncpg://", 1)
+else:
+    ASYNC_DATABASE_URL = "postgresql+asyncpg://postgres:postgres@localhost:5432/postgres"
+
+# Create async engine
+engine = create_async_engine(
+    ASYNC_DATABASE_URL,
+    echo=False,
+    future=True,
+    pool_size=5,
+    max_overflow=10
+)
+
+# Create async session factory
+async_session = sessionmaker(
+    engine, 
+    class_=AsyncSession, 
+    expire_on_commit=False
+)
+
+
+def run_async(coro):
+    """Run an async function in a sync context."""
+    try:
+        loop = asyncio.get_event_loop()
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+    
+    return loop.run_until_complete(coro)
+
+
+async def get_session():
+    """Get an async database session."""
+    async with async_session() as session:
+        yield session
+
+
+def get_db_session():
+    """Get a database session for use in Streamlit."""
+    try:
+        session_gen = get_session().__aiter__()
+        return run_async(session_gen.__anext__())
+    except StopAsyncIteration:
+        return None
+
+
+async def get_async_session():
+    """
+    Async context manager for database sessions.
+    
+    Usage:
+        async with get_async_session() as session:
+            # Use session here
+    """
+    session = async_session()
+    try:
+        yield session
+        await session.commit()
+    except Exception as e:
+        await session.rollback()
+        raise e
+    finally:
+        await session.close()
+
+
+# Dark Web Content functions
+def get_dark_web_contents(
+    page: int = 1,
+    size: int = 10,
+    content_type: Optional[List[ContentType]] = None,
+    content_status: Optional[List[ContentStatus]] = None,
+    source_name: Optional[str] = None,
+    search_query: Optional[str] = None,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> pd.DataFrame:
+    """
+    Get dark web contents as a DataFrame.
+    
+    Args:
+        page: Page number
+        size: Page size
+        content_type: Filter by content type
+        content_status: Filter by content status
+        source_name: Filter by source name
+        search_query: Search in title and content
+        from_date: Filter by scraped_at >= from_date
+        to_date: Filter by scraped_at <= to_date
+        
+    Returns:
+        pd.DataFrame: DataFrame with dark web contents
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    contents = run_async(get_contents(
+        db=session,
+        pagination=PaginationParams(page=page, size=size),
+        content_type=content_type,
+        content_status=content_status,
+        source_name=source_name,
+        search_query=search_query,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+    
+    if not contents:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    data = []
+    for content in contents:
+        data.append({
+            "id": content.id,
+            "url": content.url,
+            "title": content.title,
+            "content_type": content.content_type.value if content.content_type else None,
+            "content_status": content.content_status.value if content.content_status else None,
+            "source_name": content.source_name,
+            "source_type": content.source_type,
+            "language": content.language,
+            "scraped_at": content.scraped_at,
+            "relevance_score": content.relevance_score,
+            "sentiment_score": content.sentiment_score,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def add_dark_web_content(
+    url: str,
+    content: str,
+    title: Optional[str] = None,
+    content_type: ContentType = ContentType.OTHER,
+    source_name: Optional[str] = None,
+    source_type: Optional[str] = None,
+) -> Optional[DarkWebContent]:
+    """
+    Add a new dark web content.
+    
+    Args:
+        url: URL of the content
+        content: Text content
+        title: Title of the content
+        content_type: Type of content
+        source_name: Name of the source
+        source_type: Type of source
+        
+    Returns:
+        Optional[DarkWebContent]: Created content or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(create_content(
+        db=session,
+        url=url,
+        content=content,
+        title=title,
+        content_type=content_type,
+        source_name=source_name,
+        source_type=source_type,
+    ))
+
+
+def get_dark_web_mentions(
+    page: int = 1,
+    size: int = 10,
+    keyword: Optional[str] = None,
+    content_id: Optional[int] = None,
+    is_verified: Optional[bool] = None,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> pd.DataFrame:
+    """
+    Get dark web mentions as a DataFrame.
+    
+    Args:
+        page: Page number
+        size: Page size
+        keyword: Filter by keyword
+        content_id: Filter by content ID
+        is_verified: Filter by verification status
+        from_date: Filter by created_at >= from_date
+        to_date: Filter by created_at <= to_date
+        
+    Returns:
+        pd.DataFrame: DataFrame with dark web mentions
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    mentions = run_async(get_mentions(
+        db=session,
+        pagination=PaginationParams(page=page, size=size),
+        keyword=keyword,
+        content_id=content_id,
+        is_verified=is_verified,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+    
+    if not mentions:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    data = []
+    for mention in mentions:
+        data.append({
+            "id": mention.id,
+            "content_id": mention.content_id,
+            "keyword": mention.keyword,
+            "snippet": mention.snippet,
+            "mention_type": mention.mention_type,
+            "confidence": mention.confidence,
+            "is_verified": mention.is_verified,
+            "created_at": mention.created_at,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def add_dark_web_mention(
+    content_id: int,
+    keyword: str,
+    context: Optional[str] = None,
+    snippet: Optional[str] = None,
+) -> Optional[DarkWebMention]:
+    """
+    Add a new dark web mention.
+    
+    Args:
+        content_id: ID of the content where the mention was found
+        keyword: Keyword that was mentioned
+        context: Text surrounding the mention
+        snippet: Extract of text containing the mention
+        
+    Returns:
+        Optional[DarkWebMention]: Created mention or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(create_mention(
+        db=session,
+        content_id=content_id,
+        keyword=keyword,
+        context=context,
+        snippet=snippet,
+    ))
+
+
+# Alerts functions
+def get_alerts_df(
+    page: int = 1,
+    size: int = 10,
+    severity: Optional[List[ThreatSeverity]] = None,
+    status: Optional[List[AlertStatus]] = None,
+    category: Optional[List[AlertCategory]] = None,
+    is_read: Optional[bool] = None,
+    search_query: Optional[str] = None,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> pd.DataFrame:
+    """
+    Get alerts as a DataFrame.
+    
+    Args:
+        page: Page number
+        size: Page size
+        severity: Filter by severity
+        status: Filter by status
+        category: Filter by category
+        is_read: Filter by read status
+        search_query: Search in title and description
+        from_date: Filter by generated_at >= from_date
+        to_date: Filter by generated_at <= to_date
+        
+    Returns:
+        pd.DataFrame: DataFrame with alerts
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    alerts = run_async(get_alerts(
+        db=session,
+        pagination=PaginationParams(page=page, size=size),
+        severity=severity,
+        status=status,
+        category=category,
+        is_read=is_read,
+        search_query=search_query,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+    
+    if not alerts:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    data = []
+    for alert in alerts:
+        data.append({
+            "id": alert.id,
+            "title": alert.title,
+            "description": alert.description,
+            "severity": alert.severity.value if alert.severity else None,
+            "status": alert.status.value if alert.status else None,
+            "category": alert.category.value if alert.category else None,
+            "generated_at": alert.generated_at,
+            "source_url": alert.source_url,
+            "is_read": alert.is_read,
+            "threat_id": alert.threat_id,
+            "mention_id": alert.mention_id,
+            "assigned_to_id": alert.assigned_to_id,
+            "action_taken": alert.action_taken,
+            "resolved_at": alert.resolved_at,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def add_alert(
+    title: str,
+    description: str,
+    severity: ThreatSeverity,
+    category: AlertCategory,
+    source_url: Optional[str] = None,
+    threat_id: Optional[int] = None,
+    mention_id: Optional[int] = None,
+) -> Optional[Alert]:
+    """
+    Add a new alert.
+    
+    Args:
+        title: Alert title
+        description: Alert description
+        severity: Alert severity
+        category: Alert category
+        source_url: Source URL for the alert
+        threat_id: ID of related threat
+        mention_id: ID of related dark web mention
+        
+    Returns:
+        Optional[Alert]: Created alert or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(create_alert(
+        db=session,
+        title=title,
+        description=description,
+        severity=severity,
+        category=category,
+        source_url=source_url,
+        threat_id=threat_id,
+        mention_id=mention_id,
+    ))
+
+
+def update_alert(
+    alert_id: int,
+    status: AlertStatus,
+    action_taken: Optional[str] = None,
+) -> Optional[Alert]:
+    """
+    Update alert status.
+    
+    Args:
+        alert_id: Alert ID
+        status: New status
+        action_taken: Description of action taken
+        
+    Returns:
+        Optional[Alert]: Updated alert or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(update_alert_status(
+        db=session,
+        alert_id=alert_id,
+        status=status,
+        action_taken=action_taken,
+    ))
+
+
+def get_alert_severity_counts(
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> Dict[str, int]:
+    """
+    Get count of alerts by severity.
+    
+    Args:
+        from_date: Filter by generated_at >= from_date
+        to_date: Filter by generated_at <= to_date
+        
+    Returns:
+        Dict[str, int]: Mapping of severity to count
+    """
+    session = get_db_session()
+    
+    if not session:
+        return {}
+    
+    return run_async(get_alert_counts_by_severity(
+        db=session,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+
+
+# Threats functions
+def get_threats_df(
+    page: int = 1,
+    size: int = 10,
+    severity: Optional[List[ThreatSeverity]] = None,
+    status: Optional[List[ThreatStatus]] = None,
+    category: Optional[List[ThreatCategory]] = None,
+    search_query: Optional[str] = None,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> pd.DataFrame:
+    """
+    Get threats as a DataFrame.
+    
+    Args:
+        page: Page number
+        size: Page size
+        severity: Filter by severity
+        status: Filter by status
+        category: Filter by category
+        search_query: Search in title and description
+        from_date: Filter by discovered_at >= from_date
+        to_date: Filter by discovered_at <= to_date
+        
+    Returns:
+        pd.DataFrame: DataFrame with threats
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    threats = run_async(get_threats(
+        db=session,
+        pagination=PaginationParams(page=page, size=size),
+        severity=severity,
+        status=status,
+        category=category,
+        search_query=search_query,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+    
+    if not threats:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    data = []
+    for threat in threats:
+        data.append({
+            "id": threat.id,
+            "title": threat.title,
+            "description": threat.description,
+            "severity": threat.severity.value if threat.severity else None,
+            "status": threat.status.value if threat.status else None,
+            "category": threat.category.value if threat.category else None,
+            "source_url": threat.source_url,
+            "source_name": threat.source_name,
+            "source_type": threat.source_type,
+            "discovered_at": threat.discovered_at,
+            "affected_entity": threat.affected_entity,
+            "affected_entity_type": threat.affected_entity_type,
+            "confidence_score": threat.confidence_score,
+            "risk_score": threat.risk_score,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def add_threat(
+    title: str,
+    description: str,
+    severity: ThreatSeverity,
+    category: ThreatCategory,
+    status: ThreatStatus = ThreatStatus.NEW,
+    source_url: Optional[str] = None,
+    source_name: Optional[str] = None,
+    source_type: Optional[str] = None,
+    affected_entity: Optional[str] = None,
+    affected_entity_type: Optional[str] = None,
+    confidence_score: float = 0.0,
+    risk_score: float = 0.0,
+) -> Optional[Threat]:
+    """
+    Add a new threat.
+    
+    Args:
+        title: Threat title
+        description: Threat description
+        severity: Threat severity
+        category: Threat category
+        status: Threat status
+        source_url: URL of the source
+        source_name: Name of the source
+        source_type: Type of source
+        affected_entity: Name of affected entity
+        affected_entity_type: Type of affected entity
+        confidence_score: Confidence score (0-1)
+        risk_score: Risk score (0-1)
+        
+    Returns:
+        Optional[Threat]: Created threat or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(create_threat(
+        db=session,
+        title=title,
+        description=description,
+        severity=severity,
+        category=category,
+        status=status,
+        source_url=source_url,
+        source_name=source_name,
+        source_type=source_type,
+        affected_entity=affected_entity,
+        affected_entity_type=affected_entity_type,
+        confidence_score=confidence_score,
+        risk_score=risk_score,
+    ))
+
+
+def add_indicator(
+    threat_id: int,
+    value: str,
+    indicator_type: IndicatorType,
+    description: Optional[str] = None,
+    is_verified: bool = False,
+    context: Optional[str] = None,
+    source: Optional[str] = None,
+) -> Optional[Indicator]:
+    """
+    Add an indicator to a threat.
+    
+    Args:
+        threat_id: Threat ID
+        value: Indicator value
+        indicator_type: Indicator type
+        description: Indicator description
+        is_verified: Whether the indicator is verified
+        context: Context of the indicator
+        source: Source of the indicator
+        
+    Returns:
+        Optional[Indicator]: Created indicator or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(add_indicator_to_threat(
+        db=session,
+        threat_id=threat_id,
+        value=value,
+        indicator_type=indicator_type,
+        description=description,
+        is_verified=is_verified,
+        context=context,
+        source=source,
+    ))
+
+
+def get_threat_stats(
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> Dict[str, Any]:
+    """
+    Get threat statistics.
+    
+    Args:
+        from_date: Filter by discovered_at >= from_date
+        to_date: Filter by discovered_at <= to_date
+        
+    Returns:
+        Dict[str, Any]: Threat statistics
+    """
+    session = get_db_session()
+    
+    if not session:
+        return {}
+    
+    return run_async(get_threat_statistics(
+        db=session,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+
+
+# Reports functions
+def get_reports_df(
+    page: int = 1,
+    size: int = 10,
+    report_type: Optional[List[ReportType]] = None,
+    status: Optional[List[ReportStatus]] = None,
+    severity: Optional[List[ThreatSeverity]] = None,
+    search_query: Optional[str] = None,
+    from_date: Optional[datetime] = None,
+    to_date: Optional[datetime] = None,
+) -> pd.DataFrame:
+    """
+    Get reports as a DataFrame.
+    
+    Args:
+        page: Page number
+        size: Page size
+        report_type: Filter by report type
+        status: Filter by status
+        severity: Filter by severity
+        search_query: Search in title and summary
+        from_date: Filter by created_at >= from_date
+        to_date: Filter by created_at <= to_date
+        
+    Returns:
+        pd.DataFrame: DataFrame with reports
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    reports = run_async(get_reports(
+        db=session,
+        pagination=PaginationParams(page=page, size=size),
+        report_type=report_type,
+        status=status,
+        severity=severity,
+        search_query=search_query,
+        from_date=from_date,
+        to_date=to_date,
+    ))
+    
+    if not reports:
+        return pd.DataFrame()
+    
+    # Convert to DataFrame
+    data = []
+    for report in reports:
+        data.append({
+            "id": report.id,
+            "report_id": report.report_id,
+            "title": report.title,
+            "summary": report.summary,
+            "report_type": report.report_type.value if report.report_type else None,
+            "status": report.status.value if report.status else None,
+            "severity": report.severity.value if report.severity else None,
+            "publish_date": report.publish_date,
+            "created_at": report.created_at,
+            "time_period_start": report.time_period_start,
+            "time_period_end": report.time_period_end,
+            "author_id": report.author_id,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def add_report(
+    title: str,
+    summary: str,
+    content: str,
+    report_type: ReportType,
+    report_id: str,
+    status: ReportStatus = ReportStatus.DRAFT,
+    severity: Optional[ThreatSeverity] = None,
+    publish_date: Optional[datetime] = None,
+    time_period_start: Optional[datetime] = None,
+    time_period_end: Optional[datetime] = None,
+    keywords: Optional[List[str]] = None,
+    author_id: Optional[int] = None,
+) -> Optional[Report]:
+    """
+    Add a new report.
+    
+    Args:
+        title: Report title
+        summary: Report summary
+        content: Report content
+        report_type: Type of report
+        report_id: Custom ID for the report
+        status: Report status
+        severity: Report severity
+        publish_date: Publication date
+        time_period_start: Start of time period covered
+        time_period_end: End of time period covered
+        keywords: List of keywords related to the report
+        author_id: ID of the report author
+        
+    Returns:
+        Optional[Report]: Created report or None
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    return run_async(create_report(
+        db=session,
+        title=title,
+        summary=summary,
+        content=content,
+        report_type=report_type,
+        report_id=report_id,
+        status=status,
+        severity=severity,
+        publish_date=publish_date,
+        time_period_start=time_period_start,
+        time_period_end=time_period_end,
+        keywords=keywords,
+        author_id=author_id,
+    ))
+
+
+# Helper functions
+def get_time_range_dates(time_range: str) -> Tuple[datetime, datetime]:
+    """
+    Get start and end dates for a time range.
+    
+    Args:
+        time_range: Time range string (e.g., "Last 7 Days")
+        
+    Returns:
+        Tuple[datetime, datetime]: (start_date, end_date)
+    """
+    end_date = datetime.utcnow()
+    
+    if time_range == "Last 24 Hours":
+        start_date = end_date - timedelta(days=1)
+    elif time_range == "Last 7 Days":
+        start_date = end_date - timedelta(days=7)
+    elif time_range == "Last 30 Days":
+        start_date = end_date - timedelta(days=30)
+    elif time_range == "Last Quarter":
+        start_date = end_date - timedelta(days=90)
+    else:  # Default to last 30 days
+        start_date = end_date - timedelta(days=30)
+    
+    return start_date, end_date
+
+
+# Initialize database connection
+def init_db_connection():
+    """Initialize database connection and check if tables exist."""
+    session = get_db_session()
+    
+    if not session:
+        return False
+    
+    # Check if tables exist
+    from sqlalchemy.future import select
+    
+    try:
+        # Try to query if tables exist using SQLAlchemy text()
+        from sqlalchemy import text
+        query = text("SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'users')")
+        result = run_async(session.execute(query))
+        exists = result.scalar()
+        
+        return exists
+    except Exception as e:
+        # Tables might not exist yet
+        print(f"Error checking database: {e}")
+        return False

src/streamlit_subscription_services.py

@@ -0,0 +1,450 @@
+"""
+Streamlit integration for subscription services.
+"""
+import os
+import asyncio
+import pandas as pd
+from typing import List, Dict, Any, Optional, Union
+from datetime import datetime
+
+import stripe
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from src.models.subscription import (
+    SubscriptionPlan, UserSubscription, PaymentHistory,
+    SubscriptionTier, BillingPeriod, SubscriptionStatus, PaymentStatus
+)
+from src.api.services.subscription_service import (
+    get_subscription_plans, get_subscription_plan_by_id, get_subscription_plan_by_tier,
+    create_subscription_plan, update_subscription_plan,
+    get_user_subscription, get_user_subscription_by_id,
+    create_user_subscription, cancel_user_subscription
+)
+
+from src.streamlit_database import run_async, get_db_session
+
+# Set up Stripe API keys for client-side usage
+STRIPE_PUBLISHABLE_KEY = os.environ.get("STRIPE_PUBLISHABLE_KEY")
+
+def get_subscription_plans_df(active_only: bool = True) -> pd.DataFrame:
+    """
+    Get all subscription plans as a DataFrame.
+    
+    Args:
+        active_only: If True, only return active plans
+        
+    Returns:
+        DataFrame containing subscription plans
+    """
+    session = get_db_session()
+    
+    if not session:
+        return pd.DataFrame()
+    
+    plans = run_async(get_subscription_plans(session, active_only))
+    
+    if not plans:
+        return pd.DataFrame()
+    
+    data = []
+    for plan in plans:
+        data.append({
+            "id": plan.id,
+            "name": plan.name,
+            "tier": plan.tier.value if plan.tier else None,
+            "description": plan.description,
+            "price_monthly": plan.price_monthly,
+            "price_annually": plan.price_annually,
+            "max_alerts": plan.max_alerts,
+            "max_reports": plan.max_reports,
+            "max_searches_per_day": plan.max_searches_per_day,
+            "max_monitoring_keywords": plan.max_monitoring_keywords,
+            "max_data_retention_days": plan.max_data_retention_days,
+            "supports_api_access": plan.supports_api_access,
+            "supports_live_feed": plan.supports_live_feed,
+            "supports_dark_web_monitoring": plan.supports_dark_web_monitoring,
+            "supports_export": plan.supports_export,
+            "supports_advanced_analytics": plan.supports_advanced_analytics,
+            "is_active": plan.is_active,
+        })
+    
+    return pd.DataFrame(data)
+
+
+def get_subscription_plan(plan_id: int) -> Optional[Dict[str, Any]]:
+    """
+    Get a subscription plan by ID.
+    
+    Args:
+        plan_id: ID of the plan to get
+        
+    Returns:
+        Dictionary containing plan details or None if not found
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    plan = run_async(get_subscription_plan_by_id(session, plan_id))
+    
+    if not plan:
+        return None
+    
+    return {
+        "id": plan.id,
+        "name": plan.name,
+        "tier": plan.tier.value if plan.tier else None,
+        "description": plan.description,
+        "price_monthly": plan.price_monthly,
+        "price_annually": plan.price_annually,
+        "max_alerts": plan.max_alerts,
+        "max_reports": plan.max_reports,
+        "max_searches_per_day": plan.max_searches_per_day,
+        "max_monitoring_keywords": plan.max_monitoring_keywords,
+        "max_data_retention_days": plan.max_data_retention_days,
+        "supports_api_access": plan.supports_api_access,
+        "supports_live_feed": plan.supports_live_feed,
+        "supports_dark_web_monitoring": plan.supports_dark_web_monitoring,
+        "supports_export": plan.supports_export,
+        "supports_advanced_analytics": plan.supports_advanced_analytics,
+        "is_active": plan.is_active,
+        "stripe_product_id": plan.stripe_product_id,
+        "stripe_monthly_price_id": plan.stripe_monthly_price_id,
+        "stripe_annual_price_id": plan.stripe_annual_price_id,
+    }
+
+
+def get_user_current_subscription(user_id: int) -> Optional[Dict[str, Any]]:
+    """
+    Get a user's current subscription.
+    
+    Args:
+        user_id: ID of the user
+        
+    Returns:
+        Dictionary containing subscription details or None if not found
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    subscription = run_async(get_user_subscription(session, user_id))
+    
+    if not subscription:
+        return None
+    
+    plan = subscription.plan
+    
+    return {
+        "id": subscription.id,
+        "user_id": subscription.user_id,
+        "plan_id": subscription.plan_id,
+        "plan_name": plan.name if plan else None,
+        "plan_tier": plan.tier.value if plan and plan.tier else None,
+        "status": subscription.status.value if subscription.status else None,
+        "billing_period": subscription.billing_period.value if subscription.billing_period else None,
+        "current_period_start": subscription.current_period_start,
+        "current_period_end": subscription.current_period_end,
+        "stripe_subscription_id": subscription.stripe_subscription_id,
+        "stripe_customer_id": subscription.stripe_customer_id,
+        "created_at": subscription.created_at,
+        "canceled_at": subscription.canceled_at,
+    }
+
+
+def create_new_subscription_plan(
+    name: str,
+    tier: str,
+    description: str,
+    price_monthly: float,
+    price_annually: float,
+    max_alerts: int = 10,
+    max_reports: int = 5,
+    max_searches_per_day: int = 20,
+    max_monitoring_keywords: int = 10,
+    max_data_retention_days: int = 30,
+    supports_api_access: bool = False,
+    supports_live_feed: bool = False,
+    supports_dark_web_monitoring: bool = False,
+    supports_export: bool = False,
+    supports_advanced_analytics: bool = False,
+    create_stripe_product: bool = True
+) -> Optional[Dict[str, Any]]:
+    """
+    Create a new subscription plan.
+    
+    Args:
+        name: Name of the plan
+        tier: Tier of the plan (must be one of "free", "basic", "professional", "enterprise")
+        description: Description of the plan
+        price_monthly: Monthly price of the plan
+        price_annually: Annual price of the plan
+        max_alerts: Maximum number of alerts allowed
+        max_reports: Maximum number of reports allowed
+        max_searches_per_day: Maximum number of searches per day
+        max_monitoring_keywords: Maximum number of monitoring keywords
+        max_data_retention_days: Maximum number of days to retain data
+        supports_api_access: Whether the plan supports API access
+        supports_live_feed: Whether the plan supports live feed
+        supports_dark_web_monitoring: Whether the plan supports dark web monitoring
+        supports_export: Whether the plan supports data export
+        supports_advanced_analytics: Whether the plan supports advanced analytics
+        create_stripe_product: Whether to create a Stripe product for this plan
+        
+    Returns:
+        Dictionary containing plan details or None if creation failed
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    try:
+        # Convert tier string to enum
+        tier_enum = SubscriptionTier(tier.lower())
+    except ValueError:
+        return None
+    
+    plan = run_async(create_subscription_plan(
+        db=session,
+        name=name,
+        tier=tier_enum,
+        description=description,
+        price_monthly=price_monthly,
+        price_annually=price_annually,
+        max_alerts=max_alerts,
+        max_reports=max_reports,
+        max_searches_per_day=max_searches_per_day,
+        max_monitoring_keywords=max_monitoring_keywords,
+        max_data_retention_days=max_data_retention_days,
+        supports_api_access=supports_api_access,
+        supports_live_feed=supports_live_feed,
+        supports_dark_web_monitoring=supports_dark_web_monitoring,
+        supports_export=supports_export,
+        supports_advanced_analytics=supports_advanced_analytics,
+        create_stripe_product=create_stripe_product
+    ))
+    
+    if not plan:
+        return None
+    
+    return {
+        "id": plan.id,
+        "name": plan.name,
+        "tier": plan.tier.value if plan.tier else None,
+        "description": plan.description,
+        "price_monthly": plan.price_monthly,
+        "price_annually": plan.price_annually,
+        "max_alerts": plan.max_alerts,
+        "max_reports": plan.max_reports,
+        "max_searches_per_day": plan.max_searches_per_day,
+        "max_monitoring_keywords": plan.max_monitoring_keywords,
+        "max_data_retention_days": plan.max_data_retention_days,
+        "supports_api_access": plan.supports_api_access,
+        "supports_live_feed": plan.supports_live_feed,
+        "supports_dark_web_monitoring": plan.supports_dark_web_monitoring,
+        "supports_export": plan.supports_export,
+        "supports_advanced_analytics": plan.supports_advanced_analytics,
+        "is_active": plan.is_active,
+        "stripe_product_id": plan.stripe_product_id,
+        "stripe_monthly_price_id": plan.stripe_monthly_price_id,
+        "stripe_annual_price_id": plan.stripe_annual_price_id,
+    }
+
+
+def subscribe_user_to_plan(
+    user_id: int,
+    plan_id: int,
+    billing_period: str = "monthly",
+    create_stripe_subscription: bool = True,
+    payment_method_id: Optional[str] = None
+) -> Optional[Dict[str, Any]]:
+    """
+    Subscribe a user to a plan.
+    
+    Args:
+        user_id: ID of the user
+        plan_id: ID of the plan
+        billing_period: Billing period ("monthly" or "annually")
+        create_stripe_subscription: Whether to create a Stripe subscription
+        payment_method_id: ID of the payment method to use (required if create_stripe_subscription is True)
+        
+    Returns:
+        Dictionary containing subscription details or None if creation failed
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    try:
+        # Convert billing period string to enum
+        billing_period_enum = BillingPeriod(billing_period.lower())
+    except ValueError:
+        return None
+    
+    subscription = run_async(create_user_subscription(
+        db=session,
+        user_id=user_id,
+        plan_id=plan_id,
+        billing_period=billing_period_enum,
+        create_stripe_subscription=create_stripe_subscription,
+        payment_method_id=payment_method_id
+    ))
+    
+    if not subscription:
+        return None
+    
+    plan = subscription.plan
+    
+    return {
+        "id": subscription.id,
+        "user_id": subscription.user_id,
+        "plan_id": subscription.plan_id,
+        "plan_name": plan.name if plan else None,
+        "plan_tier": plan.tier.value if plan and plan.tier else None,
+        "status": subscription.status.value if subscription.status else None,
+        "billing_period": subscription.billing_period.value if subscription.billing_period else None,
+        "current_period_start": subscription.current_period_start,
+        "current_period_end": subscription.current_period_end,
+        "stripe_subscription_id": subscription.stripe_subscription_id,
+        "stripe_customer_id": subscription.stripe_customer_id,
+        "created_at": subscription.created_at,
+    }
+
+
+def cancel_subscription(
+    subscription_id: int,
+    cancel_stripe_subscription: bool = True
+) -> Optional[Dict[str, Any]]:
+    """
+    Cancel a subscription.
+    
+    Args:
+        subscription_id: ID of the subscription to cancel
+        cancel_stripe_subscription: Whether to cancel the Stripe subscription
+        
+    Returns:
+        Dictionary containing subscription details or None if cancellation failed
+    """
+    session = get_db_session()
+    
+    if not session:
+        return None
+    
+    subscription = run_async(cancel_user_subscription(
+        db=session,
+        subscription_id=subscription_id,
+        cancel_stripe_subscription=cancel_stripe_subscription
+    ))
+    
+    if not subscription:
+        return None
+    
+    plan = subscription.plan
+    
+    return {
+        "id": subscription.id,
+        "user_id": subscription.user_id,
+        "plan_id": subscription.plan_id,
+        "plan_name": plan.name if plan else None,
+        "plan_tier": plan.tier.value if plan and plan.tier else None,
+        "status": subscription.status.value if subscription.status else None,
+        "billing_period": subscription.billing_period.value if subscription.billing_period else None,
+        "current_period_start": subscription.current_period_start,
+        "current_period_end": subscription.current_period_end,
+        "stripe_subscription_id": subscription.stripe_subscription_id,
+        "stripe_customer_id": subscription.stripe_customer_id,
+        "created_at": subscription.created_at,
+        "canceled_at": subscription.canceled_at,
+    }
+
+
+def initialize_default_plans():
+    """Initialize default subscription plans if they don't exist."""
+    # Get existing plans
+    plans_df = get_subscription_plans_df(active_only=False)
+    
+    if not plans_df.empty:
+        # Plans already exist
+        return
+    
+    # Create default plans
+    # Free tier
+    create_new_subscription_plan(
+        name="Free",
+        tier="free",
+        description="Basic access to the platform with limited features. Perfect for individuals or small teams starting with OSINT.",
+        price_monthly=0.0,
+        price_annually=0.0,
+        max_alerts=5,
+        max_reports=2,
+        max_searches_per_day=10,
+        max_monitoring_keywords=5,
+        max_data_retention_days=7,
+        supports_api_access=False,
+        supports_live_feed=False,
+        supports_dark_web_monitoring=False,
+        supports_export=False,
+        supports_advanced_analytics=False,
+        create_stripe_product=False  # No need to create Stripe product for free tier
+    )
+    
+    # Basic tier
+    create_new_subscription_plan(
+        name="Basic",
+        tier="basic",
+        description="Enhanced access with more features. Ideal for small businesses and security teams requiring regular threat intelligence.",
+        price_monthly=29.99,
+        price_annually=299.99,
+        max_alerts=20,
+        max_reports=10,
+        max_searches_per_day=50,
+        max_monitoring_keywords=25,
+        max_data_retention_days=30,
+        supports_api_access=False,
+        supports_live_feed=True,
+        supports_dark_web_monitoring=True,
+        supports_export=True,
+        supports_advanced_analytics=False
+    )
+    
+    # Professional tier
+    create_new_subscription_plan(
+        name="Professional",
+        tier="professional",
+        description="Comprehensive access for professional users. Perfect for medium-sized organizations requiring advanced threat intelligence capabilities.",
+        price_monthly=99.99,
+        price_annually=999.99,
+        max_alerts=100,
+        max_reports=50,
+        max_searches_per_day=200,
+        max_monitoring_keywords=100,
+        max_data_retention_days=90,
+        supports_api_access=True,
+        supports_live_feed=True,
+        supports_dark_web_monitoring=True,
+        supports_export=True,
+        supports_advanced_analytics=True
+    )
+    
+    # Enterprise tier
+    create_new_subscription_plan(
+        name="Enterprise",
+        tier="enterprise",
+        description="Full access to all features with unlimited usage. Designed for large organizations with sophisticated threat intelligence requirements.",
+        price_monthly=249.99,
+        price_annually=2499.99,
+        max_alerts=0,  # Unlimited
+        max_reports=0,  # Unlimited
+        max_searches_per_day=0,  # Unlimited
+        max_monitoring_keywords=0,  # Unlimited
+        max_data_retention_days=365,
+        supports_api_access=True,
+        supports_live_feed=True,
+        supports_dark_web_monitoring=True,
+        supports_export=True,
+        supports_advanced_analytics=True
+    )