Update app.py

app.py

@@ -2,6 +2,9 @@ import streamlit as st
 import requests
 from bs4 import BeautifulSoup
 import pandas as pd
+import time
+import random
+from datetime import datetime
 
 # Set page configuration
 st.set_page_config(
@@ -69,80 +72,174 @@ def extract_data_from_html(html_content):
     return df
 
 # Function to extract data from the actual website
-@st.cache_data(ttl=3600)  # Cache data for 1 hour
-def extract_data_from_website(url="https://mopsov.twse.com.tw/mops/web/t05sr01_1"):
+@st.cache_data(ttl=1800)  # Cache data for 30 minutes
+def extract_data_from_website(url="https://mopsov.twse.com.tw/mops/web/t05sr01_1", retries=3):
+    # Rotating User-Agents to avoid detection
+    user_agents = [
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Safari/605.1.15',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:90.0) Gecko/20100101 Firefox/90.0',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36'
+    ]
+    
+    # More comprehensive headers
     headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-        'Accept-Language': 'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7'
+        'User-Agent': random.choice(user_agents),
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+        'Accept-Language': 'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Upgrade-Insecure-Requests': '1',
+        'Sec-Fetch-Dest': 'document',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-User': '?1',
+        'Cache-Control': 'max-age=0'
     }
     
-    try:
-        with st.spinner('正在從網站擷取資料...'):
-            # Make request to get the session cookies first
-            session = requests.Session()
-            session.get("https://mopsov.twse.com.tw/mops/web/index", headers=headers)
-            
-            # Now access the announcements page
-            response = session.get(url, headers=headers)
-            
-            # Create form data for POST request to get the announcements
-            form_data = {
-                'step': '1',
-                'firstin': '1',
-                'off': '1',
-                'keyword4': '',
-                'code1': '',
-                'TYPEK2': '',
-                'checkbtn': '',
-                'queryName': 'co_id',
-                'inpuType': 'co_id',
-                'TYPEK': 'all',
-                'co_id': '',
-                'year': '',
-                'month': '',
-                'day': '',
-                'b_date': '',
-                'e_date': '',
-            }
-            
-            # Make POST request
-            post_response = session.post(url, data=form_data, headers=headers)
-            
-            # Parse the HTML content
-            df = extract_data_from_html(post_response.text)
-            
-            if not df.empty:
-                st.success(f'成功擷取 {len(df)} 筆公告資料!')
-                return df
-            else:
-                st.warning('無法從網站擷取資料，切換到範例資料')
-                return None
+    # Get current date in Taiwan format (ROC calendar)
+    now = datetime.now()
+    roc_year = now.year - 1911
+    current_date = f"{roc_year}/{now.month:02d}/{now.day:02d}"
+    
+    for attempt in range(retries):
+        try:
+            with st.spinner(f'正在從網站擷取資料... (嘗試 {attempt+1}/{retries})'):
+                # Create a session to maintain cookies
+                session = requests.Session()
+                
+                # Initial visit to homepage to get cookies
+                session.get("https://mopsov.twse.com.tw/mops/web/index", headers=headers, timeout=15)
+                
+                # Small delay to avoid triggering anti-scraping measures
+                time.sleep(random.uniform(1, 3))
+                
+                # Visit the announcements page to get the form structure
+                response = session.get(url, headers=headers, timeout=15)
+                
+                # Another small delay
+                time.sleep(random.uniform(1, 2))
                 
-    except Exception as e:
-        st.error(f'訪問網站時發生錯誤: {e}')
-        return None
+                # Create form data for POST request to get the announcements
+                # Enhanced form data with more parameters
+                form_data = {
+                    'step': '1',
+                    'firstin': '1',
+                    'off': '1',
+                    'keyword4': '',
+                    'code1': '',
+                    'TYPEK2': '',
+                    'checkbtn': '',
+                    'queryName': 'co_id',
+                    'inpuType': 'co_id',
+                    'TYPEK': 'all',
+                    'co_id': '',
+                    'year': str(roc_year),  # Current ROC year
+                    'month': str(now.month),  # Current month
+                    'day': str(now.day),  # Current day
+                    'b_date': '',
+                    'e_date': '',
+                    'skey': '',
+                    'date1': '',
+                    'date2': '',
+                }
+                
+                # Make POST request
+                post_headers = headers.copy()
+                post_headers['Content-Type'] = 'application/x-www-form-urlencoded'
+                post_headers['Origin'] = 'https://mopsov.twse.com.tw'
+                post_headers['Referer'] = url
+                
+                post_response = session.post(
+                    url, 
+                    data=form_data, 
+                    headers=post_headers,
+                    timeout=20
+                )
+                
+                # Check if the response seems valid
+                if "hasBorder" in post_response.text and post_response.status_code == 200:
+                    # Parse the HTML content
+                    df = extract_data_from_html(post_response.text)
+                    
+                    if not df.empty:
+                        st.success(f'成功擷取 {len(df)} 筆公告資料!')
+                        return df
+                    else:
+                        st.info(f'網站回應成功，但找不到公告資料。可能是當日({current_date})尚無公告。')
+                        # Attempt to look for other messages in the response
+                        soup = BeautifulSoup(post_response.text, 'html.parser')
+                        messages = soup.find_all('td', {'class': 'compName'})
+                        if messages:
+                            st.info(f"網站訊息: {messages[0].text.strip()}")
+                        continue
+                else:
+                    st.warning(f'網站返回狀態碼: {post_response.status_code}。嘗試重新連接...')
+                    continue
+                    
+        except requests.exceptions.RequestException as e:
+            st.warning(f'請求錯誤 (嘗試 {attempt+1}/{retries}): {str(e)}')
+            time.sleep(2)  # Wait before retrying
+        except Exception as e:
+            st.warning(f'處理錯誤 (嘗試 {attempt+1}/{retries}): {str(e)}')
+            time.sleep(2)  # Wait before retrying
+    
+    st.warning(f'嘗試 {retries} 次後仍無法從網站擷取資料，切換到範例資料')
+    return None
 
 # Example provided in the original code
 default_html_content = """
 <table class="hasBorder"><thead><tr class="tblHead_2"><th width="10%" nowrap="">公司代號</th><th width="10%" nowrap="">公司簡稱</th><th nowrap="">發言日期</th><th width="10%" nowrap="">發言時間</th><th>主旨</th></tr></thead><tbody id="tab2"><tr class="even_2" onmouseover="this.className='mouseOn_2';" onmouseout="this.className='even_2';"><td>7724</td><td>諾亞克</td><td>114/04/01</td><td>00:06:30</td><td class="table02"><button style="width:300px;height:28px;text-align:left;background-color:transparent;border:0;cursor:pointer;" onclick="document.fm_t05sr01_1.step.value='1';document.fm_t05sr01_1.SEQ_NO.value='1';document.fm_t05sr01_1.SPOKE_TIME.value='630';document.fm_t05sr01_1.SPOKE_DATE.value='20250401';document.fm_t05sr01_1.COMPANY_NAME.value='諾亞克';document.fm_t05sr01_1.COMPANY_ID.value='7724';document.fm_t05sr01_1.skey.value='7724202504011';document.fm_t05sr01_1.hhc_co_name.value='諾亞克';openWindow(document.fm_t05sr01_1 ,'');" title="公告本公司董事會決議不分配113年度董事及員工酬勞">公告本公司董事會決議不分配113年度董事......</button></td></tr><tr class="odd_2" onmouseover="this.className='mouseOn_2';" onmouseout="this.className='odd_2';"><td>4117</td><td>普生</td><td>114/04/01</td><td>00:04:31</td><td class="table02"><button style="width:300px;height:28px;text-align:left;background-color:transparent;border:0;cursor:pointer;" onclick="document.fm_t05sr01_1.step.value='1';document.fm_t05sr01_1.SEQ_NO.value='7';document.fm_t05sr01_1.SPOKE_TIME.value='431';document.fm_t05sr01_1.SPOKE_DATE.value='20250401';document.fm_t05sr01_1.COMPANY_NAME.value='普生';document.fm_t05sr01_1.COMPANY_ID.value='4117';document.fm_t05sr01_1.skey.value='4117202503317';document.fm_t05sr01_1.hhc_co_name.value='普生';openWindow(document.fm_t05sr01_1 ,'');" title="公告本公司董事會決議不發放股利">公告本公司董事會決議不發放股利</button></td></tr></tbody></table>
 """
 
-# Sidebar with data source options
+# Add date range picker to sidebar
 st.sidebar.header("資料來源選項")
 data_source = st.sidebar.radio(
     "選擇資料來源",
     ["從網站擷取資料", "使用範例資料", "貼上HTML代碼"]
 )
 
+# Date range selector (only visible when fetching from website)
+if data_source == "從網站擷取資料":
+    st.sidebar.subheader("日期選擇")
+    
+    # Calculate ROC year (Taiwan calendar)
+    current_year = datetime.now().year
+    roc_year = current_year - 1911
+    
+    # Date inputs
+    col1, col2 = st.sidebar.columns(2)
+    with col1:
+        year = st.number_input("年度(民國)", min_value=100, max_value=roc_year, value=roc_year)
+    with col2:
+        month = st.number_input("月份", min_value=1, max_value=12, value=datetime.now().month)
+    
+    custom_date = st.sidebar.checkbox("指定日期範圍")
+    if custom_date:
+        start_date = st.sidebar.date_input("起始日期")
+        end_date = st.sidebar.date_input("結束日期")
+
 # Initialize data frame
 df = None
 
-# Process based on data source selection
+# Add progress
 if data_source == "從網站擷取資料":
-    df = extract_data_from_website()
-    if df is None:
-        st.sidebar.warning("從網站擷取資料失敗，切換到範例資料")
-        df = extract_data_from_html(default_html_content)
+    with st.expander("網路連線診斷", expanded=False):
+        st.write("檢查台灣證券交易所網站連線...")
+        try:
+            check_response = requests.get("https://mopsov.twse.com.tw/", timeout=5)
+            st.write(f"網站狀態: {'可連線 ✅' if check_response.status_code == 200 else '無法連線 ❌'}")
+            st.write(f"HTTP 狀態碼: {check_response.status_code}")
+        except Exception as e:
+            st.write(f"網站連線檢查失敗: {e}")
+    
+    fetch_data = st.button("開始擷取資料", type="primary")
+    if fetch_data:
+        # This will be enhanced to use the date parameters when implemented
+        df = extract_data_from_website()
+        if df is None:
+            st.sidebar.warning("從網站擷取資料失敗，切換到範例資料")
+            df = extract_data_from_html(default_html_content)
         
 elif data_source == "使用範例資料":
     df = extract_data_from_html(default_html_content)
@@ -158,11 +255,13 @@ if df is not None and not df.empty:
     st.subheader("台灣證券交易所公告資料")
     
     # Add search filters
-    col1, col2 = st.columns(2)
+    col1, col2, col3 = st.columns(3)
     with col1:
         search_code = st.text_input("依公司代號篩選")
     with col2:
         search_name = st.text_input("依公司名稱篩選")
+    with col3:
+        search_subject = st.text_input("依主旨關鍵字篩選")
     
     # Apply filters if provided
     filtered_df = df.copy()
@@ -170,27 +269,51 @@ if df is not None and not df.empty:
         filtered_df = filtered_df[filtered_df['公司代號'].str.contains(search_code)]
     if search_name:
         filtered_df = filtered_df[filtered_df['公司簡稱'].str.contains(search_name)]
+    if search_subject:
+        filtered_df = filtered_df[filtered_df['主旨'].str.contains(search_subject)]
     
     # Display the data
     st.dataframe(filtered_df, use_container_width=True)
     
-    # Download button
-    csv = filtered_df.to_csv(index=False).encode('utf-8-sig')
-    st.download_button(
-        label="下載為CSV",
-        data=csv,
-        file_name="twse_announcements.csv",
-        mime="text/csv",
-    )
+    # Download buttons
+    col1, col2 = st.columns(2)
+    with col1:
+        csv = filtered_df.to_csv(index=False).encode('utf-8-sig')
+        st.download_button(
+            label="下載為CSV",
+            data=csv,
+            file_name="twse_announcements.csv",
+            mime="text/csv",
+        )
+    with col2:
+        excel_buffer = pd.ExcelWriter("twse_announcements.xlsx", engine='xlsxwriter')
+        filtered_df.to_excel(excel_buffer, index=False, sheet_name='公告資料')
+        excel_buffer.close()
+        
+        with open("twse_announcements.xlsx", "rb") as f:
+            excel_data = f.read()
+        
+        st.download_button(
+            label="下載為Excel",
+            data=excel_data,
+            file_name="twse_announcements.xlsx",
+            mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        )
     
     # Display statistics
     st.subheader("資料統計")
-    col1, col2 = st.columns(2)
+    col1, col2, col3 = st.columns(3)
     with col1:
         st.metric("公告總數", len(filtered_df))
     with col2:
         company_count = filtered_df['公司代號'].nunique()
         st.metric("公司數量", company_count)
+    with col3:
+        date_counts = filtered_df['發言日期'].value_counts()
+        if not date_counts.empty:
+            latest_date = date_counts.index[0]
+            latest_count = date_counts.iloc[0]
+            st.metric(f"最新日期 ({latest_date})", latest_count)
     
     # Show announcement details on selection
     if not filtered_df.empty:
@@ -198,7 +321,7 @@ if df is not None and not df.empty:
         selected_indices = st.multiselect(
             "選擇公告", 
             options=list(range(len(filtered_df))),
-            format_func=lambda i: f"{filtered_df.iloc[i]['公司簡稱']} - {filtered_df.iloc[i]['主旨'][:20]}..."
+            format_func=lambda i: f"{filtered_df.iloc[i]['公司簡稱']} ({filtered_df.iloc[i]['公司代號']}) - {filtered_df.iloc[i]['主旨'][:20]}..."
         )
         
         if selected_indices:
@@ -214,4 +337,35 @@ else:
 
 # Footer
 st.markdown("---")
-st.markdown("台灣證券交易所公告擷取工具 | 資料來源: [台灣證券交易所](https://mopsov.twse.com.tw/mops/web/index)")
+st.markdown("台灣證券交易所公告擷取工具 | 資料來源: [台灣證券交易所](https://mopsov.twse.com.tw/mops/web/index)")
+
+# Add FAQ section at the bottom
+with st.expander("常見問題", expanded=False):
+    st.subheader("常見問題")
+    
+    st.markdown("""
+    **Q: 為什麼無法從網站擷取資料？**
+    
+    A: 可能原因包括：
+    - 台灣證券交易所網站暫時無法連接
+    - 當日尚無公告資料
+    - 網站結構可能有所變更
+    - 網路連線問題
+    
+    **Q: 資料顯示的日期格式是什麼？**
+    
+    A: 發言日期採用中華民國紀年（民國紀年），例如「114/04/01」表示西元2025年4月1日。
+    
+    **Q: 為什麼有些公告的主旨只顯示部分內容？**
+    
+    A: 當主旨內容過長時，網站顯示會自動截斷。點選公告查看詳情可能會顯示完整主旨。
+    
+    **Q: 如何取得更多歷史公告？**
+    
+    A: 本工具目前僅擷取當前頁面資料。若需查詢歷史資料，建議直接前往[台灣證券交易所](https://mopsov.twse.com.tw/mops/web/index)官方網站搜尋。
+    """)
+
+# Add version info
+st.sidebar.markdown("---")
+st.sidebar.caption("版本: 1.1.0")
+st.sidebar.caption("最後更新: 2025-04-01")