commit

.github/workflows/china gov.yml

@@ -0,0 +1,46 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Data Collection - China GOV
+
+on:
+  # schedule:
+  #   - cron: '0 16 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 14400
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install transformers
+        pip install tensorflow
+        pip install tf-keras
+    - name: Data Collection
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      run: |
+        python cbirc.py
+        python chinatax.py
+        python csrc.py
+        python mof.py
+        python gov.py
+        python mofcom.py
+        python ndrc.py
+        python pbc.py
+        python safe.py
+        python stats.py
+        python eastmoney.py

utils.py

@@ -9,10 +9,10 @@ from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
 
-# AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-# AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
-AWS_ACCESS_KEY_ID="AKIAQFXZMGHQYXKWUDWR"
-AWS_SECRET_ACCESS_KEY="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"
+AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
+AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+# AWS_ACCESS_KEY_ID="AKIAQFXZMGHQYXKWUDWR"
+# AWS_SECRET_ACCESS_KEY="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"
 
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")
 
@@ -122,7 +122,7 @@ def sentiment_computation(content):
 def upsert_content(report):
     """Upsert the content records"""
     dynamodb = get_db_connection()
-    table = dynamodb.Table('article_china')
+    table = dynamodb.Table('article_test')
         # Define the item data
     item = {
         'id': str(report['id']),