update flow

daily.py

@@ -526,9 +526,9 @@ def daily():
                             print(error)
 
 @flow(name = "Data Collection China - Daily", log_prints=True)
-def main():
+def data_collection():
     daily()
     glue_job_run()
 
 if __name__ == '__main__':
-    main()
+    data_collection()

glue.py

@@ -2,14 +2,14 @@
 import os
 import boto3
 from prefect import task
-from dotenv import load_dotenv
+from prefect.blocks.system import Secret
 
-load_dotenv()
-AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
 
-# AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
-# AWS_SECRET_ACCESS_KEY = "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"
+# AWS_ACCESS_KEY_ID = Secret.load("aws-access-key-id").get()
+# AWS_SECRET_ACCESS_KEY = Secret.load("aws-access-secret-key").get()
+
+AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
+AWS_SECRET_ACCESS_KEY = "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"
 
 def get_client_connection():
     """

utils.py

@@ -13,18 +13,17 @@ import pandas as pd
 import requests
 import boto3
 from lxml import etree
-from dotenv import load_dotenv
 from googletrans import Translator
 from transformers import pipeline
 from PyPDF2 import PdfReader
 from langdetect import detect
+from prefect.blocks.system import Secret
 
-load_dotenv()
-AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
-AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
+# AWS_ACCESS_KEY_ID = Secret.load("aws-access-key-id").get()
+# AWS_SECRET_ACCESS_KEY = Secret.load("aws-access-secret-key").get()
 
-# AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
-# AWS_SECRET_ACCESS_KEY = "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"
+AWS_ACCESS_KEY_ID = "AKIAQFXZMGHQYXKWUDWR"
+AWS_SECRET_ACCESS_KEY = "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"
 
 analyzer = pipeline("sentiment-analysis", model="ProsusAI/finbert")