| """Trigger Parquet Snapshot Glue job""" | |
| import os | |
| import logging | |
| import boto3 | |
| AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID'] | |
| AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] | |
| logging.basicConfig( | |
| format='%(asctime)s - %(levelname)s - %(funcName)s - %(message)s', | |
| datefmt="%Y-%m-%d %H:%M:%S", | |
| level=logging.INFO | |
| ) | |
| def get_client_connection(): | |
| """ | |
| Returns a client connection to the AWS Glue service. | |
| :return: AWS Glue client connection | |
| """ | |
| return boto3.client( | |
| service_name='glue', | |
| region_name='us-east-1', | |
| aws_access_key_id=AWS_ACCESS_KEY_ID, | |
| aws_secret_access_key=AWS_SECRET_ACCESS_KEY | |
| ) | |
| def glue_job_run(): | |
| """ | |
| Triggers the Glue job run for the Parquet Snapshot. | |
| :return: None | |
| """ | |
| glue = get_client_connection() | |
| response = glue.start_job_run( | |
| JobName='Article Snapshot China' | |
| ) | |
| logging.info(response) | |
| response = glue.start_job_run( | |
| JobName='Reference China' | |
| ) | |
| logging.info(response) | |
| if __name__ == "__main__": | |
| glue_job_run() | |