Open GSJ7750 opened 1 year ago
import boto3
emr_client = boto3.client('emr', region_name='ap-northeast-2')
step_name = 'competition data ingestion' jar_path = 's3://competition-for-culture-data/jar/competitionPipelines.jar' main_class = 'com.competition.saveNameWithMt20IDJob' step_args = ['--class', main_class, jar_path] # Additional arguments for spark-submit
step = { 'Name': step_name, 'ActionOnFailure': 'CONTINUE', 'HadoopJarStep': { 'Jar': 'command-runner.jar', 'Args': ['spark-submit'] + step_args } }
response = emr_client.add_job_flow_steps( JobFlowId='j-3AC8Z96GV7K7I', Steps=[step] )
if 'ResponseMetadata' in response: print('Step added successfully!') else: print('Error occurred while adding step:', response)
timezone: Asia/Seoul
schedule: cron>: 10 1 *
+ingestion: sh>: python3 /home/ec2-user/add_ingestion_step.py
https://github.com/Calenduck/Calenduck-DATA/issues/5 의 선행 티켓