Implement pipeline Batch job submission Lambdas

[scwatts]

Change summary

• add Lambda function to handle Batch job submission for pipeline runs
• store NF pipeline Batch job definition ARN as SSM parameter for retrieval in Lambdas
• expose NF pipeline Batch job queue ARN in pipeline stacks
• create IAM roles for Lambda functions and grant Batch job submission on NF pipeline Batch queue
• require sample ID and library ID for pipeline execution BASH scripts
• optionally accept a portal ID in pipeline execution BASH scripts
• correct output directory paths

Examples

star-align-nf example (click to show)

```python3 #!/usr/bin/env python3 import json import boto3 def main(): payload = { 'portal_id': '20230530abcdefgh', 'subject_id': 'SBJ00001', 'sample_id': 'PRJ230002', 'library_id': 'L2300002', 'fastq_fwd': 'gds://production/primary_data/230430_A00001_0001_AH1VLHDSX1/20230430qazwsxed/WTS_NebRNA/PRJ230002_L2300002_S1_L001_R1_001.fastq.gz', 'fastq_rev': 'gds://production/primary_data/230430_A00001_0001_AH1VLHDSX1/20230430qazwsxed/WTS_NebRNA/PRJ230002_L2300002_S1_L001_R2_001.fastq.gz', } client_lambda = boto3.client('lambda') lambda_function_name = 'star-align-nf-batch-job-submission' print(f'Invoking {lambda_function_name} with payload: {payload}') response = client_lambda.invoke( FunctionName=lambda_function_name, Payload=json.dumps(payload), ) response_message = response["Payload"].read().decode() response_data = json.loads(response_message) print(f'Received response message: ${response_data}') if __name__ == '__main__': main() ```

oncoanalyser example (click to show)

```python3 #!/usr/bin/env python3 import json import boto3 def main(): portal_id = '20230530abcdefgh' subject_id = 'SBJ00001' tumor_wgs_sample_id = 'PRJ230001' tumor_wgs_library_id = 'L2300001' tumor_wgs_bam = 'gds://production/analysis_data/SBJ00001/wgs_tumor_normal/20230515zyxwvuts/L2300001_L2300003/PRJ230001_tumor.bam' tumor_wts_sample_id = 'PRJ230002' tumor_wts_library_id = 'L2300002' tumor_wts_bam = 's3://org.umccr.data.oncoanalyser/analysis_data/SBJ00001/star-align-nf/20230515qwertyui/L2300002/PRJ230002/PRJ230002.md.bam' normal_wgs_sample_id = 'PRJ230003' normal_wgs_library_id = 'L2300003' normal_wgs_bam = 'gds://production/analysis_data/SBJ00001/wgs_tumor_normal/20230515zyxwvuts/L2300001_L2300002_dragen_somatic/PRJ230003_normal.bam' existing_wgs_dir = 's3://org.umccr.data.oncoanalyser/analysis_data/SBJ00001/oncoanalyser/20230515asdfghjk/wgs/L2300001__L2300003/SBJ00001_PRJ230001/' existing_wts_dir = 's3://org.umccr.data.oncoanalyser/analysis_data/SBJ00001/oncoanalyser/20230515zzxcvbnm/wts/L2300002/SBJ00001_PRJ230002/' payloads = { 'wgs': { 'mode': 'wgs', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wgs_sample_id': tumor_wgs_sample_id, 'tumor_wgs_library_id': tumor_wgs_library_id, 'tumor_wgs_bam': tumor_wgs_bam, 'normal_wgs_sample_id': normal_wgs_sample_id, 'normal_wgs_library_id': normal_wgs_library_id, 'normal_wgs_bam': normal_wgs_bam, }, 'wts': { 'mode': 'wts', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wts_sample_id': tumor_wts_sample_id, 'tumor_wts_library_id': tumor_wts_library_id, 'tumor_wts_bam': tumor_wts_bam, }, 'wgts': { 'mode': 'wgts', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wgs_sample_id': tumor_wgs_sample_id, 'tumor_wgs_library_id': tumor_wgs_library_id, 'tumor_wgs_bam': tumor_wgs_bam, 'normal_wgs_sample_id': normal_wgs_sample_id, 'normal_wgs_library_id': normal_wgs_library_id, 'normal_wgs_bam': normal_wgs_bam, 'tumor_wts_sample_id': tumor_wts_sample_id, 'tumor_wts_library_id': tumor_wts_library_id, 'tumor_wts_bam': tumor_wts_bam, }, 'wgts_existing_wgs': { 'mode': 'wgts_existing_wgs', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wgs_sample_id': tumor_wgs_sample_id, 'tumor_wgs_library_id': tumor_wgs_library_id, 'tumor_wgs_bam': tumor_wgs_bam, 'normal_wgs_sample_id': normal_wgs_sample_id, 'normal_wgs_library_id': normal_wgs_library_id, 'normal_wgs_bam': normal_wgs_bam, 'tumor_wts_sample_id': tumor_wts_sample_id, 'tumor_wts_library_id': tumor_wts_library_id, 'tumor_wts_bam': tumor_wts_bam, 'existing_wgs_dir': existing_wgs_dir, }, 'wgts_existing_wts': { 'mode': 'wgts_existing_wts', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wgs_sample_id': tumor_wgs_sample_id, 'tumor_wgs_library_id': tumor_wgs_library_id, 'tumor_wgs_bam': tumor_wgs_bam, 'normal_wgs_sample_id': normal_wgs_sample_id, 'normal_wgs_library_id': normal_wgs_library_id, 'normal_wgs_bam': normal_wgs_bam, 'tumor_wts_sample_id': tumor_wts_sample_id, 'tumor_wts_library_id': tumor_wts_library_id, 'tumor_wts_bam': tumor_wts_bam, 'existing_wts_dir': existing_wts_dir, }, 'wgts_existing_both': { 'mode': 'wgts_existing_both', 'portal_id': portal_id, 'subject_id': subject_id, 'tumor_wgs_sample_id': tumor_wgs_sample_id, 'tumor_wgs_library_id': tumor_wgs_library_id, 'tumor_wgs_bam': tumor_wgs_bam, 'normal_wgs_sample_id': normal_wgs_sample_id, 'normal_wgs_library_id': normal_wgs_library_id, 'normal_wgs_bam': normal_wgs_bam, 'tumor_wts_sample_id': tumor_wts_sample_id, 'tumor_wts_library_id': tumor_wts_library_id, 'tumor_wts_bam': tumor_wts_bam, 'existing_wgs_dir': existing_wgs_dir, 'existing_wts_dir': existing_wts_dir, }, } client_lambda = boto3.client('lambda') lambda_function_name = 'oncoanalyser-batch-job-submission' for payload_name, payload_data in payloads.items(): print(f'Invoking {lambda_function_name} for {payload_name} with payload: {payload_data}') response = client_lambda.invoke( FunctionName=lambda_function_name, Payload=json.dumps(payload_data), ) response_message = response["Payload"].read().decode() response_data = json.loads(response_message) print(f'Received response message: ${response_data}') if __name__ == '__main__': main() ```

[victorskl]

Portal Release - Ocicat: Implement Portal Launch Pad trigger for Oncoanalyser