Adding the ability to supply local JARs to the Glue job.

Author: EC2 Default User

lib/constructs/data-set-enrollment.ts

@@ -10,9 +10,9 @@ import * as lakeformation from 'aws-cdk-lib/aws-lakeformation';
 import * as lambda from 'aws-cdk-lib/aws-lambda';
 import * as cr from 'aws-cdk-lib/custom-resources';
 import * as fs from "fs";
+import { Asset } from 'aws-cdk-lib/aws-s3-assets';
 
 
-import { URL } from "url";
 
 
 export interface FederatedCrawlerTemplateProps extends StackProps {
@@ -172,6 +172,7 @@ export interface DataSetEnrollmentProps extends StackProps {
 		SourceAccessPolicy?: iam.Policy;
 		MaxDPUs: number;
 		WorkflowCronScheduleExpression?: string;
+		LocalJarsForGlueJob?: string[];
 }
 
 
@@ -253,7 +254,7 @@ export class DataSetEnrollment extends Construct {
 			});
 			if(props.SourceConnectionInput.name){
 				connectionArray.push(props.SourceConnectionInput.name);	
-			}
+		}
 		}
 
 
@@ -279,9 +280,38 @@ export class DataSetEnrollment extends Construct {
 		});
 		glueScript.grantRead(this.DataSetGlueRole);
 
+
+		
+	
+		var additonalJars: Array<string> = [];
+		const currentEnrollment = this;
+		if(typeof props.LocalJarsForGlueJob !== "undefined") {
+			
+			props.LocalJarsForGlueJob.forEach(function(localPath: string) {
+				
+				
+				const localName = localPath.replace(/\W/g, '');
+				const jarAsset = new s3assets.Asset(currentEnrollment, `${localName}-GlueScript`, {
+					path: localPath
+				});
+				jarAsset.grantRead(currentEnrollment.DataSetGlueRole);
+				
+				additonalJars.push(jarAsset.s3ObjectUrl);
+			});
+			
+			const additonalJarsParam = additonalJars.join(',');
+		
+			if(typeof props.GlueScriptArguments['--extra-jars'] !== "undefined"){
+				
+				props.GlueScriptArguments['--extra-jars'] = props.GlueScriptArguments['--extra-jars'] + "," + additonalJarsParam
+			}else {
+				props.GlueScriptArguments['--extra-jars'] =  additonalJarsParam
+			}
 
+		}
 
 		/// The spread operator below (...) makes the connections property conditional. Its only used for JDBC sources at the moment.
+
 		const jobParams = {
 			executionProperty: {
 				maxConcurrentRuns: 1
@@ -306,7 +336,6 @@ export class DataSetEnrollment extends Construct {
 		}
 		const etl_job = new glue.CfnJob(this, `${props.dataSetName}-EtlJob`, jobParams );
 
-		
 		const datalake_crawler = this.setupCrawler(this.Dataset_Datalake, this.DataLakeTargets, false, this.Dataset_DatalakeDatabaseName);
 
 		const datalakeEnrollmentWorkflow = new DataLakeEnrollmentWorkflow(this,`${props.dataSetName}DataLakeWorkflow`,{

lib/constructs/s3-data-set-enrollment.ts

@@ -16,6 +16,7 @@ export interface S3dataSetEnrollmentProps extends DataLakeEnrollment.DataLakeEnr
     sourceBucketDataPrefixes: string[];
     MaxDPUs: number;
     ExistingLakeFormationResource?: lakeformation.CfnResource;
+    LocalJarsForGlueJob?: string[];
 }
 
 
@@ -118,7 +119,8 @@ export class S3dataSetEnrollment extends DataLakeEnrollment{
 			    s3Targets: s3DataLakePaths
 			},
 			GlueScriptArguments: props.GlueScriptArguments,
-			WorkflowCronScheduleExpression: props.WorkflowCronScheduleExpression
+			WorkflowCronScheduleExpression: props.WorkflowCronScheduleExpression,
+			LocalJarsForGlueJob: props.LocalJarsForGlueJob
 		});