HCK-16111: feature branch to dev - external table

forward_engineering/configs/templates.js

@@ -2,7 +2,7 @@ module.exports = {
 	createDatabase: '',
 	createSchema: 'CREATE SCHEMA${ifNotExist} "${name}"${authorization}${quota};\n',
 	createExternalSchema:
-		'CREATE EXTERNAL SCHEMA${ifNotExist} "${name}" FROM ${source} DATABASE "${sourceDBName}"${sourceSchemaName}${region}${uri} IAM_ROLE "${iamRole}"${secretARN}${catalogRole}${createExternalDatabase};\n',
+		"CREATE EXTERNAL SCHEMA${ifNotExist} \"${name}\" FROM ${source}\nDATABASE '${sourceDBName}'${sourceSchemaName}${region}${uri}\nIAM_ROLE '${iamRole}'${secretARN}${catalogRole}${createExternalDatabase};\n",
 
 	createTable:
 		'CREATE ${temporary}TABLE' +
@@ -14,6 +14,12 @@ module.exports = {
 	createTableAs:
 		'CREATE ${temporary}TABLE "${schemaName}"."${name}" ${backup}${tableAttribute} AS ${query};\n${comment}${columnDescriptions}',
 
+	createExternalTable:
+		'CREATE EXTERNAL TABLE "${schemaName}"."${name}" (${columnDefinitions})${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties};${comment}${columnDescriptions}',
+
+	createExternalTableAs:
+		'CREATE EXTERNAL TABLE "${schemaName}"."${name}"${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties} AS\n${query};${comment}${columnDescriptions}',
+
 	createView:
 		'CREATE${orReplace} VIEW "${schemaName}"."${name}"(\n' +
 		'\t${column_list}\n' +

forward_engineering/ddlProvider.js

@@ -21,6 +21,10 @@ module.exports = (baseProvider, options, app) => {
 		setOrReplace,
 		getCompositeName,
 		toString,
+		parseTextArea,
+		parseProps,
+		getRowFormat,
+		getStoredAs,
 	} = require('./helpers/general')(app);
 	const {
 		decorateType,
@@ -34,6 +38,49 @@ module.exports = (baseProvider, options, app) => {
 	} = require('./helpers/columnDefinitionHelper')(app);
 	const { generateConstraint } = require('./helpers/constraintHelper')(app);
 
+	const buildExternalTable = ({ tableData, schemaName, asSelect, isActivated, comment, columnDescriptions }) => {
+		const partitionedBy = tableData.partitionedBy
+			? `\nPARTITIONED BY (${parseTextArea(tableData.partitionedBy)})`
+			: '';
+
+		const rowFormat = getRowFormat(tableData);
+		const storedAs = getStoredAs(tableData);
+
+		const location = tableData.externalLocation ? `\nLOCATION ${toString(tableData.externalLocation)}` : '';
+		const tblProperties = tableData.tableProperties
+			? `\nTABLE PROPERTIES (${parseProps(tableData.tableProperties)})`
+			: '';
+		const columnDefinitions = getColumnsDefinitions(tableData.columns, isActivated);
+
+		if (asSelect) {
+			return assignTemplates(templates.createExternalTableAs, {
+				name: tableData.name,
+				schemaName,
+				partitionedBy,
+				rowFormat,
+				storedAs,
+				location,
+				tableProperties: tblProperties,
+				query: asSelect,
+				comment: tableData.comment ? comment : '',
+				columnDescriptions,
+			});
+		}
+
+		return assignTemplates(templates.createExternalTable, {
+			name: tableData.name,
+			schemaName,
+			columnDefinitions: columnDefinitions === '' ? '' : '\n\t' + columnDefinitions,
+			partitionedBy,
+			rowFormat,
+			storedAs,
+			location,
+			tableProperties: tblProperties,
+			comment: tableData.comment ? comment : '',
+			columnDescriptions,
+		});
+	};
+
 	return {
 		createSchema({
 			name,
@@ -107,6 +154,17 @@ module.exports = (baseProvider, options, app) => {
 				tableData.columnDefinitions,
 			);
 
+			if (tableData.externalTable) {
+				return buildExternalTable({
+					tableData,
+					schemaName,
+					asSelect,
+					isActivated,
+					comment,
+					columnDescriptions,
+				});
+			}
+
 			if (asSelect) {
 				return assignTemplates(templates.createTableAs, {
 					name: tableData.name,
@@ -274,13 +332,16 @@ module.exports = (baseProvider, options, app) => {
 				iamRole: containerData.IAM_ROLE,
 				secretARN: getARN(containerData.SECRET_ARN, containerData.fromSource),
 				catalogRole:
-					containerData.CATALOG_ROLE && containerData.source === 'Data catalog'
-						? ` CATALOG_ROLE ${containerData.CATALOG_ROLE}`
+					containerData.CATALOG_ROLE && containerData.fromSource === 'Data catalog'
+						? `\nCATALOG_ROLE '${containerData.CATALOG_ROLE}'`
 						: '',
 				uri: getUri(containerData.URI, containerData.port, containerData.fromSource),
-				region: containerData.source === 'Data catalog' ? containerData.region : '',
+				region:
+					containerData.fromSource === 'Data catalog' && containerData.region
+						? `\nREGION '${containerData.region}'`
+						: '',
 				createExternalDatabase: containerData.createExternalDatabaseIfNotExists
-					? ' CREATE EXTERNAL DATABASE IF NOT EXISTS'
+					? '\nCREATE EXTERNAL DATABASE IF NOT EXISTS'
 					: '',
 				functions: Array.isArray(udfs) ? udfs.map(hydrateUdf(containerData.name)).filter(filterUdf) : [],
 				procedures: Array.isArray(procedures)
@@ -338,6 +399,17 @@ module.exports = (baseProvider, options, app) => {
 						? ''
 						: generateConstraint(sortKey, templates.compoundSortKey, jsonSchema.isActivated, { sortStyle }),
 				query: '',
+				externalTable: firstTab.externalTable,
+				partitionedBy: firstTab.partitionedBy,
+				rowFormatType: firstTab.rowFormatType,
+				rowFormatDelimited: firstTab.rowFormatDelimited,
+				rowFormatSerde: firstTab.rowFormatSerde,
+				serdeProperties: firstTab.serdeProperties,
+				storedAs: firstTab.storedAs,
+				inputFormatClass: firstTab.inputFormatClass,
+				outputFormatClass: firstTab.outputFormatClass,
+				externalLocation: firstTab.externalLocation,
+				tableProperties: firstTab.tableProperties,
 			};
 		},
 

forward_engineering/helpers/constants.js

@@ -1,5 +1,16 @@
 const DROP_STATEMENTS = ['DROP SCHEMA', 'DROP TABLE', 'DROP FUNCTION', 'DROP PROCEDURE', 'DROP COLUMN', 'DROP VIEW'];
 
+const ROW_FORMAT_TYPES = Object.freeze({
+	DELIMITED: 'DELIMITED',
+	SERDE: 'SERDE',
+});
+
+const STORED_AS_TYPES = Object.freeze({
+	INPUT_OUTPUT_FORMAT: 'INPUTFORMAT / OUTPUTFORMAT',
+});
+
 module.exports = {
 	DROP_STATEMENTS,
+	ROW_FORMAT_TYPES,
+	STORED_AS_TYPES,
 };

forward_engineering/helpers/general.js

@@ -1,5 +1,6 @@
 const _ = require('lodash');
 const { commentIfDeactivated } = require('./commentDeactivatedHelper');
+const { ROW_FORMAT_TYPES, STORED_AS_TYPES } = require('./constants');
 
 module.exports = app => {
 	const { checkAllKeysActivated, clean, tab } = app.require('@hackolade/ddl-fe-utils').general;
@@ -178,6 +179,64 @@ module.exports = app => {
 		}
 	};
 
+	const parseTextArea = text =>
+		text
+			? text
+					.split('\n')
+					.map(l => l.trim())
+					.filter(Boolean)
+					.join(', ')
+			: '';
+
+	const parseProps = text => {
+		if (!text) return '';
+
+		return text
+			.split('\n')
+			.map(line => line.trim())
+			.filter(line => line.includes('='))
+			.map(line => {
+				const [propertyKey, ...valueParts] = line.split('=');
+				const propertyValue = valueParts.join('=').trim();
+
+				return `'${escape(propertyKey.trim())}'='${escape(propertyValue)}'`;
+			})
+			.join(', ');
+	};
+
+	const getRowFormat = tableData => {
+		if (tableData.rowFormatType === ROW_FORMAT_TYPES.DELIMITED && tableData.rowFormatDelimited) {
+			return `\nROW FORMAT DELIMITED ${tableData.rowFormatDelimited}`;
+		}
+
+		if (tableData.rowFormatType === ROW_FORMAT_TYPES.SERDE && tableData.rowFormatSerde) {
+			let format = `\nROW FORMAT SERDE ${toString(tableData.rowFormatSerde)}`;
+			const serdeProps = parseProps(tableData.serdeProperties);
+
+			if (serdeProps) {
+				format += `\nWITH SERDEPROPERTIES (${serdeProps})`;
+			}
+			return format;
+		}
+
+		return '';
+	};
+
+	const getStoredAs = tableData => {
+		if (tableData.storedAs === STORED_AS_TYPES.INPUT_OUTPUT_FORMAT) {
+			if (tableData.inputFormatClass && tableData.outputFormatClass) {
+				return `\nSTORED AS INPUTFORMAT ${toString(tableData.inputFormatClass)}\nOUTPUTFORMAT ${toString(tableData.outputFormatClass)}`;
+			}
+			return '';
+		}
+
+		if (tableData.storedAs) {
+			return `\nSTORED AS ${tableData.storedAs}`;
+		}
+
+		return '';
+	};
+
 	return {
 		toString,
 		toNumber,
@@ -195,5 +254,9 @@ module.exports = app => {
 		filterProcedure,
 		setOrReplace,
 		getCompositeName,
+		parseTextArea,
+		parseProps,
+		getRowFormat,
+		getStoredAs,
 	};
 };