diff --git a/forward_engineering/configs/templates.js b/forward_engineering/configs/templates.js index c6ed14a1..3a56d712 100644 --- a/forward_engineering/configs/templates.js +++ b/forward_engineering/configs/templates.js @@ -2,7 +2,7 @@ module.exports = { createDatabase: '', createSchema: 'CREATE SCHEMA${ifNotExist} "${name}"${authorization}${quota};\n', createExternalSchema: - 'CREATE EXTERNAL SCHEMA${ifNotExist} "${name}" FROM ${source} DATABASE "${sourceDBName}"${sourceSchemaName}${region}${uri} IAM_ROLE "${iamRole}"${secretARN}${catalogRole}${createExternalDatabase};\n', + "CREATE EXTERNAL SCHEMA${ifNotExist} \"${name}\" FROM ${source}\nDATABASE '${sourceDBName}'${sourceSchemaName}${region}${uri}\nIAM_ROLE '${iamRole}'${secretARN}${catalogRole}${createExternalDatabase};\n", createTable: 'CREATE ${temporary}TABLE' + @@ -14,6 +14,12 @@ module.exports = { createTableAs: 'CREATE ${temporary}TABLE "${schemaName}"."${name}" ${backup}${tableAttribute} AS ${query};\n${comment}${columnDescriptions}', + createExternalTable: + 'CREATE EXTERNAL TABLE "${schemaName}"."${name}" (${columnDefinitions})${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties};${comment}${columnDescriptions}', + + createExternalTableAs: + 'CREATE EXTERNAL TABLE "${schemaName}"."${name}"${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties} AS\n${query};${comment}${columnDescriptions}', + createView: 'CREATE${orReplace} VIEW "${schemaName}"."${name}"(\n' + '\t${column_list}\n' + diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 11ee45ce..6414088f 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -21,6 +21,10 @@ module.exports = (baseProvider, options, app) => { setOrReplace, getCompositeName, toString, + parseTextArea, + parseProps, + getRowFormat, + getStoredAs, } = require('./helpers/general')(app); const { decorateType, @@ -34,6 +38,49 @@ module.exports = (baseProvider, options, app) => { } = require('./helpers/columnDefinitionHelper')(app); const { generateConstraint } = require('./helpers/constraintHelper')(app); + const buildExternalTable = ({ tableData, schemaName, asSelect, isActivated, comment, columnDescriptions }) => { + const partitionedBy = tableData.partitionedBy + ? `\nPARTITIONED BY (${parseTextArea(tableData.partitionedBy)})` + : ''; + + const rowFormat = getRowFormat(tableData); + const storedAs = getStoredAs(tableData); + + const location = tableData.externalLocation ? `\nLOCATION ${toString(tableData.externalLocation)}` : ''; + const tblProperties = tableData.tableProperties + ? `\nTABLE PROPERTIES (${parseProps(tableData.tableProperties)})` + : ''; + const columnDefinitions = getColumnsDefinitions(tableData.columns, isActivated); + + if (asSelect) { + return assignTemplates(templates.createExternalTableAs, { + name: tableData.name, + schemaName, + partitionedBy, + rowFormat, + storedAs, + location, + tableProperties: tblProperties, + query: asSelect, + comment: tableData.comment ? comment : '', + columnDescriptions, + }); + } + + return assignTemplates(templates.createExternalTable, { + name: tableData.name, + schemaName, + columnDefinitions: columnDefinitions === '' ? '' : '\n\t' + columnDefinitions, + partitionedBy, + rowFormat, + storedAs, + location, + tableProperties: tblProperties, + comment: tableData.comment ? comment : '', + columnDescriptions, + }); + }; + return { createSchema({ name, @@ -107,6 +154,17 @@ module.exports = (baseProvider, options, app) => { tableData.columnDefinitions, ); + if (tableData.externalTable) { + return buildExternalTable({ + tableData, + schemaName, + asSelect, + isActivated, + comment, + columnDescriptions, + }); + } + if (asSelect) { return assignTemplates(templates.createTableAs, { name: tableData.name, @@ -274,13 +332,16 @@ module.exports = (baseProvider, options, app) => { iamRole: containerData.IAM_ROLE, secretARN: getARN(containerData.SECRET_ARN, containerData.fromSource), catalogRole: - containerData.CATALOG_ROLE && containerData.source === 'Data catalog' - ? ` CATALOG_ROLE ${containerData.CATALOG_ROLE}` + containerData.CATALOG_ROLE && containerData.fromSource === 'Data catalog' + ? `\nCATALOG_ROLE '${containerData.CATALOG_ROLE}'` : '', uri: getUri(containerData.URI, containerData.port, containerData.fromSource), - region: containerData.source === 'Data catalog' ? containerData.region : '', + region: + containerData.fromSource === 'Data catalog' && containerData.region + ? `\nREGION '${containerData.region}'` + : '', createExternalDatabase: containerData.createExternalDatabaseIfNotExists - ? ' CREATE EXTERNAL DATABASE IF NOT EXISTS' + ? '\nCREATE EXTERNAL DATABASE IF NOT EXISTS' : '', functions: Array.isArray(udfs) ? udfs.map(hydrateUdf(containerData.name)).filter(filterUdf) : [], procedures: Array.isArray(procedures) @@ -338,6 +399,17 @@ module.exports = (baseProvider, options, app) => { ? '' : generateConstraint(sortKey, templates.compoundSortKey, jsonSchema.isActivated, { sortStyle }), query: '', + externalTable: firstTab.externalTable, + partitionedBy: firstTab.partitionedBy, + rowFormatType: firstTab.rowFormatType, + rowFormatDelimited: firstTab.rowFormatDelimited, + rowFormatSerde: firstTab.rowFormatSerde, + serdeProperties: firstTab.serdeProperties, + storedAs: firstTab.storedAs, + inputFormatClass: firstTab.inputFormatClass, + outputFormatClass: firstTab.outputFormatClass, + externalLocation: firstTab.externalLocation, + tableProperties: firstTab.tableProperties, }; }, diff --git a/forward_engineering/helpers/constants.js b/forward_engineering/helpers/constants.js index 8ac42621..cf525f47 100644 --- a/forward_engineering/helpers/constants.js +++ b/forward_engineering/helpers/constants.js @@ -1,5 +1,16 @@ const DROP_STATEMENTS = ['DROP SCHEMA', 'DROP TABLE', 'DROP FUNCTION', 'DROP PROCEDURE', 'DROP COLUMN', 'DROP VIEW']; +const ROW_FORMAT_TYPES = Object.freeze({ + DELIMITED: 'DELIMITED', + SERDE: 'SERDE', +}); + +const STORED_AS_TYPES = Object.freeze({ + INPUT_OUTPUT_FORMAT: 'INPUTFORMAT / OUTPUTFORMAT', +}); + module.exports = { DROP_STATEMENTS, + ROW_FORMAT_TYPES, + STORED_AS_TYPES, }; diff --git a/forward_engineering/helpers/general.js b/forward_engineering/helpers/general.js index 86225e4a..f8d43c60 100644 --- a/forward_engineering/helpers/general.js +++ b/forward_engineering/helpers/general.js @@ -1,5 +1,6 @@ const _ = require('lodash'); const { commentIfDeactivated } = require('./commentDeactivatedHelper'); +const { ROW_FORMAT_TYPES, STORED_AS_TYPES } = require('./constants'); module.exports = app => { const { checkAllKeysActivated, clean, tab } = app.require('@hackolade/ddl-fe-utils').general; @@ -178,6 +179,64 @@ module.exports = app => { } }; + const parseTextArea = text => + text + ? text + .split('\n') + .map(l => l.trim()) + .filter(Boolean) + .join(', ') + : ''; + + const parseProps = text => { + if (!text) return ''; + + return text + .split('\n') + .map(line => line.trim()) + .filter(line => line.includes('=')) + .map(line => { + const [propertyKey, ...valueParts] = line.split('='); + const propertyValue = valueParts.join('=').trim(); + + return `'${escape(propertyKey.trim())}'='${escape(propertyValue)}'`; + }) + .join(', '); + }; + + const getRowFormat = tableData => { + if (tableData.rowFormatType === ROW_FORMAT_TYPES.DELIMITED && tableData.rowFormatDelimited) { + return `\nROW FORMAT DELIMITED ${tableData.rowFormatDelimited}`; + } + + if (tableData.rowFormatType === ROW_FORMAT_TYPES.SERDE && tableData.rowFormatSerde) { + let format = `\nROW FORMAT SERDE ${toString(tableData.rowFormatSerde)}`; + const serdeProps = parseProps(tableData.serdeProperties); + + if (serdeProps) { + format += `\nWITH SERDEPROPERTIES (${serdeProps})`; + } + return format; + } + + return ''; + }; + + const getStoredAs = tableData => { + if (tableData.storedAs === STORED_AS_TYPES.INPUT_OUTPUT_FORMAT) { + if (tableData.inputFormatClass && tableData.outputFormatClass) { + return `\nSTORED AS INPUTFORMAT ${toString(tableData.inputFormatClass)}\nOUTPUTFORMAT ${toString(tableData.outputFormatClass)}`; + } + return ''; + } + + if (tableData.storedAs) { + return `\nSTORED AS ${tableData.storedAs}`; + } + + return ''; + }; + return { toString, toNumber, @@ -195,5 +254,9 @@ module.exports = app => { filterProcedure, setOrReplace, getCompositeName, + parseTextArea, + parseProps, + getRowFormat, + getStoredAs, }; }; diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 6ccadd28..186eafa1 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -124,6 +124,135 @@ making sure that you maintain a proper JSON format. "propertyType": "details", "template": "textarea" }, + { + "propertyName": "External", + "propertyKeyword": "externalTable", + "propertyType": "checkbox", + "propertyTooltip": "Specifies that the table is an external table.", + "dependency": { + "type": "not", + "values": { + "key": "temporary", + "value": true + } + } + }, + { + "propertyName": "Partitioned by", + "propertyKeyword": "partitionedBy", + "propertyTooltip": "Specify partition columns as col_name data_type, e.g.:feed_name string", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "valueType": "string", + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Row format type", + "propertyKeyword": "rowFormatType", + "propertyTooltip": "Choose between DELIMITED or SERDE", + "propertyType": "select", + "options": ["", "DELIMITED", "SERDE"], + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Row format delimited", + "propertyKeyword": "rowFormatDelimited", + "propertyType": "text", + "propertyTooltip": "Specify delimiters, e.g., FIELDS TERMINATED BY, LINES TERMINATED BY", + "dependency": { + "key": "rowFormatType", + "value": "DELIMITED" + } + }, + { + "propertyName": "Row format SerDe", + "propertyKeyword": "rowFormatSerde", + "propertyType": "text", + "dependency": { + "key": "rowFormatType", + "value": "SERDE" + } + }, + { + "propertyName": "SerDe properties", + "propertyKeyword": "serdeProperties", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "propertyTooltip": "Add SerDe properties as key=value pairs", + "dependency": { + "key": "rowFormatType", + "value": "SERDE" + } + }, + { + "propertyName": "Stored as", + "propertyKeyword": "storedAs", + "propertyTooltip": "The file format for data files.", + "propertyType": "select", + "options": [ + "", + "PARQUET", + "RCFILE", + "SEQUENCEFILE", + "TEXTFILE", + "ORC", + "AVRO", + "INPUTFORMAT / OUTPUTFORMAT" + ], + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Input Format Class", + "propertyKeyword": "inputFormatClass", + "propertyType": "text", + "dependency": { + "key": "storedAs", + "value": "INPUTFORMAT / OUTPUTFORMAT" + } + }, + { + "propertyName": "Output Format Class", + "propertyKeyword": "outputFormatClass", + "propertyType": "text", + "dependency": { + "key": "storedAs", + "value": "INPUTFORMAT / OUTPUTFORMAT" + } + }, + { + "propertyName": "Location", + "propertyKeyword": "externalLocation", + "propertyTooltip": "The path to the Amazon S3 bucket/folder or a manifest file, e.g., s3://my-bucket/folder/ or s3://my-bucket/manifest.json", + "propertyType": "text", + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Table properties", + "propertyKeyword": "tableProperties", + "propertyTooltip": "Add table properties as key=value pairs", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "valueType": "string", + "dependency": { + "key": "externalTable", + "value": true + } + }, { "propertyName": "Table role", "propertyKeyword": "tableRole", @@ -152,26 +281,54 @@ making sure that you maintain a proper JSON format. "propertyName": "Temporary", "propertyKeyword": "temporary", "propertyType": "checkbox", - "propertyTooltip": "Creates a temporary table that is visible only within the current session. The table is automatically dropped at the end of the session in which it is created." + "propertyTooltip": "Creates a temporary table that is visible only within the current session. The table is automatically dropped at the end of the session in which it is created.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "If not exists", "propertyKeyword": "ifNotExists", "propertyType": "checkbox", - "propertyTooltip": "if the specified table already exists, the command should make no changes and return a message that the table exists, rather than terminating with an error. " + "propertyTooltip": "if the specified table already exists, the command should make no changes and return a message that the table exists, rather than terminating with an error. ", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Backup", "propertyKeyword": "BACKUP", "propertyType": "checkbox", - "propertyTooltip": "Specifies whether the table should be included in automated and manual cluster snapshots." + "propertyTooltip": "Specifies whether the table should be included in automated and manual cluster snapshots.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Distribution style", "propertyKeyword": "DISTSTYLE", "propertyTooltip": "keyword that defines the data distribution style for the whole table. AUTO: Amazon Redshift assigns an optimal distribution style based on the table data. EVEN: The data in the table is spread evenly across the nodes in a cluster in a round-robin distribution. KEY: The data is distributed by the values in the distKey column. ALL: A copy of the entire table is distributed to every node. ", "propertyType": "select", - "options": ["Auto", "Even", "Key", "All"] + "options": ["Auto", "Even", "Key", "All"], + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Dist key", @@ -193,13 +350,27 @@ making sure that you maintain a proper JSON format. "maxFields": 1 } } - ] + ], + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Auto sort key", "propertyKeyword": "autoSortKey", "propertyType": "checkbox", - "propertyTooltip": "If you don't specify any sort keys options, the default is AUTO." + "propertyTooltip": "If you don't specify any sort keys options, the default is AUTO.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Sort key", @@ -221,11 +392,23 @@ making sure that you maintain a proper JSON format. } ], "dependency": { - "type": "not", - "values": { - "key": "autoSortKey", - "value": true - } + "type": "and", + "values": [ + { + "type": "not", + "values": { + "key": "autoSortKey", + "value": true + } + }, + { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } + ] } }, { @@ -235,11 +418,23 @@ making sure that you maintain a proper JSON format. "propertyType": "select", "options": ["Compound", "Interleaved"], "dependency": { - "type": "not", - "values": { - "key": "autoSortKey", - "value": true - } + "type": "and", + "values": [ + { + "type": "not", + "values": { + "key": "autoSortKey", + "value": true + } + }, + { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } + ] } }, { @@ -249,7 +444,14 @@ making sure that you maintain a proper JSON format. "propertyType": "selecthashed", "template": "entities", "withEmptyOption": true, - "excludeCurrent": true + "excludeCurrent": true, + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Include defaults",