From 34f4f724417641f5bea18e4f404ef632e50cd1cf Mon Sep 17 00:00:00 2001 From: Linsted Date: Thu, 21 May 2026 12:44:11 +0200 Subject: [PATCH 1/2] HCK-16175: add support of EXTERNAL tables for FE --- forward_engineering/configs/templates.js | 6 + forward_engineering/ddlProvider.js | 83 ++++++ forward_engineering/helpers/constants.js | 11 + forward_engineering/helpers/general.js | 27 ++ .../entity_level/entityLevelConfig.json | 236 ++++++++++++++++-- 5 files changed, 346 insertions(+), 17 deletions(-) diff --git a/forward_engineering/configs/templates.js b/forward_engineering/configs/templates.js index c6ed14a1..681cd1e1 100644 --- a/forward_engineering/configs/templates.js +++ b/forward_engineering/configs/templates.js @@ -14,6 +14,12 @@ module.exports = { createTableAs: 'CREATE ${temporary}TABLE "${schemaName}"."${name}" ${backup}${tableAttribute} AS ${query};\n${comment}${columnDescriptions}', + createExternalTable: + 'CREATE EXTERNAL TABLE "${schemaName}"."${name}" (${columnDefinitions})${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties};${comment}${columnDescriptions}', + + createExternalTableAs: + 'CREATE EXTERNAL TABLE "${schemaName}"."${name}"${partitionedBy}${rowFormat}${storedAs}${location}${tableProperties} AS\n${query};${comment}${columnDescriptions}', + createView: 'CREATE${orReplace} VIEW "${schemaName}"."${name}"(\n' + '\t${column_list}\n' + diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 11ee45ce..6ab46df3 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -5,6 +5,7 @@ const templates = require('./configs/templates'); const types = require('./configs/types'); const { commentIfDeactivated } = require('./helpers/commentDeactivatedHelper'); const { getTableAttributes, getTableConstraints, getTableLikeConstraint } = require('./helpers/tableHelper'); +const { ROW_FORMAT_TYPES, STORED_AS_TYPES } = require('./helpers/constants'); module.exports = (baseProvider, options, app) => { const { hasType } = app.require('@hackolade/ddl-fe-utils').general; @@ -21,6 +22,8 @@ module.exports = (baseProvider, options, app) => { setOrReplace, getCompositeName, toString, + parseTextArea, + parseProps, } = require('./helpers/general')(app); const { decorateType, @@ -34,6 +37,64 @@ module.exports = (baseProvider, options, app) => { } = require('./helpers/columnDefinitionHelper')(app); const { generateConstraint } = require('./helpers/constraintHelper')(app); + const buildExternalTable = ({ tableData, schemaName, asSelect, isActivated, comment, columnDescriptions }) => { + const partitionedBy = tableData.partitionedBy + ? `\nPARTITIONED BY (${parseTextArea(tableData.partitionedBy)})` + : ''; + + let rowFormat = ''; + if (tableData.rowFormatType === ROW_FORMAT_TYPES.DELIMITED && tableData.rowFormatDelimited) { + rowFormat = `\nROW FORMAT DELIMITED ${tableData.rowFormatDelimited}`; + } else if (tableData.rowFormatType === ROW_FORMAT_TYPES.SERDE && tableData.rowFormatSerde) { + rowFormat = `\nROW FORMAT SERDE ${toString(tableData.rowFormatSerde)}`; + const serdeProps = parseProps(tableData.serdeProperties); + if (serdeProps) rowFormat += `\nWITH SERDEPROPERTIES (${serdeProps})`; + } + + let storedAs = ''; + if (tableData.storedAs === STORED_AS_TYPES.INPUT_OUTPUT_FORMAT) { + if (tableData.inputFormatClass && tableData.outputFormatClass) { + storedAs = `\nSTORED AS INPUTFORMAT ${toString(tableData.inputFormatClass)}\nOUTPUTFORMAT ${toString(tableData.outputFormatClass)}`; + } + } else if (tableData.storedAs) { + storedAs = `\nSTORED AS ${tableData.storedAs}`; + } + + const location = tableData.externalLocation ? `\nLOCATION ${toString(tableData.externalLocation)}` : ''; + const tblProperties = tableData.tableProperties + ? `\nTABLE PROPERTIES (${parseProps(tableData.tableProperties)})` + : ''; + const columnDefinitions = getColumnsDefinitions(tableData.columns, isActivated); + + if (asSelect) { + return assignTemplates(templates.createExternalTableAs, { + name: tableData.name, + schemaName, + partitionedBy, + rowFormat, + storedAs, + location, + tableProperties: tblProperties, + query: asSelect, + comment: tableData.comment ? comment : '', + columnDescriptions, + }); + } + + return assignTemplates(templates.createExternalTable, { + name: tableData.name, + schemaName, + columnDefinitions: columnDefinitions === '' ? '' : '\n\t' + columnDefinitions, + partitionedBy, + rowFormat, + storedAs, + location, + tableProperties: tblProperties, + comment: tableData.comment ? comment : '', + columnDescriptions, + }); + }; + return { createSchema({ name, @@ -107,6 +168,17 @@ module.exports = (baseProvider, options, app) => { tableData.columnDefinitions, ); + if (tableData.externalTable) { + return buildExternalTable({ + tableData, + schemaName, + asSelect, + isActivated, + comment, + columnDescriptions, + }); + } + if (asSelect) { return assignTemplates(templates.createTableAs, { name: tableData.name, @@ -338,6 +410,17 @@ module.exports = (baseProvider, options, app) => { ? '' : generateConstraint(sortKey, templates.compoundSortKey, jsonSchema.isActivated, { sortStyle }), query: '', + externalTable: firstTab.externalTable, + partitionedBy: firstTab.partitionedBy, + rowFormatType: firstTab.rowFormatType, + rowFormatDelimited: firstTab.rowFormatDelimited, + rowFormatSerde: firstTab.rowFormatSerde, + serdeProperties: firstTab.serdeProperties, + storedAs: firstTab.storedAs, + inputFormatClass: firstTab.inputFormatClass, + outputFormatClass: firstTab.outputFormatClass, + externalLocation: firstTab.externalLocation, + tableProperties: firstTab.tableProperties, }; }, diff --git a/forward_engineering/helpers/constants.js b/forward_engineering/helpers/constants.js index 8ac42621..cf525f47 100644 --- a/forward_engineering/helpers/constants.js +++ b/forward_engineering/helpers/constants.js @@ -1,5 +1,16 @@ const DROP_STATEMENTS = ['DROP SCHEMA', 'DROP TABLE', 'DROP FUNCTION', 'DROP PROCEDURE', 'DROP COLUMN', 'DROP VIEW']; +const ROW_FORMAT_TYPES = Object.freeze({ + DELIMITED: 'DELIMITED', + SERDE: 'SERDE', +}); + +const STORED_AS_TYPES = Object.freeze({ + INPUT_OUTPUT_FORMAT: 'INPUTFORMAT / OUTPUTFORMAT', +}); + module.exports = { DROP_STATEMENTS, + ROW_FORMAT_TYPES, + STORED_AS_TYPES, }; diff --git a/forward_engineering/helpers/general.js b/forward_engineering/helpers/general.js index 86225e4a..b6864017 100644 --- a/forward_engineering/helpers/general.js +++ b/forward_engineering/helpers/general.js @@ -178,6 +178,31 @@ module.exports = app => { } }; + const parseTextArea = text => + text + ? text + .split('\n') + .map(l => l.trim()) + .filter(Boolean) + .join(', ') + : ''; + + const parseProps = text => { + if (!text) return ''; + + return text + .split('\n') + .map(line => line.trim()) + .filter(line => line.includes('=')) + .map(line => { + const [propertyKey, ...valueParts] = line.split('='); + const propertyValue = valueParts.join('=').trim(); + + return `'${escape(propertyKey.trim())}'='${escape(propertyValue)}'`; + }) + .join(', '); + }; + return { toString, toNumber, @@ -195,5 +220,7 @@ module.exports = app => { filterProcedure, setOrReplace, getCompositeName, + parseTextArea, + parseProps, }; }; diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 6ccadd28..186eafa1 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -124,6 +124,135 @@ making sure that you maintain a proper JSON format. "propertyType": "details", "template": "textarea" }, + { + "propertyName": "External", + "propertyKeyword": "externalTable", + "propertyType": "checkbox", + "propertyTooltip": "Specifies that the table is an external table.", + "dependency": { + "type": "not", + "values": { + "key": "temporary", + "value": true + } + } + }, + { + "propertyName": "Partitioned by", + "propertyKeyword": "partitionedBy", + "propertyTooltip": "Specify partition columns as col_name data_type, e.g.:feed_name string", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "valueType": "string", + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Row format type", + "propertyKeyword": "rowFormatType", + "propertyTooltip": "Choose between DELIMITED or SERDE", + "propertyType": "select", + "options": ["", "DELIMITED", "SERDE"], + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Row format delimited", + "propertyKeyword": "rowFormatDelimited", + "propertyType": "text", + "propertyTooltip": "Specify delimiters, e.g., FIELDS TERMINATED BY, LINES TERMINATED BY", + "dependency": { + "key": "rowFormatType", + "value": "DELIMITED" + } + }, + { + "propertyName": "Row format SerDe", + "propertyKeyword": "rowFormatSerde", + "propertyType": "text", + "dependency": { + "key": "rowFormatType", + "value": "SERDE" + } + }, + { + "propertyName": "SerDe properties", + "propertyKeyword": "serdeProperties", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "propertyTooltip": "Add SerDe properties as key=value pairs", + "dependency": { + "key": "rowFormatType", + "value": "SERDE" + } + }, + { + "propertyName": "Stored as", + "propertyKeyword": "storedAs", + "propertyTooltip": "The file format for data files.", + "propertyType": "select", + "options": [ + "", + "PARQUET", + "RCFILE", + "SEQUENCEFILE", + "TEXTFILE", + "ORC", + "AVRO", + "INPUTFORMAT / OUTPUTFORMAT" + ], + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Input Format Class", + "propertyKeyword": "inputFormatClass", + "propertyType": "text", + "dependency": { + "key": "storedAs", + "value": "INPUTFORMAT / OUTPUTFORMAT" + } + }, + { + "propertyName": "Output Format Class", + "propertyKeyword": "outputFormatClass", + "propertyType": "text", + "dependency": { + "key": "storedAs", + "value": "INPUTFORMAT / OUTPUTFORMAT" + } + }, + { + "propertyName": "Location", + "propertyKeyword": "externalLocation", + "propertyTooltip": "The path to the Amazon S3 bucket/folder or a manifest file, e.g., s3://my-bucket/folder/ or s3://my-bucket/manifest.json", + "propertyType": "text", + "dependency": { + "key": "externalTable", + "value": true + } + }, + { + "propertyName": "Table properties", + "propertyKeyword": "tableProperties", + "propertyTooltip": "Add table properties as key=value pairs", + "propertyType": "details", + "template": "textarea", + "markdown": false, + "valueType": "string", + "dependency": { + "key": "externalTable", + "value": true + } + }, { "propertyName": "Table role", "propertyKeyword": "tableRole", @@ -152,26 +281,54 @@ making sure that you maintain a proper JSON format. "propertyName": "Temporary", "propertyKeyword": "temporary", "propertyType": "checkbox", - "propertyTooltip": "Creates a temporary table that is visible only within the current session. The table is automatically dropped at the end of the session in which it is created." + "propertyTooltip": "Creates a temporary table that is visible only within the current session. The table is automatically dropped at the end of the session in which it is created.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "If not exists", "propertyKeyword": "ifNotExists", "propertyType": "checkbox", - "propertyTooltip": "if the specified table already exists, the command should make no changes and return a message that the table exists, rather than terminating with an error. " + "propertyTooltip": "if the specified table already exists, the command should make no changes and return a message that the table exists, rather than terminating with an error. ", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Backup", "propertyKeyword": "BACKUP", "propertyType": "checkbox", - "propertyTooltip": "Specifies whether the table should be included in automated and manual cluster snapshots." + "propertyTooltip": "Specifies whether the table should be included in automated and manual cluster snapshots.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Distribution style", "propertyKeyword": "DISTSTYLE", "propertyTooltip": "keyword that defines the data distribution style for the whole table. AUTO: Amazon Redshift assigns an optimal distribution style based on the table data. EVEN: The data in the table is spread evenly across the nodes in a cluster in a round-robin distribution. KEY: The data is distributed by the values in the distKey column. ALL: A copy of the entire table is distributed to every node. ", "propertyType": "select", - "options": ["Auto", "Even", "Key", "All"] + "options": ["Auto", "Even", "Key", "All"], + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Dist key", @@ -193,13 +350,27 @@ making sure that you maintain a proper JSON format. "maxFields": 1 } } - ] + ], + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Auto sort key", "propertyKeyword": "autoSortKey", "propertyType": "checkbox", - "propertyTooltip": "If you don't specify any sort keys options, the default is AUTO." + "propertyTooltip": "If you don't specify any sort keys options, the default is AUTO.", + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Sort key", @@ -221,11 +392,23 @@ making sure that you maintain a proper JSON format. } ], "dependency": { - "type": "not", - "values": { - "key": "autoSortKey", - "value": true - } + "type": "and", + "values": [ + { + "type": "not", + "values": { + "key": "autoSortKey", + "value": true + } + }, + { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } + ] } }, { @@ -235,11 +418,23 @@ making sure that you maintain a proper JSON format. "propertyType": "select", "options": ["Compound", "Interleaved"], "dependency": { - "type": "not", - "values": { - "key": "autoSortKey", - "value": true - } + "type": "and", + "values": [ + { + "type": "not", + "values": { + "key": "autoSortKey", + "value": true + } + }, + { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } + ] } }, { @@ -249,7 +444,14 @@ making sure that you maintain a proper JSON format. "propertyType": "selecthashed", "template": "entities", "withEmptyOption": true, - "excludeCurrent": true + "excludeCurrent": true, + "dependency": { + "type": "not", + "values": { + "key": "externalTable", + "value": true + } + } }, { "propertyName": "Include defaults", From 6518beea055095ce38a83582d4f9e70333123d3b Mon Sep 17 00:00:00 2001 From: ivan-m-dev Date: Thu, 21 May 2026 13:30:40 +0200 Subject: [PATCH 2/2] HCK-16175: fix Sonar issue --- forward_engineering/ddlProvider.js | 22 +++------------- forward_engineering/helpers/general.js | 36 ++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 6ab46df3..3cac73be 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -5,7 +5,6 @@ const templates = require('./configs/templates'); const types = require('./configs/types'); const { commentIfDeactivated } = require('./helpers/commentDeactivatedHelper'); const { getTableAttributes, getTableConstraints, getTableLikeConstraint } = require('./helpers/tableHelper'); -const { ROW_FORMAT_TYPES, STORED_AS_TYPES } = require('./helpers/constants'); module.exports = (baseProvider, options, app) => { const { hasType } = app.require('@hackolade/ddl-fe-utils').general; @@ -24,6 +23,8 @@ module.exports = (baseProvider, options, app) => { toString, parseTextArea, parseProps, + getRowFormat, + getStoredAs, } = require('./helpers/general')(app); const { decorateType, @@ -42,23 +43,8 @@ module.exports = (baseProvider, options, app) => { ? `\nPARTITIONED BY (${parseTextArea(tableData.partitionedBy)})` : ''; - let rowFormat = ''; - if (tableData.rowFormatType === ROW_FORMAT_TYPES.DELIMITED && tableData.rowFormatDelimited) { - rowFormat = `\nROW FORMAT DELIMITED ${tableData.rowFormatDelimited}`; - } else if (tableData.rowFormatType === ROW_FORMAT_TYPES.SERDE && tableData.rowFormatSerde) { - rowFormat = `\nROW FORMAT SERDE ${toString(tableData.rowFormatSerde)}`; - const serdeProps = parseProps(tableData.serdeProperties); - if (serdeProps) rowFormat += `\nWITH SERDEPROPERTIES (${serdeProps})`; - } - - let storedAs = ''; - if (tableData.storedAs === STORED_AS_TYPES.INPUT_OUTPUT_FORMAT) { - if (tableData.inputFormatClass && tableData.outputFormatClass) { - storedAs = `\nSTORED AS INPUTFORMAT ${toString(tableData.inputFormatClass)}\nOUTPUTFORMAT ${toString(tableData.outputFormatClass)}`; - } - } else if (tableData.storedAs) { - storedAs = `\nSTORED AS ${tableData.storedAs}`; - } + const rowFormat = getRowFormat(tableData); + const storedAs = getStoredAs(tableData); const location = tableData.externalLocation ? `\nLOCATION ${toString(tableData.externalLocation)}` : ''; const tblProperties = tableData.tableProperties diff --git a/forward_engineering/helpers/general.js b/forward_engineering/helpers/general.js index b6864017..f8d43c60 100644 --- a/forward_engineering/helpers/general.js +++ b/forward_engineering/helpers/general.js @@ -1,5 +1,6 @@ const _ = require('lodash'); const { commentIfDeactivated } = require('./commentDeactivatedHelper'); +const { ROW_FORMAT_TYPES, STORED_AS_TYPES } = require('./constants'); module.exports = app => { const { checkAllKeysActivated, clean, tab } = app.require('@hackolade/ddl-fe-utils').general; @@ -203,6 +204,39 @@ module.exports = app => { .join(', '); }; + const getRowFormat = tableData => { + if (tableData.rowFormatType === ROW_FORMAT_TYPES.DELIMITED && tableData.rowFormatDelimited) { + return `\nROW FORMAT DELIMITED ${tableData.rowFormatDelimited}`; + } + + if (tableData.rowFormatType === ROW_FORMAT_TYPES.SERDE && tableData.rowFormatSerde) { + let format = `\nROW FORMAT SERDE ${toString(tableData.rowFormatSerde)}`; + const serdeProps = parseProps(tableData.serdeProperties); + + if (serdeProps) { + format += `\nWITH SERDEPROPERTIES (${serdeProps})`; + } + return format; + } + + return ''; + }; + + const getStoredAs = tableData => { + if (tableData.storedAs === STORED_AS_TYPES.INPUT_OUTPUT_FORMAT) { + if (tableData.inputFormatClass && tableData.outputFormatClass) { + return `\nSTORED AS INPUTFORMAT ${toString(tableData.inputFormatClass)}\nOUTPUTFORMAT ${toString(tableData.outputFormatClass)}`; + } + return ''; + } + + if (tableData.storedAs) { + return `\nSTORED AS ${tableData.storedAs}`; + } + + return ''; + }; + return { toString, toNumber, @@ -222,5 +256,7 @@ module.exports = app => { getCompositeName, parseTextArea, parseProps, + getRowFormat, + getStoredAs, }; };