Add Support for Vector Data Type in SQL for REST

.pipelines/mssql-pipelines.yml

@@ -177,7 +177,7 @@ jobs:
       # for the linux job above.
       data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;
       InstallerUrl: https://download.microsoft.com/download/7/c/1/7c14e92e-bdcb-4f89-b7cf-93543e7112d1/SqlLocalDB.msi
-      SqlVersionCode: '15.0'
+      SqlVersionCode: '17.0'
 
   steps:
   - template: templates/mssql-test-steps.yml
@@ -200,7 +200,7 @@ jobs:
       # for the linux job above.
       data-source.connection-string: Server=(localdb)\MSSQLLocalDB;Persist Security Info=False;Integrated Security=True;MultipleActiveResultSets=False;Connection Timeout=30;TrustServerCertificate=True;
       InstallerUrl: https://download.microsoft.com/download/7/c/1/7c14e92e-bdcb-4f89-b7cf-93543e7112d1/SqlLocalDB.msi
-      SqlVersionCode: '15.0'
+      SqlVersionCode: '17.0'
 
   steps:
   - template: templates/mssql-test-steps.yml

config-generators/mssql-commands.txt

@@ -16,6 +16,8 @@ add Broker --config "dab-config.MsSql.json" --source brokers --permissions "anon
 add WebsiteUser --config "dab-config.MsSql.json" --source website_users --permissions "anonymous:create,read,delete,update"
 add WebsiteUser_MM --config "dab-config.MsSql.json" --source website_users_mm --graphql "websiteuser_mm:websiteusers_mm" --permissions "anonymous:*"
 add SupportedType --config "dab-config.MsSql.json" --source type_table --permissions "anonymous:create,read,delete,update"
+add VectorType --config "dab-config.MsSql.json" --source vector_type_table --rest true --graphql false --permissions "anonymous:create,read,delete,update"
+update VectorType --config "dab-config.MsSql.json" --permissions "authenticated:create,read,delete,update"
 add stocks_price --config "dab-config.MsSql.json" --source stocks_price --permissions "authenticated:create,read,update,delete"
 update stocks_price --config "dab-config.MsSql.json" --permissions "anonymous:read"
 update stocks_price --config "dab-config.MsSql.json" --permissions "TestNestedFilterFieldIsNull_ColumnForbidden:read" --fields.exclude "price"

scripts/start-mssql-server.bash

@@ -27,7 +27,7 @@ echo "forceencryption = 1" >> $CERT_DIR/mssql.conf
 cat $CERT_DIR/mssql.conf
 
 # Start mssql-server by volume mounting the cert, key and conf files.
-docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$DOCKER_SQL_PASS" -p 1433:1433 --name customerdb -h customerdb -v $CERT_DIR/mssql.conf:/var/opt/mssql/mssql.conf -v $CERT_DIR/mssql.pem:/var/opt/mssql/mssql.pem -v $CERT_DIR/mssql.key:/var/opt/mssql/mssql.key -d mcr.microsoft.com/mssql/server:2019-latest
+docker run -e "ACCEPT_EULA=Y" -e "MSSQL_SA_PASSWORD=$DOCKER_SQL_PASS" -p 1433:1433 --name customerdb -h customerdb -v $CERT_DIR/mssql.conf:/var/opt/mssql/mssql.conf -v $CERT_DIR/mssql.pem:/var/opt/mssql/mssql.pem -v $CERT_DIR/mssql.key:/var/opt/mssql/mssql.key -d mcr.microsoft.com/mssql/server:2025-latest
 sleep 30
 docker logs customerdb
 

src/Core/Models/SqlTypeConstants.cs

@@ -48,6 +48,7 @@ public static class SqlTypeConstants
         { "datetime2", true },            // SqlDbType.DateTime2
         { "datetimeoffset", true },       // SqlDbType.DateTimeOffset
         { "", false },                    // SqlDbType.Udt and SqlDbType.Structured provided by SQL as empty strings (unsupported)
-        { "numeric", true}                // Not present in SqlDbType, however can be returned by sql functions like LAG and should map to decimal.
+        { "numeric", true},                // Not present in SqlDbType, however can be returned by sql functions like LAG and should map to decimal.
+        { "vector", true }                // SqlDbType.Vector
     };
 }

src/Core/Resolvers/MsSqlQueryBuilder.cs

@@ -666,8 +666,7 @@ AND ty.name IN
                                     N'hierarchyid',
                                     N'sql_variant',
                                     N'xml',
-                                    N'rowversion',
-                                    N'vector'
+                                    N'rowversion'
                                 )
                         ) THEN 1
                         ELSE 0
@@ -712,8 +711,7 @@ AND ty.name IN
                                     N'hierarchyid',
                                     N'sql_variant',
                                     N'xml',
-                                    N'rowversion',
-                                    N'vector'
+                                    N'rowversion'
                                 )
                         )
                     )

src/Core/Resolvers/MsSqlQueryExecutor.cs

@@ -18,6 +18,7 @@
 using Azure.Identity;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Data.SqlClient;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 
 namespace Azure.DataApiBuilder.Core.Resolvers
@@ -695,6 +696,19 @@ public override SqlCommand PrepareDbCommand(
                         parameter.Size = parameterEntry.Value.Length.Value;
                     }
 
+                    // if sqldbtype is vector then set the value as an SqlVector object
+                    if (parameter.SqlDbType is SqlDbType.Vector)
+                    {
+                        List<float> values = new();
+                        foreach (float val in (Array)parameter.Value)
+                        {
+                            values.Add(val);
+                        }
+
+                        SqlVector<float> value = new(values.ToArray());
+                        parameter.Value = value;
+                    }
+
                     cmd.Parameters.Add(parameter);
                 }
             }

src/Core/Resolvers/QueryExecutor.cs

@@ -13,6 +13,7 @@
 using Azure.DataApiBuilder.Core.Models;
 using Azure.DataApiBuilder.Service.Exceptions;
 using Microsoft.AspNetCore.Http;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 using Polly;
 using Polly.Retry;
@@ -502,7 +503,7 @@ public async Task<DbResultSet>
                             {
                                 if (!ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled())
                                 {
-                                    dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                                    dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
                                 }
                                 else
                                 {
@@ -554,7 +555,7 @@ public DbResultSet
                             {
                                 if (!ConfigProvider.GetConfig().MaxResponseSizeLogicEnabled())
                                 {
-                                    dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                                    dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
                                 }
                                 else
                                 {
@@ -822,7 +823,7 @@ internal int StreamDataIntoDbResultSetRow(DbDataReader dbDataReader, DbResultSet
             {
                 dataRead = columnSize;
                 ValidateSize(availableBytes, dataRead);
-                dbResultSetRow.Columns.Add(columnName, dbDataReader[columnName]);
+                dbResultSetRow.Columns.Add(columnName, GetColumnInformation(dbDataReader, columnName));
             }
 
             return dataRead;
@@ -885,6 +886,22 @@ private void ValidateSize(long availableSizeBytes, long sizeToBeReadBytes)
             }
         }
 
+        /// <summary>
+        /// Helper function to get column information from the DbDataReader and handle special cases like SqlVector<float>.
+        /// </summary>
+        /// <param name="dbDataReader"></param>
+        /// <param name="columnName"></param>
+        /// <returns></returns>
+        private static object GetColumnInformation(DbDataReader dbDataReader, string columnName)
+        {
+            if (dbDataReader[columnName] is SqlVector<float> columnValue)
+            {
+                return columnValue.Memory;
+            }
+
+            return dbDataReader[columnName];
+        }
+
         internal virtual void AddDbExecutionTimeToMiddlewareContext(long time)
         {
             HttpContext? httpContext = HttpContextAccessor?.HttpContext;

src/Core/Resolvers/Sql Query Structures/BaseSqlQueryStructure.cs

@@ -4,6 +4,7 @@
 using System.Data;
 using System.Globalization;
 using System.Net;
+using System.Text.Json;
 using Azure.DataApiBuilder.Auth;
 using Azure.DataApiBuilder.Config.DatabasePrimitives;
 using Azure.DataApiBuilder.Config.ObjectModel;
@@ -452,10 +453,50 @@ protected static object ParseParamAsSystemType(string param, Type systemType)
                 "Guid" => Guid.Parse(param),
                 "TimeOnly" => TimeOnly.Parse(param),
                 "TimeSpan" => TimeOnly.Parse(param),
+                "Single[]" => ParseArrayIntoSystemType(param, systemType),
                 _ => throw new NotSupportedException($"{systemType.Name} is not supported")
             };
         }
 
+        /// <summary>
+        /// Takes the array of the parameter we are going to parse and converts each element to the specified system type.
+        /// </summary>
+        /// <param name="param"></param>
+        /// <param name="systemType"></param>
+        /// <returns></returns>
+        /// <exception cref="NotSupportedException"></exception>
+        /// <exception cref="FormatException"></exception>
+        private static object ParseArrayIntoSystemType(string param, Type systemType)
+        {
+            Type typeOfArray;
+            switch (systemType.Name)
+            {
+                case "Single[]":
+                    typeOfArray = typeof(Single);
+                    break;
+
+                default:
+                    throw new NotSupportedException($"{systemType.Name} is not supported");
+            }
+
+            try
+            {
+                List<object> list = new();
+                object[] values = JsonSerializer.Deserialize<object[]>(param) ?? Array.Empty<object>();
+                for (int i = 0; i < values.Length; i++)
+                {
+                    string stringValue = values[i]?.ToString() ?? string.Empty;
+                    values[i] = ParseParamAsSystemType(stringValue, typeOfArray);
+                }
+
+                return values;
+            }
+            catch
+            {
+                throw new FormatException($"Expected an array for {systemType.Name} but got an unexpected value");
+            }
+        }
+
         /// <summary>
         /// Very similar to GQLArgumentToDictParams but only extracts the argument names from
         /// the specified field which means that the method does not need a middleware context

src/Core/Services/MetadataProviders/MsSqlMetadataProvider.cs

@@ -15,6 +15,7 @@
 using Azure.DataApiBuilder.Service.Exceptions;
 using Azure.DataApiBuilder.Service.GraphQLBuilder;
 using Microsoft.Data.SqlClient;
+using Microsoft.Data.SqlTypes;
 using Microsoft.Extensions.Logging;
 using static Azure.DataApiBuilder.Service.GraphQLBuilder.GraphQLNaming;
 
@@ -114,6 +115,15 @@ protected override void PopulateColumnDefinitionWithHasDefaultAndDbType(
                     columnDefinition.DbType = TypeHelper.GetDbTypeFromSystemType(columnDefinition.SystemType);
 
                     string sqlDbTypeName = (string)columnInfo["DATA_TYPE"];
+
+                    if (columnDefinition.SystemType == typeof(SqlVector<Single>))
+                    {
+                        sqlDbTypeName = "vector";   // Currently the "DATA_TYPE" column returns "varbinary" for vector type columns. This is a known issue https://learn.microsoft.com/en-us/sql/t-sql/data-types/vector-data-type?view=sql-server-ver17&tabs=csharp#known-issues
+                        columnDefinition.IsArrayType = true;
+                        columnDefinition.ElementSystemType = typeof(Single);
+                        columnDefinition.SystemType = columnDefinition.ElementSystemType.MakeArrayType();
+                    }
+
                     if (Enum.TryParse(sqlDbTypeName, ignoreCase: true, out SqlDbType sqlDbType))
                     {
                         // The DbType enum in .NET does not distinguish between VarChar and NVarChar. Both are mapped to DbType.String.

src/Core/Services/OpenAPI/OpenApiDocumentor.cs

@@ -1484,12 +1484,18 @@ private static OpenApiSchema CreateComponentSchema(
                 if (metadataProvider.TryGetBackingColumn(entityName, field, out string? backingColumnValue) && !string.IsNullOrEmpty(backingColumnValue))
                 {
                     string typeMetadata = string.Empty;
+                    string subTypeMetadata = string.Empty;
                     string formatMetadata = string.Empty;
                     string? fieldDescription = null;
 
                     if (dbObject.SourceDefinition.Columns.TryGetValue(backingColumnValue, out ColumnDefinition? columnDef))
                     {
                         typeMetadata = TypeHelper.GetJsonDataTypeFromSystemType(columnDef.SystemType).ToString().ToLower();
+
+                        if (string.Equals(typeMetadata, JsonDataType.Array.ToString().ToLower(), StringComparison.OrdinalIgnoreCase))
+                        {
+                            subTypeMetadata = TypeHelper.GetJsonDataTypeFromSystemType(columnDef.ElementSystemType!).ToString().ToLower();
+                        }
                     }
 
                     if (entityConfig?.Fields != null)
@@ -1502,7 +1508,8 @@ private static OpenApiSchema CreateComponentSchema(
                     {
                         Type = typeMetadata,
                         Format = formatMetadata,
-                        Description = fieldDescription
+                        Description = fieldDescription,
+                        Items = !string.IsNullOrWhiteSpace(subTypeMetadata) ? new OpenApiSchema() { Type = subTypeMetadata } : null
                     });
                 }
             }

src/Core/Services/TypeHelper.cs

@@ -8,6 +8,7 @@
 using Azure.DataApiBuilder.Core.Services.OpenAPI;
 using Azure.DataApiBuilder.Service.Exceptions;
 using HotChocolate.Language;
+using Microsoft.Data.SqlTypes;
 using Microsoft.OData.Edm;
 
 namespace Azure.DataApiBuilder.Core.Services
@@ -46,7 +47,8 @@ public static class TypeHelper
             [typeof(byte[])] = DbType.Binary,
             [typeof(TimeOnly)] = DbType.Time,
             [typeof(TimeSpan)] = DbType.Time,
-            [typeof(object)] = DbType.Object
+            [typeof(object)] = DbType.Object,
+            [typeof(SqlVector<Single>)] = DbType.Single
         };
 
         /// <summary>
@@ -77,7 +79,8 @@ public static class TypeHelper
             [typeof(TimeOnly)] = JsonDataType.String,
             [typeof(object)] = JsonDataType.Object,
             [typeof(DateTime)] = JsonDataType.String,
-            [typeof(DateTimeOffset)] = JsonDataType.String
+            [typeof(DateTimeOffset)] = JsonDataType.String,
+            [typeof(Single[])] = JsonDataType.Array
         };
 
         /// <summary>
@@ -111,7 +114,8 @@ public static class TypeHelper
             [SqlDbType.TinyInt] = typeof(byte),
             [SqlDbType.UniqueIdentifier] = typeof(Guid),
             [SqlDbType.VarBinary] = typeof(byte[]),
-            [SqlDbType.VarChar] = typeof(string)
+            [SqlDbType.VarChar] = typeof(string),
+            [SqlDbType.Vector] = typeof(float)
         };
 
         private static Dictionary<SqlDbType, DbType> _sqlDbDateTimeTypeToDbType = new()

src/Service.Tests/DatabaseSchema-MsSql.sql

@@ -40,6 +40,7 @@ DROP TABLE IF EXISTS stocks;
 DROP TABLE IF EXISTS comics;
 DROP TABLE IF EXISTS brokers;
 DROP TABLE IF EXISTS type_table;
+DROP TABLE IF EXISTS vector_type_table;
 DROP TABLE IF EXISTS trees;
 DROP TABLE IF EXISTS fungi;
 DROP TABLE IF EXISTS empty_table;
@@ -232,6 +233,12 @@ CREATE TABLE type_table(
     uuid_types uniqueidentifier DEFAULT newid()
 );
 
+CREATE TABLE vector_type_table(
+    id int IDENTITY(5001, 1) PRIMARY KEY,
+    vector_data vector(3),
+    vector_data_max vector(1998)
+);
+
 CREATE TABLE trees (
     treeId int PRIMARY KEY,
     species varchar(max),
@@ -608,6 +615,23 @@ VALUES
 INSERT INTO type_table(id, uuid_types) values(10, 'D1D021A8-47B4-4AE4-B718-98E89C41A161');
 SET IDENTITY_INSERT type_table OFF
 
+SET IDENTITY_INSERT vector_type_table ON
+INSERT INTO vector_type_table(id, vector_data)
+VALUES
+    (1, '[0.5, 0.25, 0.75]'),
+    (2, '[1.5, -2.5, 3.5]'),
+    (3, NULL),
+    (4, '[1.0, 2.0, 3.0]'),
+    (5, '[4.0, 5.0, 6.0]'),
+    (6, '[7.0, 8.0, 9.0]');
+
+INSERT INTO vector_type_table(id, vector_data_max)
+VALUES (7, CAST('[' + (
+    SELECT STRING_AGG(CAST(value AS NVARCHAR(MAX)), ',') WITHIN GROUP (ORDER BY value)
+    FROM GENERATE_SERIES(1, 1998)
+) + ']' AS vector(1998)));
+SET IDENTITY_INSERT vector_type_table OFF
+
 SET IDENTITY_INSERT sales ON
 INSERT INTO sales(id, item_name, subtotal, tax) VALUES (1, 'Watch', 249.00, 20.59), (2, 'Montior', 120.50, 11.12);
 SET IDENTITY_INSERT sales OFF