Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@
`getFloat`/`setFloat` and `getObject` accessors, and reported as such by `ResultSetMetaData` and `DatabaseMetaData`.
Previously reading or writing a `BFloat16` column failed with an
unsupported-data-type error. (https://github.com/ClickHouse/clickhouse-java/issues/2279)
- **[client-v2, jdbc-v2]** Added support for the experimental `QBit(element_type, dimension)` vector data type
(ClickHouse `25.10+`; the `allow_experimental_qbit_type` server setting is required to create a column), with
`BFloat16`, `Float32`, or `Float64` element types. A `QBit` value is transmitted over `RowBinary` exactly like
`Array(element_type)`, so it is read and written as a Java array of the element type (`float[]` for
`BFloat16`/`Float32`, `double[]` for `Float64`) through generic records, binary readers, and POJO binding. In the
JDBC driver (`jdbc-v2`) `QBit` maps to `java.sql.Types.ARRAY` and is returned as a `java.sql.Array` from
`getObject`/`getArray`. Previously `QBit` was an unimplemented type constant and reading or writing such a column
failed. Reading `QBit` through the `Native` output format is not supported — the server transmits it there using a
different internal layout — and fails fast with a clear error; use a `RowBinary` format instead.
(https://github.com/ClickHouse/clickhouse-java/issues/2610)
- **[client-v2, jdbc-v2]** Added TLS cipher suite selection. `Client.Builder.setSSLCipherSuites(String...)` (client-v2)
and the comma-separated `ssl_cipher_suites` connection property (client-v2 and jdbc-v2) restrict the cipher suites
enabled on secure connections; when unset, the transport defaults are used. Cipher-suite selection is independent of the
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ public final class ClickHouseColumn implements Serializable {
private static final String KEYWORD_NESTED = ClickHouseDataType.Nested.name();
private static final String KEYWORD_VARIANT = ClickHouseDataType.Variant.name();
private static final String KEYWORD_JSON = ClickHouseDataType.JSON.name();
private static final String KEYWORD_QBIT = ClickHouseDataType.QBit.name();

private int columnCount;
private int columnIndex;
Expand Down Expand Up @@ -146,6 +147,17 @@ private static ClickHouseColumn update(ClickHouseColumn column) {
}
}
break;
case QBit:
// QBit(element_type, dimension) is a one-level array of its element type on the
// wire; the dimension parameter is kept as the column precision.
if (!column.nested.isEmpty()) {
column.arrayLevel = 1;
column.arrayBaseColumn = column.nested.get(0);
}
if (size > 1) {
column.precision = Integer.parseInt(column.parameters.get(1).trim());
}
break;
case Bool:
column.template = ClickHouseBoolValue.ofNull();
break;
Expand Down Expand Up @@ -567,6 +579,26 @@ protected static int readColumn(String args, int startIndex, int len, String nam
fixedLength = false;
estimatedLength++;
}
} else if (args.startsWith(KEYWORD_QBIT, i)) {
int index = args.indexOf('(', i + KEYWORD_QBIT.length());
if (index < i) {
throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE);
}
List<String> params = new LinkedList<>();
i = ClickHouseUtils.readParameters(args, index, len, params);
if (params.size() < 2) {
throw new IllegalArgumentException(
"QBit requires an element type and a dimension, e.g. QBit(Float32, 8)");
}
// QBit(element_type, dimension) is transmitted over RowBinary exactly like
// Array(element_type): a var-int length followed by that many element values. The
// first parameter is the element type and drives the nested (item) column.
List<ClickHouseColumn> nestedColumns = new LinkedList<>();
nestedColumns.add(ClickHouseColumn.of("", params.get(0)));
column = new ClickHouseColumn(ClickHouseDataType.QBit, name, args.substring(startIndex, i),
nullable, lowCardinality, params, nestedColumns);
fixedLength = false;
estimatedLength++;
}

if (column == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -508,4 +508,33 @@ public Object[][] testJSONBinaryFormat_dp() {
{"JSON(max_dynamic_types=3,max_dynamic_paths=3, SKIP REGEXP '^-.*',SKIP ff, flags Array(Array(Array(Int8))), SKIP alt_count)", 2, Arrays.asList("flags")},
};
}

@DataProvider(name = "qbitTypesProvider")
private static Object[][] qbitTypesProvider() {
return new Object[][] {
{ "QBit(BFloat16, 4)", ClickHouseDataType.BFloat16, 4 },
{ "QBit(Float32, 8)", ClickHouseDataType.Float32, 8 },
{ "QBit(Float64, 1536)", ClickHouseDataType.Float64, 1536 },
};
}

@Test(groups = { "unit" }, dataProvider = "qbitTypesProvider")
public void testParseQBit(String typeName, ClickHouseDataType elementType, int dimension) {
ClickHouseColumn column = ClickHouseColumn.of("vec", typeName);
Assert.assertEquals(column.getDataType(), ClickHouseDataType.QBit);
Assert.assertEquals(column.getOriginalTypeName(), typeName);
// QBit(element_type, dimension) is a one-level array of its element type on the wire.
Assert.assertEquals(column.getArrayNestedLevel(), 1);
Assert.assertNotNull(column.getArrayBaseColumn());
Assert.assertEquals(column.getArrayBaseColumn().getDataType(), elementType);
Assert.assertEquals(column.getNestedColumns().size(), 1);
Assert.assertEquals(column.getNestedColumns().get(0).getDataType(), elementType);
// The fixed vector dimension is retained as the column precision.
Assert.assertEquals(column.getPrecision(), dimension);
}

@Test(groups = { "unit" }, expectedExceptions = IllegalArgumentException.class)
public void testParseQBitRequiresDimension() {
ClickHouseColumn.of("vec", "QBit(Float32)");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
return true;
}

private boolean readBlock() throws IOException {

Check failure on line 71 in client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java

View check run for this annotation

SonarQubeCloud / SonarCloud Code Analysis

Refactor this method to reduce its Cognitive Complexity from 16 to the 15 allowed.

See more on https://sonarcloud.io/project/issues?id=ClickHouse_clickhouse-java&issues=AZ-J9-1qRYQuxmCUBDar&open=AZ-J9-1qRYQuxmCUBDar&pullRequest=2939
int nColumns;
try {
nColumns = BinaryStreamReader.readVarInt(input);
Expand All @@ -91,6 +91,21 @@
names.add(column.getColumnName());
types.add(column.getDataType().name());

if (containsQBit(column)) {
// QBit is transmitted in the Native format using its internal bit-transposed
// Tuple(FixedString(...)) layout, which is NOT the Array(element_type)-like
// representation used in RowBinary (the only representation this reader decodes for
// QBit). Reading it through the columnar/per-row paths below would misread those bytes
// and desynchronize the block, corrupting the columns that follow. Fail loudly instead
// of silently decoding garbage. This also covers a QBit nested inside another type
// (e.g. Map(String, QBit(...))). QBit can be read through a RowBinary format.
throw new ClientException("Reading column '" + column.getColumnName() + "' ("
+ column.getOriginalTypeName() + ") from the Native format is not supported "
+ "because it contains a QBit type: QBit is serialized in the Native format "
+ "using an internal layout this reader does not decode. Use a RowBinary format "
+ "(e.g. RowBinaryWithNamesAndTypes) to read QBit values");
}

List<Object> values = new ArrayList<>(nRows);
if (column.isArray()) {
int[] sizes = new int[nRows];
Expand All @@ -116,6 +131,26 @@
return true;
}

/**
* Returns {@code true} if {@code column} is a {@code QBit} or contains a {@code QBit} anywhere in
* its nested type tree (e.g. {@code Array(QBit(...))}, {@code Tuple(..., QBit(...))},
* {@code Map(String, QBit(...))}). {@code QBit} uses a different, internal wire layout in the
* Native format than in RowBinary, so this reader cannot decode it and rejects such columns
* up-front rather than misreading the block. {@code Nullable}/{@code LowCardinality} wrappers are
* flags on the column, so a wrapped {@code QBit} still reports {@code dataType == QBit} here.
*/
private static boolean containsQBit(ClickHouseColumn column) {
if (column.getDataType() == ClickHouseDataType.QBit) {
return true;
}
for (ClickHouseColumn nested : column.getNestedColumns()) {
if (nested != column && containsQBit(nested)) {
return true;
}
}
return false;
}

private static class Block {
final List<String> names;
final List<String> types;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
return (T) readJsonData(input, actualColumn);
}
// case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
case QBit:
// QBit(element_type, dimension) is transmitted like Array(element_type).
Comment thread
polyglotAI-bot marked this conversation as resolved.
case Array:
if (typeHint == null) { typeHint = arrayDefaultTypeHint;}
return convertArray(readArray(actualColumn), typeHint);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ public class SerializerUtils {
public static void serializeData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
//Serialize the value to the stream based on the data type
switch (column.getDataType()) {
case QBit:
serializeQBitData(stream, value, column);
break;
case Array:
serializeArrayData(stream, value, column);
break;
Expand Down Expand Up @@ -470,6 +473,52 @@ public static void serializeArrayData(OutputStream stream, Object value, ClickHo
}
}

/**
* Serializes a {@code QBit(element_type, dimension)} value. On the wire a {@code QBit} is
* transmitted exactly like {@code Array(element_type)} — a var-int length followed by that many
* element values — but the element count is fixed and must equal the declared dimension. The
* count is validated up-front so a wrong-sized (including empty) vector fails fast on the client
* with a clear message instead of a late server {@code SERIALIZATION_ERROR}, mirroring the
* client-side length enforcement already applied to the other fixed-size type,
* {@code FixedString(N)}. A non-null value that is neither a Java array nor a {@code List} cannot
* carry a vector and is rejected as well — otherwise it would fall through to
* {@link #serializeArrayData} and write no bytes for the column, desynchronizing the
* {@code RowBinary} stream and corrupting the columns that follow.
* <p>
* A {@code null} value is rejected for the same reason: a fixed-dimension {@code QBit} cannot be
* represented by {@code null}. A top-level {@code null} non-nullable {@code QBit} is already
* rejected by
* {@link com.clickhouse.client.api.data_formats.RowBinaryFormatSerializer#writeValuePreamble},
* but a {@code QBit} nested inside a container ({@code Tuple}/{@code Map}/{@code Array}) is written
* through {@link #serializeNestedData}, which does not route a non-nullable element through that
* preamble; without this guard the {@code null} would delegate to {@link #serializeArrayData} and
* be written as a zero-length vector (var-int {@code 0}), again desynchronizing the stream. (A
* {@code Nullable(QBit)} {@code null} never reaches here: its null-marker is written earlier by the
* preamble or by {@link #serializeNestedData}, which then return.)
*/
private static void serializeQBitData(OutputStream stream, Object value, ClickHouseColumn column) throws IOException {
if (value == null) {
throw new IllegalArgumentException("QBit column '" + column.getColumnName()
+ "' cannot be null; expected exactly " + column.getPrecision() + " elements");
}
int length;
if (value.getClass().isArray()) {
length = Array.getLength(value);
} else if (value instanceof List) {
length = ((List<?>) value).size();
} else {
throw new IllegalArgumentException("QBit column '" + column.getColumnName()
+ "' expects a Java array or List of its element type but got "
+ value.getClass().getName());
}
int dimension = column.getPrecision();
if (length != dimension) {
throw new IllegalArgumentException("QBit column '" + column.getColumnName()
+ "' expects exactly " + dimension + " elements but got " + length);
}
serializeArrayData(stream, value, column);
}
Comment thread
polyglotAI-bot marked this conversation as resolved.

/**
DO NOT USE - part of internal API that will be changed
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ public String convertToString(Object value, ClickHouseColumn column) {
case IPv4:
case IPv6:
return ipvToString(value, column);
case QBit:
// QBit is rendered as an array literal of its element type, like Array(element_type).
case Array:
return arrayToString(value, column);
case Point:
Expand Down
Loading
Loading