diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java index c93963fa1..f14c2a44e 100644 --- a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java +++ b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java @@ -177,7 +177,7 @@ public static Map, Integer> buildVariantMapping(List>> DATA_TYPE_TO_CLASS = dataTypeClassMap(); + public static final Map>> DATA_TYPE_TO_CLASS = dataTypeClassMap(); static Map>> dataTypeClassMap() { Map>> map = new HashMap<>(); diff --git a/client-v2/pom.xml b/client-v2/pom.xml index 0c6409cdf..57ff943a8 100644 --- a/client-v2/pom.xml +++ b/client-v2/pom.xml @@ -88,8 +88,26 @@ com.fasterxml.jackson.core jackson-databind - test ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + provided + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + com.google.code.gson + gson + ${gson.version} + provided ${project.parent.groupId} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java index 58d94da9b..211288a30 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/Client.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/Client.java @@ -1651,6 +1651,7 @@ public CompletableFuture query(String sqlQuery, Map buildRequestSettings(Map opSettings) return requestSettings; } + /** + * Applies format-specific server-side settings to the already merged request settings. + * Must be called after {@link #buildRequestSettings(Map)} and after the request format has been resolved + * (either provided by the caller or defaulted), so that the inspected format reflects the final value. + * + *

For {@link ClickHouseFormat#JSONEachRow}, callers may opt in to plain JSON numbers by setting + * {@link ClientConfigProperties#JSON_DISABLE_NUMBER_QUOTING}. Explicit server settings are otherwise + * left untouched.

+ *
    + *
  • {@code output_format_json_quote_64bit_integers}
    • + *
  • {@code output_format_json_quote_64bit_floats}
    • + *
  • {@code output_format_json_quote_decimals}
    • + *
+ */ + private static void applyFormatSpecificSettings(QuerySettings requestSettings) { + boolean disableNumberQuoting = ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING + .getOrDefault(requestSettings.getAllSettings()); + if (requestSettings.getFormat() == ClickHouseFormat.JSONEachRow && disableNumberQuoting) { + requestSettings.serverSetting("output_format_json_quote_64bit_integers", "0"); + requestSettings.serverSetting("output_format_json_quote_64bit_floats", "0"); + requestSettings.serverSetting("output_format_json_quote_decimals", "0"); + } + } + private Duration durationSince(long sinceNanos) { return Duration.ofNanos(System.nanoTime() - sinceNanos); } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java index e548a90f9..5a61fcebf 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java @@ -190,6 +190,11 @@ public Object parseValue(String value) { */ HTTP_SEND_PARAMS_IN_BODY("client.http.use_form_request_for_query", Boolean.class, "false"), + /** + * When enabled for JSONEachRow queries, asks ClickHouse to emit large integer, + * floating-point, and decimal values as JSON numbers instead of quoted strings. + */ + JSON_DISABLE_NUMBER_QUOTING("json_disable_number_quoting", Boolean.class, "false"), /** * Prefix for custom settings. Should be aligned with server configuration. diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java index 51e1b1df0..c95bde792 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java @@ -1,618 +1,21 @@ package com.clickhouse.client.api.data_formats; -import com.clickhouse.client.api.metadata.TableSchema; -import com.clickhouse.data.value.ClickHouseBitmap; -import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue; -import com.clickhouse.data.value.ClickHouseGeoPointValue; -import com.clickhouse.data.value.ClickHouseGeoPolygonValue; -import com.clickhouse.data.value.ClickHouseGeoRingValue; - -import java.math.BigDecimal; -import java.math.BigInteger; -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.time.OffsetDateTime; -import java.time.ZonedDateTime; -import java.time.temporal.TemporalAmount; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -public interface ClickHouseBinaryFormatReader extends AutoCloseable { - - /** - * Reads a single value from the stream. - * - * @param - * @return - */ - T readValue(int colIndex); - - /** - * Reads a row to an array of objects. - * - * @param colName - * @param - * @return - */ - T readValue(String colName); - - boolean hasValue(String colName); - - boolean hasValue(int colIndex); - - /** - * Checks if there are more rows to read. - * - * @return - */ - boolean hasNext(); - - /** - * Moves cursor to the next row. Must be called before reading the first row. Returns reference to - * an internal record representation. It means that next call to the method will affect value in returned Map. - * This is done for memory usage optimization. - * Method is intended to be used only by the client not an application. - * - * @return reference to a map filled with column values or null if no more records are available - */ - Map next(); - - /** - * Reads column with name `colName` as a string. - * - * @param colName - column name - * @return - */ - String getString(String colName); - - /** - * Reads column with name `colName` as a byte. - * - * @param colName - column name - * @return - */ - byte getByte(String colName); - - /** - * Reads column with name `colName` as a short. - * - * @param colName - column name - * @return - */ - short getShort(String colName); - - /** - * Reads column with name `colName` as an integer. - * - * @param colName - column name - * @return - */ - int getInteger(String colName); - - /** - * Reads column with name `colName` as a long. - * - * @param colName - column name - * @return - */ - long getLong(String colName); - - /** - * Reads column with name `colName` as a float. - * Warning: this method may lose precision for float values. - * - * @param colName - * @return - */ - float getFloat(String colName); - - /** - * Reads column with name `colName` as a double. - * Warning: this method may lose precision for double values. - * - * @param colName - * @return - */ - double getDouble(String colName); - - /** - * Reads column with name `colName` as a boolean. - * - * @param colName - * @return - */ - boolean getBoolean(String colName); - - /** - * Reads column with name `colName` as a BigInteger. - * - * @param colName - * @return - */ - BigInteger getBigInteger(String colName); - - /** - * Reads column with name `colName` as a BigDecimal. - * - * @param colName - * @return - */ - BigDecimal getBigDecimal(String colName); - - /** - * Returns the value of the specified column as an Instant. Timezone is derived from the column definition. - * If no timezone is specified in the column definition then UTC will be used. - * - * If column value is Date or Date32 it will return an Instant with time set to 00:00:00. - * If column value is DateTime or DateTime32 it will return an Instant with the time part. - * - * @param colName - * @return - */ - Instant getInstant(String colName); - - /** - * Returns the value of the specified column as a ZonedDateTime. Timezone is derived from the column definition. - * If no timezone is specified in the column definition then UTC will be used. - * - * If column value is Date or Date32 it will return a ZonedDateTime with time set to 00:00:00. - * If column value is DateTime or DateTime32 it will return a ZonedDateTime with the time part. - * - * @param colName - * @return - */ - ZonedDateTime getZonedDateTime(String colName); - - /** - * Returns the value of the specified column as a Duration. - * - * If a stored value is bigger than Long.MAX_VALUE then exception will be thrown. In such case - * use asBigInteger() method. - * - * If value of IntervalQuarter then Duration will be in the unit of Months. - * - * @param colName - * @return Duration in the unit of column type. - */ - Duration getDuration(String colName); - - - /** - * Returns the value of the specified column as an Inet4Address. - * - * @param colName - * @return - */ - Inet4Address getInet4Address(String colName); - - /** - * Returns the value of the specified column as an Inet6Address. - * - * @param colName - * @return - */ - Inet6Address getInet6Address(String colName); - - /** - * Returns the value of the specified column as a UUID. - * - * @param colName - * @return - */ - UUID getUUID(String colName); - - /** - * Returns the value of the specified column as a ClickHouseGeoPointValue. - * - * @param colName - * @return - */ - ClickHouseGeoPointValue getGeoPoint(String colName); - - /** - * Returns the value of the specified column as a ClickHouseGeoRingValue. - * - * @param colName - * @return - */ - ClickHouseGeoRingValue getGeoRing(String colName); - - /** - * Returns the value of the specified column as a ClickHouseGeoPolygonValue. - * - * @param colName - * @return - */ - ClickHouseGeoPolygonValue getGeoPolygon(String colName); - - /** - * Returns the value of the specified column as a ClickHouseGeoMultiPolygonValue. - * - * @param colName - * @return - */ - ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(String colName); - - /** - * @see #getList(int) - * @param colName - column name - * @return list of values, or {@code null} if the value is null - */ - List getList(String colName); - - /** - * @see #getByteArray(int) - * @param colName - column name - * @return array of bytes, or {@code null} if the value is null - */ - byte[] getByteArray(String colName); - - /** - * @see #getIntArray(int) - * @param colName - column name - * @return array of int values, or {@code null} if the value is null - */ - int[] getIntArray(String colName); - - /** - * @see #getLongArray(int) - * @param colName - column name - * @return array of long values, or {@code null} if the value is null - */ - long[] getLongArray(String colName); - - /** - * @see #getFloatArray(int) - * @param colName - column name - * @return array of float values, or {@code null} if the value is null - */ - float[] getFloatArray(String colName); - - /** - * @see #getDoubleArray(int) - * @param colName - column name - * @return array of double values, or {@code null} if the value is null - */ - double[] getDoubleArray(String colName); - - /** - * @see #getBooleanArray(int) - * @param colName - column name - * @return array of boolean values, or {@code null} if the value is null - */ - boolean[] getBooleanArray(String colName); - - /** - * @see #getShortArray(int) - * @param colName - column name - * @return array of short values, or {@code null} if the value is null - */ - short[] getShortArray(String colName); - - /** - * @see #getStringArray(int) - * @param colName - column name - * @return array of string values, or {@code null} if the value is null - */ - String[] getStringArray(String colName); - - /** - * @see #getObjectArray(int) - * @param colName - column name - * @return array of objects, or {@code null} if the value is null - */ - Object[] getObjectArray(String colName); - - /** - * Reads column with name `colName` as a string. - * - * @param index - * @return - */ - String getString(int index); - - /** - * Reads column with name `colName` as a byte. - * - * @param index - * @return - */ - byte getByte(int index); - - /** - * Reads column with name `colName` as a short. - * - * @param index - * @return - */ - short getShort(int index); - - /** - * Reads column with name `colName` as an integer. - * - * @param index - * @return - */ - int getInteger(int index); - - /** - * Reads column with name `colName` as a long. - * - * @param index - * @return - */ - long getLong(int index); - - /** - * Reads column with name `colName` as a float. - * Warning: this method may lose precision for float values. - * - * @param index - * @return - */ - float getFloat(int index); - - /** - * Reads column with name `colName` as a double. - * Warning: this method may lose precision for double values. - * - * @param index - * @return - */ - double getDouble(int index); - - /** - * Reads column with name `colName` as a boolean. - * - * @param index - * @return - */ - boolean getBoolean(int index); - - /** - * Reads column with name `colName` as a BigInteger. - * - * @param index - * @return - */ - BigInteger getBigInteger(int index); - - /** - * Reads column with name `colName` as a BigDecimal. - * - * @param index - * @return - */ - BigDecimal getBigDecimal(int index); - - /** - * Returns the value of the specified column as an Instant. Timezone is derived from the column definition. - * If no timezone is specified in the column definition then UTC will be used. - * - * If column value is Date or Date32 it will return an Instant with time set to 00:00:00. - * If column value is DateTime or DateTime32 it will return an Instant with the time part. - * - * @param index - * @return - */ - Instant getInstant(int index); - - /** - * Returns the value of the specified column as a ZonedDateTime. Timezone is derived from the column definition. - * If no timezone is specified in the column definition then UTC will be used. - * - * If column value is Date or Date32 it will return a ZonedDateTime with time set to 00:00:00. - * If column value is DateTime or DateTime32 it will return a ZonedDateTime with the time part. - * - * @param index - * @return - */ - ZonedDateTime getZonedDateTime(int index); - - /** - * Returns the value of the specified column as a Duration. - * If a stored value is bigger than Long.MAX_VALUE then exception will be thrown. In such case - * use asBigInteger() method. - * If value of IntervalQuarter then Duration will be in the unit of Months. - * - * @param index - * @return Duration in the unit of column type. - */ - Duration getDuration(int index); - - - /** - * Returns the value of the specified column as an Inet4Address. - * - * @param index - * @return - */ - Inet4Address getInet4Address(int index); - - /** - * Returns the value of the specified column as an Inet6Address. - * - * @param index - * @return - */ - Inet6Address getInet6Address(int index); - - /** - * Returns the value of the specified column as a UUID. - * - * @param index - * @return - */ - UUID getUUID(int index); - - /** - * Returns the value of the specified column as a ClickHouseGeoPointValue. - * - * @param index - * @return - */ - ClickHouseGeoPointValue getGeoPoint(int index); - - /** - * Returns the value of the specified column as a ClickHouseGeoRingValue. - * - * @param index - * @return - */ - ClickHouseGeoRingValue getGeoRing(int index); - - /** - * Returns the value of the specified column as a ClickHouseGeoPolygonValue. - * - * @param index - * @return - */ - ClickHouseGeoPolygonValue getGeoPolygon(int index); - - /** - * Returns the value of the specified column as a ClickHouseGeoMultiPolygonValue. - * - * @param index - * @return - */ - ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(int index); - - /** - * Returns the value of the specified column as a {@link List}. Suitable for reading Array columns of any type. - *

For nested arrays (e.g. {@code Array(Array(Int64))}), returns a {@code List>}. - * For nullable arrays (e.g. {@code Array(Nullable(Int32))}), list elements may be {@code null}.

- * - * @param index - column index (1-based) - * @return list of values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the column is not an array type - */ - List getList(int index); - - /** - * Returns the value of the specified column as a {@code byte[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of bytes, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a byte array - */ - byte[] getByteArray(int index); - - /** - * Returns the value of the specified column as an {@code int[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of int values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to an int array - */ - int[] getIntArray(int index); - - /** - * Returns the value of the specified column as a {@code long[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of long values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a long array - */ - long[] getLongArray(int index); - - /** - * Returns the value of the specified column as a {@code float[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of float values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a float array - */ - float[] getFloatArray(int index); - - /** - * Returns the value of the specified column as a {@code double[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of double values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a double array - */ - double[] getDoubleArray(int index); - - /** - * Returns the value of the specified column as a {@code boolean[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of boolean values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a boolean array - */ - boolean[] getBooleanArray(int index); - - /** - * Returns the value of the specified column as a {@code short[]}. Suitable for 1D Array columns only. - * - * @param index - column index (1-based) - * @return array of short values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a short array - */ - short[] getShortArray(int index); - - /** - * Returns the value of the specified column as a {@code String[]}. Suitable for 1D Array columns only. - * Cannot be used for none string element types. - * - * @param index - column index (1-based) - * @return array of string values, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the column is not an array type - */ - String[] getStringArray(int index); - - /** - * Returns the value of the specified column as an {@code Object[]}. Suitable for multidimensional Array columns. - * Nested arrays are recursively converted to {@code Object[]}. - * Note: result is not cached so avoid repetitive calls on same column. - * - * @param index - column index (1-based) - * @return array of objects, or {@code null} if the value is null - * @throws com.clickhouse.client.api.ClientException if the column is not an array type - */ - Object[] getObjectArray(int index); - - Object[] getTuple(int index); - - Object[] getTuple(String colName); - - byte getEnum8(String colName); - - byte getEnum8(int index); - - short getEnum16(String colName); - - short getEnum16(int index); - - LocalDate getLocalDate(String colName); - - LocalDate getLocalDate(int index); - - LocalTime getLocalTime(String colName); - - LocalTime getLocalTime(int index); - - LocalDateTime getLocalDateTime(String colName); - - LocalDateTime getLocalDateTime(int index); - - OffsetDateTime getOffsetDateTime(String colName); - - OffsetDateTime getOffsetDateTime(int index); - - TableSchema getSchema(); - - ClickHouseBitmap getClickHouseBitmap(String colName); - - ClickHouseBitmap getClickHouseBitmap(int index); - - TemporalAmount getTemporalAmount(int index); - - TemporalAmount getTemporalAmount(String colName); +/** + * Reader for ClickHouse binary output formats (such as {@code Native}, + * {@code RowBinary}, {@code RowBinaryWithNames}, and + * {@code RowBinaryWithNamesAndTypes}). + * + *

Row navigation, schema access, and typed accessors are inherited from + * {@link ClickHouseFormatReader}; this interface specializes the contract for + * binary-encoded result streams and is the type returned by the binary + * factory methods on {@link com.clickhouse.client.api.Client}. Readers for + * text-oriented output formats (for example {@code JSONEachRow}) implement + * {@link ClickHouseTextFormatReader} instead.

+ * + *

Instances are produced by + * {@link com.clickhouse.client.api.Client#newBinaryFormatReader(com.clickhouse.client.api.query.QueryResponse)} + * and + * {@link com.clickhouse.client.api.Client#newBinaryFormatReader(com.clickhouse.client.api.query.QueryResponse, com.clickhouse.client.api.metadata.TableSchema)}.

+ */ +public interface ClickHouseBinaryFormatReader extends ClickHouseFormatReader { } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseFormatReader.java new file mode 100644 index 000000000..2464798fe --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseFormatReader.java @@ -0,0 +1,629 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.api.metadata.TableSchema; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoPointValue; +import com.clickhouse.data.value.ClickHouseGeoPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoRingValue; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +/** + * Common contract for row-by-row format readers regardless of the underlying + * wire encoding. Implementations are produced for binary formats (see + * {@link ClickHouseBinaryFormatReader}) and text formats (see + * {@link ClickHouseTextFormatReader}). + * + *

The methods declared here describe row navigation, schema access, and + * typed accessors by column name and 1-based column index. Concrete readers + * may not support every accessor for every format; unsupported accessors are + * expected to throw {@link UnsupportedOperationException}.

+ */ +public interface ClickHouseFormatReader extends AutoCloseable { + + /** + * Reads a single value from the stream. + * + * @param + * @return + */ + T readValue(int colIndex); + + /** + * Reads a row to an array of objects. + * + * @param colName + * @param + * @return + */ + T readValue(String colName); + + boolean hasValue(String colName); + + boolean hasValue(int colIndex); + + /** + * Checks if there are more rows to read. + * + * @return + */ + boolean hasNext(); + + /** + * Moves cursor to the next row. Must be called before reading the first row. Returns reference to + * an internal record representation. It means that next call to the method will affect value in returned Map. + * This is done for memory usage optimization. + * Method is intended to be used only by the client not an application. + * + * @return reference to a map filled with column values or null if no more records are available + */ + Map next(); + + /** + * Reads column with name `colName` as a string. + * + * @param colName - column name + * @return + */ + String getString(String colName); + + /** + * Reads column with name `colName` as a byte. + * + * @param colName - column name + * @return + */ + byte getByte(String colName); + + /** + * Reads column with name `colName` as a short. + * + * @param colName - column name + * @return + */ + short getShort(String colName); + + /** + * Reads column with name `colName` as an integer. + * + * @param colName - column name + * @return + */ + int getInteger(String colName); + + /** + * Reads column with name `colName` as a long. + * + * @param colName - column name + * @return + */ + long getLong(String colName); + + /** + * Reads column with name `colName` as a float. + * Warning: this method may lose precision for float values. + * + * @param colName + * @return + */ + float getFloat(String colName); + + /** + * Reads column with name `colName` as a double. + * Warning: this method may lose precision for double values. + * + * @param colName + * @return + */ + double getDouble(String colName); + + /** + * Reads column with name `colName` as a boolean. + * + * @param colName + * @return + */ + boolean getBoolean(String colName); + + /** + * Reads column with name `colName` as a BigInteger. + * + * @param colName + * @return + */ + BigInteger getBigInteger(String colName); + + /** + * Reads column with name `colName` as a BigDecimal. + * + * @param colName + * @return + */ + BigDecimal getBigDecimal(String colName); + + /** + * Returns the value of the specified column as an Instant. Timezone is derived from the column definition. + * If no timezone is specified in the column definition then UTC will be used. + * + * If column value is Date or Date32 it will return an Instant with time set to 00:00:00. + * If column value is DateTime or DateTime32 it will return an Instant with the time part. + * + * @param colName + * @return + */ + Instant getInstant(String colName); + + /** + * Returns the value of the specified column as a ZonedDateTime. Timezone is derived from the column definition. + * If no timezone is specified in the column definition then UTC will be used. + * + * If column value is Date or Date32 it will return a ZonedDateTime with time set to 00:00:00. + * If column value is DateTime or DateTime32 it will return a ZonedDateTime with the time part. + * + * @param colName + * @return + */ + ZonedDateTime getZonedDateTime(String colName); + + /** + * Returns the value of the specified column as a Duration. + * + * If a stored value is bigger than Long.MAX_VALUE then exception will be thrown. In such case + * use asBigInteger() method. + * + * If value of IntervalQuarter then Duration will be in the unit of Months. + * + * @param colName + * @return Duration in the unit of column type. + */ + Duration getDuration(String colName); + + + /** + * Returns the value of the specified column as an Inet4Address. + * + * @param colName + * @return + */ + Inet4Address getInet4Address(String colName); + + /** + * Returns the value of the specified column as an Inet6Address. + * + * @param colName + * @return + */ + Inet6Address getInet6Address(String colName); + + /** + * Returns the value of the specified column as a UUID. + * + * @param colName + * @return + */ + UUID getUUID(String colName); + + /** + * Returns the value of the specified column as a ClickHouseGeoPointValue. + * + * @param colName + * @return + */ + ClickHouseGeoPointValue getGeoPoint(String colName); + + /** + * Returns the value of the specified column as a ClickHouseGeoRingValue. + * + * @param colName + * @return + */ + ClickHouseGeoRingValue getGeoRing(String colName); + + /** + * Returns the value of the specified column as a ClickHouseGeoPolygonValue. + * + * @param colName + * @return + */ + ClickHouseGeoPolygonValue getGeoPolygon(String colName); + + /** + * Returns the value of the specified column as a ClickHouseGeoMultiPolygonValue. + * + * @param colName + * @return + */ + ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(String colName); + + /** + * @see #getList(int) + * @param colName - column name + * @return list of values, or {@code null} if the value is null + */ + List getList(String colName); + + /** + * @see #getByteArray(int) + * @param colName - column name + * @return array of bytes, or {@code null} if the value is null + */ + byte[] getByteArray(String colName); + + /** + * @see #getIntArray(int) + * @param colName - column name + * @return array of int values, or {@code null} if the value is null + */ + int[] getIntArray(String colName); + + /** + * @see #getLongArray(int) + * @param colName - column name + * @return array of long values, or {@code null} if the value is null + */ + long[] getLongArray(String colName); + + /** + * @see #getFloatArray(int) + * @param colName - column name + * @return array of float values, or {@code null} if the value is null + */ + float[] getFloatArray(String colName); + + /** + * @see #getDoubleArray(int) + * @param colName - column name + * @return array of double values, or {@code null} if the value is null + */ + double[] getDoubleArray(String colName); + + /** + * @see #getBooleanArray(int) + * @param colName - column name + * @return array of boolean values, or {@code null} if the value is null + */ + boolean[] getBooleanArray(String colName); + + /** + * @see #getShortArray(int) + * @param colName - column name + * @return array of short values, or {@code null} if the value is null + */ + short[] getShortArray(String colName); + + /** + * @see #getStringArray(int) + * @param colName - column name + * @return array of string values, or {@code null} if the value is null + */ + String[] getStringArray(String colName); + + /** + * @see #getObjectArray(int) + * @param colName - column name + * @return array of objects, or {@code null} if the value is null + */ + Object[] getObjectArray(String colName); + + /** + * Reads column with name `colName` as a string. + * + * @param index + * @return + */ + String getString(int index); + + /** + * Reads column with name `colName` as a byte. + * + * @param index + * @return + */ + byte getByte(int index); + + /** + * Reads column with name `colName` as a short. + * + * @param index + * @return + */ + short getShort(int index); + + /** + * Reads column with name `colName` as an integer. + * + * @param index + * @return + */ + int getInteger(int index); + + /** + * Reads column with name `colName` as a long. + * + * @param index + * @return + */ + long getLong(int index); + + /** + * Reads column with name `colName` as a float. + * Warning: this method may lose precision for float values. + * + * @param index + * @return + */ + float getFloat(int index); + + /** + * Reads column with name `colName` as a double. + * Warning: this method may lose precision for double values. + * + * @param index + * @return + */ + double getDouble(int index); + + /** + * Reads column with name `colName` as a boolean. + * + * @param index + * @return + */ + boolean getBoolean(int index); + + /** + * Reads column with name `colName` as a BigInteger. + * + * @param index + * @return + */ + BigInteger getBigInteger(int index); + + /** + * Reads column with name `colName` as a BigDecimal. + * + * @param index + * @return + */ + BigDecimal getBigDecimal(int index); + + /** + * Returns the value of the specified column as an Instant. Timezone is derived from the column definition. + * If no timezone is specified in the column definition then UTC will be used. + * + * If column value is Date or Date32 it will return an Instant with time set to 00:00:00. + * If column value is DateTime or DateTime32 it will return an Instant with the time part. + * + * @param index + * @return + */ + Instant getInstant(int index); + + /** + * Returns the value of the specified column as a ZonedDateTime. Timezone is derived from the column definition. + * If no timezone is specified in the column definition then UTC will be used. + * + * If column value is Date or Date32 it will return a ZonedDateTime with time set to 00:00:00. + * If column value is DateTime or DateTime32 it will return a ZonedDateTime with the time part. + * + * @param index + * @return + */ + ZonedDateTime getZonedDateTime(int index); + + /** + * Returns the value of the specified column as a Duration. + * If a stored value is bigger than Long.MAX_VALUE then exception will be thrown. In such case + * use asBigInteger() method. + * If value of IntervalQuarter then Duration will be in the unit of Months. + * + * @param index + * @return Duration in the unit of column type. + */ + Duration getDuration(int index); + + + /** + * Returns the value of the specified column as an Inet4Address. + * + * @param index + * @return + */ + Inet4Address getInet4Address(int index); + + /** + * Returns the value of the specified column as an Inet6Address. + * + * @param index + * @return + */ + Inet6Address getInet6Address(int index); + + /** + * Returns the value of the specified column as a UUID. + * + * @param index + * @return + */ + UUID getUUID(int index); + + /** + * Returns the value of the specified column as a ClickHouseGeoPointValue. + * + * @param index + * @return + */ + ClickHouseGeoPointValue getGeoPoint(int index); + + /** + * Returns the value of the specified column as a ClickHouseGeoRingValue. + * + * @param index + * @return + */ + ClickHouseGeoRingValue getGeoRing(int index); + + /** + * Returns the value of the specified column as a ClickHouseGeoPolygonValue. + * + * @param index + * @return + */ + ClickHouseGeoPolygonValue getGeoPolygon(int index); + + /** + * Returns the value of the specified column as a ClickHouseGeoMultiPolygonValue. + * + * @param index + * @return + */ + ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(int index); + + /** + * Returns the value of the specified column as a {@link List}. Suitable for reading Array columns of any type. + *

For nested arrays (e.g. {@code Array(Array(Int64))}), returns a {@code List>}. + * For nullable arrays (e.g. {@code Array(Nullable(Int32))}), list elements may be {@code null}.

+ * + * @param index - column index (1-based) + * @return list of values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the column is not an array type + */ + List getList(int index); + + /** + * Returns the value of the specified column as a {@code byte[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of bytes, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a byte array + */ + byte[] getByteArray(int index); + + /** + * Returns the value of the specified column as an {@code int[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of int values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to an int array + */ + int[] getIntArray(int index); + + /** + * Returns the value of the specified column as a {@code long[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of long values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a long array + */ + long[] getLongArray(int index); + + /** + * Returns the value of the specified column as a {@code float[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of float values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a float array + */ + float[] getFloatArray(int index); + + /** + * Returns the value of the specified column as a {@code double[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of double values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a double array + */ + double[] getDoubleArray(int index); + + /** + * Returns the value of the specified column as a {@code boolean[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of boolean values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a boolean array + */ + boolean[] getBooleanArray(int index); + + /** + * Returns the value of the specified column as a {@code short[]}. Suitable for 1D Array columns only. + * + * @param index - column index (1-based) + * @return array of short values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the value cannot be converted to a short array + */ + short[] getShortArray(int index); + + /** + * Returns the value of the specified column as a {@code String[]}. Suitable for 1D Array columns only. + * Cannot be used for none string element types. + * + * @param index - column index (1-based) + * @return array of string values, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the column is not an array type + */ + String[] getStringArray(int index); + + /** + * Returns the value of the specified column as an {@code Object[]}. Suitable for multidimensional Array columns. + * Nested arrays are recursively converted to {@code Object[]}. + * Note: result is not cached so avoid repetitive calls on same column. + * + * @param index - column index (1-based) + * @return array of objects, or {@code null} if the value is null + * @throws com.clickhouse.client.api.ClientException if the column is not an array type + */ + Object[] getObjectArray(int index); + + Object[] getTuple(int index); + + Object[] getTuple(String colName); + + byte getEnum8(String colName); + + byte getEnum8(int index); + + short getEnum16(String colName); + + short getEnum16(int index); + + LocalDate getLocalDate(String colName); + + LocalDate getLocalDate(int index); + + LocalTime getLocalTime(String colName); + + LocalTime getLocalTime(int index); + + LocalDateTime getLocalDateTime(String colName); + + LocalDateTime getLocalDateTime(int index); + + OffsetDateTime getOffsetDateTime(String colName); + + OffsetDateTime getOffsetDateTime(int index); + + TableSchema getSchema(); + + ClickHouseBitmap getClickHouseBitmap(String colName); + + ClickHouseBitmap getClickHouseBitmap(int index); + + TemporalAmount getTemporalAmount(int index); + + TemporalAmount getTemporalAmount(String colName); +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseTextFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseTextFormatReader.java new file mode 100644 index 000000000..399820e54 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseTextFormatReader.java @@ -0,0 +1,20 @@ +package com.clickhouse.client.api.data_formats; + +/** + * Reader for ClickHouse text output formats (such as + * {@code JSONEachRow}). + * + *

Row navigation, schema access, and typed accessors are inherited from + * {@link ClickHouseFormatReader}; this interface specializes the contract for + * text-encoded result streams

+ * + *

Implementation of a reader may not support every accessor declared on + * {@link ClickHouseFormatReader}; unsupported accessors are expected to throw + * {@link UnsupportedOperationException}.

+ */ +public interface ClickHouseTextFormatReader extends ClickHouseFormatReader { + + String currentRowAsString(); + + Object currentRowAsObject(); +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/GsonJsonParserFactory.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/GsonJsonParserFactory.java new file mode 100644 index 000000000..603cb3078 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/GsonJsonParserFactory.java @@ -0,0 +1,64 @@ +package com.clickhouse.client.api.data_formats; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; +import com.google.gson.reflect.TypeToken; +import com.google.gson.stream.JsonReader; +import com.google.gson.stream.JsonToken; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class GsonJsonParserFactory implements JsonParserFactory { + private final Gson gson; + + public GsonJsonParserFactory() { + GsonBuilder builder = new GsonBuilder(); + customize(builder); + builder.setLenient(); // JSONEachRow needs lenient reader for multiple root objects + this.gson = builder.create(); + } + + protected void customize(GsonBuilder builder) { + // JSONEachRow numbers may represent UInt64 or Decimal values, so avoid + // Gson's default Double materialization and preserve the original value. + builder.setObjectToNumberStrategy(ToNumberPolicy.BIG_DECIMAL); + } + + @Override + public JsonParser createJsonParser(InputStream in) { + return new JsonParserImpl(gson.newJsonReader(new InputStreamReader(in, StandardCharsets.UTF_8))); + } + + private class JsonParserImpl implements JsonParser { + + private final JsonReader reader; + + public JsonParserImpl(JsonReader jsonReader) { + this.reader = jsonReader; + } + + + @Override + public Map nextRow() throws Exception { + try { + if (reader.peek() == JsonToken.END_DOCUMENT) { + return null; + } + } catch (java.io.EOFException e) { + return null; + } + + return GsonJsonParserFactory.this.gson.fromJson(reader, new TypeToken>() { + }.getType()); + } + + @Override + public void close() throws Exception { + reader.close(); + } + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java new file mode 100644 index 000000000..0f9fa444a --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReader.java @@ -0,0 +1,646 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.api.ClientException; +import com.clickhouse.client.api.data_formats.internal.NumberConverter; +import com.clickhouse.client.api.internal.SchemaUtils; +import com.clickhouse.client.api.metadata.TableSchema; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.value.ClickHouseBitmap; +import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoPointValue; +import com.clickhouse.data.value.ClickHouseGeoPolygonValue; +import com.clickhouse.data.value.ClickHouseGeoRingValue; + +import java.lang.reflect.Array; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.time.temporal.TemporalAmount; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class JSONEachRowFormatReader implements ClickHouseTextFormatReader { + private final JsonParser parser; + private TableSchema schema; + private Map currentRow; + private Map nextRow; + private boolean hasNext; + + public JSONEachRowFormatReader(JsonParser parser) { + this.parser = parser; + try { + this.nextRow = parser.nextRow(); + this.hasNext = this.nextRow != null; + if (nextRow != null) { + List columns = new ArrayList<>(); + for (String key : nextRow.keySet()) { + // For JSONEachRow we don't know the exact ClickHouse type, so we use a reasonable default. + // We can try to guess based on the value type in the first row. + columns.add(ClickHouseColumn.of(key, SchemaUtils.inferDataType(nextRow.get(key)), false)); + } + this.schema = new TableSchema(columns); + } else { + this.schema = new TableSchema(new ArrayList<>()); + } + } catch (Exception e) { + throw new RuntimeException("Failed to initialize JSON reader", e); + } + } + + @Override + public T readValue(int colIndex) { + return (T) currentRow.get(schema.columnIndexToName(colIndex)); + } + + @Override + public T readValue(String colName) { + return (T) currentRow.get(colName); + } + + @Override + public boolean hasValue(String colName) { + return currentRow.containsKey(colName) && currentRow.get(colName) != null; + } + + @Override + public boolean hasValue(int colIndex) { + return hasValue(schema.columnIndexToName(colIndex)); + } + + @Override + public boolean hasNext() { + return hasNext; + } + + @Override + public Map next() { + if (!hasNext) { + currentRow = null; + return null; + } + + currentRow = nextRow; + readNextRow(); + return currentRow; + } + + private void readNextRow() { + try { + nextRow = parser.nextRow(); + hasNext = nextRow != null; + } catch (Exception e) { + hasNext = false; + nextRow = null; + throw new RuntimeException("Failed to read next JSON row", e); + } + } + + @Override + public String currentRowAsString() { + throw new UnsupportedOperationException(); + } + + @Override + public Object currentRowAsObject() { + throw new UnsupportedOperationException(); + } + + @Override + public String getString(String colName) { + Object val = currentRow.get(colName); + return val == null ? null : val.toString(); + } + + @Override + public byte getByte(String colName) { + return ((Number) currentRow.get(colName)).byteValue(); + } + + @Override + public short getShort(String colName) { + return ((Number) currentRow.get(colName)).shortValue(); + } + + @Override + public int getInteger(String colName) { + return ((Number) currentRow.get(colName)).intValue(); + } + + @Override + public long getLong(String colName) { + return ((Number) currentRow.get(colName)).longValue(); + } + + @Override + public float getFloat(String colName) { + return ((Number) currentRow.get(colName)).floatValue(); + } + + @Override + public double getDouble(String colName) { + return ((Number) currentRow.get(colName)).doubleValue(); + } + + @Override + public boolean getBoolean(String colName) { + Object val = currentRow.get(colName); + if (val instanceof Boolean) { + return (Boolean) val; + } + if (val instanceof Number) { + // Match AbstractBinaryFormatReader (SerializerUtils.convertToBoolean): + // any non-zero integral value is true, zero is false. Fractional + // values keep the same behavior because they are truncated by + // longValue() before the zero check. + return ((Number) val).longValue() != 0; + } + if (val == null) { + throw new ClientException("Column '" + colName + "' has null value and cannot be converted to boolean"); + } + throw new ClientException("Cannot convert value of type " + val.getClass().getName() + + " in column '" + colName + "' to boolean"); + } + + @Override + public BigInteger getBigInteger(String colName) { + Object val = currentRow.get(colName); + if (val == null) return null; + if (val instanceof BigInteger) return (BigInteger) val; + return new BigDecimal(val.toString()).toBigInteger(); + } + + @Override + public BigDecimal getBigDecimal(String colName) { + Object val = currentRow.get(colName); + if (val == null) return null; + if (val instanceof BigDecimal) return (BigDecimal) val; + return new BigDecimal(val.toString()); + } + + @Override + public Instant getInstant(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ZonedDateTime getZonedDateTime(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Duration getDuration(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Inet4Address getInet4Address(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public Inet6Address getInet6Address(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public UUID getUUID(String colName) { + return UUID.fromString(currentRow.get(colName).toString()); + } + + @Override + public ClickHouseGeoPointValue getGeoPoint(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoRingValue getGeoRing(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoPolygonValue getGeoPolygon(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public List getList(String colName) { + Object val = currentRow.get(colName); + if (val == null) { + return null; + } + if (!(val instanceof List)) { + throw new ClientException("Column '" + colName + "' is not of array type (actual: " + + val.getClass().getName() + ")"); + } + return (List) val; + } + + @Override + public byte[] getByteArray(String colName) { + return getPrimitiveArray(colName, byte.class); + } + + @Override + public int[] getIntArray(String colName) { + return getPrimitiveArray(colName, int.class); + } + + @Override + public long[] getLongArray(String colName) { + return getPrimitiveArray(colName, long.class); + } + + @Override + public float[] getFloatArray(String colName) { + return getPrimitiveArray(colName, float.class); + } + + @Override + public double[] getDoubleArray(String colName) { + return getPrimitiveArray(colName, double.class); + } + + @Override + public boolean[] getBooleanArray(String colName) { + return getPrimitiveArray(colName, boolean.class); + } + + @Override + public short[] getShortArray(String colName) { + return getPrimitiveArray(colName, short.class); + } + + @Override + public String[] getStringArray(String colName) { + List list = asArrayList(colName); + if (list == null) { + return null; + } + String[] out = new String[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object el = list.get(i); + out[i] = el == null ? null : el.toString(); + } + return out; + } + + @Override + public Object[] getObjectArray(String colName) { + List list = asArrayList(colName); + return list == null ? null : list.toArray(new Object[0]); + } + + /** + * Returns the value of the given column as a {@code List}, or {@code null} + * if the value is missing. Throws {@link ClientException} when the column + * exists but is not an array. + */ + private List asArrayList(String colName) { + Object val = currentRow.get(colName); + if (val == null) { + return null; + } + if (!(val instanceof List)) { + throw new ClientException("Column '" + colName + "' is not of array type (actual: " + + val.getClass().getName() + ")"); + } + return (List) val; + } + + @SuppressWarnings("unchecked") + private T getPrimitiveArray(String colName, Class componentType) { + List list = asArrayList(colName); + if (list == null) { + return null; + } + try { + Object array = Array.newInstance(componentType, list.size()); + for (int i = 0; i < list.size(); i++) { + Object el = list.get(i); + if (el == null) { + throw new ClientException("Column '" + colName + + "' contains a null element which cannot fit into an array of primitive " + + componentType.getName()); + } + Array.set(array, i, coerceToComponent(el, componentType)); + } + return (T) array; + } catch (ClassCastException | IllegalArgumentException e) { + throw new ClientException("Value of column '" + colName + + "' cannot be converted to an array of " + componentType.getName(), e); + } + } + + /** + * Coerces a parsed JSON element to a boxed primitive type. JSON parsers + * may materialize numeric array elements as different boxed types + * (e.g. {@code Integer}, {@code Long}, {@code Double}, {@code BigDecimal}), + * so element-level conversion is necessary before populating a typed + * primitive array. The {@code componentType} is always one of the eight + * Java primitives passed by {@link #getPrimitiveArray}; unsupported + * component types are rejected explicitly to keep the helper total. + */ + private static Object coerceToComponent(Object value, Class componentType) { + if (componentType == byte.class) { + return NumberConverter.toByte(value); + } + if (componentType == short.class) { + return NumberConverter.toShort(value); + } + if (componentType == int.class) { + return NumberConverter.toInt(value); + } + if (componentType == long.class) { + return NumberConverter.toLong(value); + } + if (componentType == float.class) { + return NumberConverter.toFloat(value); + } + if (componentType == double.class) { + return NumberConverter.toDouble(value); + } + if (componentType == boolean.class) { + if (value instanceof Boolean) { + return value; + } + if (value instanceof Number) { + return ((Number) value).longValue() != 0; + } + throw new IllegalArgumentException( + "Cannot convert " + value.getClass().getName() + " to boolean array element"); + } + throw new IllegalArgumentException("Unsupported component type: " + componentType.getName()); + } + + @Override + public String getString(int index) { + return getString(schema.columnIndexToName(index)); + } + + @Override + public byte getByte(int index) { + return getByte(schema.columnIndexToName(index)); + } + + @Override + public short getShort(int index) { + return getShort(schema.columnIndexToName(index)); + } + + @Override + public int getInteger(int index) { + return getInteger(schema.columnIndexToName(index)); + } + + @Override + public long getLong(int index) { + return getLong(schema.columnIndexToName(index)); + } + + @Override + public float getFloat(int index) { + return getFloat(schema.columnIndexToName(index)); + } + + @Override + public double getDouble(int index) { + return getDouble(schema.columnIndexToName(index)); + } + + @Override + public boolean getBoolean(int index) { + return getBoolean(schema.columnIndexToName(index)); + } + + @Override + public BigInteger getBigInteger(int index) { + return getBigInteger(schema.columnIndexToName(index)); + } + + @Override + public BigDecimal getBigDecimal(int index) { + return getBigDecimal(schema.columnIndexToName(index)); + } + + @Override + public Instant getInstant(int index) { + return getInstant(schema.columnIndexToName(index)); + } + + @Override + public ZonedDateTime getZonedDateTime(int index) { + return getZonedDateTime(schema.columnIndexToName(index)); + } + + @Override + public Duration getDuration(int index) { + return getDuration(schema.columnIndexToName(index)); + } + + @Override + public Inet4Address getInet4Address(int index) { + return getInet4Address(schema.columnIndexToName(index)); + } + + @Override + public Inet6Address getInet6Address(int index) { + return getInet6Address(schema.columnIndexToName(index)); + } + + @Override + public UUID getUUID(int index) { + return getUUID(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoPointValue getGeoPoint(int index) { + return getGeoPoint(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoRingValue getGeoRing(int index) { + return getGeoRing(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoPolygonValue getGeoPolygon(int index) { + return getGeoPolygon(schema.columnIndexToName(index)); + } + + @Override + public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(int index) { + return getGeoMultiPolygon(schema.columnIndexToName(index)); + } + + @Override + public List getList(int index) { + return getList(schema.columnIndexToName(index)); + } + + @Override + public byte[] getByteArray(int index) { + return getByteArray(schema.columnIndexToName(index)); + } + + @Override + public int[] getIntArray(int index) { + return getIntArray(schema.columnIndexToName(index)); + } + + @Override + public long[] getLongArray(int index) { + return getLongArray(schema.columnIndexToName(index)); + } + + @Override + public float[] getFloatArray(int index) { + return getFloatArray(schema.columnIndexToName(index)); + } + + @Override + public double[] getDoubleArray(int index) { + return getDoubleArray(schema.columnIndexToName(index)); + } + + @Override + public boolean[] getBooleanArray(int index) { + return getBooleanArray(schema.columnIndexToName(index)); + } + + @Override + public short[] getShortArray(int index) { + return getShortArray(schema.columnIndexToName(index)); + } + + @Override + public String[] getStringArray(int index) { + return getStringArray(schema.columnIndexToName(index)); + } + + @Override + public Object[] getObjectArray(int index) { + return getObjectArray(schema.columnIndexToName(index)); + } + + @Override + public Object[] getTuple(int index) { + return getTuple(schema.columnIndexToName(index)); + } + + @Override + public Object[] getTuple(String colName) { + Object value = currentRow.get(colName); + if (value == null) { + return null; + } + if (value instanceof List) { + return ((List) value).toArray(new Object[0]); + } + return (Object[]) value; + } + + @Override + public byte getEnum8(String colName) { + return getByte(colName); + } + + @Override + public byte getEnum8(int index) { + return getByte(index); + } + + @Override + public short getEnum16(String colName) { + return getShort(colName); + } + + @Override + public short getEnum16(int index) { + return getShort(index); + } + + @Override + public LocalDate getLocalDate(String colName) { + return LocalDate.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalDate getLocalDate(int index) { + return getLocalDate(schema.columnIndexToName(index)); + } + + @Override + public LocalTime getLocalTime(String colName) { + return LocalTime.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalTime getLocalTime(int index) { + return getLocalTime(schema.columnIndexToName(index)); + } + + @Override + public LocalDateTime getLocalDateTime(String colName) { + return LocalDateTime.parse(currentRow.get(colName).toString()); + } + + @Override + public LocalDateTime getLocalDateTime(int index) { + return getLocalDateTime(schema.columnIndexToName(index)); + } + + @Override + public OffsetDateTime getOffsetDateTime(String colName) { + return OffsetDateTime.parse(currentRow.get(colName).toString()); + } + + @Override + public OffsetDateTime getOffsetDateTime(int index) { + return getOffsetDateTime(schema.columnIndexToName(index)); + } + + @Override + public TableSchema getSchema() { + return schema; + } + + @Override + public ClickHouseBitmap getClickHouseBitmap(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public ClickHouseBitmap getClickHouseBitmap(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public TemporalAmount getTemporalAmount(int index) { + throw new UnsupportedOperationException(); + } + + @Override + public TemporalAmount getTemporalAmount(String colName) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws Exception { + parser.close(); + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JacksonJsonParserFactory.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JacksonJsonParserFactory.java new file mode 100644 index 000000000..0730be9c0 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JacksonJsonParserFactory.java @@ -0,0 +1,51 @@ +package com.clickhouse.client.api.data_formats; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Map; + +public class JacksonJsonParserFactory implements JsonParserFactory { + private final ObjectMapper mapper; + + public JacksonJsonParserFactory() { + this.mapper = createMapper(); + } + + protected ObjectMapper createMapper() { + // override this method to customize object mapper + return new ObjectMapper(); + } + + @Override + public JsonParser createJsonParser(InputStream in) throws IOException { + return new JsonParserImpl(mapper.createParser(in)); + } + + private class JsonParserImpl implements JsonParser { + + private final com.fasterxml.jackson.core.JsonParser parser; + + public JsonParserImpl(com.fasterxml.jackson.core.JsonParser parser) { + this.parser = parser; + } + + @Override + public Map nextRow() throws Exception { + // Jackson's streaming parser skips whitespace (including the newlines that + // separate JSONEachRow objects), so reaching EOF is the only reason + // nextToken() returns null. Any non-START_OBJECT token here would indicate + // malformed input and is reported by mapper.readValue(...). + if (parser.nextToken() == null) { + return null; + } + return mapper.readValue(parser, Map.class); + } + + @Override + public void close() throws Exception { + parser.close(); + } + } +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParser.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParser.java new file mode 100644 index 000000000..e0a271b44 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParser.java @@ -0,0 +1,17 @@ +package com.clickhouse.client.api.data_formats; + +import java.util.Map; + +/** + * Interface for JSON row processors. + */ +public interface JsonParser extends AutoCloseable { + + + /** + * Reads next row from the input stream. + * @return map of column names to values, or null if no more rows + * @throws Exception if an error occurs during parsing + */ + Map nextRow() throws Exception; +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParserFactory.java b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParserFactory.java new file mode 100644 index 000000000..94bd3e5c9 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/data_formats/JsonParserFactory.java @@ -0,0 +1,17 @@ +package com.clickhouse.client.api.data_formats; + +import java.io.IOException; +import java.io.InputStream; + +public interface JsonParserFactory { + + + /** + * Implementation should create only instance of actual JSON parser. + * This method is called for each request and should avoid long initialization or + * create big objects + * @param in - stream of bytes to parse as JSON + * @return instance of {@link JsonParser} + */ + JsonParser createJsonParser(InputStream in) throws IOException; +} diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/SchemaUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/SchemaUtils.java new file mode 100644 index 000000000..ed0ce67b6 --- /dev/null +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/SchemaUtils.java @@ -0,0 +1,87 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.data.ClickHouseDataType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public final class SchemaUtils { + private static final ClickHouseDataType DEFAULT_DATA_TYPE = ClickHouseDataType.String; + + private static final List DATA_TYPE_PRIORITY = ImmutableList.of( + ClickHouseDataType.Bool, + ClickHouseDataType.Int8, + ClickHouseDataType.Int16, + ClickHouseDataType.Int32, + ClickHouseDataType.Int64, + ClickHouseDataType.Int256, + ClickHouseDataType.Float32, + ClickHouseDataType.Float64, + ClickHouseDataType.Decimal, + ClickHouseDataType.String, + ClickHouseDataType.UUID, + ClickHouseDataType.IPv4, + ClickHouseDataType.IPv6, + ClickHouseDataType.DateTime64, + ClickHouseDataType.Date, + ClickHouseDataType.IntervalNanosecond, + ClickHouseDataType.IntervalDay, + ClickHouseDataType.Time64, + ClickHouseDataType.Point, + ClickHouseDataType.Ring, + ClickHouseDataType.Polygon, + ClickHouseDataType.MultiPolygon, + ClickHouseDataType.Tuple, + ClickHouseDataType.Geometry + ); + + private static final Map, ClickHouseDataType> CLASS_TO_DATA_TYPE = buildClassToDataTypeMap(); + + private SchemaUtils() { + } + + public static ClickHouseDataType inferDataType(Object value) { + if (value == null) { + return DEFAULT_DATA_TYPE; + } + + // JSONEachRow has no type metadata, so structural values only infer the + // top-level family; nested key/value/element types remain best-effort. + if (value instanceof Map) { + return ClickHouseDataType.Map; + } + + Class valueClass = ClickHouseDataType.toObjectType(value.getClass()); + if (value instanceof List || valueClass.isArray()) { + return ClickHouseDataType.Array; + } + + ClickHouseDataType dataType = CLASS_TO_DATA_TYPE.get(valueClass); + return dataType == null ? DEFAULT_DATA_TYPE : dataType; + } + + private static Map, ClickHouseDataType> buildClassToDataTypeMap() { + Map, ClickHouseDataType> map = new LinkedHashMap<>(); + + for (ClickHouseDataType dataType : DATA_TYPE_PRIORITY) { + addTypeMappings(map, dataType); + } + + return ImmutableMap.copyOf(map); + } + + private static void addTypeMappings(Map, ClickHouseDataType> map, ClickHouseDataType dataType) { + Set> javaClasses = ClickHouseDataType.DATA_TYPE_TO_CLASS.get(dataType); + if (javaClasses == null) { + return; + } + + for (Class javaClass : javaClasses) { + map.putIfAbsent(ClickHouseDataType.toObjectType(javaClass), dataType); + } + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index eaa675349..a308d8949 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -332,7 +332,7 @@ public void testDefaultSettings() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 34); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 35); // to check everything is set. Increment when new added. } try (Client client = new Client.Builder() @@ -365,7 +365,7 @@ public void testDefaultSettings() { .setSocketSndbuf(100000) .build()) { Map config = client.getConfiguration(); - Assert.assertEquals(config.size(), 35); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 36); // to check everything is set. Increment when new added. Assert.assertEquals(config.get(ClientConfigProperties.DATABASE.getKey()), "mydb"); Assert.assertEquals(config.get(ClientConfigProperties.MAX_EXECUTION_TIME.getKey()), "10"); Assert.assertEquals(config.get(ClientConfigProperties.COMPRESSION_LZ4_UNCOMPRESSED_BUF_SIZE.getKey()), "300000"); @@ -432,7 +432,7 @@ public void testWithOldDefaults() { Assert.assertEquals(config.get(p.getKey()), p.getDefaultValue(), "Default value doesn't match"); } } - Assert.assertEquals(config.size(), 34); // to check everything is set. Increment when new added. + Assert.assertEquals(config.size(), 35); // to check everything is set. Increment when new added. } } diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..a747f5bf1 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java @@ -0,0 +1,1177 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseFormat; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Random; +import java.util.UUID; +import java.util.function.BiFunction; +import java.util.function.BiPredicate; +import java.util.function.Consumer; + +public abstract class AbstractJSONEachRowFormatReaderTests extends BaseIntegrationTest { + + /** Number of rows generated per primitive column. */ + private static final int ROW_COUNT = 5; + + /** Fixed seed so generated random values are stable across runs and parser implementations. */ + private static final long RANDOM_SEED = 0xC0FFEEL; + + /** Shared list of primitive cases, populated once per JVM and reused by every test class. */ + private static final List PRIMITIVE_CASES = + buildPrimitiveCases(new Random(RANDOM_SEED)); + + protected Client client; + + private String primitivesTable; + + @BeforeClass(groups = {"integration"}) + public void setUpPrimitivesTable() throws Exception { + primitivesTable = "test_json_each_row_primitives_" + + getClass().getSimpleName().toLowerCase(Locale.ROOT); + + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + try (Client setupClient = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isCloud()) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .build()) { + + setupClient.execute("DROP TABLE IF EXISTS " + primitivesTable).get().close(); + + StringBuilder create = new StringBuilder("CREATE TABLE ") + .append(primitivesTable) + .append(" (id UInt32"); + for (PrimitiveTypeCase c : PRIMITIVE_CASES) { + create.append(", ").append(c.columnName).append(' ').append(c.chType); + } + create.append(") ENGINE = MergeTree ORDER BY id"); + setupClient.execute(create.toString()).get().close(); + + StringBuilder insert = new StringBuilder("INSERT INTO ") + .append(primitivesTable).append(" VALUES "); + for (int row = 0; row < ROW_COUNT; row++) { + if (row > 0) { + insert.append(", "); + } + insert.append('(').append(row); + for (PrimitiveTypeCase c : PRIMITIVE_CASES) { + insert.append(", ").append(c.sqlLiterals.get(row)); + } + insert.append(')'); + } + setupClient.execute(insert.toString()).get().close(); + } + } + + @AfterClass(groups = {"integration"}) + public void tearDownPrimitivesTable() throws Exception { + if (primitivesTable == null) { + return; + } + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + try (Client teardownClient = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isCloud()) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .build()) { + teardownClient.execute("DROP TABLE IF EXISTS " + primitivesTable).get().close(); + } + } + + @BeforeMethod(groups = {"integration"}) + public void setUp() { + ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); + client = new Client.Builder() + .addEndpoint(Protocol.HTTP, node.getHost(), node.getPort(), isCloud()) + .setUsername("default") + .setPassword(ClickHouseServerForTest.getPassword()) + .build(); + } + + @AfterMethod(groups = {"integration"}) + public void tearDown() { + if (client != null) { + client.close(); + } + } + + private QuerySettings newJsonEachRowSettings() { + return new QuerySettings() + .setFormat(ClickHouseFormat.JSONEachRow); + } + + /** + * Settings used by the primitive accessor tests. Integer and floating-point values + * are returned unquoted so they materialise as {@code Number} instances (whose + * {@code longValue}/{@code doubleValue} calls match the typed accessors). + * Decimal values are kept quoted (the ClickHouse default) so that JSON parsers + * that materialise unquoted JSON numbers as {@code Double} (e.g. Jackson with + * default settings) do not lose precision on large {@code Decimal} values. + */ + private QuerySettings newJsonEachRowSettingsForPrimitives() { + return newJsonEachRowSettings() + .serverSetting("output_format_json_quote_64bit_integers", "0") + .serverSetting("output_format_json_quote_64bit_floats", "0") + .serverSetting("output_format_json_quote_decimals", "1"); + } + + protected abstract ClickHouseTextFormatReader createReader(QueryResponse response) throws IOException; + + /** + * Builds a reader directly over the provided JSONEachRow byte stream, bypassing + * the server. This is used to exercise error paths (such as corrupted input) + * deterministically across both parser factories. + */ + protected abstract ClickHouseTextFormatReader createReader(InputStream input) throws IOException; + + // ------------------------------------------------------------------ + // Parameterized primitive value tests + // ------------------------------------------------------------------ + + @DataProvider(name = "primitiveTypeCases") + public Object[][] primitiveTypeCases() { + Object[][] rows = new Object[PRIMITIVE_CASES.size()][1]; + for (int i = 0; i < PRIMITIVE_CASES.size(); i++) { + rows[i][0] = PRIMITIVE_CASES.get(i); + } + return rows; + } + + /** + * Runs the same assertions for every primitive type by reading the column populated + * in {@link #setUpPrimitivesTable()}. For each row it asserts that the value can be + * read via the type-appropriate accessor (both by name and by index) and that a set + * of accessors that cannot convert the value throw an exception. + */ + @Test(groups = {"integration"}, dataProvider = "primitiveTypeCases") + public void testPrimitiveTypeAccessors(PrimitiveTypeCase tc) throws Exception { + String sql = "SELECT " + tc.columnName + " FROM " + primitivesTable + " ORDER BY id"; + + try (QueryResponse response = + client.query(sql, newJsonEachRowSettingsForPrimitives()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue(reader.hasNext(), + "missing row " + row + " for " + tc); + Assert.assertNotNull(reader.next(), + "null row " + row + " for " + tc); + + Object expected = tc.expectedValues.get(row); + + Object actualByName = tc.readByName.apply(reader, tc.columnName); + Assert.assertTrue(tc.equality.test(actualByName, expected), + "row " + row + " by name for " + tc + + ": expected=" + expected + ", actual=" + actualByName); + + Object actualByIndex = tc.readByIndex.apply(reader, 1); + Assert.assertTrue(tc.equality.test(actualByIndex, expected), + "row " + row + " by index for " + tc + + ": expected=" + expected + ", actual=" + actualByIndex); + } + + Assert.assertFalse(reader.hasNext(), "extra rows for " + tc); + Assert.assertNull(reader.next(), "extra row payload for " + tc); + } + + try (QueryResponse response = + client.query(sql, newJsonEachRowSettingsForPrimitives()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertNotNull(reader.next(), "row needed for incompatibility checks: " + tc); + + for (IncompatibleAccessor accessor : tc.incompatibleAccessors) { + assertAccessorThrows(reader, accessor.byName, + accessor.name + " by name on " + tc); + assertAccessorThrows(reader, accessor.byIndex, + accessor.name + " by index on " + tc); + } + } + } + + private static void assertAccessorThrows(ClickHouseTextFormatReader reader, + Consumer call, + String context) { + try { + call.accept(reader); + Assert.fail("Expected exception when invoking " + context); + } catch (RuntimeException expected) { + // Any RuntimeException is acceptable - DateTimeParseException, + // NumberFormatException, ClassCastException, IllegalArgumentException + // and UnsupportedOperationException are all valid signals that the + // accessor cannot convert the stored value. + } + } + + // ------------------------------------------------------------------ + // Remaining non-value-focused tests + // ------------------------------------------------------------------ + + @Test(groups = {"integration"}) + public void testSchemaInference() throws Exception { + // Numeric inference depends on parser materialization, so this test checks + // that numerics do not collapse to String and stable scalar types still map. + String sql = "SELECT toInt64(42) as col_int, toFloat64(3.14) as col_float, " + + "true as col_bool, 'val' as col_str"; + + try (QueryResponse response = client.query(sql, newJsonEachRowSettings()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertNotNull(reader.getSchema()); + Assert.assertEquals(reader.getSchema().getColumns().size(), 4); + + Assert.assertNotEquals(reader.getSchema().getColumnByIndex(1).getDataType(), ClickHouseDataType.String); + Assert.assertNotEquals(reader.getSchema().getColumnByIndex(2).getDataType(), ClickHouseDataType.String); + Assert.assertEquals(reader.getSchema().getColumnByIndex(3).getDataType(), ClickHouseDataType.Bool); + Assert.assertEquals(reader.getSchema().getColumnByIndex(4).getDataType(), ClickHouseDataType.String); + } + } + + @Test(groups = {"integration"}) + public void testEmptyData() throws Exception { + String sql = "SELECT * FROM remote('127.0.0.1', system.one) WHERE dummy > 1"; + + try (QueryResponse response = client.query(sql, newJsonEachRowSettings()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertFalse(reader.hasNext()); + Assert.assertNull(reader.next()); + Assert.assertEquals(reader.getSchema().getColumns().size(), 0); + } + } + + @Test(groups = {"integration"}) + public void testReadValueAndHasValue() throws Exception { + String sql = "SELECT 7 as id, 'abc' as name, CAST(NULL AS Nullable(String)) as missing"; + + try (QueryResponse response = client.query(sql, newJsonEachRowSettings()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + reader.next(); + + Number id = reader.readValue("id"); + Assert.assertNotNull(id); + Assert.assertEquals(id.intValue(), 7); + Assert.assertEquals((String) reader.readValue(2), "abc"); + + Assert.assertTrue(reader.hasValue("id")); + Assert.assertTrue(reader.hasValue(2)); + Assert.assertFalse(reader.hasValue("missing")); + Assert.assertFalse(reader.hasValue(3)); + Assert.assertFalse(reader.hasValue("not_a_column")); + } + } + + @Test(groups = {"integration"}) + public void testListAccessor() throws Exception { + String sql = "SELECT [1, 2, 3] as arr"; + + try (QueryResponse response = client.query(sql, newJsonEachRowSettings()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + reader.next(); + + List values = reader.getList("arr"); + Assert.assertNotNull(values); + Assert.assertEquals(values.size(), 3); + Assert.assertEquals(values.get(0).intValue(), 1); + Assert.assertEquals(values.get(1).intValue(), 2); + Assert.assertEquals(values.get(2).intValue(), 3); + + List byIndex = reader.getList(1); + Assert.assertNotNull(byIndex); + Assert.assertEquals(byIndex.size(), 3); + } + } + + @Test(groups = {"integration"}, expectedExceptions = IllegalArgumentException.class) + public void testNewBinaryFormatReaderRejectsJsonEachRow() throws Exception { + String sql = "SELECT 1 as id"; + + try (QueryResponse response = client.query(sql, newJsonEachRowSettings()).get()) { + client.newBinaryFormatReader(response); + } + } + + // ------------------------------------------------------------------ + // Corrupted-stream coverage for the readNextRow error path + // ------------------------------------------------------------------ + + /** + * Builds a reader directly over a hand-crafted JSONEachRow stream that + * contains a valid first object followed by malformed bytes. The first + * call to {@code next()} buffered row #1 in the constructor and then + * tries to read row #2, which is malformed; the reader must surface a + * {@link RuntimeException} and refuse to advance any further. + * + *

This is structured as a unit-style test so it does not depend on the + * server, but it stays in the {@code integration} group alongside the + * other reader tests because it is parser-factory specific.

+ */ + @Test(groups = {"integration"}) + public void testCorruptedStreamFailsReadNextRow() throws Exception { + // The third "row" deliberately contains an unterminated string after + // a partial object so that lenient parsers cannot quietly accept it. + // The newline separation matches real JSONEachRow framing. + String body = "{\"id\":1,\"name\":\"first\"}\n" + + "{\"id\":2,\"name\":\"second\"}\n" + + "{\"id\":3,\"name\":\"unterminated"; + + try (InputStream input = new ByteArrayInputStream(body.getBytes(StandardCharsets.UTF_8)); + ClickHouseTextFormatReader reader = createReader(input)) { + + Assert.assertTrue(reader.hasNext(), "first row should be buffered"); + + Assert.assertNotNull(reader.next(), "row 1 should be readable"); + Assert.assertEquals(reader.getString("name"), "first"); + + try { + reader.next(); + Assert.fail("Expected RuntimeException reading malformed row"); + } catch (RuntimeException expected) { + // any RuntimeException is acceptable - both Jackson and Gson + // surface different concrete exception types here, but the + // reader wraps them into a RuntimeException. + } + + Assert.assertFalse(reader.hasNext(), + "reader must report end-of-stream after a parse failure"); + Assert.assertNull(reader.next(), + "next() must return null after a parse failure, not retry the stream"); + } + } + + // ------------------------------------------------------------------ + // Boolean accessor: numeric -> boolean coverage + // ------------------------------------------------------------------ + + /** + * Verifies that {@code getBoolean} converts numeric ClickHouse columns to + * boolean using the "non-zero is true" rule, mirroring + * {@code AbstractBinaryFormatReader} semantics. The test pushes literal + * 0/1 values through several integer widths and also exercises the + * native Bool column. + */ + @Test(groups = {"integration"}) + public void testGetBooleanFromNumericValues() throws Exception { + String sql = "SELECT " + + "toInt8(0) AS v_int8_zero, toInt8(1) AS v_int8_one, " + + "toInt32(0) AS v_int32_zero, toInt32(42) AS v_int32_nonzero, " + + "toInt64(0) AS v_int64_zero, toInt64(-7) AS v_int64_nonzero, " + + "toUInt64(1) AS v_uint64_one, " + + "true AS v_bool_true, false AS v_bool_false"; + + try (QueryResponse response = + client.query(sql, newJsonEachRowSettingsForPrimitives()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertNotNull(reader.next()); + + Assert.assertFalse(reader.getBoolean("v_int8_zero")); + Assert.assertTrue(reader.getBoolean("v_int8_one")); + Assert.assertFalse(reader.getBoolean("v_int32_zero")); + Assert.assertTrue(reader.getBoolean("v_int32_nonzero")); + Assert.assertFalse(reader.getBoolean("v_int64_zero")); + Assert.assertTrue(reader.getBoolean("v_int64_nonzero")); + Assert.assertTrue(reader.getBoolean("v_uint64_one")); + Assert.assertTrue(reader.getBoolean("v_bool_true")); + Assert.assertFalse(reader.getBoolean("v_bool_false")); + + // The same values must also be readable by 1-based column index. + Assert.assertFalse(reader.getBoolean(1)); + Assert.assertTrue(reader.getBoolean(2)); + } + } + + // ------------------------------------------------------------------ + // Array accessor coverage + // ------------------------------------------------------------------ + + /** + * Exercises the typed Array accessors against a row of mixed Array(...) + * columns. Verifies that: + * + *
    + *
  • {@code getList} returns the parser-native list.
    • + *
  • The typed primitive accessors coerce parsed JSON numbers into the + * requested primitive type, regardless of whether the parser + * materialized elements as {@code Integer}, {@code Long}, + * {@code Double}, or {@code BigDecimal}.
    • + *
  • The typed accessors throw on non-array columns.
    • + *
+ */ + @Test(groups = {"integration"}) + public void testArrayAccessors() throws Exception { + String sql = "SELECT " + + "[1, 2, 3]::Array(Int32) AS col_int_arr, " + + "[10, 20, 30]::Array(Int64) AS col_long_arr, " + + "[1, 2]::Array(Int16) AS col_short_arr, " + + "[7, 8]::Array(Int8) AS col_byte_arr, " + + "[1.5, 2.5]::Array(Float64) AS col_double_arr, " + + "[1.0, 2.0]::Array(Float32) AS col_float_arr, " + + "['a', 'b', 'c']::Array(String) AS col_string_arr, " + + "[true, false, true]::Array(Bool) AS col_bool_arr, " + + "toInt32(1) AS col_not_array"; + + try (QueryResponse response = + client.query(sql, newJsonEachRowSettingsForPrimitives()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertNotNull(reader.next()); + + List intList = reader.getList("col_int_arr"); + Assert.assertNotNull(intList); + Assert.assertEquals(intList.size(), 3); + + // Each typed accessor is exercised by name first and then by 1-based + // column index so the index-based overloads are also covered against + // the inferred schema produced from the first row. + Assert.assertEquals(reader.getIntArray("col_int_arr"), new int[] {1, 2, 3}); + Assert.assertEquals(reader.getIntArray(1), new int[] {1, 2, 3}); + Assert.assertEquals(reader.getLongArray("col_long_arr"), new long[] {10L, 20L, 30L}); + Assert.assertEquals(reader.getLongArray(2), new long[] {10L, 20L, 30L}); + Assert.assertEquals(reader.getShortArray("col_short_arr"), + new short[] {(short) 1, (short) 2}); + Assert.assertEquals(reader.getShortArray(3), new short[] {(short) 1, (short) 2}); + Assert.assertEquals(reader.getByteArray("col_byte_arr"), new byte[] {(byte) 7, (byte) 8}); + Assert.assertEquals(reader.getByteArray(4), new byte[] {(byte) 7, (byte) 8}); + Assert.assertEquals(reader.getDoubleArray("col_double_arr"), + new double[] {1.5d, 2.5d}, 1e-9); + Assert.assertEquals(reader.getDoubleArray(5), new double[] {1.5d, 2.5d}, 1e-9); + Assert.assertEquals(reader.getFloatArray("col_float_arr"), + new float[] {1.0f, 2.0f}, 1e-6f); + Assert.assertEquals(reader.getFloatArray(6), new float[] {1.0f, 2.0f}, 1e-6f); + Assert.assertEquals(reader.getStringArray("col_string_arr"), + new String[] {"a", "b", "c"}); + Assert.assertEquals(reader.getStringArray(7), new String[] {"a", "b", "c"}); + Assert.assertEquals(reader.getBooleanArray("col_bool_arr"), + new boolean[] {true, false, true}); + Assert.assertEquals(reader.getBooleanArray(8), new boolean[] {true, false, true}); + + Assert.assertEquals(reader.getObjectArray("col_int_arr").length, 3); + Assert.assertEquals(reader.getObjectArray(1).length, 3); + + // Non-array columns must surface a RuntimeException rather than + // silently returning null or a malformed array. + try { + reader.getIntArray("col_not_array"); + Assert.fail("Expected exception on scalar column"); + } catch (RuntimeException expected) { + // ok + } + try { + reader.getList("col_not_array"); + Assert.fail("Expected exception on scalar column"); + } catch (RuntimeException expected) { + // ok + } + } + } + + /** + * Locks in the contract that a {@code null} element coming from the server + * (e.g. through {@code Array(Nullable(...))}) cannot be silently turned + * into a zero/false slot of a Java primitive array. The reader must throw + * a {@link RuntimeException} so callers don't lose the distinction between + * "missing value" and "actual zero". + */ + @Test(groups = {"integration"}) + public void testArrayAccessorsRejectNullElementsFromServer() throws Exception { + String sql = "SELECT " + + "[1, NULL, 3]::Array(Nullable(Int32)) AS col_int_arr, " + + "[true, NULL]::Array(Nullable(Bool)) AS col_bool_arr, " + + "['x', NULL, 'z']::Array(Nullable(String)) AS col_string_arr"; + + try (QueryResponse response = + client.query(sql, newJsonEachRowSettingsForPrimitives()).get(); + ClickHouseTextFormatReader reader = createReader(response)) { + + Assert.assertNotNull(reader.next()); + + try { + reader.getIntArray("col_int_arr"); + Assert.fail("Expected exception on null element in primitive array"); + } catch (RuntimeException expected) { + // ok + } + try { + reader.getBooleanArray("col_bool_arr"); + Assert.fail("Expected exception on null element in primitive array"); + } catch (RuntimeException expected) { + // ok + } + + // getStringArray and getObjectArray preserve nulls because the + // resulting array can hold null references. + Assert.assertEquals(reader.getStringArray("col_string_arr"), + new String[] {"x", null, "z"}); + Object[] objs = reader.getObjectArray("col_string_arr"); + Assert.assertEquals(objs.length, 3); + Assert.assertNull(objs[1]); + + // getList must surface the raw list with the null element intact. + List intList = reader.getList("col_int_arr"); + Assert.assertNotNull(intList); + Assert.assertEquals(intList.size(), 3); + Assert.assertNull(intList.get(1)); + } + } + + // ------------------------------------------------------------------ + // Test case definitions for primitive types + // ------------------------------------------------------------------ + + /** + * Describes one primitive ClickHouse column under test. Each case carries the SQL + * literals used to populate the row, the expected Java values, the accessor + * functions that are expected to succeed, and a list of accessors that must throw + * for values of this type. + */ + public static final class PrimitiveTypeCase { + final String columnName; + final String chType; + final List sqlLiterals; + final List expectedValues; + final BiFunction readByName; + final BiFunction readByIndex; + final BiPredicate equality; + final List incompatibleAccessors; + + PrimitiveTypeCase(String columnName, String chType, + List sqlLiterals, + List expectedValues, + BiFunction readByName, + BiFunction readByIndex, + BiPredicate equality, + List incompatibleAccessors) { + this.columnName = columnName; + this.chType = chType; + this.sqlLiterals = sqlLiterals; + this.expectedValues = expectedValues; + this.readByName = readByName; + this.readByIndex = readByIndex; + this.equality = equality; + this.incompatibleAccessors = incompatibleAccessors; + } + + @Override + public String toString() { + return columnName + " (" + chType + ")"; + } + } + + /** Pair of accessor invocations (by name and by index) that must throw for the case's column. */ + public static final class IncompatibleAccessor { + final String name; + final Consumer byName; + final Consumer byIndex; + + IncompatibleAccessor(String name, + Consumer byName, + Consumer byIndex) { + this.name = name; + this.byName = byName; + this.byIndex = byIndex; + } + } + + private static List buildPrimitiveCases(Random rnd) { + List cases = new ArrayList<>(); + + // ---- Signed integers --------------------------------------------------- + cases.add(intCase("col_int8", "Int8", + Arrays.asList( + Byte.MIN_VALUE, Byte.MAX_VALUE, (byte) 0, + (byte) (rnd.nextInt(256) - 128), + (byte) (rnd.nextInt(256) - 128)), + Number::byteValue, + (r, n) -> r.getByte(n), + (r, i) -> r.getByte(i))); + + cases.add(intCase("col_int16", "Int16", + Arrays.asList( + Short.MIN_VALUE, Short.MAX_VALUE, (short) 0, + (short) (rnd.nextInt(65536) - 32768), + (short) (rnd.nextInt(65536) - 32768)), + Number::shortValue, + (r, n) -> r.getShort(n), + (r, i) -> r.getShort(i))); + + cases.add(intCase("col_int32", "Int32", + Arrays.asList( + Integer.MIN_VALUE, Integer.MAX_VALUE, 0, + rnd.nextInt(), rnd.nextInt()), + Number::intValue, + (r, n) -> r.getInteger(n), + (r, i) -> r.getInteger(i))); + + cases.add(intCase("col_int64", "Int64", + Arrays.asList( + Long.MIN_VALUE, Long.MAX_VALUE, 0L, + rnd.nextLong(), rnd.nextLong()), + Number::longValue, + (r, n) -> r.getLong(n), + (r, i) -> r.getLong(i))); + + // ---- Unsigned integers ------------------------------------------------- + cases.add(intCase("col_uint8", "UInt8", + Arrays.asList( + (short) 0, (short) 255, (short) 128, + (short) rnd.nextInt(256), + (short) rnd.nextInt(256)), + Number::shortValue, + (r, n) -> r.getShort(n), + (r, i) -> r.getShort(i))); + + cases.add(intCase("col_uint16", "UInt16", + Arrays.asList( + 0, 65535, 32768, + rnd.nextInt(65536), + rnd.nextInt(65536)), + Number::intValue, + (r, n) -> r.getInteger(n), + (r, i) -> r.getInteger(i))); + + cases.add(intCase("col_uint32", "UInt32", + Arrays.asList( + 0L, 4294967295L, 1L, + (long) rnd.nextInt() & 0xFFFFFFFFL, + (long) rnd.nextInt() & 0xFFFFFFFFL), + Number::longValue, + (r, n) -> r.getLong(n), + (r, i) -> r.getLong(i))); + + // UInt64 max (2^64 - 1) does not fit into a signed long, so we read it as + // BigInteger and use BigInteger equality everywhere. + List uint64Values = Arrays.asList( + BigInteger.ZERO, + new BigInteger("18446744073709551615"), + BigInteger.ONE, + new BigInteger(63, rnd), + new BigInteger(64, rnd)); + cases.add(new PrimitiveTypeCase( + "col_uint64", "UInt64", + sqlLiteralsFromValues(uint64Values, v -> "toUInt64('" + v + "')"), + uint64Values, + (r, n) -> r.getBigInteger(n), + (r, i) -> r.getBigInteger(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForNumericValue("col_uint64", 1))); + + // ---- Floating-point ---------------------------------------------------- + List floatValues = Arrays.asList( + 0.0f, Float.MAX_VALUE, -Float.MAX_VALUE, + rnd.nextFloat() * 1_000f - 500f, + rnd.nextFloat() * 1_000f - 500f); + cases.add(new PrimitiveTypeCase( + "col_float32", "Float32", + sqlLiteralsFromValues(floatValues, + v -> "toFloat32(" + Float.toString(v) + ")"), + toObjectList(floatValues), + (r, n) -> r.getFloat(n), + (r, i) -> r.getFloat(i), + AbstractJSONEachRowFormatReaderTests::approximatelyEqualsFloat, + incompatibleForNumericValue("col_float32", 1))); + + List doubleValues = Arrays.asList( + 0.0d, Double.MAX_VALUE, -Double.MAX_VALUE, + rnd.nextDouble() * 1_000d - 500d, + rnd.nextDouble() * 1_000d - 500d); + cases.add(new PrimitiveTypeCase( + "col_float64", "Float64", + sqlLiteralsFromValues(doubleValues, + v -> "toFloat64(" + Double.toString(v) + ")"), + toObjectList(doubleValues), + (r, n) -> r.getDouble(n), + (r, i) -> r.getDouble(i), + AbstractJSONEachRowFormatReaderTests::approximatelyEqualsDouble, + incompatibleForNumericValue("col_float64", 1))); + + // ---- Decimal ----------------------------------------------------------- + List decimalValues = Arrays.asList( + new BigDecimal("0.0000"), + new BigDecimal("99999999999999.9999"), + new BigDecimal("-99999999999999.9999"), + new BigDecimal(rnd.nextLong() % 1_000_000_000L) + .movePointLeft(4), + new BigDecimal(rnd.nextLong() % 1_000_000_000L) + .movePointLeft(4)); + cases.add(new PrimitiveTypeCase( + "col_decimal", "Decimal(18, 4)", + sqlLiteralsFromValues(decimalValues, + v -> "toDecimal64('" + v.toPlainString() + "', 4)"), + toObjectList(decimalValues), + (r, n) -> r.getBigDecimal(n), + (r, i) -> r.getBigDecimal(i), + AbstractJSONEachRowFormatReaderTests::equalsBigDecimal, + incompatibleForNumericValue("col_decimal", 1))); + + // ---- Bool -------------------------------------------------------------- + List boolValues = Arrays.asList(false, true, false, rnd.nextBoolean(), rnd.nextBoolean()); + cases.add(new PrimitiveTypeCase( + "col_bool", "Bool", + sqlLiteralsFromValues(boolValues, Object::toString), + toObjectList(boolValues), + (r, n) -> r.getBoolean(n), + (r, i) -> r.getBoolean(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForBoolValue("col_bool", 1))); + + // ---- String ------------------------------------------------------------ + List stringValues = Arrays.asList( + "", + "hello world", + randomAsciiString(rnd, 32), + randomAsciiString(rnd, 16), + "line1\nline2\twith special chars: 'quoted'"); + cases.add(new PrimitiveTypeCase( + "col_string", "String", + sqlLiteralsFromValues(stringValues, + AbstractJSONEachRowFormatReaderTests::toClickHouseStringLiteral), + toObjectList(stringValues), + (r, n) -> r.getString(n), + (r, i) -> r.getString(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForStringValue("col_string", 1))); + + // ---- Date -------------------------------------------------------------- + List dateValues = Arrays.asList( + LocalDate.of(1970, 1, 1), // Date min + LocalDate.of(2149, 6, 6), // Date max + LocalDate.of(2000, 1, 1), + randomDate(rnd), + randomDate(rnd)); + cases.add(new PrimitiveTypeCase( + "col_date", "Date", + sqlLiteralsFromValues(dateValues, v -> "toDate('" + v + "')"), + toObjectList(dateValues), + (r, n) -> r.getLocalDate(n), + (r, i) -> r.getLocalDate(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForDateValue("col_date", 1))); + + // ---- UUID -------------------------------------------------------------- + List uuidValues = Arrays.asList( + new UUID(0L, 0L), + new UUID(-1L, -1L), + UUID.fromString("11111111-2222-3333-4444-555555555555"), + new UUID(rnd.nextLong(), rnd.nextLong()), + new UUID(rnd.nextLong(), rnd.nextLong())); + cases.add(new PrimitiveTypeCase( + "col_uuid", "UUID", + sqlLiteralsFromValues(uuidValues, v -> "toUUID('" + v + "')"), + toObjectList(uuidValues), + (r, n) -> r.getUUID(n), + (r, i) -> r.getUUID(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForUuidValue("col_uuid", 1))); + + // ---- Enum8 / Enum16 ---------------------------------------------------- + // ClickHouse serialises enum columns into JSONEachRow as their string + // labels (not as the underlying numeric value). The reader therefore + // exposes them through getString, while numeric and temporal accessors + // remain incompatible. The 1-based column index passed to the + // incompatible-accessor helper is always 1 because each parameterized + // run selects a single column. + List enum8Values = Arrays.asList( + "red", "green", "blue", "red", "green"); + cases.add(new PrimitiveTypeCase( + "col_enum8", "Enum8('red' = 1, 'green' = 2, 'blue' = 3)", + sqlLiteralsFromValues(enum8Values, + AbstractJSONEachRowFormatReaderTests::toClickHouseStringLiteral), + toObjectList(enum8Values), + (r, n) -> r.getString(n), + (r, i) -> r.getString(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForEnumValue("col_enum8", 1))); + + List enum16Values = Arrays.asList( + "alpha", "beta", "gamma", "alpha", "beta"); + cases.add(new PrimitiveTypeCase( + "col_enum16", "Enum16('alpha' = 100, 'beta' = 200, 'gamma' = 300)", + sqlLiteralsFromValues(enum16Values, + AbstractJSONEachRowFormatReaderTests::toClickHouseStringLiteral), + toObjectList(enum16Values), + (r, n) -> r.getString(n), + (r, i) -> r.getString(i), + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForEnumValue("col_enum16", 1))); + + return cases; + } + + // ------------------------------------------------------------------ + // Case factories and helpers + // ------------------------------------------------------------------ + + /** + * Builds a case for a signed/unsigned integer column whose expected Java type is a + * boxed integer. Values are passed via {@code toXxx('literal')} casts so the SQL + * parser does not need to evaluate large unary expressions. + */ + private static PrimitiveTypeCase intCase( + String columnName, String chType, + List values, + java.util.function.Function normalize, + BiFunction readByName, + BiFunction readByIndex) { + + List expected = new ArrayList<>(values.size()); + for (Number v : values) { + expected.add(normalize.apply(v)); + } + List literals = sqlLiteralsFromValues(values, + v -> "to" + chType + "('" + v + "')"); + return new PrimitiveTypeCase(columnName, chType, literals, expected, + readByName, readByIndex, + AbstractJSONEachRowFormatReaderTests::equalsByEquals, + incompatibleForNumericValue(columnName, 1)); + } + + private static List sqlLiteralsFromValues(List values, + java.util.function.Function toLiteral) { + List literals = new ArrayList<>(values.size()); + for (T v : values) { + literals.add(toLiteral.apply(v)); + } + return literals; + } + + private static List toObjectList(List values) { + return new ArrayList<>(values); + } + + // ---- Equality helpers ------------------------------------------------------ + + private static boolean equalsByEquals(Object actual, Object expected) { + if (expected == null) { + return actual == null; + } + return expected.equals(actual); + } + + private static boolean equalsBigDecimal(Object actual, Object expected) { + if (expected == null) { + return actual == null; + } + if (!(actual instanceof BigDecimal)) { + return false; + } + return ((BigDecimal) expected).compareTo((BigDecimal) actual) == 0; + } + + private static boolean approximatelyEqualsFloat(Object actual, Object expected) { + if (actual == null || expected == null) { + return actual == expected; + } + float a = ((Number) actual).floatValue(); + float e = ((Number) expected).floatValue(); + if (Float.compare(a, e) == 0) { + return true; + } + if (Float.isInfinite(a) || Float.isInfinite(e) || Float.isNaN(a) || Float.isNaN(e)) { + return Float.compare(a, e) == 0; + } + float tolerance = Math.max(Math.ulp(e) * 4f, Math.abs(e) * 1e-6f); + return Math.abs(a - e) <= tolerance; + } + + private static boolean approximatelyEqualsDouble(Object actual, Object expected) { + if (actual == null || expected == null) { + return actual == expected; + } + double a = ((Number) actual).doubleValue(); + double e = ((Number) expected).doubleValue(); + if (Double.compare(a, e) == 0) { + return true; + } + if (Double.isInfinite(a) || Double.isInfinite(e) || Double.isNaN(a) || Double.isNaN(e)) { + return Double.compare(a, e) == 0; + } + double tolerance = Math.max(Math.ulp(e) * 4d, Math.abs(e) * 1e-12d); + return Math.abs(a - e) <= tolerance; + } + + // ---- Incompatible accessor sets ------------------------------------------- + + /** + * Accessors that always fail (or fail for non-numeric content). For numeric columns + * we rely on date/UUID/temporal accessors to throw because the value cannot be + * parsed as a date or UUID. For unsupported accessors we expect + * {@link UnsupportedOperationException} from the reader implementation. + */ + private static List incompatibleForNumericValue(String name, int index) { + return Arrays.asList( + new IncompatibleAccessor("getLocalDate", + r -> r.getLocalDate(name), + r -> r.getLocalDate(index)), + new IncompatibleAccessor("getLocalTime", + r -> r.getLocalTime(name), + r -> r.getLocalTime(index)), + new IncompatibleAccessor("getLocalDateTime", + r -> r.getLocalDateTime(name), + r -> r.getLocalDateTime(index)), + new IncompatibleAccessor("getUUID", + r -> r.getUUID(name), + r -> r.getUUID(index)), + new IncompatibleAccessor("getZonedDateTime", + r -> r.getZonedDateTime(name), + r -> r.getZonedDateTime(index)), + new IncompatibleAccessor("getInstant", + r -> r.getInstant(name), + r -> r.getInstant(index)), + new IncompatibleAccessor("getInet4Address", + r -> r.getInet4Address(name), + r -> r.getInet4Address(index))); + } + + private static List incompatibleForBoolValue(String name, int index) { + // Numeric accessors fail because Boolean is not a Number, and string parsers + // fail because "true" / "false" cannot be parsed as a number, date, or UUID. + return Arrays.asList( + new IncompatibleAccessor("getByte", + r -> r.getByte(name), + r -> r.getByte(index)), + new IncompatibleAccessor("getInteger", + r -> r.getInteger(name), + r -> r.getInteger(index)), + new IncompatibleAccessor("getLong", + r -> r.getLong(name), + r -> r.getLong(index)), + new IncompatibleAccessor("getBigInteger", + r -> r.getBigInteger(name), + r -> r.getBigInteger(index)), + new IncompatibleAccessor("getBigDecimal", + r -> r.getBigDecimal(name), + r -> r.getBigDecimal(index)), + new IncompatibleAccessor("getLocalDate", + r -> r.getLocalDate(name), + r -> r.getLocalDate(index)), + new IncompatibleAccessor("getUUID", + r -> r.getUUID(name), + r -> r.getUUID(index)), + new IncompatibleAccessor("getZonedDateTime", + r -> r.getZonedDateTime(name), + r -> r.getZonedDateTime(index))); + } + + private static List incompatibleForStringValue(String name, int index) { + // String content here is not numeric, not a date, and not a UUID, so numeric + // and temporal accessors must throw. getBoolean is also expected to throw + // because the JSONEachRow reader does not silently convert string content + // through Boolean.parseBoolean (matching AbstractBinaryFormatReader-style + // strictness about incompatible types). + return Arrays.asList( + new IncompatibleAccessor("getByte", + r -> r.getByte(name), + r -> r.getByte(index)), + new IncompatibleAccessor("getInteger", + r -> r.getInteger(name), + r -> r.getInteger(index)), + new IncompatibleAccessor("getLong", + r -> r.getLong(name), + r -> r.getLong(index)), + new IncompatibleAccessor("getDouble", + r -> r.getDouble(name), + r -> r.getDouble(index)), + new IncompatibleAccessor("getBoolean", + r -> r.getBoolean(name), + r -> r.getBoolean(index)), + new IncompatibleAccessor("getBigInteger", + r -> r.getBigInteger(name), + r -> r.getBigInteger(index)), + new IncompatibleAccessor("getBigDecimal", + r -> r.getBigDecimal(name), + r -> r.getBigDecimal(index)), + new IncompatibleAccessor("getLocalDate", + r -> r.getLocalDate(name), + r -> r.getLocalDate(index)), + new IncompatibleAccessor("getUUID", + r -> r.getUUID(name), + r -> r.getUUID(index)), + new IncompatibleAccessor("getZonedDateTime", + r -> r.getZonedDateTime(name), + r -> r.getZonedDateTime(index))); + } + + private static List incompatibleForDateValue(String name, int index) { + // Date columns arrive as strings, so numeric accessors throw ClassCastException + // and time-only / date-time accessors fail to parse the YYYY-MM-DD shape. + return Arrays.asList( + new IncompatibleAccessor("getInteger", + r -> r.getInteger(name), + r -> r.getInteger(index)), + new IncompatibleAccessor("getLong", + r -> r.getLong(name), + r -> r.getLong(index)), + new IncompatibleAccessor("getDouble", + r -> r.getDouble(name), + r -> r.getDouble(index)), + new IncompatibleAccessor("getBigDecimal", + r -> r.getBigDecimal(name), + r -> r.getBigDecimal(index)), + new IncompatibleAccessor("getLocalTime", + r -> r.getLocalTime(name), + r -> r.getLocalTime(index)), + new IncompatibleAccessor("getLocalDateTime", + r -> r.getLocalDateTime(name), + r -> r.getLocalDateTime(index)), + new IncompatibleAccessor("getUUID", + r -> r.getUUID(name), + r -> r.getUUID(index)), + new IncompatibleAccessor("getZonedDateTime", + r -> r.getZonedDateTime(name), + r -> r.getZonedDateTime(index))); + } + + private static List incompatibleForEnumValue(String name, int index) { + // ClickHouse serialises Enum8/Enum16 columns as their string label in + // JSONEachRow output, so numeric and temporal accessors must throw and + // getBoolean must also throw (the reader does not silently parse + // string content into a boolean). getEnum8/getEnum16 forward to + // getByte/getShort, so they must throw on a string value as well. + return Arrays.asList( + new IncompatibleAccessor("getByte", + r -> r.getByte(name), + r -> r.getByte(index)), + new IncompatibleAccessor("getShort", + r -> r.getShort(name), + r -> r.getShort(index)), + new IncompatibleAccessor("getInteger", + r -> r.getInteger(name), + r -> r.getInteger(index)), + new IncompatibleAccessor("getLong", + r -> r.getLong(name), + r -> r.getLong(index)), + new IncompatibleAccessor("getDouble", + r -> r.getDouble(name), + r -> r.getDouble(index)), + new IncompatibleAccessor("getBoolean", + r -> r.getBoolean(name), + r -> r.getBoolean(index)), + new IncompatibleAccessor("getBigInteger", + r -> r.getBigInteger(name), + r -> r.getBigInteger(index)), + new IncompatibleAccessor("getBigDecimal", + r -> r.getBigDecimal(name), + r -> r.getBigDecimal(index)), + new IncompatibleAccessor("getEnum8", + r -> r.getEnum8(name), + r -> r.getEnum8(index)), + new IncompatibleAccessor("getEnum16", + r -> r.getEnum16(name), + r -> r.getEnum16(index)), + new IncompatibleAccessor("getLocalDate", + r -> r.getLocalDate(name), + r -> r.getLocalDate(index)), + new IncompatibleAccessor("getUUID", + r -> r.getUUID(name), + r -> r.getUUID(index))); + } + + private static List incompatibleForUuidValue(String name, int index) { + return Arrays.asList( + new IncompatibleAccessor("getInteger", + r -> r.getInteger(name), + r -> r.getInteger(index)), + new IncompatibleAccessor("getLong", + r -> r.getLong(name), + r -> r.getLong(index)), + new IncompatibleAccessor("getDouble", + r -> r.getDouble(name), + r -> r.getDouble(index)), + new IncompatibleAccessor("getBigDecimal", + r -> r.getBigDecimal(name), + r -> r.getBigDecimal(index)), + new IncompatibleAccessor("getLocalDate", + r -> r.getLocalDate(name), + r -> r.getLocalDate(index)), + new IncompatibleAccessor("getLocalDateTime", + r -> r.getLocalDateTime(name), + r -> r.getLocalDateTime(index)), + new IncompatibleAccessor("getZonedDateTime", + r -> r.getZonedDateTime(name), + r -> r.getZonedDateTime(index))); + } + + // ---- Random value helpers ------------------------------------------------- + + private static String randomAsciiString(Random rnd, int length) { + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + // Printable ASCII range except quote (32-126 except 39 and 92). + char c; + do { + c = (char) (32 + rnd.nextInt(95)); + } while (c == '\'' || c == '\\'); + sb.append(c); + } + return sb.toString(); + } + + private static LocalDate randomDate(Random rnd) { + // Stay well inside the Date range (1970-01-01..2149-06-06) to avoid time-zone + // edge effects when the server re-serialises the value into JSON. + int year = 1971 + rnd.nextInt(170); + int month = 1 + rnd.nextInt(12); + int day = 1 + rnd.nextInt(LocalDate.of(year, month, 1).lengthOfMonth()); + return LocalDate.of(year, month, day); + } + + private static String toClickHouseStringLiteral(String value) { + StringBuilder sb = new StringBuilder(value.length() + 2); + sb.append('\''); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '\'': + sb.append("\\'"); + break; + case '\\': + sb.append("\\\\"); + break; + case '\n': + sb.append("\\n"); + break; + case '\t': + sb.append("\\t"); + break; + case '\r': + sb.append("\\r"); + break; + default: + sb.append(c); + break; + } + } + sb.append('\''); + return sb.toString(); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..57f4e1abe --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/GsonJSONEachRowFormatReaderTests.java @@ -0,0 +1,74 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.api.insert.InsertResponse; +import com.clickhouse.client.api.insert.InsertSettings; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.data.ClickHouseFormat; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonWriter; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStreamWriter; +import java.util.Collections; +import java.util.Map; + +@Test(groups = {"integration"}) +public class GsonJSONEachRowFormatReaderTests extends AbstractJSONEachRowFormatReaderTests { + + private static Gson gson = new GsonBuilder().create(); + + private JsonParserFactory parserFactory = new GsonJsonParserFactory(); + + @Override + protected ClickHouseTextFormatReader createReader(QueryResponse response) throws IOException { + return new JSONEachRowFormatReader(parserFactory.createJsonParser(response.getInputStream())); + } + + @Override + protected ClickHouseTextFormatReader createReader(InputStream input) throws IOException { + return new JSONEachRowFormatReader(parserFactory.createJsonParser(input)); + } + + @Test(groups = {"integration"}) + public void testRowToObject() throws Exception { + + TestDTO_1[] data = new TestDTO_1[] { + new TestDTO_1("key1", 0.2, -0.2, Collections.singletonMap("p1", 10)), + new TestDTO_1("key2", 0.3, -0.5, Collections.singletonMap("p1", 9)), + }; + + final String table = "test_row_to_object_json"; + final String createStmt = "CREATE TABLE IF NOT EXISTS " + table + " (key String, sensor1 Decimal, sensor2 Decimal, params JSON) Engine MergeTree Order By (key)"; + client.execute(createStmt).get().close(); + client.execute("TRUNCATE " + table).get().close(); + + try (InsertResponse response = client.insert(table, out -> { + try (JsonWriter jsonWriter = gson.newJsonWriter(new OutputStreamWriter(out))) { + jsonWriter.setLenient(true); + for (TestDTO_1 value : data) { + gson.toJson(value, TestDTO_1.class, jsonWriter); + } + } + }, ClickHouseFormat.JSONEachRow, new InsertSettings()).get()) { + System.out.println("inserted rows" + response.getWrittenRows() ); + } + } + + @Data + @AllArgsConstructor + public static class TestDTO_1 { + + private String key; + + private Double sensor1; + + private Double sensor2; + + private Map params; + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReaderTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReaderTest.java new file mode 100644 index 000000000..f45fe8676 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JSONEachRowFormatReaderTest.java @@ -0,0 +1,717 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.data.ClickHouseDataType; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Test(groups = {"unit"}) +public class JSONEachRowFormatReaderTest { + + /** Simple in-memory parser that yields a fixed list of rows. */ + private static final class StubJsonParser implements JsonParser { + private final List> rows; + private boolean closed; + private int index; + + StubJsonParser(List> rows) { + this.rows = new ArrayList<>(rows); + } + + @Override + public Map nextRow() { + return index < rows.size() ? rows.get(index++) : null; + } + + @Override + public void close() { + closed = true; + } + + boolean isClosed() { + return closed; + } + } + + private static Map row(Object... pairs) { + Map r = new LinkedHashMap<>(); + for (int i = 0; i < pairs.length; i += 2) { + r.put((String) pairs[i], pairs[i + 1]); + } + return r; + } + + private static JSONEachRowFormatReader readerOf(Map... rows) { + return new JSONEachRowFormatReader(new StubJsonParser(Arrays.asList(rows))); + } + + // --------------------------------------------------------------------- + // Schema inference + // --------------------------------------------------------------------- + + @Test + public void testSchemaInferenceForIntegerLikeValues() { + JSONEachRowFormatReader reader = readerOf(row( + "as_integer", 1, + "as_long", 2L, + "as_big_integer", BigInteger.TEN)); + + Assert.assertEquals(reader.getSchema().getColumnByName("as_integer").getDataType(), + ClickHouseDataType.Int32); + Assert.assertEquals(reader.getSchema().getColumnByName("as_long").getDataType(), + ClickHouseDataType.Int64); + Assert.assertEquals(reader.getSchema().getColumnByName("as_big_integer").getDataType(), + ClickHouseDataType.Int256); + } + + @Test + public void testSchemaInferenceForFractionalValues() { + JSONEachRowFormatReader reader = readerOf(row( + "as_double", 1.5d, + "as_float", 2.5f, + "as_big_decimal", new BigDecimal("3.14"))); + + Assert.assertEquals(reader.getSchema().getColumnByName("as_double").getDataType(), + ClickHouseDataType.Float64); + Assert.assertEquals(reader.getSchema().getColumnByName("as_float").getDataType(), + ClickHouseDataType.Float32); + Assert.assertEquals(reader.getSchema().getColumnByName("as_big_decimal").getDataType(), + ClickHouseDataType.Decimal); + } + + @Test + public void testSchemaInferenceUsesJavaTypeForWholeFractionalValues() { + JSONEachRowFormatReader reader = readerOf(row( + "as_double_whole", 5.0d, + "as_float_whole", 7.0f, + "as_big_decimal_whole", new BigDecimal("42"))); + + Assert.assertEquals(reader.getSchema().getColumnByName("as_double_whole").getDataType(), + ClickHouseDataType.Float64); + Assert.assertEquals(reader.getSchema().getColumnByName("as_float_whole").getDataType(), + ClickHouseDataType.Float32); + Assert.assertEquals(reader.getSchema().getColumnByName("as_big_decimal_whole").getDataType(), + ClickHouseDataType.Decimal); + } + + @Test + public void testGuessDataTypeForOutOfRangeDoubleIsFloat64() { + // Values outside the long range cannot be represented as Int64; the + // reader must fall back to Float64 even when they are mathematically + // whole numbers. + JSONEachRowFormatReader reader = readerOf(row( + "too_big", 1.0e20d, + "infinite", Double.POSITIVE_INFINITY)); + + Assert.assertEquals(reader.getSchema().getColumnByName("too_big").getDataType(), + ClickHouseDataType.Float64); + Assert.assertEquals(reader.getSchema().getColumnByName("infinite").getDataType(), + ClickHouseDataType.Float64); + } + + @Test + public void testSchemaInferenceForUnsupportedNumberSubtypesUsesDefault() { + // AtomicInteger is a Number, but it is not part of ClickHouseDataType.DATA_TYPE_TO_CLASS. + JSONEachRowFormatReader reader = readerOf(row("custom", new AtomicInteger(5))); + Assert.assertEquals(reader.getSchema().getColumnByName("custom").getDataType(), + ClickHouseDataType.String); + } + + @Test + public void testSchemaInferenceForBooleanIsBool() { + JSONEachRowFormatReader reader = readerOf(row("flag", Boolean.TRUE)); + Assert.assertEquals(reader.getSchema().getColumnByName("flag").getDataType(), + ClickHouseDataType.Bool); + } + + @Test + public void testSchemaInferenceForStructuredAndSpecialValues() { + UUID uuid = UUID.fromString("11111111-2222-3333-4444-555555555555"); + JSONEachRowFormatReader reader = readerOf(row( + "as_string", "hello", + "as_list", Arrays.asList(1, 2, 3), + "as_array", new double[] {1.0d, 2.0d}, + "as_map", Collections.singletonMap("k", "v"), + "as_uuid", uuid, + "as_date", LocalDate.of(2024, 1, 2), + "as_datetime", LocalDateTime.of(2024, 1, 2, 3, 4), + "as_null", null)); + + Assert.assertEquals(reader.getSchema().getColumnByName("as_string").getDataType(), + ClickHouseDataType.String); + Assert.assertEquals(reader.getSchema().getColumnByName("as_list").getDataType(), + ClickHouseDataType.Array); + Assert.assertEquals(reader.getSchema().getColumnByName("as_array").getDataType(), + ClickHouseDataType.Array); + Assert.assertEquals(reader.getSchema().getColumnByName("as_map").getDataType(), + ClickHouseDataType.Map); + Assert.assertEquals(reader.getSchema().getColumnByName("as_uuid").getDataType(), + ClickHouseDataType.UUID); + Assert.assertEquals(reader.getSchema().getColumnByName("as_date").getDataType(), + ClickHouseDataType.Date); + Assert.assertEquals(reader.getSchema().getColumnByName("as_datetime").getDataType(), + ClickHouseDataType.DateTime64); + Assert.assertEquals(reader.getSchema().getColumnByName("as_null").getDataType(), + ClickHouseDataType.String); + } + + @Test + public void testEmptyResultSetExposesEmptySchema() { + JSONEachRowFormatReader reader = new JSONEachRowFormatReader( + new StubJsonParser(Collections.emptyList())); + + Assert.assertNotNull(reader.getSchema()); + Assert.assertEquals(reader.getSchema().getColumns().size(), 0); + Assert.assertFalse(reader.hasNext()); + Assert.assertNull(reader.next()); + } + + @Test + public void testReaderInitializationWrapsParserFailures() { + JsonParser failing = new JsonParser() { + @Override + public Map nextRow() throws Exception { + throw new IllegalStateException("boom"); + } + @Override + public void close() { } + }; + try { + new JSONEachRowFormatReader(failing); + Assert.fail("Expected RuntimeException"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Failed to initialize JSON reader"), + "Unexpected message: " + e.getMessage()); + Assert.assertTrue(e.getCause() instanceof IllegalStateException); + } + } + + @Test + public void testNextRowFailureIsWrapped() throws Exception { + JsonParser parser = new JsonParser() { + private int call; + @Override + public Map nextRow() { + if (call++ == 0) { + return row("id", 1); + } + throw new IllegalStateException("kaboom"); + } + @Override + public void close() { } + }; + try (JSONEachRowFormatReader reader = new JSONEachRowFormatReader(parser)) { + try { + reader.next(); + Assert.fail("Expected RuntimeException"); + } catch (RuntimeException e) { + Assert.assertTrue(e.getMessage().contains("Failed to read next JSON row"), + "Unexpected message: " + e.getMessage()); + } + } + } + + // --------------------------------------------------------------------- + // Row navigation, readValue, hasValue + // --------------------------------------------------------------------- + + @Test + public void testHasNextAndNext() throws Exception { + Map r1 = row("id", 1); + Map r2 = row("id", 2); + + try (JSONEachRowFormatReader reader = readerOf(r1, r2)) { + Assert.assertTrue(reader.hasNext()); + Assert.assertSame(reader.next(), r1); + Assert.assertTrue(reader.hasNext()); + Assert.assertSame(reader.next(), r2); + Assert.assertFalse(reader.hasNext()); + Assert.assertNull(reader.next()); + } + } + + @Test + public void testReadValueByIndexAndName() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row("id", 42, "name", "abc"))) { + reader.next(); + Number byIndex = reader.readValue(1); + Assert.assertNotNull(byIndex); + Assert.assertEquals(byIndex.intValue(), 42); + Assert.assertEquals((String) reader.readValue("name"), "abc"); + Assert.assertEquals((String) reader.readValue(2), "abc"); + } + } + + @Test + public void testHasValue() throws Exception { + Map r = new HashMap<>(); + r.put("present", "value"); + r.put("nullable", null); + + try (JSONEachRowFormatReader reader = new JSONEachRowFormatReader( + new StubJsonParser(Collections.singletonList(r)))) { + reader.next(); + Assert.assertTrue(reader.hasValue("present")); + Assert.assertFalse(reader.hasValue("nullable")); + Assert.assertFalse(reader.hasValue("missing")); + // The schema only contains the keys observed in the first row, so + // any column index resolved to a name that is present must be + // truthy and any nullable column must be falsy. + Assert.assertEquals(reader.hasValue(1), reader.hasValue(reader.getSchema().columnIndexToName(1))); + } + } + + @Test + public void testCloseDelegatesToParser() throws Exception { + StubJsonParser parser = new StubJsonParser(Collections.singletonList(row("id", 1))); + JSONEachRowFormatReader reader = new JSONEachRowFormatReader(parser); + Assert.assertFalse(parser.isClosed()); + reader.close(); + Assert.assertTrue(parser.isClosed()); + } + + // --------------------------------------------------------------------- + // Typed accessors + // --------------------------------------------------------------------- + + @Test + public void testNumericAccessors() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row( + "b", 120, + "s", 30000, + "i", 1_000_000, + "l", 10_000_000_000L, + "f", 1.5d, + "d", 2.5d))) { + reader.next(); + + Assert.assertEquals(reader.getByte("b"), (byte) 120); + Assert.assertEquals(reader.getByte(1), (byte) 120); + Assert.assertEquals(reader.getShort("s"), (short) 30000); + Assert.assertEquals(reader.getShort(2), (short) 30000); + Assert.assertEquals(reader.getInteger("i"), 1_000_000); + Assert.assertEquals(reader.getInteger(3), 1_000_000); + Assert.assertEquals(reader.getLong("l"), 10_000_000_000L); + Assert.assertEquals(reader.getLong(4), 10_000_000_000L); + Assert.assertEquals(reader.getFloat("f"), 1.5f, 0.0001f); + Assert.assertEquals(reader.getFloat(5), 1.5f, 0.0001f); + Assert.assertEquals(reader.getDouble("d"), 2.5d, 0.0001d); + Assert.assertEquals(reader.getDouble(6), 2.5d, 0.0001d); + + Assert.assertEquals(reader.getEnum8("b"), (byte) 120); + Assert.assertEquals(reader.getEnum8(1), (byte) 120); + Assert.assertEquals(reader.getEnum16("s"), (short) 30000); + Assert.assertEquals(reader.getEnum16(2), (short) 30000); + } + } + + @Test + public void testStringAccessor() throws Exception { + Map r = new LinkedHashMap<>(); + r.put("s", "hello"); + r.put("missing", null); + + try (JSONEachRowFormatReader reader = new JSONEachRowFormatReader( + new StubJsonParser(Collections.singletonList(r)))) { + reader.next(); + Assert.assertEquals(reader.getString("s"), "hello"); + Assert.assertEquals(reader.getString(1), "hello"); + Assert.assertNull(reader.getString("missing")); + Assert.assertNull(reader.getString(2)); + } + } + + @Test + public void testBooleanAccessor() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row( + "from_bool", Boolean.TRUE, + "from_zero", 0, + "from_nonzero", 1, + "from_long", 42L, + "from_big_int", new BigInteger("9000000000"), + "from_big_dec_zero", BigDecimal.ZERO))) { + reader.next(); + Assert.assertTrue(reader.getBoolean("from_bool")); + Assert.assertFalse(reader.getBoolean("from_zero")); + Assert.assertTrue(reader.getBoolean("from_nonzero")); + Assert.assertTrue(reader.getBoolean("from_long")); + Assert.assertTrue(reader.getBoolean("from_big_int")); + Assert.assertFalse(reader.getBoolean("from_big_dec_zero")); + Assert.assertTrue(reader.getBoolean(1)); + } + } + + @Test + public void testBooleanAccessorRejectsStringValue() throws Exception { + // Aligning with AbstractBinaryFormatReader expectations: text values + // that are not numeric or boolean are rejected rather than silently + // funneled through Boolean.parseBoolean, which would silently treat + // any non-"true" string as false. + try (JSONEachRowFormatReader reader = readerOf(row("v", "true"))) { + reader.next(); + try { + reader.getBoolean("v"); + Assert.fail("Expected exception for string value"); + } catch (RuntimeException expected) { + // ok + } + try { + reader.getBoolean(1); + Assert.fail("Expected exception for string value"); + } catch (RuntimeException expected) { + // ok + } + } + } + + @Test + public void testBooleanAccessorRejectsNullValue() throws Exception { + Map r = new LinkedHashMap<>(); + r.put("v", null); + try (JSONEachRowFormatReader reader = new JSONEachRowFormatReader( + new StubJsonParser(Collections.singletonList(r)))) { + reader.next(); + try { + reader.getBoolean("v"); + Assert.fail("Expected exception for null value"); + } catch (RuntimeException expected) { + // ok + } + } + } + + @Test + public void testBigNumberAccessors() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row( + "from_big_integer", new BigInteger("123456789012345"), + "from_string_int", "987654321", + "from_big_decimal", new BigDecimal("12345.6789"), + "from_string_dec", "0.5"))) { + reader.next(); + + Assert.assertEquals(reader.getBigInteger("from_big_integer"), + new BigInteger("123456789012345")); + Assert.assertEquals(reader.getBigInteger("from_string_int"), + new BigInteger("987654321")); + Assert.assertEquals(reader.getBigInteger(1), new BigInteger("123456789012345")); + Assert.assertNull(reader.getBigInteger("not_a_column")); + + Assert.assertEquals(reader.getBigDecimal("from_big_decimal").compareTo(new BigDecimal("12345.6789")), 0); + Assert.assertEquals(reader.getBigDecimal("from_string_dec").compareTo(new BigDecimal("0.5")), 0); + Assert.assertEquals(reader.getBigDecimal(3).compareTo(new BigDecimal("12345.6789")), 0); + } + } + + @Test + public void testTemporalAccessors() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row( + "d", "2024-05-06", + "t", "07:08:09", + "dt", "2024-05-06T07:08:09", + "odt", "2024-05-06T07:08:09+02:00"))) { + reader.next(); + + Assert.assertEquals(reader.getLocalDate("d"), java.time.LocalDate.of(2024, 5, 6)); + Assert.assertEquals(reader.getLocalDate(1), java.time.LocalDate.of(2024, 5, 6)); + Assert.assertEquals(reader.getLocalTime("t"), java.time.LocalTime.of(7, 8, 9)); + Assert.assertEquals(reader.getLocalTime(2), java.time.LocalTime.of(7, 8, 9)); + Assert.assertEquals(reader.getLocalDateTime("dt"), + java.time.LocalDateTime.of(2024, 5, 6, 7, 8, 9)); + Assert.assertEquals(reader.getLocalDateTime(3), + java.time.LocalDateTime.of(2024, 5, 6, 7, 8, 9)); + Assert.assertEquals(reader.getOffsetDateTime("odt"), + java.time.OffsetDateTime.parse("2024-05-06T07:08:09+02:00")); + Assert.assertEquals(reader.getOffsetDateTime(4), + java.time.OffsetDateTime.parse("2024-05-06T07:08:09+02:00")); + } + } + + @Test + public void testUuidAndListAccessors() throws Exception { + UUID uuid = UUID.fromString("11111111-2222-3333-4444-555555555555"); + try (JSONEachRowFormatReader reader = readerOf(row( + "u", uuid.toString(), + "arr", Arrays.asList(1, 2, 3), + "tuple", Arrays.asList("a", 1)))) { + reader.next(); + + Assert.assertEquals(reader.getUUID("u"), uuid); + Assert.assertEquals(reader.getUUID(1), uuid); + + List list = reader.getList("arr"); + Assert.assertEquals(list, Arrays.asList(1, 2, 3)); + Assert.assertEquals(reader.getList(2), Arrays.asList(1, 2, 3)); + + Assert.assertEquals(reader.getTuple("tuple"), new Object[] {"a", 1}); + Assert.assertEquals(reader.getTuple(3), new Object[] {"a", 1}); + } + } + + // --------------------------------------------------------------------- + // Unsupported operations + // --------------------------------------------------------------------- + + @Test + public void testUnsupportedAccessorsThrow() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row("v", "x"))) { + reader.next(); + + assertUnsupported(() -> reader.getInstant("v")); + assertUnsupported(() -> reader.getInstant(1)); + assertUnsupported(() -> reader.getZonedDateTime("v")); + assertUnsupported(() -> reader.getZonedDateTime(1)); + assertUnsupported(() -> reader.getDuration("v")); + assertUnsupported(() -> reader.getDuration(1)); + assertUnsupported(() -> reader.getInet4Address("v")); + assertUnsupported(() -> reader.getInet4Address(1)); + assertUnsupported(() -> reader.getInet6Address("v")); + assertUnsupported(() -> reader.getInet6Address(1)); + assertUnsupported(() -> reader.getGeoPoint("v")); + assertUnsupported(() -> reader.getGeoPoint(1)); + assertUnsupported(() -> reader.getGeoRing("v")); + assertUnsupported(() -> reader.getGeoRing(1)); + assertUnsupported(() -> reader.getGeoPolygon("v")); + assertUnsupported(() -> reader.getGeoPolygon(1)); + assertUnsupported(() -> reader.getGeoMultiPolygon("v")); + assertUnsupported(() -> reader.getGeoMultiPolygon(1)); + assertUnsupported(() -> reader.getClickHouseBitmap("v")); + assertUnsupported(() -> reader.getClickHouseBitmap(1)); + assertUnsupported(() -> reader.getTemporalAmount("v")); + assertUnsupported(() -> reader.getTemporalAmount(1)); + } + } + + @Test + public void testArrayAccessorsReturnNullForNullValue() throws Exception { + Map r = new LinkedHashMap<>(); + r.put("v", null); + try (JSONEachRowFormatReader reader = new JSONEachRowFormatReader( + new StubJsonParser(Collections.singletonList(r)))) { + reader.next(); + + // By name: every array accessor must propagate the null cleanly + // rather than throw NullPointerException or fabricate an empty array. + Assert.assertNull(reader.getList("v")); + Assert.assertNull(reader.getByteArray("v")); + Assert.assertNull(reader.getShortArray("v")); + Assert.assertNull(reader.getIntArray("v")); + Assert.assertNull(reader.getLongArray("v")); + Assert.assertNull(reader.getFloatArray("v")); + Assert.assertNull(reader.getDoubleArray("v")); + Assert.assertNull(reader.getBooleanArray("v")); + Assert.assertNull(reader.getStringArray("v")); + Assert.assertNull(reader.getObjectArray("v")); + + // By 1-based index: the same null-propagation guarantee must hold + // for the index-based overloads, which delegate to the name-based + // implementations through the inferred schema. + Assert.assertNull(reader.getList(1)); + Assert.assertNull(reader.getByteArray(1)); + Assert.assertNull(reader.getShortArray(1)); + Assert.assertNull(reader.getIntArray(1)); + Assert.assertNull(reader.getLongArray(1)); + Assert.assertNull(reader.getFloatArray(1)); + Assert.assertNull(reader.getDoubleArray(1)); + Assert.assertNull(reader.getBooleanArray(1)); + Assert.assertNull(reader.getStringArray(1)); + Assert.assertNull(reader.getObjectArray(1)); + } + } + + @Test + public void testArrayAccessorsRejectNullElement() throws Exception { + // A null element in a JSON array cannot be stored into a Java + // primitive array slot, so getPrimitiveArray must surface this rather + // than substitute a zero/false value silently. The String and Object + // overloads are allowed to keep the null element. + try (JSONEachRowFormatReader reader = readerOf(row( + "ints", Arrays.asList(1, null, 3), + "bools", Arrays.asList(Boolean.TRUE, null), + "strs", Arrays.asList("a", null, "c"), + "objs", Arrays.asList("x", null)))) { + reader.next(); + + assertThrowsRuntime(() -> reader.getIntArray("ints")); + assertThrowsRuntime(() -> reader.getIntArray(1)); + assertThrowsRuntime(() -> reader.getLongArray("ints")); + assertThrowsRuntime(() -> reader.getShortArray("ints")); + assertThrowsRuntime(() -> reader.getByteArray("ints")); + assertThrowsRuntime(() -> reader.getFloatArray("ints")); + assertThrowsRuntime(() -> reader.getDoubleArray("ints")); + assertThrowsRuntime(() -> reader.getBooleanArray("bools")); + + // The non-primitive container accessors keep nulls. + Assert.assertEquals(reader.getStringArray("strs"), new String[] {"a", null, "c"}); + Object[] objs = reader.getObjectArray("objs"); + Assert.assertEquals(objs.length, 2); + Assert.assertNull(objs[1]); + } + } + + @Test + public void testArrayAccessorsRejectNonArrayValues() throws Exception { + try (JSONEachRowFormatReader reader = readerOf(row("v", "x"))) { + reader.next(); + assertThrowsRuntime(() -> reader.getList("v")); + assertThrowsRuntime(() -> reader.getList(1)); + assertThrowsRuntime(() -> reader.getIntArray("v")); + assertThrowsRuntime(() -> reader.getIntArray(1)); + assertThrowsRuntime(() -> reader.getLongArray("v")); + assertThrowsRuntime(() -> reader.getLongArray(1)); + assertThrowsRuntime(() -> reader.getShortArray("v")); + assertThrowsRuntime(() -> reader.getShortArray(1)); + assertThrowsRuntime(() -> reader.getByteArray("v")); + assertThrowsRuntime(() -> reader.getByteArray(1)); + assertThrowsRuntime(() -> reader.getFloatArray("v")); + assertThrowsRuntime(() -> reader.getFloatArray(1)); + assertThrowsRuntime(() -> reader.getDoubleArray("v")); + assertThrowsRuntime(() -> reader.getDoubleArray(1)); + assertThrowsRuntime(() -> reader.getBooleanArray("v")); + assertThrowsRuntime(() -> reader.getBooleanArray(1)); + assertThrowsRuntime(() -> reader.getStringArray("v")); + assertThrowsRuntime(() -> reader.getStringArray(1)); + assertThrowsRuntime(() -> reader.getObjectArray("v")); + assertThrowsRuntime(() -> reader.getObjectArray(1)); + } + } + + @Test + public void testArrayAccessorsByIndex() throws Exception { + // Mirrors testArrayAccessorsCoercePrimitiveElements but addresses every + // typed array accessor through its 1-based column index. This ensures + // the index-based overloads delegate correctly through the inferred + // schema, even for arrays where the schema only records "Array" without + // a specific element type. + try (JSONEachRowFormatReader reader = readerOf(row( + "ints", Arrays.asList(1, 2, 3), + "longs", Arrays.asList(10L, 20L, 30L), + "shorts", Arrays.asList((short) 4, (short) 5), + "bytes", Arrays.asList((byte) 6, (byte) 7), + "doubles", Arrays.asList(1.5d, 2.5d), + "floats", Arrays.asList(1.0f, 2.0f), + "bools", Arrays.asList(Boolean.TRUE, Boolean.FALSE), + "strs", Arrays.asList("a", "b")))) { + reader.next(); + + Assert.assertEquals(reader.getIntArray(1), new int[] {1, 2, 3}); + Assert.assertEquals(reader.getLongArray(2), new long[] {10L, 20L, 30L}); + Assert.assertEquals(reader.getShortArray(3), new short[] {(short) 4, (short) 5}); + Assert.assertEquals(reader.getByteArray(4), new byte[] {(byte) 6, (byte) 7}); + Assert.assertEquals(reader.getDoubleArray(5), new double[] {1.5d, 2.5d}, 1e-9); + Assert.assertEquals(reader.getFloatArray(6), new float[] {1.0f, 2.0f}, 1e-6f); + Assert.assertEquals(reader.getBooleanArray(7), new boolean[] {true, false}); + Assert.assertEquals(reader.getStringArray(8), new String[] {"a", "b"}); + + // getList and getObjectArray by index, on the first column. + List list = reader.getList(1); + Assert.assertEquals(list, Arrays.asList(1, 2, 3)); + Object[] objs = reader.getObjectArray(1); + Assert.assertEquals(objs.length, 3); + } + } + + @Test + public void testGetBooleanArrayFromNumericList() throws Exception { + // Exercises the Number branch in coerceToComponent for boolean[]: the + // reader treats 0 as false and any non-zero value as true, regardless + // of whether the parser materialized the element as Integer, Long, or + // BigDecimal. + try (JSONEachRowFormatReader reader = readerOf(row( + "as_int", Arrays.asList(0, 1, 2, -1), + "as_long", Arrays.asList(0L, 5L), + "as_big_decimal", Arrays.asList(BigDecimal.ZERO, new BigDecimal("3"))))) { + reader.next(); + + Assert.assertEquals(reader.getBooleanArray("as_int"), + new boolean[] {false, true, true, true}); + Assert.assertEquals(reader.getBooleanArray("as_long"), + new boolean[] {false, true}); + Assert.assertEquals(reader.getBooleanArray("as_big_decimal"), + new boolean[] {false, true}); + } + } + + @Test + public void testGetBooleanArrayRejectsNonBooleanNonNumberElements() throws Exception { + // String elements cannot be coerced into boolean[] (we don't accept + // string-to-boolean parsing on the scalar getBoolean accessor either), + // so getBooleanArray must surface a RuntimeException sourced from the + // illegal-coerce branch in coerceToComponent. + try (JSONEachRowFormatReader reader = readerOf(row( + "v", Arrays.asList("true", "false")))) { + reader.next(); + assertThrowsRuntime(() -> reader.getBooleanArray("v")); + assertThrowsRuntime(() -> reader.getBooleanArray(1)); + } + } + + @Test + public void testArrayAccessorsCoercePrimitiveElements() throws Exception { + // Different parsers materialize numbers as different boxed types + // (Integer, Long, Double, BigDecimal). The reader must coerce each + // element to the requested primitive type without losing data. + try (JSONEachRowFormatReader reader = readerOf(row( + "ints", Arrays.asList(1, 2, 3), + "longs", Arrays.asList(10L, 20L, 30L), + "doubles", Arrays.asList(1.5d, 2.5d), + "floats", Arrays.asList(1.0f, 2.0f), + "shorts", Arrays.asList((short) 10, (short) 20), + "bytes", Arrays.asList((byte) 1, (byte) 2), + "bools", Arrays.asList(Boolean.TRUE, Boolean.FALSE, Boolean.TRUE), + "strs", Arrays.asList("a", "b"), + "big_decs", Arrays.asList(new BigDecimal("4"), new BigDecimal("5"))))) { + reader.next(); + + Assert.assertEquals(reader.getIntArray("ints"), new int[] {1, 2, 3}); + Assert.assertEquals(reader.getLongArray("longs"), new long[] {10L, 20L, 30L}); + Assert.assertEquals(reader.getDoubleArray("doubles"), new double[] {1.5d, 2.5d}, 1e-9); + Assert.assertEquals(reader.getFloatArray("floats"), new float[] {1.0f, 2.0f}, 1e-6f); + Assert.assertEquals(reader.getShortArray("shorts"), new short[] {(short) 10, (short) 20}); + Assert.assertEquals(reader.getByteArray("bytes"), new byte[] {(byte) 1, (byte) 2}); + Assert.assertEquals(reader.getBooleanArray("bools"), new boolean[] {true, false, true}); + Assert.assertEquals(reader.getStringArray("strs"), new String[] {"a", "b"}); + + // BigDecimal -> int[] requires element-level coercion. + Assert.assertEquals(reader.getIntArray("big_decs"), new int[] {4, 5}); + + // getObjectArray preserves boxed element types. + Object[] objs = reader.getObjectArray("ints"); + Assert.assertEquals(objs.length, 3); + + // getList returns the parser-native list reference. + List list = reader.getList("ints"); + Assert.assertEquals(list, Arrays.asList(1, 2, 3)); + } + } + + private static void assertThrowsRuntime(Runnable r) { + try { + r.run(); + Assert.fail("Expected RuntimeException"); + } catch (RuntimeException expected) { + // ok + } + } + + private static void assertUnsupported(Runnable r) { + try { + r.run(); + Assert.fail("Expected UnsupportedOperationException"); + } catch (UnsupportedOperationException expected) { + // ok + } + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java new file mode 100644 index 000000000..03aaab38b --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/JacksonJSONEachRowFormatReaderTests.java @@ -0,0 +1,23 @@ +package com.clickhouse.client.api.data_formats; + +import com.clickhouse.client.api.query.QueryResponse; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.io.InputStream; + +@Test(groups = {"integration"}) +public class JacksonJSONEachRowFormatReaderTests extends AbstractJSONEachRowFormatReaderTests { + + private JsonParserFactory parserFactory = new JacksonJsonParserFactory(); + + @Override + protected ClickHouseTextFormatReader createReader(QueryResponse response) throws IOException { + return new JSONEachRowFormatReader(parserFactory.createJsonParser(response.getInputStream())); + } + + @Override + protected ClickHouseTextFormatReader createReader(InputStream input) throws IOException { + return new JSONEachRowFormatReader(parserFactory.createJsonParser(input)); + } +} diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParserTest.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParserTest.java new file mode 100644 index 000000000..1ad5778bc --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/JacksonJsonParserTest.java @@ -0,0 +1,102 @@ +package com.clickhouse.client.api.data_formats.internal; + +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JsonParser; +import com.fasterxml.jackson.core.JsonParseException; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +@Test(groups = {"unit"}) +public class JacksonJsonParserTest { + + private JacksonJsonParserFactory parserFactory = new JacksonJsonParserFactory(); + + private static InputStream input(String json) { + return new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)); + } + + @Test + public void testReadsSingleRow() throws Exception { + try (JsonParser parser = parserFactory.createJsonParser( + input("{\"id\": 1, \"name\": \"a\"}"))) { + Map row = parser.nextRow(); + Assert.assertNotNull(row); + Assert.assertEquals(((Number) row.get("id")).intValue(), 1); + Assert.assertEquals(row.get("name"), "a"); + + // After the only row, EOF must be reported as null. + Assert.assertNull(parser.nextRow()); + } + } + + @Test + public void testReadsMultipleRowsSeparatedByWhitespace() throws Exception { + // JSONEachRow output is a sequence of JSON objects separated by + // newlines. Jackson skips whitespace between tokens, so the parser + // must transparently advance to each subsequent object. + String body = "{\"id\":1}\n{\"id\":2}\n {\"id\":3}\n"; + try (JsonParser parser = parserFactory.createJsonParser(input(body))) { + for (int expected : Arrays.asList(1, 2, 3)) { + Map row = parser.nextRow(); + Assert.assertNotNull(row, "row " + expected + " should not be null"); + Assert.assertEquals(((Number) row.get("id")).intValue(), expected); + } + Assert.assertNull(parser.nextRow()); + } + } + + @Test + public void testEmptyInputReturnsNull() throws Exception { + try (JsonParser parser = parserFactory.createJsonParser(input(""))) { + Assert.assertNull(parser.nextRow()); + } + } + + @Test + public void testWhitespaceOnlyInputReturnsNull() throws Exception { + try (JsonParser parser = parserFactory.createJsonParser(input(" \n\n "))) { + Assert.assertNull(parser.nextRow()); + } + } + + @Test + public void testRepeatedNextRowAfterExhaustionRemainsNull() throws Exception { + try (JsonParser parser = parserFactory.createJsonParser(input("{\"id\":1}"))) { + Assert.assertNotNull(parser.nextRow()); + Assert.assertNull(parser.nextRow()); + Assert.assertNull(parser.nextRow()); + } + } + + @Test(expectedExceptions = JsonParseException.class) + public void testMalformedInputPropagatesParseException() throws Exception { + try (JsonParser parser = parserFactory.createJsonParser(input("{not valid json"))) { + parser.nextRow(); + } + } + + @Test + public void testCloseClosesUnderlyingStream() throws Exception { + AtomicBoolean closed = new AtomicBoolean(false); + InputStream stream = new ByteArrayInputStream("{\"id\":1}".getBytes(StandardCharsets.UTF_8)) { + @Override + public void close() throws IOException { + closed.set(true); + super.close(); + } + }; + + JsonParser parser = parserFactory.createJsonParser(stream); + parser.close(); + Assert.assertTrue(closed.get(), "Underlying input stream should be closed"); + } + +} diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index 2651f88e9..2f81d2eb5 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -7,6 +7,7 @@ import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseServerForTest; import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; import com.clickhouse.client.api.ClientException; import com.clickhouse.client.api.ServerException; import com.clickhouse.client.api.command.CommandSettings; @@ -380,6 +381,28 @@ public void testQueryJSONEachRow() throws ExecutionException, InterruptedExcepti } } + @Test(groups = {"integration"}) + public void testJsonEachRowNumberQuoteSettingsAreOptIn() throws Exception { + String sql = "SELECT toInt64(1234567890123) AS v"; + + QuerySettings settings = new QuerySettings() + .setFormat(ClickHouseFormat.JSONEachRow) + .serverSetting("output_format_json_quote_64bit_integers", "1"); + try (QueryResponse response = client.query(sql, settings).get(); + BufferedReader reader = new BufferedReader(new InputStreamReader(response.getInputStream()))) { + Assert.assertTrue(reader.readLine().contains("\"v\":\"1234567890123\"")); + } + + QuerySettings unquotedSettings = new QuerySettings() + .setFormat(ClickHouseFormat.JSONEachRow) + .serverSetting("output_format_json_quote_64bit_integers", "1") + .setOption(ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING.getKey(), true); + try (QueryResponse response = client.query(sql, unquotedSettings).get(); + BufferedReader reader = new BufferedReader(new InputStreamReader(response.getInputStream()))) { + Assert.assertTrue(reader.readLine().contains("\"v\":1234567890123")); + } + } + @DataProvider(name = "rowBinaryFormats") ClickHouseFormat[] getRowBinaryFormats() { return new ClickHouseFormat[]{ diff --git a/docs/client-v2-json-support.md b/docs/client-v2-json-support.md new file mode 100644 index 000000000..233c0ff59 --- /dev/null +++ b/docs/client-v2-json-support.md @@ -0,0 +1,558 @@ +# JSONEachRow Support in `client-v2` and `jdbc-v2` + +This document specifies the `JSONEachRow` output-format support introduced in +`client-v2` and exposed through `jdbc-v2`. It defines the public API, +configuration properties, runtime dependencies, type mapping, and current +limitations. + +## Motivation + +ClickHouse provides several JSON-oriented column types (`JSON`, `Variant`, +`Dynamic`) and structured types (`Array`, `Tuple`, `Map`). When such values +are returned through binary formats they are commonly materialized as +serialized strings, which requires every caller to embed and configure its own +JSON parser and complicates the propagation of nested objects through JDBC. + +`JSONEachRow` is the row-oriented JSON output format of ClickHouse: each row +is emitted as a self-contained JSON object separated by line breaks. With the +appropriate server settings, numeric values are emitted as JSON numbers and +nested objects are preserved without additional encoding. Supporting this +format directly in the Java clients enables: + +- materializing columns of type `JSON` as `Map` instances; +- preserving nested objects, arrays, and tuples without an additional parsing + step; +- exposing structured JSON payloads through the standard JDBC `ResultSet` + contract. + +Combining `JSONEachRow` output with a pluggable Jackson or Gson parser +provides additional advantages beyond what the format alone delivers: + +- **Streaming row parsing.** Jackson's `JsonParser` and Gson's `JsonReader` + consume the response stream incrementally. `JSONEachRowFormatReader` + materializes one row at a time, so peak memory consumption is bounded by + the size of the current row rather than by the size of the result set. +- **Reuse of an existing JSON dependency on the classpath.** Applications + that already depend on Jackson or Gson for unrelated purposes can instantiate + the matching `JsonParserFactory` and avoid contributing a second JSON library + to the runtime classpath. Only the library JARs are shared; the default + factories use their own `ObjectMapper` or `Gson` instance unless callers + provide a subclass with custom settings. +- **Choice between processors.** Jackson and Gson are selected independently + and can be swapped at deployment time. Applications may pick the processor + that best matches their existing classpath and operational constraints, + without changing application code that consumes the reader or the JDBC + `ResultSet`. + +## Summary of changes + +`client-v2`: + +- Introduces a common `com.clickhouse.client.api.data_formats.ClickHouseFormatReader` + interface that declares all row navigation, schema access, and typed + accessors. The pre-existing `ClickHouseBinaryFormatReader` becomes a + format-family sub-interface for binary output formats and inherits its + full method set unchanged from `ClickHouseFormatReader`. +- Adds `com.clickhouse.client.api.data_formats.ClickHouseTextFormatReader`, + a sibling sub-interface for text output formats. +- Adds `com.clickhouse.client.api.data_formats.JSONEachRowFormatReader`, + which implements `ClickHouseTextFormatReader` over a streaming JSON + parser. +- `Client.newBinaryFormatReader(...)` continues to construct the binary + readers (`Native`, `RowBinary`, `RowBinaryWithNames`, + `RowBinaryWithNamesAndTypes`) and rejects text formats with + `IllegalArgumentException`. `JSONEachRow` callers construct + `JSONEachRowFormatReader` directly from a `JsonParser`. +- Introduces a JSON parser SPI under + `com.clickhouse.client.api.data_formats`, consisting of `JsonParser`, + `JsonParserFactory`, `JacksonJsonParserFactory`, and + `GsonJsonParserFactory`. +- Adds an opt-in client flag for `JSONEachRow` requests that asks ClickHouse + to emit large integers, floats, and decimals as plain JSON numbers (see + [JSON number output settings](#json-number-output-settings)). +- Declares Jackson and Gson as `provided` Maven dependencies, so that + applications must include the chosen processor on their own classpath. + +`jdbc-v2`: + +- Modifies `StatementImpl.executeQuery(...)` to accept `JSONEachRow` as a + valid output format. All other text formats remain unsupported. +- Adds `DriverProperties.JSON_PARSER_FACTORY` + (key `jdbc_json_parser_factory`) for selecting the `JsonParserFactory` + implementation by fully-qualified class name. +- Declares Jackson and Gson as `provided` dependencies, consistent with + `client-v2`. + +Two runnable examples are included in the repository: +`examples/client-v2-json-processors` and `examples/jdbc-v2-json-processors`. + +## Public API + +### `ClickHouseFormatReader`, `ClickHouseBinaryFormatReader`, `ClickHouseTextFormatReader` + +```java +package com.clickhouse.client.api.data_formats; + +public interface ClickHouseFormatReader extends AutoCloseable { ... } + +public interface ClickHouseBinaryFormatReader extends ClickHouseFormatReader { } + +public interface ClickHouseTextFormatReader extends ClickHouseFormatReader { } +``` + +`ClickHouseFormatReader` is the common contract for row-by-row format +readers regardless of the underlying wire encoding. The two sub-interfaces +specialize that contract by output-format family: callers receive a +`ClickHouseBinaryFormatReader` when the response is in a binary format and a +`ClickHouseTextFormatReader` when it is in a text format. All accessor +methods declared today live on the common parent; future format-specific +extensions are expected to be added on the corresponding sub-interface +without changing the shared surface, so code written against +`ClickHouseBinaryFormatReader` continues to compile against the same +inherited methods. + +### `JSONEachRowFormatReader` + +```java +package com.clickhouse.client.api.data_formats; + +public class JSONEachRowFormatReader implements ClickHouseTextFormatReader { ... } +``` + +The reader is instantiated from an `InputStream`-backed `JsonParser`. Callers +usually create that parser through `JacksonJsonParserFactory`, +`GsonJsonParserFactory`, or an application subclass of one of those factories. + +`JSONEachRow` is a text format, so the reader implements +`ClickHouseTextFormatReader`. Callers that need to handle both binary and +text readers uniformly can program against `ClickHouseFormatReader`. + +### `JsonParser` SPI + +```java +package com.clickhouse.client.api.data_formats; + +public interface JsonParser extends AutoCloseable { + Map nextRow() throws Exception; +} +``` + +Two factories are provided: + +- `JacksonJsonParserFactory` uses `com.fasterxml.jackson.core` and + `com.fasterxml.jackson.databind` to stream JSON objects. +- `GsonJsonParserFactory` uses `com.google.gson` with a lenient `JsonReader`, + which accepts a sequence of top-level JSON objects separated by whitespace, + as produced by `JSONEachRow`. + +`JsonParserFactory.createJsonParser(InputStream)` creates a parser for each +response stream. The factory may hold reusable parser configuration, but the +returned `JsonParser` is request-scoped and owns the stream reader it creates. + +The shipped implementations construct their own `ObjectMapper` and `Gson` +instances with default settings. To customize the underlying library +(Jackson modules, feature flags, Gson `TypeAdapter`s, number policies, etc.) +extend the corresponding factory and override its protected hook: + +- `JacksonJsonParserFactory` exposes `protected ObjectMapper createMapper()`. + Override it to return a fully configured `ObjectMapper`; the factory uses + the returned instance for all subsequent row parsing. +- `GsonJsonParserFactory` exposes `protected void customize(GsonBuilder + builder)`. Override it to configure the `GsonBuilder` before the factory + applies `setLenient()` and calls `build()`. + +Customization that does not need to influence the underlying parser can +still be performed on the caller side, after the row has been materialized +as `Map`. + +### JDBC parser factory property + +`jdbc-v2` selects the parser factory through +`DriverProperties.JSON_PARSER_FACTORY`: + +| Property key | Value | +| ------------------------- | ----- | +| `jdbc_json_parser_factory` | Fully-qualified class name implementing `JsonParserFactory` | + +The named class is loaded reflectively when the connection is created and must +have a public no-argument constructor. There is no equivalent `client-v2` +configuration key; direct client users pass a factory instance to their own +reader construction code. + +### JSON number quoting flag + +`client-v2` can opt in to numeric JSON output settings through +`ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING`: + +| Property key | Default | Effect | +| --------------------------------------------- | ------- | ------ | +| `json_disable_number_quoting` | `false` | When `true` and the resolved request format is `JSONEachRow`, sets `output_format_json_quote_64bit_integers=0`, `output_format_json_quote_64bit_floats=0`, and `output_format_json_quote_decimals=0` for that request. | + +The flag can be set on the client builder or on a specific `QuerySettings` +instance. It does not change `output_format_json_quote_denormals`. + +## Runtime dependencies + +`client-v2` and `jdbc-v2` declare the JSON libraries with `provided` scope, +so that they are not contributed to the runtime classpath of applications +that do not require them. Applications must add the JSON library used by the +selected factory to their runtime classpath: + +- Jackson — `com.fasterxml.jackson.core:jackson-databind`, + `com.fasterxml.jackson.core:jackson-core`, + `com.fasterxml.jackson.core:jackson-annotations` + (required when using `JacksonJsonParserFactory`); or +- Gson — `com.google.code.gson:gson` + (required when using `GsonJsonParserFactory`). + +The repository builds against Jackson `2.17.2` and Gson `2.10.1`. The parser +implementations rely only on the streaming token API and a single +`Map` materialization call, so other recent versions of +either library are expected to be compatible. + +When the selected JSON library is not present on the classpath, construction +or first use of the corresponding factory fails with the dependency-loading +error raised by the JVM or the JSON library. + +## Usage in `client-v2` + +```java +Client client = new Client.Builder() + .addEndpoint("http://localhost:8123") + .setUsername("default") + .setPassword("") + .setDefaultDatabase("default") + .serverSetting("allow_experimental_json_type", "1") + .build(); + +JsonParserFactory parserFactory = new JacksonJsonParserFactory(); + +QuerySettings settings = new QuerySettings() + .setFormat(ClickHouseFormat.JSONEachRow) + .setOption(ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING.getKey(), true); + +try (QueryResponse response = client.query( + "SELECT id, name, active, score, payload FROM events ORDER BY id", + settings).get(); + ClickHouseTextFormatReader reader = new JSONEachRowFormatReader( + parserFactory.createJsonParser(response.getInputStream()))) { + while (reader.next() != null) { + int id = reader.getInteger("id"); + String name = reader.getString("name"); + boolean active = reader.getBoolean("active"); + double score = reader.getDouble("score"); + Map payload = reader.readValue("payload"); // JSON column + // ... + } +} +``` + +Notes: + +- Set `ClickHouseFormat.JSONEachRow` in `QuerySettings`. Do not rely on an SQL + `FORMAT JSONEachRow` clause for direct `client-v2` examples when you also + want client-side JSON number output settings, because those settings are + applied only when the request settings identify the format as `JSONEachRow`. +- `client.newBinaryFormatReader(response)` continues to return a + `ClickHouseBinaryFormatReader` for binary output formats and rejects text + formats such as `JSONEachRow` with `IllegalArgumentException`. Callers that + need to handle both can program against the shared `ClickHouseFormatReader` + parent interface. +- `Map` is the canonical materialization for JSON columns + and for the row itself, as produced by the selected library. JSON arrays + are returned as `List`; nested JSON objects are returned as nested + `Map` instances. The exact Java types of leaf values are + whatever Jackson or Gson chose during parsing. + +## Usage in `jdbc-v2` + +The output format is selected by appending `FORMAT JSONEachRow` to the SQL +statement. The driver does not rewrite the SQL and does not apply a default +format on the caller's behalf. + +```java +Properties props = new Properties(); +props.setProperty("user", "default"); +props.setProperty("password", ""); +props.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), + JacksonJsonParserFactory.class.getName()); +// The JSON column type is experimental on the server side. +props.setProperty(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_floats"), "0"); +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_decimals"), "0"); + +try (Connection conn = DriverManager.getConnection( + "jdbc:clickhouse://localhost:8123/default", props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + "SELECT id, name, active, score, payload " + + "FROM events ORDER BY id FORMAT JSONEachRow")) { + + while (rs.next()) { + int id = rs.getInt("id"); + String name = rs.getString("name"); + boolean active = rs.getBoolean("active"); + double score = rs.getDouble("score"); + Object payload = rs.getObject("payload"); // Map / List / scalar + } +} +``` + +Behavior: + +- When `FORMAT JSONEachRow` is not specified, `jdbc-v2` continues to use the + binary default. `StatementImpl` accepts only `RowBinaryWithNamesAndTypes` + and `JSONEachRow` as output formats; any other text format causes + `SQLException("Only RowBinaryWithNameAndTypes and JSONEachRow are supported + for output format. ...")` to be thrown. +- `ResultSet.getObject(...)` returns parser-native `Map`, `List`, and scalar + values without an additional string round-trip. JSON arrays are returned as + the `List` implementation produced by the selected JSON library. Because + `JSONEachRow` has no array element metadata, `ResultSet.getArray(...)` is + not supported for these inferred JSON arrays. +- Temporal typed JDBC accessors follow the current `JSONEachRowFormatReader` + text-accessor support. `ResultSet.getString(...)` can be used to read the + server-formatted temporal text, but `getTimestamp(...)`, + `getObject(..., Timestamp.class)`, and related temporal conversions are not + guaranteed for `FORMAT JSONEachRow` result sets. Use the binary default + format when JDBC temporal typed accessors are required, or read the value as + a string/object and convert it in application code. +- The JSON processor is selected at the connection level through the + `jdbc_json_parser_factory` driver property. It cannot be changed per + statement, in line with the lifecycle of other connection options. +- Because JDBC selects `JSONEachRow` through SQL text, set the JSON output + server settings explicitly as connection properties when integer or decimal + numeric accessors are used. + +## JSON number output settings + +`Client.applyFormatSpecificSettings(...)` runs after request settings have +been merged and after the request format has been resolved. When the format +is `JSONEachRow` and +`ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING` is enabled, the +following server-side settings are set to `0` for the request: + +| Setting | Value | Rationale | +| ----------------------------------------- | ------------ | -------------------------------------------------------------------------- | +| `output_format_json_quote_64bit_integers` | `0` | Emits `Int64` and `UInt64` as JSON numbers rather than quoted strings. | +| `output_format_json_quote_64bit_floats` | `0` | Emits 64-bit floating-point values as JSON numbers. | +| `output_format_json_quote_decimals` | `0` | Emits decimals as JSON numbers, allowing materialization as `BigDecimal` or `Double`. | + +These overrides are scoped to the individual request and apply only when both +conditions are true: the request format in `QuerySettings` is `JSONEachRow`, +and `json_disable_number_quoting` is enabled through the client +or request settings. Explicit server settings are otherwise preserved. + +Denormal floating-point values (`NaN`, `Inf`, `-Inf`) are not yet handled by +the built-in JSON reader. The client does not set +`output_format_json_quote_denormals`; keep the server default or set +`output_format_json_quote_denormals=1` so these values are quoted, then handle +them as strings at the application boundary. + +JDBC callers that use SQL `FORMAT JSONEachRow` should set the same numeric +server settings explicitly through connection properties when integer or +decimal numeric accessors are used. + +## Row parsing, schema, and typed accessors + +### Row parsing is delegated to the chosen library + +The reader does not implement its own JSON parser. Each row is materialized +by the configured library: + +- the Jackson backend calls `ObjectMapper.readValue(parser, Map.class)` on + Jackson's streaming `JsonParser`; +- the Gson backend calls `gson.fromJson(reader, TypeToken>)` + on a lenient Gson `JsonReader`. + +The result of each call is a `Map` whose values have the +runtime Java types chosen by the library for the parsed JSON tokens — +typically `Number` (for example `Integer`, `Long`, `Double`, `BigDecimal`), +`Boolean`, `String`, `List` for JSON arrays, nested +`Map` for JSON objects, and `null` for JSON `null`. Numeric +representation, widening rules, handling of large integers, and any other +JSON-to-Java decisions are governed entirely by the library. The reader +neither inspects raw JSON tokens nor overrides the library's parsing +behavior. + +### Integer precision with Gson + +ClickHouse `Int64` and `UInt64` values can exceed the exactly representable +integer range of a JSON floating-point number. When +`json_disable_number_quoting` is enabled, the client asks +ClickHouse to emit them as JSON numbers for `JSONEachRow`, so the selected +JSON library's number materialization policy matters. + +Jackson's default `Map.class` materialization keeps ordinary integer tokens as +integer `Number` implementations. Gson's default `Map` +materialization can surface JSON numbers as floating-point values, which may +round integers larger than `2^53` before `getLong(...)` sees them. + +If you use Gson and need integer precision, provide a custom factory that +configures Gson's object number strategy: + +```java +public final class PreciseGsonJsonParserFactory extends GsonJsonParserFactory { + @Override + protected void customize(GsonBuilder builder) { + builder.setObjectToNumberStrategy(com.google.gson.ToNumberPolicy.LONG_OR_DOUBLE); + } +} +``` + +Use that factory directly with `client-v2`: + +```java +JsonParserFactory parserFactory = new PreciseGsonJsonParserFactory(); +``` + +For JDBC, put the factory class name in the connection properties: + +```java +props.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), + PreciseGsonJsonParserFactory.class.getName()); +``` + +`ToNumberPolicy.LONG_OR_DOUBLE` preserves values that fit in `long` as +`Long`. If exact decimal representation is more important than returning +`Long` for integer tokens, use `ToNumberPolicy.BIG_DECIMAL` and convert +explicitly at the application boundary. + +### Minimal schema discovery + +`JSONEachRow` does not include a schema header. To populate a minimal +`TableSchema` for the typed accessors, the reader inspects the **Java +types** of the first row's values, after the library has produced them, and +maps each to a `ClickHouseDataType`: + +| Java type produced by the library | Inferred ClickHouse type | +| ---------------------------------------------------------------------- | ------------------------ | +| `Integer` | `Int32` | +| `Long` | `Int64` | +| `BigInteger` | `Int256` | +| `Float` | `Float32` | +| `Double` | `Float64` | +| `BigDecimal` | `Decimal` | +| `Boolean` | `Bool` | +| `List` or Java array | `Array` | +| `Map` | `Map` | +| Any other value (`String`, `null`, unsupported number subtypes, ...) | `String` | + +Implications: + +- Schema discovery is performed once, on the first row. Empty result sets + produce a schema with no columns. +- Column names are taken verbatim from the JSON keys of the first row, in + iteration order. +- The discovered schema is intended only to support the typed accessors + (`getInteger`, `getString`, and so on). Server-side column metadata such + as precision, nullability, and codec is not reconstructed. +- Whether a JSON number is materialized as `Integer`, `Long`, `Double`, or + `BigDecimal` is a property of the chosen library, not of the reader. + Applications that need a specific numeric representation should select + the processor whose default behavior matches their expectations. + +### Typed accessors + +The typed accessors declared on the read interface are implemented as +follows: + +| Accessor | Behavior | +| --------------------------------------------- | ------------------------------------------------------------- | +| `readValue` / `next` | Returns the row as a `Map`. | +| `getString` | Returns `Object#toString()` of the JSON value, or `null`. | +| `getByte` / `getShort` / `getInteger` / `getLong` / `getFloat` / `getDouble` | Casts through `Number`. | +| `getBoolean` | Accepts `Boolean`, non-zero `Number`, or parses a string. | +| `getBigInteger` / `getBigDecimal` | Routes through `BigDecimal(String)`. | +| `getLocalDate` / `getLocalTime` / `getLocalDateTime` / `getOffsetDateTime` | Uses the corresponding `parse(...)` method on the string value. | +| `getUUID` | Uses `UUID.fromString(...)` on the string value. | +| `getList` | Returns the JSON array as `List`. | +| `getTuple` | Returns the row value cast to `Object[]`. | +| `getEnum8` / `getEnum16` | Delegates to `getByte` / `getShort`. | + +Accessor limitations to keep in mind: + +- `getTuple(...)` does not adapt parser-native `List` or `Map` values. Since + JSON arrays are usually materialized as `List` and JSON objects as `Map`, + callers should use `readValue(...)` for tuple-like JSON values and convert + them explicitly. + +The following accessors are not supported by the current implementation and +throw `UnsupportedOperationException`: + +- `getInstant`, `getZonedDateTime`, `getDuration`, `getTemporalAmount`; +- `getInet4Address`, `getInet6Address`; +- `getGeoPoint`, `getGeoRing`, `getGeoPolygon`, `getGeoMultiPolygon`; +- the typed array accessors `getByteArray`, `getIntArray`, `getLongArray`, + `getFloatArray`, `getDoubleArray`, `getBooleanArray`, `getShortArray`, + `getStringArray`, `getObjectArray`; +- `getClickHouseBitmap`. + +For these types, callers should obtain the parsed value through +`readValue(...)` or `getList(...)` and convert it explicitly. + +## Streaming and lifetime + +- `JacksonJsonParserFactory` and `GsonJsonParserFactory` delegate parsing to the + underlying library and consume one row at a time from the response + `InputStream`. Memory consumption is proportional to the size of the + current row and is independent of the size of the result set. +- `JSONEachRowFormatReader` reads the first row eagerly during construction + in order to inspect the Java types of its values and populate the minimal + `TableSchema` described above. For empty result sets, the reader exposes + an empty `TableSchema`, and `hasNext()` returns `false`. +- `close()` is propagated to the underlying parser, which closes the input + stream it owns. Callers are responsible for closing the originating + `QueryResponse` (or JDBC `ResultSet`). + +## Compatibility considerations + +- The parser factory classes are additive. Applications that do not request + `JSONEachRow` and do not instantiate or configure a parser factory are + unaffected. +- The default request format is unchanged. The existing binary readers + (`Native`, `RowBinary`, `RowBinaryWithNames`, `RowBinaryWithNamesAndTypes`) + retain their previous behavior. +- The reader hierarchy now distinguishes binary and text formats: + `ClickHouseBinaryFormatReader` and `ClickHouseTextFormatReader` are sibling + sub-interfaces of the new `ClickHouseFormatReader`. The accessor surface is + unchanged; callers that hold a `ClickHouseBinaryFormatReader` reference for + binary formats are unaffected. `Client.newBinaryFormatReader(...)` rejects + `JSONEachRow` with `IllegalArgumentException`; construct + `JSONEachRowFormatReader` directly for JSONEachRow streams. +- Jackson and Gson are now declared with `provided` scope in `client-v2` and + `jdbc-v2`. Applications that previously inherited Jackson transitively from + these modules in `test` scope must declare the chosen processor explicitly + on their runtime classpath. +- `jdbc_json_parser_factory` is a new JDBC driver property and is only needed + by connections that execute `FORMAT JSONEachRow` queries. + +## Examples + +Two runnable Gradle examples are provided under `examples/`: + +- `examples/client-v2-json-processors` exercises the `client-v2` API + directly, switching between Jackson and Gson factories against a shared sample + table that contains primitive columns and one `payload JSON` column. + Entry point: `ClientV2JsonProcessorsExample`. +- `examples/jdbc-v2-json-processors` performs the same flow through the JDBC + driver, with `FORMAT JSONEachRow` appended to the `SELECT` statement and + parser factory selection applied through connection properties. Entry point: + `JdbcV2JsonProcessorsExample`. + +Both examples include a sample dataset under +`src/main/resources/sample_data.csv` and require a running ClickHouse server +with `allow_experimental_json_type=1`. + +## Tests + +- `client-v2/src/test/java/com/clickhouse/client/api/data_formats/AbstractJSONEachRowFormatReaderTests.java` + defines a parameterized integration test executed for both processors via + the subclasses `JacksonJSONEachRowFormatReaderTests` and + `GsonJSONEachRowFormatReaderTests`. The suite covers basic parsing, schema + inference, primitive type accessors, and empty result sets. +- `jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java` adds + `testJSONEachRowFormat`, which exercises + `Statement.executeQuery("... FORMAT JSONEachRow")` through the JDBC driver + against both parser factories. diff --git a/docs/features.md b/docs/features.md index d44549284..5fd2b3af9 100644 --- a/docs/features.md +++ b/docs/features.md @@ -15,6 +15,7 @@ This document lists stable, user-visible behavior in `client-v2` and `jdbc-v2` t - Parameterized SQL: Accepts named query parameters and can send them through supported HTTP request encodings. - Result materialization helpers: Provides streaming `Records`, generic row access, and convenience APIs that materialize all rows into generic records or typed POJOs. - Binary format readers: Reads ClickHouse binary result formats including `Native`, `RowBinary`, `RowBinaryWithNames`, and `RowBinaryWithNamesAndTypes`. +- JSONEachRow text reader: Can stream `JSONEachRow` responses through a caller-supplied `JsonParser`, with Jackson and Gson parser factory implementations available as optional classpath dependencies, and infers a best-effort schema from the first row. - Data type conversion: Maps ClickHouse types to Java values for binary reads, POJO binding, and SQL parameter formatting, including date/time handling. - Geometry type support: For ClickHouse `25.11+`, where `Geometry` changed from a string alias to `Variant(Point, Ring, LineString, MultiLineString, Polygon, MultiPolygon)`, the client reads and writes `Geometry` values through generic records, binary readers, POJO binding, and SQL parameter formatting, using Java array dimensionality to represent the geometry shape. - Insert APIs: Supports inserting registered POJOs, raw streams, and callback-driven writers, with optional column lists and format selection. @@ -41,6 +42,8 @@ Compatibility-sensitive traits: - `Geometry` handling is shape-sensitive: supported values are 1D through 4D Java arrays representing the nested geometry variants, and unsupported shapes or non-array values are rejected during serialization. - `Geometry` write inference is dimension-based rather than fully type-specific: point, ring/line string, polygon/multi-line string, and multi-polygon are selected from array depth, so writing `Geometry` cannot currently distinguish `Ring` from `LineString` or `Polygon` from `MultiLineString`. - Session precedence is part of the contract: client session defaults apply to each request, operation settings may override them, and only the client `session_id` is mutable at runtime while other client session properties remain fixed for the lifetime of the client. +- JSONEachRow reading depends on the selected parser factory and request format settings: parser materialization determines Java value types, the reader infers minimal schema from the first row, and JSON number server settings are applied only when `QuerySettings` resolves to `ClickHouseFormat.JSONEachRow` and `json_disable_number_quoting` is enabled. +- JSONEachRow schema inference is intentionally best-effort: scalar values use Java-to-ClickHouse type mappings, while JSON arrays and objects are identified structurally as `Array` and `Map`. For arrays, maps, and some nested or ambiguous values, the inferred type may not include the most specific element, key, value, or nested ClickHouse type. ## `jdbc-v2` @@ -59,7 +62,7 @@ Compatibility-sensitive traits: - Prepared statements: Supports `?` parameters through client-side SQL rendering and validates that all parameters are bound before execution. - SQL parsing and classification: Classifies SQL to distinguish queries, updates, inserts, `USE`, and role-changing statements, with selectable parser backends. - JDBC escape processing: Translates supported JDBC escape syntax for dates, timestamps, and functions before execution. -- Result set streaming: Streams result sets from ClickHouse binary formats, enforces max-row limits, and manages result-set lifecycle correctly. +- Result set streaming: Streams result sets from ClickHouse binary formats and `FORMAT JSONEachRow`, enforces max-row limits, and manages result-set lifecycle correctly. - Result-set metadata: Exposes JDBC `ResultSetMetaData` backed by ClickHouse column schema. - Database metadata: Implements JDBC `DatabaseMetaData` for ClickHouse catalogs, schemas, tables, columns, and related capability reporting. - Parameter metadata: Reports prepared-statement parameter counts. @@ -79,6 +82,7 @@ Compatibility-sensitive traits: - String-like ClickHouse values have stable JDBC expectations: `String`, `FixedString`, and `Enum` values are returned as strings, while `UUID` is available both as `getString()` and `getObject(..., UUID.class)`. - `Geometry` has a stable JDBC mapping: metadata reports SQL type `ARRAY` with type name `Geometry`, read paths return nested Java arrays rather than custom wrappers, and write paths depend on the caller preserving the intended point/array nesting shape. - JDBC `Geometry` writes share the same ambiguity as the client serializer: variant selection is inferred from nesting depth, so `Ring` versus `LineString` and `Polygon` versus `MultiLineString` are not currently distinguishable when writing through the generic `Geometry` path. +- JDBC `FORMAT JSONEachRow` support is opt-in through the `jdbc_json_parser_factory` driver property, whose value must be a fully-qualified `JsonParserFactory` class name with a public no-argument constructor; JSONEachRow numeric and structured value behavior follows the selected parser and configured server output settings. Inferred JSON arrays are returned from `ResultSet.getObject(...)` as parser-native `List` values rather than JDBC `Array` values because JSONEachRow does not include element metadata. JDBC temporal typed accessors such as `getTimestamp(...)` are not guaranteed for JSONEachRow result sets; callers that need stable JDBC temporal conversions should use the binary default format or perform application-level conversion from string/object values. - Binary parameters passed through `setBytes()` are encoded as ClickHouse `unhex(...)` expressions rather than text literals; empty byte arrays map to an empty string expression. - Stream and reader setters (`setAsciiStream`, `setUnicodeStream`, `setBinaryStream`, `setCharacterStream`, `setNCharacterStream`) are treated as text input encoded with the same string-escaping rules, including length-based truncation when a length is supplied. - `getString()` formatting for temporal values is stable output: `Date` uses `yyyy-MM-dd`, `DateTime` uses `yyyy-MM-dd HH:mm:ss`, and `DateTime64` preserves fractional precision, all interpreted in server timezone context where applicable. diff --git a/examples/client-v2-json-processors/.gitignore b/examples/client-v2-json-processors/.gitignore new file mode 100644 index 000000000..f8b92c3aa --- /dev/null +++ b/examples/client-v2-json-processors/.gitignore @@ -0,0 +1,2 @@ +.gradle +build diff --git a/examples/client-v2-json-processors/README.md b/examples/client-v2-json-processors/README.md new file mode 100644 index 000000000..ff8360664 --- /dev/null +++ b/examples/client-v2-json-processors/README.md @@ -0,0 +1,138 @@ +# Client V2 JSON Processors Example + +## Overview + +This standalone example shows how to consume a `JSONEachRow` response with the +`client-v2` `JSONEachRowFormatReader` and a `JsonParserFactory`. The two +factories shipped with the client are the customization points: + +- `JacksonJsonParserFactory` exposes a `protected ObjectMapper createMapper()` + hook — override it to return a fully configured `ObjectMapper` (modules, + feature flags, custom deserializers, etc.). +- `GsonJsonParserFactory` exposes a `protected void customize(GsonBuilder)` + hook — override it to configure the `GsonBuilder` (number policy, type + adapters, etc.). The factory still applies `setLenient()` on its own + afterwards, which is required for the stream-of-objects shape of + `JSONEachRow`. + +The example is structured as a small component: + +- `ClientV2JsonProcessorsExample(Client client)` holds the shared `Client` and + exposes regular instance methods (`recreateTable()`, `loadSampleData()`, + `readAll(label, factory)`, `run()`), so the class can be copied as-is into + another project and have its individual methods invoked. +- Sample rows are kept in a plain `Object[][]` constant, separate from the + SQL, so the read path stays focused on the parser factory. +- Two small subclasses, `CustomJacksonParserFactory` and + `CustomGsonParserFactory`, demonstrate the protected-hook customization. + Both also implement a tiny `PayloadConverter` interface defined inside the + example: their configured `ObjectMapper` / `Gson` is reused to convert the + raw `payload` `Map` produced by the underlying library into a typed + `Payload` POJO. The default factories do not implement the interface, so + `readAll(...)` logs the raw map for them — making the contrast between the + default behavior and the customized behavior visible in the output. + +Each read call in `run()` follows the same three-step shape: + +1. **Create the factory** — `new JacksonJsonParserFactory()` / + `new GsonJsonParserFactory()` for defaults, or an instance of a custom + subclass. +2. **Customize if needed** — only inside the subclass, by overriding the + protected hook. +3. **Execute** — `readAll(label, factory)` runs the `SELECT` and feeds the + response stream through + `new JSONEachRowFormatReader(factory.createJsonParser(...))`. + +The client example selects the output format with +`new QuerySettings().setFormat(ClickHouseFormat.JSONEachRow)`. Use that form +instead of appending `FORMAT JSONEachRow` to the SQL when calling `client-v2` +directly when you enable client-side JSON number output settings, because +those settings depend on the request format. + +## Integer Precision + +ClickHouse 64-bit integers can be larger than the exact integer range of a +JSON floating-point number. Jackson's default map materialization preserves +ordinary integer tokens as integer `Number` values. Gson's default +`Map` materialization may surface numbers as floating-point +values, which can round large integers before `getLong(...)` sees them. + +For Gson, extend `GsonJsonParserFactory` and configure the object number +strategy: + +```java +public final class PreciseGsonJsonParserFactory extends GsonJsonParserFactory { + @Override + protected void customize(GsonBuilder builder) { + builder.setObjectToNumberStrategy(com.google.gson.ToNumberPolicy.LONG_OR_DOUBLE); + } +} +``` + +The included `CustomGsonParserFactory` uses this pattern. Use +`ToNumberPolicy.BIG_DECIMAL` instead when exact decimal representation matters +more than receiving integer tokens as `Long`. + +## Requirements + +- JDK 17 or newer +- A running ClickHouse server reachable from the machine running the example +- A locally installed `client-v2` snapshot from this repository + +## How to Run + +From this directory: + +```shell +gradle run +``` + +Connection properties can be supplied as system properties: + +- `-DchEndpoint` — endpoint to connect to (default: `http://localhost:8123`) +- `-DchUser` — ClickHouse user name (default: `default`) +- `-DchPassword` — ClickHouse user password (default: empty) +- `-DchDatabase` — ClickHouse database name (default: `default`) + +Example with custom connection properties: + +```shell +gradle run \ + -DchEndpoint=http://localhost:8123 \ + -DchUser=default \ + -DchPassword= \ + -DchDatabase=default +``` + +## Executable Example + +`com.clickhouse.examples.client_v2.json_processors.ClientV2JsonProcessorsExample` + +Steps performed by `run()`: + +1. `recreateTable()` — drops and re-creates `client_v2_json_processors_example` + with primitive columns and one `payload JSON` column. +2. `loadSampleData()` — inserts the rows from the `SAMPLE_ROWS` array as a + single batched `INSERT`. +3. `readAll(...)` is invoked four times, each time with a different + `JsonParserFactory`: + - default `JacksonJsonParserFactory`; + - `CustomJacksonParserFactory`, which overrides `createMapper()` to + tolerate unknown properties and preserve big integers and decimals + exactly, and implements `PayloadConverter` to convert each row's + `payload` `Map` into a `Payload` POJO via + `ObjectMapper.convertValue(...)`; + - default `GsonJsonParserFactory`; + - `CustomGsonParserFactory`, which overrides `customize(GsonBuilder)` to + use a `LONG_OR_DOUBLE` number policy and disable HTML escaping, and + implements `PayloadConverter` to convert each row's `payload` `Map` + into a `Payload` POJO via `gson.fromJson(gson.toJsonTree(...))`. + +Logged rows include the payload value's runtime class so the difference +between the default factories (which surface a `LinkedHashMap` / +`LinkedTreeMap`) and the customized factories (which surface a `Payload`) +shows up directly in the output. + +The build keeps both `jackson-databind` and `gson` on the classpath so the +example can switch between processors at runtime. Production applications +only need to keep the processor they actually use. diff --git a/examples/client-v2-json-processors/build.gradle.kts b/examples/client-v2-json-processors/build.gradle.kts new file mode 100644 index 000000000..e9f39b34e --- /dev/null +++ b/examples/client-v2-json-processors/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + application +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation(libs.clickhouseClient) + + // Keep both processors on the classpath so the example can switch between them. + implementation(libs.jacksonDatabind) + implementation(libs.gson) + + implementation(libs.slf4jApi) + runtimeOnly(libs.slf4jSimple) +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +application { + mainClass = "com.clickhouse.examples.client_v2.json_processors.ClientV2JsonProcessorsExample" +} + +tasks.named("run") { + listOf("chEndpoint", "chUser", "chPassword", "chDatabase").forEach { key -> + System.getProperty(key)?.let { value -> + systemProperty(key, value) + } + } +} diff --git a/examples/client-v2-json-processors/gradle.properties b/examples/client-v2-json-processors/gradle.properties new file mode 100644 index 000000000..5ad69748c --- /dev/null +++ b/examples/client-v2-json-processors/gradle.properties @@ -0,0 +1 @@ +org.gradle.configuration-cache=true diff --git a/examples/client-v2-json-processors/gradle/libs.versions.toml b/examples/client-v2-json-processors/gradle/libs.versions.toml new file mode 100644 index 000000000..7379edc0e --- /dev/null +++ b/examples/client-v2-json-processors/gradle/libs.versions.toml @@ -0,0 +1,12 @@ +[versions] +clickhouseClient = "0.9.8-SNAPSHOT" +jackson = "2.18.6" +gson = "2.10.1" +slf4j = "2.0.17" + +[libraries] +clickhouseClient = { module = "com.clickhouse:client-v2", version.ref = "clickhouseClient" } +jacksonDatabind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } +gson = { module = "com.google.code.gson:gson", version.ref = "gson" } +slf4jApi = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +slf4jSimple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" } diff --git a/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.jar b/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..d997cfc60 Binary files /dev/null and b/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.properties b/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..c61a118f7 --- /dev/null +++ b/examples/client-v2-json-processors/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/examples/client-v2-json-processors/gradlew b/examples/client-v2-json-processors/gradlew new file mode 100755 index 000000000..739907dfd --- /dev/null +++ b/examples/client-v2-json-processors/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/2d6327017519d23b96af35865dc997fcb544fb40/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/examples/client-v2-json-processors/gradlew.bat b/examples/client-v2-json-processors/gradlew.bat new file mode 100644 index 000000000..e509b2dd8 --- /dev/null +++ b/examples/client-v2-json-processors/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/client-v2-json-processors/settings.gradle.kts b/examples/client-v2-json-processors/settings.gradle.kts new file mode 100644 index 000000000..8b35c469e --- /dev/null +++ b/examples/client-v2-json-processors/settings.gradle.kts @@ -0,0 +1,5 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" +} + +rootProject.name = "ch-java-client-v2-json-processors" diff --git a/examples/client-v2-json-processors/src/main/java/com/clickhouse/examples/client_v2/json_processors/ClientV2JsonProcessorsExample.java b/examples/client-v2-json-processors/src/main/java/com/clickhouse/examples/client_v2/json_processors/ClientV2JsonProcessorsExample.java new file mode 100644 index 000000000..6951bb6e2 --- /dev/null +++ b/examples/client-v2-json-processors/src/main/java/com/clickhouse/examples/client_v2/json_processors/ClientV2JsonProcessorsExample.java @@ -0,0 +1,283 @@ +package com.clickhouse.examples.client_v2.json_processors; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.command.CommandResponse; +import com.clickhouse.client.api.data_formats.ClickHouseTextFormatReader; +import com.clickhouse.client.api.data_formats.GsonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JSONEachRowFormatReader; +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JsonParserFactory; +import com.clickhouse.client.api.query.QueryResponse; +import com.clickhouse.client.api.query.QuerySettings; +import com.clickhouse.data.ClickHouseFormat; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +/** + * Demonstrates how to consume a {@code JSONEachRow} response with the client-v2 + * {@link JSONEachRowFormatReader} and a {@link JsonParserFactory}. + * + *

The class is intentionally written as a regular component (instance methods, + * shared {@link Client} field) so it can be copied as-is into other projects and + * have its individual methods invoked.

+ * + *

Two factories ship with the client and serve as the customization points: + * {@link com.clickhouse.client.api.data_formats.JacksonJsonParserFactory} and {@link com.clickhouse.client.api.data_formats.GsonJsonParserFactory}. Extend + * either of them and override the protected hook + * ({@code createMapper()} for Jackson, {@code customize(GsonBuilder)} for Gson) + * to plug in any library-level customization.

+ */ +public class ClientV2JsonProcessorsExample { + + private static final Logger LOG = LoggerFactory.getLogger(ClientV2JsonProcessorsExample.class); + + private static final String TABLE = "client_v2_json_processors_example"; + + /** + * Sample dataset: {@code { id, name, active, score, payload }}. + */ + private static final Object[][] SAMPLE_ROWS = { + {1, "alpha", true, 1.5, "{\"city\":\"Berlin\",\"tags\":[\"a\",\"b\"]}"}, + {2, "beta", false, 2.5, "{\"city\":\"Paris\", \"tags\":[\"c\"]}"}, + {3, "gamma", true, 3.5, "{\"city\":\"Tokyo\", \"tags\":[]}"}, + }; + + private final Client client; + + public ClientV2JsonProcessorsExample(Client client) { + this.client = client; + } + + public static void main(String[] args) throws Exception { + try (Client client = buildClient()) { + new ClientV2JsonProcessorsExample(client).run(); + } + } + + /** + * Runs the full demo: prepares the table, loads sample rows, reads them four times. + */ + public void run() throws Exception { + recreateTable(); + loadSampleData(); + + // 1. Default Jackson: use the shipped factory as-is. + readAll("Jackson (default)", new JacksonJsonParserFactory()); + + // 2. Customized Jackson: extend the factory and override createMapper(). + readAll("Jackson (custom)", new CustomJacksonParserFactory()); + + // 3. Default Gson: use the shipped factory as-is. + readAll("Gson (default)", new GsonJsonParserFactory()); + + // 4. Customized Gson: extend the factory and override customize(GsonBuilder). + readAll("Gson (custom)", new CustomGsonParserFactory()); + } + + /** + * Reads every row from {@link #TABLE} using a {@code JSONEachRow} stream + * decoded with the supplied {@link JsonParserFactory}. + * + *

When the factory also implements {@link PayloadConverter} (as the two + * custom factories below do), the raw {@code payload} value is fed through + * {@link PayloadConverter#toPayload(Object)} so the row is logged with a + * typed {@link Payload} POJO instead of the bare {@code Map} + * produced by the underlying library.

+ */ + public void readAll(String label, JsonParserFactory factory) throws Exception { + LOG.info("--- Reading rows with {} ---", label); + + QuerySettings settings = new QuerySettings() + .setFormat(ClickHouseFormat.JSONEachRow) + .setOption(ClientConfigProperties.JSON_DISABLE_NUMBER_QUOTING.getKey(), true); + String sql = "SELECT id, name, active, score, payload FROM " + TABLE + " ORDER BY id"; + + PayloadConverter converter = factory instanceof PayloadConverter ? (PayloadConverter) factory : null; + + try (QueryResponse response = client.query(sql, settings).get(); + ClickHouseTextFormatReader reader = new JSONEachRowFormatReader( + factory.createJsonParser(response.getInputStream()))) { + + while (reader.next() != null) { + Object rawPayload = reader.readValue("payload"); + Object payload = converter != null ? converter.toPayload(rawPayload) : rawPayload; + LOG.info(" id={}, name={}, active={}, score={}, payload={} ({})", + reader.getInteger("id"), + reader.getString("name"), + reader.getBoolean("active"), + reader.getDouble("score"), + payload, + payload == null ? "null" : payload.getClass().getSimpleName()); + } + } + } + + public void recreateTable() throws Exception { + execute("DROP TABLE IF EXISTS " + TABLE); + execute("CREATE TABLE " + TABLE + " (" + + "id UInt32, name String, active Bool, score Float64, payload JSON" + + ") ENGINE = MergeTree ORDER BY id"); + } + + /** + * Inserts {@link #SAMPLE_ROWS} into {@link #TABLE} as a single batched INSERT. + */ + public void loadSampleData() throws Exception { + StringBuilder sql = new StringBuilder("INSERT INTO ").append(TABLE) + .append(" (id, name, active, score, payload) VALUES"); + for (int i = 0; i < SAMPLE_ROWS.length; i++) { + Object[] row = SAMPLE_ROWS[i]; + sql.append(i == 0 ? " " : ", ") + .append('(').append(row[0]) + .append(", ").append(sqlString((String) row[1])) + .append(", ").append(row[2]) + .append(", ").append(row[3]) + .append(", ").append(sqlString((String) row[4])) + .append(')'); + } + execute(sql.toString()); + } + + private void execute(String sql) throws Exception { + try (CommandResponse ignored = client.execute(sql).get()) { + LOG.debug("Executed: {}", sql); + } + } + + private static Client buildClient() { + return new Client.Builder() + .addEndpoint(System.getProperty("chEndpoint", "http://localhost:8123")) + .setUsername(System.getProperty("chUser", "default")) + .setPassword(System.getProperty("chPassword", "")) + .setDefaultDatabase(System.getProperty("chDatabase", "default")) + .serverSetting("allow_experimental_json_type", "1") + .build(); + } + + private static String sqlString(String value) { + return "'" + value.replace("'", "''") + "'"; + } + + // --------------------------------------------------------------------- + // Customized factories + // --------------------------------------------------------------------- + + /** + * Customized {@link JacksonJsonParserFactory}. Override {@code createMapper()} + * to return any {@link ObjectMapper} you want — modules, feature flags, + * deserializers, etc. all carry over to row parsing. + * + *

This example tolerates new server-side keys and preserves big integers + * and decimals exactly inside the {@code payload} JSON column. It also + * implements {@link PayloadConverter} so the same configured mapper is + * reused to convert the row's {@code payload} {@code Map} into a typed + * {@link Payload} POJO via {@link ObjectMapper#convertValue(Object, Class)}.

+ */ + public static final class CustomJacksonParserFactory extends JacksonJsonParserFactory implements PayloadConverter { + + private static final ObjectMapper MAPPER = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS, true) + .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true) + .build(); + + @Override + protected ObjectMapper createMapper() { + return MAPPER; + } + + @Override + public Payload toPayload(Object rawPayload) { + return MAPPER.convertValue(rawPayload, Payload.class); + } + } + + /** + * Customized {@link GsonJsonParserFactory}. Override + * {@code customize(GsonBuilder)} and configure the builder; the factory + * applies {@code setLenient()} on its own afterward (which is required for + * the stream-of-objects shape of {@code JSONEachRow}). + * + *

This example parses integer-shaped JSON numbers as {@code Long} (the + * default is {@code Double}, which loses precision for large {@code Int64} + * values) and disables HTML escaping on round-trips. It also implements + * {@link PayloadConverter} so the same configured {@link Gson} is reused + * to convert the row's {@code payload} {@code Map} into a typed + * {@link Payload} POJO via {@code fromJson(toJsonTree(...))}.

+ */ + public static final class CustomGsonParserFactory extends GsonJsonParserFactory implements PayloadConverter { + + private static final Gson GSON = new GsonBuilder() + .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .disableHtmlEscaping() + .setLenient() + .create(); + + @Override + protected void customize(GsonBuilder builder) { + builder.setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .disableHtmlEscaping(); + } + + @Override + public Payload toPayload(Object rawPayload) { + return GSON.fromJson(GSON.toJsonTree(rawPayload), Payload.class); + } + } + + // --------------------------------------------------------------------- + // Domain types used to demonstrate POJO materialization of payload + // --------------------------------------------------------------------- + + /** + * Optional hook implemented by customized factories that know how to turn + * the raw {@code payload} value (a {@code Map} produced by + * the underlying JSON library) into a typed {@link Payload} POJO. The + * default factories do not implement it, so {@link #readAll(String, JsonParserFactory)} + * logs the raw map for them. + */ + public interface PayloadConverter { + Payload toPayload(Object rawPayload); + } + + /** POJO shape of the {@code payload} JSON column used by the sample data. */ + public static final class Payload { + + private String city; + private List tags; + + public Payload() { + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + @Override + public String toString() { + return "Payload{city='" + city + "', tags=" + tags + '}'; + } + } +} diff --git a/examples/jdbc-dispatcher-demo/.gitignore b/examples/jdbc-dispatcher-demo/.gitignore new file mode 100644 index 000000000..67bcc2f72 --- /dev/null +++ b/examples/jdbc-dispatcher-demo/.gitignore @@ -0,0 +1,2 @@ +.gradle/ +build/ diff --git a/examples/jdbc-v2-json-processors/.gitignore b/examples/jdbc-v2-json-processors/.gitignore new file mode 100644 index 000000000..f8b92c3aa --- /dev/null +++ b/examples/jdbc-v2-json-processors/.gitignore @@ -0,0 +1,2 @@ +.gradle +build diff --git a/examples/jdbc-v2-json-processors/README.md b/examples/jdbc-v2-json-processors/README.md new file mode 100644 index 000000000..80f03b24f --- /dev/null +++ b/examples/jdbc-v2-json-processors/README.md @@ -0,0 +1,193 @@ +# JDBC V2 JSON Processors Example + +## Overview + +This standalone example shows how to consume `FORMAT JSONEachRow` responses +through `jdbc-v2` with the two factories shipped under +`com.clickhouse.client.api.data_formats`: + +- `JacksonJsonParserFactory` +- `GsonJsonParserFactory` + +### How the JDBC driver selects a factory + +The driver picks the parser factory **by fully-qualified class name** from +the `jdbc_json_parser_factory` driver property. +The value is the FQN of a class that implements `JsonParserFactory`; the +driver loads it reflectively and instantiates it through a **public no-arg +constructor**. There is no enum-style selector. + +Selection is **connection-level**: the factory cannot be swapped on an +existing connection. The driver instantiates the named class once during +connection creation and reuses that instance for every `JSONEachRow` +response served by the connection. + +### Customization is done by extending the factory + +Because the driver instantiates the named class with a no-arg constructor, +customization cannot be expressed as constructor arguments. The supported +approach is: + +1. Subclass `JacksonJsonParserFactory` or `GsonJsonParserFactory` in your + own code. +2. Override the protected hook: + - `protected ObjectMapper createMapper()` on `JacksonJsonParserFactory` — + return any fully-configured `ObjectMapper` (modules, feature flags, + deserializers). + - `protected void customize(GsonBuilder builder)` on + `GsonJsonParserFactory` — configure the `GsonBuilder` (number policy, + `TypeAdapter`s, date format, ...). The factory still applies + `setLenient()` on its own afterwards, which is required for + `JSONEachRow`. +3. Set `JSON_PARSER_FACTORY` to the FQN of the subclass. + +This example carries both subclasses as `public static final` nested classes +inside `JdbcV2JsonProcessorsExample` (`CustomJacksonParserFactory` and +`CustomGsonParserFactory`). The example feeds their FQNs to the driver via +`factoryClass.getName()`, which for nested classes returns the +`Outer$Inner` binary form — accepted by `Class.forName(...)` and by the +driver. If you set `JSON_PARSER_FACTORY` manually (e.g. from a config file +or JDBC URL) and your subclass is nested, you must use the same `$`-form; +top-level classes use the ordinary dot-separated FQN. + +Both custom subclasses also implement a tiny `PayloadConverter` interface +defined inside the example: their configured `ObjectMapper` / `Gson` is +reused to convert the raw `payload` value produced by the underlying +library into a typed `Payload` POJO. Because the JDBC driver only exposes +the factory through the connection (not as a Java object), `readAll(...)` +detects the interface on the factory **class** and instantiates its own +converter via the same public no-arg constructor the driver uses. The +default factories do not implement the interface, so `readAll(...)` logs +the raw map for them — making the contrast between the default behavior +and the customized behavior visible in the output. + +### Component shape + +`JdbcV2JsonProcessorsExample` is written as a small component: + +- `JdbcV2JsonProcessorsExample(String url, String user, String password)` + holds the connection settings and exposes regular instance methods + (`recreateTable()`, `loadSampleData()`, `readAll(label, factoryClass)`, + `run()`), so the class can be copied as-is into another project and have + its individual methods invoked. +- Sample rows are kept in a plain `Object[][]` constant, separate from the + SQL, so the read path stays focused on the parser-factory wiring. + +Each read call in `run()` follows the same three-step shape: + +1. **Pick a factory class** — `JacksonJsonParserFactory.class` / + `GsonJsonParserFactory.class` for defaults, or one of the nested custom + subclasses. +2. **Customize if needed** — only inside the subclass, by overriding the + protected hook. +3. **Execute** — `readAll(label, factoryClass)` opens a fresh connection + with `JSON_PARSER_FACTORY=`, runs the `SELECT ... FORMAT JSONEachRow` + and iterates the `ResultSet`. + +Because JDBC selects `JSONEachRow` through SQL text, set the JSON output +server settings explicitly on the connection when numeric accessors are used: + +```java +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_floats"), "0"); +props.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_decimals"), "0"); +``` + +Denormal floating-point values (`NaN`, `Inf`, `-Inf`) are not handled by the +built-in JSON reader yet. Keep `output_format_json_quote_denormals=1` and +handle those values as strings if your queries can return them. + +## Integer Precision + +ClickHouse 64-bit integers can be larger than the exact integer range of a +JSON floating-point number. Jackson's default map materialization preserves +ordinary integer tokens as integer `Number` values. Gson's default +`Map` materialization may surface numbers as floating-point +values, which can round large integers before `ResultSet.getLong(...)` sees +them. + +For Gson, extend `GsonJsonParserFactory` and configure the object number +strategy: + +```java +public final class PreciseGsonJsonParserFactory extends GsonJsonParserFactory { + @Override + protected void customize(GsonBuilder builder) { + builder.setObjectToNumberStrategy(com.google.gson.ToNumberPolicy.LONG_OR_DOUBLE); + } +} +``` + +Then configure JDBC with the factory class name: + +```java +props.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), + PreciseGsonJsonParserFactory.class.getName()); +``` + +The included `CustomGsonParserFactory` uses this pattern. Use +`ToNumberPolicy.BIG_DECIMAL` instead when exact decimal representation matters +more than receiving integer tokens as `Long`. + +## Requirements + +- JDK 17 or newer +- A running ClickHouse server reachable from the machine running the example +- A locally installed `jdbc-v2` snapshot from this repository + +## How to Run + +From this directory: + +```shell +gradle run +``` + +Connection properties can be supplied as system properties: + +- `-DchUrl` — JDBC URL (default: `jdbc:clickhouse://localhost:8123/default`) +- `-DchUser` — ClickHouse user name (default: `default`) +- `-DchPassword` — ClickHouse user password (default: empty) + +Example with custom connection properties: + +```shell +gradle run \ + -DchUrl=jdbc:clickhouse://localhost:8123/default \ + -DchUser=default \ + -DchPassword= +``` + +## Executable Example + +`com.clickhouse.examples.jdbc_v2.json_processors.JdbcV2JsonProcessorsExample` + +Steps performed by `run()`: + +1. `recreateTable()` — drops and re-creates `jdbc_v2_json_processors_example` + with primitive columns and one `payload JSON` column. +2. `loadSampleData()` — inserts the rows from the `SAMPLE_ROWS` array as a + single batched `INSERT`. +3. `readAll(...)` is invoked four times, each time pointing + `JSON_PARSER_FACTORY` at a different class: + - `JacksonJsonParserFactory` — default Jackson; + - `JdbcV2JsonProcessorsExample.CustomJacksonParserFactory` — nested + subclass overriding `createMapper()` to tolerate unknown properties + and preserve big integers and decimals exactly, also implementing + `PayloadConverter` to convert each row's `payload` `Map` into a + `Payload` POJO via `ObjectMapper.convertValue(...)`; + - `GsonJsonParserFactory` — default Gson; + - `JdbcV2JsonProcessorsExample.CustomGsonParserFactory` — nested + subclass overriding `customize(GsonBuilder)` to use a `LONG_OR_DOUBLE` + number policy and disable HTML escaping, also implementing + `PayloadConverter` to convert each row's `payload` `Map` into a + `Payload` POJO via `gson.fromJson(gson.toJsonTree(...))`. + +Logged rows include the payload value's runtime class so the difference +between the default factories (which surface a `LinkedHashMap` / +`LinkedTreeMap`) and the customized factories (which surface a `Payload`) +shows up directly in the output. + +The build keeps both `jackson-databind` and `gson` on the classpath so the +example can switch between processors at runtime. Production applications +only need to keep the processor they actually use. diff --git a/examples/jdbc-v2-json-processors/build.gradle.kts b/examples/jdbc-v2-json-processors/build.gradle.kts new file mode 100644 index 000000000..07ed830d2 --- /dev/null +++ b/examples/jdbc-v2-json-processors/build.gradle.kts @@ -0,0 +1,37 @@ +plugins { + application +} + +repositories { + mavenLocal() + mavenCentral() +} + +dependencies { + implementation(libs.jdbcV2) + + // Keep both processors on the classpath so the example can switch between them. + implementation(libs.jacksonDatabind) + implementation(libs.gson) + + implementation(libs.slf4jApi) + runtimeOnly(libs.slf4jSimple) +} + +java { + toolchain { + languageVersion = JavaLanguageVersion.of(17) + } +} + +application { + mainClass = "com.clickhouse.examples.jdbc_v2.json_processors.JdbcV2JsonProcessorsExample" +} + +tasks.named("run") { + listOf("chUrl", "chUser", "chPassword").forEach { key -> + System.getProperty(key)?.let { value -> + systemProperty(key, value) + } + } +} diff --git a/examples/jdbc-v2-json-processors/gradle.properties b/examples/jdbc-v2-json-processors/gradle.properties new file mode 100644 index 000000000..5ad69748c --- /dev/null +++ b/examples/jdbc-v2-json-processors/gradle.properties @@ -0,0 +1 @@ +org.gradle.configuration-cache=true diff --git a/examples/jdbc-v2-json-processors/gradle/libs.versions.toml b/examples/jdbc-v2-json-processors/gradle/libs.versions.toml new file mode 100644 index 000000000..6ac8a57a6 --- /dev/null +++ b/examples/jdbc-v2-json-processors/gradle/libs.versions.toml @@ -0,0 +1,12 @@ +[versions] +jdbcV2 = "0.9.8-SNAPSHOT" +jackson = "2.18.6" +gson = "2.10.1" +slf4j = "2.0.17" + +[libraries] +jdbcV2 = { module = "com.clickhouse:jdbc-v2", version.ref = "jdbcV2" } +jacksonDatabind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" } +gson = { module = "com.google.code.gson:gson", version.ref = "gson" } +slf4jApi = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +slf4jSimple = { module = "org.slf4j:slf4j-simple", version.ref = "slf4j" } diff --git a/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.jar b/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..d997cfc60 Binary files /dev/null and b/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.jar differ diff --git a/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.properties b/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..c61a118f7 --- /dev/null +++ b/examples/jdbc-v2-json-processors/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,7 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-9.4.1-bin.zip +networkTimeout=10000 +validateDistributionUrl=true +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/examples/jdbc-v2-json-processors/gradlew b/examples/jdbc-v2-json-processors/gradlew new file mode 100755 index 000000000..739907dfd --- /dev/null +++ b/examples/jdbc-v2-json-processors/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/2d6327017519d23b96af35865dc997fcb544fb40/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC2039,SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command: +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# and any embedded shellness will be escaped. +# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be +# treated as '${Hostname}' itself on the command line. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/examples/jdbc-v2-json-processors/gradlew.bat b/examples/jdbc-v2-json-processors/gradlew.bat new file mode 100644 index 000000000..e509b2dd8 --- /dev/null +++ b/examples/jdbc-v2-json-processors/gradlew.bat @@ -0,0 +1,93 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem +@rem SPDX-License-Identifier: Apache-2.0 +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. 1>&2 +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 +echo. 1>&2 +echo Please set the JAVA_HOME variable in your environment to match the 1>&2 +echo location of your Java installation. 1>&2 + +goto fail + +:execute +@rem Setup the command line + + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/examples/jdbc-v2-json-processors/settings.gradle.kts b/examples/jdbc-v2-json-processors/settings.gradle.kts new file mode 100644 index 000000000..4cb564e86 --- /dev/null +++ b/examples/jdbc-v2-json-processors/settings.gradle.kts @@ -0,0 +1,5 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "1.0.0" +} + +rootProject.name = "ch-java-jdbc-v2-json-processors" diff --git a/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java b/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java new file mode 100644 index 000000000..bccb90c92 --- /dev/null +++ b/examples/jdbc-v2-json-processors/src/main/java/com/clickhouse/examples/jdbc_v2/json_processors/JdbcV2JsonProcessorsExample.java @@ -0,0 +1,329 @@ +package com.clickhouse.examples.jdbc_v2.json_processors; + +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.data_formats.GsonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JsonParserFactory; +import com.clickhouse.jdbc.Driver; +import com.clickhouse.jdbc.DriverProperties; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.json.JsonMapper; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.ToNumberPolicy; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; +import java.util.Properties; + +/** + * Demonstrates how to consume {@code FORMAT JSONEachRow} responses through + * {@code jdbc-v2} with the two factories shipped under + * {@link com.clickhouse.client.api.data_formats}. + * + *

The class is intentionally written as a regular component (instance + * methods, shared connection settings as fields) so it can be copied as-is + * into other projects and have its individual methods invoked.

+ * + *

The JDBC driver selects a parser factory from the + * {@link DriverProperties#JSON_PARSER_FACTORY} driver property — its value is + * the fully-qualified class name of a {@link JsonParserFactory} + * implementation, which the driver loads reflectively and instantiates via + * a public no-arg constructor. There is no enum-style selector.

+ * + *

For Jackson/Gson customization the recommended approach is therefore to + * extend the shipped factory and override its protected hook + * ({@code createMapper()} or {@code customize(GsonBuilder)}), then point + * {@code JSON_PARSER_FACTORY} at the FQN of the subclass. See + * {@link CustomJacksonParserFactory} and {@link CustomGsonParserFactory} + * below.

+ */ +public class JdbcV2JsonProcessorsExample { + + private static final Logger LOG = LoggerFactory.getLogger(JdbcV2JsonProcessorsExample.class); + + private static final String TABLE = "jdbc_v2_json_processors_example"; + + /** + * Sample dataset: {@code { id, name, active, score, payload }}. + */ + private static final Object[][] SAMPLE_ROWS = { + {1, "alpha", true, 1.5, "{\"city\":\"Berlin\",\"tags\":[\"a\",\"b\"]}"}, + {2, "beta", false, 2.5, "{\"city\":\"Paris\", \"tags\":[\"c\"]}"}, + {3, "gamma", true, 3.5, "{\"city\":\"Tokyo\", \"tags\":[]}"}, + }; + + private final String url; + private final String user; + private final String password; + + public JdbcV2JsonProcessorsExample(String url, String user, String password) { + this.url = url; + this.user = user; + this.password = password; + } + + public static void main(String[] args) throws Exception { + // jdbc-v2 does not self-register from a static initializer; standalone + // examples must register the driver explicitly before calling DriverManager. + Driver.load(); + + String url = System.getProperty("chUrl", "jdbc:clickhouse://localhost:8123/default"); + String user = System.getProperty("chUser", "default"); + String password = System.getProperty("chPassword", ""); + new JdbcV2JsonProcessorsExample(url, user, password).run(); + } + + /** + * Runs the full demo: prepares the table, loads sample rows, reads them four times. + */ + public void run() throws Exception { + recreateTable(); + loadSampleData(); + + // 1. Default Jackson: factory FQN points to the shipped class. + readAll("Jackson (default)", JacksonJsonParserFactory.class); + + // 2. Customized Jackson: factory FQN points to a subclass of + // JacksonJsonParserFactory whose no-arg constructor overrides createMapper(). + readAll("Jackson (custom)", CustomJacksonParserFactory.class); + + // 3. Default Gson. + readAll("Gson (default)", GsonJsonParserFactory.class); + + // 4. Customized Gson: subclass overriding customize(GsonBuilder). + readAll("Gson (custom)", CustomGsonParserFactory.class); + } + + /** + * Reads every row from {@link #TABLE} through a fresh JDBC connection + * configured to use the supplied {@link JsonParserFactory} implementation. + * + *

Selection is connection-level: the factory cannot be swapped on an + * existing connection. The driver instantiates the named class once during + * connection creation and reuses that instance for every {@code JSONEachRow} + * response served by the connection.

+ * + *

When {@code factoryClass} also implements {@link PayloadConverter} + * (as the two custom factories below do), the method also instantiates a + * converter — through the same public no-arg constructor the driver uses — + * and feeds {@code rs.getObject("payload")} through + * {@link PayloadConverter#toPayload(Object)}, so the row is logged with a + * typed {@link Payload} POJO instead of the bare {@code Map} + * produced by the underlying library.

+ */ + public void readAll(String label, Class factoryClass) throws Exception { + LOG.info("--- Reading rows with {} ({}) ---", label, factoryClass.getName()); + + Properties props = baseProperties(); + props.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), factoryClass.getName()); + + PayloadConverter converter = PayloadConverter.class.isAssignableFrom(factoryClass) + ? (PayloadConverter) factoryClass.getDeclaredConstructor().newInstance() + : null; + + String sql = "SELECT id, name, active, score, payload FROM " + TABLE + + " ORDER BY id FORMAT JSONEachRow"; + + try (Connection connection = DriverManager.getConnection(url, props); + Statement statement = connection.createStatement(); + ResultSet rs = statement.executeQuery(sql)) { + + while (rs.next()) { + Object rawPayload = rs.getObject("payload"); + Object payload = converter != null ? converter.toPayload(rawPayload) : rawPayload; + LOG.info(" id={}, name={}, active={}, score={}, payload={} ({})", + rs.getInt("id"), + rs.getString("name"), + rs.getBoolean("active"), + rs.getDouble("score"), + payload, + payload == null ? "null" : payload.getClass().getSimpleName()); + } + } + } + + public void recreateTable() throws SQLException { + try (Connection connection = DriverManager.getConnection(url, baseProperties()); + Statement statement = connection.createStatement()) { + statement.execute("DROP TABLE IF EXISTS " + TABLE); + statement.execute("CREATE TABLE " + TABLE + " (" + + "id UInt32, name String, active Bool, score Float64, payload JSON" + + ") ENGINE = MergeTree ORDER BY id"); + } + } + + /** + * Inserts {@link #SAMPLE_ROWS} into {@link #TABLE} as a single batched INSERT. + */ + public void loadSampleData() throws SQLException { + StringBuilder sql = new StringBuilder("INSERT INTO ").append(TABLE) + .append(" (id, name, active, score, payload) VALUES"); + for (int i = 0; i < SAMPLE_ROWS.length; i++) { + Object[] row = SAMPLE_ROWS[i]; + sql.append(i == 0 ? " " : ", ") + .append('(').append(row[0]) + .append(", ").append(sqlString((String) row[1])) + .append(", ").append(row[2]) + .append(", ").append(row[3]) + .append(", ").append(sqlString((String) row[4])) + .append(')'); + } + try (Connection connection = DriverManager.getConnection(url, baseProperties()); + Statement statement = connection.createStatement()) { + statement.executeUpdate(sql.toString()); + } + } + + private Properties baseProperties() { + Properties properties = new Properties(); + properties.setProperty("user", user); + properties.setProperty("password", password); + properties.setProperty(ClientConfigProperties.serverSetting("allow_experimental_json_type"), "1"); + properties.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_integers"), "0"); + properties.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_64bit_floats"), "0"); + properties.setProperty(ClientConfigProperties.serverSetting("output_format_json_quote_decimals"), "0"); + return properties; + } + + private static String sqlString(String value) { + return "'" + value.replace("'", "''") + "'"; + } + + // --------------------------------------------------------------------- + // Customized factories + // --------------------------------------------------------------------- + + /** + * Customized {@link JacksonJsonParserFactory}. Override {@code createMapper()} + * to return any {@link ObjectMapper} you want — modules, feature flags, + * deserializers, etc. all carry over to row parsing. + * + *

This class must be {@code public static} with a public no-arg + * constructor because the JDBC driver loads it reflectively via the + * {@code jdbc_json_parser_factory} driver property; the {@code .getName()} + * of a nested class is the {@code Outer$Inner} binary form, which + * {@code Class.forName(...)} accepts.

+ * + *

This example tolerates new server-side keys and preserves big integers + * and decimals exactly inside the {@code payload} JSON column. It also + * implements {@link PayloadConverter} so the same configured mapper is + * reused to convert the row's {@code payload} {@code Map} into a typed + * {@link Payload} POJO via {@link ObjectMapper#convertValue(Object, Class)}.

+ */ + public static final class CustomJacksonParserFactory extends JacksonJsonParserFactory implements PayloadConverter { + + private static final ObjectMapper MAPPER = JsonMapper.builder() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .configure(DeserializationFeature.USE_BIG_INTEGER_FOR_INTS, true) + .configure(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS, true) + .build(); + + @Override + protected ObjectMapper createMapper() { + return MAPPER; + } + + @Override + public Payload toPayload(Object rawPayload) { + return MAPPER.convertValue(rawPayload, Payload.class); + } + } + + /** + * Customized {@link GsonJsonParserFactory}. Override + * {@code customize(GsonBuilder)} and configure the builder; the factory + * applies {@code setLenient()} on its own afterwards (which is required for + * the stream-of-objects shape of {@code JSONEachRow}). + * + *

This class must be {@code public static} with a public no-arg + * constructor because the JDBC driver loads it reflectively via the + * {@code jdbc_json_parser_factory} driver property; the {@code .getName()} + * of a nested class is the {@code Outer$Inner} binary form, which + * {@code Class.forName(...)} accepts.

+ * + *

This example parses integer-shaped JSON numbers as {@code Long} (the + * default is {@code Double}, which loses precision for large {@code Int64} + * values) and disables HTML escaping on round-trips. It also implements + * {@link PayloadConverter} so the same configured {@link Gson} is reused + * to convert the row's {@code payload} {@code Map} into a typed + * {@link Payload} POJO via {@code fromJson(toJsonTree(...))}.

+ */ + public static final class CustomGsonParserFactory extends GsonJsonParserFactory implements PayloadConverter { + + private static final Gson GSON = new GsonBuilder() + .setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .disableHtmlEscaping() + .setLenient() + .create(); + + @Override + protected void customize(GsonBuilder builder) { + builder.setObjectToNumberStrategy(ToNumberPolicy.LONG_OR_DOUBLE) + .disableHtmlEscaping(); + } + + @Override + public Payload toPayload(Object rawPayload) { + return GSON.fromJson(GSON.toJsonTree(rawPayload), Payload.class); + } + } + + // --------------------------------------------------------------------- + // Domain types used to demonstrate POJO materialization of payload + // --------------------------------------------------------------------- + + /** + * Optional hook implemented by customized factories that know how to turn + * the raw {@code payload} value (a {@code Map} produced by + * the underlying JSON library) into a typed {@link Payload} POJO. The + * default factories do not implement it, so + * {@link #readAll(String, Class)} logs the raw map for them. + * + *

The interface is invoked from application code, not from the JDBC + * driver itself: {@code readAll(...)} detects it on the factory class and + * instantiates its own converter through the same public no-arg constructor + * the driver uses for row parsing.

+ */ + public interface PayloadConverter { + Payload toPayload(Object rawPayload); + } + + /** POJO shape of the {@code payload} JSON column used by the sample data. */ + public static final class Payload { + + private String city; + private List tags; + + public Payload() { + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public List getTags() { + return tags; + } + + public void setTags(List tags) { + this.tags = tags; + } + + @Override + public String toString() { + return "Payload{city='" + city + "', tags=" + tags + '}'; + } + } +} diff --git a/jdbc-v2/pom.xml b/jdbc-v2/pom.xml index a8f02f2e9..05b035694 100644 --- a/jdbc-v2/pom.xml +++ b/jdbc-v2/pom.xml @@ -50,14 +50,35 @@ ${guava.version} - com.fasterxml.jackson.core jackson-databind - test ${jackson.version} + provided + + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + provided + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + provided + + + + com.google.code.gson + gson + ${gson.version} + provided + + + ${project.parent.groupId} clickhouse-client @@ -89,18 +110,6 @@ test - - com.fasterxml.jackson.core - jackson-core - ${jackson.version} - test - - - com.fasterxml.jackson.core - jackson-databind - ${jackson.version} - test - com.fasterxml.jackson.dataformat jackson-dataformat-yaml @@ -211,4 +220,4 @@ - \ No newline at end of file + diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java index 4f41865ce..13822a2ff 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ConnectionImpl.java @@ -2,6 +2,7 @@ import com.clickhouse.client.api.Client; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.data_formats.JsonParserFactory; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.client.api.query.QuerySettings; @@ -16,6 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; import java.sql.Array; import java.sql.Blob; import java.sql.CallableStatement; @@ -69,6 +71,8 @@ public class ConnectionImpl implements Connection, JdbcV2Wrapper { private final FeatureManager featureManager; + private final JsonParserFactory jsonParserFactory; + public ConnectionImpl(String url, Properties info) throws SQLException { try { this.url = url;//Raw URL @@ -119,6 +123,10 @@ public ConnectionImpl(String url, Properties info) throws SQLException { this.sqlParser = SqlParserFacade.getParser(config.getDriverProperty(DriverProperties.SQL_PARSER.getKey(), DriverProperties.SQL_PARSER.getDefaultValue()), config); this.featureManager = new FeatureManager(this.config); + + final String jsonParserFactoryName = config.getDriverProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), null); + this.jsonParserFactory = jsonParserFactoryName == null ? null : instantiateJsonParserFactory( + config.getDriverProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), null)); } catch (SQLException e) { throw e; } catch (Exception e) { @@ -126,6 +134,31 @@ public ConnectionImpl(String url, Properties info) throws SQLException { } } + private JsonParserFactory instantiateJsonParserFactory(String className) throws SQLException { + if (className == null || className.trim().isEmpty()) { + throw new SQLException("Value of '" + DriverProperties.JSON_PARSER_FACTORY.getKey() + + "' is empty string but should be a FQN of factory class."); + } + try { + Class factoryClass = this.getClass().getClassLoader().loadClass(className); + if (!JsonParserFactory.class.isAssignableFrom(factoryClass)) { + throw new SQLException("Class '" + className + "' should implement " + JsonParserFactory.class.getName()); + } + + return (JsonParserFactory) factoryClass.getDeclaredConstructor().newInstance(); + } catch (ClassNotFoundException e) { + throw new SQLException("Class '" + className + "' (implementing JsonParserFactory ) not found. Check " + + DriverProperties.JSON_PARSER_FACTORY.getKey() + " property", e); + } catch (InvocationTargetException | InstantiationException | IllegalAccessException | + NoSuchMethodException e) { + throw new SQLException("Failed to instantiate '" + className + "'. Check class implementation.", e); + } + } + + public JsonParserFactory getJsonParserFactory() { + return jsonParserFactory; + } + public SqlParserFacade getSqlParser() { return sqlParser; } @@ -534,7 +567,7 @@ public Properties getClientInfo() throws SQLException { * Creating multilevel arrays may be confusing. * Spec doesn't tell much about it so there may be different variants. * Note: createArrayOf() expect type name be for element of the array and for - * Array(Array(Int8)) it should be Int8 according to spec. However element type + * Array(Array(Int8)) it should be Int8 according to spec. However, element type * of 1st level array is Array(Int8) * @param typeName the SQL name of the type the elements of the array map to. The typeName is a * database-specific name which may be the name of a built-in type, a user-defined type or a standard SQL type supported by this database. This diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java index f18071e3f..7beedf50b 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/DriverProperties.java @@ -129,6 +129,13 @@ public enum DriverProperties { */ CLUSTER_NAME("jdbc_cluster_name", null), + /** + * Defines which {@link com.clickhouse.client.api.data_formats.JsonParserFactory} implementation the connection + * should use when the response is in {@code JSONEachRow} format. Value is the fully-qualified class name of + * the factory class. + */ + JSON_PARSER_FACTORY("jdbc_json_parser_factory", null), + ; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java index e57282967..4a84842a2 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java @@ -1,7 +1,7 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.DataTypeUtils; -import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.data_formats.ClickHouseFormatReader; import com.clickhouse.client.api.metadata.TableSchema; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.data.ClickHouseColumn; @@ -44,6 +44,7 @@ import java.time.ZonedDateTime; import java.util.Calendar; import java.util.Collections; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; @@ -51,7 +52,7 @@ public class ResultSetImpl implements ResultSet, JdbcV2Wrapper { private static final Logger log = LoggerFactory.getLogger(ResultSetImpl.class); private ResultSetMetaDataImpl metaData; - protected ClickHouseBinaryFormatReader reader; + protected ClickHouseFormatReader reader; private QueryResponse response; private boolean closed; private final StatementImpl parentStatement; @@ -73,7 +74,7 @@ public class ResultSetImpl implements ResultSet, JdbcV2Wrapper { private Consumer onDataTransferException; - public ResultSetImpl(StatementImpl parentStatement, QueryResponse response, ClickHouseBinaryFormatReader reader, + public ResultSetImpl(StatementImpl parentStatement, QueryResponse response, ClickHouseFormatReader reader, Consumer onDataTransferException) throws SQLException { this.parentStatement = parentStatement; this.response = response; @@ -1495,8 +1496,14 @@ public T getObjectImpl(String columnLabel, Class type, Map T getObjectImpl(String columnLabel, Class type, Map) { + throw new SQLException("JSONEachRow arrays are returned as parser-native List values. " + + "Use getObject(...) to read this column."); } - return (T) JdbcUtils.convert(reader.readValue(columnLabel), type, column); + return (T) JdbcUtils.convert(value, type, column); } else { wasNull = true; return null; diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java index 2801450ed..471582b0d 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/StatementImpl.java @@ -1,11 +1,13 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.ClientConfigProperties; -import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader; +import com.clickhouse.client.api.data_formats.ClickHouseFormatReader; +import com.clickhouse.client.api.data_formats.JSONEachRowFormatReader; import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.query.QueryResponse; import com.clickhouse.client.api.query.QuerySettings; import com.clickhouse.client.api.sql.SQLUtils; +import com.clickhouse.data.ClickHouseFormat; import com.clickhouse.jdbc.internal.ExceptionUtils; import com.clickhouse.jdbc.internal.FeatureManager; import com.clickhouse.jdbc.internal.ParsedStatement; @@ -177,11 +179,22 @@ protected ResultSetImpl executeQueryImpl(String sql, QuerySettings settings) thr response = connection.getClient().query(lastStatementSql, mergedSettings).get(queryTimeout, TimeUnit.SECONDS); } - if (response.getFormat().isText()) { - throw new SQLException("Only RowBinaryWithNameAndTypes is supported for output format. Please check your query.", + ClickHouseFormatReader reader; + if (response.getFormat() == ClickHouseFormat.JSONEachRow) { + if (connection.getJsonParserFactory() == null) { + throw new SQLException("Response is in JSONEachRow format, but " + + DriverProperties.JSON_PARSER_FACTORY.getKey() + " is not configured. Set " + + DriverProperties.JSON_PARSER_FACTORY.getKey() + " to a JsonParserFactory implementation.", + ExceptionUtils.SQL_STATE_CLIENT_ERROR); + } + reader = new JSONEachRowFormatReader(connection.getJsonParserFactory().createJsonParser(response.getInputStream())); + } else if (!response.getFormat().isText()) { + reader = connection.getClient().newBinaryFormatReader(response); + } else { + throw new SQLException("Only RowBinaryWithNameAndTypes and JSONEachRow are supported for output format. Please check your query.", ExceptionUtils.SQL_STATE_CLIENT_ERROR); } - ClickHouseBinaryFormatReader reader = connection.getClient().newBinaryFormatReader(response); + if (reader.getSchema() == null) { long writtenRows = 0L; try { diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index d4d1c4987..2bc43ac1b 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -18,12 +18,7 @@ import java.nio.charset.StandardCharsets; import java.sql.DriverPropertyInfo; import java.sql.SQLException; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; +import java.util.*; import java.util.function.Supplier; import java.util.regex.Pattern; import java.util.stream.Collectors; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java index 4c3262dd6..f5e28c6dd 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ResultSetImplTest.java @@ -1,6 +1,7 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; import com.clickhouse.data.ClickHouseVersion; import org.testng.Assert; import org.testng.annotations.Test; @@ -25,6 +26,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.util.List; import java.util.Properties; import static org.testng.Assert.assertEquals; @@ -264,6 +266,50 @@ public void testCursorPosition() throws SQLException { } } + @Test(groups = {"integration"}) + public void testJsonEachRowCursorPositionDetectsLastRow() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), JacksonJsonParserFactory.class.getName()); + properties.setProperty(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), "JSONEachRow"); + try (Connection conn = getJdbcConnection(properties); Statement stmt = conn.createStatement()) { + int limit = 13; + try (ResultSet rs = stmt.executeQuery("SELECT number FROM system.numbers LIMIT " + limit)) { + + for (int i = 0; i < limit - 1; i++) { + Assert.assertTrue(rs.next()); + Assert.assertFalse(rs.isLast()); + } + + Assert.assertTrue(rs.next()); + Assert.assertTrue(rs.isLast()); + + Assert.assertFalse(rs.next()); + Assert.assertFalse(rs.isLast()); + } + } + } + + @Test(groups = {"integration"}) + public void testJsonEachRowGetObjectReturnsParserNativeArray() throws SQLException { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), JacksonJsonParserFactory.class.getName()); + try (Connection conn = getJdbcConnection(properties); Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT [1, 2, 3] AS arr FORMAT JSONEachRow")) { + Assert.assertTrue(rs.next()); + Object value = rs.getObject("arr"); + Assert.assertTrue(value instanceof List, "Expected parser-native List but got " + value.getClass()); + + List list = (List) value; + Assert.assertEquals(list.size(), 3); + Assert.assertEquals(((Number) list.get(0)).intValue(), 1); + Assert.assertEquals(((Number) list.get(1)).intValue(), 2); + Assert.assertEquals(((Number) list.get(2)).intValue(), 3); + + Assert.expectThrows(SQLException.class, () -> rs.getArray("arr")); + } + } + } + @Test(groups = {"integration"}) public void testFetchDirectionsAndSize() throws SQLException { diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java index 4995caf16..347102372 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/StatementTest.java @@ -1,6 +1,9 @@ package com.clickhouse.jdbc; import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.data_formats.JsonParserFactory; +import com.clickhouse.client.api.data_formats.GsonJsonParserFactory; +import com.clickhouse.client.api.data_formats.JacksonJsonParserFactory; import com.clickhouse.client.api.internal.ServerSettings; import com.clickhouse.client.api.query.GenericRecord; import com.clickhouse.data.ClickHouseVersion; @@ -9,7 +12,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; -import org.testng.SkipException; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -719,6 +721,44 @@ public void testTextFormatInResponse() throws Exception { } } + @Test(groups = {"integration"}, dataProvider = "testJSONEachRowFormatDP") + public void testJSONEachRowFormat(Class parserFactory) throws Exception { + Properties properties = new Properties(); + properties.setProperty(DriverProperties.JSON_PARSER_FACTORY.getKey(), parserFactory.getName()); + try (Connection conn = getJdbcConnection(properties)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("SELECT 1 AS num, 'test' AS str FORMAT JSONEachRow")) { + assertTrue(rs.next()); + assertEquals(rs.getInt("num"), 1); + assertEquals(rs.getString("str"), "test"); + assertFalse(rs.next()); + } + } + } + } + + @Test(groups = {"integration"}) + public void testJSONEachRowFormatRequiresParserFactory() throws Exception { + try (Connection conn = getJdbcConnection(); + Statement stmt = conn.createStatement()) { + try { + stmt.executeQuery("SELECT 1 AS num FORMAT JSONEachRow"); + fail("Expected SQLException"); + } catch (SQLException e) { + assertTrue(e.getMessage().contains(DriverProperties.JSON_PARSER_FACTORY.getKey()), + "Unexpected message: " + e.getMessage()); + } + } + } + + @DataProvider + public static Object[][] testJSONEachRowFormatDP() { + return new Object[][] { + {JacksonJsonParserFactory.class}, + {GsonJsonParserFactory.class}, + }; + } + @Test(groups = "integration") void testWithClause() throws Exception { int count = 0; diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java index a5a3e972f..ef368524e 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/internal/JdbcConfigurationTest.java @@ -122,7 +122,8 @@ public void testParseURLValid(String jdbcURL, Properties properties, { JdbcConfiguration configuration = new JdbcConfiguration(jdbcURL, properties); assertEquals(configuration.getConnectionUrl(), connectionURL, "URL: " + jdbcURL); - assertEquals(configuration.clientProperties, expectedClientProps, "URL: " + jdbcURL); + assertEquals(configuration.clientProperties, expectedClientProps, "expected: " + expectedClientProps + + " actual: " + configuration.clientProperties); Client.Builder bob = new Client.Builder(); configuration.applyClientProperties(bob); Client client = bob.build();