package ch;
import com.clickhouse.client.api.Client;
import com.clickhouse.client.api.insert.*;
import com.clickhouse.client.api.metadata.TableSchema;
import com.clickhouse.client.api.query.GenericRecord;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.*;
import org.testcontainers.utility.DockerImageName;
import java.util.List;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.*;
@Testcontainers
public class ClickHouseClientTest {
private static final DockerImageName DOCKER_IMAGE_NAME =
DockerImageName.parse("clickhouse/clickhouse-server").withTag("24.9-alpine");
@Container
GenericContainer container = new GenericContainer(DOCKER_IMAGE_NAME).withExposedPorts(8123);
public static class Pojo {
private long a;
public Pojo(long a) {
this.a = a;
}
public long getA() {
return a;
}
public void setA(long a) {
this.a = a;
}
}
@Test
void testDocumentation() throws Exception {
Client client = new Client.Builder()
.addEndpoint("http://" + container.getHost() + ":" + container.getMappedPort(8123))
.setUsername("default")
.setPassword("")
.compressServerResponse(false)
.build();
assertDoesNotThrow(() -> client.execute("""
CREATE TABLE test_table
( A Int64 )
ENGINE = MergeTree
ORDER BY A
SETTINGS non_replicated_deduplication_window = 100;
""").get(1, TimeUnit.SECONDS));
TableSchema schema = client.getTableSchema("test_table");
client.register(Pojo.class, schema);
InsertResponse response1 = client.insert("test_table", List.of(new Pojo(1L)), new InsertSettings().setDeduplicationToken("test"))
.get(1, TimeUnit.SECONDS);
assertEquals(1L, response1.getWrittenRows());
InsertResponse response2 = client.insert("test_table", List.of(new Pojo(1L)), new InsertSettings().setDeduplicationToken("test1"))
.get(1, TimeUnit.SECONDS);
assertEquals(1L, response2.getWrittenRows());
InsertResponse response3 = client.insert("test_table", List.of(new Pojo(2L)), new InsertSettings().setDeduplicationToken("test"))
.get(1, TimeUnit.SECONDS);
assertEquals(1L, response3.getWrittenRows());
List<GenericRecord> records = client.queryAll("SELECT * FROM test_table");
assertEquals(2, records.size());
Long[] values = records.stream().map(r -> r.getLong(1)).toArray(Long[]::new);
assertArrayEquals(new Long[]{1L, 1L}, values); // actual [1, 2]
}
}
Test output
```
> Task :compileTestJava
> Task :testClasses
[Test worker] INFO org.testcontainers.images.PullPolicy - Image pull policy will be performed by: DefaultPullPolicy()
[Test worker] INFO org.testcontainers.utility.ImageNameSubstitutor - Image name substitution will be performed by: DefaultImageNameSubstitutor (composite of 'ConfigurationFileImageNameSubstitutor' and 'PrefixingImageNameSubstitutor')
[Test worker] INFO org.testcontainers.DockerClientFactory - Testcontainers version: 1.20.3
[Test worker] INFO org.testcontainers.dockerclient.DockerClientProviderStrategy - Loaded org.testcontainers.dockerclient.UnixSocketClientProviderStrategy from ~/.testcontainers.properties, will try it first
[Test worker] INFO org.testcontainers.dockerclient.DockerClientProviderStrategy - Found Docker environment with local Unix socket (unix:///var/run/docker.sock)
[Test worker] INFO org.testcontainers.DockerClientFactory - Docker host IP address is localhost
[Test worker] INFO org.testcontainers.DockerClientFactory - Connected to docker:
Server Version: 27.2.0
API Version: 1.47
Operating System: Docker Desktop
Total Memory: 11951 MB
Labels:
com.docker.desktop.address=unix:///Users/joschi/Library/Containers/com.docker.docker/Data/docker-cli.sock
[Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Creating container for image: testcontainers/ryuk:0.9.0
[Test worker] INFO org.testcontainers.utility.RegistryAuthLocator - Failure when attempting to lookup auth config. Please ignore if you don't have images in an authenticated registry. Details: (dockerImageName: testcontainers/ryuk:0.9.0, configFile: /Users/joschi/.docker/config.json, configEnv: DOCKER_AUTH_CONFIG). Falling back to docker-java default behaviour. Exception message: Unrecognized token 'credentials': was expecting ('true', 'false' or 'null')
at [Source: credentials not found in native keychain; line: 1, column: 12]
[Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Container testcontainers/ryuk:0.9.0 is starting: 09955f62f364318a8feb30b01110c9c4021ac3c84d87a98847eb189458f346e6
[Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Container testcontainers/ryuk:0.9.0 started in PT0.259562S
[Test worker] INFO org.testcontainers.utility.RyukResourceReaper - Ryuk started - will monitor and terminate Testcontainers containers on JVM exit
[Test worker] INFO org.testcontainers.DockerClientFactory - Checking the system...
[Test worker] INFO org.testcontainers.DockerClientFactory - ✔︎ Docker server version should be at least 1.6.0
[Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Creating container for image: clickhouse/clickhouse-server:24.9-alpine
[Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Container clickhouse/clickhouse-server:24.9-alpine is starting: b91f1da8f3bda808031b2955a6c04fe230964ad866bd9d1acc67ca01303d6987
[Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Container clickhouse/clickhouse-server:24.9-alpine started in PT0.413358S
[Test worker] INFO com.clickhouse.client.api.Client - Using server timezone: UTC
[Test worker] INFO com.clickhouse.client.api.Client - Connection reuse strategy: FIFO
[Test worker] INFO com.clickhouse.client.api.Client - client compression: false, server compression: false, http compression: false
[Test worker] INFO com.clickhouse.client.api.Client - Using new http client implementation
array contents differ at index [1], expected: <1> but was: <2>
Expected :1
Actual :2
org.opentest4j.AssertionFailedError: array contents differ at index [1], expected: <1> but was: <2>
at org.junit.jupiter.api.AssertionFailureBuilder.build(AssertionFailureBuilder.java:151)
at org.junit.jupiter.api.AssertionFailureBuilder.buildAndThrow(AssertionFailureBuilder.java:132)
at org.junit.jupiter.api.AssertArrayEquals.failArraysNotEqual(AssertArrayEquals.java:440)
at org.junit.jupiter.api.AssertArrayEquals.assertArrayElementsEqual(AssertArrayEquals.java:389)
at org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:346)
at org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:159)
at org.junit.jupiter.api.AssertArrayEquals.assertArrayEquals(AssertArrayEquals.java:155)
at org.junit.jupiter.api.Assertions.assertArrayEquals(Assertions.java:1456)
at ch.ClickHouseClientTest.testDocumentation(ClickHouseClientTest.java:76)
at java.base/java.lang.reflect.Method.invoke(Method.java:580)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1596)
at java.base/java.util.ArrayList.forEach(ArrayList.java:1596)
> Task :test FAILED
ClickHouseClientTest > testDocumentation() FAILED
org.opentest4j.AssertionFailedError at ClickHouseClientTest.java:76
1 test completed, 1 failed
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':test'.
BUILD FAILED in 1s
2 actionable tasks: 2 executed
```
Expected behaviour
Code example
assertArrayEquals(new Long[]{1L, 1L}, values); // should succeed with actual [1, 1]
Using the example from the documentation with clickhouse-client works as expected:
# docker run -d -it --rm --name ch -p 8123:8123 clickhouse/clickhouse-server:24.9-alpine
6c99738ec5fa297a2a37ceae592bdf4e89d24211f8bc5511e77d54e245748d50
# docker exec -it ch clickhouse-client
ClickHouse client version 24.9.2.42 (official build).
Connecting to localhost:9000 as user default.
Connected to ClickHouse server version 24.9.2.
6c99738ec5fa :) CREATE TABLE test_table
( A Int64 )
ENGINE = MergeTree
ORDER BY A
SETTINGS non_replicated_deduplication_window = 100;
CREATE TABLE test_table
(
`A` Int64
)
ENGINE = MergeTree
ORDER BY A
SETTINGS non_replicated_deduplication_window = 100
Query id: 52995350-2cf8-4d31-a825-20bb65681ff7
Ok.
0 rows in set. Elapsed: 0.010 sec.
6c99738ec5fa :) INSERT INTO test_table SETTINGS insert_deduplication_token = 'test' VALUES (1);
INSERT INTO test_table
SETTINGS insert_deduplication_token = 'test'
FORMAT Values
Query id: bd548b4c-703b-48d8-90c4-954e4d605425
Ok.
1 row in set. Elapsed: 0.008 sec.
6c99738ec5fa :) INSERT INTO test_table SETTINGS insert_deduplication_token = 'test1' VALUES (1);
INSERT INTO test_table
SETTINGS insert_deduplication_token = 'test1'
FORMAT Values
Query id: c7fca64e-0f43-4b2a-850d-2a1381ac1c3d
Ok.
1 row in set. Elapsed: 0.007 sec.
6c99738ec5fa :) INSERT INTO test_table SETTINGS insert_deduplication_token = 'test' VALUES (2);
INSERT INTO test_table
SETTINGS insert_deduplication_token = 'test'
FORMAT Values
Query id: 9aa0a717-7764-4ed4-8cf9-b478f79f9e58
Ok.
1 row in set. Elapsed: 0.006 sec.
6c99738ec5fa :) SELECT * FROM test_table;
SELECT *
FROM test_table
Query id: efa2dc76-0cf9-4f71-ad7b-64aaf7ca8906
┌─A─┐
1. │ 1 │
└───┘
┌─A─┐
2. │ 1 │
└───┘
2 rows in set. Elapsed: 0.005 sec.
6c99738ec5fa :) Bye.
Error log
None. The insert settings are simply not passed along to ClickHouse.
Describe the bug
client-v2
seems to ignore most settings provided viaInsertSettings
onClient#insert(...)
, such asinsert_deduplication_token
.This looks very similar to the issue reported in https://github.com/ClickHouse/clickhouse-java/issues/1868 and fixed in https://github.com/ClickHouse/clickhouse-java/pull/1869.
Steps to reproduce
Minimal reproducer using the example from the documentation for
insert_deduplication_token
:build.gradle.kts
src/test/java/ch/ClickHouseClientTest.java
Test output
``` > Task :compileTestJava > Task :testClasses [Test worker] INFO org.testcontainers.images.PullPolicy - Image pull policy will be performed by: DefaultPullPolicy() [Test worker] INFO org.testcontainers.utility.ImageNameSubstitutor - Image name substitution will be performed by: DefaultImageNameSubstitutor (composite of 'ConfigurationFileImageNameSubstitutor' and 'PrefixingImageNameSubstitutor') [Test worker] INFO org.testcontainers.DockerClientFactory - Testcontainers version: 1.20.3 [Test worker] INFO org.testcontainers.dockerclient.DockerClientProviderStrategy - Loaded org.testcontainers.dockerclient.UnixSocketClientProviderStrategy from ~/.testcontainers.properties, will try it first [Test worker] INFO org.testcontainers.dockerclient.DockerClientProviderStrategy - Found Docker environment with local Unix socket (unix:///var/run/docker.sock) [Test worker] INFO org.testcontainers.DockerClientFactory - Docker host IP address is localhost [Test worker] INFO org.testcontainers.DockerClientFactory - Connected to docker: Server Version: 27.2.0 API Version: 1.47 Operating System: Docker Desktop Total Memory: 11951 MB Labels: com.docker.desktop.address=unix:///Users/joschi/Library/Containers/com.docker.docker/Data/docker-cli.sock [Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Creating container for image: testcontainers/ryuk:0.9.0 [Test worker] INFO org.testcontainers.utility.RegistryAuthLocator - Failure when attempting to lookup auth config. Please ignore if you don't have images in an authenticated registry. Details: (dockerImageName: testcontainers/ryuk:0.9.0, configFile: /Users/joschi/.docker/config.json, configEnv: DOCKER_AUTH_CONFIG). Falling back to docker-java default behaviour. Exception message: Unrecognized token 'credentials': was expecting ('true', 'false' or 'null') at [Source: credentials not found in native keychain; line: 1, column: 12] [Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Container testcontainers/ryuk:0.9.0 is starting: 09955f62f364318a8feb30b01110c9c4021ac3c84d87a98847eb189458f346e6 [Test worker] INFO tc.testcontainers/ryuk:0.9.0 - Container testcontainers/ryuk:0.9.0 started in PT0.259562S [Test worker] INFO org.testcontainers.utility.RyukResourceReaper - Ryuk started - will monitor and terminate Testcontainers containers on JVM exit [Test worker] INFO org.testcontainers.DockerClientFactory - Checking the system... [Test worker] INFO org.testcontainers.DockerClientFactory - ✔︎ Docker server version should be at least 1.6.0 [Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Creating container for image: clickhouse/clickhouse-server:24.9-alpine [Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Container clickhouse/clickhouse-server:24.9-alpine is starting: b91f1da8f3bda808031b2955a6c04fe230964ad866bd9d1acc67ca01303d6987 [Test worker] INFO tc.clickhouse/clickhouse-server:24.9-alpine - Container clickhouse/clickhouse-server:24.9-alpine started in PT0.413358S [Test worker] INFO com.clickhouse.client.api.Client - Using server timezone: UTC [Test worker] INFO com.clickhouse.client.api.Client - Connection reuse strategy: FIFO [Test worker] INFO com.clickhouse.client.api.Client - client compression: false, server compression: false, http compression: false [Test worker] INFO com.clickhouse.client.api.Client - Using new http client implementation array contents differ at index [1], expected: <1> but was: <2> Expected :1 Actual :2Expected behaviour
Code example
Using the example from the documentation with
clickhouse-client
works as expected:Error log
None. The insert settings are simply not passed along to ClickHouse.
Configuration
Environment
com.clickhouse:client-v2:0.7.0
openjdk 21.0.5 2024-10-15 LTS
ClickHouse server
CREATE TABLE
statements for tables involved: Seeinsert_deduplication_token