search

apache / arrow

Apache Arrow is a multi-language toolbox for accelerated data interchange and in-memory processing
https://arrow.apache.org/
Apache License 2.0
13.88k stars 3.38k forks source link

arrow flight sql jdbc drive with Lz4Compression #41457

Open Curricane opened 2 months ago

Curricane commented 2 months ago

Describe the usage question you have. Please include as many useful details as possible.

I'm new to java jdbc and flight. But i want to use flight sql jdbc drive to connect a db. this is my code

package com.test;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Properties;
import org.apache.arrow.compression.CommonsCompressionFactory;
import org.apache.arrow.compression.Lz4CompressionCodec;

public class App {
    public static void main(String[] args) {
        try {
            // Load the JDBC driver
            Class.forName("org.apache.arrow.driver.jdbc.ArrowFlightJdbcDriver");

            String url = "jdbc:arrow-flight-sql://localhost:50050?useEncryption=0";

            Properties p = new Properties();
            p.setProperty("user", "admin");
            p.setProperty("password", "password");
            Connection connection = DriverManager.getConnection(url, p);

            // Create a statement
            Statement statement = connection.createStatement();

            // Execute a query
            String query = "SELECT 1"; // Replace "your_table" with your actual table name
            ResultSet resultSet = statement.executeQuery(query);

            // Process the result set
            while (resultSet.next()) {
                System.out.println(resultSet.getInt(1)); // Adjust the index or use column names depending on your
                                                            // schema
            }

            // Close the resources
            resultSet.close();
            statement.close();
            connection.close();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

and my pom.xml

<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.fusiongrid</groupId>
    <artifactId>arrow-flight-sql-example</artifactId>
    <version>1.0-SNAPSHOT</version>
    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <build>
        <plugins>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>exec-maven-plugin</artifactId>
                <version>1.6.0</version>
                <configuration>
                    <mainClass>com.test.App</mainClass>
                    <executable>java</executable>
                    <arguments>
                        <argument>--add-opens=java.base/java.nio=ALL-UNNAMED</argument>
                        <argument>-Darrow.memory.debug.allocator=true</argument>
                    </arguments>
                </configuration>
            </plugin>
        </plugins>
    </build>

    <dependencies>
        <dependency>
            <groupId>org.apache.arrow</groupId>
            <artifactId>flight-sql-jdbc-driver</artifactId>
            <version>16.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.arrow</groupId>
            <artifactId>arrow-compression</artifactId>
            <version>16.0.0</version>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>2.0.13</version>
        </dependency>

        <dependency>
        <groupId>ch.qos.logback</groupId>
        <artifactId>logback-classic</artifactId>
        <version>1.5.5</version>
    </dependency>

    </dependencies>
</project>

when i run it, a error show like that

WARNING: Unknown module: org.apache.arrow.memory.core specified to --add-opens
4月 30, 2024 2:38:01 下午 org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.BaseAllocator <clinit>
信息: Debug mode disabled. Enable with the VM option -Darrow.memory.debug.allocator=true.
4月 30, 2024 2:38:01 下午 org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.DefaultAllocationManagerOption getDefaultAllocationManagerFactory
信息: allocation manager type not specified, using netty as the default type
4月 30, 2024 2:38:01 下午 org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.memory.CheckAllocator reportResult
信息: Using DefaultAllocationManager at memory/netty/DefaultAllocationManagerFactory.class
java.sql.SQLException: Error while executing SQL "SELECT 1": java.lang.IllegalArgumentException: Please add arrow-compression module to use CommonsCompressionFactory for LZ4_FRAME
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.Helper.createException(Helper.java:56)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.Helper.createException(Helper.java:41)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:164)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:228)
    at com.test.App.main(App.java:29)
Caused by: java.lang.RuntimeException: java.lang.IllegalArgumentException: Please add arrow-compression module to use CommonsCompressionFactory for LZ4_FRAME
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.Helper.wrap(Helper.java:37)
    at org.apache.arrow.driver.jdbc.utils.FlightEndpointDataQueue.next(FlightEndpointDataQueue.java:117)
    at org.apache.arrow.driver.jdbc.utils.FlightEndpointDataQueue.next(FlightEndpointDataQueue.java:155)
    at org.apache.arrow.driver.jdbc.ArrowFlightJdbcFlightStreamResultSet.getNextEndpointStream(ArrowFlightJdbcFlightStreamResultSet.java:260)
    at org.apache.arrow.driver.jdbc.ArrowFlightJdbcFlightStreamResultSet.loadNewFlightStream(ArrowFlightJdbcFlightStreamResultSet.java:131)
    at org.apache.arrow.driver.jdbc.ArrowFlightJdbcFlightStreamResultSet.populateData(ArrowFlightJdbcFlightStreamResultSet.java:146)
    at org.apache.arrow.driver.jdbc.ArrowFlightJdbcFlightStreamResultSet.execute(ArrowFlightJdbcFlightStreamResultSet.java:138)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.AvaticaConnection$1.execute(AvaticaConnection.java:669)
    at org.apache.arrow.driver.jdbc.ArrowFlightMetaImpl.prepareAndExecute(ArrowFlightMetaImpl.java:205)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.AvaticaConnection.prepareAndExecuteInternal(AvaticaConnection.java:677)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:157)
    ... 2 more
Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalArgumentException: Please add arrow-compression module to use CommonsCompressionFactory for LZ4_FRAME
    at java.base/java.util.concurrent.FutureTask.report(FutureTask.java:122)
    at java.base/java.util.concurrent.FutureTask.get(FutureTask.java:191)
    at org.apache.arrow.driver.jdbc.utils.FlightEndpointDataQueue.next(FlightEndpointDataQueue.java:106)
    ... 11 more
Caused by: java.lang.IllegalArgumentException: Please add arrow-compression module to use CommonsCompressionFactory for LZ4_FRAME
    at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.compression.NoCompressionCodec$Factory.createCodec(NoCompressionCodec.java:69)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.vector.VectorLoader.load(VectorLoader.java:82)
    at org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flight.FlightStream.next(FlightStream.java:270)
    at org.apache.arrow.driver.jdbc.utils.FlightEndpointDataQueue.lambda$enqueue$2(FlightEndpointDataQueue.java:187)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
    at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
    at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:317)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
    at org.apache.arrow.driver.jdbc.shaded.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
    at java.base/java.lang.Thread.run(Thread.java:1570)

Component(s)

FlightRPC

vibhatha commented 2 months ago

cc @lidavidm

lidavidm commented 2 months ago

We don't really support compression here. A PR would be welcome to enable it.