search

jaehyeon-kim / dbt-on-aws

dbt (data build tool) projects targeting AWS analytics services (redshift, glue, emr, athena) and open table formats
25 stars 8 forks source link

The runner class can't be built - POM error #2

Open isabela-angelo-ze-delivery opened 1 year ago

isabela-angelo-ze-delivery commented 1 year ago

Hello!

I am trying to use this repo to run a thrift server indefinitely but I get an error when I build the runner class.

Expected Behavior

JAR file in target path

Actual Behavior

[INFO] Error stacktraces are turned on. [INFO] Scanning for projects... [ERROR] [ERROR] Some problems were encountered while processing the POMs: [FATAL] Non-resolvable parent POM for io.jaehyeon:spark-thrift-server:1.0.0-SNAPSHOT: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM @ line 3, column 10 @ [ERROR] The build could not read 1 project -> [Help 1] org.apache.maven.project.ProjectBuildingException: Some problems were encountered while processing the POMs: [FATAL] Non-resolvable parent POM for io.jaehyeon:spark-thrift-server:1.0.0-SNAPSHOT: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM @ line 3, column 10

at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:388)
at org.apache.maven.graph.DefaultGraphBuilder.collectProjects (DefaultGraphBuilder.java:349)
at org.apache.maven.graph.DefaultGraphBuilder.getProjectsForMavenReactor (DefaultGraphBuilder.java:340)
at org.apache.maven.graph.DefaultGraphBuilder.build (DefaultGraphBuilder.java:76)
at org.apache.maven.DefaultMaven.buildGraph (DefaultMaven.java:453)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:196)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:172)
at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:100)
at org.apache.maven.cli.MavenCli.execute (MavenCli.java:821)
at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:270)
at org.apache.maven.cli.MavenCli.main (MavenCli.java:192)
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke (DirectMethodHandleAccessor.java:104)
at java.lang.reflect.Method.invoke (Method.java:578)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:282)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:225)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:406)
at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:347)

[ERROR] [ERROR] The project io.jaehyeon:spark-thrift-server:1.0.0-SNAPSHOT (/Users/zeuser/codes/dbt-on-aws/emr-eks/hive-on-spark-in-kubernetes/examples/spark-thrift-server/pom.xml) has 1 error [ERROR] Non-resolvable parent POM for io.jaehyeon:spark-thrift-server:1.0.0-SNAPSHOT: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT and 'parent.relativePath' points at wrong local POM @ line 3, column 10 -> [Help 2] org.apache.maven.model.resolution.UnresolvableModelException: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:174) at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:218) at org.apache.maven.model.building.DefaultModelBuilder.readParentExternally (DefaultModelBuilder.java:1009) at org.apache.maven.model.building.DefaultModelBuilder.readParent (DefaultModelBuilder.java:801) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:327) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:243) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:446) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:409) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:366) at org.apache.maven.graph.DefaultGraphBuilder.collectProjects (DefaultGraphBuilder.java:349) at org.apache.maven.graph.DefaultGraphBuilder.getProjectsForMavenReactor (DefaultGraphBuilder.java:340) at org.apache.maven.graph.DefaultGraphBuilder.build (DefaultGraphBuilder.java:76) at org.apache.maven.DefaultMaven.buildGraph (DefaultMaven.java:453) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:196) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:172) at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:100) at org.apache.maven.cli.MavenCli.execute (MavenCli.java:821) at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:270) at org.apache.maven.cli.MavenCli.main (MavenCli.java:192) at jdk.internal.reflect.DirectMethodHandleAccessor.invoke (DirectMethodHandleAccessor.java:104) at java.lang.reflect.Method.invoke (Method.java:578) at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:282) at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:225) at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:406) at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:347) Caused by: org.eclipse.aether.resolution.ArtifactResolutionException: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolve (DefaultArtifactResolver.java:506) at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifacts (DefaultArtifactResolver.java:278) at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifact (DefaultArtifactResolver.java:255) at org.eclipse.aether.internal.impl.DefaultRepositorySystem.resolveArtifact (DefaultRepositorySystem.java:296) at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:172) at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:218) at org.apache.maven.model.building.DefaultModelBuilder.readParentExternally (DefaultModelBuilder.java:1009) at org.apache.maven.model.building.DefaultModelBuilder.readParent (DefaultModelBuilder.java:801) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:327) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:243) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:446) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:409) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:366) at org.apache.maven.graph.DefaultGraphBuilder.collectProjects (DefaultGraphBuilder.java:349) at org.apache.maven.graph.DefaultGraphBuilder.getProjectsForMavenReactor (DefaultGraphBuilder.java:340) at org.apache.maven.graph.DefaultGraphBuilder.build (DefaultGraphBuilder.java:76) at org.apache.maven.DefaultMaven.buildGraph (DefaultMaven.java:453) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:196) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:172) at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:100) at org.apache.maven.cli.MavenCli.execute (MavenCli.java:821) at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:270) at org.apache.maven.cli.MavenCli.main (MavenCli.java:192) at jdk.internal.reflect.DirectMethodHandleAccessor.invoke (DirectMethodHandleAccessor.java:104) at java.lang.reflect.Method.invoke (Method.java:578) at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:282) at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:225) at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:406) at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:347) Caused by: org.eclipse.aether.transfer.ArtifactNotFoundException: Could not find artifact io.jaehyeon:jaehyeon-parent:pom:1.0.0-SNAPSHOT at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolve (DefaultArtifactResolver.java:496) at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifacts (DefaultArtifactResolver.java:278) at org.eclipse.aether.internal.impl.DefaultArtifactResolver.resolveArtifact (DefaultArtifactResolver.java:255) at org.eclipse.aether.internal.impl.DefaultRepositorySystem.resolveArtifact (DefaultRepositorySystem.java:296) at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:172) at org.apache.maven.project.ProjectModelResolver.resolveModel (ProjectModelResolver.java:218) at org.apache.maven.model.building.DefaultModelBuilder.readParentExternally (DefaultModelBuilder.java:1009) at org.apache.maven.model.building.DefaultModelBuilder.readParent (DefaultModelBuilder.java:801) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:327) at org.apache.maven.model.building.DefaultModelBuilder.build (DefaultModelBuilder.java:243) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:446) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:409) at org.apache.maven.project.DefaultProjectBuilder.build (DefaultProjectBuilder.java:366) at org.apache.maven.graph.DefaultGraphBuilder.collectProjects (DefaultGraphBuilder.java:349) at org.apache.maven.graph.DefaultGraphBuilder.getProjectsForMavenReactor (DefaultGraphBuilder.java:340) at org.apache.maven.graph.DefaultGraphBuilder.build (DefaultGraphBuilder.java:76) at org.apache.maven.DefaultMaven.buildGraph (DefaultMaven.java:453) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:196) at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:172) at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:100) at org.apache.maven.cli.MavenCli.execute (MavenCli.java:821) at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:270) at org.apache.maven.cli.MavenCli.main (MavenCli.java:192) at jdk.internal.reflect.DirectMethodHandleAccessor.invoke (DirectMethodHandleAccessor.java:104) at java.lang.reflect.Method.invoke (Method.java:578) at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:282) at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:225) at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:406) at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:347) [ERROR] [ERROR] Re-run Maven using the -X switch to enable full debug logging. [ERROR] [ERROR] For more information about the errors and possible solutions, please read the following articles: [ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/ProjectBuildingException [ERROR] [Help 2] http://cwiki.apache.org/confluence/display/MAVEN/UnresolvableModelException

Steps to Reproduce the Problem

  1. Clone this repo
  2. CD into emr-eks/hive-on-spark-in-kubernetes/examples/spark-thrift-server
  3. Run mvn -e -DskipTests=true clean install

Specifications

YonatanKiron commented 1 year ago

@isabela-angelo-ze-delivery Did you find any workaround? or are you able to deploy thrift in EKS somehow else?

isabela-angelo-ze-delivery commented 1 year ago

Hello @YonatanKiron ! I just deleted the parent part and the building worked. I have copied and pasted the code below. I am not sure if all the required dependencies is already in there, but I tested some spark code using the new Thrift class and it executed successfully. I only gave up using this approach with dbt because I use Hudi to write in data lake and the integration between Hudi and dbt is not working as far as I know.

The working pom:

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <groupId>io.jaehyeon.hive</groupId>
    <version>1.0.0-SNAPSHOT</version>
    <modelVersion>4.0.0</modelVersion>
    <artifactId>spark-thrift-server</artifactId>
    <name>Spark Thrift Server</name>
    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
        <junit.version>4.12</junit.version>
        <scala.version>2.12.10</scala.version>
        <scala.binary.version>2.12</scala.binary.version>
        <spark.version>3.3.0</spark.version>
        <hadoop.version>3.2.1</hadoop.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>${junit.version}</version>
            <scope>test</scope>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-compiler</artifactId>
            <version>${scala.version}</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-reflect</artifactId>
            <version>${scala.version}</version>
        </dependency>
        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
        </dependency>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scalap</artifactId>
            <version>${scala.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-yarn_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-repl_${scala.binary.version}</artifactId>
            <version>${spark.version}</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-aws</artifactId>
            <version>${hadoop.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-core</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-annotations</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>${hadoop.version}</version>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-javadoc-plugin</artifactId>
                <version>2.9</version>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-release-plugin</artifactId>
                <version>2.4.1</version>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-source-plugin</artifactId>
                <version>3.2.0</version>
                <executions>
                    <execution>
                        <id>attach-sources</id>
                        <phase>verify</phase>
                        <goals>
                            <goal>jar-no-fork</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.0</version>
                <configuration>
                    <artifactSet>
                        <excludes>
                            <exclude>org.apache.spark</exclude>
                            <exclude>junit:junit</exclude>
                            <exclude>jmock:*</exclude>
                            <exclude>*:xml-apis</exclude>
                            <exclude>log4j:log4j:jar:</exclude>
                        </excludes>
                    </artifactSet>
                    <createDependencyReducedPom>false</createDependencyReducedPom>
                    <shadedArtifactAttached>true</shadedArtifactAttached>
                    <shadedClassifierName>spark-job</shadedClassifierName>
                    <relocations>
                        <relocation>
                            <pattern>com.google</pattern>
                            <shadedPattern>io.jaehyeon.shaded.google</shadedPattern>
                            <excludes>
                                <exclude>com.google.common.base.Optional</exclude>
                            </excludes>
                        </relocation>
                    </relocations>
                    <transformers>
                        <transformer
                                implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
                        <transformer
                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                            <resource>META-INF/spring.handlers</resource>
                        </transformer>
                        <transformer
                                implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                            <resource>META-INF/spring.schemas</resource>
                        </transformer>
                    </transformers>
                    <filters>
                        <filter>
                            <artifact>*:*</artifact>
                            <excludes>
                                <exclude>org/datanucleus/**</exclude>
                                <exclude>META-INF/*.SF</exclude>
                                <exclude>META-INF/*.DSA</exclude>
                                <exclude>META-INF/*.RSA</exclude>
                            </excludes>
                        </filter>
                    </filters>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>