SnuK87 / keycloak-kafka

Keycloak module to produce events to kafka
Apache License 2.0
98 stars 36 forks source link

Enable SASL mechanism for kafkaProducer #25

Closed ArcticKeeper closed 1 year ago

ArcticKeeper commented 1 year ago

Hello dears. I'm totally new to programming and especially Java. I want to enable SASL authentication without ssl and i made this changes in your KafkaProducerFactory.java:

package com.github.snuk87.keycloak.kafka;

import java.util.Map;
import java.util.Properties;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.common.serialization.StringSerializer;
import org.apache.kafka.common.security.scram.ScramLoginModule;

public final class KafkaProducerFactory {

        private KafkaProducerFactory() {

        }

        public static Producer<String, String> createProducer(String clientId, String bootstrapServer,
            Map<String, Object> optionalProperties) {
                Properties props = new Properties();
                props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServer);
                props.put(ProducerConfig.CLIENT_ID_CONFIG, clientId);
                props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
                props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

                optionalProperties.forEach(props::put);

                props.put("security.protocol", "SASL_PLAINTEXT");
                props.put("sasl.mechanism", "SCRAM-SHA-512");
                props.put("sasl.jaas.config", "org.apache.kafka.common.security.scram.ScramLoginModule required username=\"user\" password=\"pass\";");

                // fix Class org.apache.kafka.common.serialization.StringSerializer could not be
                // found. see https://stackoverflow.com/a/50981469
                Thread.currentThread().setContextClassLoader(null);

                return new KafkaProducer<>(props);
        }
}

It was built succesfully but when I'm trying to start kafka-spi in keycloak i'm recieving this error:

keycloak_1  | 2023-04-14 14:02:10,403 ERROR [org.keycloak.services.error.KeycloakErrorHandler] (executor-thread-0) Uncaught server error: org.apache.kafka.common.KafkaException: Failed to construct kafka producer
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:440)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:291)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:318)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:303)
keycloak_1  |   at com.github.snuk87.keycloak.kafka.KafkaProducerFactory.createProducer(KafkaProducerFactory.java:36)
keycloak_1  |   at com.github.snuk87.keycloak.kafka.KafkaEventListenerProvider.<init>(KafkaEventListenerProvider.java:53)
keycloak_1  |   at com.github.snuk87.keycloak.kafka.KafkaEventListenerProviderFactory.create(KafkaEventListenerProviderFactory.java:29)
keycloak_1  |   at com.github.snuk87.keycloak.kafka.KafkaEventListenerProviderFactory.create(KafkaEventListenerProviderFactory.java:12)
keycloak_1  |   at org.keycloak.services.DefaultKeycloakSession.getProvider(DefaultKeycloakSession.java:287)
keycloak_1  |   at jdk.internal.reflect.GeneratedMethodAccessor20.invoke(Unknown Source)
keycloak_1  |   at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
keycloak_1  |   at java.base/java.lang.reflect.Method.invoke(Method.java:566)
keycloak_1  |   at org.jboss.resteasy.core.ContextParameterInjector$GenericDelegatingProxy.invoke(ContextParameterInjector.java:166)
keycloak_1  |   at com.sun.proxy.$Proxy56.getProvider(Unknown Source)
keycloak_1  |   at org.keycloak.services.resources.admin.AdminEventBuilder.lambda$addListeners$0(AdminEventBuilder.java:101)
keycloak_1  |   at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.accept(ForEachOps.java:183)
keycloak_1  |   at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:177)
keycloak_1  |   at java.base/java.util.Iterator.forEachRemaining(Iterator.java:133)
keycloak_1  |   at java.base/java.util.Spliterators$IteratorSpliterator.forEachRemaining(Spliterators.java:1801)
keycloak_1  |   at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484)
keycloak_1  |   at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474)
keycloak_1  |   at java.base/java.util.stream.ForEachOps$ForEachOp.evaluateSequential(ForEachOps.java:150)
keycloak_1  |   at java.base/java.util.stream.ForEachOps$ForEachOp$OfRef.evaluateSequential(ForEachOps.java:173)
keycloak_1  |   at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234)
keycloak_1  |   at java.base/java.util.stream.ReferencePipeline.forEach(ReferencePipeline.java:497)
keycloak_1  |   at org.keycloak.services.resources.admin.AdminEventBuilder.addListeners(AdminEventBuilder.java:100)
keycloak_1  |   at org.keycloak.services.resources.admin.AdminEventBuilder.refreshRealmEventsConfig(AdminEventBuilder.java:84)
keycloak_1  |   at org.keycloak.services.resources.admin.RealmAdminResource.updateRealmEventsConfig(RealmAdminResource.java:705)
keycloak_1  |   at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
keycloak_1  |   at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
keycloak_1  |   at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
keycloak_1  |   at java.base/java.lang.reflect.Method.invoke(Method.java:566)
keycloak_1  |   at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:170)
keycloak_1  |   at org.jboss.resteasy.core.MethodInjectorImpl.invoke(MethodInjectorImpl.java:130)
keycloak_1  |   at org.jboss.resteasy.core.ResourceMethodInvoker.internalInvokeOnTarget(ResourceMethodInvoker.java:660)
keycloak_1  |   at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTargetAfterFilter(ResourceMethodInvoker.java:524)
keycloak_1  |   at org.jboss.resteasy.core.ResourceMethodInvoker.lambda$invokeOnTarget$2(ResourceMethodInvoker.java:474)
keycloak_1  |   at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:364)
keycloak_1  |   at org.jboss.resteasy.core.ResourceMethodInvoker.invokeOnTarget(ResourceMethodInvoker.java:476)
keycloak_1  |   at org.jboss.resteasy.core.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:434)
keycloak_1  |   at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:192)
keycloak_1  |   at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:152)
keycloak_1  |   at org.jboss.resteasy.core.ResourceLocatorInvoker.invokeOnTargetObject(ResourceLocatorInvoker.java:183)
keycloak_1  |   at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:141)
keycloak_1  |   at org.jboss.resteasy.core.ResourceLocatorInvoker.invoke(ResourceLocatorInvoker.java:32)
keycloak_1  |   at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:492)
keycloak_1  |   at org.jboss.resteasy.core.SynchronousDispatcher.lambda$invoke$4(SynchronousDispatcher.java:261)
keycloak_1  |   at org.jboss.resteasy.core.SynchronousDispatcher.lambda$preprocess$0(SynchronousDispatcher.java:161)
keycloak_1  |   at org.jboss.resteasy.core.interception.jaxrs.PreMatchContainerRequestContext.filter(PreMatchContainerRequestContext.java:364)
keycloak_1  |   at org.jboss.resteasy.core.SynchronousDispatcher.preprocess(SynchronousDispatcher.java:164)
keycloak_1  |   at org.jboss.resteasy.core.SynchronousDispatcher.invoke(SynchronousDispatcher.java:247)
keycloak_1  |   at io.quarkus.resteasy.runtime.standalone.RequestDispatcher.service(RequestDispatcher.java:73)
keycloak_1  |   at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.dispatch(VertxRequestHandler.java:151)
keycloak_1  |   at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.handle(VertxRequestHandler.java:82)
keycloak_1  |   at io.quarkus.resteasy.runtime.standalone.VertxRequestHandler.handle(VertxRequestHandler.java:42)
keycloak_1  |   at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1212)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:163)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:141)
keycloak_1  |   at io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2.handle(StaticResourcesRecorder.java:67)
keycloak_1  |   at io.quarkus.vertx.http.runtime.StaticResourcesRecorder$2.handle(StaticResourcesRecorder.java:55)
keycloak_1  |   at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1212)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:163)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:141)
keycloak_1  |   at io.quarkus.vertx.http.runtime.VertxHttpRecorder$5.handle(VertxHttpRecorder.java:380)
keycloak_1  |   at io.quarkus.vertx.http.runtime.VertxHttpRecorder$5.handle(VertxHttpRecorder.java:358)
keycloak_1  |   at io.vertx.ext.web.impl.RouteState.handleContext(RouteState.java:1212)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImplBase.iterateNext(RoutingContextImplBase.java:163)
keycloak_1  |   at io.vertx.ext.web.impl.RoutingContextImpl.next(RoutingContextImpl.java:141)
keycloak_1  |   at org.keycloak.quarkus.runtime.integration.web.QuarkusRequestFilter.lambda$createBlockingHandler$1(QuarkusRequestFilter.java:90)
keycloak_1  |   at io.vertx.core.impl.ContextImpl.lambda$null$0(ContextImpl.java:159)
keycloak_1  |   at io.vertx.core.impl.AbstractContext.dispatch(AbstractContext.java:100)
keycloak_1  |   at io.vertx.core.impl.ContextImpl.lambda$executeBlocking$1(ContextImpl.java:157)
keycloak_1  |   at io.quarkus.vertx.core.runtime.VertxCoreRecorder$13.runWith(VertxCoreRecorder.java:545)
keycloak_1  |   at org.jboss.threads.EnhancedQueueExecutor$Task.run(EnhancedQueueExecutor.java:2449)
keycloak_1  |   at org.jboss.threads.EnhancedQueueExecutor$ThreadBody.run(EnhancedQueueExecutor.java:1478)
keycloak_1  |   at org.jboss.threads.DelegatingRunnable.run(DelegatingRunnable.java:29)
keycloak_1  |   at org.jboss.threads.ThreadLocalResettingRunnable.run(ThreadLocalResettingRunnable.java:29)
keycloak_1  |   at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
keycloak_1  |   at java.base/java.lang.Thread.run(Thread.java:829)
keycloak_1  | Caused by: org.apache.kafka.common.KafkaException: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:184)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:192)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.clientChannelBuilder(ChannelBuilders.java:81)
keycloak_1  |   at org.apache.kafka.clients.ClientUtils.createChannelBuilder(ClientUtils.java:105)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.newSender(KafkaProducer.java:448)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:429)
keycloak_1  |   ... 78 more
keycloak_1  | Caused by: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:731)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:672)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:670)
keycloak_1  |   at java.base/java.security.AccessController.doPrivileged(Native Method)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:670)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:581)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.AbstractLogin.login(AbstractLogin.java:60)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.<init>(LoginManager.java:62)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.acquireLoginManager(LoginManager.java:105)
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:170)
keycloak_1  |   ... 83 more

Please help me understand what am I doing wrong?

ArcticKeeper commented 1 year ago

@SnuK87 Hello dear. Can you please help with this?

SnuK87 commented 1 year ago

There is no need to change the code to configure the kafka client. You can do it through parameters or env variables. It's described here. In your case you can add --spi-events-listener-kafka-security-protocol SASL_PLAINTEXT as parameter.

I assume there is an issue with the class loader workaround here: https://github.com/SnuK87/keycloak-kafka/blob/master/src/main/java/com/github/snuk87/keycloak/kafka/KafkaStandardProducerFactory.java#L26

Do you have a docker-compose or anything where I can test this?

ArcticKeeper commented 1 year ago

There is no need to change the code to configure the kafka client. You can do it through parameters or env variables. It's described here. In your case you can add --spi-events-listener-kafka-security-protocol SASL_PLAINTEXT as parameter.

I assume there is an issue with the class loader workaround here: https://github.com/SnuK87/keycloak-kafka/blob/master/src/main/java/com/github/snuk87/keycloak/kafka/KafkaStandardProducerFactory.java#L26

Do you have a docker-compose or anything where I can test this?

Thank you for your reply. I'm using cloud managed kafka instance with sasl_mechanism: "SCRAM-SHA-512" and security_protocol: SASL_PLAINTEXT. I've already tried to add this --spi-events-listener-kafka-security-protocol SASL_PLAINTEXT in the docker image entrypoint but he accepts only jaas-template.

i'm using your source code from version 1.1.1 cause our production keycloak cannot work with 17 java compiler. here is my keycloak compose:

version: '3'
services:
  keycloak:
    image: keycloak_with_kafka:latest
    ports:
      - "8080:8080"
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: "pass"
      KEYCLOAK_LOGLEVEL: "DEBUG"
      KAFKA_TOPIC: keycloak-events
      KAFKA_CLIENT_ID: keycloak
      KAFKA_BOOTSTRAP_SERVERS: "kafka-instance:9092"
      KAFKA_EVENTS: LOGIN_ERROR,LOGIN,REGISTER,LOGOUT
    command: start-dev

I was built image from source code with changes above.

ArcticKeeper commented 1 year ago

@SnuK87 Hello. as you ask i'm sending docker-compose with keycloak and kafka

version: '3'
services:
  keycloak:
    depends_on:
      - "kafka"
    image: keycloak-kafka:latest
    ports:
      - "8080:8080"
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: "pass"
      KEYCLOAK_LOGLEVEL: "DEBUG"
      KAFKA_TOPIC: keycloak-events
      KAFKA_CLIENT_ID: keycloak
      KAFKA_BOOTSTRAP_SERVERS: "kafka:9094"
      KAFKA_EVENTS: LOGIN_ERROR,LOGIN,REGISTER,LOGOUT
    command: start-dev '--spi-events-listener-kafka-sasl-login-class org.apache.kafka.common.security.plain.PlainLoginModule' '--spi-events-listener-kafka-sasl-mechanism PLAIN' '--spi-events-listener-kafka-security-protocol SASL_PLAINTEXT' "--spi-events-listener-kafka-sasl-jaas-config org.apache.kafka.common.security.plain.PlainLoginModule\ required\ username=\"user\"\ password=\"user\"\;"

  zookeeper:
    restart: always
    image: bitnami/zookeeper:latest
    ports:
      - "2181:2181"
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes

  kafka:
    depends_on:
      - "zookeeper"
    image: bitnami/kafka:latest
    ports:
      - "9092:9092"
      - "9094:9094"
    environment:
      KAFKA_CFG_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://kafka:9094
      KAFKA_CFG_LISTENERS: INSIDE://:9092,OUTSIDE://:9094
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:SASL_PLAINTEXT
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INSIDE
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CREATE_TOPICS: "keycloak-events:1:1"
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_CLIENT_USERS: user
      KAFKA_CLIENT_PASSWORDS: pass
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./kafka_jaas.conf:/opt/bitnami/kafka/conf/kafka_jaas.conf:ro

This is "kafka_jaas.conf" contents:

 KafkaServer {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="keycloak"
    password="keycloak";
};

When i try to enable kafka plugin from keycloak it crashes with this error:

keycloak_1   | Caused by: org.apache.kafka.common.KafkaException: java.lang.ClassCastException: class org.apache.kafka.common.security.plain.PlainLoginModule cannot be cast to class org.apache.kafka.common.security.auth.Login (org.apache.kafka.common.security.plain.PlainLoginModule and org.apache.kafka.common.security.auth.Login are in unnamed module of loader io.quarkus.bootstrap.runner.RunnerClassLoader @7a7b0070)
ArcticKeeper commented 1 year ago

Hello again, @SnuK87. Any news about this problem? I've build the latest 1.1.3 version of this module for 19.0.1 keycloak. Here is the configuration of docker-compose:

version: '3'
services:
  keycloak:
    image: keykafka:1.1.3
    ports:
      - "8080:8080"
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: "password"
      KEYCLOAK_LOGLEVEL: "DEBUG"
      KAFKA_TOPIC: keycloak-events
      KAFKA_CLIENT_ID: keycloak
      KAFKA_BOOTSTRAP_SERVERS: "kafka-nstance:9092"
      KAFKA_EVENTS: LOGIN_ERROR,LOGIN,REGISTER,LOGOUT
    command: start-dev "--spi-events-listener-kafka-sasl-jaas-config ./kafka_jaas.conf" '--spi-events-listener-kafka-sasl-mechanism PLAIN' '--spi-events-listener-kafka-security-protocol SASL_PLAINTEXT'

This is the exception i get when i use this jaas_config:

KafkaClient {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="keycloak"
    password="keycloak";
};
keycloak_1  | 2023-05-05 11:43:05,242 INFO  [org.apache.kafka.clients.producer.KafkaProducer] (executor-thread-0) [Producer clientId=keycloak] Instantiated an idempotent producer.
keycloak_1  | 2023-05-05 11:43:05,263 INFO  [org.apache.kafka.clients.producer.KafkaProducer] (executor-thread-0) [Producer clientId=keycloak] Closing the Kafka producer with timeoutMillis = 0 ms.
keycloak_1  | 2023-05-05 11:43:05,265 INFO  [org.apache.kafka.common.metrics.Metrics] (executor-thread-0) Metrics scheduler closed
keycloak_1  | 2023-05-05 11:43:05,266 INFO  [org.apache.kafka.common.metrics.Metrics] (executor-thread-0) Closing reporter org.apache.kafka.common.metrics.JmxReporter
keycloak_1  | 2023-05-05 11:43:05,266 INFO  [org.apache.kafka.common.metrics.Metrics] (executor-thread-0) Metrics reporters closed
keycloak_1  | 2023-05-05 11:43:05,268 INFO  [org.apache.kafka.common.utils.AppInfoParser] (executor-thread-0) App info kafka.producer for keycloak unregistered
keycloak_1  | 2023-05-05 11:43:05,277 ERROR [io.quarkus.vertx.http.runtime.QuarkusErrorHandler] (executor-thread-0) HTTP Request to /admin/realms/master/events/config failed, error id: 57ba190d-39a5-4699-9d2d-07da5d2720b0-1: java.lang.NullPointerException

Also this exception with different jaas_config:

sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required username="keycloak" password="keycloak";

also tried this jaas.conf:

KafkaClient {
    org.apache.kafka.common.security.plain.PlainLoginModule required
    username="keycloak"
    password="keycloak";
};
keycloak_1  | Caused by: java.lang.IllegalArgumentException: Login module control flag not specified in JAAS config

Not found anything about this flag in google

SnuK87 commented 1 year ago

@ArcticKeeper I tried your configuration with a standalone kafka client and I get the same exception. So I believe that this issue is not related to this module or keycloak but to your configuration or the kafka-client.

Did you get this setup running with any other kafka client?

ArcticKeeper commented 1 year ago

@SnuK87 i think there is a problem with parsing jaas_config.conf but i don't know how to check it.

SnuK87 commented 1 year ago

Then I'll close this now because the problem is not related to this module. I would recommend to ask on stackoverflow. Feel free to reopen this issue when you have a running setup that doesn't work with this module.

ArcticKeeper commented 1 year ago

@SnuK87 I'm reopening this issue cause i think i found the root of all this errors. Now i build a 1.1.4 version of your listener and this is the compose file now:

version: '3'
services:
  keycloak:
    image: keykafka:1.1.4
    ports:
      - "8080:8080"
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: "password"
      KEYCLOAK_LOGLEVEL: "DEBUG"
      KAFKA_TOPIC: keycloak-events
      KAFKA_CLIENT_ID: keycloak
      KAFKA_BOOTSTRAP_SERVERS: "kafka:9092"
      KAFKA_EVENTS: LOGIN_ERROR,LOGIN,REGISTER,LOGOUT
      KAFKA_SASL_MECHANISM: "SCRAM-SHA-256"
      KAFKA_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
      KAFKA_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.scram.ScramLoginModule required username=user password=password ;"
    command: start-dev

I'm build latest version from 1.1.4 soures and change only this in pom.xml

    <properties>
        <maven.compiler.source>11</maven.compiler.source>
        <maven.compiler.target>11</maven.compiler.target>
        <keycloak.version>19.0.1</keycloak.version>
        <kafka.version>3.4.0</kafka.version>
        <junit.version>5.9.2</junit.version>
    </properties>

This is the errors:

keycloak_1  | Caused by: org.apache.kafka.common.KafkaException: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:184)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:192)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.clientChannelBuilder(ChannelBuilders.java:81)
keycloak_1  |   at org.apache.kafka.clients.ClientUtils.createChannelBuilder(ClientUtils.java:105)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.newSender(KafkaProducer.java:511)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:454)
keycloak_1  |   ... 70 more
keycloak_1  | Caused by: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:731)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:672)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:670)
keycloak_1  |   at java.base/java.security.AccessController.doPrivileged(Native Method)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:670)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:581)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.AbstractLogin.login(AbstractLogin.java:60)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.<init>(LoginManager.java:62)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.acquireLoginManager(LoginManager.java:105)
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:170)
keycloak_1  |   ... 75 more

Also i've mentioned this topic about this error: https://github.com/ermadan/kafkalytic/issues/14 Guys from link above fixed it inside the code. Maybe this can help.

My opinion is that there is no any mentions and import of this classes: import org.apache.kafka.common.*; in code but i'm not sure.

This is 100% working config.

SnuK87 commented 1 year ago

@ArcticKeeper Could you please provide a full example including your kafka setup?

ArcticKeeper commented 1 year ago

Hello @SnuK87 you can take compose with kyecloak from the comment above, here is the Dockerfile for keycloak:

FROM quay.io/keycloak/keycloak:19.0.1 as builder

ENV KC_HEALTH_ENABLED=true
ENV KC_METRICS_ENABLED=true
ENV KC_FEATURES=token-exchange
ENV KC_DB=postgres
RUN curl -sL https://github.com/aerogear/keycloak-metrics-spi/releases/download/2.5.3/keycloak-metrics-spi-2.5.3.jar -o /opt/keycloak/providers/keycloak-metrics-spi.jar
COPY ./keycloak-kafka-1.1.4-jar-with-dependencies.jar /opt/keycloak/providers/keycloak-kafka-1.1.4-jar-with-dependencies.jar
RUN /opt/keycloak/bin/kc.sh build

FROM quay.io/keycloak/keycloak:19.0.1
COPY --from=builder /opt/keycloak/ /opt/keycloak/
USER keycloak
WORKDIR /opt/keycloak
ENTRYPOINT ["/opt/keycloak/bin/kc.sh"]

Here is the Kafka config(SCRAM_SHA_256 & 512 are enabled) and it's working without SSL:

KafkaServer {
org.apache.kafka.common.security.scram.ScramLoginModule required
username="user"
password="password";
user_mdb_admin="password";
};

Here is some code on Go, we tested and it's working perfect with our Kafka instance, both Producer and Consumer:

func GetKafkaProducerClient() sarama.SyncProducer {
    brokers := GetVariable("KAFKA_BROKER_URL", true)
    splitBrokers := strings.Split(brokers, ",")
    fmt.Printf(splitBrokers[0])
    conf := sarama.NewConfig()
    conf.Producer.RequiredAcks = sarama.WaitForAll
    conf.Producer.Return.Successes = true
    conf.Version = sarama.V0_10_0_0
    conf.ClientID = "sasl_scram_client"
    conf.Net.SASL.Enable = true
    conf.Net.SASL.Handshake = true
    conf.Net.SASL.User = GetVariable("KAFKA_USER", true)
    conf.Net.SASL.Password = GetVariable("KAFKA_PASSWORD", true)
    conf.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} }
    conf.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)

    certs := x509.NewCertPool()
    pemPath := GetVariable("KAFKA_YC_CERT_PATH", false)
    if len(pemPath) == 0 {
        pemPath = "/etc/ssl/certs/yandexcloud.crt"
    }
    pemData, err := ioutil.ReadFile(pemPath)
    if err != nil {
        fmt.Println("Couldn't load cert: ", err.Error())
        // Handle the error
    }
    certs.AppendCertsFromPEM(pemData)

    conf.Net.TLS.Enable = true
    conf.Net.TLS.Config = &tls.Config{
        InsecureSkipVerify: true,
        RootCAs:            certs,
    }

    syncProducer, err := sarama.NewSyncProducer(splitBrokers, conf)
    if err != nil {
        fmt.Println("Couldn't create producer: ", err.Error())
        os.Exit(0)
    }
    return syncProducer
}

If you can give me your email, i send you cred's for managed kafka instance and you can test it yourself. =)

SnuK87 commented 1 year ago

@ArcticKeeper Kafka is not starting with the provided config. I need a working docker-compose file for kafka (NOT keycloak)

If you have working code then feel free to translate it to java and open a pull request.

ArcticKeeper commented 1 year ago

Hello @SnuK87. I've managed to up kafka with SASL_PLAINTEXT and PLAIN not SCRAM, but the error is the same. Here is compose file:

version: '3.7'
services:
  keycloak:
    image: keykafka:1.1.4
    ports:
      - "8080:8080"
    depends_on:
      - kafka
    environment:
      KEYCLOAK_ADMIN: admin
      KEYCLOAK_ADMIN_PASSWORD: "12345"
      KEYCLOAK_LOGLEVEL: "DEBUG"
      KAFKA_TOPIC: keycloak-events
      KAFKA_CLIENT_ID: keycloak
      KAFKA_BOOTSTRAP_SERVERS: "kafka:9092"
      KAFKA_EVENTS: LOGIN_ERROR,LOGIN,REGISTER,LOGOUT
      KAFKA_SASL_MECHANISM: "PLAIN"
      KAFKA_SECURITY_PROTOCOL: "SASL_PLAINTEXT"
      KAFKA_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.plain.PlainLoginModule required username=alice password=alice-secret ;"
    command: start-dev
  zookeeper:
    image: zookeeper:3.5.6
    hostname: zookeeper
    container_name: zookeeper-sasl
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SASL_ENABLED: "false"
  kafka:
    image: wurstmeister/kafka:latest
    restart: unless-stopped
    container_name: "kafka-sasl"
    ports:
      - "9092:9092"
    expose:
      - "9093"
    depends_on:
      - zookeeper
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-sasl:2181/kafka
      KAFKA_BROKER_ID: 0
      KAFKA_ADVERTISED_HOST_NAME: kafka-sasl
      KAFKA_ADVERTISED_LISTENERS: SASL_PLAINTEXT://kafka-sasl:9093,OUTSIDE://localhost:9092
      KAFKA_LISTENERS: SASL_PLAINTEXT://0.0.0.0:9093,OUTSIDE://0.0.0.0:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: SASL_PLAINTEXT:SASL_PLAINTEXT,OUTSIDE:SASL_PLAINTEXT
      ZOOKEEPER_SASL_ENABLED: "false"
      KAFKA_INTER_BROKER_LISTENER_NAME: SASL_PLAINTEXT
      KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
      KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
      KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/kafka_server_jaas.conf"
    volumes:
      - ./kafka-config/kafka_server_jaas.conf:/etc/kafka/kafka_server_jaas.conf
      - ./kafka-config/kafka_client.conf:/etc/kafka/kafka_client.conf

Here is the config(kafka_client.conf):

security.protocol=SASL_PLAINTEXT
sasl.mechanism=PLAIN
sasl.jaas.config=org.apache.kafka.common.security.plain.PlainLoginModule required \
        username="alice" \
        password="alice-secret";

kafka_server_jaas.conf:

KafkaServer {
  org.apache.kafka.common.security.plain.PlainLoginModule required
  username="admin"
  password="admin-secret"
  user_admin="admin-secret"
  user_alice="alice-secret";
};
Client{};

Same error here cause of the deps:

keycloak_1   | Caused by: org.apache.kafka.common.KafkaException: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.plain.PlainLoginModule
keycloak_1   |  at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:184)
keycloak_1   |  at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:192)
keycloak_1   |  at org.apache.kafka.common.network.ChannelBuilders.clientChannelBuilder(ChannelBuilders.java:81)
keycloak_1   |  at org.apache.kafka.clients.ClientUtils.createChannelBuilder(ClientUtils.java:105)
keycloak_1   |  at org.apache.kafka.clients.producer.KafkaProducer.newSender(KafkaProducer.java:511)
keycloak_1   |  at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:454)
keycloak_1   |  ... 78 more
keycloak_1   | Caused by: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.plain.PlainLoginModule
keycloak_1   |  at java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:731)
keycloak_1   |  at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:672)
keycloak_1   |  at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:670)
keycloak_1   |  at java.base/java.security.AccessController.doPrivileged(Native Method)
keycloak_1   |  at java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:670)
keycloak_1   |  at java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:581)
keycloak_1   |  at org.apache.kafka.common.security.authenticator.AbstractLogin.login(AbstractLogin.java:60)
keycloak_1   |  at org.apache.kafka.common.security.authenticator.LoginManager.<init>(LoginManager.java:62)
keycloak_1   |  at org.apache.kafka.common.security.authenticator.LoginManager.acquireLoginManager(LoginManager.java:105)
keycloak_1   |  at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:170)
keycloak_1   |  ... 83 more

Kafka is out of the question, problem on plugin side.

SnuK87 commented 1 year ago

@ArcticKeeper Thanks. I'll take a look at it when I find some time.

My point is, that it's easier to debug this with a standalone kafka-client instead of using keycloak. This plugin just uses the normal kafka-client from apache. So when the standalone client is running then you can adapt the implementation in this plugin.

SnuK87 commented 1 year ago

@ArcticKeeper I was able to reproduce the error... This error only occurs when you compile the plugin with Java 11. It works fine when you compile with java 17+. The released .jar is also working fine.

I won't look into this any further because Java 11 active support will end in September and this issue seems to be related to running the kafka-client with Java 11.

Still feel free to open a pull request if you find a solution that works with both version.

ArcticKeeper commented 1 year ago

@SnuK87 Hello. Can you please provide your working configuration with keycloak 21.1.1 and 1.1.4 plugin build with 17 Java? Now i've tried keycloak 21.1.1 with your plugin 1.1.4 release and got the same error.

ERROR [io.quarkus.vertx.http.runtime.QuarkusErrorHandler] (executor-thread-4) HTTP Request to /admin/realms/master/events/config failed, error id: bdfa975f-1575-4d8f-aa71-5541f02001b9-1: java.lang.NullPointerException: Cannot invoke "java.lang.ClassLoader.getResources(String)" because "cl" is null
keycloak_1  | Caused by: org.apache.kafka.common.KafkaException: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:184)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.create(ChannelBuilders.java:192)
keycloak_1  |   at org.apache.kafka.common.network.ChannelBuilders.clientChannelBuilder(ChannelBuilders.java:81)
keycloak_1  |   at org.apache.kafka.clients.ClientUtils.createChannelBuilder(ClientUtils.java:105)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.newSender(KafkaProducer.java:511)
keycloak_1  |   at org.apache.kafka.clients.producer.KafkaProducer.<init>(KafkaProducer.java:454)
keycloak_1  |   ... 65 more
keycloak_1  | Caused by: javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.scram.ScramLoginModule
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invoke(LoginContext.java:739)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:679)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext$4.run(LoginContext.java:677)
keycloak_1  |   at java.base/java.security.AccessController.doPrivileged(AccessController.java:712)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.invokePriv(LoginContext.java:677)
keycloak_1  |   at java.base/javax.security.auth.login.LoginContext.login(LoginContext.java:587)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.AbstractLogin.login(AbstractLogin.java:60)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.<init>(LoginManager.java:62)
keycloak_1  |   at org.apache.kafka.common.security.authenticator.LoginManager.acquireLoginManager(LoginManager.java:105)
keycloak_1  |   at org.apache.kafka.common.network.SaslChannelBuilder.configure(SaslChannelBuilder.java:170)
keycloak_1  |   ... 70 more
SnuK87 commented 1 year ago

@ArcticKeeper Can you please take a look at this #30

Build the jar of the branch and try to use it in the Dockerfile that you postet before. For me this worked on Java17 and Java11.

ArcticKeeper commented 1 year ago

@SnuK87 finally it works perfectly, thanks! Also tested with SASL_SSL ad it works too. =)

kovacshuni commented 1 year ago

Hi I have a very similar problem:

javax.security.auth.login.LoginException: No LoginModule found for org.apache.kafka.common.security.plain.PlainLoginModule

I'm looking at the fix

kovacshuni commented 1 year ago

it worked for me too! Thanks @SnuK87 @ArcticKeeper

SnuK87 commented 1 year ago

Thanks for your feedback. I just prepared a new Release with the changes