Kafka TSV JSON Spec "Failed to submit supervisor"

[MrMoronIV]

Affected Version

Druid 0.16.0

Description

Nano-quickstart: when trying to submit a JSON spec it gives me:

Failed to submit supervisor: Class org.apache.druid.segment.indexing.RealtimeTuningConfig is not assignable to org.apache.druid.indexing.kafka.supervisor.KafkaSupervisorTuningConfig (through reference chain: org.apache.druid.indexing.kafka.supervisor.KafkaSupervisorSpec["tuningConfig"])

My extensions:

druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches","druid-avro-extensions","druid-kafka-extraction-namespace"]

The spec:

{
  "type": "kafka",
  "dataSchema": {
    "metricsSpec": [
      {
        "name": "count",
        "type": "count"
      },
      {
        "name": "sum_br_colordepth",
        "type": "longSum",
        "fieldName": "br_colordepth"
      },
      {
        "name": "sum_br_cookies",
        "type": "longSum",
        "fieldName": "br_cookies"
      },
      {
        "name": "sum_br_features_director",
        "type": "longSum",
        "fieldName": "br_features_director"
      },
      {
        "name": "sum_br_features_flash",
        "type": "longSum",
        "fieldName": "br_features_flash"
      },
      {
        "name": "sum_br_features_gears",
        "type": "longSum",
        "fieldName": "br_features_gears"
      },
      {
        "name": "sum_br_features_java",
        "type": "longSum",
        "fieldName": "br_features_java"
      },
      {
        "name": "sum_br_features_pdf",
        "type": "longSum",
        "fieldName": "br_features_pdf"
      },
      {
        "name": "sum_br_features_quicktime",
        "type": "longSum",
        "fieldName": "br_features_quicktime"
      },
      {
        "name": "sum_br_features_realplayer",
        "type": "longSum",
        "fieldName": "br_features_realplayer"
      },
      {
        "name": "sum_br_features_silverlight",
        "type": "longSum",
        "fieldName": "br_features_silverlight"
      },
      {
        "name": "sum_br_features_windowsmedia",
        "type": "longSum",
        "fieldName": "br_features_windowsmedia"
      },
      {
        "name": "sum_br_viewheight",
        "type": "longSum",
        "fieldName": "br_viewheight"
      },
      {
        "name": "sum_br_viewwidth",
        "type": "longSum",
        "fieldName": "br_viewwidth"
      },
      {
        "name": "sum_doc_height",
        "type": "longSum",
        "fieldName": "doc_height"
      },
      {
        "name": "sum_doc_width",
        "type": "longSum",
        "fieldName": "doc_width"
      },
      {
        "name": "sum_domain_sessionidx",
        "type": "longSum",
        "fieldName": "domain_sessionidx"
      },
      {
        "name": "sum_dvce_screenheight",
        "type": "longSum",
        "fieldName": "dvce_screenheight"
      },
      {
        "name": "sum_dvce_screenwidth",
        "type": "longSum",
        "fieldName": "dvce_screenwidth"
      },
      {
        "name": "sum_page_urlport",
        "type": "longSum",
        "fieldName": "page_urlport"
      },
      {
        "name": "sum_user_fingerprint",
        "type": "longSum",
        "fieldName": "user_fingerprint"
      }
    ],
    "granularitySpec": {
      "segmentGranularity": "MINUTE",
      "queryGranularity": "MINUTE",
      "rollup": true
    },
    "parser": {
      "type": "string",
      "parseSpec": {
        "format": "tsv",
        "timestampSpec": {
          "column": "derived_tstamp",
          "format": "auto"
        },
        "hasHeaderRow": false,
        "columns": [
          "appId",
          "platform",
          "etl_tstamp",
          "collector_tstamp",
          "dvce_created_tstamp",
          "event",
          "event_id",
          "txn_id",
          "name_tracker",
          "v_tracker",
          "v_collector",
          "v_etl",
          "user_id",
          "user_ipaddress",
          "user_fingerprint",
          "domain_userid",
          "domain_sessionidx",
          "network_userid",
          "geo_country",
          "geo_region",
          "geo_city",
          "geo_zipcode",
          "geo_latitude",
          "geo_longitude",
          "geo_region_name",
          "ip_isp",
          "ip_organization",
          "ip_domain",
          "ip_netspeed",
          "page_url",
          "page_title",
          "page_referrer",
          "page_urlscheme",
          "page_urlhost",
          "page_urlport",
          "page_urlpath",
          "page_urlquery",
          "page_urlfragment",
          "refr_urlscheme",
          "refr_urlhost",
          "refr_urlport",
          "refr_urlpath",
          "refr_urlquery",
          "refr_urlfragment",
          "refr_medium",
          "refr_source",
          "refr_term",
          "mkt_medium",
          "mkt_source",
          "mkt_term",
          "mkt_content",
          "mkt_campaign",
          "contexts",
          "se_category",
          "se_action",
          "se_label",
          "se_property",
          "se_value",
          "unstruct_event",
          "tr_orderid",
          "tr_affiliation",
          "tr_total",
          "tr_tax",
          "tr_shipping",
          "tr_city",
          "tr_state",
          "tr_country",
          "ti_orderid",
          "ti_sku",
          "ti_name",
          "ti_category",
          "ti_price",
          "ti_quantity",
          "pp_xoffset_min",
          "pp_xoffset_max",
          "pp_yoffset_min",
          "pp_yoffset_max",
          "useragent",
          "br_name",
          "br_family",
          "br_version",
          "br_type",
          "br_renderengine",
          "br_lang",
          "br_features_pdf",
          "br_features_flash",
          "br_features_java",
          "br_features_director",
          "br_features_quicktime",
          "br_features_realplayer",
          "br_features_windowsmedia",
          "br_features_gears",
          "br_features_silverlight",
          "br_cookies",
          "br_colordepth",
          "br_viewwidth",
          "br_viewheight",
          "os_name",
          "os_family",
          "os_manufacturer",
          "os_timezone",
          "dvce_type",
          "dvce_ismobile",
          "dvce_screenwidth",
          "dvce_screenheight",
          "doc_charset",
          "doc_width",
          "doc_height",
          "tr_currency",
          "tr_total_base",
          "tr_tax_base",
          "tr_shipping_base",
          "ti_currency",
          "ti_price_base",
          "base_currency",
          "geo_timezone",
          "mkt_clickid",
          "mkt_network",
          "etl_tags",
          "dvce_sent_tstamp",
          "refr_domain_userid",
          "refr_dvce_tstamp",
          "derived_contexts",
          "domain_sessionid",
          "derived_tstamp",
          "event_vendor",
          "event_name",
          "event_format",
          "event_version",
          "event_fingerprint",
          "true_tstamp",
          "pii"
        ],
        "dimensionsSpec": {
          "dimensions": []
        }
      }
    },
    "dataSource": "data_stream"
  },
  "tuningConfig": {
    "type": "realtime",
    "windowPeriod": "PT10M",
    "intermediatePersistPeriod": "PT10M",
    "maxRowsInMemory": "100000",
    "logParseExceptions": true
  },
  "ioConfig": {
    "topic": "data_stream",
    "consumerProperties": {
     // this works
    },
    "taskCount": 1,
    "replicas": 1,
    "taskDuration": "PT5M",
    "type": "kafka"
  }
}

[v-vishwa]

@MrMoronIV In you ingestion spec, I see you have

"tuningConfig": {
    "type": "realtime",

I think, Type should be Kafka, please see the doc below: https://druid.apache.org/docs/latest/development/extensions-core/kafka-ingestion

[MrMoronIV]

I replaced the whole tuningConfig block with:

"tuningConfig": {
    "type": "kafka",
    "maxRowsPerSegment": 5000000
  },

and it now works, sweet!