search

dacort / metabase-athena-driver

An Amazon Athena driver for Metabase 0.32 and later
Apache License 2.0
225 stars 32 forks source link

0.36 support #52

Closed dacort closed 4 years ago

dacort commented 4 years ago

Add database-position key to schema for 0.36 compatibility. Resolves #50

AwolDes commented 4 years ago

Hey @dacort, we're running version 0.37.0.2 and I am still receving the error described in: https://discourse.metabase.com/t/database-sync-issue-with-new-version-0-36-0/11028/7.

There is a new column, but it is only a string (not a nested struct as described in https://github.com/dacort/metabase-athena-driver/issues/50#issuecomment-667470921)

Let me know if I can provide anymore info!

Error

[e93b342b-79f7-4164-845c-c2aa9d84ba2b] 2020-11-17T09:43:03+10:00 ERROR metabase.sync.util Error syncing Fields for Table 'Table 705 'applogs.app_events''
clojure.lang.ExceptionInfo: Output of table-metadata does not match schema: 
       {:fields #{{:database-position missing-required-key}}}  
 {:type :schema.core/error, :schema {:name (constrained Str "Non-blank string"), :schema (maybe (constrained Str "Non-blank string")), :fields #{{:base-type (pred "Valid field type"), #schema.core.OptionalKey{:k :pk?} java.lang.Boolean, :name (constrained Str "Non-blank string"), #schema.core.OptionalKey{:k :field-comment} (maybe (constrained Str "Non-blank string")), #schema.core.OptionalKey{:k :custom} {Any Any}, :database-type (maybe (constrained Str "Non-blank string")), #schema.core.OptionalKey{:k :nested-fields} #{(recursive (var metabase.sync.interface/TableMetadataField))}, #schema.core.OptionalKey{:k :special-type} (maybe (pred "Valid field type")), :database-position (constrained Int "Integer greater than or equal to zero")}}, #schema.core.OptionalKey{:k :description} (maybe (constrained Str "Non-blank string"))}, :value {:name "app_events", :schema "applogs", :fields #{{:name "user_roles", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "impersonator_org_name", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "impersonator_id", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "hour", :database-type "string", :base-type :type/Text} {:name "auth_user_id", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "total_partner_size", :database-type "integer", :base-type :type/Integer, :field-comment "from deserializer"} {:name "event_source", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "partner_ids", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "organisation_name", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "organisation_id", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "impersonated_event", :database-type "boolean", :base-type :type/Boolean, :field-comment "from deserializer"} {:name "org_size", :database-type "integer", :base-type :type/Integer, :field-comment "from deserializer"} {:name "user_id", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "impersonator_org_id", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "payload", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "event_version", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "user_employment_type", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "feature", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "event_name", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "session_span", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "month", :database-type "string", :base-type :type/Text} {:name "organisation_country", :database-type "string", :base-type :type/Text} {:name "recorded_at", :database-type "timestamp", :base-type :type/DateTime, :field-comment "from deserializer"} {:name "is_partnered_org", :database-type "boolean", :base-type :type/Boolean, :field-comment "from deserializer"} {:name "region", :database-type "string", :base-type :type/Text, :field-comment "from deserializer"} {:name "day", :database-type "string", :base-type :type/Text} {:name "year", :database-type "string", :base-type :type/Text}}}, :error {:fields #{{:database-position missing-required-key}}}}
    at metabase.sync.fetch_metadata$fn__58051$table_metadata__58056.invoke(fetch_metadata.clj:15)
    at metabase.sync.sync_metadata.fields.fetch_metadata$fn__58373$db_metadata__58378$fn__58379.invoke(fetch_metadata.clj:86)
    at metabase.sync.sync_metadata.fields.fetch_metadata$fn__58373$db_metadata__58378.invoke(fetch_metadata.clj:82)
    at metabase.sync.sync_metadata.fields$fn__58855$sync_fields_for_table_BANG___58864$fn__58867$fn__58868.invoke(fields.clj:75)
    at metabase.sync.util$do_with_error_handling.invokeStatic(util.clj:149)
    at metabase.sync.util$do_with_error_handling.invoke(util.clj:142)
    at metabase.sync.sync_metadata.fields$fn__58855$sync_fields_for_table_BANG___58864$fn__58867.invoke(fields.clj:74)
    at metabase.sync.sync_metadata.fields$fn__58855$sync_fields_for_table_BANG___58864.invoke(fields.clj:68)
    at clojure.core$partial$fn__5839.invoke(core.clj:2624)
    at clojure.core$map$fn__5866.invoke(core.clj:2753)
    at clojure.lang.LazySeq.sval(LazySeq.java:42)
    at clojure.lang.LazySeq.seq(LazySeq.java:51)
    at clojure.lang.RT.seq(RT.java:535)
    at clojure.core$seq__5402.invokeStatic(core.clj:137)
    at clojure.core$filter$fn__5893.invoke(core.clj:2809)
    at clojure.lang.LazySeq.sval(LazySeq.java:42)
    at clojure.lang.LazySeq.seq(LazySeq.java:51)
    at clojure.lang.Cons.next(Cons.java:39)
    at clojure.lang.RT.boundedLength(RT.java:1792)
    at clojure.lang.RestFn.applyTo(RestFn.java:130)
    at clojure.core$apply.invokeStatic(core.clj:667)
    at clojure.core$apply.invoke(core.clj:660)
    at metabase.sync.sync_metadata.fields$fn__58893$sync_fields_BANG___58898$fn__58899.invoke(fields.clj:88)
    at metabase.sync.sync_metadata.fields$fn__58893$sync_fields_BANG___58898.invoke(fields.clj:80)
    at metabase.sync.util$fn__44021$run_step_with_metadata__44026$fn__44030$fn__44032.invoke(util.clj:345)
    at metabase.sync.util$with_start_and_finish_logging_STAR_.invokeStatic(util.clj:102)
    at metabase.sync.util$with_start_and_finish_logging_STAR_.invoke(util.clj:96)
    at metabase.sync.util$with_start_and_finish_debug_logging.invokeStatic(util.clj:119)
    at metabase.sync.util$with_start_and_finish_debug_logging.invoke(util.clj:116)
    at metabase.sync.util$fn__44021$run_step_with_metadata__44026$fn__44030.invoke(util.clj:342)
    at metabase.sync.util$fn__44021$run_step_with_metadata__44026.invoke(util.clj:337)
    at metabase.sync.util$fn__44213$run_sync_operation__44218$fn__44219$fn__44220.invoke(util.clj:430)
    at clojure.core$mapv$fn__8445.invoke(core.clj:6912)
    at clojure.lang.PersistentVector.reduce(PersistentVector.java:343)
    at clojure.core$reduce.invokeStatic(core.clj:6827)
    at clojure.core$mapv.invokeStatic(core.clj:6903)
    at clojure.core$mapv.invoke(core.clj:6903)
    at metabase.sync.util$fn__44213$run_sync_operation__44218$fn__44219.invoke(util.clj:430)
    at metabase.sync.util$fn__44213$run_sync_operation__44218.invoke(util.clj:424)
    at metabase.sync.sync_metadata$fn__59567$sync_db_metadata_BANG___59572$fn__59573$fn__59574.invoke(sync_metadata.clj:51)
    at metabase.sync.util$do_with_error_handling.invokeStatic(util.clj:149)
    at metabase.sync.util$do_with_error_handling.invoke(util.clj:142)
    at metabase.sync.util$do_with_error_handling.invokeStatic(util.clj:145)
    at metabase.sync.util$do_with_error_handling.invoke(util.clj:142)
    at clojure.core$partial$fn__5839.invoke(core.clj:2623)
    at metabase.driver$fn__22491.invokeStatic(driver.clj:550)
    at metabase.driver$fn__22491.invoke(driver.clj:550)
    at clojure.lang.MultiFn.invoke(MultiFn.java:239)
    at metabase.sync.util$sync_in_context$fn__43933.invoke(util.clj:138)
    at metabase.sync.util$with_db_logging_disabled$fn__43930.invoke(util.clj:129)
    at metabase.sync.util$with_start_and_finish_logging_STAR_.invokeStatic(util.clj:102)
    at metabase.sync.util$with_start_and_finish_logging_STAR_.invoke(util.clj:96)
    at metabase.sync.util$with_start_and_finish_logging$fn__43919.invoke(util.clj:114)
    at metabase.sync.util$with_sync_events$fn__43914.invoke(util.clj:88)
    at metabase.sync.util$with_duplicate_ops_prevented$fn__43905.invoke(util.clj:67)
    at metabase.sync.util$do_sync_operation.invokeStatic(util.clj:164)
    at metabase.sync.util$do_sync_operation.invoke(util.clj:161)
    at metabase.sync.sync_metadata$fn__59567$sync_db_metadata_BANG___59572$fn__59573.invoke(sync_metadata.clj:50)
    at metabase.sync.sync_metadata$fn__59567$sync_db_metadata_BANG___59572.invoke(sync_metadata.clj:47)
    at metabase.api.database$fn__60309$fn__60311.invoke(database.clj:622)
    at clojure.core$binding_conveyor_fn$fn__5754.invoke(core.clj:2030)
    at clojure.lang.AFn.call(AFn.java:18)
    at java.base/java.util.concurrent.FutureTask.run(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
    at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
    at java.base/java.lang.Thread.run(Unknown Source)

Diagnostic

{
  "browser-info": {
    "language": "en-US",
    "platform": "Linux x86_64",
    "userAgent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:82.0) Gecko/20100101 Firefox/82.0",
    "vendor": ""
  },
  "system-info": {
    "file.encoding": "UTF-8",
    "java.runtime.name": "OpenJDK Runtime Environment",
    "java.runtime.version": "11.0.7+10",
    "java.vendor": "AdoptOpenJDK",
    "java.vendor.url": "https://adoptopenjdk.net/",
    "java.version": "11.0.7",
    "java.vm.name": "OpenJDK 64-Bit Server VM",
    "java.vm.version": "11.0.7+10",
    "os.name": "Linux",
    "os.version": "4.14.193-149.317.amzn2.x86_64",
    "user.language": "en",
    "user.timezone": "GMT"
  },
  "metabase-info": {
    "databases": [
      "athena",
      "postgres"
    ],
    "hosting-env": "unknown",
    "application-database": "postgres",
    "application-database-details": {
      "database": {
        "name": "PostgreSQL",
        "version": "10.13"
      },
      "jdbc-driver": {
        "name": "PostgreSQL JDBC Driver",
        "version": "42.2.8"
      }
    },
    "run-mode": "prod",
    "version": {
      "date": "2020-10-26",
      "tag": "v0.37.0.2",
      "branch": "release-x.37.x",
      "hash": "ba7be09"
    },
    "settings": {
      "report-timezone": null
    }
  }
}