search

lior-k / fast-elasticsearch-vector-scoring

Score documents using embedding-vectors dot-product or cosine-similarity with ES Lucene engine
Apache License 2.0
395 stars 112 forks source link

Unable to query in Elastic 6.8.1 #33

Closed jwgwalton closed 4 years ago

jwgwalton commented 4 years ago

Hi @lior-k

i have set up an elastic 6.8.1 docker image and installed the es-6.8.1 branch.

docker run -p 9200:9200 -p 9300:9300 -v ${elastic_plugin_folder}/target/releases:/tmp/cosine_sim -e discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.8.1

and installed it 

docker exec -it trusting_banach /bin/bash

elasticsearch-plugin install file:///tmp/cosine_sim/elasticsearch-binary-vector-scoring-6.8.1.zip

i can see that it is installed

[root@8a2db7afc779 elasticsearch]# bin/elasticsearch-plugin list
elasticsearch-binary-vector-scoring

When i try to query it i'm getting the following error

query = {
  "query": {
    "function_score": {
      "boost_mode": "replace",
      "script_score": {
        "script": {
        "source": "binary_vector_score",
          "lang": "knn",
          "params": {
            "cosine": False,
            "field": "title_vec_b64",
            "vector": [-0.11091560125350952,
 -0.012368256226181984,
 -0.05769440531730652,
 0.10475035011768341,
 0.08878926932811737,
 -0.07440587133169174,
 0.08918392658233643,
 -0.016563668847084045,
 0.05303652584552765,
 0.010436886921525002,
 -0.06147288531064987,
 0.07994012534618378,
 -0.09865640848875046,
 0.01937752775847912,
 -0.004414134658873081,
 0.04878390580415726,
 0.1500605344772339,
 -0.049730271100997925,
 0.10887563228607178,
 0.07925839722156525,
 -0.03000705875456333,
 -0.026256615296006203,
 0.01621185801923275,
 0.039685823023319244,
 -0.001498837023973465,
 -0.0267797764390707,
 0.030115759000182152,
 0.005555476061999798,
 0.046774279326200485,
 0.19266989827156067,
 -0.02019449509680271,
 0.006202386692166328,
 0.015504976734519005,
 0.06700988858938217,
 -0.019451739266514778,
 0.018186839297413826,
 0.014493770897388458,
 -0.06764443963766098,
 0.09920039027929306,
 0.08250521868467331,
 0.03336912393569946,
 0.11773338913917542,
 -0.03351331502199173,
 0.07179781794548035,
 -0.18448804318904877,
 -0.0197548046708107,
 -0.017714664340019226,
 0.12445169687271118,
 0.02748987078666687,
 0.01750885136425495,
 -0.059615299105644226,
 0.0033058077096939087,
 0.08752410113811493,
 0.0942256897687912,
 -0.1654362976551056,
 -0.06923578679561615,
 0.04007047042250633,
 0.026420842856168747,
 -0.04796523600816727,
 0.13311980664730072,
 0.018357206135988235,
 -0.01189229916781187,
 0.06303215026855469,
 0.034484561532735825,
 0.0814635381102562,
 0.08240565657615662,
 -0.003505193628370762,
 0.04957431182265282,
 -0.11623919755220413,
 0.028763558715581894,
 -0.05360335111618042,
 0.01727301813662052,
 -0.010051139630377293,
 0.08175407350063324,
 -0.02495856210589409,
 0.04708561301231384,
 0.12423346191644669,
 -0.019826268777251244,
 -0.01785324141383171,
 0.033566221594810486,
 0.03995600715279579,
 0.16404400765895844,
 -0.05547409504652023,
 -0.0010914923623204231,
 0.004697272554039955,
 0.07940614223480225,
 -0.03247788920998573,
 0.01867639645934105,
 0.06854085624217987,
 -0.06572879105806351,
 -0.024124089628458023,
 0.16388298571109772,
 -0.024828284978866577,
 0.0249160323292017,
 -0.013938387855887413,
 0.020326070487499237,
 0.006996334530413151,
 0.023765256628394127,
 -0.005187366157770157,
 0.09260062873363495,
 0.005694741848856211,
 -0.11202815920114517,
 0.023884126916527748,
 -0.051436565816402435,
 -0.04748039320111275,
 -0.019691068679094315,
 -0.11744862794876099,
 -0.056783631443977356,
 0.0620306134223938,
 -0.0157561544328928,
 -0.03777662664651871,
 0.09631360322237015,
 -0.1594366431236267,
 -0.02411549910902977,
 0.018708499148488045,
 -0.011242246255278587,
 -0.05065244063735008,
 0.0920867845416069,
 -0.045188501477241516,
 0.02484002336859703,
 -0.061890728771686554,
 0.04936341196298599,
 -0.07623309642076492,
 0.06964029371738434,
 -0.09506729245185852,
 0.1260889172554016,
 -0.05375320091843605,
 -0.028824951499700546,
 0.07088449597358704,
 -0.0563826858997345,
 0.06858667731285095,
 -0.09268361330032349,
 -0.07293982058763504,
 -0.021877095103263855,
 0.04640849679708481,
 -0.08267684280872345,
 0.1712718904018402,
 0.08969420939683914,
 0.12968459725379944,
 0.06828564405441284,
 -0.07446791976690292,
 0.04404822364449501,
 0.09619954228401184,
 0.014123346656560898,
 -0.05217517167329788,
 -0.033148810267448425,
 -0.1279505342245102,
 0.17056585848331451,
 0.07830381393432617,
 -0.02065315656363964,
 0.018212372437119484,
 -0.05478692054748535,
 0.0008633993566036224,
 -0.11331601440906525,
 -0.0970207080245018,
 -0.019216883927583694,
 0.02283022552728653,
 -0.09400290250778198,
 0.019097940996289253,
 0.09848718345165253,
 0.01893545687198639,
 0.012185772880911827,
 0.10844842344522476,
 -0.12887753546237946,
 0.06064837425947189,
 -0.06580662727355957,
 -0.007481692358851433,
 -0.03142566978931427,
 -0.06736018508672714,
 -0.052196428179740906,
 0.019975129514932632,
 0.04874351993203163,
 -0.2319813221693039,
 0.16128496825695038,
 0.07921537011861801,
 0.02356587164103985,
 0.21304066479206085,
 0.04416591674089432,
 -0.06543312966823578,
 0.10867374390363693,
 -0.059921473264694214,
 0.004250713624060154,
 -0.04853147640824318,
 -0.08667157590389252,
 0.08203355222940445,
 0.02892845682799816,
 0.11836115270853043,
 0.002109323628246784,
 0.04675440117716789,
 0.08557461202144623,
 0.03520556166768074,
 0.005658429116010666,
 0.07631449401378632,
 -0.11538150906562805,
 0.08367155492305756,
 -0.04233440011739731,
 0.026365965604782104,
 0.11810635775327682,
 -0.008726771920919418,
 -0.006178293377161026,
 -0.032966498285532,
 0.009704645723104477,
 0.10991569608449936,
 0.07687313854694366,
 0.045167192816734314,
 -0.10288700461387634,
 0.08407768607139587,
 -0.0060685910284519196,
 -0.004732324741780758,
 0.13219217956066132,
 -0.01730496622622013,
 -0.05590490251779556,
 -0.022574706003069878,
 -0.06257370859384537,
 0.025937484577298164,
 0.18327824771404266,
 0.062333844602108,
 -0.12896911799907684,
 0.016680652275681496,
 0.03501032292842865,
 -0.039892345666885376,
 -0.0010370910167694092,
 -0.059304557740688324,
 -0.007233946584165096,
 -0.008397813886404037,
 0.06103808432817459,
 -0.10880440473556519,
 -0.1236773282289505,
 -0.08680218458175659,
 0.03410463407635689,
 -0.0372888557612896,
 0.04124067723751068,
 0.053624026477336884,
 0.020399751141667366,
 -0.09959948062896729,
 0.03808313235640526,
 0.004636439960449934,
 0.033109381794929504,
 -0.009622188284993172,
 -0.12189603596925735,
 0.08236785978078842,
 -0.05138041079044342,
 -0.06599799543619156,
 -0.059065062552690506,
 0.06687866151332855,
 -0.012619949877262115,
 -0.07758430391550064,
 -0.04862333834171295,
 -0.12815603613853455,
 0.04510051757097244,
 0.06221261993050575,
 -0.01896725967526436,
 -0.11125870048999786,
 0.07486312836408615,
 0.07476145774126053,
 0.07034306228160858,
 -0.21859833598136902,
 -0.024630384519696236,
 0.02307208999991417,
 -0.060816820710897446,
 -0.0612434521317482,
 -0.037258923053741455,
 0.027293622493743896,
 -0.01798875629901886,
 -0.10089060664176941,
 -0.030837880447506905,
 -0.054076097905635834,
 0.09301923215389252,
 -0.03546946868300438,
 -0.01776343211531639,
 -0.02929227240383625,
 -0.00369332917034626,
 -0.08080127090215683,
 0.005425403825938702,
 0.006371726747602224,
 0.07843196392059326,
 -0.06922762095928192,
 0.09849295020103455,
 -0.05541716888546944,
 -0.02776404842734337,
 0.04205353558063507,
 -0.01546469610184431,
 -0.0618584007024765,
 -0.13895854353904724,
 -0.036999598145484924,
 -0.012432890012860298,
 -0.01635652780532837,
 -0.06800690293312073,
 0.07489623129367828,
 0.008075803518295288,
 -0.06403892487287521,
 0.06564713269472122,
 -0.09574733674526215,
 0.06792283803224564,
 -0.09947624057531357,
 0.056209221482276917,
 0.09886199235916138,
 0.10442547500133514,
 -0.019203905016183853,
 -0.22731733322143555]
          }
        }
      }
    }
  },
  "size": 100
}

response = requests.post(f"http://localhost:9200/infringement/_search", json=query)
response.json()

{'error': {'root_cause': [{'type': 'query_shard_exception',
    'reason': 'script_score: the script could not be loaded',
    'index_uuid': 'Y88sphZVRteeoVEj1xxvMA',
    'index': 'infringement'}],
  'type': 'search_phase_execution_exception',
  'reason': 'all shards failed',
  'phase': 'query',
  'grouped': True,
  'failed_shards': [{'shard': 0,
    'index': 'infringement',
    'node': 'FT2lVCDISOSXfUfAfxqyhw',
    'reason': {'type': 'query_shard_exception',
     'reason': 'script_score: the script could not be loaded',
     'index_uuid': 'Y88sphZVRteeoVEj1xxvMA',
     'index': 'infringement',
     'caused_by': {'type': 'illegal_argument_exception',
      'reason': 'script_lang not supported [knn]'}}}]},
 'status': 400}

The documentation has the query body for ES 5.X and ES 7.1:. Is there anything different for 6.8.1?

jwgwalton commented 4 years ago

There is data in elastic which has the vector encoded in base 64 with the code provided.

# Convert vectors to base64 strings
import base64
import numpy as np

dfloat32 = np.dtype('>f4')

def encode_array(arr):
    base64_str = base64.b64encode(np.array(arr).astype(dfloat32)).decode("utf-8")
    return base64_str
jwgwalton commented 4 years ago

The logs for the docker container do not show anything either.

docker run -p 9200:9200 -p 9300:9300 -v /home/jwgwalton/dev/test-elastic-search-plugin:/tmp/cosine_sim -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.8.1
OpenJDK 64-Bit Server VM warning: Option UseConcMarkSweepGC was deprecated in version 9.0 and will likely be removed in a future release.
[2019-11-28T13:02:05,457][INFO ][o.e.e.NodeEnvironment    ] [FT2lVCD] using [1] data paths, mounts [[/ (overlay)]], net usable_space [108.5gb], net total_space [224.8gb], types [overlay]
[2019-11-28T13:02:05,459][INFO ][o.e.e.NodeEnvironment    ] [FT2lVCD] heap size [989.8mb], compressed ordinary object pointers [true]
[2019-11-28T13:02:05,460][INFO ][o.e.n.Node               ] [FT2lVCD] node name derived from node ID [FT2lVCDISOSXfUfAfxqyhw]; set [node.name] to override
[2019-11-28T13:02:05,461][INFO ][o.e.n.Node               ] [FT2lVCD] version[6.8.1], pid[1], build[default/docker/1fad4e1/2019-06-18T13:16:52.517138Z], OS[Linux/4.15.0-20-generic/amd64], JVM[Oracle Corporation/OpenJDK 64-Bit Server VM/12.0.1/12.0.1+12]
[2019-11-28T13:02:05,461][INFO ][o.e.n.Node               ] [FT2lVCD] JVM arguments [-Xms1g, -Xmx1g, -XX:+UseConcMarkSweepGC, -XX:CMSInitiatingOccupancyFraction=75, -XX:+UseCMSInitiatingOccupancyOnly, -Des.networkaddress.cache.ttl=60, -Des.networkaddress.cache.negative.ttl=10, -XX:+AlwaysPreTouch, -Xss1m, -Djava.awt.headless=true, -Dfile.encoding=UTF-8, -Djna.nosys=true, -XX:-OmitStackTraceInFastThrow, -Dio.netty.noUnsafe=true, -Dio.netty.noKeySetOptimization=true, -Dio.netty.recycler.maxCapacityPerThread=0, -Dlog4j.shutdownHookEnabled=false, -Dlog4j2.disable.jmx=true, -Djava.io.tmpdir=/tmp/elasticsearch-6847990490396971235, -XX:+HeapDumpOnOutOfMemoryError, -XX:HeapDumpPath=data, -XX:ErrorFile=logs/hs_err_pid%p.log, -Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m, -Djava.locale.providers=COMPAT, -XX:UseAVX=2, -Des.cgroups.hierarchy.override=/, -Des.path.home=/usr/share/elasticsearch, -Des.path.conf=/usr/share/elasticsearch/config, -Des.distribution.flavor=default, -Des.distribution.type=docker]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [aggs-matrix-stats]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [analysis-common]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [ingest-common]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [ingest-geoip]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [ingest-user-agent]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [lang-expression]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [lang-mustache]
[2019-11-28T13:02:06,912][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [lang-painless]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [mapper-extras]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [parent-join]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [percolator]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [rank-eval]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [reindex]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [repository-url]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [transport-netty4]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [tribe]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-ccr]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-core]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-deprecation]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-graph]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-ilm]
[2019-11-28T13:02:06,913][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-logstash]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-ml]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-monitoring]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-rollup]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-security]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-sql]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-upgrade]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] loaded module [x-pack-watcher]
[2019-11-28T13:02:06,914][INFO ][o.e.p.PluginsService     ] [FT2lVCD] no plugins loaded
[2019-11-28T13:02:06,999][WARN ][o.e.d.c.j.Joda           ] [FT2lVCD] 'y' year should be replaced with 'u'. Use 'y' for year-of-era. Prefix your date format with '8' to use the new specifier.
[2019-11-28T13:02:09,627][INFO ][o.e.x.s.a.s.FileRolesStore] [FT2lVCD] parsed [0] roles from file [/usr/share/elasticsearch/config/roles.yml]
[2019-11-28T13:02:10,070][INFO ][o.e.x.m.p.l.CppLogMessageHandler] [FT2lVCD] [controller/87] [Main.cc@109] controller (64 bit): Version 6.8.1 (Build 6e3432237cefa4) Copyright (c) 2019 Elasticsearch BV
[2019-11-28T13:02:10,576][INFO ][o.e.d.DiscoveryModule    ] [FT2lVCD] using discovery type [single-node] and host providers [settings]
[2019-11-28T13:02:11,213][INFO ][o.e.n.Node               ] [FT2lVCD] initialized
[2019-11-28T13:02:11,213][INFO ][o.e.n.Node               ] [FT2lVCD] starting ...
[2019-11-28T13:02:11,391][INFO ][o.e.t.TransportService   ] [FT2lVCD] publish_address {172.17.0.2:9300}, bound_addresses {0.0.0.0:9300}
[2019-11-28T13:02:11,404][WARN ][o.e.b.BootstrapChecks    ] [FT2lVCD] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
[2019-11-28T13:02:11,485][INFO ][o.e.h.n.Netty4HttpServerTransport] [FT2lVCD] publish_address {172.17.0.2:9200}, bound_addresses {0.0.0.0:9200}
[2019-11-28T13:02:11,485][INFO ][o.e.n.Node               ] [FT2lVCD] started
[2019-11-28T13:02:11,609][WARN ][o.e.x.s.a.s.m.NativeRoleMappingStore] [FT2lVCD] Failed to clear cache for realms [[]]
[2019-11-28T13:02:11,650][INFO ][o.e.g.GatewayService     ] [FT2lVCD] recovered [0] indices into cluster_state
[2019-11-28T13:02:11,770][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.watches] for index patterns [.watches*]
[2019-11-28T13:02:11,808][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.watch-history-9] for index patterns [.watcher-history-9*]
[2019-11-28T13:02:11,827][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.triggered_watches] for index patterns [.triggered_watches*]
[2019-11-28T13:02:11,859][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.monitoring-logstash] for index patterns [.monitoring-logstash-6-*]
[2019-11-28T13:02:11,895][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.monitoring-es] for index patterns [.monitoring-es-6-*]
[2019-11-28T13:02:11,914][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.monitoring-alerts] for index patterns [.monitoring-alerts-6]
[2019-11-28T13:02:11,945][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.monitoring-beats] for index patterns [.monitoring-beats-6-*]
[2019-11-28T13:02:11,968][INFO ][o.e.c.m.MetaDataIndexTemplateService] [FT2lVCD] adding template [.monitoring-kibana] for index patterns [.monitoring-kibana-6-*]
[2019-11-28T13:02:12,094][INFO ][o.e.l.LicenseService     ] [FT2lVCD] license [80c414f1-8ffb-4606-b107-64e47341398f] mode [basic] - valid
[2019-11-28T13:07:10,478][WARN ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][young][270][6] duration [4.4s], collections [1]/[7.3s], total [4.4s]/[4.5s], memory [356.4mb]->[82.3mb]/[989.8mb], all_pools {[young] [273mb]->[8.3mb]/[273mb]}{[survivor] [29.1mb]->[19.7mb]/[34.1mb]}{[old] [54.2mb]->[54.2mb]/[682.6mb]}
[2019-11-28T13:07:10,520][WARN ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][270] overhead, spent [4.4s] collecting in the last [7.3s]
[2019-11-28T13:07:31,857][WARN ][o.e.d.r.a.a.i.RestPutMappingAction] [FT2lVCD] [types removal] The parameter include_type_name should be explicitly specified in create index requests to prepare for 7.0. In 7.0 include_type_name will default to 'false', and requests are expected to omit the type name in mapping definitions.
[2019-11-28T13:07:32,076][WARN ][o.e.d.c.m.MetaDataCreateIndexService] [FT2lVCD] the default number of shards will change from [5] to [1] in 7.0.0; if you wish to continue using the default of [5] shards, you must manage this on the create index request or with an index template
[2019-11-28T13:07:32,368][INFO ][o.e.c.m.MetaDataCreateIndexService] [FT2lVCD] [infringement] creating index, cause [api], templates [], shards [5]/[1], mappings [_doc]
[2019-11-28T13:07:36,100][INFO ][o.e.c.m.MetaDataMappingService] [FT2lVCD] [infringement/Y88sphZVRteeoVEj1xxvMA] update_mapping [_doc]
[2019-11-28T13:07:45,094][WARN ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][young][301][7] duration [1.3s], collections [1]/[2.2s], total [1.3s]/[5.9s], memory [320.7mb]->[228.1mb]/[989.8mb], all_pools {[young] [246.7mb]->[18.3mb]/[273mb]}{[survivor] [19.7mb]->[29.3mb]/[34.1mb]}{[old] [54.2mb]->[182.2mb]/[682.6mb]}
[2019-11-28T13:07:45,095][WARN ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][301] overhead, spent [1.3s] collecting in the last [2.2s]
[2019-11-28T13:13:24,062][INFO ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][young][637][8] duration [849ms], collections [1]/[1.9s], total [849ms]/[6.7s], memory [483.1mb]->[244.3mb]/[989.8mb], all_pools {[young] [271.6mb]->[2.2mb]/[273mb]}{[survivor] [29.3mb]->[34.1mb]/[34.1mb]}{[old] [182.2mb]->[208.3mb]/[682.6mb]}
[2019-11-28T13:13:24,063][INFO ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][637] overhead, spent [849ms] collecting in the last [1.9s]
[2019-11-28T13:23:18,575][WARN ][o.e.m.j.JvmGcMonitorService] [FT2lVCD] [gc][1229] overhead, spent [531ms] collecting in the last [1s]
jwgwalton commented 4 years ago

I trimmed out some of the entries but just to show there is data with encoded vectors in elastic.

{'took': 4,
 'timed_out': False,
 '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0},
 'hits': {'total': 20,
  'max_score': 1.0,
  'hits': [  {'_index': 'infringement',
    '_type': '_doc',
    '_id': '111',
    '_score': 1.0,
    '_source': {'client_id': 'string',
     'url': 'url',
     'title': 'title',
     'product_description': 'product_description',
     'site_group': 'site_group',
     'region': 'region',
     'keywords': "['keywords']",
     'language': 'en',
     'enforcement_probability': 0.8012800716887034,
     'title_vec_b64': 'uoyAUDrCou08RQj+PRqb4Ttug6q9UcEDvJ1p8DxqcFO6PCUivIYyljxF/0G8rD32vDAiubyI77a8QTwuvKw0BDxD17Y8kiTnO+fXvjylbBA6T+/iO9E/9LufuLa8f+VkPSC9Xj1DbGG8DvHWPNc6gzwP1fI83eE2u2bRnrwyfck7qFzEPG2uMLtwr5K8ZAHTvBtn1rwL7rM5iG0ePQcG9LudMOi7k4LfOw1GVrwdwQM8JfnYPT3RM7uoW808qt6UPCordjumAm48dgXzvPknkDwZcYa8ZmdNvIiuNLsOah47mfz+vBEcG7uKLOo6nC0uvDMFTTtw2YA8g+XWPIF2vjrMtVO7iyjTPPc0BzyJPk28eGDou+J33jzDdAY78o8NPGWh9jtTSbe8HnC8PCmoDjr+tHo8spH5vJzeCDrVkNC7v4lWvDenPLuxKmI5/AAGu7xOerqeczu9Az4HPHctQzwkR6s8e2wkPIUoDbybEnu8nnNlu/G3ITyb1f48tK7qPIJ06zk+FrM7mxmePBuwl7yPIje8YzkHvJGLazwFxFi72vJ9OvJGrTwlQ3g8PvcoOlbFOTy3Q2q8jmkIOxLrAbwOyzq8ryu2vJonLDuUF6O7usEyvCPKnDvbXAo8f1tfvJ2V/zwW0oa8DRxUOx8/EDuO8S48sX8EPL7FqrxA8oc8utcVvMNeyTx8hKs7zJKWPJo82LzOrTA8lB2aO6kdSD0fz0q9A6KOPIKjwTrDyjs7+hF6uyaHCjumR807rVq0vA1en7xPcLO84P8BPFQgAzumDpY6BpF5vCo0frqe8q47UuInu3bdejqjJ9e6/XhBO4VXTLsyYJi8w+VPOq4DZLuJZPK8F2NBvQ4Cejyy9dM7YdDuO2NXcLy0rAO8eOUGusxOTzxaVoE8eLEVPAL/u7wx2ne4xkwYOgmYYDr8b9w8jUVhvFTpKrwNqGE7Qz0WO8WjUjzDb0M7qG1tvQiZ/LyREe69G4X7vNlRhDtBSR88IiXxOyW9drv+X9C8zI/eO6nxBzw0A9A70iQwPGO88zu4xXo61TZfO66gxz0MBFK8CmMBO1bRyjtp24O8j9ANu+ot8Lt3VBg8GlPfPFR2RzrUqMQ76S1ZvGtZHLskjtY8tCMOu/qZxDukS0e7kdW3vGL4ZTtZ35i7g7tEu9E/HLtyPI08RtaTOx11cjwyp4C7gUbSOyz8ZbsNg348DjLPO5DR1TyH5CQ8bmUoO7qgiDupB1a8kSnNPCYTkrzMoe48E+mkvBnqWbuSC4G7kwn9OzNAXryd6kq7jYX/vIcPejzIKTQ7y/IrOwRukDvihjs8KNyru/WiBjziXg87ggvDO+caCTy67HS77vI+vDhMerwj39A86MqVuwSMHjulEra7sp9WuugTir09qnu8Wnj7vHxX1jt+BIG80bP2u8KutDxCn089PQVhvIbibTyny4u8x1S+PIJxUrpJXGK8R/7nPDGFVr1XHk+9AwX/u4AAXrxHLwe73ecPvAEYzzsaO407RF/6vQhSZ7wbHa28m75xur1qdzv+3nE7EYUZvFMqtrwGHsQ4o65GPFzbGTxG/De8fYtQPDIFDjwhtf28smhG',
     'description_vec_64': 'uoyAUDrCou08RQj+PRqb4Ttug6q9UcEDvJ1p8DxqcFO6PCUivIYyljxF/0G8rD32vDAiubyI77a8QTwuvKw0BDxD17Y8kiTnO+fXvjylbBA6T+/iO9E/9LufuLa8f+VkPSC9Xj1DbGG8DvHWPNc6gzwP1fI83eE2u2bRnrwyfck7qFzEPG2uMLtwr5K8ZAHTvBtn1rwL7rM5iG0ePQcG9LudMOi7k4LfOw1GVrwdwQM8JfnYPT3RM7uoW808qt6UPCordjumAm48dgXzvPknkDwZcYa8ZmdNvIiuNLsOah47mfz+vBEcG7uKLOo6nC0uvDMFTTtw2YA8g+XWPIF2vjrMtVO7iyjTPPc0BzyJPk28eGDou+J33jzDdAY78o8NPGWh9jtTSbe8HnC8PCmoDjr+tHo8spH5vJzeCDrVkNC7v4lWvDenPLuxKmI5/AAGu7xOerqeczu9Az4HPHctQzwkR6s8e2wkPIUoDbybEnu8nnNlu/G3ITyb1f48tK7qPIJ06zk+FrM7mxmePBuwl7yPIje8YzkHvJGLazwFxFi72vJ9OvJGrTwlQ3g8PvcoOlbFOTy3Q2q8jmkIOxLrAbwOyzq8ryu2vJonLDuUF6O7usEyvCPKnDvbXAo8f1tfvJ2V/zwW0oa8DRxUOx8/EDuO8S48sX8EPL7FqrxA8oc8utcVvMNeyTx8hKs7zJKWPJo82LzOrTA8lB2aO6kdSD0fz0q9A6KOPIKjwTrDyjs7+hF6uyaHCjumR807rVq0vA1en7xPcLO84P8BPFQgAzumDpY6BpF5vCo0frqe8q47UuInu3bdejqjJ9e6/XhBO4VXTLsyYJi8w+VPOq4DZLuJZPK8F2NBvQ4Cejyy9dM7YdDuO2NXcLy0rAO8eOUGusxOTzxaVoE8eLEVPAL/u7wx2ne4xkwYOgmYYDr8b9w8jUVhvFTpKrwNqGE7Qz0WO8WjUjzDb0M7qG1tvQiZ/LyREe69G4X7vNlRhDtBSR88IiXxOyW9drv+X9C8zI/eO6nxBzw0A9A70iQwPGO88zu4xXo61TZfO66gxz0MBFK8CmMBO1bRyjtp24O8j9ANu+ot8Lt3VBg8GlPfPFR2RzrUqMQ76S1ZvGtZHLskjtY8tCMOu/qZxDukS0e7kdW3vGL4ZTtZ35i7g7tEu9E/HLtyPI08RtaTOx11cjwyp4C7gUbSOyz8ZbsNg348DjLPO5DR1TyH5CQ8bmUoO7qgiDupB1a8kSnNPCYTkrzMoe48E+mkvBnqWbuSC4G7kwn9OzNAXryd6kq7jYX/vIcPejzIKTQ7y/IrOwRukDvihjs8KNyru/WiBjziXg87ggvDO+caCTy67HS77vI+vDhMerwj39A86MqVuwSMHjulEra7sp9WuugTir09qnu8Wnj7vHxX1jt+BIG80bP2u8KutDxCn089PQVhvIbibTyny4u8x1S+PIJxUrpJXGK8R/7nPDGFVr1XHk+9AwX/u4AAXrxHLwe73ecPvAEYzzsaO407RF/6vQhSZ7wbHa28m75xur1qdzv+3nE7EYUZvFMqtrwGHsQ4o65GPFzbGTxG/De8fYtQPDIFDjwhtf28smhG'}}]}}
lior-k commented 4 years ago

Interesting. I don't see the plug-in in the ES initialization log (that you attached). Did you restart ES after installing the plug-in? If not that explain it. It must be restarted after the plug-in installation. (Note that it must be done from the bash inside your docker container)

Let me know if it helps

On Thu, Nov 28, 2019, 4:14 PM jwgwalton notifications@github.com wrote:

I trimmed out some of the entries but just to show there is data with encoded vectors in elastic.

{'took': 4, 'timed_out': False, '_shards': {'total': 5, 'successful': 5, 'skipped': 0, 'failed': 0}, 'hits': {'total': 20, 'max_score': 1.0, 'hits': [ {'_index': 'infringement', '_type': '_doc', '_id': '111', '_score': 1.0, '_source': {'client_id': 'string', 'url': 'url', 'title': 'title', 'product_description': 'product_description', 'site_group': 'site_group', 'region': 'region', 'keywords': "['keywords']", 'language': 'en', 'enforcement_probability': 0.8012800716887034, 'title_vec_b64': 'uoyAUDrCou08RQj+PRqb4Ttug6q9UcEDvJ1p8DxqcFO6PCUivIYyljxF/0G8rD32vDAiubyI77a8QTwuvKw0BDxD17Y8kiTnO+fXvjylbBA6T+/iO9E/9LufuLa8f+VkPSC9Xj1DbGG8DvHWPNc6gzwP1fI83eE2u2bRnrwyfck7qFzEPG2uMLtwr5K8ZAHTvBtn1rwL7rM5iG0ePQcG9LudMOi7k4LfOw1GVrwdwQM8JfnYPT3RM7uoW808qt6UPCordjumAm48dgXzvPknkDwZcYa8ZmdNvIiuNLsOah47mfz+vBEcG7uKLOo6nC0uvDMFTTtw2YA8g+XWPIF2vjrMtVO7iyjTPPc0BzyJPk28eGDou+J33jzDdAY78o8NPGWh9jtTSbe8HnC8PCmoDjr+tHo8spH5vJzeCDrVkNC7v4lWvDenPLuxKmI5/AAGu7xOerqeczu9Az4HPHctQzwkR6s8e2wkPIUoDbybEnu8nnNlu/G3ITyb1f48tK7qPIJ06zk+FrM7mxmePBuwl7yPIje8YzkHvJGLazwFxFi72vJ9OvJGrTwlQ3g8PvcoOlbFOTy3Q2q8jmkIOxLrAbwOyzq8ryu2vJonLDuUF6O7usEyvCPKnDvbXAo8f1tfvJ2V/zwW0oa8DRxUOx8/EDuO8S48sX8EPL7FqrxA8oc8utcVvMNeyTx8hKs7zJKWPJo82LzOrTA8lB2aO6kdSD0fz0q9A6KOPIKjwTrDyjs7+hF6uyaHCjumR807rVq0vA1en7xPcLO84P8BPFQgAzumDpY6BpF5vCo0frqe8q47UuInu3bdejqjJ9e6/XhBO4VXTLsyYJi8w+VPOq4DZLuJZPK8F2NBvQ4Cejyy9dM7YdDuO2NXcLy0rAO8eOUGusxOTzxaVoE8eLEVPAL/u7wx2ne4xkwYOgmYYDr8b9w8jUVhvFTpKrwNqGE7Qz0WO8WjUjzDb0M7qG1tvQiZ/LyREe69G4X7vNlRhDtBSR88IiXxOyW9drv+X9C8zI/eO6nxBzw0A9A70iQwPGO88zu4xXo61TZfO66gxz0MBFK8CmMBO1bRyjtp24O8j9ANu+ot8Lt3VBg8GlPfPFR2RzrUqMQ76S1ZvGtZHLskjtY8tCMOu/qZxDukS0e7kdW3vGL4ZTtZ35i7g7tEu9E/HLtyPI08RtaTOx11cjwyp4C7gUbSOyz8ZbsNg348DjLPO5DR1TyH5CQ8bmUoO7qgiDupB1a8kSnNPCYTkrzMoe48E+mkvBnqWbuSC4G7kwn9OzNAXryd6kq7jYX/vIcPejzIKTQ7y/IrOwRukDvihjs8KNyru/WiBjziXg87ggvDO+caCTy67HS77vI+vDhMerwj39A86MqVuwSMHjulEra7sp9WuugTir09qnu8Wnj7vHxX1jt+BIG80bP2u8KutDxCn089PQVhvIbibTyny4u8x1S+PIJxUrpJXGK8R/7nPDGFVr1XHk+9AwX/u4AAXrxHLwe73ecPvAEYzzsaO407RF/6vQhSZ7wbHa28m75xur1qdzv+3nE7EYUZvFMqtrwGHsQ4o65GPFzbGTxG/De8fYtQPDIFDjwhtf28smhG', 'description_vec_64': 'uoyAUDrCou08RQj+PRqb4Ttug6q9UcEDvJ1p8DxqcFO6PCUivIYyljxF/0G8rD32vDAiubyI77a8QTwuvKw0BDxD17Y8kiTnO+fXvjylbBA6T+/iO9E/9LufuLa8f+VkPSC9Xj1DbGG8DvHWPNc6gzwP1fI83eE2u2bRnrwyfck7qFzEPG2uMLtwr5K8ZAHTvBtn1rwL7rM5iG0ePQcG9LudMOi7k4LfOw1GVrwdwQM8JfnYPT3RM7uoW808qt6UPCordjumAm48dgXzvPknkDwZcYa8ZmdNvIiuNLsOah47mfz+vBEcG7uKLOo6nC0uvDMFTTtw2YA8g+XWPIF2vjrMtVO7iyjTPPc0BzyJPk28eGDou+J33jzDdAY78o8NPGWh9jtTSbe8HnC8PCmoDjr+tHo8spH5vJzeCDrVkNC7v4lWvDenPLuxKmI5/AAGu7xOerqeczu9Az4HPHctQzwkR6s8e2wkPIUoDbybEnu8nnNlu/G3ITyb1f48tK7qPIJ06zk+FrM7mxmePBuwl7yPIje8YzkHvJGLazwFxFi72vJ9OvJGrTwlQ3g8PvcoOlbFOTy3Q2q8jmkIOxLrAbwOyzq8ryu2vJonLDuUF6O7usEyvCPKnDvbXAo8f1tfvJ2V/zwW0oa8DRxUOx8/EDuO8S48sX8EPL7FqrxA8oc8utcVvMNeyTx8hKs7zJKWPJo82LzOrTA8lB2aO6kdSD0fz0q9A6KOPIKjwTrDyjs7+hF6uyaHCjumR807rVq0vA1en7xPcLO84P8BPFQgAzumDpY6BpF5vCo0frqe8q47UuInu3bdejqjJ9e6/XhBO4VXTLsyYJi8w+VPOq4DZLuJZPK8F2NBvQ4Cejyy9dM7YdDuO2NXcLy0rAO8eOUGusxOTzxaVoE8eLEVPAL/u7wx2ne4xkwYOgmYYDr8b9w8jUVhvFTpKrwNqGE7Qz0WO8WjUjzDb0M7qG1tvQiZ/LyREe69G4X7vNlRhDtBSR88IiXxOyW9drv+X9C8zI/eO6nxBzw0A9A70iQwPGO88zu4xXo61TZfO66gxz0MBFK8CmMBO1bRyjtp24O8j9ANu+ot8Lt3VBg8GlPfPFR2RzrUqMQ76S1ZvGtZHLskjtY8tCMOu/qZxDukS0e7kdW3vGL4ZTtZ35i7g7tEu9E/HLtyPI08RtaTOx11cjwyp4C7gUbSOyz8ZbsNg348DjLPO5DR1TyH5CQ8bmUoO7qgiDupB1a8kSnNPCYTkrzMoe48E+mkvBnqWbuSC4G7kwn9OzNAXryd6kq7jYX/vIcPejzIKTQ7y/IrOwRukDvihjs8KNyru/WiBjziXg87ggvDO+caCTy67HS77vI+vDhMerwj39A86MqVuwSMHjulEra7sp9WuugTir09qnu8Wnj7vHxX1jt+BIG80bP2u8KutDxCn089PQVhvIbibTyny4u8x1S+PIJxUrpJXGK8R/7nPDGFVr1XHk+9AwX/u4AAXrxHLwe73ecPvAEYzzsaO407RF/6vQhSZ7wbHa28m75xur1qdzv+3nE7EYUZvFMqtrwGHsQ4o65GPFzbGTxG/De8fYtQPDIFDjwhtf28smhG'}}]}}

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://github.com/lior-k/fast-elasticsearch-vector-scoring/issues/33?email_source=notifications&email_token=ABGGISDQ5OJWOJISRWLHZCLQV7G6DA5CNFSM4JSU3ML2YY3PNVWWK3TUL52HS4DFVREXG43VMVBW63LNMVXHJKTDN5WW2ZLOORPWSZGOEFMXEHQ#issuecomment-559510046, or unsubscribe https://github.com/notifications/unsubscribe-auth/ABGGISHA5322F6ESWCYSMMLQV7G6DANCNFSM4JSU3MLQ .

jwgwalton commented 4 years ago

Cheers lior-k, it was that.