search

Graylog2 / graylog2-server

Free and open log management
https://www.graylog.org
Other
7.37k stars 1.06k forks source link

Slow Elasticsearch Queries since sync 'permission' Checks #9328

Open HenryTheSir opened 3 years ago

HenryTheSir commented 3 years ago

The query which is send to ES uses since this PRs ( https://github.com/Graylog2/graylog2-server/pull/6573 / https://github.com/Graylog2/graylog2-server/pull/6651 ) an Terms Query for the Stream Permissions. In Environments with a high amount of streams, data in ES and an 'empty' Stream Selection every allowed stream ID gets listed in the Terms Query.

disclosure: The given examples were send directly to ES without graylog involved, Graylog was just involved to determine the actual query building in graylog itself.

Expected Behavior

Fast searches which are not slowed because of slow query.

Current Behavior

Searches get build like this: 

{
  "from": 0,
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "bool": {
            "filter": [
              {
                "query_string": {
                  "query": "message:exception",
                  "fields": [],
                  "type": "best_fields",
                  "tie_breaker": 0,
                  "default_operator": "or",
                  "max_determinized_states": 10000,
                  "allow_leading_wildcard": true,
                  "enable_position_increments": true,
                  "fuzziness": "AUTO",
                  "fuzzy_prefix_length": 0,
                  "fuzzy_max_expansions": 50,
                  "phrase_slop": 0,
                  "escape": false,
                  "auto_generate_synonyms_phrase_query": true,
                  "fuzzy_transpositions": true,
                  "boost": 1
                }
              },
              {
                "match_all": {
                  "boost": 1
                }
              }
            ],
            "adjust_pure_negative": true,
            "boost": 1
          }
        },
        {
          "range": {
            "timestamp": {
              "from": "2020-10-18 15:24:05.093",
              "to": "2020-10-19 15:24:05.093",
              "include_lower": true,
              "include_upper": true,
              "boost": 1
            }
          }
        },
        {
          "terms": {
            "streams": [
 "000000000000000000000001",
"000000000000000000000002",
"000000000000000000000003"
            ],
            "boost": 1
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  },
  "aggregations": {
    "agg-1": {
      "date_histogram": {
        "field": "timestamp",
        "format": "date_time",
        "interval": "30m",
        "offset": 0,
        "order": {
          "_key": "asc"
        },
        "keyed": false,
        "min_doc_count": 0
      }
    },
    "timestamp-min": {
      "min": {
        "field": "timestamp"
      }
    },
    "timestamp-max": {
      "max": {
        "field": "timestamp"
      }
    }
  }
}

Terms directive is in my example 120 streams, I cleaned the IDs out, because in another Env this IDs would not match. Just add your own IDs in the Terms directive.

This Search took first time:

{

    "took": 11012,
    "timed_out": false,
    "_shards": {
        "total": 605,
        "successful": 605,
        "skipped": 355,
        "failed": 0
    },
    "hits": {
        "total": 3170077,
        "max_score": 0,
        "hits": [ ]
    },`
second time:
`{

    "took": 8718,
    "timed_out": false,
    "_shards": {
        "total": 605,
        "successful": 605,
        "skipped": 355,
        "failed": 0
    },
    "hits": {
        "total": 3170077,
        "max_score": 0,
        "hits": [ ]
    },
    "aggregations": {

The same query without the Terms aggregation finishes in 3 seconds and in the second attempt (proably cached) in 100ms.

If we built one global filter query the results are fixer too: Query: 

{
  "from": 0,
  "size": 0,
  "query": {
    "bool": {
      "must": [
        {
          "bool": {
            "filter": [
              {
                "query_string": {
                  "query": "message:exception AND (streams:000000000000000000000001 OR streams:000000000000000000000002 OR streams:000000000000000000000003)",
                  "fields": [],
                  "type": "best_fields",
                  "tie_breaker": 0,
                  "default_operator": "or",
                  "max_determinized_states": 10000,
                  "allow_leading_wildcard": true,
                  "enable_position_increments": true,
                  "fuzziness": "AUTO",
                  "fuzzy_prefix_length": 0,
                  "fuzzy_max_expansions": 50,
                  "phrase_slop": 0,
                  "escape": false,
                  "auto_generate_synonyms_phrase_query": true,
                  "fuzzy_transpositions": true,
                  "boost": 1
                }
              },
              {
                "match_all": {
                  "boost": 1
                }
              }
            ],
            "adjust_pure_negative": true,
            "boost": 1
          }
        },
        {
          "range": {
            "timestamp": {
              "from": "2020-10-18 15:24:05.093",
              "to": "2020-10-19 15:24:05.093",
              "include_lower": true,
              "include_upper": true,
              "boost": 1
            }
          }
        }
      ],
      "adjust_pure_negative": true,
      "boost": 1
    }
  },
  "aggregations": {
    "agg-1": {
      "date_histogram": {
        "field": "timestamp",
        "format": "date_time",
        "interval": "30m",
        "offset": 0,
        "order": {
          "_key": "asc"
        },
        "keyed": false,
        "min_doc_count": 0
      }
    },
    "timestamp-min": {
      "min": {
        "field": "timestamp"
      }
    },
    "timestamp-max": {
      "max": {
        "field": "timestamp"
      }
    }
  }
}

Here again insert your own Stream IDs!

Result comes as follow: 

{

    "took": 1168,
    "timed_out": false,
    "_shards": {
        "total": 605,
        "successful": 605,
        "skipped": 355,
        "failed": 0
    },
    "hits": {
        "total": 3170077,
        "max_score": 0,
        "hits": [ ]
    },
    "aggregations": {

Possible Solution

Do not know exactly. Faster but ugly solution would be to chain the stream IDs with an 'OR'

e.g. sth like above.

Steps to Reproduce (for bugs)

  1. Create high amount of streams
  2. Search without stream selection and permission granted to nearly all streams
  3. watch thread pools for search in ES /_cat/thread_pool/search?v&s=node_name
  4. Thread Pool for search is used very much with just one query.

Context

The Terms query seems to start a internal ES search for every Term given in the Terms query. This is observed using the ES endpoint for search Threadpools (/_cat/thread_pool/search?v&s=node_name) Which results in high used search thread_pools and a slow search result.

Your Environment

About 30-40TB in ES Cluster and nearly all streams pointing to the same index set

dennisoelkers commented 3 years ago

Hey @HenryTheSir!

Thanks for sharing that observation. I don't know yet why the usage of the terms filter should be that much slower, but I will try to reproduce the scenario and see what the reason is and how we can improve it. I will keep you updated!

dennisoelkers commented 3 years ago

@HenryTheSir, I cannot reproduce the timing difference in a local setup. The run times for both results are nearly identical (which is also what I am expecting). Can you share a bit more about the differing thread pools usage you are seeing?

HenryTheSir commented 3 years ago

Here are some more details about the setup:

node_name name   active queue rejected
arbiter   search      0     0        0
hot01     search      0     0        0
hot02     search      0     0        0
hot03     search      0     0        0
hot04     search      0     0        0
hot05     search      0     0        0
hot06     search      0     0        0
hot07     search      0     0        0
hot08     search      0     0        0
master01  search      0     0        0
master02  search      0     0        0
warm01    search     10    93        0
warm02    search     10     0        0
warm03    search     10    45        0
warm04    search     10     4        0
warm05    search     13    79        0
warm06    search     13     5        0
warm07    search     13   110        0
warm08    search     13     3        0
warm09    search     13    85        0
warm10    search     11     0        0
{

    "took": 17564,
    "timed_out": false,
    "_shards": {
        "total": 605,
        "successful": 605,
        "skipped": 340,
        "failed": 0
    },
    "hits": {
        "total": 13445690,
        "max_score": 0,
        "hits": [ ]
    },
    "aggregations": {
        "agg-1": { ,

Same query with AND (streams: OR ... OR streams:stream_id_120)

node_name name   active queue rejected
arbiter   search      0     0        0
hot01     search      0     0        0
hot02     search      0     0        0
hot03     search      0     0        0
hot04     search      0     0        0
hot05     search      0     0        0
hot06     search      0     0        0
hot07     search      0     0        0
hot08     search      0     0        0
master01  search      0     0        0
master02  search      0     0        0
warm01    search      0     0        0
warm02    search      0     0        0
warm03    search      0     0        0
warm04    search      0     0        0
warm05    search      0     0        0
warm06    search      0     0        0
warm07    search      0     0        0
warm08    search      0     0        0
warm09    search      0     0        0
warm10    search      0     0        0

(I was probably to slow to get the used thread pools ) 

{

    "took": 1022,
    "timed_out": false,
    "_shards": {
        "total": 605,
        "successful": 605,
        "skipped": 340,
        "failed": 0
    },
    "hits": {
        "total": 13445690,
        "max_score": 0,
        "hits": [ ]

Do not know how I can make this more reproducable for you.

It doesn't matter by the way in which order (first terms or first 'OR / AND') the requests are made, so this is no cache issue that the 'OR'/'AND' query is much faster.

HenryTheSir commented 3 years ago

If one adds an "profile": true to the ES Query one can see, that the "type": "TermInSetQuery" needs the most time to be evaluated.

TermInSetQuery : "time_in_nanos": 1 603 556 826 BooleanQuery (If all streams get concated with OR): "time_in_nanos": 21 719 496

Truncated example snipped from one shard

    "type": "TermInSetQuery",
    "description": "streams:(000000000000000000000001 000000000000000000000002 000000000000000000000003 <truncated>)",
    "time_in_nanos": 1603556826,
    "breakdown": {
        "score": 0,
        "build_scorer_count": 150,
        "match_count": 0,
        "create_weight": 1361,
        "next_doc": 0,
        "match": 0,
        "create_weight_count": 1,
        "next_doc_count": 0,
        "score_count": 0,
        "build_scorer": 1603547722,
        "advance": 7529,
        "advance_count": 63
    }

To be noticed is that no child processes are spawnd.

{

    "type": "BooleanQuery",
    "description": "streams:<truncated> streams:<truncated> streams:<truncated>",
    "time_in_nanos": 21719496,
    "breakdown": {
        "score": 0,
        "build_scorer_count": 159,
        "match_count": 0,
        "create_weight": 205703,
        "next_doc": 0,
        "match": 0,
        "create_weight_count": 1,
        "next_doc_count": 0,
        "score_count": 0,
        "build_scorer": 16640549,
        "advance": 4872931,
        "advance_count": 153
    },
    "children": [
        {
            "type": "TermQuery",
            "description": "streams:<id>",
            "time_in_nanos": 297552,
            "breakdown": {
                "score": 0,
                "build_scorer_count": 53,
                "match_count": 0,
                "create_weight": 606,
                "next_doc": 0,
                "match": 0,
                "create_weight_count": 1,
                "next_doc_count": 0,
                "score_count": 0,
                "build_scorer": 296892,
                "advance": 0,
                "advance_count": 0
            }
        }
        ,
        {
            "type": "TermQuery",
            "description": "streams:<id>",
            "time_in_nanos": 121915,
            "breakdown": {
                "score": 0,
                "build_scorer_count": 53,
                "match_count": 0,
                "create_weight": 184,
                "next_doc": 0,
                "match": 0,
                "create_weight_count": 1,
                "next_doc_count": 0,
                "score_count": 0,
                "build_scorer": 121677,
                "advance": 0,
                "advance_count": 0
            }
        } <<<truncated

Where in the OR query subprocesses are spawned

To be mentioned is, that the terms query needs at least 16 Stream IDs, otherwise Lucene/ES switches internally to an ConstantScoreQuery with BooleanQuery and TermQuery. If I reduce my stream_ids to this level (below 16) and keep the streamids wich hold the most data in the query the query performance boosts up while getting nearly the exact result.

dennisoelkers commented 3 years ago

To be mentioned is, that the terms query needs at least 16 Stream IDs, otherwise Lucene/ES switches internally to an ConstantScoreQuery with BooleanQuery and TermQuery. If I reduce my stream_ids to this level (below 16) and keep the streamids wich hold the most data in the query the query performance boosts up while getting nearly the exact result.

That was the information I was missing. From the snippets I have seen, I had the impression that the performance regression could be reproduced by using a couple of stream ids, but it seems to occur only for 16+ streams. I still think that an unknown additional factor, specific to your environment comes into play, which makes the OR-query so much more efficient for you compared to others, because the performance improvement you are seeing is still not reproducible for me.

dennisoelkers commented 3 years ago

@HenryTheSir: I am wondering if the ES node query cache might interfer with the results. To rule this out, I would like to ask you if you could execute two queries:

First this one:

{
    "from": 0,
    "size": 0,
    "query": {
        "bool": {
            "filter": {
                "query_string": {
                    "query": "streams:000000000000000000000001 OR <and so on, for your 16+ stream ids>
                }
            }
        }
    },
    "aggregations": {
        "timestamp-min": {
            "min": {
                "field": "timestamp"
            }
        },
        "timestamp-max": {
            "max": {
                "field": "timestamp"
            }
        }
    }
}

Then this one:

{
    "from": 0,
    "size": 0,
    "query": {
        "bool": {
            "filter": {
                "terms": {
                    "streams": [
                        <your 16+ stream ids here>
                    ],
                    "boost": 1
                }
            }
        }
    },
    "aggregations": {
        "timestamp-min": {
            "min": {
                "field": "timestamp"
            }
        },
        "timestamp-max": {
            "max": {
                "field": "timestamp"
            }
        }
    }
}

The runtimes of these queries are interesting, ideally you would run them several times, mixing those two.

HenryTheSir commented 3 years ago

@dennisoelkers : Sorry for the delayed answer: here are my 'bench' results against an minimalized szenarion (otherwise I would probably take down complete ES cluster).

Query was sent against one index, here are the results 10 iterations and always switching between terms and or query. 


or
{"took":17,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":16,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":17,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":44,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":3,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":17,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":3,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":16,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":17,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":15,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":2,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":16,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":4,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
or
{"took":22,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}
terms
{"took":5,"timed_out":false,"_shards":{"total":4,"successful":4,"skipped":0,"failed":0},"hits":{"total":386432561,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.521122760583E12,"value_as_string":"2018-03-15 14:06:00.583"},"timestamp-max":{"value":1.61003847E12,"value_as_string":"2021-01-07 16:54:30.000"}}}

for i  in {1..10}
do
   echo "or" 
   curl -X POST  $(hostname):9200/<indexname>/_search -H "Content-Type: application/json" --data @or_body.json 
   echo "" 
   echo "terms" 
   curl -X POST  $(hostname):9200/<indexname>/_search -H "Content-Type: application/json" --data @terms_body.json 
   echo "" 
done

Interesting results if the query hits just one index. I will take the risk and run against full cluster. 10 seconds pause between each search 

or
{"took":54776,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39359934508,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":58229,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39361144590,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}
or
{"took":9546,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39361930056,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":8953,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39362361631,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}
or
{"took":9380,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39362696078,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}
terms
{"took":9218,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39362845246,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}
or
{"took":9793,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39363176711,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}
terms
{"took":9033,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39363274074,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":20419,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39363714857,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":8799,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39364329393,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":28481,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39364644294,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":29443,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39365275045,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":10252,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39365885623,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":14412,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39366264945,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":19939,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39366742971,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":19491,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39367026863,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":9525,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39367597440,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":8878,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39367889883,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
or
{"took":9341,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39368245675,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11,"value_as_string":"1975-01-29 22:25:20.057"},"timestamp-max":{"value":1.950299992E12,"value_as_string":"2031-10-20 21:59:52.000"}}}
terms
{"took":8799,"timed_out":false,"num_reduce_phases":2,"_shards":{"total":706,"successful":706,"skipped":0,"failed":0},"hits":{"total":39368385942,"max_score":0.0,"hits":[]},"aggregations":{"timestamp-min":{"value":1.60266320057E11},"timestamp-max":{"value":1.950299992E12}}}