chrisronline
commented
6 years ago Hi @VincentWiz,
Can you paste the mappings for .kibana?
GET .kibana/_mappings
Closed VincentWiz closed 6 years ago
chrisronline
commented
6 years ago Hi @VincentWiz,
Can you paste the mappings for .kibana?
GET .kibana/_mappings
cmg-dev
commented
6 years ago I've got the same behaviour on Mac OS. I used the Elastic-Stack Docker repo.
https://github.com/elastic/stack-docker
This should make it more reproduceable.
chrisronline
commented
6 years ago @cmg-dev Can you provide your mappings too, for .kibana?
cmg-dev
commented
6 years ago @chrisronline Sure.
{
".kibana": {
"mappings": {
"doc": {
"properties": {
"config": {
"dynamic": "true",
"properties": {
"buildNum": {
"type": "keyword"
},
"xPackMonitoring:showBanner": {
"type": "boolean"
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"optionsJSON": {
"type": "text"
},
"panelsJSON": {
"type": "text"
},
"refreshInterval": {
"properties": {
"display": {
"type": "keyword"
},
"pause": {
"type": "boolean"
},
"section": {
"type": "integer"
},
"value": {
"type": "integer"
}
}
},
"timeFrom": {
"type": "keyword"
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"graph-workspace": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"numLinks": {
"type": "integer"
},
"numVertices": {
"type": "integer"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
},
"wsState": {
"type": "text"
}
}
},
"index-pattern": {
"properties": {
"fieldFormatMap": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"fields": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"timeFieldName": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"title": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
"search": {
"properties": {
"columns": {
"type": "keyword"
},
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"sort": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"fields": {
"keyword": {
"ignore_above": 2048,
"type": "keyword"
}
},
"type": "text"
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"savedSearchId": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
"type": "text"
}
}
}
}
}
}
}
}I've got another Docker-Compose Setup running, that is 2 weeks older.
This does not show this behaviour
The _mappings from this instance look like:
{
".kibana": {
"mappings": {
"doc": {
"dynamic": "strict",
"properties": {
"config": {
"dynamic": "true",
"properties": {
"buildNum": {
"type": "keyword"
},
"defaultIndex": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
},
"xPackMonitoring:allowReport": {
"type": "boolean"
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"optionsJSON": {
"type": "text"
},
"panelsJSON": {
"type": "text"
},
"refreshInterval": {
"properties": {
"display": {
"type": "keyword"
},
"pause": {
"type": "boolean"
},
"section": {
"type": "integer"
},
"value": {
"type": "integer"
}
}
},
"timeFrom": {
"type": "keyword"
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"graph-workspace": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"numLinks": {
"type": "integer"
},
"numVertices": {
"type": "integer"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
},
"wsState": {
"type": "text"
}
}
},
"index-pattern": {
"properties": {
"fieldFormatMap": {
"type": "text"
},
"fields": {
"type": "text"
},
"intervalName": {
"type": "keyword"
},
"notExpandable": {
"type": "boolean"
},
"sourceFilters": {
"type": "text"
},
"timeFieldName": {
"type": "keyword"
},
"title": {
"type": "text"
}
}
},
"search": {
"properties": {
"columns": {
"type": "keyword"
},
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"sort": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "keyword"
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"fields": {
"keyword": {
"ignore_above": 2048,
"type": "keyword"
}
},
"type": "text"
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"savedSearchId": {
"type": "keyword"
},
"title": {
"type": "text"
},
"uiStateJSON": {
"type": "text"
},
"version": {
"type": "integer"
},
"visState": {
"type": "text"
}
}
}
}
}
}
}
}And probably more usefull, the diff of both _mappings
diff kibana_broken.json kibana_healthy.json yields:
4a5
> "dynamic": "strict",
12c13,22
< "xPackMonitoring:showBanner": {
---
> "defaultIndex": {
> "fields": {
> "keyword": {
> "ignore_above": 256,
> "type": "keyword"
> }
> },
> "type": "text"
> },
> "xPackMonitoring:allowReport": {
106,111d115
< "fields": {
< "keyword": {
< "ignore_above": 256,
< "type": "keyword"
< }
< },
115,120d118
< "fields": {
< "keyword": {
< "ignore_above": 256,
< "type": "keyword"
< }
< },
123,129c121,127
< "timeFieldName": {
< "fields": {
< "keyword": {
< "ignore_above": 256,
< "type": "keyword"
< }
< },
---
> "intervalName": {
> "type": "keyword"
> },
> "notExpandable": {
> "type": "boolean"
> },
> "sourceFilters": {
131a130,132
> "timeFieldName": {
> "type": "keyword"
> },
133,138d133
< "fields": {
< "keyword": {
< "ignore_above": 256,
< "type": "keyword"
< }
< },
221,227c216
< "fields": {
< "keyword": {
< "ignore_above": 256,
< "type": "keyword"
< }
< },
< "type": "text"
---
> "type": "keyword"
VincentWiz
commented
6 years ago Hey,
Here is the mapping :
{
".kibana-6": {
"mappings": {
"doc": {
"properties": {
"config": {
"properties": {
"buildNum": {
"type": "long"
},
"dateFormat:dow": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"defaultIndex": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"discover:aggs:terms:size": {
"type": "long"
},
"indexPattern:placeholder": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"indexPatterns:warnAboutUnsupportedTimePatterns": {
"type": "boolean"
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "long"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"optionsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"panelsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timeRestore": {
"type": "boolean"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"index-pattern": {
"properties": {
"fields": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"notExpandable": {
"type": "boolean"
},
"timeFieldName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"search": {
"properties": {
"columns": {
"type": "keyword"
},
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"sort": {
"type": "keyword"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 2048
}
}
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
},
"visState": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
dqhung215
commented
6 years ago I also had the same problem. What is the solution for it? Please advise!
cmg-dev
commented
6 years ago @dqhung215 Could you try it with the latest version from this repository:
https://github.com/elastic/stack-docker
This now works for me, i think. Try to migrate my own docker-compose-setup to this.
VincentWiz
commented
6 years ago Do we have an idea for those who are not running on Docker? :)
cmg-dev
commented
6 years ago @VincentWiz Probably, if we can isolate the problem ;)
That's what I'm trying to do, with the docker-setup.
This Issue is related, I believe https://github.com/elastic/stack-docker/issues/15.
VincentWiz
commented
6 years ago @cmg-dev Thanks a lot, i'll look at it
chrisronline
commented
6 years ago I'm not sure what's exactly happening to cause it, but I think I know why it's failing.
"type": {
"fields": {
"keyword": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}The above mapping will not work with the code properly. The one from the guide is:
"type": {
"type": "keyword"
},The type property needs to be unanalyzed so a bool filter will work, from this code.
@dqhung215 @cmg-dev Can I assume you both followed the migration steps exactly too?
cmg-dev
commented
6 years ago @chrisronline I have no previous Kibana Setup.
I'm Experiencing this issue, but maybe it has a different cause.
I can easily verify/reproduce it, by using the Docker-Compose setup, provided on https://github.com/elastic/stack-docker.
I have the feeling, that during the startup of the systems, something happens and a race condition of some kind appears.
I will let you know, if I find something out.
chrisronline
commented
6 years ago @cmg-dev Possibly. I just went through a fresh install using the docker setup and didn't experience the problem at all. If you try the process again, does it happen every time?
cmg-dev
commented
6 years ago @chrisronline Firstly, that are good news, I guess.
To your question: No unfortunately not.
I asked several co-worker it seems, that the problem focuses on the ones with newer hardware.
We're all running MacBooks but some - lile me - are using the latest models. Can't find any other meaningful differences :(
chrisronline
commented
6 years ago @cmg-dev
Did the workaround I mentioned work for you? I will dig more into this, but I want to make sure I can unblock you if possible.
zaakiy
commented
6 years ago @chrisronline Your workaround is aligned with the instructions at https://www.elastic.co/guide/en/kibana/6.0/migrating-6.0-index.html which I followed.
Kibana does not show me any existing Index Patterns nor does it show any new index pattern I try to create, but every time I try to create a new Index Pattern, the number of documents in my Kibana index increases by 1.
This leads me to believe that my old index patterns are still in there, but Kibana 6 is not reading them properly.
chrisronline
commented
6 years ago @zaakiy What does your .kibana mapping look like?
zaakiy
commented
6 years ago @chrisronline
Here it is (I'm seeing lot's of ignore_above 256 even though I followed the migration instructions):
{
".kibana-6": {
"mappings": {
"doc": {
"properties": {
"config": {
"properties": {
"buildNum": {
"type": "long"
},
"context:defaultSize": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"context:step": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"courier:ignoreFilterIfFieldNotInIndex": {
"type": "boolean"
},
"defaultIndex": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"discover:aggs:terms:size": {
"type": "long"
},
"doc_table:highlight": {
"type": "boolean"
},
"doc_table:highlight:all_fields": {
"type": "boolean"
},
"filters:pinnedByDefault": {
"type": "boolean"
},
"indexPattern:placeholder": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"indexPatterns:warnAboutUnsupportedTimePatterns": {
"type": "boolean"
},
"notifications:banner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"notifications:lifetime:banner": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"visualization:tileMap:maxPrecision": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "long"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"optionsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"panelsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timeRestore": {
"type": "boolean"
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"index-pattern": {
"properties": {
"fieldFormatMap": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"fields": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"notExpandable": {
"type": "boolean"
},
"timeFieldName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"search": {
"properties": {
"columns": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "long"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"sort": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "keyword"
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
},
"visState": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}68 documents in the kibana index
Try to create an index pattern
Screenshot after index pattern is "created". Note that there is still no index pattern list on the left hand side. Also note that the index is not starred, which means that the Kibana UI knows that this is not the first index. So Kibana is aware of other indexes but is not displaying them.
Now there are 68 documents in the kibana index, an increase of one. EDIT: meant to say 69
Click on remove index pattern
Back to 68 documents
In summary, Kibana is creating the index pattern but not displaying either old index patterns or newly created ones. However Kibana is aware that other index patterns exist because it does not automatically create the new index pattern as default (starred).
zaakiy
commented
6 years ago @chrisronline In case it helps, I happened to save the results of me reindex operation which demonstrates that I used the new template as per the Migration instructions. EDIT: this would have cause errors if I hadn't used the new template which I'm guessing you probably already know.
# curl -XPOST 'localhost:9200/_reindex?pretty' -H 'Content-Type: application/json' -d'
> {
> "source": {
> "index": ".kibana"
> },
> "dest": {
> "index": ".kibana-6"
> },
> "script": {
> "source": "ctx._source = [ ctx._type : ctx._source ]; ctx._source.type = ctx._type; ctx._id = ctx._type + \":\" + ctx._id; ctx._type = \"doc\"; "
> }
> }
> '{
"took" : 3745,
"timed_out" : false,
"total" : 56,
"updated" : 0,
"created" : 56,
"deleted" : 0,
"batches" : 1,
"version_conflicts" : 0,
"noops" : 0,
"retries" : {
"bulk" : 0,
"search" : 0
},
"throttled_millis" : 0,
"requests_per_second" : -1.0,
"throttled_until_millis" : 0,
"failures" : [ ]
}@zaakiy You are correct. The problem is the type property in the mapping. It needs to be just a keyword instead of text with a keyword multi-field.
Try these steps and let me know if it fixes it:
1) Create a .kibana-6.1 index based on your existing mappings, but make sure type looks like:
"type": {
"type": "keyword"
},2) Reindex from .kibana-6 into .kibana-6.1:
POST _reindex
{
"source": {
"index": ".kibana-6"
},
"dest": {
"index": ".kibana-6.1"
}
}3) Realias
POST /_aliases
{
"actions" : [
{ "add": { "index": ".kibana-6.1", "alias": ".kibana" } },
]
}These steps will leave you with two indices, .kibana-6 and .kibana-6.1.
Now, try loading the index pattern page in Kibana. Let's see if that fixes it for you
zaakiy
commented
6 years ago @chrisronline Thanks it worked.
( I'm struggling to come up with a hypothesis as to why it did not work for me before. I obviously must have not set up the template correctly. )
Many thanks.
tylersmalley
commented
6 years ago @zaakiy @cmg-dev @VincentWiz @dqhung215,
The only way I can see this being possible is if the .kibana-6 index was not created with the mappings. This is done in step two of the manual migration.
When you were able to reproduce with stack-docker, were you using a new Elasticsearch cluster or re-using the one which had the issue? The Kibana data/mapping is persisted to the Elasticsearch cluster, re-installing just Kibana will not change this.
VincentWiz
commented
6 years ago Hey @tylersmalley , I did all steps written in the migration manual. I didn't try on docker, i'll let people who did answer your question.
T3rm1
commented
6 years ago I have the same problem. Updated from 5.4.3 to 6.1.3. Cannot create new index patterns.
Here is my .kibana-6 index:
{
".kibana-6-reindex": {
"mappings": {
"doc": {
"properties": {
"config": {
"properties": {
"buildNum": {
"type": "long"
},
"dashboard:defaultDarkTheme": {
"type": "boolean"
},
"dateFormat:dow": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"defaultIndex": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"discover:aggs:terms:size": {
"type": "long"
},
"timelion:showTutorial": {
"type": "boolean"
}
}
},
"dashboard": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "long"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"optionsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"panelsJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"refreshInterval": {
"properties": {
"display": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"pause": {
"type": "boolean"
},
"section": {
"type": "long"
},
"value": {
"type": "long"
}
}
},
"timeFrom": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"timeRestore": {
"type": "boolean"
},
"timeTo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"index-pattern": {
"properties": {
"fields": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"notExpandable": {
"type": "boolean"
},
"timeFieldName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"search": {
"properties": {
"columns": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"hits": {
"type": "long"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"sort": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
}
}
},
"server": {
"properties": {
"uuid": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"timelion-sheet": {
"properties": {
"description": {
"type": "text"
},
"hits": {
"type": "integer"
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text"
}
}
},
"timelion_chart_height": {
"type": "integer"
},
"timelion_columns": {
"type": "integer"
},
"timelion_interval": {
"type": "keyword"
},
"timelion_other_interval": {
"type": "keyword"
},
"timelion_rows": {
"type": "integer"
},
"timelion_sheet": {
"type": "text"
},
"title": {
"type": "text"
},
"version": {
"type": "integer"
}
}
},
"type": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"updated_at": {
"type": "date"
},
"url": {
"properties": {
"accessCount": {
"type": "long"
},
"accessDate": {
"type": "date"
},
"createDate": {
"type": "date"
},
"url": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 2048
}
}
}
}
},
"visualization": {
"properties": {
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"kibanaSavedObjectMeta": {
"properties": {
"searchSourceJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"savedSearchId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"title": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"uiStateJSON": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"version": {
"type": "long"
},
"visState": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
}
}
}
}
}
}
itkovian
commented
6 years ago Hmm, I have the mentioned
"type" : {
"type" : "keyword"
},but I cannot create new index patterns or scripted fields :/
I'm at kibana-6.1.0, having migrated from 5.6.
Well, I do not see them, even though the api (api/saved_objects/?type=index-pattern) does show it:
17 |
-- | --
id | "96ecb090-18ed-11e8-bf2c-b724d4cbd20f"
type | "index-pattern"
updated_at | "2018-02-23T23:02:15.320Z"
version | 1
attributes |
title | "torque-2018"
timeFieldName | "@timestamp"
rhoboat
commented
6 years ago @itkovian So, were you able to successfully create that index pattern?
itkovian
commented
6 years ago Well, hitting the create button just stays on the same page. And it does not show up in the list of index patterns at all. So I'm not sure :)
Well, now to does show up, but clicking it gives an error:
Error: 403 Response
value/<@https://tangela1.ugent.be/bundles/kibana.bundle.js?v=16337:37:84505
processQueue@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:132456
scheduleProcessQueue/<@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:133349
$digest@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:144239
$apply@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:147007
done@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:100015
completeRequest@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:104697
createHttpBackend/</xhr.onload@https://tangela1.ugent.be/bundles/commons.bundle.js?v=16337:29:105435
forzagreen
commented
6 years ago I had this issue on Kibana 6.2.2, but not on 6.0.1 Both are Dockers containers.
chrisronline
commented
6 years ago @itkovian @ForzaGreen
Can you paste your entire .kibana mappings? And also include the result of:
POST .kibana/_search
{
"query": {
"term": {
"type": {
"value": "index-pattern"
}
}
}
}
ksing5
commented
6 years ago Please check the 'elasticSearch' console. In my case the local drive running elastic seach had just 5% free space and thus was unable to create index patterns
stacey-gammon
commented
6 years ago Is this still an issue?
stacey-gammon
commented
6 years ago It sounds like a lot of the above issues might be situational, possibly upgrades that got mangled, and do not all point to a single bug. I'm inclined to close this, since it looks like there hasn't been too much activity, and there hasn't been a concrete bug discovered, and also because we are on 6.4 now.
Please feel free to re-open with more details if anyone feels this is in error!
lanerjo
commented
6 years ago I am having a very similar issue on a fresh install of 6.3.2
sampathlovely
commented
5 years ago Hi I am not able to create an index in kibana normal csv file. i am very new to this subject please advise.
When i running mapping below the error i am facing.
{ ".kibana_1" : { "mappings" : { "doc" : { "dynamic" : "strict", "properties" : { "canvas-workpad" : { "dynamic" : "false", "properties" : { "@created" : { "type" : "date" }, "@timestamp" : { "type" : "date" }, "id" : { "type" : "text", "index" : false }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword" } } } } }, "config" : { "dynamic" : "true", "properties" : { "buildNum" : { "type" : "keyword" }, "dashboard:defaultDarkTheme" : { "type" : "boolean" }, "defaultIndex" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "format:number:defaultPattern" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "format:percent:defaultPattern" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } }, "history:limit" : { "type" : "long" }, "timelion:showTutorial" : { "type" : "boolean" }, "visualization:regionmap:showWarnings" : { "type" : "boolean" }, "visualization:tileMap:maxPrecision" : { "type" : "long" } } }, "dashboard" : { "properties" : { "description" : { "type" : "text" }, "hits" : { "type" : "integer" }, "kibanaSavedObjectMeta" : { "properties" : { "searchSourceJSON" : { "type" : "text" } } }, "optionsJSON" : { "type" : "text" }, "panelsJSON" : { "type" : "text" }, "refreshInterval" : { "properties" : { "display" : { "type" : "keyword" }, "pause" : { "type" : "boolean" }, "section" : { "type" : "integer" }, "value" : { "type" : "integer" } } }, "timeFrom" : { "type" : "keyword" }, "timeRestore" : { "type" : "boolean" }, "timeTo" : { "type" : "keyword" }, "title" : { "type" : "text" }, "uiStateJSON" : { "type" : "text" }, "version" : { "type" : "integer" } } }, "graph-workspace" : { "properties" : { "description" : { "type" : "text" }, "kibanaSavedObjectMeta" : { "properties" : { "searchSourceJSON" : { "type" : "text" } } }, "numLinks" : { "type" : "integer" }, "numVertices" : { "type" : "integer" }, "title" : { "type" : "text" }, "version" : { "type" : "integer" }, "wsState" : { "type" : "text" } } }, "index-pattern" : { "properties" : { "fieldFormatMap" : { "type" : "text" }, "fields" : { "type" : "text" }, "intervalName" : { "type" : "keyword" }, "notExpandable" : { "type" : "boolean" }, "sourceFilters" : { "type" : "text" }, "timeFieldName" : { "type" : "keyword" }, "title" : { "type" : "text" }, "type" : { "type" : "keyword" }, "typeMeta" : { "type" : "keyword" } } }, "kql-telemetry" : { "properties" : { "optInCount" : { "type" : "long" }, "optOutCount" : { "type" : "long" } } }, "migrationVersion" : { "dynamic" : "true", "properties" : { "index-pattern" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 256 } } } } }, "namespace" : { "type" : "keyword" }, "search" : { "properties" : { "columns" : { "type" : "keyword" }, "description" : { "type" : "text" }, "hits" : { "type" : "integer" }, "kibanaSavedObjectMeta" : { "properties" : { "searchSourceJSON" : { "type" : "text" } } }, "sort" : { "type" : "keyword" }, "title" : { "type" : "text" }, "version" : { "type" : "integer" } } }, "server" : { "properties" : { "uuid" : { "type" : "keyword" } } }, "space" : { "properties" : { "_reserved" : { "type" : "boolean" }, "color" : { "type" : "keyword" }, "description" : { "type" : "text" }, "initials" : { "type" : "keyword" }, "name" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 2048 } } } } }, "telemetry" : { "properties" : { "enabled" : { "type" : "boolean" } } }, "timelion-sheet" : { "properties" : { "description" : { "type" : "text" }, "hits" : { "type" : "integer" }, "kibanaSavedObjectMeta" : { "properties" : { "searchSourceJSON" : { "type" : "text" } } }, "timelion_chart_height" : { "type" : "integer" }, "timelion_columns" : { "type" : "integer" }, "timelion_interval" : { "type" : "keyword" }, "timelion_other_interval" : { "type" : "keyword" }, "timelion_rows" : { "type" : "integer" }, "timelion_sheet" : { "type" : "text" }, "title" : { "type" : "text" }, "version" : { "type" : "integer" } } }, "type" : { "type" : "keyword" }, "updated_at" : { "type" : "date" }, "url" : { "properties" : { "accessCount" : { "type" : "long" }, "accessDate" : { "type" : "date" }, "createDate" : { "type" : "date" }, "url" : { "type" : "text", "fields" : { "keyword" : { "type" : "keyword", "ignore_above" : 2048 } } } } }, "visualization" : { "properties" : { "description" : { "type" : "text" }, "kibanaSavedObjectMeta" : { "properties" : { "searchSourceJSON" : { "type" : "text" } } }, "savedSearchId" : { "type" : "keyword" }, "title" : { "type" : "text" }, "uiStateJSON" : { "type" : "text" }, "version" : { "type" : "integer" }, "visState" : { "type" : "text" } } } } } } } }
sampathlovely
commented
5 years ago Hi one more issue i am facing I am not able to config index pattern in elastic search.
This is input code:
input{ file{ path=>"C:\Logstach\logstash-6.4.2/kibana2csv" start_position => "beginning" sincedb_path=>"/dev/null" } } filter{ csv{ separator=>";" columns=>["coreid";"University_Name";"Region";"Country";"City";"Campus";"program_id";"Program_Name";"Program_Type";"Program_Level";"Program_Subject";"Program_Category";"Program_Start_Month";"Program_Duration";"total_fee_USD";"Total_Fee_Domestic_Students";"Total_Fee_International_Students";"Total_Fee_Currency";"Minimum_SAT";"Maximum_SAT";"Minimum_GMAT";"Maximum_GMAT";"Avg_GMAT";"Minimum_GRE";"Maximum_GRE";"Minimum_GPA";"Maximum_GPA";"Minimum_ATAR";"Maximum_ATAR";"Minimum_STAT";"Maximum_STAT";"Minimum_IELTS";"Maximum_IELTS";"Minimum_TOEFL";"Maximum_TOEFL";"Diversity_International";"Minimum_work_experience";"Average_Work_experience";"Ethnicity";"Activities_Sports";"Gender_Distribution";"public_private";"In_campus_Housing";"Scholarships";"Alumni_Starting_Salaries";"Student_faculty_ratio";"Graduation_Rate";"Institution_Logo";"Institution_Campus_URL";"AdmissionContact";"AdmissionContactEmail";"AdmissionContactPhone";"Program_Description";"QSIUWUR";"QSIUTL";"QSIUAFD";"QSIUEM";"QSIUST";"Sponsored_University";"TU_TM";"Norm_GMAT";"Norm_SAT";"Norm_GRE";"Norm_ATAR";"Norm_STAT";"Norm_Min_Score";"Rankings";"Stars";"Test_Required";"Global_Min_SAT";"Global_Min_GMAT";"Global_Min_GRE";"Global_Min_GPA";"Global_Min_ATAR";"Global_Min_STAT";"Global_Min_IELTS";"Global_Min_TOEFL";"QSIUDI";"QSIU_DFee";"QSIU_IFee";"Norm_CGPA";"Country_Code";"Banner_URL";"Video_URL";"Dom_USD";"International_USD";"processed_Fee_USD";"SubRegion";"Class_Size";"Apply_Flag";"Instruction_Medium";"Count_Faculty";"Count_Students_International";"Age_Institution"] } Mutate{convert=>["program_id", "integer"]} Mutate{convert=>["coreid", "integer"]} } output{ elasticsearch{ "hosts" => "localhost:9200" "index" => "kibana2csv" "document_type" => "data" } stdout{} }
This is cmd error:
Microsoft Windows [Version 10.0.17134.590] (c) 2018 Microsoft Corporation. All rights reserved.
C:\Logstach\logstash-6.4.2\bin>logstash -f Excel.txt.config 'findstr' is not recognized as an internal or external command, operable program or batch file. Sending Logstash logs to C:/Logstach/logstash-6.4.2/logs which is now configured via log4j2.properties [2019-03-01T17:08:17,614][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified [2019-03-01T17:08:17,703][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.4.2"} [2019-03-01T17:08:17,718][INFO ][logstash.config.source.local.configpathloader] No config files found in path {:path=>"C:/Logstach/logstash-6.4.2/bin/Excel.txt.config"} [2019-03-01T17:08:17,719][ERROR][logstash.config.sourceloader] **No configuration found in the configured sources.** [2019-03-01T17:08:17,872][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
After upgrading from Kibana 5.6 to Kibana 6 (and ElasticSearch), I can't create an index pattern. In the chrome console : "Cannot read property 'indexPatterns' of undefined" when clicking on "Create". But nothing is wrong in Kibana logs or ES logs. No error messages when setting it to default, but the index pattern just doesn't appear in the list.
Kibana version: 6.0
Elasticsearch version: 6.0
Server OS version: Ubuntu 14.04
Browser version: Chrome 62.0
Browser OS version:
Original install method (e.g. download page, yum, from source, etc.):
The pattern is well created at http://kibanaIP:Port/api/saved_objects/index-pattern/322c1d30-cf73-11e7-9bdf-959368ea42e0
By checking at http://kibanaIP:Port/api/saved_objects/?type=index-pattern I have " {"page":1,"per_page":20,"total":0,"saved_objects":[]} " that is strange because i can't see the pattern.
But asking http://10.1.12.32:5601/api/saved_objects/, i can see my pattern (and visualisations).