Connection error - Githubissues

dmac707 commented 9 years ago

Hi Adrien,

I am very interested in your plug-in but haven't successfully connected to my ElasticSearch instance with it. When clicking on scanner la base I get the following error:

Traceback (most recent call last): File "~/.qgis2/python/plugins/ElasticSearchConnector/elasticsearch_connector.py", line 112, in onConnectClick if mapping[field]["type"] == "geo_point" or mapping[field]["type"] == "geo_shape": KeyError: 'type'

I am running elasticsearch 1.5.1 and QGIS 2.8.1 on Ubuntu 14.04 LTS. The data was imported to ES using ogr2ogr from PostGIS.

Regards,

David

AdrienVH commented 9 years ago

Hi David, It's awesome that you tried the plugin ! I'm happy :+1: The occured error seems to be related to your mappings... I think that one field of one of your types of one of your indices doesn't have any "type" (eg. integer, string, date, etc.). The plugin needs to know those types to recognize a "geo" field (geo_shape / geo_point) among all the fields of each type of each index. Could you show us your mappings ? You can take a look at them on "http://localhost:9200/_mappings?pretty"... Thank you again for your feedback ! Adrien

dmac707 commented 9 years ago

Hi Adrian,

Thank you for getting back to me. I did think that how GDAL created the mappings was a bit dubious. My only reason for using GDAL is because rivers are being deprecated by ElasticSearch. Here are my mappings

{
  "build.elasticsearchaddresses" : {
    "mappings" : { }
  },
  ".kibana" : {
    "mappings" : {
      "config" : {
        "properties" : {
          "buildNum" : {
            "type" : "long"
          },
          "defaultIndex" : {
            "type" : "string"
          }
        }
      },
      "index-pattern" : {
        "properties" : {
          "customFormats" : {
            "type" : "string"
          },
          "fields" : {
            "type" : "string"
          },
          "intervalName" : {
            "type" : "string"
          },
          "timeFieldName" : {
            "type" : "string"
          },
          "title" : {
            "type" : "string"
          }
        }
      },
      "visualization" : {
        "properties" : {
          "description" : {
            "type" : "string"
          },
          "kibanaSavedObjectMeta" : {
            "properties" : {
              "searchSourceJSON" : {
                "type" : "string"
              }
            }
          },
          "savedSearchId" : {
            "type" : "string"
          },
          "title" : {
            "type" : "string"
          },
          "version" : {
            "type" : "integer"
          },
          "visState" : {
            "type" : "string"
          }
        }
      }
    }
  },
  "addressbase_premium" : {
    "mappings" : {
      "FeatureCollection" : {
        "properties" : {
          "geometry" : {
            "properties" : {
              "coordinates" : {
                "type" : "geo_point",
                "store" : true
              },
              "type" : {
                "type" : "string",
                "store" : true
              }
            }
          },
          "properties" : {
            "properties" : {
              "abp_id" : {
                "type" : "integer",
                "store" : true
              },
              "address_caps" : {
                "type" : "string",
                "store" : true
              },
              "authoritycode" : {
                "type" : "integer",
                "store" : true
              },
              "ccgcode" : {
                "type" : "string",
                "store" : true
              },
              "ccgname" : {
                "type" : "string",
                "store" : true
              },
              "class_desc" : {
                "type" : "string",
                "store" : true
              },
              "classification" : {
                "type" : "string",
                "store" : true
              },
              "easting" : {
                "type" : "float",
                "store" : true
              },
              "end_date" : {
                "type" : "date",
                "store" : true,
                "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
              },
              "lat" : {
                "type" : "integer",
                "store" : true
              },
              "lat_long" : {
                "type" : "string",
                "store" : true
              },
              "local_admin_area_code" : {
                "type" : "string",
                "store" : true
              },
              "local_admin_area_name" : {
                "type" : "string",
                "store" : true
              },
              "locality_name" : {
                "type" : "string",
                "store" : true
              },
              "logical_status" : {
                "type" : "integer",
                "store" : true
              },
              "logical_status_desc" : {
                "type" : "string",
                "store" : true
              },
              "lon" : {
                "type" : "integer",
                "store" : true
              },
              "multi_occ_count" : {
                "type" : "integer",
                "store" : true
              },
              "nhs_area_team" : {
                "type" : "string",
                "store" : true
              },
              "nhs_area_team_code" : {
                "type" : "string",
                "store" : true
              },
              "nhs_region" : {
                "type" : "string",
                "store" : true
              },
              "nhs_region_code" : {
                "type" : "string",
                "store" : true
              },
              "northing" : {
                "type" : "float",
                "store" : true
              },
              "parish_code" : {
                "type" : "string",
                "store" : true
              },
              "parish_name" : {
                "type" : "string",
                "store" : true
              },
              "postally_addressable" : {
                "type" : "string",
                "store" : true
              },
              "postcode_locator" : {
                "type" : "string",
                "store" : true
              },
              "start_date" : {
                "type" : "date",
                "store" : true,
                "format" : "yyyy/MM/dd HH:mm:ss||yyyy/MM/dd"
              },
              "street_description" : {
                "type" : "string",
                "store" : true
              },
              "udprn" : {
                "type" : "integer",
                "store" : true
              },
              "uprn" : {
                "type" : "integer",
                "store" : true
              },
              "ward_code" : {
                "type" : "string",
                "store" : true
              },
              "ward_name" : {
                "type" : "string",
                "store" : true
              }
            }
          },
          "type" : {
            "type" : "string",
            "store" : true
          }
        }
      }
    }
  }
}

AdrienVH commented 9 years ago

Hi !

Oh i see it now... Your mappings are very interesting cause they show me a different construction. I thought that mappings were always "flat-structured", like this :

{
    "index1" : {
        "properties" : {
            "field1" : {
                "type" : "string"
            },
            "field2" : {
                "type" : "integer"
            },
            "field2" : {
                "type" : "geo_shape"
            }
        }
    },
    "index2" : {
        "properties" : {
            "field1" : {
                "type" : "float"
            },
            "field2" : {
                "type" : "string"
            },
            "field2" : {
                "type" : "geo_point"
            }
        }
    }
}

But, i saw in your mappings something "deep-structured", like that :

{
    "index1" : {
        "properties" : {
            "field1" : {
                "field2" : {
                    "type" : "string"
                },
                "field3" : {
                    "type" : "geo_point"
                }
            },
            "field4" : {
                "type" : "integer"
            }
        }
    }
}

I think that i need to update my plugin in order to take account of those "deep-structured" mappings !

If the field definition has no type, it's because there are one or several fields inside this field. But i need to go recursive cause it's possible that we found a very complex tree of fields in fields in field, etc. I need to work on this part of my plugin...

Adrien

dmac707 commented 9 years ago

Best of luck. I look forward to testing the next version.

pedrosk commented 6 years ago

Adrien,

I upgraded to the new QGIS just to use your plugin. I run into this problem. Do you think it would be possible to have a version where I can define what I care for?

Like this: /apc_latlong5/temp1/_search?pretty=true' ^^^ Is what I use when I run a query from CLI. Thanks or if you have a suggestion how to make the plugin work I would be more than happy

{
  "mapkib" : {
    "mappings" : {
      "salas" : {
        "properties" : {
          "UTC_time" : {
            "type" : "date"
          },
          "available_mem" : {
            "type" : "half_float"
          },
          "date_now" : {
            "type" : "date"
          },
          "kibip" : {
            "type" : "ip"
          },
          "local_time" : {
            "type" : "short"
          },
          "memoryMB" : {
            "type" : "half_float"
          },
          "memorykb" : {
            "type" : "half_float"
          }
        }
      },
      "kibCL" : {
        "properties" : {
          "UTC_time" : {
            "type" : "date"
          },
          "available_mem" : {
            "type" : "half_float"
          },
          "date_now" : {
            "type" : "date"
          },
          "local_time" : {
            "type" : "short"
          },
          "memoryMB" : {
            "type" : "half_float"
          },
          "memorykb" : {
            "type" : "half_float"
          }
        }
      }
    }
  },
  "mapes" : {
    "mappings" : {
      "salas" : {
        "properties" : {
          "UTC_time" : {
            "type" : "date"
          },
          "available_mem" : {
            "type" : "half_float"
          },
          "date_now" : {
            "type" : "date"
          },
          "esid" : {
            "type" : "keyword"
          },
          "esip" : {
            "type" : "ip"
          },
          "kibip" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "local_time" : {
            "type" : "short"
          },
          "memoryESMB" : {
            "type" : "half_float"
          },
          "memoryESkb" : {
            "type" : "half_float"
          }
        }
      }
    }
  },
  "apc" : {
    "mappings" : {
      "temp1" : {
        "properties" : {
          "IP" : {
            "type" : "ip"
          },
          "UTC_time" : {
            "type" : "date"
          },
          "date_now" : {
            "type" : "date"
          },
          "humidExt" : {
            "type" : "half_float"
          },
          "humidExt2" : {
            "type" : "half_float"
          },
          "humidInt" : {
            "type" : "half_float"
          },
          "local_time" : {
            "type" : "short"
          },
          "tempExt1" : {
            "type" : "half_float"
          },
          "tempExt2" : {
            "type" : "half_float"
          },
          "tempInt" : {
            "type" : "half_float"
          }
        }
      }
    }
  },
  "gis" : {
    "mappings" : {
      "CL" : {
        "properties" : {
          "GISip" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          },
          "UTC_time" : {
            "type" : "date"
          },
          "available_mem" : {
            "type" : "half_float"
          },
          "date_now" : {
            "type" : "date"
          },
          "kibip" : {
            "type" : "ip"
          },
          "local_time" : {
            "type" : "short"
          },
          "memoryGISMB" : {
            "type" : "half_float"
          },
          "memoryGISkb" : {
            "type" : "half_float"
          }
        }
      }
    }
  },
  ".kibana" : {
    "mappings" : {
      "timelion-sheet" : {
        "dynamic" : "strict",
        "properties" : {
          "description" : {
            "type" : "text"
          },
          "hits" : {
            "type" : "integer"
          },
          "kibanaSavedObjectMeta" : {
            "properties" : {
              "searchSourceJSON" : {
                "type" : "text"
              }
            }
          },
          "timelion_chart_height" : {
            "type" : "integer"
          },
          "timelion_columns" : {
            "type" : "integer"
          },
          "timelion_interval" : {
            "type" : "keyword"
          },
          "timelion_other_interval" : {
            "type" : "keyword"
          },
          "timelion_rows" : {
            "type" : "integer"
          },
          "timelion_sheet" : {
            "type" : "text"
          },
          "title" : {
            "type" : "text"
          },
          "version" : {
            "type" : "integer"
          }
        }
      },
      "visualization" : {
        "dynamic" : "strict",
        "properties" : {
          "description" : {
            "type" : "text"
          },
          "kibanaSavedObjectMeta" : {
            "properties" : {
              "searchSourceJSON" : {
                "type" : "text"
              }
            }
          },
          "savedSearchId" : {
            "type" : "keyword"
          },
          "title" : {
            "type" : "text"
          },
          "uiStateJSON" : {
            "type" : "text"
          },
          "version" : {
            "type" : "integer"
          },
          "visState" : {
            "type" : "text"
          }
        }
      },
      "url" : {
        "dynamic" : "strict",
        "properties" : {
          "accessCount" : {
            "type" : "long"
          },
          "accessDate" : {
            "type" : "date"
          },
          "createDate" : {
            "type" : "date"
          },
          "url" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 2048
              }
            }
          }
        }
      },
      "search" : {
        "dynamic" : "strict",
        "properties" : {
          "columns" : {
            "type" : "keyword"
          },
          "description" : {
            "type" : "text"
          },
          "hits" : {
            "type" : "integer"
          },
          "kibanaSavedObjectMeta" : {
            "properties" : {
              "searchSourceJSON" : {
                "type" : "text"
              }
            }
          },
          "sort" : {
            "type" : "keyword"
          },
          "title" : {
            "type" : "text"
          },
          "version" : {
            "type" : "integer"
          }
        }
      },
      "index-pattern" : {
        "dynamic" : "strict",
        "properties" : {
          "fieldFormatMap" : {
            "type" : "text"
          },
          "fields" : {
            "type" : "text"
          },
          "intervalName" : {
            "type" : "keyword"
          },
          "notExpandable" : {
            "type" : "boolean"
          },
          "sourceFilters" : {
            "type" : "text"
          },
          "timeFieldName" : {
            "type" : "keyword"
          },
          "title" : {
            "type" : "text"
          }
        }
      },
      "server" : {
        "dynamic" : "strict",
        "properties" : {
          "uuid" : {
            "type" : "keyword"
          }
        }
      },
      "_default_" : {
        "dynamic" : "strict"
      },
      "config" : {
        "dynamic" : "true",
        "properties" : {
          "buildNum" : {
            "type" : "keyword"
          },
          "defaultIndex" : {
            "type" : "text",
            "fields" : {
              "keyword" : {
                "type" : "keyword",
                "ignore_above" : 256
              }
            }
          }
        }
      },
      "dashboard" : {
        "dynamic" : "strict",
        "properties" : {
          "description" : {
            "type" : "text"
          },
          "hits" : {
            "type" : "integer"
          },
          "kibanaSavedObjectMeta" : {
            "properties" : {
              "searchSourceJSON" : {
                "type" : "text"
              }
            }
          },
          "optionsJSON" : {
            "type" : "text"
          },
          "panelsJSON" : {
            "type" : "text"
          },
          "refreshInterval" : {
            "properties" : {
              "display" : {
                "type" : "keyword"
              },
              "pause" : {
                "type" : "boolean"
              },
              "section" : {
                "type" : "integer"
              },
              "value" : {
                "type" : "integer"
              }
            }
          },
          "timeFrom" : {
            "type" : "keyword"
          },
          "timeRestore" : {
            "type" : "boolean"
          },
          "timeTo" : {
            "type" : "keyword"
          },
          "title" : {
            "type" : "text"
          },
          "uiStateJSON" : {
            "type" : "text"
          },
          "version" : {
            "type" : "integer"
          }
        }
      }
    }
  },
  "apc_latlong5" : {
    "mappings" : {
      "temp1" : {
        "properties" : {
          "IP" : {
            "type" : "ip"
          },
          "UTC_time" : {
            "type" : "date"
          },
          "date_now" : {
            "type" : "date"
          },
          "humidExt" : {
            "type" : "half_float"
          },
          "local_time" : {
            "type" : "short"
          },
          "location" : {
            "type" : "geo_point",
            "ignore_malformed" : true
          },
          "tempExt1" : {
            "type" : "half_float"
          }
        }
      }
    }
  }
}

 I run ElasticSearch 5.6 
KeyError: 'properties' 
Traceback (most recent call last):
  File "/home/pedro/.qgis2/python/plugins/ElasticSearchConnector/elasticsearch_connector.py", line 109, in onConnectClick
    mapping = mappings[type]["properties"]
KeyError: 'properties'

Python version: 2.7.6 (default, Nov 23 2017, 15:53:45) [GCC 4.8.4] 
QGIS version: 2.14.3-Essen Essen, exported 

Python Path:
/usr/share/qgis/python/plugins/processing
/usr/share/qgis/python
/home/pedro/.qgis2/python
/home/pedro/.qgis2/python/plugins
/usr/share/qgis/python/plugins
/usr/lib/python2.7
/usr/lib/python2.7/plat-x86_64-linux-gnu
/usr/lib/python2.7/lib-tk
/usr/lib/python2.7/lib-old
/usr/lib/python2.7/lib-dynload
/usr/local/lib/python2.7/dist-packages
/usr/lib/python2.7/dist-packages
/usr/lib/python2.7/dist-packages/PILcompat
/usr/lib/python2.7/dist-packages/gst-0.10
/usr/lib/python2.7/dist-packages/gtk-2.0
/usr/lib/pymodules/python2.7
/home/pedro/.qgis2//python
/usr/share/qgis/python/plugins/fTools/tools
/home/pedro/GIS/CL/qgis

AdrienVH / ElasticSearchConnector

Connection error #3