search

pelias / docker

Run the Pelias geocoder in docker containers, including example projects.
MIT License
338 stars 226 forks source link

Interpolation error #348

Open MiesSuomesta opened 5 months ago

MiesSuomesta commented 5 months ago

Hi All!

Describe the bug I'm trying to get Pelias working, but now it fails to locate SQL database, and error does not include what & where.

Steps to Reproduce docker compose up in pelias project directory.

Expected behavior Similar outcome of pelias comparison tool

Environment:

Pastebin/Screenshots kuva

Additional context Pelias config as follows:

{
  "esclient": {
    "apiVersion": "7.x",
    "keepAlive": true,
    "requestTimeout": "120000",
    "hosts": [{
      "env": "development",
      "protocol": "http",
      "host": "localhost",
      "port": 9200
    }],
    "log": [{
      "type": "stdio",
      "json": false,
      "level": [ "error", "warning", "info", "debug" ]
    }]
  },
  "elasticsearch": {
    "settings": {
      "index": {
        "number_of_replicas": "0",
        "number_of_shards": "5",
        "refresh_interval": "1m"
      }
    }
  },
  "interpolation": {
    "client": {
      "adapter": "null"
    }
  },
  "dbclient": {
    "statFrequency": 10000,
    "batchSize": 500
  },
  "api": {
    "services": {
      "placeholder":   { "url": "http://placeholder:4100" },
      "pip":           { "url": "http://pip:4200" },
      "interpolation": { "url": "http://interpolation:4300" },
      "libpostal":     { "url": "http://libpostal:4400" }
    },
    "accessLog": "common",
    "host": "http://pelias.mapzen.com/",
    "indexName": "pelias",
    "version": "1.0",
    "targets": {
      "auto_discover": true,
      "canonical_sources": ["whosonfirst", "openstreetmap", "openaddresses", "geonames"],
      "layers_by_source": {
        "openstreetmap": [ "address", "venue", "street" ],
        "openaddresses": [ "address" ],
        "geonames": [
          "country", "macroregion", "region", "county", "localadmin", "locality", "borough",
          "neighbourhood", "venue"
        ],
        "whosonfirst": [
          "continent", "empire", "country", "dependency", "macroregion", "region", "locality",
          "localadmin", "macrocounty", "county", "macrohood", "borough", "neighbourhood",
          "microhood", "disputed", "venue", "postalcode", "ocean", "marinearea"
        ]
      },
      "source_aliases": {
        "osm": [ "openstreetmap" ],
        "oa":  [ "openaddresses" ],
        "gn":  [ "geonames" ],
        "wof": [ "whosonfirst" ]
      },
      "layer_aliases": {
        "coarse": [
          "continent", "empire", "country", "dependency", "macroregion", "region", "locality",
          "localadmin", "macrocounty", "county", "macrohood", "borough", "neighbourhood",
          "microhood", "disputed", "postalcode", "ocean", "marinearea"
        ]
      }
    }
  },
  "schema": {
    "indexName": "pelias"
  },
  "logger": {
    "level": "info",
    "timestamp": true,
    "colorize": true
  },
  "acceptance-tests": {
    "endpoints": {
      "local": "http://localhost:3100/v1/"
    }
  },
  "imports": {
    "adminLookup": {
      "enabled": true,
      "maxConcurrentRequests": 100,
      "usePostalCities": true
    },
    "blacklist": {
      "files": []
    },
    "csv": {
    },
    "geonames": {
      "datapath": "/data/geonames",
      "countryCode": "FI"
    },
    "openstreetmap": {
      "datapath": "/data/openstreetmap",
      "leveldbpath": "/data",
      "import": [{
        "filename": "finland-latest.osm.pbf"
      }]
    },
    "openaddresses": {
      "datapath": "/data/openaddresses",
      "token": "oa.bbbcf5787bb4251445883cc417f811ba02b9fd64809fd56c5a972171fbcfb2f6",
      "files": [ "fi/countrywide-fi.csv", "fi/countrywide-sv.csv" ]
    },
    "polyline": {
      "datapath": "/data/polyline",
      "files": []
    },
    "whosonfirst": {
      "datapath": "/data/whosonfirst",
      "importPostalcodes": true
    }
  }
}

Elasticsearch conf:

network.host: 0.0.0.0
indices.breaker.fielddata.limit: 85%
indices.fielddata.cache.size: 75%

# My additions:
xpack.ml.enabled: false
bootstrap.memory_lock: false

docker-compose.yml

version: '3'
networks:
  default:
    driver: bridge
services:
  libpostal:
    image: pelias/libpostal-service
    container_name: pelias_libpostal
    restart: always
    ports: [ "4400:4400" ]
  schema:
    image: pelias/schema:master
    container_name: pelias_schema
    volumes:
      - "./pelias.json:/code/pelias.json"
  api:
    image: pelias/api:master
    container_name: pelias_api
    restart: always
    environment:
      - "PORT=4000"
      - http.cors.allow-origin=*
    ports: [ "4000:4000" ]
    volumes:
      - "./pelias.json:/code/pelias.json"
  placeholder:
    image: pelias/placeholder:master
    container_name: pelias_placeholder
    restart: always
    environment:
      - PORT=4100
      - PLACEHOLDER_DATA=/data/placeholder
    ports: [ "4100:4100" ]
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
      - "./blacklist/:/data/blacklist"
  whosonfirst:
    image: pelias/whosonfirst:master
    container_name: pelias_whosonfirst
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  openstreetmap:
    image: pelias/openstreetmap:master
    container_name: pelias_openstreetmap
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  openaddresses:
    image: pelias/openaddresses:master
    container_name: pelias_openaddresses
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  csv-importer:
    image: pelias/csv-importer:master
    container_name: pelias_csv_importer
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
      - "./blacklist/:/data/blacklist"
  polylines:
    image: pelias/polylines:master
    container_name: pelias_polylines
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  interpolation:
    image: pelias/interpolation:master
    container_name: pelias_interpolation
    restart: always
    environment: [ "PORT=4300" ]
    ports: [ "4300:4300" ]
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  pip:
    image: pelias/pip-service:master
    container_name: pelias_pip-service
    restart: always
    environment: [ "PORT=4200" ]
    ports: [ "4200:4200" ]
    volumes:
      - "./pelias.json:/code/pelias.json"
      - "${DATA_DIR}:/data"
  elasticsearch:
    image: pelias/elasticsearch:7.16.1
    container_name: pelias_elasticsearch
    restart: always
    ports: [ "9200:9200", "9300:9300" ]
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - xpack.ml.enabled=false
      - http.cors.allow-origin=*
      - ES_HEAP_SIZE=20g
      - MAX_LOCKED_MEMORY=unlimited
    mem_limit: 20g
    volumes:
      - ./elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
      - "${DATA_DIR}:/usr/share/elasticsearch/data"
      - ./pelias.json:/code/pelias.json
    ulimits:
      memlock:
        soft: -1
        hard: -1
      nofile:
        soft: 65536
        hard: 65536
    cap_add: [ "IPC_LOCK" ]
    security_opt:
      - seccomp=unconfined

Thanks in advance!

EDIT 1: Updated the issue with current configs I use for pelias.

MiesSuomesta commented 5 months ago

kuva

How to fix the issue? It cant be very big problem cause the response is not refusal or so ...

MiesSuomesta commented 5 months ago

I think I found reason .. I've not updated the pelias data with the pelias download util and then importting it .. I'm just now trying that ..

MiesSuomesta commented 5 months ago

My directory structure is as follows:

pelias-docker 
├── cmd
│   ├── docker.sh
│   ├── download.sh
│   ├── elastic.sh
│   ├── import.sh
│   ├── prepare.sh
│   ├── system.sh
│   └── test.sh
├── common
│   └── preview
│       └── index.html
├── images
│   ├── elasticsearch
│   │   ├── 2.4
│   │   │   ├── Dockerfile
│   │   │   └── elasticsearch.yml
│   │   ├── 5.6.12
│   │   │   ├── Dockerfile
│   │   │   └── elasticsearch.yml
│   │   ├── 6.8.5
│   │   │   ├── Dockerfile
│   │   │   └── elasticsearch.yml
│   │   ├── 7.17.15
│   │   │   ├── Dockerfile
│   │   │   └── elasticsearch.yml
│   │   ├── 7.5.1
│   │   │   ├── Dockerfile
│   │   │   └── elasticsearch.yml
│   │   └── 8.12.2-beta
│   │       ├── Dockerfile
│   │       └── elasticsearch.yml
│   └── valhalla
│       └── Dockerfile
├── lib
│   ├── cli.sh
│   └── env.sh
├── LICENSE
├── pelias
├── projects
│   └── finland
│       ├── blacklist
│       ├── data
│       │   ├── blacklist
│       │   ├── csv-importer
│       │   ├── elasticsearch
│       │   ├── interpolation
│       │   │   ├── address.db
│       │   │   ├── address.db.gz
│       │   │   ├── build.meta
│       │   │   ├── builds
│       │   │   │   └── current -> /data/interpolation
│       │   │   ├── conflate_oa.err
│       │   │   ├── conflate_oa.out
│       │   │   ├── conflate_oa.skip
│       │   │   ├── conflate_osm.err
│       │   │   ├── conflate_osm.out
│       │   │   ├── finland-latest.osm.pbf
│       │   │   ├── polyline.err
│       │   │   ├── polyline.out
│       │   │   ├── sqlite
│       │   │   │   └── store.sqlite3
│       │   │   ├── street.db
│       │   │   ├── street.db.gz
│       │   │   ├── vertices.err
│       │   │   ├── vertices.out
│       │   │   └── vertices.skip
│       │   ├── nodes
│       │   │   └── 0
│       │   │       ├── indices
│       │   │       │   ├── 7XpCDJBERlCuvX8tLcatTA
│       │   │       │   │   ├── 0
│       │   │       │   │   │   ├── index
│       │   │       │   │   │   │   ├── _0.cfe
│       │   │       │   │   │   │   ├── _0.cfs
│       │   │       │   │   │   │   ├── _0.si
│       │   │       │   │   │   │   ├── _1.cfe
│       │   │       │   │   │   │   ├── _1.cfs
│       │   │       │   │   │   │   ├── _1.si
│       │   │       │   │   │   │   ├── segments_3
│       │   │       │   │   │   │   └── write.lock
│       │   │       │   │   │   ├── _state
│       │   │       │   │   │   │   ├── retention-leases-50.st
│       │   │       │   │   │   │   └── state-48.st
│       │   │       │   │   │   └── translog
│       │   │       │   │   │       ├── translog-51.tlog
│       │   │       │   │   │       └── translog.ckp
│       │   │       │   │   └── _state
│       │   │       │   │       └── state-101.st
│       │   │       │   ├── MaK3962lQCKsmaMtKcDHTQ
│       │   │       │   │   ├── 0
│       │   │       │   │   │   ├── index
│       │   │       │   │   │   │   ├── _6.cfe
│       │   │       │   │   │   │   ├── _6.cfs
│       │   │       │   │   │   │   ├── _6.si
│       │   │       │   │   │   │   ├── _7.cfe
│       │   │       │   │   │   │   ├── _7.cfs
│       │   │       │   │   │   │   ├── _7.si
│       │   │       │   │   │   │   ├── _9.cfe
│       │   │       │   │   │   │   ├── _9.cfs
│       │   │       │   │   │   │   ├── _9.si
│       │   │       │   │   │   │   ├── _a.cfe
│       │   │       │   │   │   │   ├── _a.cfs
│       │   │       │   │   │   │   ├── _a.si
│       │   │       │   │   │   │   ├── _b.cfe
│       │   │       │   │   │   │   ├── _b.cfs
│       │   │       │   │   │   │   ├── _b.si
│       │   │       │   │   │   │   ├── _c.cfe
│       │   │       │   │   │   │   ├── _c.cfs
│       │   │       │   │   │   │   ├── _c.si
│       │   │       │   │   │   │   ├── _d.cfe
│       │   │       │   │   │   │   ├── _d.cfs
│       │   │       │   │   │   │   ├── _d.si
│       │   │       │   │   │   │   ├── segments_a
│       │   │       │   │   │   │   └── write.lock
│       │   │       │   │   │   ├── _state
│       │   │       │   │   │   │   ├── retention-leases-50.st
│       │   │       │   │   │   │   └── state-48.st
│       │   │       │   │   │   └── translog
│       │   │       │   │   │       ├── translog-57.tlog
│       │   │       │   │   │       └── translog.ckp
│       │   │       │   │   └── _state
│       │   │       │   │       └── state-97.st
│       │   │       │   └── MwoiuYgJR_O7mLZ4H1GbVA
│       │   │       │       ├── 0
│       │   │       │       │   ├── index
│       │   │       │       │   │   ├── _0.cfe
│       │   │       │       │   │   ├── _0.cfs
│       │   │       │       │   │   ├── _0.si
│       │   │       │       │   │   ├── _1.cfe
│       │   │       │       │   │   ├── _1.cfs
│       │   │       │       │   │   ├── _1.si
│       │   │       │       │   │   ├── _2.cfe
│       │   │       │       │   │   ├── _2.cfs
│       │   │       │       │   │   ├── _2.si
│       │   │       │       │   │   ├── _3.cfe
│       │   │       │       │   │   ├── _3.cfs
│       │   │       │       │   │   ├── _3.si
│       │   │       │       │   │   ├── _4.cfe
│       │   │       │       │   │   ├── _4.cfs
│       │   │       │       │   │   ├── _4.si
│       │   │       │       │   │   ├── _6.cfe
│       │   │       │       │   │   ├── _6.cfs
│       │   │       │       │   │   ├── _6.si
│       │   │       │       │   │   ├── segments_8
│       │   │       │       │   │   └── write.lock
│       │   │       │       │   ├── _state
│       │   │       │       │   │   ├── retention-leases-50.st
│       │   │       │       │   │   └── state-48.st
│       │   │       │       │   └── translog
│       │   │       │       │       ├── translog-55.tlog
│       │   │       │       │       └── translog.ckp
│       │   │       │       └── _state
│       │   │       │           └── state-102.st
│       │   │       ├── node.lock
│       │   │       ├── snapshot_cache
│       │   │       │   ├── segments_1d
│       │   │       │   └── write.lock
│       │   │       └── _state
│       │   │           ├── _cu.cfe
│       │   │           ├── _cu.cfs
│       │   │           ├── _cu.si
│       │   │           ├── _cv.cfe
│       │   │           ├── _cv.cfs
│       │   │           ├── _cv.si
│       │   │           ├── _cy.cfe
│       │   │           ├── _cy.cfs
│       │   │           ├── _cy.si
│       │   │           ├── manifest-48.st
│       │   │           ├── node-48.st
│       │   │           ├── segments_fv
│       │   │           └── write.lock
│       │   ├── openaddresses
│       │   │   ├── fi
│       │   │   │   ├── countrywide-fi.csv
│       │   │   │   ├── countrywide-fi.vrt
│       │   │   │   ├── countrywide-sv.csv
│       │   │   │   └── countrywide-sv.vrt
│       │   │   └── README.txt
│       │   ├── openstreetmap
│       │   │   ├── finland-latest.osm.pbf
│       │   │   └── planet-latest.osm.pbf
│       │   ├── pip
│       │   │   └── sqlite
│       │   │       └── store.sqlite3
│       │   ├── placeholder
│       │   │   └── store.sqlite3
│       │   ├── polylines
│       │   │   └── finland.0sv
│       │   ├── tiger
│       │   │   └── downloads
│       │   └── whosonfirst
│       │       └── sqlite
│       │           ├── store.sqlite3
│       │           ├── whosonfirst-data-admin-latest.db
│       │           └── whosonfirst-data-postalcode-latest.db
│       ├── docker-compose.yml
│       ├── elasticsearch.yml
│       ├── package.json
│       ├── package-lock.json
│       ├── pelias.json
│       └── README.md
└── README.md

Hope it helps sone one else having issues :D ...

arnesetzer commented 2 months ago

What does pelias elastic status and docker compose ps -a say? "No living connection" indicates that the elastic search db is not running (properly).