search

ozlerhakan / mongolastic

:traffic_light: A dataset migration tool from MongoDB to Elasticsearch and vice versa.
MIT License
137 stars 34 forks source link

Error connecting (?) to ES #14

Closed rodrigoalviani closed 8 years ago

rodrigoalviani commented 8 years ago

My config file:

misc:
        dindex:
                name: dbCars
                as: dbcars
        ctype:
                name: Cars
                as: cars
        direction: me
mongo:
        host: 10.11.28.7
        port: 27017
elastic:
        host: 192.168.2.32
        port: 9200
        longToString: true

Error:

0 [main] INFO com.kodcu.config.FileConfiguration  -
Config Output:
{elastic=Elastic{host='192.168.2.32', port=9200, dateFormat=null, longToString=true}, misc=Misc{batch=500, direction='me', dindex=Namespace{as='dbcars', name='dbCars'}, ctype=Namespace{as='cars', name='Cars'}, dropDataset=true}, mongo=Mongo{host='10.11.28.7', port=27017, query='{}', auth=null}}

224 [main] INFO org.elasticsearch.plugins  - [Mad Dog Rassitano] modules [], plugins [], sites []
247 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [force_merge], type [fixed], size [1], queue_size [null]
257 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [percolate], type [fixed], size [4], queue_size [1k]
273 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [fetch_shard_started], type [scaling], min [1], size [8], keep_alive [5m]
274 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [listener], type [fixed], size [2], queue_size [null]
275 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [index], type [fixed], size [4], queue_size [200]
275 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [refresh], type [scaling], min [1], size [2], keep_alive [5m]
275 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [suggest], type [fixed], size [4], queue_size [1k]
276 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [generic], type [cached], keep_alive [30s]
278 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [warmer], type [scaling], min [1], size [2], keep_alive [5m]
279 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [search], type [fixed], size [7], queue_size [1k]
279 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [flush], type [scaling], min [1], size [2], keep_alive [5m]
280 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [fetch_shard_store], type [scaling], min [1], size [8], keep_alive [5m]
280 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [management], type [scaling], min [1], size [5], keep_alive [5m]
281 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [get], type [fixed], size [4], queue_size [1k]
281 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [bulk], type [fixed], size [4], queue_size [50]
282 [main] DEBUG org.elasticsearch.threadpool  - [Mad Dog Rassitano] creating thread_pool [snapshot], type [scaling], min [1], size [2], keep_alive [5m]
718 [main] DEBUG org.elasticsearch.common.network  - configuration:

lo0
        inet 127.0.0.1 netmask:255.0.0.0 scope:host
        inet6 fe80::1 prefixlen:64 scope:link
        inet6 ::1 prefixlen:128 scope:host
        UP MULTICAST LOOPBACK mtu:16384 index:1

en1
        inet 192.168.2.32 netmask:255.255.255.0 broadcast:192.168.2.255 scope:site
        inet6 fe80::aa86:ddff:feb0:d36d prefixlen:64 scope:link
        hardware A8:86:DD:B0:D3:6D
        UP MULTICAST mtu:1500 index:5

awdl0
        inet6 fe80::84e9:fdff:fee1:4c12 prefixlen:64 scope:link
        hardware 86:E9:FD:E1:4C:12
        UP MULTICAST mtu:1484 index:9

748 [main] DEBUG org.elasticsearch.common.netty  - using gathering [true]
798 [main] DEBUG org.elasticsearch.client.transport  - [Mad Dog Rassitano] node_sampler_interval[5s]
818 [main] DEBUG org.elasticsearch.netty.channel.socket.nio.SelectorUtil  - Using select timeout of 500
818 [main] DEBUG org.elasticsearch.netty.channel.socket.nio.SelectorUtil  - Epoll-bug workaround enabled = false
850 [main] DEBUG org.elasticsearch.client.transport  - [Mad Dog Rassitano] adding address [{#transport#-1}{192.168.2.32}{192.168.2.32:9200}]
896 [main] DEBUG org.elasticsearch.transport.netty  - [Mad Dog Rassitano] connected to node [{#transport#-1}{192.168.2.32}{192.168.2.32:9200}]
5935 [main] INFO org.elasticsearch.client.transport  - [Mad Dog Rassitano] failed to get node info for {#transport#-1}{192.168.2.32}{192.168.2.32:9200}, disconnecting...
ReceiveTimeoutTransportException[[][192.168.2.32:9200][cluster:monitor/nodes/liveness] request_id [0] timed out after [5004ms]]
    at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:679)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
5937 [main] DEBUG org.elasticsearch.transport.netty  - [Mad Dog Rassitano] disconnecting from [{#transport#-1}{192.168.2.32}{192.168.2.32:9200}] due to explicit disconnect call
5941 [elasticsearch[Mad Dog Rassitano][generic][T#1]] DEBUG org.elasticsearch.transport.netty  - [Mad Dog Rassitano] connected to node [{#transport#-1}{192.168.2.32}{192.168.2.32:9200}]
5997 [main] INFO org.mongodb.driver.cluster  - Cluster created with settings {hosts=[10.11.28.7:27017], mode=MULTIPLE, requiredClusterType=UNKNOWN, serverSelectionTimeout='5000 ms', maxWaitQueueSize=500}
5998 [main] INFO org.mongodb.driver.cluster  - Adding discovered server 10.11.28.7:27017 to client view of cluster
6068 [main] DEBUG org.mongodb.driver.cluster  - Updating cluster description to  {type=UNKNOWN, servers=[{address=10.11.28.7:27017, type=UNKNOWN, state=CONNECTING}]
6118 [main] INFO org.mongodb.driver.cluster  - No server chosen by ReadPreferenceServerSelector{readPreference=primaryPreferred} from cluster description ClusterDescription{type=UNKNOWN, connectionMode=MULTIPLE, all=[ServerDescription{address=10.11.28.7:27017, type=UNKNOWN, state=CONNECTING}]}. Waiting for 5000 ms before timing out
6730 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.connection  - Opened connection [connectionId{localValue:1, serverValue:802}] to 10.11.28.7:27017
6730 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] DEBUG org.mongodb.driver.cluster  - Checking status of 10.11.28.7:27017
6889 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Monitor thread successfully connected to server with description ServerDescription{address=10.11.28.7:27017, type=REPLICA_SET_PRIMARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[2, 6, 5]}, minWireVersion=0, maxWireVersion=2, maxDocumentSize=16777216, roundTripTimeNanos=157645062, setName='rs0', canonicalAddress=mongohml:27017, hosts=[mongohml:27018, mongohml:27017, mongohml:27019], passives=[], arbiters=[], primary='mongohml:27017', tagSet=TagSet{[]}, electionId=null, setVersion=3}
6890 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Discovered cluster type of REPLICA_SET
6891 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Adding discovered server mongohml:27018 to client view of cluster
6893 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Adding discovered server mongohml:27017 to client view of cluster
6893 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Adding discovered server mongohml:27019 to client view of cluster
6894 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Server 10.11.28.7:27017 is no longer a member of the replica set.  Removing from client view of cluster.
6896 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] INFO org.mongodb.driver.cluster  - Canonical address mongohml:27017 does not match server address.  Removing 10.11.28.7:27017 from client view of cluster
6896 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] DEBUG org.mongodb.driver.cluster  - Updating cluster description to  {type=REPLICA_SET, servers=[{address=mongohml:27017, type=UNKNOWN, state=CONNECTING}, {address=mongohml:27018, type=UNKNOWN, state=CONNECTING}, {address=mongohml:27019, type=UNKNOWN, state=CONNECTING}]
6897 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-10.11.28.7:27017] DEBUG org.mongodb.driver.connection  - Closing connection connectionId{localValue:1, serverValue:802}
8610 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27019] INFO org.mongodb.driver.connection  - Opened connection [connectionId{localValue:4, serverValue:810}] to mongohml:27019
8610 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27018] INFO org.mongodb.driver.connection  - Opened connection [connectionId{localValue:2, serverValue:803}] to mongohml:27018
8610 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] INFO org.mongodb.driver.connection  - Opened connection [connectionId{localValue:3, serverValue:803}] to mongohml:27017
8611 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27018] DEBUG org.mongodb.driver.cluster  - Checking status of mongohml:27018
8611 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27019] DEBUG org.mongodb.driver.cluster  - Checking status of mongohml:27019
8612 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] DEBUG org.mongodb.driver.cluster  - Checking status of mongohml:27017
8770 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27019] INFO org.mongodb.driver.cluster  - Monitor thread successfully connected to server with description ServerDescription{address=mongohml:27019, type=REPLICA_SET_SECONDARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[2, 6, 5]}, minWireVersion=0, maxWireVersion=2, maxDocumentSize=16777216, roundTripTimeNanos=157484839, setName='rs0', canonicalAddress=mongohml:27019, hosts=[mongohml:27018, mongohml:27017, mongohml:27019], passives=[], arbiters=[], primary='mongohml:27017', tagSet=TagSet{[]}, electionId=null, setVersion=3}
8770 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] INFO org.mongodb.driver.cluster  - Monitor thread successfully connected to server with description ServerDescription{address=mongohml:27017, type=REPLICA_SET_PRIMARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[2, 6, 5]}, minWireVersion=0, maxWireVersion=2, maxDocumentSize=16777216, roundTripTimeNanos=157568194, setName='rs0', canonicalAddress=mongohml:27017, hosts=[mongohml:27018, mongohml:27017, mongohml:27019], passives=[], arbiters=[], primary='mongohml:27017', tagSet=TagSet{[]}, electionId=null, setVersion=3}
8770 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27018] INFO org.mongodb.driver.cluster  - Monitor thread successfully connected to server with description ServerDescription{address=mongohml:27018, type=REPLICA_SET_SECONDARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[2, 6, 5]}, minWireVersion=0, maxWireVersion=2, maxDocumentSize=16777216, roundTripTimeNanos=157648746, setName='rs0', canonicalAddress=mongohml:27018, hosts=[mongohml:27018, mongohml:27017, mongohml:27019], passives=[], arbiters=[], primary='mongohml:27017', tagSet=TagSet{[]}, electionId=null, setVersion=3}
8772 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27019] DEBUG org.mongodb.driver.cluster  - Updating cluster description to  {type=REPLICA_SET, servers=[{address=mongohml:27017, type=UNKNOWN, state=CONNECTING}, {address=mongohml:27018, type=UNKNOWN, state=CONNECTING}, {address=mongohml:27019, type=REPLICA_SET_SECONDARY, roundTripTime=157,5 ms, state=CONNECTED}]
8773 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27018] DEBUG org.mongodb.driver.cluster  - Updating cluster description to  {type=REPLICA_SET, servers=[{address=mongohml:27017, type=UNKNOWN, state=CONNECTING}, {address=mongohml:27018, type=REPLICA_SET_SECONDARY, roundTripTime=157,6 ms, state=CONNECTED}, {address=mongohml:27019, type=REPLICA_SET_SECONDARY, roundTripTime=157,5 ms, state=CONNECTED}]
8775 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] INFO org.mongodb.driver.cluster  - Setting max set version to 3 from replica set primary mongohml:27017
8775 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] INFO org.mongodb.driver.cluster  - Discovered replica set primary mongohml:27017
8776 [cluster-ClusterId{value='5790d7d7ecfbab71ef9eda9e', description='null'}-mongohml:27017] DEBUG org.mongodb.driver.cluster  - Updating cluster description to  {type=REPLICA_SET, servers=[{address=mongohml:27017, type=REPLICA_SET_PRIMARY, roundTripTime=157,6 ms, state=CONNECTED}, {address=mongohml:27018, type=REPLICA_SET_SECONDARY, roundTripTime=157,6 ms, state=CONNECTED}, {address=mongohml:27019, type=REPLICA_SET_SECONDARY, roundTripTime=157,5 ms, state=CONNECTED}]
9405 [main] INFO org.mongodb.driver.connection  - Opened connection [connectionId{localValue:5, serverValue:811}] to mongohml:27019
9411 [main] DEBUG org.mongodb.driver.protocol.command  - Sending command {count : BsonString{value='Cars'}} to database dbCars on connection [connectionId{localValue:5, serverValue:811}] to server mongohml:27019
9574 [main] DEBUG org.mongodb.driver.protocol.command  - Command execution completed
9574 [main] INFO com.kodcu.provider.MongoToElasticProvider  - Mongo collection count: 261834
9578 [main] INFO com.kodcu.main.Mongolastic  - Load duration: 9576ms
Exception in thread "main" NoNodeAvailableException[None of the configured nodes are available: [{#transport#-1}{192.168.2.32}{192.168.2.32:9200}]]
    at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:290)
    at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:207)
    at org.elasticsearch.client.transport.support.TransportProxyClient.execute(TransportProxyClient.java:55)
    at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:288)
    at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:359)
    at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1226)
    at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:86)
    at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:56)
    at com.kodcu.service.ElasticBulkService.dropDataSet(ElasticBulkService.java:94)
    at com.kodcu.provider.Provider.transfer(Provider.java:22)
    at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:61)
    at java.util.Optional.ifPresent(Optional.java:159)
    at com.kodcu.main.Mongolastic.start(Mongolastic.java:50)
    at com.kodcu.main.Mongolastic.main(Mongolastic.java:38)
ozlerhakan commented 8 years ago

Hi @rodrigoalviani ,

The log says that :

5935 [main] INFO org.elasticsearch.client.transport  - [Mad Dog Rassitano] failed to get node info for {#transport#-1}{192.168.2.32}{192.168.2.32:9200}, disconnecting...

This issue is not related with mongolastic. Default transport port of es is 9300.

rodrigoalviani commented 8 years ago

Hi @ozlerhakan,

Locally I connect to ES on 9200, where its is set?

ozlerhakan commented 8 years ago

Yes, but mongolastic communicates through es's tcp transport port.

ozlerhakan commented 8 years ago

do you solve your issue? @rodrigoalviani

rodrigoalviani commented 8 years ago

@ozlerhakan Not solved, even with the door open I got the same error, telnet connect in 9300 smoothly. But anyway, I wrote an application that makes the transfer of MongoDB data for the ES. You can close, Thanks!

MaxCreeger32 commented 6 years ago

hello, I am sorry to tell that, but I have exactly the same problem : 

Exception in thread "main" NoNodeAvailableException[None of the configured nodes are available: [{#transport#-1}{xQcBppiDSRyq7Xm8YalPmg}{127.0.0.1}{127.0.0.1:9300}]]
        at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:347)
        at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:245)
        at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:60)
        at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:360)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:394)
        at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1247)
        at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:46)
        at com.kodcu.provider.ElasticToMongoProvider.getCount(ElasticToMongoProvider.java:39)
        at com.kodcu.provider.Provider.transfer(Provider.java:16)
        at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:69)
        at java.util.Optional.ifPresent(Optional.java:159)
        at com.kodcu.main.Mongolastic.start(Mongolastic.java:58)
        at com.kodcu.main.Mongolastic.main(Mongolastic.java:35)
[INFO ] [2018-05-14 23:10:33] [cluster-ClusterId{value='5af9fb492dd4b341253b85cf', description='null'}-127.0.0.1:27017] [INFO]: - Exception in monitor thread while connecting to server 127.0.0.1:27017
com.mongodb.MongoSecurityException: Exception authenticating
        at com.mongodb.connection.NativeAuthenticator.authenticate(NativeAuthenticator.java:48) ~[mongolastic.jar:?]
        at com.mongodb.connection.InternalStreamConnectionInitializer.authenticateAll(InternalStreamConnectionInitializer.java:109) ~[mongolastic.jar:?]
        at com.mongodb.connection.InternalStreamConnectionInitializer.initialize(InternalStreamConnectionInitializer.java:46) ~[mongolastic.jar:?]
        at com.mongodb.connection.InternalStreamConnection.open(InternalStreamConnection.java:116) ~[mongolastic.jar:?]
        at com.mongodb.connection.DefaultServerMonitor$ServerMonitorRunnable.run(DefaultServerMonitor.java:113) [mongolastic.jar:?]
        at java.lang.Thread.run(Thread.java:748) [?:1.8.0_171]
Caused by: com.mongodb.MongoCommandException: Command failed with error 18: 'auth failed' on server 127.0.0.1:27017. The full response is { "ok" : 0.0, "errmsg" : "auth failed", "code" : 18, "codeName" : "AuthenticationFailed" }
        at com.mongodb.connection.CommandHelper.createCommandFailureException(CommandHelper.java:170) ~[mongolastic.jar:?]
        at com.mongodb.connection.CommandHelper.receiveCommandResult(CommandHelper.java:123) ~[mongolastic.jar:?]
        at com.mongodb.connection.CommandHelper.executeCommand(CommandHelper.java:32) ~[mongolastic.jar:?]
        at com.mongodb.connection.NativeAuthenticator.authenticate(NativeAuthenticator.java:46) ~[mongolastic.jar:?]
        ... 5 more

My config file

misc:   
    dindex:  
        name: teleinfo  
        as: test_teleinfo  
    ctype:  
        name: conso  
        as: test_conso  
    direction: em  
mongo:  
    host: 127.0.0.1  
    port: 27017  
    auth:  
        user: bber  
        pwd: "xxxxxxxx"  
        source: admin  
        mechanism: plain  
elastic:  
    host: 127.0.0.1  
    port: 9300  
    dateFormat: "dd/MM/yyyy-HH:mm"  
    longToString: true

if it can helps I use ElasticSeach 5.5 Thanks for your help

ozlerhakan commented 6 years ago

Hi @MaxCreeger32 ,

Which auth do you use in your mongod? it says that { "ok" : 0.0, "errmsg" : "auth failed", "code" : 18, "codeName" : "AuthenticationFailed" }

MaxCreeger32 commented 6 years ago

sorry, yes I have seen that problem that is solved now : but I still have the ElasticSearch connection problem : 

[INFO ] [2018-05-15 08:17:12] [main] [INFO]: -
Config Output:
{elastic=Elastic{host='127.0.0.1', port=9300, clusterName=null, dateFormat=dd/MM/yyyy-HH:mm, longToString=true, auth=null}, misc=Misc{batch=200, direction='em', dindex=Namespace{as='test_teleinfo', name='teleinfo'}, ctype=Namespace{as='test_conso', name='cons
o'}, dropDataset=true}, mongo=Mongo{host='127.0.0.1', port=27017, query='{}', project='null', auth=Auth [user=bber, pwd=xxxxxxx, source=admin, mechanism=plain]}}

[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - no modules loaded
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.index.reindex.ReindexPlugin]
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.join.ParentJoinPlugin]
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.percolator.PercolatorPlugin]
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.script.mustache.MustachePlugin]
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.transport.Netty4Plugin]
[INFO ] [2018-05-15 08:17:13] [main] [INFO]: - loaded plugin [org.elasticsearch.xpack.XPackPlugin]
[INFO ] [2018-05-15 08:17:14] [main] [INFO]: - Cluster created with settings {hosts=[127.0.0.1:27017], mode=MULTIPLE, requiredClusterType=UNKNOWN, serverSelectionTimeout='5000 ms', maxWaitQueueSize=500}
[INFO ] [2018-05-15 08:17:14] [main] [INFO]: - Adding discovered server 127.0.0.1:27017 to client view of cluster
[INFO ] [2018-05-15 08:17:14] [main] [INFO]: - Load duration: 1394ms
Exception in thread "main" NoNodeAvailableException[None of the configured nodes are available: [{#transport#-1}{JRfQErvgQ8edu5xPzAV4Qg}{127.0.0.1}{127.0.0.1:9300}]]
        at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:347)
        at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:245)
        at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:60)
        at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:360)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:394)
        at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1247)
        at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:46)
        at com.kodcu.provider.ElasticToMongoProvider.getCount(ElasticToMongoProvider.java:39)
        at com.kodcu.provider.Provider.transfer(Provider.java:16)
        at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:69)
        at java.util.Optional.ifPresent(Optional.java:159)
        at com.kodcu.main.Mongolastic.start(Mongolastic.java:58)
        at com.kodcu.main.Mongolastic.main(Mongolastic.java:35)
ozlerhakan commented 6 years ago

Have you looked at these threads: https://stackoverflow.com/questions/25912572/java-elasticsearch-none-of-the-configured-nodes-are-available , https://stackoverflow.com/questions/42826735/nonodeavailableexception-none-of-the-configured-nodes-are-available?rq=1

you may need to specify a cluster name in the mongolastic config file:

...
elastic:
    clusterName: elastictest
...
MaxCreeger32 commented 6 years ago

thanks for the links ! I have set the cluster name in the mongolastic file : 

misc:
    dindex:
        name: teleinfo
        as: test_teleinfo
    ctype:
        name: conso
        as: test_conso
    direction: em
mongo:
    host: 127.0.0.1
    port: 27017
    auth:
        user: bber
        pwd: "cab32b79"
        source: admin
        mechanism: plain
elastic:
    host: 127.0.0.1
    port: 9300
    dateFormat: "dd/MM/yyyy-HH:mm"
    longToString: true
    clusterName: elasticsearch

and the cluster name in my elastic search is correct : 

$ curl http://127.0.0.1:9200
{
  "name" : "vZtXodp",
  "cluster_name" : "elasticsearch",
  "cluster_uuid" : "NEa7hb8cSPGbnYWU8hp87A",
  "version" : {
    "number" : "5.5.2",
    "build_hash" : "b2f0c09",
    "build_date" : "2017-08-14T12:33:14.154Z",
    "build_snapshot" : false,
    "lucene_version" : "6.6.0"
  },
  "tagline" : "You Know, for Search"
}

but I still have : 

$ java -jar mongolastic.jar -f config.yml
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: -
Config Output:
{elastic=Elastic{host='127.0.0.1', port=9300, clusterName=elasticsearch, dateFormat=dd/MM/yyyy-HH:mm, longToString=true, auth=null}, misc=Misc{batch=200, direction='em', dindex=Namespace{as='test_teleinfo', name='teleinfo'}, ctype=Namespace{as='test_conso', n
ame='conso'}, dropDataset=true}, mongo=Mongo{host='127.0.0.1', port=27017, query='{}', project='null', auth=Auth [user=bber, pwd=xxxxxxxx, source=admin, mechanism=plain]}}

[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - no modules loaded
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.index.reindex.ReindexPlugin]
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.join.ParentJoinPlugin]
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.percolator.PercolatorPlugin]
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.script.mustache.MustachePlugin]
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.transport.Netty4Plugin]
[INFO ] [2018-05-15 12:43:13] [main] [INFO]: - loaded plugin [org.elasticsearch.xpack.XPackPlugin]
[INFO ] [2018-05-15 12:43:14] [main] [INFO]: - Cluster created with settings {hosts=[127.0.0.1:27017], mode=MULTIPLE, requiredClusterType=UNKNOWN, serverSelectionTimeout='5000 ms', maxWaitQueueSize=500}
[INFO ] [2018-05-15 12:43:14] [main] [INFO]: - Adding discovered server 127.0.0.1:27017 to client view of cluster
[INFO ] [2018-05-15 12:43:14] [main] [INFO]: - Load duration: 1468ms
Exception in thread "main" NoNodeAvailableException[None of the configured nodes are available: [{#transport#-1}{e2-U48vzS1y96o8z2A2llw}{127.0.0.1}{127.0.0.1:9300}]]
        at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:347)
        at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:245)
        at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:60)
        at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:360)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405)
        at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:394)
        at org.elasticsearch.client.support.AbstractClient$IndicesAdmin.execute(AbstractClient.java:1247)
        at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:46)
        at com.kodcu.provider.ElasticToMongoProvider.getCount(ElasticToMongoProvider.java:39)
        at com.kodcu.provider.Provider.transfer(Provider.java:16)
        at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:69)
        at java.util.Optional.ifPresent(Optional.java:159)
        at com.kodcu.main.Mongolastic.start(Mongolastic.java:58)
        at com.kodcu.main.Mongolastic.main(Mongolastic.java:35)

as i see in your code, it seems to be somewhere in the elastic packages not yours ... i do not understand why I still have this error.

ozlerhakan commented 6 years ago

hmm, which version of mongolastic do you use? could you try previous jar files? maybe this is a problem for the latest jar

MaxCreeger32 commented 6 years ago

It seems to be the version 1.4.3 as the indication in the link is : https://github.com/ozlerhakan/mongolastic/releases/download/v1.4.3/mongolastic.jar

MaxCreeger32 commented 6 years ago

oh i just see in the release that there is a new one ! I will test with this one

MaxCreeger32 commented 6 years ago

It seems that the version 1.4.2 is OK ! 👍 I still have problems, but it is mongDb problem that has nothing connections with this problem (I did not create the target database 😝 ).

MaxCreeger32 commented 6 years ago

thanks for all your support

MaxCreeger32 commented 6 years ago

I have just seen that in tyhe histry of ElasticConfiguration.java : 

-            InetSocketTransportAddress ista = new InetSocketTransportAddress(InetAddress.getByName(config.getElastic().getHost()), config.getElastic().getPort());  
+            TransportAddress ista = new TransportAddress(InetAddress.getByName(config.getElastic().getHost()), config.getElastic().getPort());

may be it is the TransportAdress class that have a problem in comparison of the InetSocketTransportAddress

ozlerhakan commented 6 years ago

Great @MaxCreeger32 ! good to hear that it works with 1.4.2 :)

may be it is the TransportAdress class that have a problem in comparison of the InetSocketTransportAddress

Hmm, if we have another case for that issue, I'll look a solution

Semant1ka commented 6 years ago

Hi! I am observing the same problem for version 1.4.4 (btw, download path on main page pointing on 1.4.3)

My error looks like:

[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - 
Config Output:
{elastic=Elastic{host='127.0.0.1', port=9300, clusterName=docker-cluster, dateFormat=null, longToString=false, auth=null}, misc=Misc{batch=200, direction='me', dindex=Namespace{as='null', name='api'}, ctype=Namespace{as='null', name='conversations'}, dropDataset=false}, mongo=Mongo{host='0.0.0.0', port=27017, query='{}', project='null', auth=null}}

[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - no modules loaded
[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - loaded plugin [org.elasticsearch.index.reindex.ReindexPlugin]
[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - loaded plugin [org.elasticsearch.join.ParentJoinPlugin]
[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - loaded plugin [org.elasticsearch.percolator.PercolatorPlugin]
[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - loaded plugin [org.elasticsearch.script.mustache.MustachePlugin]
[INFO ] [2018-09-11 14:12:53] [main] [INFO]: - loaded plugin [org.elasticsearch.transport.Netty4Plugin]
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - Cluster created with settings {hosts=[0.0.0.0:27017], mode=MULTIPLE, requiredClusterType=UNKNOWN, serverSelectionTimeout='5000 ms', maxWaitQueueSize=500}
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - Adding discovered server 0.0.0.0:27017 to client view of cluster
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - No server chosen by ReadPreferenceServerSelector{readPreference=ReadPreference{name=primaryPreferred}} from cluster description ClusterDescription{type=UNKNOWN, connectionMode=MULTIPLE, serverDescriptions=[ServerDescription{address=0.0.0.0:27017, type=UNKNOWN, state=CONNECTING}]}. Waiting for 5000 ms before timing out
[INFO ] [2018-09-11 14:12:56] [cluster-ClusterId{value='5b9805a85e14334ebbdad732', description='null'}-0.0.0.0:27017] [INFO]: - Opened connection [connectionId{localValue:1, serverValue:85}] to 0.0.0.0:27017
[INFO ] [2018-09-11 14:12:56] [cluster-ClusterId{value='5b9805a85e14334ebbdad732', description='null'}-0.0.0.0:27017] [INFO]: - Monitor thread successfully connected to server with description ServerDescription{address=0.0.0.0:27017, type=STANDALONE, state=CONNECTED, ok=true, version=ServerVersion{versionList=[4, 0, 2]}, minWireVersion=0, maxWireVersion=7, maxDocumentSize=16777216, roundTripTimeNanos=7181257}
[INFO ] [2018-09-11 14:12:56] [cluster-ClusterId{value='5b9805a85e14334ebbdad732', description='null'}-0.0.0.0:27017] [INFO]: - Discovered cluster type of STANDALONE
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - Opened connection [connectionId{localValue:2, serverValue:86}] to 0.0.0.0:27017
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - Mongo collection count: 1
[INFO ] [2018-09-11 14:12:56] [main] [INFO]: - Transferring data began to elasticsearch.
[WARN ] [2018-09-11 14:12:57] [main] [WARN]: - Failed to execute bulk request 1.
org.elasticsearch.client.transport.NoNodeAvailableException: None of the configured nodes are available: [{#transport#-1}{UiDmeqXiREmHZ9AiHB8SAg}{127.0.0.1}{127.0.0.1:9300}]
    at org.elasticsearch.client.transport.TransportClientNodesService.ensureNodesAreAvailable(TransportClientNodesService.java:347) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.client.transport.TransportClientNodesService.execute(TransportClientNodesService.java:245) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.client.transport.TransportProxyClient.execute(TransportProxyClient.java:60) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.client.transport.TransportClient.doExecute(TransportClient.java:371) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:405) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.client.support.AbstractClient.bulk(AbstractClient.java:482) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.Retry$RetryHandler.execute(Retry.java:206) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.Retry.withBackoff(Retry.java:64) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkRequestHandler.execute(BulkRequestHandler.java:65) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkProcessor.execute(BulkProcessor.java:339) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkProcessor.awaitClose(BulkProcessor.java:239) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.service.ElasticBulkService.close(ElasticBulkService.java:79) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.lambda$proceedService$0(Mongolastic.java:70) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.provider.Provider.transfer(Provider.java:35) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:69) ~[mongolastic.1.4.4.jar:?]
    at java.util.Optional.ifPresent(Optional.java:172) [?:?]
    at com.kodcu.main.Mongolastic.start(Mongolastic.java:58) [mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.main(Mongolastic.java:35) [mongolastic.1.4.4.jar:?]
[ERROR] [2018-09-11 14:12:57] [main] [ERROR]: - Transfer failed.
[ERROR] [2018-09-11 14:12:57] [main] [ERROR]: - None of the configured nodes are available: [{#transport#-1}{UiDmeqXiREmHZ9AiHB8SAg}{127.0.0.1}{127.0.0.1:9300}]
org.elasticsearch.client.transport.NoNodeAvailableException: None of the configured nodes are available: [{#transport#-1}{UiDmeqXiREmHZ9AiHB8SAg}{127.0.0.1}{127.0.0.1:9300}]
    at com.kodcu.listener.BulkProcessorListener.afterBulk(BulkProcessorListener.java:33) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkRequestHandler.execute(BulkRequestHandler.java:96) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkProcessor.execute(BulkProcessor.java:339) ~[mongolastic.1.4.4.jar:?]
    at org.elasticsearch.action.bulk.BulkProcessor.awaitClose(BulkProcessor.java:239) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.service.ElasticBulkService.close(ElasticBulkService.java:79) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.lambda$proceedService$0(Mongolastic.java:70) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.provider.Provider.transfer(Provider.java:35) ~[mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.proceedService(Mongolastic.java:69) ~[mongolastic.1.4.4.jar:?]
    at java.util.Optional.ifPresent(Optional.java:172) [?:?]
    at com.kodcu.main.Mongolastic.start(Mongolastic.java:58) [mongolastic.1.4.4.jar:?]
    at com.kodcu.main.Mongolastic.main(Mongolastic.java:35) [mongolastic.1.4.4.jar:?]
[INFO ] [2018-09-11 14:12:57] [main] [INFO]: - Closed connection [connectionId{localValue:2, serverValue:86}] to 0.0.0.0:27017 because the pool has been closed.
[INFO ] [2018-09-11 14:12:57] [main] [INFO]: - Load duration: 3746ms

docker-compose.txt migration.txt