search

micahstubbs / drift

an ES2015 port of h2o-flow
MIT License
1 stars 2 forks source link

build model | deep learning | ERROR FETCHING INITIAL MODEL BUILDER STATE #93

Closed micahstubbs closed 7 years ago

micahstubbs commented 7 years ago

screen shot 2017-01-10 at 5 15 28 pm

micahstubbs commented 7 years ago

h2o-3 output

micahstubbs commented 7 years ago 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: POST /3/ModelBuilders/gbm/parameters, parms: {col_sample_rate=1, keep_cross_validation_fold_assignment=false, nfolds=0, quantile_alpha=0.5, nbins_top_level=1024, score_tree_interval=0, max_confusion_matrix_size=20, stopping_tolerance=0.001, class_sampling_factors=, categorical_encoding=AUTO, r2_stopping=1.7976931348623157e+308, col_sample_rate_change_per_level=1, min_split_improvement=0.00001, histogram_type=AUTO, sample_rate_per_class=, min_rows=10, seed=-1, max_hit_ratio_k=0, max_depth=5, pred_noise_bandwidth=0, huber_alpha=0.9, score_each_iteration=false, max_runtime_secs=0, tweedie_power=1.5, col_sample_rate_per_tree=1, sample_rate=1, max_abs_leafnode_pred=1.7976931348623157e+308, stopping_metric=AUTO, ignore_const_cols=true, learn_rate=0.1, nbins_cats=1024, build_tree_one_node=false, model_id=gbm-function v4(options, buf, offset) {
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     // Deprecated - 'format' argument, as supported in v1.2
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     var i = buf && offset || 0;
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     if (typeof(options) === 'string') {
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:       buf = (options === 'binary') ? new BufferClass(16) : null;
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:       options = null;
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     }
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     options = options || {};
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     var rnds = options.random || (options.rng || _rng)();
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     // Per 4.4, set bits for version and `clock_seq_hi_and_reserved`
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     rnds[6] = (rnds[6] & 0x0f) | 0x40;
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     rnds[8] = (rnds[8] & 0x3f) | 0x80;
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     // Copy bytes to buffer, if provided
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     if (buf) {
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:       for (var ii = 0; ii < 16; ii++) {
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:         buf[i + ii] = rnds[ii];
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:       }
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     }
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO: 
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:     return buf || unparse(rnds);
01-11 15:31:40.151 172.16.2.26:54321     27143  #11510-18 INFO:   }, fold_assignment=AUTO, nbins=20, distribution=AUTO, checkpoint=, max_after_balance_size=5, balance_classes=false, stopping_rounds=0, learn_rate_annealing=1, keep_cross_validation_predictions=false, ntrees=50}
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR: java.lang.IllegalArgumentException: Key length would be 682
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.Key.<init>(Key.java:246)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.Key.make(Key.java:264)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.Key.make(Key.java:287)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.Key.make(Key.java:292)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.Schema.parse(Schema.java:491)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.Schema.setField(Schema.java:356)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.Schema.fillFromParms(Schema.java:307)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.Schema.fillFromParms(Schema.java:237)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:46)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:16)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.RequestServer.serve(RequestServer.java:412)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.RequestServer.doGeneric(RequestServer.java:263)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at water.api.RequestServer.doPost(RequestServer.java:200)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at javax.servlet.http.HttpServlet.service(HttpServlet.java:755)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at javax.servlet.http.HttpServlet.service(HttpServlet.java:848)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:503)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.Server.handle(Server.java:370)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:982)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1043)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)
01-11 15:31:40.152 172.16.2.26:54321     27143  #11510-18 ERRR:     at java.lang.Thread.run(Thread.java:745)
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: Caught exception: 
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: 
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: ERROR MESSAGE:
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: 
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: Key length would be 682
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: 
01-11 15:31:40.154 172.16.2.26:54321     27143  #11510-18 ERRR: ; Stacktrace: [java.lang.IllegalArgumentException: Key length would be 682,     water.Key.<init>(Key.java:246),     water.Key.make(Key.java:264),     water.Key.make(Key.java:287),     water.Key.make(Key.java:292),     water.api.Schema.parse(Schema.java:491),     water.api.Schema.setField(Schema.java:356),     water.api.Schema.fillFromParms(Schema.java:307),     water.api.Schema.fillFromParms(Schema.java:237),     water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:46),     water.api.ModelBuilderHandler.handle(ModelBuilderHandler.java:16),     water.api.RequestServer.serve(RequestServer.java:412),     water.api.RequestServer.doGeneric(RequestServer.java:263),     water.api.RequestServer.doPost(RequestServer.java:200),     javax.servlet.http.HttpServlet.service(HttpServlet.java:755),     javax.servlet.http.HttpServlet.service(HttpServlet.java:848),     org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684),     org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:503),     org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137),     org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557),     org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231),     org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086),     org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:429),     org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193),     org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020),     org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135),     org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154),     org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116),     org.eclipse.jetty.server.Server.handle(Server.java:370),     org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494),     org.eclipse.jetty.server.BlockingHttpConnection.handleRequest(BlockingHttpConnection.java:53),     org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:982),     org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:1043),     org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:865),     org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240),     org.eclipse.jetty.server.BlockingHttpConnection.handle(BlockingHttpConnection.java:72),     org.eclipse.jetty.server.bio.SocketConnector$ConnectorEndPoint.run(SocketConnector.java:264),     org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608),     org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543),     java.lang.Thread.run(Thread.java:745)]
01-11 15:31:40.285 172.16.2.26:54321     27143  #11510-18 INFO: POST /3/LogAndEcho, parms: {message=FLOW: Error fetching initial model builder state; Error calling POST /3/ModelBuilders/gbm/parameters with opts {"model_id":"gbm-function v4(options, b...; HTTP connection failure: status=error, code=400, error=Bad Request}
01-11 15:31:40.287 172.16.2.26:54321     27143  #11510-18 INFO: FLOW: Error fetching initial model builder state; Error calling POST /3/ModelBuilders/gbm/parameters with opts {"model_id":"gbm-function v4(options, b...; HTTP connection failure: status=error, code=400, error=Bad Request
micahstubbs commented 7 years ago

broken at ef8cc265172522097493031432816006a6eaf59a on 1/10

micahstubbs commented 7 years ago

working at ae3bd16cce3a907a5a48c40e19c87f5532c1010e at end of day on 1/8

micahstubbs commented 7 years ago

working at 9e17f7ef54c8b75520a03350ff3c4ac1ad855b83 at end of day on 1/9

micahstubbs commented 7 years ago

working at e8def0037932341844d0f61613f003f79c999908 on 1/10

micahstubbs commented 7 years ago

working at b9523978b3271470f9bbb98e0a10e82401f31123 on 1/10

micahstubbs commented 7 years ago

broken at 1074c880303b7364dcc2762941d72c59bcf3dcd2 on 1/10

micahstubbs commented 7 years ago

working at eb95d2547273ecc906ccecbedfeb3858b32e21b9 on 1/10

micahstubbs commented 7 years ago

broken at 51cceb8f3b940c24525a79081cdb6b3ead3d0805 on 1/10

abstract out uuid

this commit introduced the bug