search

apache / linkis

Apache Linkis builds a computation middleware layer to facilitate connection, governance and orchestration between the upper applications and the underlying data engines.
https://linkis.apache.org/
Apache License 2.0
3.3k stars 1.17k forks source link

自己编译打包的环境运行报错，日志里面没有查询到具体报错信息 #1068

Closed harvey2738 closed 2 years ago

harvey2738 commented 2 years ago

数据平台环境： hdp Spark2.3.2，hive 3.1.0，hadoop 3.1.0

按照官方文档修改了相应的pom以及打包完成，以及部署。

shell配置部署用户为hadoop，

linkis应用启动用户为root

修改了相关linkis-cli的配置执行相应cli命令如下：

linkis-cli-spark-sql -e 'show databases'

任务能提交成功，但是执行失败，错误日志如下：

2021-11-08 18:10:23,279 INFO AbstractFitter(55) - Parsing command: "-engineType spark-2.3.2 -codeType sql -code select 1" into template: "UNIVERSAL" 2021-11-08 18:10:23,285 INFO ParsedTplValidator(46) - start validating command "linkis-cli", template "UNIVERSAL" 2021-11-08 18:10:23,285 INFO ParsedTplValidator(50) - Start params-check 2021-11-08 18:10:23,285 INFO ParsedTplValidator(52) - params-check ok. 2021-11-08 18:10:23,340 INFO PropsFilesScanner(38) - Start scanning for properties files. Root path = "/appcom/Install/LinkisInstall/conf/linkis-cli/" 2021-11-08 18:10:23,362 INFO PropsFilesScanner(56) - Scanned properties files="[/appcom/Install/LinkisInstall/conf/linkis-cli/linkis-cli.properties]" 2021-11-08 18:10:23,366 INFO LinkisClientApplication(228) - ==========std_var============ { "cmdParams": { "cmdType": "UNIVERSAL", "paramItemMap": { "wds.linkis.client.common.status": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.status", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.help": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.help", "defaultValue": false, "hasVal": false }, "wds.linkis.client.common.output.path": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.output.path", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.user.conf": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.user.conf", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.kill": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.kill", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.tokenKey": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.tokenKey", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.creator": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.creator", "defaultValue": "", "hasVal": false }, "wds.linkis.client.label.engineType": { "keyPrefix": "wds.linkis.client.label", "key": "wds.linkis.client.label.engineType", "value": "spark-2.3.2", "defaultValue": "", "hasVal": true }, "wds.linkis.client.source.scriptPath": { "keyPrefix": "wds.linkis.client.source", "key": "wds.linkis.client.source.scriptPath", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.submitUser": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.submitUser", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.proxyUser": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.proxyUser", "defaultValue": "", "hasVal": false }, "wds.linkis.client.param.conf": { "keyPrefix": "wds.linkis.client.param.conf", "key": "wds.linkis.client.param.conf", "value": {}, "hasVal": false }, "wds.linkis.client.param.var": { "keyPrefix": "wds.linkis.client.param.var", "key": "wds.linkis.client.param.var", "value": {}, "hasVal": false }, "wds.linkis.client.label.codeType": { "keyPrefix": "wds.linkis.client.label", "key": "wds.linkis.client.label.codeType", "value": "sql", "defaultValue": "", "hasVal": true }, "wds.linkis.client.common.gatewayUrl": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.gatewayUrl", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.code.path": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.code.path", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.tokenValue": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.tokenValue", "defaultValue": "", "hasVal": false }, "wds.linkis.client.common.authStrategy": { "keyPrefix": "wds.linkis.client.common", "key": "wds.linkis.client.common.authStrategy", "defaultValue": "", "hasVal": false }, "wds.linkis.client.source": { "keyPrefix": "wds.linkis.client.source", "key": "wds.linkis.client.source", "value": {}, "hasVal": false }, "wds.linkis.client.label": { "keyPrefix": "wds.linkis.client.label", "key": "wds.linkis.client.label", "value": {}, "hasVal": false }, "wds.linkis.client.exec.code": { "keyPrefix": "wds.linkis.client.exec", "key": "wds.linkis.client.exec.code", "value": "select 1", "defaultValue": "", "hasVal": true } }, "extraProperties": {} }, "defaultConf": { "wds.linkis.client.common.gatewayUrl": "http://127.0.0.1:9010", "wds.linkis.client.common.tokenKey": "Validation-Code", "wds.linkis.client.common.tokenValue": "BML-AUTH", "wds.linkis.client.common.creator": "LINKISCLI", "wds.linkis.client.common.authStrategy": "token" }, "subMapCache": {} } 2021-11-08 18:10:23,386 INFO LinkisClientApplication(269) - ==========job============ { "executionMap": { "code": "select 1" }, "paramConfMap": {}, "paramRunTimeMap": {}, "paramVarsMap": {}, "labelMap": { "userCreator": "root-LINKISCLI", "codeType": "sql", "engineType": "spark-2.3.2" }, "sourceMap": { "scriptPath": "LinkisCli" }, "submitUser": "root", "proxyUser": "root", "subExecutionType": "NONE", "outputWay": "STANDARD" } 2021-11-08 18:10:23,397 INFO UjesClientDriverBuilder(71) - ==========ujes_Ctx============ { "gatewayUrl": "http://127.0.0.1:9010", "connectionTimeout": 30000, "discoveryEnabled": false, "loadbalancerEnabled": true, "readTimeoutMills": 30000, "tokenKey": "Validation-Code", "tokenValue": "BML-AUTH", "authenticationStrategyStr": "token", "maxConnectionSize": 5, "retryEnabled": false, "dwsVersion": "v1" } 2021-11-08 18:10:23,767 INFO UjesClientDriver(99) - Linkis ujes client inited. 2021-11-08 18:10:23,769 INFO UjesClientDriverBuilder(71) - ==========ujes_Ctx============ { "gatewayUrl": "http://127.0.0.1:9010", "connectionTimeout": 30000, "discoveryEnabled": false, "loadbalancerEnabled": true, "readTimeoutMills": 30000, "tokenKey": "Validation-Code", "tokenValue": "BML-AUTH", "authenticationStrategyStr": "token", "maxConnectionSize": 5, "retryEnabled": false, "dwsVersion": "v1" } 2021-11-08 18:10:23,773 INFO UjesClientDriver(99) - Linkis ujes client inited. 2021-11-08 18:10:23,787 INFO UjesClientDriver(144) - Request info to Linkis: { "com$webank$wedatasphere$linkis$ujes$client$request$UserAction$$user": "root", "formParams": {}, "payload": { "executionContent": { "code": "select 1" }, "source": { "scriptPath": "LinkisCli" }, "params": { "configuration": { "startup": {} }, "variable": {} }, "executeUser": "root", "labels": { "userCreator": "root-LINKISCLI", "codeType": "sql", "engineType": "spark-2.3.2" } }, "queryParams": {}, "com$webank$wedatasphere$linkis$httpclient$request$HttpAction$$headerParams": {}, "com$webank$wedatasphere$linkis$httpclient$request$HttpAction$$cookies": [] } 2021-11-08 18:10:24,148 WARN BDPConfiguration$(49) - **** Notice: The Linkis configuration file linkis.properties is not exists! * 2021-11-08 18:10:24,149 WARN BDPConfiguration$(49) - **** Notice: The Linkis serverConf file null is not exists! * 2021-11-08 18:10:25,087 INFO UjesClientDriver(162) - Response info from Linkis: { "com$webank$wedatasphere$linkis$ujes$client$request$UserAction$$user": "root", "com$webank$wedatasphere$linkis$httpclient$dws$request$DWSHttpAction$$dwsVersion": "v1", "formParams": {}, "payload": { "executionContent": { "code": "select 1" }, "source": { "scriptPath": "LinkisCli" }, "params": { "configuration": { "startup": {} }, "variable": {} }, "executeUser": "root", "labels": { "userCreator": "root-LINKISCLI", "codeType": "sql", "engineType": "spark-2.3.2" } }, "queryParams": {}, "com$webank$wedatasphere$linkis$httpclient$request$HttpAction$$headerParams": { "Token-User": "root", "Token-Code": "BML-AUTH" }, "com$webank$wedatasphere$linkis$httpclient$request$HttpAction$$cookies": [] } 2021-11-08 18:10:27,149 INFO LinkisSubmitExecutor(119) - Jobinfo from Linkis: { "task": { "taskID": 22, "instance": "cb-hdp-37-100:9104", "execId": "LINKISCLI_root_spark_1", "umUser": "root", "executionCode": "select 1", "progress": "1.0", "logPath": "hdfs:///tmp/linkis/root/log/LINKISCLI/2021-11-08/22.log", "status": "Failed", "engineType": "spark", "errDesc": "21304, Task is Failed,errorMsg: errCode: 12003 ,desc: cb-hdp-37-100:9101_2 Failed to async get EngineNode RMErrorException: errCode: 11006 ,desc: Failed to request external resourceConnectException: Connection refused (Connection refused) ,ip: cb-hdp-37-1", "executeApplicationName": "spark", "requestApplicationName": "LINKISCLI", "runType": "sql", "paramsJson": "{\"configuration\":{\"startup\":{},\"runtime\":{\"wds.linkis.resultSet.store.path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\",\"source\":{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"},\"job\":{\"resultsetIndex\":0,\"#rt_rs_store_path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\"}}},\"variable\":{}}", "costTime": 1000, "strongerExecId": "exec_id018018linkis-cg-entrancecb-hdp-37-100:9104LINKISCLI_root_spark_1", "sourceJson": "{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"}", "sourceTailor": "LinkisCli-127.0.0.1" }, "com$webank$wedatasphere$linkis$ujes$client$response$JobInfoResult$$requestPersistTask": { "taskID": 22, "instance": "cb-hdp-37-100:9104", "execId": "LINKISCLI_root_spark_1", "umUser": "root", "executionCode": "select 1", "progress": 1.0, "logPath": "hdfs:///tmp/linkis/root/log/LINKISCLI/2021-11-08/22.log", "status": "Failed", "createdTime": "Nov 8, 2021 6:10:24 PM", "updatedTime": "Nov 8, 2021 6:10:25 PM", "errCode": 0, "errDesc": "21304, Task is Failed,errorMsg: errCode: 12003 ,desc: cb-hdp-37-100:9101_2 Failed to async get EngineNode RMErrorException: errCode: 11006 ,desc: Failed to request external resourceConnectException: Connection refused (Connection refused) ,ip: cb-hdp-37-1", "executeApplicationName": "spark", "requestApplicationName": "LINKISCLI", "runType": "sql", "engineType": "sql", "engineStartTime": "Nov 8, 2021 6:10:25 PM" }, "com$webank$wedatasphere$linkis$ujes$client$request$UserAction$$user": "root", "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$resultMap": { "method": "/api/jobhistory/{id}/get", "status": 0, "message": "OK", "data": { "task": { "taskID": 22, "instance": "cb-hdp-37-100:9104", "execId": "LINKISCLI_root_spark_1", "umUser": "root", "executionCode": "select 1", "progress": "1.0", "logPath": "hdfs:///tmp/linkis/root/log/LINKISCLI/2021-11-08/22.log", "status": "Failed", "engineType": "spark", "errDesc": "21304, Task is Failed,errorMsg: errCode: 12003 ,desc: cb-hdp-37-100:9101_2 Failed to async get EngineNode RMErrorException: errCode: 11006 ,desc: Failed to request external resourceConnectException: Connection refused (Connection refused) ,ip: cb-hdp-37-1", "executeApplicationName": "spark", "requestApplicationName": "LINKISCLI", "runType": "sql", "paramsJson": "{\"configuration\":{\"startup\":{},\"runtime\":{\"wds.linkis.resultSet.store.path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\",\"source\":{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"},\"job\":{\"resultsetIndex\":0,\"#rt_rs_store_path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\"}}},\"variable\":{}}", "costTime": 1000, "strongerExecId": "exec_id018018linkis-cg-entrancecb-hdp-37-100:9104LINKISCLI_root_spark_1", "sourceJson": "{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"}", "sourceTailor": "LinkisCli-127.0.0.1" } } }, "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$responseBody": "{\"method\":\"/api/jobhistory/{id}/get\",\"status\":0,\"message\":\"OK\",\"data\":{\"task\":{\"taskID\":22,\"instance\":\"cb-hdp-37-100:9104\",\"execId\":\"LINKISCLI_root_spark_1\",\"umUser\":\"root\",\"engineInstance\":null,\"executionCode\":\"select 1\",\"progress\":\"1.0\",\"logPath\":\"hdfs:///tmp/linkis/root/log/LINKISCLI/2021-11-08/22.log\",\"resultLocation\":null,\"status\":\"Failed\",\"createdTime\":1636366224000,\"updatedTime\":1636366225000,\"engineType\":\"spark\",\"errCode\":null,\"errDesc\":\"21304, Task is Failed,errorMsg: errCode: 12003 ,desc: cb-hdp-37-100:9101_2 Failed to async get EngineNode RMErrorException: errCode: 11006 ,desc: Failed to request external resourceConnectException: Connection refused (Connection refused) ,ip: cb-hdp-37-1\",\"executeApplicationName\":\"spark\",\"requestApplicationName\":\"LINKISCLI\",\"runType\":\"sql\",\"paramsJson\":\"{\\"configuration\\":{\\"startup\\":{},\\"runtime\\":{\\"wds.linkis.resultSet.store.path\\":\\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\\",\\"source\\":{\\"scriptPath\\":\\"LinkisCli\\",\\"requestIP\\":\\"127.0.0.1\\"},\\"job\\":{\\"resultsetIndex\\":0,\\"#rt_rs_store_path\\":\\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\\"}}},\\"variable\\":{}}\",\"costTime\":1000,\"strongerExecId\":\"exec_id018018linkis-cg-entrancecb-hdp-37-100:9104LINKISCLI_root_spark_1\",\"sourceJson\":\"{\\"scriptPath\\":\\"LinkisCli\\",\\"requestIP\\":\\"127.0.0.1\\"}\",\"sourceTailor\":\"LinkisCli-127.0.0.1\",\"engineStartTime\":null,\"labels\":null,\"subJobs\":null}}}", "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$statusCode": 200, "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$url": "/api/rest_j/v1/jobhistory/22/get", "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$contentType": "application/json;charset=utf-8", "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$status": 0, "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$message": "OK", "com$webank$wedatasphere$linkis$httpclient$dws$response$DWSResult$$data": { "task": { "taskID": 22, "instance": "cb-hdp-37-100:9104", "execId": "LINKISCLI_root_spark_1", "umUser": "root", "executionCode": "select 1", "progress": "1.0", "logPath": "hdfs:///tmp/linkis/root/log/LINKISCLI/2021-11-08/22.log", "status": "Failed", "engineType": "spark", "errDesc": "21304, Task is Failed,errorMsg: errCode: 12003 ,desc: cb-hdp-37-100:9101_2 Failed ### to async get EngineNode RMErrorException: errCode: 11006 ,desc: Failed to request external resourceConnectException: Connection refused (Connection refused) ,ip: cb-hdp-37-1",**** "executeApplicationName": "spark", "requestApplicationName": "LINKISCLI", "runType": "sql", "paramsJson": "{\"configuration\":{\"startup\":{},\"runtime\":{\"wds.linkis.resultSet.store.path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\",\"source\":{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"},\"job\":{\"resultsetIndex\":0,\"#rt_rs_store_path\":\"hdfs:///tmp/linkis/root/linkis/20211108_181024/LINKISCLI/22\"}}},\"variable\":{}}", "costTime": 1000, "strongerExecId": "exec_id018018linkis-cg-entrancecb-hdp-37-100:9104LINKISCLI_root_spark_1", "sourceJson": "{\"scriptPath\":\"LinkisCli\",\"requestIP\":\"127.0.0.1\"}", "sourceTailor": "LinkisCli-127.0.0.1" } }, "bitmap$0": false } 2021-11-08 18:10:27,150 INFO LinkisSubmitExecutor(154) - Job submission status: Final status=FAILED Updated taskID=22 ExecID=exec_id018018linkis-cg-entrancecb-hdp-37-100:9104LINKISCLI_root_spark_1 2021-11-08 18:10:27,182 WARN SyncSubmission(154) - Exception thrown when trying to query final result. Status will change to FAILED com.webank.wedatasphere.linkis.cli.core.exception.ExecutorException: EXE0021,Error occured during execution: Get ResultSet Failed: job Status is not "Succeed", . at com.webank.wedatasphere.linkis.cli.application.driver.UjesClientDriver.queryResultSetPaths(UjesClientDriver.java:428) ~[linkis-cli-application-1.0.2.jar:?] at com.webank.wedatasphere.linkis.cli.application.interactor.execution.executor.LinkisSubmitExecutor.doGetFinalResult(LinkisSubmitExecutor.java:173) ~[linkis-cli-application-1.0.2.jar:?] at com.webank.wedatasphere.linkis.cli.core.interactor.execution.SyncSubmission.ExecWithAsyncBackend(SyncSubmission.java:152) [linkis-cli-core-1.0.2.jar:?] at com.webank.wedatasphere.linkis.cli.core.interactor.execution.SyncSubmission.execute(SyncSubmission.java:76) [linkis-cli-core-1.0.2.jar:?] at com.webank.wedatasphere.linkis.cli.application.LinkisClientApplication.exec(LinkisClientApplication.java:349) [linkis-cli-application-1.0.2.jar:?] at com.webank.wedatasphere.linkis.cli.application.LinkisClientApplication.main(LinkisClientApplication.java:381) [linkis-cli-application-1.0.2.jar:?] 2021-11-08 18:10:27,214 INFO LinkisSubmitExecutor(101) - 

    task 22 status is FAILED, progress : 1.0

2021-11-08 18:10:29,215 INFO LinkisJobResultPresenter(57) - Job status is not success but 'FAILED'. Will not try to retrieve any Result

大致搜索下错误信息代码：

image

能否有大神，帮忙查看这个错误信息。

peacewong commented 2 years ago

Hello, you can check the logs of linkisManager and EngineConnManager