search

crawlab-team / crawlab

Distributed web crawler admin platform for spiders management regardless of languages and frameworks. 分布式爬虫管理平台，支持任何语言和框架
https://www.crawlab.cn
BSD 3-Clause "New" or "Revised" License
11.37k stars 1.79k forks source link

根据教程的单节点配置，部署失败 #1384

Open HamGuy opened 1 year ago

HamGuy commented 1 year ago

Bug 描述 iStoreOS( OpenWrt)，x64平台，docker 部署，使用latest镜像，部署失败

复现步骤 该 Bug 复现步骤如下

  1. 使用官方文档提供的单节点配置文件, 通过 docker-compose 部署
  2. 部署后一直重启，查询日子报错error open /var/log/crawlab: no such file or directory

配置文件

version: '3.3'
services:
  master:
    image: crawlabteam/crawlab
    container_name: crawlab_master
    restart: always
    environment:
      CRAWLAB_NODE_MASTER: "Y"  # Y: .........
      CRAWLAB_MONGO_HOST: "mongo"  # mongo host address. ... Docker-Compose ........................ service ......
      CRAWLAB_MONGO_PORT: "27017"  # mongo port
      CRAWLAB_MONGO_DB: "crawlab"  # mongo database
      CRAWLAB_MONGO_USERNAME: "username"  # mongo username
      CRAWLAB_MONGO_PASSWORD: "password"  # mongo password
      CRAWLAB_MONGO_AUTHSOURCE: "admin"  # mongo auth source
    volumes:
      - "/root/crawlab/master:/root/.crawlab"  # ......... crawlab .........
      - "/root/crawlab/master:/data"  # ......... crawlab ......
      - "/root/crawlab/logs:/var/logs/crawlab" # ......... crawlab ............
    ports:
      - "8080:8080"  # ...... api ......
    depends_on:
      - mongo

  mongo:
    image: mongo:4.2
    restart: always
    environment:
      MONGO_INITDB_ROOT_USERNAME: "username"  # mongo username
      MONGO_INITDB_ROOT_PASSWORD: "password"  # mongo password
    volumes:
      - "/root/crawlab/mongo/data/db:/data/db"  # ......... mongo ......
    ports:
      - "27017:27017"  # ...... mongo ..................

错误日志

docker compose up
[+] Running 21/21
 ✔ mongo Pulled                                                                                                                                                                                                                     10.0s 
 ✔ master 19 layers [⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿⣿]      0B/0B      Pulled                                                                                                                                                                   436.8s 
   ✔ 01085d60b3a6 Pull complete                                                                                                                                                                                                     24.4s 
   ✔ 182e0f0e374a Pull complete                                                                                                                                                                                                      5.3s 
   ✔ 643cd8e07f77 Pull complete                                                                                                                                                                                                     84.9s 
   ✔ 9da605dbf7dc Pull complete                                                                                                                                                                                                     84.1s 
   ✔ da1a6484a9d9 Pull complete                                                                                                                                                                                                    107.1s 
   ✔ a0c1e73a7c8c Pull complete                                                                                                                                                                                                    366.5s 
   ✔ 21aa4801c6c3 Pull complete                                                                                                                                                                                                    151.9s 
   ✔ 9bd19f30921d Pull complete                                                                                                                                                                                                    191.5s 
   ✔ c32adcef8c93 Pull complete                                                                                                                                                                                                    229.4s 
   ✔ add54245ee76 Pull complete                                                                                                                                                                                                    197.7s 
   ✔ 1e2bdef749c5 Pull complete                                                                                                                                                                                                    203.5s 
   ✔ 3a9885698325 Pull complete                                                                                                                                                                                                    207.6s 
   ✔ d4081f0e8fe6 Pull complete                                                                                                                                                                                                    213.0s 
   ✔ fba38324c183 Pull complete                                                                                                                                                                                                    218.4s 
   ✔ 485f3330e045 Pull complete                                                                                                                                                                                                    239.8s 
   ✔ 9591a924de18 Pull complete                                                                                                                                                                                                    271.4s 
   ✔ 47085380bbda Pull complete                                                                                                                                                                                                    265.4s 
   ✔ 075e8a6af6e2 Pull complete                                                                                                                                                                                                    307.8s 
   ✔ 16a28cce47f3 Pull complete                                                                                                                                                                                                    276.3s 
[+] Running 3/2
 ✔ Network crawlab_default    Created                                                                                                                                                                                                0.0s 
 ✔ Container crawlab-mongo-1  Created                                                                                                                                                                                                0.4s 
 ✔ Container crawlab_master   Created                                                                                                                                                                                                0.0s 
Attaching to crawlab-mongo-1, crawlab_master
crawlab-mongo-1  | 
crawlab-mongo-1  | 
crawlab-mongo-1  | WARNING: MongoDB 5.0+ requires a CPU with AVX support, and your current system does not appear to have that!
crawlab-mongo-1  |   see https://jira.mongodb.org/browse/SERVER-54407
crawlab-mongo-1  |   see also https://www.mongodb.com/community/forums/t/mongodb-5-0-cpu-intel-g4650-compatibility/116610/2
crawlab-mongo-1  |   see also https://github.com/docker-library/mongo/issues/485#issuecomment-891991814
crawlab-mongo-1  | 
crawlab-mongo-1  | 
crawlab-mongo-1  | 2023-08-22T15:38:15.345+0000 I  CONTROL  [main] Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'
crawlab-mongo-1  | 2023-08-22T15:38:15.352+0000 W  ASIO     [main] No TransportLayer configured during NetworkInterface startup
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] MongoDB starting : pid=1 port=27017 dbpath=/data/db 64-bit host=d1171834ec25
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] db version v4.2.24
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] git version: 5e4ec1d24431fcdd28b579a024c5c801b8cde4e2
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] OpenSSL version: OpenSSL 1.1.1  11 Sep 2018
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] allocator: tcmalloc
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] modules: none
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] build environment:
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten]     distmod: ubuntu1804
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten]     distarch: x86_64
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten]     target_arch: x86_64
crawlab-mongo-1  | 2023-08-22T15:38:15.353+0000 I  CONTROL  [initandlisten] options: { net: { bindIp: "*" }, security: { authorization: "enabled" } }
crawlab-mongo-1  | 2023-08-22T15:38:15.354+0000 I  STORAGE  [initandlisten] Detected data files in /data/db created by the 'wiredTiger' storage engine, so setting the active storage engine to 'wiredTiger'.
crawlab-mongo-1  | 2023-08-22T15:38:15.354+0000 I  STORAGE  [initandlisten] wiredtiger_open config: create,cache_size=7419M,cache_overflow=(file_max=0M),session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000,close_scan_interval=10,close_handle_minimum=250),statistics_log=(wait=0),verbose=[recovery_progress,checkpoint_progress],
crawlab_master   | replaced api url in file: crawlab-ui.14fd3a25.js
crawlab_master   | 2023/08/22 23:38:15 error open /var/log/crawlab: no such file or directory
crawlab_master   | goroutine 19 [running]:
crawlab_master   | runtime/debug.Stack()
crawlab_master   |      /usr/local/go/src/runtime/debug/stack.go:24 +0x65
crawlab_master   | runtime/debug.PrintStack()
crawlab_master   |      /usr/local/go/src/runtime/debug/stack.go:16 +0x19
crawlab_master   | github.com/crawlab-team/crawlab-core/utils.ListDir({0x1c79f7d?, 0x0?})
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/utils/file.go:88 +0x70
crawlab_master   | github.com/crawlab-team/crawlab-core/task/log.(*FileLogDriver).cleanup(0xc0004ac6a0)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/task/log/file_driver.go:188 +0x4a
crawlab_master   | created by github.com/crawlab-team/crawlab-core/task/log.(*FileLogDriver).Init
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/task/log/file_driver.go:34 +0x56
crawlab_master   | open /var/log/crawlab: no such file or directory
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/go-trace@v0.1.1/trace.go:6 github.com/crawlab-team/go-trace.PrintError()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/task/log/file_driver.go:190 github.com/crawlab-team/crawlab-core/task/log.(*FileLogDriver).cleanup()
crawlab_master   | /usr/local/go/src/runtime/asm_amd64.s:1571 runtime.goexit()
crawlab-mongo-1  | 2023-08-22T15:38:16.078+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:78950][1:0x7f4f3f772b00], txn-recover: Recovering log 5 through 6
crawlab-mongo-1  | 2023-08-22T15:38:16.129+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:129354][1:0x7f4f3f772b00], txn-recover: Recovering log 6 through 6
crawlab-mongo-1  | 2023-08-22T15:38:16.199+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:199424][1:0x7f4f3f772b00], txn-recover: Main recovery loop: starting at 5/6272 to 6/256
crawlab-mongo-1  | 2023-08-22T15:38:16.301+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:300992][1:0x7f4f3f772b00], txn-recover: Recovering log 5 through 6
crawlab-mongo-1  | 2023-08-22T15:38:16.359+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:359068][1:0x7f4f3f772b00], txn-recover: Recovering log 6 through 6
crawlab-mongo-1  | 2023-08-22T15:38:16.415+0000 I  STORAGE  [initandlisten] WiredTiger message [1692718696:415773][1:0x7f4f3f772b00], txn-recover: Set global recovery timestamp: (0, 0)
crawlab-mongo-1  | 2023-08-22T15:38:16.426+0000 I  RECOVERY [initandlisten] WiredTiger recoveryTimestamp. Ts: Timestamp(0, 0)
crawlab-mongo-1  | 2023-08-22T15:38:16.429+0000 I  STORAGE  [initandlisten] Timestamp monitor starting
crawlab-mongo-1  | 2023-08-22T15:38:16.431+0000 I  CONTROL  [initandlisten] 
crawlab-mongo-1  | 2023-08-22T15:38:16.431+0000 I  CONTROL  [initandlisten] ** WARNING: soft rlimits too low. rlimits set to 63437 processes, 1048576 files. Number of processes should be at least 524288 : 0.5 times number of files.
crawlab-mongo-1  | 2023-08-22T15:38:16.435+0000 I  STORAGE  [initandlisten] Flow Control is enabled on this deployment.
crawlab-mongo-1  | 2023-08-22T15:38:16.436+0000 I  FTDC     [initandlisten] Initializing full-time diagnostic data capture with directory '/data/db/diagnostic.data'
crawlab-mongo-1  | 2023-08-22T15:38:16.438+0000 I  NETWORK  [listener] Listening on /tmp/mongodb-27017.sock
crawlab-mongo-1  | 2023-08-22T15:38:16.438+0000 I  NETWORK  [listener] Listening on 0.0.0.0
crawlab-mongo-1  | 2023-08-22T15:38:16.438+0000 I  NETWORK  [listener] waiting for connections on port 27017
crawlab_master   | server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: mongo:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp 172.20.0.2:27017: connect: connection refused }, ] }
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/go-trace@v0.1.1/trace.go:13 github.com/crawlab-team/go-trace.TraceError()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/user/service.go:224 github.com/crawlab-team/crawlab-core/user.NewUserService()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/user/service.go:242 github.com/crawlab-team/crawlab-core/user.GetUserService()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/user/service.go:252 github.com/crawlab-team/crawlab-core/user.ProvideGetUserService.func1()
crawlab_master   | /usr/local/go/src/reflect/value.go:556 reflect.Value.call()
crawlab_master   | /usr/local/go/src/reflect/value.go:339 reflect.Value.Call()
crawlab_master   | /go/pkg/mod/go.uber.org/dig@v1.10.0/dig.go:284 go.uber.org/dig.defaultInvoker()
crawlab_master   | /go/pkg/mod/go.uber.org/dig@v1.10.0/dig.go:710 go.uber.org/dig.(*node).Call()
crawlab_master   | /go/pkg/mod/go.uber.org/dig@v1.10.0/param.go:245 go.uber.org/dig.paramSingle.Build()
crawlab_master   | /go/pkg/mod/go.uber.org/dig@v1.10.0/param.go:201 go.uber.org/dig.paramList.BuildList()
crawlab_master   | /go/pkg/mod/go.uber.org/dig@v1.10.0/dig.go:432 go.uber.org/dig.(*Container).Invoke()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:224 github.com/crawlab-team/crawlab-core/controllers.newUserContext()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:25 github.com/crawlab-team/crawlab-core/controllers.getUserActions()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:243 github.com/crawlab-team/crawlab-core/controllers.newUserController()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/init.go:18 github.com/crawlab-team/crawlab-core/controllers.InitControllers()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/utils.go:26 github.com/crawlab-team/crawlab-core/apps.initModule()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:39 github.com/crawlab-team/crawlab-core/apps.(*Api).Init()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:113 github.com/crawlab-team/crawlab-core/apps.NewApi()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:123 github.com/crawlab-team/crawlab-core/apps.GetApi()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/server.go:120 github.com/crawlab-team/crawlab-core/apps.NewServer()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/server.go:148 github.com/crawlab-team/crawlab-core/apps.GetServer()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/cmd/server.go:22 github.com/crawlab-team/crawlab-core/cmd.glob..func1()
crawlab_master   | /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:860 github.com/spf13/cobra.(*Command).execute()
crawlab_master   | /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:974 github.com/spf13/cobra.(*Command).ExecuteC()
crawlab_master   | /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:902 github.com/spf13/cobra.(*Command).Execute()
crawlab_master   | /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/cmd/root.go:23 github.com/crawlab-team/crawlab-core/cmd.Execute()
crawlab_master   | /go/src/app/main.go:8 main.main()
crawlab_master   | /usr/local/go/src/runtime/proc.go:250 runtime.main()
crawlab_master   | /usr/local/go/src/runtime/asm_amd64.s:1571 runtime.goexit()
crawlab_master   | panic: could not build arguments for function "github.com/crawlab-team/crawlab-core/controllers".newUserContext.func1 (/go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:224): failed to build interfaces.UserService: received non-nil error from function "github.com/crawlab-team/crawlab-core/user".ProvideGetUserService.func1 (/go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/user/service.go:251): server selection error: server selection timeout, current topology: { Type: Unknown, Servers: [{ Addr: mongo:27017, Type: Unknown, Last error: connection() error occured during connection handshake: dial tcp 172.20.0.2:27017: connect: connection refused }, ] }
crawlab_master   | 
crawlab_master   | goroutine 1 [running]:
crawlab_master   | github.com/crawlab-team/crawlab-core/controllers.newUserContext()
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:231 +0x125
crawlab_master   | 
crawlab_master   | github.com/crawlab-team/crawlab-core/controllers.getUserActions()
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:25 +0x1d
crawlab_master   | github.com/crawlab-team/crawlab-core/controllers.newUserController()
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/user.go:243 +0x85
crawlab_master   | github.com/crawlab-team/crawlab-core/controllers.InitControllers()
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/controllers/init.go:18 +0xfc
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.initModule({0x1c7290f, 0xb}, 0x38?)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/utils.go:26 +0x30
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.(*Api).Init(0x416167?)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:39 +0x31
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.NewApi()
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:113 +0x56
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.GetApi(...)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/api.go:123
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.NewServer({0x0, 0x0, 0x0?})
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/server.go:120 +0x1fe
crawlab_master   | github.com/crawlab-team/crawlab-core/apps.GetServer(...)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/apps/server.go:148
crawlab_master   | github.com/crawlab-team/crawlab-core/cmd.glob..func1(0x2dd1a00?, {0x1c5f3c2?, 0x0?, 0x0?})
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/cmd/server.go:22 +0x37
crawlab_master   | github.com/spf13/cobra.(*Command).execute(0x2dd1a00, {0x2ebadb8, 0x0, 0x0})
crawlab_master   |      /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:860 +0x663
crawlab_master   | github.com/spf13/cobra.(*Command).ExecuteC(0x2dd1780)
crawlab_master   |      /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:974 +0x3b4
crawlab_master   | github.com/spf13/cobra.(*Command).Execute(...)
crawlab_master   |      /go/pkg/mod/github.com/spf13/cobra@v1.3.0/command.go:902
crawlab_master   | github.com/crawlab-team/crawlab-core/cmd.Execute(...)
crawlab_master   |      /go/pkg/mod/github.com/crawlab-team/crawlab-core@v0.6.3-0.20230804030437-664646cf1665/cmd/root.go:23
crawlab_master   | main.main()
crawlab_master   |      /go/src/app/main.go:8 +0x25
crawlab_master exited with code 0

1349

bqzzd1 commented 7 months ago

同样的问题，用k8s, 你解决了嘛

tikazyq commented 7 months ago

看样子是mongo没起来