Go dependencies between v1 and v2 launching an experiment - [BUG]

[vincenzo-dip8]

Describe the bug I'm trying to use the instructions in the https://diablobench.github.io/fresh-install at the Install yourself sections and use the final command

./bin/eurosys workload-native-10.yaml setup.txt

I’m facing several issues with go during the deploy, because minion (v2) set the go version to 1.17.8 (while in diablo-v1 use 1.14 or greater). The step is when v2 take from repo v1 the Makefile and make these step (when minion use install-diablo script):

go mod tidy
make reqs
make diablo

what I noticed are problems with: golang.org/x/lint golang.org/x/tools/go/ast/astutil golang.org/x/tools/internal/typeparams golang.org/x/tools/go/gcexportdata

Screenshots

Part of the output.log file

GO111MODULE=off GO111MODULE=off go get -v golang.org/x/lint/golint
get "golang.org/x/lint/golint": found meta tag vcs.metaImport{Prefix:"golang.org/x/lint", VCS:"git", RepoRoot:"https://go.googlesource.com/lint"} at //golang.org/x/lint/golint?go-get=1
get "golang.org/x/lint/golint": verifying non-authoritative meta tag
golang.org/x/lint (download)
created GOPATH=/root/install/diablo/.go; see 'go help gopath'
get "golang.org/x/tools/go/ast/astutil": found meta tag vcs.metaImport{Prefix:"golang.org/x/tools", VCS:"git", RepoRoot:"https://go.googlesource.com/tools"} at //golang.org/x/tools/go/ast/astutil?go-get=1
get "golang.org/x/tools/go/ast/astutil": verifying non-authoritative meta tag
golang.org/x/tools (download)
get "golang.org/x/tools/go/gcexportdata": found meta tag vcs.metaImport{Prefix:"golang.org/x/tools", VCS:"git", RepoRoot:"https://go.googlesource.com/tools"} at //golang.org/x/tools/go/gcexportdata?go-get=1
get "golang.org/x/tools/go/gcexportdata": verifying non-authoritative meta tag
golang.org/x/tools/internal/tokeninternal
golang.org/x/tools/go/ast/astutil
golang.org/x/tools/internal/typeparams
# golang.org/x/tools/internal/typeparams
.go/src/golang.org/x/tools/internal/typeparams/common.go:45:8: undefined: ast.IndexListExpr
.go/src/golang.org/x/tools/internal/typeparams/common.go:66:11: undefined: ast.IndexListExpr
.go/src/golang.org/x/tools/internal/typeparams/common.go:160:32: undefined: types.Context
.go/src/golang.org/x/tools/internal/typeparams/coretype.go:111:39: undefined: types.Term
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:63:30: undefined: types.TypeParam
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:63:51: undefined: types.Term
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:81:51: undefined: types.Term
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:91:26: undefined: types.Union
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:91:43: undefined: types.Term
.go/src/golang.org/x/tools/internal/typeparams/normalize.go:95:41: undefined: types.Term
.go/src/golang.org/x/tools/internal/typeparams/common.go:66:11: too many errors
# golang.org/x/tools/go/ast/astutil
.go/src/golang.org/x/tools/go/ast/astutil/enclosing.go:323:23: n.Type.TypeParams undefined (type *ast.FuncType has no field or method TypeParams)
.go/src/golang.org/x/tools/go/ast/astutil/enclosing.go:378:8: undefined: ast.IndexListExpr
.go/src/golang.org/x/tools/go/ast/astutil/enclosing.go:589:8: undefined: ast.IndexListExpr
.go/src/golang.org/x/tools/go/ast/astutil/rewrite.go:253:8: undefined: ast.IndexListExpr
.go/src/golang.org/x/tools/go/ast/astutil/rewrite.go:294:18: n.TypeParams undefined (type *ast.FuncType has no field or method TypeParams)
.go/src/golang.org/x/tools/go/ast/astutil/rewrite.go:409:18: n.TypeParams undefined (type *ast.TypeSpec has no field or method TypeParams)
make: *** [Makefile:15: reqs] Error 2
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.11:22.sock -p 22 root@10.1.0.11 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.7:22.sock -p 22 root@10.1.0.7 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.16:22.sock -p 22 root@10.1.0.16 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.10:22.sock -p 22 root@10.1.0.10 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.9:22.sock -p 22 root@10.1.0.9 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.15:22.sock -p 22 root@10.1.0.15 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.14:22.sock -p 22 root@10.1.0.14 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.12:22.sock -p 22 root@10.1.0.12 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.13:22.sock -p 22 root@10.1.0.13 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.8:22.sock -p 22 root@10.1.0.8 'rm' '.minion-task'
ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ControlMaster=auto -o ControlPersist=60 -o ControlPath=/tmp/minion-ssh.root@10.1.0.2:22.sock -p 22 root@10.1.0.2 'rm' '.minion-task'
./bin/eurosys: failed to install diablo
Please type './bin/eurosys --help' for more information

Docker Container Environment OS: Both Ubuntu 20 and 22.04

Others I'm trying to understand how you use minion (v2) in the paper at https://arxiv.org/abs/2311.09440, because i notice that from the "minion paper" you switch from AWS to a simple testbed that i think use just SSH.

Since i'm going to do the same, i need to understand the steps to not use AWS. In particular, are the guidelines given in the FreshInstallation/InstallYourself section useful? With setup.txt file to be used in order to execute the experiment i need on more than one machine/container simply with ssh @NatoliChris @gauthier-voron

Also, I'm using the Dockerfile below to solve other dependencies issues that happen during the deploy, but then i stop on the bug above.

FROM ubuntu:20.04

ENV DEBIAN_FRONTEND=noninteractive

RUN apt-get update -y && \
    apt-get install -y \
        openssh-client openssh-server \
        net-tools netcat \
        cloud-utils \
        curl wget \
        git python3-pip \
        bash \
        perl-base libconfig-yaml-perl libcapture-tiny-perl libbz2-dev libreadline-dev libssl-dev libffi-dev libsqlite3-dev libncurses5-dev libncursesw5-dev liblzma-dev \
        sudo

SHELL ["/bin/bash", "-c"]

RUN echo "ALL ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/nopasswd

COPY id_ed25519 /root/.ssh/id_ed25519
COPY id_ed25519.pub /root/.ssh/id_ed25519.pub
COPY id_ed25519.pub /root/.ssh/authorized_keys

RUN echo "StrictHostKeyChecking no" >> /root/.ssh/config

RUN chmod 700 /root/.ssh && \
    chmod 600 /root/.ssh/id_ed25519.pub && \
    chmod 600 /root/.ssh/authorized_keys && \ 
    chmod 600 /root/.ssh/config

RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config && \
    sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config 

EXPOSE 22

RUN curl -L https://install.perlbrew.pl | bash
RUN source /root/perl5/perlbrew/etc/bashrc
RUN exec bash; \
    perlbrew install --notest --force perl-5.34.0; \
    perlbrew switch perl-5.34.0

# Install other Perl dependencies
RUN wget -O YAML-1.24.tar.gz http://search.cpan.org/CPAN/authors/id/T/TI/TINITA/YAML-1.24.tar.gz; \
    tar -xzvf YAML-1.24.tar.gz; \
    cd YAML-1.24; \
    perl Makefile.PL; \
    make; \
    make install; \
    cd /; \
    wget -O IO-Tee-0.65.tar.gz http://search.cpan.org/CPAN/authors/id/N/NE/NEILB/IO-Tee-0.65.tar.gz; \
    tar -xzvf IO-Tee-0.65.tar.gz; \
    cd IO-Tee-0.65; \
    perl Makefile.PL; \
    make; \
    make install; \
    cd /; \
    cpan -i Capture::Tiny; \
    cpan -i XML::Simple; \
    cpan YAML;  
    cpan JSON

RUN curl https://pyenv.run | bash; \
    echo 'export PATH="$HOME/.pyenv/bin:$PATH"' >> /root/.bashrc; \
    echo 'eval "$(pyenv init --path)"' >> /root/.bashrc; \
    echo 'eval "$(pyenv virtualenv-init -)"' >> /root/.bashrc; \
    source /root/.bashrc
RUN exec bash; \
    pyenv install -s 3.10.6; \
    pyenv global 3.10.6; \
    pip install PyYAML

RUN apt-get clean; rm -rf /var/lib/apt/lists/

[vincenzo-dip8]

I tried installing all these versions of Go:

• 1.14
• 1.17.8
• 1.21.6

The results was always the same showed above

[vincenzo-dip8]

I found that the problem is related to

make reqs

I'm referring to the aec branch used in the Fresh Installation guide. Simply commenting that code line i was able to move on with the experiment. So, i want to recap the change i made in otder to run the bin/eurosys:

• scripts/script/remote/linux/apt/install-diablo: comment on make reqs
• scripts/script/remote/linux/apt/install-geth-accounts-worker: update the go version to 1.20.5
• scripts/script/remote/linux/apt/install-solana: after ./scripts/cargo-install-all.sh --validator-only . added export PATH="/install/solana"/bin:"$PATH"

[vincenzo-dip8]

Fixes i notice (on the 'aec' branch):

• scripts/script/remote/linux/apt/install-diablo: comment on make reqs
• scripts/script/remote/linux/apt/install-geth-accounts-worker: update the go version to 1.20.5
• scripts/script/remote/linux/apt/install-solana: after "./scripts/cargo-install-all.sh --validator-only ." added "export PATH="/install/solana"/bin:"$PATH""

To FIX:

• scripts/script/remote/linux/apt/install-avalanche: the checkout for 'aec' seems to not exist (as tag), we have to pass to go version 1.20.10, install jq, always get error when go get ..avalanche-network-runner@1d61e94 where

  HEAD is now at 7840200f8 Merge pull request #1026 from ava-labs/dev
  fatal: no tag exactly matches '7840200f865b6d6629866330d5fdf95bd4010ae9'
  Downloading dependencies...
  go: github.com/ava-labs/avalanchego-operator@v0.0.0-20211115144351-99f07d2570bf: invalid version: git ls-remote -q origin in /install/avalanche/go/pkg/mod/cache/vcs/0d4bed232e5a07a14f862c3021d2ff8945d42678ff73dd641e3d6ea0cc529f27: exit status 128:
      fatal: could not read Username for 'https://github.com': terminal prompts disabled
  Confirm the import path was entered correctly.
  If this is a private repository, see https://golang.org/doc/faq#git_https for additional information.

otherwise, by simply commenting the @xxxxxxxxx, then

# avalanche-tools
./main.go:124:37: cannot use nodeID (variable of type ids.NodeID) as ids.ShortID value in argument to append
./main.go:144:37: nodeID.PrefixedString undefined (type ids.NodeID has no field or method PrefixedString)
./main.go:144:62: undefined: constants.NodeIDPrefix
./main.go:166:13: cannot use 0x1337133713371337 (untyped int constant 1384596537706222391) as *"math/big".Int value in struct literal
./main.go:175:3: too many arguments in call to network.NewAvalancheGoGenesis
        have (logging.NoLog, uint32, nil, []"github.com/ava-labs/avalanche-network-runner/network".AddrAndBalance, []ids.ShortID)
        want (uint32, []"github.com/ava-labs/avalanche-network-runner/network".AddrAndBalance, []"github.com/ava-labs/avalanche-network-runner/network".AddrAndBalance, []ids.NodeID)

• scripts/script/remote/linux/apt/install-solang: change the final dir on "cmake -G Ninja -DLLVM:..." from llvm to ../llvm, still fatal error on last cargo build --release where:

  
  The following warnings were emitted during compilation:

warning: src/linker/linker.cpp:2:10: fatal error: lld/Common/Driver.h: No such file or directory warning: 2 | #include "lld/Common/Driver.h" warning: | ^~~~~ warning: compilation terminated.

error: failed to run custom build command for solang v0.1.11 (/install/solang)

Caused by: process didn't exit successfully: /install/solang/target/release/build/solang-e2cdb07673293bdd/build-script-build (exit status: 1) --- stdout TARGET = Some("x86_64-unknown-linux-gnu") OPT_LEVEL = Some("3") HOST = Some("x86_64-unknown-linux-gnu") cargo:rerun-if-env-changed=CXX_x86_64-unknown-linux-gnu CXX_x86_64-unknown-linux-gnu = None cargo:rerun-if-env-changed=CXX_x86_64_unknown_linux_gnu CXX_x86_64_unknown_linux_gnu = None cargo:rerun-if-env-changed=HOST_CXX HOST_CXX = None cargo:rerun-if-env-changed=CXX CXX = None cargo:rerun-if-env-changed=CRATE_CC_NO_DEFAULTS CRATE_CC_NO_DEFAULTS = None DEBUG = Some("false") CARGO_CFG_TARGET_FEATURE = Some("fxsr,sse,sse2") cargo:rerun-if-env-changed=CXXFLAGS_x86_64-unknown-linux-gnu CXXFLAGS_x86_64-unknown-linux-gnu = None cargo:rerun-if-env-changed=CXXFLAGS_x86_64_unknown_linux_gnu CXXFLAGS_x86_64_unknown_linux_gnu = None cargo:rerun-if-env-changed=HOST_CXXFLAGS HOST_CXXFLAGS = None cargo:rerun-if-env-changed=CXXFLAGS CXXFLAGS = None running: "c++" "-O3" "-ffunction-sections" "-fdata-sections" "-fPIC" "-m64" "-Wall" "-Wextra" "-Wno-unused-parameter" "-I/usr/lib/llvm-10/include" "-std=c++14" "-fno-exceptions" "-D_GNU_SOURCE" "-DSTDC_CONSTANT_MACROS" "-DSTDC_FORMAT_MACROS" "-D__STDC_LIMIT_MACROS" "-o" "/install/solang/target/release/build/solang-83c3d7e7774b6631/out/src/linker/linker.o" "-c" "src/linker/linker.cpp" cargo:warning=src/linker/linker.cpp:2:10: fatal error: lld/Common/Driver.h: No such file or directory

cargo:warning= 2 | #include "lld/Common/Driver.h"

cargo:warning= | ^~~~~

cargo:warning=compilation terminated.

exit status: 1

--- stderr

error occurred: Command "c++" "-O3" "-ffunction-sections" "-fdata-sections" "-fPIC" "-m64" "-Wall" "-Wextra" "-Wno-unused-parameter" "-I/usr/lib/llvm-10/include" "-std=c++14" "-fno-exceptions" "-D_GNU_SOURCE" "-DSTDC_CONSTANT_MACROS" "-DSTDC_FORMAT_MACROS" "-D__STDC_LIMIT_MACROS" "-o" "/install/solang/target/release/build/solang-83c3d7e7774b6631/out/src/linker/linker.o" "-c" "src/linker/linker.cpp" with args "c++" did not execute successfully (status code exit status: 1).

warning: build failed, waiting for other jobs to finish...