eleven-pixel commented 1 year ago

~/supra_seal$ ./exec.sh

SECTOR_SIZE=32GiB
'[' '' == 512MiB ']'
./build.sh 32GiB
SECTOR_SIZE=-DSECTOR_SIZE_32GiB
'[' 32GiB == 512MiB ']'
CC=cc
CXX=c++
NVCC=nvcc ++++ which nvcc +++ dirname /usr/local/cuda-12.2/bin/nvcc ++ dirname /usr/local/cuda-12.2/bin
CUDA=/usr/local/cuda-12.2
SPDK=deps/spdk-v22.09
CUDA_ARCH='-arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0' ++ c++ -dM -E -x c++ /dev/null ++ awk '{ if($2=="__cplusplus" && $3<"2017") print "-std=c++17"; }'
CXXSTD=
INCLUDE='-Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include'
CFLAGS='-DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX'
CXXFLAGS='-DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread'
LDFLAGS='-fno-omit-frame-pointer -Wl,-z,relro,-z,now -Wl,-z,noexecstack -fuse-ld=bfd -Ldeps/spdk-v22.09/build/lib -Wl,--whole-archive -Wl,--no-as-needed -lspdk_bdev_malloc -lspdk_bdev_null -lspdk_bdev_nvme -lspdk_bdev_passthru -lspdk_bdev_lvol -lspdk_bdev_raid -lspdk_bdev_error -lspdk_bdev_gpt -lspdk_bdev_split -lspdk_bdev_delay -lspdk_bdev_zone_block -lspdk_blobfs_bdev -lspdk_blobfs -lspdk_blob_bdev -lspdk_lvol -lspdk_blob -lspdk_nvme -lspdk_bdev_ftl -lspdk_ftl -lspdk_bdev_aio -lspdk_bdev_virtio -lspdk_virtio -lspdk_vfio_user -lspdk_accel_ioat -lspdk_ioat -lspdk_scheduler_dynamic -lspdk_env_dpdk -lspdk_scheduler_dpdk_governor -lspdk_scheduler_gscheduler -lspdk_sock_posix -lspdk_event -lspdk_event_bdev -lspdk_bdev -lspdk_notify -lspdk_dma -lspdk_event_accel -lspdk_accel -lspdk_event_vmd -lspdk_vmd -lspdk_event_sock -lspdk_init -lspdk_thread -lspdk_trace -lspdk_sock -lspdk_rpc -lspdk_jsonrpc -lspdk_json -lspdk_util -lspdk_log -Wl,--no-whole-archive deps/spdk-v22.09/build/lib/libspdk_env_dpdk.a -Wl,--whole-archive deps/spdk-v22.09/dpdk/build/lib/librte_bus_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_cryptodev.a deps/spdk-v22.09/dpdk/build/lib/librte_dmadev.a deps/spdk-v22.09/dpdk/build/lib/librte_eal.a deps/spdk-v22.09/dpdk/build/lib/librte_ethdev.a deps/spdk-v22.09/dpdk/build/lib/librte_hash.a deps/spdk-v22.09/dpdk/build/lib/librte_kvargs.a deps/spdk-v22.09/dpdk/build/lib/librte_mbuf.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_net.a deps/spdk-v22.09/dpdk/build/lib/librte_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_power.a deps/spdk-v22.09/dpdk/build/lib/librte_rcu.a deps/spdk-v22.09/dpdk/build/lib/librte_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_telemetry.a deps/spdk-v22.09/dpdk/build/lib/librte_vhost.a -Wl,--no-whole-archive -lnuma -ldl -Ldeps/spdk-v22.09/isa-l/.libs -lisal -pthread -lrt -luuid -lssl -lcrypto -lm -laio'
'[' '!' -d /var/tmp/supra_seal ']'
rm -fr obj
mkdir -p obj
rm -fr bin
mkdir -p bin
mkdir -p deps
'[' '!' -d deps/spdk-v22.09 ']'
'[' '!' -d deps/sppark ']'
'[' '!' -d deps/blst ']'
'[' '!' -d c2/bellperson ']'
cc -c sha/sha_ext_mbx2.S -o obj/sha_ext_mbx2.o
xxd -i poseidon/constants/constants_2
xxd -i poseidon/constants/constants_4
xxd -i poseidon/constants/constants_8
xxd -i poseidon/constants/constants_11
xxd -i poseidon/constants/constants_16
xxd -i poseidon/constants/constants_24
xxd -i poseidon/constants/constants_36
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/sppark/util -o obj/pc1.o -c pc1/pc1.cpp
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -o obj/streaming_node_reader_nvme.o -c nvme/streaming_node_reader_nvme.cpp
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -o obj/ring_t.o -c nvme/ring_t.cpp
c++ -g -O2 -c sealing/sector_parameters.cpp -o obj/sector_parameters.o
nvcc -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -DNO_SPDK -Xcompiler -Wno-subobject-linkage -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -dc pc2/cuda/pc2.cu -o obj/pc2.o
nvcc -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -DNO_SPDK -Xcompiler -Wno-subobject-linkage -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -dlink pc2/cuda/pc2.cu -o obj/pc2_link.o
wait
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -c sealing/supra_seal.cpp -o obj/supra_seal.o -Wno-subobject-linkage
ar rvs obj/libsupraseal.a obj/pc1.o obj/pc2.o obj/pc2_link.o obj/ring_t.o obj/streaming_node_reader_nvme.o obj/supra_seal.o obj/sector_parameters.o obj/sha_ext_mbx2.o ar: creating obj/libsupraseal.a a - obj/pc1.o a - obj/pc2.o a - obj/pc2_link.o a - obj/ring_t.o a - obj/streaming_node_reader_nvme.o a - obj/supra_seal.o a - obj/sector_parameters.o a - obj/sha_ext_mbx2.o
c++ -DSECTOR_SIZE_32GiB -g -O2 -Ideps/spdk-v22.09/include -Ideps/spdk-v22.09/isa-l/.. -Ideps/spdk-v22.09/dpdk/build/include -DADX -fno-omit-frame-pointer -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers -fno-strict-aliasing -march=native -Wformat -Wformat-security -D_GNU_SOURCE -fPIC -fstack-protector -fno-common -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2 -DSPDK_GIT_COMMIT=4be6d3043 -pthread -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -o bin/seal demos/main.cpp -Lobj -lsupraseal -fno-omit-frame-pointer -Wl,-z,relro,-z,now -Wl,-z,noexecstack -fuse-ld=bfd -Ldeps/spdk-v22.09/build/lib -Wl,--whole-archive -Wl,--no-as-needed -lspdk_bdev_malloc -lspdk_bdev_null -lspdk_bdev_nvme -lspdk_bdev_passthru -lspdk_bdev_lvol -lspdk_bdev_raid -lspdk_bdev_error -lspdk_bdev_gpt -lspdk_bdev_split -lspdk_bdev_delay -lspdk_bdev_zone_block -lspdk_blobfs_bdev -lspdk_blobfs -lspdk_blob_bdev -lspdk_lvol -lspdk_blob -lspdk_nvme -lspdk_bdev_ftl -lspdk_ftl -lspdk_bdev_aio -lspdk_bdev_virtio -lspdk_virtio -lspdk_v+ c++ -pthread -g -O3 -march=native -Wall -Wextra -Werror -Wno-subobject-linkage tools/tree_r.cpp -o bin/tree_r_cpu -Iposeidon -Ideps/sppark -Ideps/blst/src -L deps/blst -lblst fio_user -lspdk_accel_ioat -lspdk_ioat -lspdk_scheduler_dynamic -lspdk_env_dpdk -lspdk_scheduler_dpdk_governor -lspdk_scheduler_gscheduler -lspdk_sock_posix -lspdk_event -lspdk_event_bdev -lspdk_bdev -lspdk_notify -lspdk_dma -lspdk_event_accel -lspdk_accel -lspdk_event_vmd -lspdk_vmd -lspdk_event_sock -lspdk_init -lspdk_thread -lspdk_trace -lspdk_sock -lspdk_rpc -lspdk_jsonrpc -lspdk_json -lspdk_util -lspdk_log -Wl,--no-whole-archive deps/spdk-v22.09/build/lib/libspdk_env_dpdk.a -Wl,--whole-archive deps/spdk-v22.09/dpdk/build/lib/librte_bus_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_cryptodev.a deps/spdk-v22.09/dpdk/build/lib/librte_dmadev.a deps/spdk-v22.09/dpdk/build/lib/librte_eal.a deps/spdk-v22.09/dpdk/build/lib/librte_ethdev.a deps/spdk-v22.09/dpdk/build/lib/librte_hash.a deps/spdk-v22.09/dpdk/build/lib/librte_kvargs.a deps/spdk-v22.09/dpdk/build/lib/librte_mbuf.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool.a deps/spdk-v22.09/dpdk/build/lib/librte_mempool_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_net.a deps/spdk-v22.09/dpdk/build/lib/librte_pci.a deps/spdk-v22.09/dpdk/build/lib/librte_power.a deps/spdk-v22.09/dpdk/build/lib/librte_rcu.a deps/spdk-v22.09/dpdk/build/lib/librte_ring.a deps/spdk-v22.09/dpdk/build/lib/librte_telemetry.a deps/spdk-v22.09/dpdk/build/lib/librte_vhost.a -Wl,--no-whole-archive -lnuma -ldl -Ldeps/spdk-v22.09/isa-l/.libs -lisal -pthread -lrt -luuid -lssl -lcrypto -lm -laio -Ldeps/blst -lblst -L/usr/local/cuda-12.2/lib64 -lcudart_static -lgmp -lconfig++
wait
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -g -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -Xcompiler -march=native -O3 -x cu tools/tree_r.cpp -o bin/tree_r -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -g -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -Xcompiler -march=native -O3 -x cu tools/pc2.cu -o bin/pc2 -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
nvcc -DSECTOR_SIZE_32GiB -DNO_SPDK -DSTREAMING_NODE_READER_FILES -arch=sm_80 -gencode arch=compute_70,code=sm_70 -t0 -std=c++17 -g -O3 -Xcompiler -march=native -Xcompiler -Wall -Xcompiler -Wextra -Xcompiler -Werror -Xcompiler -Wno-subobject-linkage -Xcompiler -Wno-unused-parameter -x cu tools/tree_r.cpp -o bin/tree_r -Iposeidon -Ideps/sppark -Ideps/sppark/util -Ideps/blst/src -L deps/blst -lblst -lconfig++
SECTOR_SIZE_FEATURE=
'[' 32GiB == 512MiB ']'
cd demos/rust
env 'RUSTFLAGS=-C target-cpu=native' cargo +nightly build --release Compiling supra-seal-demo v0.1.0 (/home/worker/supra_seal/demos/rust) Finished release [optimized] target(s) in 4.16s
sudo ./target/release/supra-seal-demo Initializing spdk using config supra_seal.cfg sectors 2 coord 8 hashers 1 sectors 4 coord 8 hashers 2 sectors 8 coord 8 hashers 4 sectors 16 coord 8 hashers 8 sectors 32 coord 8 hashers 14 coord 16 hashers 2 sectors 64 coord 8 hashers 14 coord 16 hashers 14 coord 24 hashers 4 sectors 128 coord 8 hashers 14 coord 16 hashers 14 coord 24 hashers 14 coord 32 hashers 14 coord 40 hashers 8 TELEMETRY: No legacy callbacks, legacy socket not created Attaching to 0000:01:00.0 Sealing Parameters SECTOR_SIZE 34359738368 NODE_SIZE 32 NODE_WORDS 8 NODE_COUNT 1073741824 PARENT_COUNT_BASE 6 PARENT_COUNT_EXP 8 PARENT_COUNT 14 PARENT_SIZE 4 LAYER_COUNT 11 NODES_PER_HASHER 2 PARENT_BUFFER_NODES 3407872 NODE_BUFFER_NODES 524288

Max Offset returned 6fc7d256 Slot size returned 2c0000000 for 128 sectors max_offset 1875366486 and slot_size 11811160064 2023-07-24 15:35:48 1690184148 Batch 0 locked slot 0 2023-07-24 15:35:48 1690184148 Batch 0 start PC1 supra-seal-demo: pc1/../nvme/spdk_ptr_t.hpp:29: void spdk_ptr_t::alloc(size_t) [with T = batch_t<page_t<sealing_config_t<128> >, 13>; size_t = long unsigned int]: Assertion `ptr != nullptr' failed. ./exec.sh: line 22: 53222 Aborted sudo ./target/release/supra-seal-demo

How do i fix this error？

OS:Ubuntu 20.04.3 LTS

GCC:gcc version 11.4.0 (Ubuntu 11.4.0-2ubuntu1~20.04)

simonatsn commented 1 year ago

It looks like spdk failed to allocate storage for PC1. Can you check for the proper allocation of huge pages? How much memory is in the system overall?

grep Huge /proc/meminfo

Look for:

HugePages_Total: 128 HugePages_Free: 128

eleven-pixel commented 1 year ago

I have already set up huge page

:~/supra_seal$ grep Huge /proc/meminfo AnonHugePages: 0 kB ShmemHugePages: 0 kB FileHugePages: 0 kB HugePages_Total: 128 HugePages_Free: 128 HugePages_Rsvd: 0 HugePages_Surp: 0 Hugepagesize: 2048 kB Hugetlb: 262144 kB

The system has a total of 1T of memory

eleven-pixel commented 1 year ago

sudo ./scripts/setup.sh status Hugepages node hugesize free / total node0 1048576kB 0 / 0 node0 2048kB 128 / 128

Type BDF Vendor Device NUMA Driver Device Block devices NVMe 0000:01:00.0 1b96 2400 0 uio_pci_generic - - NVMe 0000:02:00.0 1b96 2400 0 nvme nvme1 nvme1n1 NVMe 0000:03:00.0 144d a808 0 nvme nvme2 nvme2n1

Is it because there are two node0?

simonatsn commented 1 year ago

It could be. If you end up on the node with no pages allocated it seems like that would be a problem. Can you allocate pages on both nodes, or direct it to run on the node with pages allocated?

eleven-pixel commented 1 year ago

Try it later, thank you

dot-asm commented 1 year ago

If problem persists, just re-open the issue. Cheers.

supranational / supra_seal

Assertion `ptr != nullptr' failed #16

Look for: