Why training so slow - Githubissues

JiashengHong commented 6 years ago

I use 64 batch, each takes me nearly 40s. Is it normal? I use GTX-1050ti, and the GPU load is low. default Can you help me? Please!!! Thank you!!!

JiashengHong commented 6 years ago

My Makefile: GPU=1 CUDNN=1 CUDNN_HALF=0 OPENCV=1 AVX=0 OPENMP=0 LIBSO=0

# set GPU=1 and CUDNN=1 to speedup on GPU # set CUDNN_HALF=1 to further speedup 3 x times (Mixed-precision using Tensor Cores) on GPU Tesla V100, Titan V, DGX-2 # set AVX=1 and OPENMP=1 to speedup on CPU (if error occurs then set AVX=0)

DEBUG=1

ARCH= -gencode arch=compute_30,code=sm_30 / -gencode arch=compute_35,code=sm_35 / -gencode arch=compute_50,code=[sm_50,compute_50] / -gencode arch=compute_52,code=[sm_52,compute_52] / -gencode arch=compute_61,code=[sm_61,compute_61]

OS := $(shell uname)

# Tesla V100 # ARCH= -gencode arch=compute_70,code=[sm_70,compute_70]

# GTX 1080, GTX 1070, GTX 1060, GTX 1050, GTX 1030, Titan Xp, Tesla P40, Tesla P4 # ARCH= -gencode arch=compute_61,code=sm_61 -gencode arch=compute_61,code=compute_61

# GP100/Tesla P100 ?DGX-1 # ARCH= -gencode arch=compute_60,code=sm_60

# For Jetson TX1, Tegra X1, DRIVE CX, DRIVE PX - uncomment: # ARCH= -gencode arch=compute_53,code=[sm_53,compute_53]

# For Jetson Tx2 or Drive-PX2 uncomment: # ARCH= -gencode arch=compute_62,code=[sm_62,compute_62]

VPATH=./src/ EXEC=darknet OBJDIR=./obj/

ifeq ($(LIBSO), 1) LIBNAMESO=darknet.so APPNAMESO=uselib endif

CC=gcc CPP=g++ NVCC=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v8.0/bin/nvcc.exe OPTS=-Ofast LDFLAGS= -lm -pthread COMMON= CFLAGS=-Wall -Wfatal-errors -Wno-unused-result -Wno-unknown-pragmas

ifeq ($(DEBUG), 1) OPTS= -O0 -g else ifeq ($(AVX), 1) CFLAGS+= -ffp-contract=fast -mavx -msse4.1 -msse4a endif endif

CFLAGS+=$(OPTS)

ifeq ($(OPENCV), 1) COMMON+= -DOPENCV CFLAGS+= -DOPENCV LDFLAGS+= pkg-config --libs opencv COMMON+= pkg-config --cflags opencv endif

ifeq ($(OPENMP), 1) CFLAGS+= -fopenmp LDFLAGS+= -lgomp endif

ifeq ($(GPU), 1) COMMON+= -DGPU -I/usr/local/cuda/include/ CFLAGS+= -DGPU ifeq ($(OS),Darwin) #MAC LDFLAGS+= -L/usr/local/cuda/lib -lcuda -lcudart -lcublas -lcurand else LDFLAGS+= -L/usr/local/cuda/lib64 -lcuda -lcudart -lcublas -lcurand endif endif

ifeq ($(CUDNN), 1) COMMON+= -DCUDNN ifeq ($(OS),Darwin) #MAC CFLAGS+= -DCUDNN -I/usr/local/cuda/include LDFLAGS+= -L/usr/local/cuda/lib -lcudnn else CFLAGS+= -DCUDNN -I/usr/local/cudnn/include LDFLAGS+= -L/usr/local/cudnn/lib64 -lcudnn endif endif

ifeq ($(CUDNN_HALF), 1) COMMON+= -DCUDNN_HALF CFLAGS+= -DCUDNN_HALF ARCH+= -gencode arch=compute_70,code=[sm_70,compute_70] endif

OBJ=http_stream.o gemm.o utils.o cuda.o convolutional_layer.o list.o image.o activations.o im2col.o col2im.o blas.o crop_layer.o dropout_layer.o maxpool_layer.o softmax_layer.o data.o matrix.o network.o connected_layer.o cost_layer.o parser.o option_list.o darknet.o detection_layer.o captcha.o route_layer.o writing.o box.o nightmare.o normalization_layer.o avgpool_layer.o coco.o dice.o yolo.o detector.o layer.o compare.o classifier.o local_layer.o swag.o shortcut_layer.o activation_layer.o rnn_layer.o gru_layer.o rnn.o rnn_vid.o crnn_layer.o demo.o tag.o cifar.o go.o batchnorm_layer.o art.o region_layer.o reorg_layer.o reorg_old_layer.o super.o voxel.o tree.o yolo_layer.o upsample_layer.o ifeq ($(GPU), 1) LDFLAGS+= -lstdc++ OBJ+=convolutional_kernels.o activation_kernels.o im2col_kernels.o col2im_kernels.o blas_kernels.o crop_layer_kernels.o dropout_layer_kernels.o maxpool_layer_kernels.o network_kernels.o avgpool_layer_kernels.o endif

OBJS = $(addprefix $(OBJDIR), $(OBJ)) DEPS = $(wildcard src/*.h) Makefile

all: obj backup results $(EXEC) $(LIBNAMESO) $(APPNAMESO)

ifeq ($(LIBSO), 1) CFLAGS+= -fPIC

$(LIBNAMESO): $(OBJS) src/yolo_v2_class.hpp src/yolo_v2_class.cpp $(CPP) -shared -std=c++11 -fvisibility=hidden -DYOLODLL_EXPORTS $(COMMON) $(CFLAGS) $(OBJS) src/yolo_v2_class.cpp -o $@ $(LDFLAGS)

$(APPNAMESO): $(LIBNAMESO) src/yolo_v2_class.hpp src/yolo_console_dll.cpp $(CPP) -std=c++11 $(COMMON) $(CFLAGS) -o $@ src/yolo_console_dll.cpp $(LDFLAGS) -L ./ -l:$(LIBNAMESO) endif

$(EXEC): $(OBJS) $(CPP) $(COMMON) $(CFLAGS) $^ -o $@ $(LDFLAGS)

$(OBJDIR)%.o: %.c $(DEPS) $(CC) $(COMMON) $(CFLAGS) -c $< -o $@

$(OBJDIR)%.o: %.cpp $(DEPS) $(CPP) $(COMMON) $(CFLAGS) -c $< -o $@

$(OBJDIR)%.o: %.cu $(DEPS) $(NVCC) $(ARCH) $(COMMON) --compiler-options "$(CFLAGS)" -c $< -o $@

obj: mkdir -p obj backup: mkdir -p backup results: mkdir -p results

.PHONY: clean

clean: rm -rf $(OBJS) $(EXEC) $(LIBNAMESO) $(APPNAMESO)

TheMikeyR commented 6 years ago

Can you post your config? and training command?

JiashengHong commented 6 years ago

@TheMikeyR yolov3.cfg: [net] #Testing #batch=64 #subdivisions=64 Training batch=64 subdivisions=32 width=416 height=416 channels=3 momentum=0.9 decay=0.0005 angle=0 saturation = 1.5 exposure = 1.5 hue=.1

learning_rate=0.001 #burn_in=1000 max_batches = 500200 policy=steps steps=100,800,1000 scales=10,.1,.1

[convolutional] batch_normalize=1 filters=32 size=3 stride=1 pad=1 activation=leaky

# Downsample

[convolutional] batch_normalize=1 filters=64 size=3 stride=2 pad=1 activation=leaky

[convolutional] batch_normalize=1 filters=32 size=1 stride=1 pad=1 activation=leaky

[convolutional] batch_normalize=1 filters=64 size=3 stride=1 pad=1 activation=leaky