Why do more threads take longer?

[csrdxbb]

Today I test the NNPACK in ARMv8 machine, I found when I improve the thread number, the time increases. I am very confused and I am no sure what is the problem, the script just like this:

#include <iostream>
#include <sys/time.h>

#include <vector>
#include "nnpack.h"

using namespace std;

float test_nnpack(size_t bs, size_t threads)
{
    enum nnp_status init_status = nnp_initialize();
    if (init_status != nnp_status_success)
    {
        return 0;
    }

    enum nnp_convolution_algorithm algorithm;
    enum nnp_convolution_transform_strategy strategy=nnp_convolution_transform_strategy_tuple_based;
    const size_t batch_size = 1;
    const size_t input_channels = 16;
    const size_t output_channels = 16;
        const size_t kernel_num = 3;
    const struct nnp_padding input_padding = {1, 1, 1, 1};
    const struct nnp_size input_size = {224, 224};
    const struct nnp_size kernel_size = {3, 3};
    const struct nnp_size stride = {.width=1, .height=1};
    const struct nnp_size output_size = {
        .width = (input_padding.left + input_size.width + input_padding.right - kernel_size.width)/stride.width + 1,
        .height = (input_padding.top + input_size.height + input_padding.bottom - kernel_size.height)/stride.height + 1
    };
    float *input, *kernel, *output, *bias;

    input = (float *)malloc(batch_size * input_channels * input_size.height * input_size.width * sizeof(float));
    kernel = (float *)malloc(input_channels * output_channels * kernel_size.height * kernel_size.width * sizeof(float));
    output = (float *)malloc(batch_size * output_channels * output_size.height * output_size.width * sizeof(float));
    bias = (float *)malloc(output_channels * sizeof(float));

    pthreadpool_t threadpool = nullptr;
        if (true) {   
            threadpool = pthreadpool_create(32);  
            printf("Threads: %zu\n", pthreadpool_get_threads_count(threadpool));  
        }   

    struct nnp_profile computation_profile;

    int i, j, c, iter;
    struct timeval start, end;

    for (c = 0; c < input_channels; c++)
    {
        for (i = 0; i < input_size.height; i++)
        {
            for (j = 0; j < input_size.width; j++)
            {
                input[c * input_size.height * input_size.width + i * input_size.width + j] = (i * input_size.width + j) * 0.1;
            }
        }
    }

    for(i = 0; i < output_channels; i++)
    {
        for (j = 0; j < input_channels * kernel_size.height * kernel_size.width; j++)
        {
            kernel[i * input_channels * kernel_size.height * kernel_size.width + j] = 0.1;
        }
    }

    for (i = 0; i < output_channels; i++)
    {
        bias[i] = 1.0;
    }

        iter = 1;
    gettimeofday(&start, nullptr); 
    for (i = 0; i < iter; i++)
    {
        algorithm = nnp_convolution_algorithm_wt8x8;
        nnp_convolution_output(algorithm,
                                  batch_size,
                                  input_channels,
                                  output_channels,
                                  input_size,
                                  input_padding,
                                  kernel_size,
                                  input,
                                  kernel,
                                  bias,
                                  output,
                                  threadpool,
                                  nullptr);
    }
    gettimeofday(&end, nullptr);
    long second = end.tv_sec - start.tv_sec;
    long usecond = end.tv_usec - start.tv_usec;
    float mtime = (second * 1000 + usecond / 1000.0);
    cout << "Winograd convolution elapsed time:" << mtime << "ms"  << endl;
        cout << output[10] << endl;

    gettimeofday(&start, nullptr);
    for (i = 0; i < iter; i++)
    {
        algorithm = nnp_convolution_algorithm_ft8x8;
        nnp_convolution_output(algorithm,
                                  batch_size,
                                  input_channels,
                                  output_channels,
                                  input_size,
                                  input_padding,
                                  kernel_size,
                                  input,
                                  kernel,
                                  bias,
                                  output,
                                  threadpool,
                                  nullptr);
    }
    gettimeofday(&end, nullptr);
    second = end.tv_sec - start.tv_sec;
    usecond = end.tv_usec - start.tv_usec;
    mtime = (second * 1000 + usecond / 1000.0);
    cout << "FFT8x8 convolution elapsed time:" << mtime << "ms"  << endl;
        cout << output[10] << endl;        

    gettimeofday(&start, nullptr);
    for (i = 0; i < iter; i++)
    {
        algorithm = nnp_convolution_algorithm_implicit_gemm;
        nnp_convolution_output(algorithm,
                                  batch_size,
                                  input_channels,
                                  output_channels,
                                  input_size,
                                  input_padding,
                                  kernel_size,
                                  input,
                                  kernel,
                                  bias,
                                  output,
                                  threadpool,
                                  nullptr);
    }
    gettimeofday(&end, nullptr);
    second = end.tv_sec - start.tv_sec;
    usecond = end.tv_usec - start.tv_usec;
    mtime = (second * 1000 + usecond / 1000.0);
    cout << "GEMM convolution elapsed time:" << mtime << "ms"  << endl;
        cout << output[10] << endl; 

    gettimeofday(&start, nullptr);
    for (i = 0; i < iter; i++)
    {
        algorithm = nnp_convolution_algorithm_direct;
        nnp_convolution_output(algorithm,
                                  batch_size,
                                  input_channels,
                                  output_channels,
                                  input_size,
                                  input_padding,
                                  kernel_size,
                                  input,
                                  kernel,
                                  bias,
                                  output,
                                  threadpool,
                                  nullptr);
    }
    gettimeofday(&end, nullptr);
    second = end.tv_sec - start.tv_sec;
    usecond = end.tv_usec - start.tv_usec;
    mtime = (second * 1000 + usecond / 1000.0);
    cout << "Direct convolution elapsed time:" << mtime << "ms"  << endl;
        cout << output[10] << endl; 

    return 0;
}
int main(int argc, char* argv[])
{
        size_t batch_size = atoi(argv[1]);
        size_t thread_num = atoi(argv[2]);
    test_nnpack(batch_size,thread_num);
    return 0;
}

My machine are numa arch, however I am sure the 32 threads run on same node so there is no numa remote access issue. Please tell me how to improve the time ? Thanks!