search

open-compass / opencompass

OpenCompass is an LLM evaluation platform, supporting a wide range of models (Llama3, Mistral, InternLM2,GPT-4,LLaMa2, Qwen,GLM, Claude, etc) over 100+ datasets.
https://opencompass.org.cn/
Apache License 2.0
4.1k stars 436 forks source link

[Bug] 按文档使用gpt3.5 测试数据集报错 #1270

Open luckfu opened 4 months ago

luckfu commented 4 months ago

先决条件

问题类型

我正在使用官方支持的任务/模型/数据集进行评估。

环境

/bin/sh: 1: /usr/local/cuda/bin/nvcc: not found
/bin/sh: 1: gcc: not found
{'CUDA available': True,
 'CUDA_HOME': '/usr/local/cuda',
 'GCC': 'n/a',
 'GPU 0,1': 'NVIDIA A800 80GB PCIe',
 'MMEngine': '0.10.4',
 'MUSA available': False,
 'NVCC': 'Not Available',
 'PyTorch': '2.3.0+cu121',
 'PyTorch compiling details': 'PyTorch built with:\n'
                              '  - GCC 9.3\n'
                              '  - C++ Version: 201703\n'
                              '  - Intel(R) oneAPI Math Kernel Library Version '
                              '2022.2-Product Build 20220804 for Intel(R) 64 '
                              'architecture applications\n'
                              '  - Intel(R) MKL-DNN v3.3.6 (Git Hash '
                              '86e6af5974177e513fd3fee58425e1063e7f1361)\n'
                              '  - OpenMP 201511 (a.k.a. OpenMP 4.5)\n'
                              '  - LAPACK is enabled (usually provided by '
                              'MKL)\n'
                              '  - NNPACK is enabled\n'
                              '  - CPU capability usage: AVX2\n'
                              '  - CUDA Runtime 12.1\n'
                              '  - NVCC architecture flags: '
                              '-gencode;arch=compute_50,code=sm_50;-gencode;arch=compute_60,code=sm_60;-gencode;arch=compute_70,code=sm_70;-gencode;arch=compute_75,code=sm_75;-gencode;arch=compute_80,code=sm_80;-gencode;arch=compute_86,code=sm_86;-gencode;arch=compute_90,code=sm_90\n'
                              '  - CuDNN 8.9.2\n'
                              '  - Magma 2.6.1\n'
                              '  - Build settings: BLAS_INFO=mkl, '
                              'BUILD_TYPE=Release, CUDA_VERSION=12.1, '
                              'CUDNN_VERSION=8.9.2, '
                              'CXX_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/c++, '
                              'CXX_FLAGS= -D_GLIBCXX_USE_CXX11_ABI=0 '
                              '-fabi-version=11 -fvisibility-inlines-hidden '
                              '-DUSE_PTHREADPOOL -DNDEBUG -DUSE_KINETO '
                              '-DLIBKINETO_NOROCTRACER -DUSE_FBGEMM '
                              '-DUSE_QNNPACK -DUSE_PYTORCH_QNNPACK '
                              '-DUSE_XNNPACK -DSYMBOLICATE_MOBILE_DEBUG_HANDLE '
                              '-O2 -fPIC -Wall -Wextra -Werror=return-type '
                              '-Werror=non-virtual-dtor -Werror=bool-operation '
                              '-Wnarrowing -Wno-missing-field-initializers '
                              '-Wno-type-limits -Wno-array-bounds '
                              '-Wno-unknown-pragmas -Wno-unused-parameter '
                              '-Wno-unused-function -Wno-unused-result '
                              '-Wno-strict-overflow -Wno-strict-aliasing '
                              '-Wno-stringop-overflow -Wsuggest-override '
                              '-Wno-psabi -Wno-error=pedantic '
                              '-Wno-error=old-style-cast -Wno-missing-braces '
                              '-fdiagnostics-color=always -faligned-new '
                              '-Wno-unused-but-set-variable '
                              '-Wno-maybe-uninitialized -fno-math-errno '
                              '-fno-trapping-math -Werror=format '
                              '-Wno-stringop-overflow, LAPACK_INFO=mkl, '
                              'PERF_WITH_AVX=1, PERF_WITH_AVX2=1, '
                              'PERF_WITH_AVX512=1, TORCH_VERSION=2.3.0, '
                              'USE_CUDA=ON, USE_CUDNN=ON, USE_CUSPARSELT=1, '
                              'USE_EXCEPTION_PTR=1, USE_GFLAGS=OFF, '
                              'USE_GLOG=OFF, USE_GLOO=ON, USE_MKL=ON, '
                              'USE_MKLDNN=ON, USE_MPI=OFF, USE_NCCL=1, '
                              'USE_NNPACK=ON, USE_OPENMP=ON, USE_ROCM=OFF, '
                              'USE_ROCM_KERNEL_ASSERT=OFF, \n',
 'Python': '3.10.14 (main, May  6 2024, 19:42:50) [GCC 11.2.0]',
 'TorchVision': '0.18.0+cu121',
 'numpy_random_seed': 2147483648,
 'opencompass': '0.2.5+',
 'sys.platform': 'linux'}

重现问题 - 代码/配置示例

from mmengine.config import read_base
from opencompass.models import OpenAI
from opencompass.partitioners import NaivePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask

with read_base():
    # choose a list of datasets
    from .datasets.collections.chat_medium import datasets
    # and output the results in a choosen format
    from .summarizers.medium import summarizer

api_meta_template = dict(
    round=[
            dict(role='HUMAN', api_role='HUMAN'),
            dict(role='BOT', api_role='BOT', generate=True),
    ],
)

models = [
    dict(abbr='GPT-3.5-turbo-0613',
        type=OpenAI, path='gpt-3.5-turbo-0613',
        key='ENV',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
        meta_template=api_meta_template,
        query_per_second=1,
        max_out_len=2048, max_seq_len=4096, batch_size=8),
]

infer = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=LocalRunner,
        max_num_workers=8,
        task=dict(type=OpenICLInferTask)),
)

重现问题 - 命令或脚本

python run.py ./configs/eval_gpt3.5.py -w outputs/demo --debug

重现问题 - 错误信息

python run.py ./configs/eval_gpt3.5.py -w outputs/demo --debug
Traceback (most recent call last):
  File "/data/opencompass/run.py", line 4, in <module>
    main()
  File "/data/opencompass/opencompass/cli/main.py", line 220, in main
    cfg = get_config_from_arg(args)
  File "/data/opencompass/opencompass/utils/run.py", line 82, in get_config_from_arg
    config = Config.fromfile(args.config, format_python_code=False)
  File "/data/miniconda3/lib/python3.10/site-packages/mmengine/config/config.py", line 494, in fromfile
    raise e
  File "/data/miniconda3/lib/python3.10/site-packages/mmengine/config/config.py", line 492, in fromfile
    cfg_dict, imported_names = Config._parse_lazy_import(filename)
  File "/data/miniconda3/lib/python3.10/site-packages/mmengine/config/config.py", line 1081, in _parse_lazy_import
    _base_cfg_dict, _base_imported_names = Config._parse_lazy_import(  # noqa: E501
  File "/data/miniconda3/lib/python3.10/site-packages/mmengine/config/config.py", line 1074, in _parse_lazy_import
    raise ConfigParsingError(
mmengine.config.utils.ConfigParsingError: ./configs/datasets/collections/../TheoremQA/TheoremQA_gen_7009de.py not found! It means that incorrect module is defined in `with read_base(): = from ..TheoremQA.TheoremQA_gen_7009de import ...`, please make sure the base config module is valid and is consistent with the prior import logic

其他信息

没有改动

liushz commented 4 months ago

Due to a recent update in OpenCompass, certain datasets for ./configs/eval_gpt3.5.py are no longer supported. If you wish to test GPT3.5 using a demonstration configuration, please navigate to ./configs/eval_gpt3.5.py and modify it like:

# Test GSM8k dataset with GPT3.5
from mmengine.config import read_base
from opencompass.models import OpenAI
from opencompass.partitioners import NaivePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask

with read_base():
    # choose a list of datasets
    from .datasets.gsm8k.gsm8k_gen import datasets

api_meta_template = dict(
    round=[
            dict(role='HUMAN', api_role='HUMAN'),
            dict(role='BOT', api_role='BOT', generate=True),
    ],
)

models = [
    dict(abbr='GPT-3.5-turbo-0613',
        type=OpenAI, path='gpt-3.5-turbo-0613',
        key='ENV',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
        meta_template=api_meta_template,
        query_per_second=1,
        max_out_len=2048, max_seq_len=4096, batch_size=8),
]

infer = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=LocalRunner,
        max_num_workers=8,
        task=dict(type=OpenICLInferTask)),
)

This is a config for test GSM8k dataset with GPT3.5, you can replace the dataset to whatever dataset you want to test.

mxdlzg commented 4 months ago

Due to a recent update in OpenCompass, certain datasets for ./configs/eval_gpt3.5.py are no longer supported. If you wish to test GPT3.5 using a demonstration configuration, please navigate to ./configs/eval_gpt3.5.py and modify it like:

# Test GSM8k dataset with GPT3.5
from mmengine.config import read_base
from opencompass.models import OpenAI
from opencompass.partitioners import NaivePartitioner
from opencompass.runners import LocalRunner
from opencompass.tasks import OpenICLInferTask

with read_base():
    # choose a list of datasets
    from .datasets.gsm8k.gsm8k_gen import datasets

api_meta_template = dict(
    round=[
            dict(role='HUMAN', api_role='HUMAN'),
            dict(role='BOT', api_role='BOT', generate=True),
    ],
)

models = [
    dict(abbr='GPT-3.5-turbo-0613',
        type=OpenAI, path='gpt-3.5-turbo-0613',
        key='ENV',  # The key will be obtained from $OPENAI_API_KEY, but you can write down your key here as well
        meta_template=api_meta_template,
        query_per_second=1,
        max_out_len=2048, max_seq_len=4096, batch_size=8),
]

infer = dict(
    partitioner=dict(type=NaivePartitioner),
    runner=dict(
        type=LocalRunner,
        max_num_workers=8,
        task=dict(type=OpenICLInferTask)),
)

This is a config for test GSM8k dataset with GPT3.5, you can replace the dataset to whatever dataset you want to test.

When will the documentation be updated? I just came into this library yesterday, and the mismatch between the documentation and the actual behavior made me very confused.

liushz commented 4 months ago

Which section of the documentation allows you to use this configuration for the demo? ./configs/eval_gpt3.5.py