root@gpu03:~/Megatron-LM/tools/openwebtext# python3 cleanup_dataset.py /workspace/data/merged_output.json /workspace/data/merged_cleand.json
Traceback (most recent call last):
File "cleanup_dataset.py", line 12, in
from tokenizer import Tokenizer
File "/root/Megatron-LM/tools/openwebtext/tokenizer.py", line 13, in
from megatron.core.datasets.megatron_tokenizer import MegatronTokenizer
ModuleNotFoundError: No module named 'megatron'
root@gpu03:~/Megatron-LM/tools/openwebtext#
root@gpu03:~/Megatron-LM/tools/openwebtext# python3 cleanup_dataset.py /workspace/data/merged_output.json /workspace/data/merged_cleand.json Traceback (most recent call last): File "cleanup_dataset.py", line 12, in
from tokenizer import Tokenizer
File "/root/Megatron-LM/tools/openwebtext/tokenizer.py", line 13, in
from megatron.core.datasets.megatron_tokenizer import MegatronTokenizer
ModuleNotFoundError: No module named 'megatron'
root@gpu03:~/Megatron-LM/tools/openwebtext#
请问这个要怎么解决呢？