Closed Ssunbell closed 1 year ago
from datasets import load_dataset dataset = load_dataset("json", data_files="../train/train_v1.0.json", field="data")
이거 하면 동일하게 되는듯?
ㅇㅇ 그거랑 동일한 형식인데 bounding box랑 image pixel값만 추가해서 넣어둠
from datasets import load_dataset
datasets = load_dataset("Trailblazer-Yoo/boostcamp-docvqa")
>>> DatasetDict({
val: Dataset({
features: ['questionId', 'question', 'image', 'docId', 'ucsf_document_id', 'ucsf_document_page_no', 'answers', 'data_split', 'words', 'boxes'],
num_rows: 5349
})
train: Dataset({
features: ['questionId', 'question', 'image', 'docId', 'ucsf_document_id', 'ucsf_document_page_no', 'answers', 'data_split', 'words', 'boxes'],
num_rows: 39463
})
})
from datasets import load_dataset
test_dataset = load_dataset("Trailblazer-Yoo/boostcamp-docvqa-test")
>>> DatasetDict({
test: Dataset({
features: ['questionId', 'question', 'image', 'docId', 'ucsf_document_id', 'ucsf_document_page_no', 'data_split', 'words', 'boxes'],
num_rows: 5188
})
})
목표
klue dataset처럼 script 방식으로 데이터셋 만들기
dataset에 올릴 파일은 다음과 같음
''' {'questionId': 49153, 'question': 'What is the ‘actual’ value per 1000, during the year 1975?', 'image': 'documents/pybv0228_81.png', 'docId': 14465, 'answers': ['0.28'], 'box': [36,32,10,2], 'input_words' : [hello] } '''