Open ocampor opened 4 years ago
Dear Sir, I would like to build up a dataset builder based on CSIQ dataset. In the first stage, I tried to download CSIQ dataset with the following codes: import os.path
import tensorflow as tf import tensorflow_datasets.public_api as tfds
CITATION = r""" @article{larson2010most, title={Most apparent distortion: full-reference image quality assessment and the role of strategy}, author={Larson, Eric Cooper and Chandler, Damon Michael}, journal={Journal of electronic imaging}, volume={19}, number={1}, pages={011006}, year={2010}, publisher={International Society for Optics and Photonics} } """ DESCRIPTION = """ The CSIQ image database is a popular database for testing image quality assessment algorithms and other aspects of image quality. The database consists of 30 original images, each distorted using one of six types of distortions, each at four to five different levels of distortion. The CSIQ images were subjectively rated based on a linear displacement of the images across four calibrated LCD monitors placed side-by-side with equal viewing distance to the observer. The database contains 5000 subjective ratings from 35 different observers, and the ratings are reported in the form of DMOS (a larger denotes greater visual distortion compared to the corresponding reference image). """ URL = u"http://vision.eng.shizuoka.ac.jp/mod/page/view.php?id=23" LICENSE = """ ©2020 Damon M. Chandler CSIQ Lab | Dept. of Electrical and Electronic Engineering | College of Engineering | Shizuoka University
〒432-8561 | 浜松市中区城北3-5-1 | 静岡大学 工学部 電気電子工学科
Portions of this site contain images © David Vignoni. LGPL 2.1.
The Essential theme for Moodle is developed by Gareth J Barnard """ SUPERVISED_KEYS = ("distorted_image", "dmos")
class CSIQ(tfds.core.GeneratorBasedBuilder): VERSION = tfds.core.Version("1.0.0")
def _info(self):
return tfds.core.DatasetInfo(
builder=self,
description=DESCRIPTION,
features=tfds.features.FeaturesDict({
"dst_idx": tf.int32,#tfds.features.Text(),
"dst_type":tfds.features.Text(),
#"index": tf.int32,
"Distorted_image": tfds.features.Image(),
"Reference_image": tfds.features.Image(),
"dst_lev": tf.int32,
"dmos_std": tf.float32,
"dmos": tf.float32,
}),
supervised_keys=SUPERVISED_KEYS,
homepage=URL,
citation=CITATION,
#redistribution_info={
# 'license': LICENSE,
#},
)
def _split_generators(self, manager):
#"CSIQ_LBL"= "http://vision.eng.shizuoka.ac.jp/pluginfile.php/69/mod_page/content/25/csiq.DMOS.xlsx"
extracted_path = manager.download_and_extract({
"CSIQ_DEST": "http://vision.eng.shizuoka.ac.jp/csiq/dst_imgs.zip",
"CSIQ_REF": "http://vision.eng.shizuoka.ac.jp/csiq/src_imgs.zip",
#"force_checksums_validation": bool = False,
#"register_checksums": bool = False
})
extracted_path['CSIQ_DEST']
extracted_path['CSIQ_REF']
#images_path = os.path.join(extracted_path, "CSIQ_DEST","CSIQ_REF")
#images_path = os.path.join(Datadir)
directory_url = 'http://vision.eng.shizuoka.ac.jp/pluginfile.php/69/mod_page/content/25/'
file_names = ['csiq.DMOS.xlsx']
file_paths = [
tf.keras.utils.get_file(file_name, directory_url + file_name)
for file_name in file_names
]
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
gen_kwargs={
"images_path": images_path,
#"labels": os.path.join(images_path,"dmos1.csv")
"labels": tf.io.read_file(file_paths)
#"labels":pd.read_csv(r"C:\CSIQ_all\dmos1.csv")
},
)
]
def _generate_examples(self, images_path, labels):
with tf.io.gfile.GFile(labels) as f:
lines = f.readlines()
for image_id, line in enumerate(lines[1:]):
values = line.split(",")
yield image_id, {
"dst_idx": values[0],
"dst_type": values[1],
"Distorted": os.path.join(images_path, values[2]),
"Reference": os.path.join(images_path, values[3]),
"DST_lev": values[4],
"dmos_std": values[5],
"dmos": values[6],
#"dmos_realigned": values[0],
#"dmos_realigned_std": values[0],
}
WHile building a pipeline, with following commands: builder = CSIQ() builder.download_and_prepare() ds = builder.as_dataset(shuffle_files=True)['train'] ds = ds.shuffle(1024).batch(1) next(iter(ds)).keys() for features in ds.take(1): distorted_image = features['Distorted'] reference_image = features['Reference'] dmos = tf.round(features['dmos'][0], 2)#disable for Tid2013 distortion = features['distortion'][0] # disable for Tid2013
print(f'The distortion of the image is {dmos} with'
f'a distortion {distortion} and shape {distorted_image.shape}')
show_images([reference_image, distorted_image])
It results: Checksum error.
NonMatchingChecksumError Traceback (most recent call last)
Hi @KamalLamichhane, I want to download the CSIQ databaes, but the link to it can not be located anymore. Do you have any solution to deal with it?
Thanks
Datasets: