search

gdcc / easyDataverse

🪐 - Lightweight Dataverse interface in Python to upload, download and update datasets found in Dataverse installations.
MIT License
16 stars 5 forks source link

Error at creating a Dataset from_url #4

Closed atrisovic closed 1 year ago

atrisovic commented 1 year ago

Hey!

I successfully install easyDataverse, but get an error when creating a dataset. This is my code:

from easyDataverse import Dataset
dataset = Dataset.from_url("https://demo.dataverse.org/dataset.xhtml?persistentId=doi:10.70122/FK2/Y73N6C")

This is the error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[10], line 1
----> 1 dataset = Dataset.from_url("https://demo.dataverse.org/dataset.xhtml?persistentId=doi:10.70122/FK2/Y73N6C")

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:201, in pydantic.decorator.ValidatedFunction.execute()

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:359, in Dataset.from_url(cls, url, filedir, download_files, api_token, lib_name)
    353     raise ValueError(
    354         f"Given URL '{url}' is not a valid Dataverse URL since no 'persistenID' is given"
    355     )
    357 dataverse_url = f"https://{parsed_url.hostname}/"
--> 359 return cls.from_dataverse_doi(
    360     doi=doi[0],
    361     filedir=filedir,
    362     lib_name=lib_name,
    363     dataverse_url=dataverse_url,
    364     api_token=api_token,
    365     download_files=download_files,
    366 )

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:40, in pydantic.decorator.validate_arguments.validate.wrapper_function()

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:134, in pydantic.decorator.ValidatedFunction.call()

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/pydantic/decorator.py:201, in pydantic.decorator.ValidatedFunction.execute()

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:416, in Dataset.from_dataverse_doi(cls, doi, filedir, filenames, download_files, lib_name, dataverse_url, api_token)
    412     if not dataverse_url:
    413         raise ValueError(
    414             "Dataverse URL has not been specified in argument 'dataverse_url'. Please specify it to download datasets from your desired installation."
    415         )
--> 416     return cls._fetch_without_lib(
    417         dataset=cls(),
    418         doi=doi,
    419         filedir=filedir,
    420         dataverse_url=dataverse_url,
    421         api_token=api_token,
    422         filenames=filenames,
    423     )
    425 elif not lib_name and "EASYDATAVERSE_LIB_NAME" in os.environ:
    426     dataverse_url, api_token = cls._fetch_env_vars(api_token)

File ~/PycharmProjects/easyDataverse/easyDataverse/core/dataset.py:473, in Dataset._fetch_without_lib(**kwargs)
    468     except KeyError:
    469         warnings.warn(
    470             "No 'API_TOKEN' found in the environment. Please be aware, that you might not have the rights to download this dataset."
    471         )
--> 473 return download_from_dataverse_without_lib(**kwargs)

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/downloader.py:99, in download_from_dataverse_without_lib(dataset, doi, filedir, dataverse_url, api_token, filenames)
     96 dataset.p_id = doi
     98 # Step 2: Extract all metadatablocks from the given dataset
---> 99 blocks = [
    100     create_block_definitions(block_name, block, dataverse_url)
    101     for block_name, block in metadatablocks.items()
    102 ]
    104 # Step 3: Populate data and assign to dataset
    105 for block in blocks:

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/downloader.py:100, in <listcomp>(.0)
     96 dataset.p_id = doi
     98 # Step 2: Extract all metadatablocks from the given dataset
     99 blocks = [
--> 100     create_block_definitions(block_name, block, dataverse_url)
    101     for block_name, block in metadatablocks.items()
    102 ]
    104 # Step 3: Populate data and assign to dataset
    105 for block in blocks:

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:95, in create_block_definitions(block_name, block, dataverse_url)
     93 # Turn raw definitions into classes
     94 for field in block["fields"]:
---> 95     _process_field(field, lookup, cls_def, add_funs)
     97 # Now, create the class
     98 block_cls = create_model(
     99     block_name.capitalize(), __base__=(DataverseBase,), **cls_def
    100 )()

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:138, in _process_field(field, lookup, cls_def, add_funs)
    131 PROCESS_MAPPING = {
    132     "primitive": _process_primitive,
    133     "controlledVocabulary": _process_primitive,
    134     "compound": _process_compound,
    135 }
    137 fun = PROCESS_MAPPING[field["typeClass"]]
--> 138 fun(field, lookup, cls_def, add_funs)

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:162, in _process_compound(field, lookup, cls_def, add_funs)
    159 field_name = _camel_to_snake(field_name)
    161 # Generate add method
--> 162 add_funs[f"add_{field_name}"] = _generate_add_method(cls, field_name)
    164 cls_def[field_name] = (dtype, Field(**field_meta))

File ~/PycharmProjects/easyDataverse/easyDataverse/tools/downloader/nolibutils.py:245, in _generate_add_method(target_cls, field)
    242 add_fun = copy.deepcopy(_generic_add_function)
    243 add_fun.__name__ = f"add_{field}"
--> 245 return forge.sign(
    246     forge.self,
    247     *[
    248         forge.kwarg(name, type=dtype, default=forge.empty)
    249         for name, dtype in target_cls.__annotations__.items()
    250     ],
    251     forge.kwarg("_target_cls", default=target_cls, bound=True),
    252     forge.kwarg("_field", default=field, bound=True),
    253 )(add_fun)

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py:331, in Revision.__call__(self, callable)
    328         return callable(*mapped.args, **mapped.kwargs)
    330 next_.validate()
--> 331 inner.__mapper__ = Mapper(next_, callable)  # type: ignore
    332 inner.__signature__ = inner.__mapper__.public_signature  # type: ignore
    333 return inner

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py:62, in Mapper.__init__(self, fsignature, callable)
     54 def __init__(
     55         self,
     56         fsignature: FSignature,
   (...)
     59     # pylint: disable=W0622, redefined-builtin
     60     # pylint: disable=W0621, redefined-outer-name
     61     private_signature = inspect.signature(callable)
---> 62     public_signature = fsignature.native
     63     parameter_map = self.map_parameters(fsignature, private_signature)
     64     context_param = get_context_parameter(fsignature)

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py:1231, in FSignature.native(self)
   1224 @property
   1225 def native(self) -> inspect.Signature:
   1226     """
   1227     Provides a representation of this :class:`~forge.FSignature` as an
   1228     instance of :class:`inspect.Signature`
   1229     """
   1230     return inspect.Signature(
-> 1231         [param.native for param in self if not param.bound],
   1232         return_annotation=self.return_annotation,
   1233     )

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py:1231, in <listcomp>(.0)
   1224 @property
   1225 def native(self) -> inspect.Signature:
   1226     """
   1227     Provides a representation of this :class:`~forge.FSignature` as an
   1228     instance of :class:`inspect.Signature`
   1229     """
   1230     return inspect.Signature(
-> 1231         [param.native for param in self if not param.bound],
   1232         return_annotation=self.return_annotation,
   1233     )

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py:355, in FParameter.native(self)
    353 if not self.name:
    354     raise TypeError('Cannot generate an unnamed parameter')
--> 355 return inspect.Parameter(
    356     name=self.name,
    357     kind=self.kind,
    358     default=empty.ccoerce_native(self.default),
    359     annotation=empty.ccoerce_native(self.type),
    360 )

File ~/opt/miniconda3/envs/easyDVconda/lib/python3.10/inspect.py:2673, in Parameter.__init__(self, name, kind, default, annotation)
   2670     name = 'implicit{}'.format(name[1:])
   2672 if not name.isidentifier():
-> 2673     raise ValueError('{!r} is not a valid parameter name'.format(name))
   2675 self._name = name

ValueError: 'e-mail' is not a valid parameter name

What am I doing wrong? x-)

atrisovic commented 1 year ago

The same error happens in this example:

dataset = Dataset.from_dataverse_doi(
  doi="doi:10.7910/DVN/SYNPBS",
  dataverse_url="https://dataverse.harvard.edu"
)
atrisovic commented 1 year ago

Also in the command line:

(easyDVconda) ant746@MacBook:~/PycharmProjects/easyDataverse/temp (main) $ dataverse fetch https://demo.dataverse.org/dataset.xhtml?persistentId=doi:10.70122/FK2/Y73N6C
/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py:469: UserWarning: No 'API_TOKEN' found in the environment. Please be aware, that you might not have the rights to download this dataset.
  warnings.warn(
Traceback (most recent call last):

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/bin/dataverse", line 8, in <module>
    sys.exit(main())

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/cli.py", line 94, in main
    app()

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/cli.py", line 86, in fetch
    dataset = Dataset.from_url(url, path)

  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function

  File "pydantic/decorator.py", line 134, in pydantic.decorator.ValidatedFunction.call

  File "pydantic/decorator.py", line 201, in pydantic.decorator.ValidatedFunction.execute

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 359, in from_url
    return cls.from_dataverse_doi(

  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function

  File "pydantic/decorator.py", line 134, in pydantic.decorator.ValidatedFunction.call

  File "pydantic/decorator.py", line 201, in pydantic.decorator.ValidatedFunction.execute

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 416, in from_dataverse_doi
    return cls._fetch_without_lib(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 473, in _fetch_without_lib
    return download_from_dataverse_without_lib(**kwargs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/downloader.py", line 99, in download_from_dataverse_without_lib
    blocks = [

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/downloader.py", line 100, in <listcomp>
    create_block_definitions(block_name, block, dataverse_url)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 95, in create_block_definitions
    _process_field(field, lookup, cls_def, add_funs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 138, in _process_field
    fun(field, lookup, cls_def, add_funs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 162, in _process_compound
    add_funs[f"add_{field_name}"] = _generate_add_method(cls, field_name)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 245, in _generate_add_method
    return forge.sign(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py", line 331, in __call__
    inner.__mapper__ = Mapper(next_, callable)  # type: ignore

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py", line 62, in __init__
    public_signature = fsignature.native

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 1231, in native
    [param.native for param in self if not param.bound],

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 1231, in <listcomp>
    [param.native for param in self if not param.bound],

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 355, in native
    return inspect.Parameter(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/inspect.py", line 2673, in __init__
    raise ValueError('{!r} is not a valid parameter name'.format(name))

ValueError: 'e-mail' is not a valid parameter name

(easyDVconda) ant746@MacBook:~/PycharmProjects/easyDataverse/temp (main) $ export API_TOKEN=SoMeToKeN
(easyDVconda) ant746@MacBook:~/PycharmProjects/easyDataverse/temp (main) $ 
(easyDVconda) ant746@MacBook:~/PycharmProjects/easyDataverse/temp (main) $ dataverse fetch https://demo.dataverse.org/dataset.xhtml?persistentId=doi:10.70122/FK2/Y73N6C
/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py:469: UserWarning: No 'API_TOKEN' found in the environment. Please be aware, that you might not have the rights to download this dataset.
  warnings.warn(
Traceback (most recent call last):

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/bin/dataverse", line 8, in <module>
    sys.exit(main())

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/cli.py", line 94, in main
    app()

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/cli.py", line 86, in fetch
    dataset = Dataset.from_url(url, path)

  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function

  File "pydantic/decorator.py", line 134, in pydantic.decorator.ValidatedFunction.call

  File "pydantic/decorator.py", line 201, in pydantic.decorator.ValidatedFunction.execute

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 359, in from_url
    return cls.from_dataverse_doi(

  File "pydantic/decorator.py", line 40, in pydantic.decorator.validate_arguments.validate.wrapper_function

  File "pydantic/decorator.py", line 134, in pydantic.decorator.ValidatedFunction.call

  File "pydantic/decorator.py", line 201, in pydantic.decorator.ValidatedFunction.execute

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 416, in from_dataverse_doi
    return cls._fetch_without_lib(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/core/dataset.py", line 473, in _fetch_without_lib
    return download_from_dataverse_without_lib(**kwargs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/downloader.py", line 99, in download_from_dataverse_without_lib
    blocks = [

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/downloader.py", line 100, in <listcomp>
    create_block_definitions(block_name, block, dataverse_url)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 95, in create_block_definitions
    _process_field(field, lookup, cls_def, add_funs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 138, in _process_field
    fun(field, lookup, cls_def, add_funs)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 162, in _process_compound
    add_funs[f"add_{field_name}"] = _generate_add_method(cls, field_name)

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/easyDataverse/tools/downloader/nolibutils.py", line 245, in _generate_add_method
    return forge.sign(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py", line 331, in __call__
    inner.__mapper__ = Mapper(next_, callable)  # type: ignore

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_revision.py", line 62, in __init__
    public_signature = fsignature.native

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 1231, in native
    [param.native for param in self if not param.bound],

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 1231, in <listcomp>
    [param.native for param in self if not param.bound],

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/site-packages/forge/_signature.py", line 355, in native
    return inspect.Parameter(

  File "/Users/ant746/opt/miniconda3/envs/easyDVconda/lib/python3.10/inspect.py", line 2673, in __init__
    raise ValueError('{!r} is not a valid parameter name'.format(name))

ValueError: 'e-mail' is not a valid parameter name
atrisovic commented 1 year ago

I use Python 3.10

I get the same error when I install easyDataverse from source and when I install it with pip

JR-1991 commented 1 year ago

Thanks for the issue! I proposed a pull request #7 to address this bug and tested your code. It works on my side now except I have no authorization to fetch the dataset. Could you try to fetch it using the PR branch?