search

iDigBio / idigbio-python-client

A python client for the idigbio v2 API
MIT License
10 stars 1 forks source link

The python client only exposes the "data" section if "fields_exclude" parameter is specified #2

Closed danstoner closed 8 years ago

danstoner commented 9 years ago

The python client only exposes the "data" section in results when using "fields_exclude" on an unrelated field.

The default parameter passed by the client is 

fields_exclude=["data.*"]

Even if I use

fields=["data"]

or

fields=["data.*"]

in order to try to include the data fields, the default exclude overrides the include.

It appears that in order to get back any "data" fields I have to pass in any fields_exclude parameter (to overrride the default).

danstoner commented 9 years ago

iDigBio Search API Example. No specified "fields" or "fields_exclude" parameters.

Notice that the search API includes the "data" section in the results by default, with dwc terms inside.

http://beta-search.idigbio.org/v2/search/records?rq={%22scientificname%22:%20%22puma%20concolor%22}&limit=1

http://beta-search.idigbio.org/v2/search/records?rq={%22scientificname%22:%20%22puma%20concolor%22}&limit=1

Results:

{

    "itemCount": 1737,
    "lastModified": "2015-04-18T05:10:39.976Z",
    "items": 

[

{

    "uuid": "bc5510a4-4cc5-4731-a023-8827fdf58e61",
    "etag": "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
    "data": 

{

    "dwc:startDayOfYear": "102",
    "dwc:latestEpochOrHighestSeries": "WISCONSIN MID",
    "dwc:specificEpithet": "concolor",
    "dwc:countryCode": "US",
    "dwc:county": "Eddy",
    "dwc:recordedBy": "Harris, Arthur H. (Van Goebel)",
    "dwc:order": "Carnivora",
    "dcterms:references": "http://portal.vertnet.org/o/utep/fossils?id=5-298",
    "dwc:individualCount": "1",
    "dwc:occurrenceID": "urn:catalog:UTEP:Fossils:5-298",
    "dcterms:language": "en",
    "dwc:verbatimElevation": "1280 M",
    "id": "urn:catalog:UTEP:Fossils:5-298",
    "dwc:establishmentMeans": "native",
    "dwc:stateProvince": "New Mexico",
    "dwc:locationRemarks": "See site 22 for Dry Cave references.",
    "dwc:eventDate": "1968-04-11",
    "dwc:country": "United States",
    "dwc:collectionCode": "Fossils",
    "dwc:occurrenceStatus": "present",
    "dwc:kingdom": "Animalia",
    "dwc:maximumElevationInMeters": "1280",
    "dwc:basisOfRecord": "PreservedSpecimen",
    "dcterms:accessRights": "http://vertnet.org/resources/norms.html",
    "dwc:genus": "Puma",
    "dwc:continent": "North America",
    "dwc:family": "Felidae",
    "dwc:identifiedBy": "Harris, Arthur H.",
    "dwc:higherClassification": "Mammalia | Carnivora | Felidae | Felinae | Puma",
    "dwc:vernacularName": "Mountain Lion",
    "dwc:phylum": "Chordata",
    "dwc:locality": "Sabertooth Camel Maze, Dry Cave; Pecos River",
    "dwc:institutionID": "http://grbio.org/cool/gq95-3z97",
    "dwc:institutionCode": "UTEP",
    "dwc:taxonRank": "species",
    "dwc:class": "Mammalia",
    "dwc:catalogNumber": "5-298",
    "dwc:nomenclaturalCode": "ICZN",
    "dcterms:type": "PhysicalObject",
    "dwc:higherGeography": "| USA | NM | EDDY |  |  |  |",
    "dwc:month": "4",
    "dwc:endDayOfYear": "102",
    "dwc:verbatimLocality": "USA | NM | Eddy | Sabertooth Camel Maze, Dry Cave; Pecos River",
    "dwc:earliestEpochOrLowestSeries": "WISCONSIN MID",
    "dwc:preparations": "MC I R",
    "dwc:minimumElevationInMeters": "1280",
    "dcterms:modified": "2015-02-20",
    "dwc:scientificName": "Puma concolor",
    "dwc:day": "11",
    "dwc:year": "1968"

},
"indexTerms": 
{

    "individualcount": 1,
    "family": "felidae",
    "recordset": "15b5a0a1-3883-4789-a351-294f00282137",
    "dqs": 0.48333333333333334,
    "stateprovince": "new mexico",
    "earliestepochorlowestseries": "wisconsin mid",
    "county": "eddy",
    "phylum": "chordata",
    "catalognumber": "5-298",
    "specificepithet": "concolor",
    "continent": "north america",
    "uuid": "bc5510a4-4cc5-4731-a023-8827fdf58e61",
    "countrycode": "usa",
    "basisofrecord": "preservedspecimen",
    "collector": "harris, arthur h. (van goebel)",
    "institutioncode": "utep",
    "datemodified": "2015-03-10T20:46:15.130000+00:00",
    "datecollected": "1968-04-11T00:00:00+00:00",
    "etag": "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
    "verbatimlocality": "usa | nm | eddy | sabertooth camel maze, dry cave; pecos river",
    "minelevation": 1280,
    "hasImage": false,
    "kingdom": "animalia",
    "maxelevation": 1280,
    "highertaxon": "mammalia | carnivora | felidae | felinae | puma",
    "commonname": "mountain lion",
    "scientificname": "puma concolor",
    "class": "mammalia",
    "occurrenceid": "urn:catalog:utep:fossils:5-298",
    "institutionid": "http://grbio.org/cool/gq95-3z97",
    "country": "united states",
    "locality": "sabertooth camel maze, dry cave; pecos river",
    "collectioncode": "fossils",
    "eventdate": "1968-04-11",
    "latestepochorhighestseries": "wisconsin mid",
    "flags": 

[

    "idigbio_isocountrycode_added"

],
"recordids": 

            [
                "15b5a0a1-3883-4789-a351-294f00282137\\urn:catalog:utep:fossils:5-298"
            ],
            "genus": "puma",
            "order": "carnivora"
        }
    }

],
"attribution": 
    [ "* attribution trimmed *" ]
}

iDigBio Search API Example. Exclude the "data" section via "fields_exclude" parameter.

The "data" section can be excluded from results with the "fields_exclude" parameter.

http://beta-search.idigbio.org/v2/search/records?rq={%22scientificname%22:%20%22puma%20concolor%22}&limit=1&fields_exclude=[%22data%22]

http://beta-search.idigbio.org/v2/search/records?rq={%22scientificname%22:%20%22puma%20concolor%22}&limit=1&fields_exclude=[%22data%22]

Results:

{

    "itemCount": 1737,
    "lastModified": "2015-04-18T05:10:39.976Z",
    "items": 

[

{

    "uuid": "bc5510a4-4cc5-4731-a023-8827fdf58e61",
    "etag": "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
    "data": { },
    "indexTerms": 

{

    "countrycode": "usa",
    "etag": "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
    "recordset": "15b5a0a1-3883-4789-a351-294f00282137",
    "specificepithet": "concolor",
    "institutioncode": "utep",
    "dqs": 0.48333333333333334,
    "basisofrecord": "preservedspecimen",
    "institutionid": "http://grbio.org/cool/gq95-3z97",
    "maxelevation": 1280,
    "collector": "harris, arthur h. (van goebel)",
    "phylum": "chordata",
    "latestepochorhighestseries": "wisconsin mid",
    "order": "carnivora",
    "catalognumber": "5-298",
    "kingdom": "animalia",
    "family": "felidae",
    "highertaxon": "mammalia | carnivora | felidae | felinae | puma",
    "datemodified": "2015-03-10T20:46:15.130000+00:00",
    "earliestepochorlowestseries": "wisconsin mid",
    "scientificname": "puma concolor",
    "hasImage": false,
    "flags": 

[

    "idigbio_isocountrycode_added"

],
"verbatimlocality": "usa | nm | eddy | sabertooth camel maze, dry cave; pecos river",
"occurrenceid": "urn:catalog:utep:fossils:5-298",
"recordids": 

            [
                "15b5a0a1-3883-4789-a351-294f00282137\\urn:catalog:utep:fossils:5-298"
            ],
            "class": "mammalia",
            "genus": "puma",
            "individualcount": 1,
            "country": "united states",
            "collectioncode": "fossils",
            "county": "eddy",
            "continent": "north america",
            "minelevation": 1280,
            "locality": "sabertooth camel maze, dry cave; pecos river",
            "commonname": "mountain lion",
            "stateprovince": "new mexico",
            "uuid": "bc5510a4-4cc5-4731-a023-8827fdf58e61",
            "eventdate": "1968-04-11",
            "datecollected": "1968-04-11T00:00:00+00:00"
        }
    }

],
"attribution": 
    [ "* attribution trimmed *" ]
}
danstoner commented 9 years ago

iDigBio python client Example.

Specified "fields" parameter containing ["data"] does not cause "data" to appear in results due to default "fields_exclude" parameter.

import idigbio
api=idigbio.json()
record_dict = api.search_records(rq={"scientificname": "puma concolor"}, limit=1, fields="[data]")
for each in record_dict["items"]:
    print json.dumps(each)

Output has no "data" field due to the search_records actually also silently sending the parameter:

fields_exclude=["data.*"]

Result:

{
   "indexTerms" : {
      "hasImage" : false,
      "highertaxon" : "mammalia | carnivora | felidae | felinae | puma",
      "collectioncode" : "fossils",
      "verbatimlocality" : "usa | nm | eddy | sabertooth camel maze, dry cave; pecos river",
      "commonname" : "mountain lion",
      "family" : "felidae",
      "earliestepochorlowestseries" : "wisconsin mid",
      "genus" : "puma",
      "county" : "eddy",
      "kingdom" : "animalia",
      "uuid" : "bc5510a4-4cc5-4731-a023-8827fdf58e61",
      "scientificname" : "puma concolor",
      "maxelevation" : 1280,
      "dqs" : 0.483333333333333,
      "phylum" : "chordata",
      "institutioncode" : "utep",
      "recordids" : [
         "15b5a0a1-3883-4789-a351-294f00282137\\urn:catalog:utep:fossils:5-298"
      ],
      "latestepochorhighestseries" : "wisconsin mid",
      "basisofrecord" : "preservedspecimen",
      "countrycode" : "usa",
      "occurrenceid" : "urn:catalog:utep:fossils:5-298",
      "continent" : "north america",
      "order" : "carnivora",
      "specificepithet" : "concolor",
      "eventdate" : "1968-04-11",
      "country" : "united states",
      "datemodified" : "2015-03-10T20:46:15.130000+00:00",
      "class" : "mammalia",
      "flags" : [
         "idigbio_isocountrycode_added"
      ],
      "catalognumber" : "5-298",
      "individualcount" : 1,
      "recordset" : "15b5a0a1-3883-4789-a351-294f00282137",
      "datecollected" : "1968-04-11T00:00:00+00:00",
      "minelevation" : 1280,
      "stateprovince" : "new mexico",
      "collector" : "harris, arthur h. (van goebel)",
      "institutionid" : "http://grbio.org/cool/gq95-3z97",
      "etag" : "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
      "locality" : "sabertooth camel maze, dry cave; pecos river"
   },
   "uuid" : "bc5510a4-4cc5-4731-a023-8827fdf58e61",
   "data" : {},
   "etag" : "d7af11977f2a7d6d9b692db9ef497e73c1e5772d"
}

Repeating the query but adding an empty exclude parameter allows the query to work as expected.

record_dict = api.search_records(rq={"scientificname": "puma concolor"}, fields="[data]", limit=1, fields_exclude=[""])

Result:

{
   "uuid" : "bc5510a4-4cc5-4731-a023-8827fdf58e61",
   "data" : {
      "dwc:eventDate" : "1968-04-11",
      "dwc:institutionID" : "http://grbio.org/cool/gq95-3z97",
      "dwc:higherClassification" : "Mammalia | Carnivora | Felidae | Felinae | Puma",
      "dwc:specificEpithet" : "concolor",
      "dwc:countryCode" : "US",
      "dwc:latestEpochOrHighestSeries" : "WISCONSIN MID",
      "dwc:stateProvince" : "New Mexico",
      "dwc:phylum" : "Chordata",
      "dwc:endDayOfYear" : "102",
      "dwc:occurrenceID" : "urn:catalog:UTEP:Fossils:5-298",
      "dwc:startDayOfYear" : "102",
      "dwc:occurrenceStatus" : "present",
      "dwc:preparations" : "MC I R",
      "dwc:country" : "United States",
      "dwc:scientificName" : "Puma concolor",
      "dwc:county" : "Eddy",
      "dwc:recordedBy" : "Harris, Arthur H. (Van Goebel)",
      "dwc:institutionCode" : "UTEP",
      "dwc:taxonRank" : "species",
      "dwc:verbatimLocality" : "USA | NM | Eddy | Sabertooth Camel Maze, Dry Cave; Pecos River",
      "dwc:individualCount" : "1",
      "dwc:catalogNumber" : "5-298",
      "dwc:verbatimElevation" : "1280 M",
      "dwc:establishmentMeans" : "native",
      "dcterms:modified" : "2015-02-20",
      "dwc:year" : "1968",
      "dwc:collectionCode" : "Fossils",
      "dwc:family" : "Felidae",
      "dwc:basisOfRecord" : "PreservedSpecimen",
      "dwc:locationRemarks" : "See site 22 for Dry Cave references.",
      "dwc:identifiedBy" : "Harris, Arthur H.",
      "dwc:genus" : "Puma",
      "dwc:maximumElevationInMeters" : "1280",
      "dwc:class" : "Mammalia",
      "dwc:kingdom" : "Animalia",
      "dwc:order" : "Carnivora",
      "dcterms:language" : "en",
      "dwc:nomenclaturalCode" : "ICZN",
      "dwc:locality" : "Sabertooth Camel Maze, Dry Cave; Pecos River",
      "dcterms:references" : "http://portal.vertnet.org/o/utep/fossils?id=5-298",
      "dcterms:type" : "PhysicalObject",
      "dwc:vernacularName" : "Mountain Lion",
      "dwc:month" : "4",
      "dwc:minimumElevationInMeters" : "1280",
      "dwc:earliestEpochOrLowestSeries" : "WISCONSIN MID",
      "dcterms:accessRights" : "http://vertnet.org/resources/norms.html",
      "id" : "urn:catalog:UTEP:Fossils:5-298",
      "dwc:higherGeography" : "| USA | NM | EDDY |  |  |  |",
      "dwc:continent" : "North America",
      "dwc:day" : "11"
   },
   "indexTerms" : {
      "flags" : [
         "idigbio_isocountrycode_added"
      ],
      "maxelevation" : 1280,
      "earliestepochorlowestseries" : "wisconsin mid",
      "institutioncode" : "utep",
      "order" : "carnivora",
      "datemodified" : "2015-03-10T20:46:15.130000+00:00",
      "recordset" : "15b5a0a1-3883-4789-a351-294f00282137",
      "etag" : "d7af11977f2a7d6d9b692db9ef497e73c1e5772d",
      "countrycode" : "usa",
      "stateprovince" : "new mexico",
      "family" : "felidae",
      "collectioncode" : "fossils",
      "individualcount" : 1,
      "phylum" : "chordata",
      "class" : "mammalia",
      "highertaxon" : "mammalia | carnivora | felidae | felinae | puma",
      "commonname" : "mountain lion",
      "occurrenceid" : "urn:catalog:utep:fossils:5-298",
      "verbatimlocality" : "usa | nm | eddy | sabertooth camel maze, dry cave; pecos river",
      "dqs" : 0.483333333333333,
      "datecollected" : "1968-04-11T00:00:00+00:00",
      "institutionid" : "http://grbio.org/cool/gq95-3z97",
      "eventdate" : "1968-04-11",
      "uuid" : "bc5510a4-4cc5-4731-a023-8827fdf58e61",
      "recordids" : [
         "15b5a0a1-3883-4789-a351-294f00282137\\urn:catalog:utep:fossils:5-298"
      ],
      "hasImage" : false,
      "scientificname" : "puma concolor",
      "catalognumber" : "5-298",
      "latestepochorhighestseries" : "wisconsin mid",
      "genus" : "puma",
      "continent" : "north america",
      "country" : "united states",
      "specificepithet" : "concolor",
      "kingdom" : "animalia",
      "basisofrecord" : "preservedspecimen",
      "county" : "eddy",
      "minelevation" : 1280,
      "collector" : "harris, arthur h. (van goebel)",
      "locality" : "sabertooth camel maze, dry cave; pecos river"
   },
   "etag" : "d7af11977f2a7d6d9b692db9ef497e73c1e5772d"
}