rcongiu / Hive-JSON-Serde

Read - Write JSON SerDe for Apache Hive.
Other
732 stars 391 forks source link

STRUCT and STRUCT KEY exists hive reserved words #158

Closed markluopp closed 7 years ago

markluopp commented 8 years ago

json sample:

{
   "message":"3T stuff. Lightly used. Was on the bike for 10 hours worth of riding only then I cracked the frame so now parting out to fund a new build. \n\n3T Ergosum Ltd Bars SOLD\n\n3T Arx II stem.  SOLD.\n\n3T Carbon bottle cage. Some marks internally from sliding a bottle in and out, otherwise, the exterior is 'as new'. Paid $65 will take $45 + postage. \n\nOr pickup 2763, Quakers Hill NSW",
   "permalink_url":"xxxx",
   "from":{
      "name":"Henri Le Comte",
      "id":"10153385684979159"
   },
   "created_time":"2014-03-23T07:34:09+0000",
   "updated_time":"2014-03-23T07:54:57+0000",
   "type":"photo",
   "status_type":"added_photos",
   "is_expired":false,
   "id":"142825039256674_231613730377804",
   "to":{
      "data":[
         {
            "id":"142825039256674",
            "name":"Bicycle Market Sydney"
         }
      ]
   },
   "likes":{
      "data":[
         {
            "id":"10153660785916861",
            "name":"Mike Critch"
         }
      ],
      "paging":{
         "cursors":{
            "before":"MTAxNTM2NjA3ODU5MTY4NjEZD",
            "after":"MTAxNTM2NjA3ODU5MTY4NjEZD"
         }
      }
   },
   "comments":{
      "data":[
         {
            "created_time":"2014-03-23T07:37:41+0000",
            "from":{
               "name":"Ethan Dean Doughty",
               "id":"1383606691655138"
            },
            "message":"I'm interested in the stem",
            "id":"231614240377753"
         },
         {
            "created_time":"2014-03-23T07:38:12+0000",
            "from":{
               "name":"Henri Le Comte",
               "id":"10153385684979159"
            },
            "message":"Stem just sold sorry Ethan, missed it by about a minute.",
            "id":"231614270377750"
         },
         {
            "created_time":"2014-03-23T07:38:17+0000",
            "from":{
               "name":"Henri Le Comte",
               "id":"10153385684979159"
            },
            "message":"Bars & Cage still avail.",
            "id":"231614287044415"
         },
         {
            "created_time":"2014-03-23T07:51:34+0000",
            "from":{
               "name":"Christopher Law",
               "id":"1277857902259323"
            },
            "message":"Bars alloy or cf?",
            "id":"231615900377587"
         },
         {
            "created_time":"2014-03-23T07:52:29+0000",
            "from":{
               "name":"Henri Le Comte",
               "id":"10153385684979159"
            },
            "message":"Alloy.",
            "id":"231616013710909"
         },
         {
            "created_time":"2014-03-23T07:52:39+0000",
            "from":{
               "name":"Henri Le Comte",
               "id":"10153385684979159"
            },
            "message":"240grams if memory serves",
            "id":"231616043710906"
         },
         {
            "created_time":"2014-03-23T07:54:57+0000",
            "from":{
               "name":"Henri Le Comte",
               "id":"10153385684979159"
            },
            "message":"Just weighed them Christopher, 245 grams on my scale",
            "id":"231616363710874"
         }
      ],
      "paging":{
         "cursors":{
            "before":"WTI5dGJXVnVkRjlqZAFhKemIzSTZANak14TmpFME1qUXdNemMzTnpVek9qRXpPVFUxTmpBeU5qRT0ZD",
            "after":"WTI5dGJXVnVkRjlqZAFhKemIzSTZANak14TmpFMk16WXpOekV3T0RjME9qRXpPVFUxTmpFeU9UYz0ZD"
         }
      }
   }
}

hive ddl:

create external table test(
  message string COMMENT 'from deserializer',
  permalink_url string COMMENT 'from deserializer',
  from_user STRUCT<name:string,id:string> COMMENT 'from deserializer',
  created_time string COMMENT 'from deserializer',
  updated_time string COMMENT 'from deserializer',
  type string COMMENT 'from deserializer',
  status_type string COMMENT 'from deserializer',
  is_expired string COMMENT 'from deserializer',
  id string COMMENT 'from deserializer',
  shares STRUCT<count:int> COMMENT 'from deserializer',
  likes STRUCT<likes_detail:ARRAY<STRUCT<id:bigint,name:string>>> COMMENT 'from deserializer',
  comments STRUCT<post_data:ARRAY<STRUCT<created_time:string,from_user:struct<name:string,id:bigint>,message:string,id:bigint>>> COMMENT 'from deserializer')
ROW FORMAT SERDE
  'org.openx.data.jsonserde.JsonSerDe'
WITH SERDEPROPERTIES ("mapping.from_user"="from")
STORED AS INPUTFORMAT
  'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
  'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
  'xxxx/post';
rcongiu commented 7 years ago

What's the issue here ? Closing, but feel free to reopen with more info.