twitter / twurl

OAuth-enabled curl for the Twitter API
https://developer.twitter.com
MIT License
1.79k stars 303 forks source link

Invalid Json output? #54

Closed wcpreston closed 9 years ago

wcpreston commented 9 years ago

I have been using twurl for over a year to run a series of twitter searches. I feed the json output to a json2csv system, and it started complaining about a month ago that the json output is invalid. I've taken the output of a number of searches and fed them into multiple json2csv systems online and they all say invalid json.

Did something happen between Nov 5 & 15th that would have changed this? I updated to the latest version of twurl and that didn't fix it.

I know squat about json, so I can't look at it and see what's wrong. All I know is all the json converters are complaining it's not valid. I'll cut and paste an example below.

{"statuses":[{"metadata":{"iso_language_code":"en","result_type":"recent"},"created_at":"Wed Dec 17 21:50:34 +0000 2014","id":545335269185294337,"id_str":"545335269185294337","text":"This is DIY data disaster recovery. @ AHK Electronic Sheetmetal http:\/\/t.co\/HfDCqKIu5p","source":"\u003ca href=\"http:\/\/instagram.com\" rel=\"nofollow\"\u003eInstagram\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":14364184,"id_str":"14364184","name":"Jason Sholl","screen_name":"jasonsholl","location":"San Jose, CA","profile_location":null,"description":"Founder of @ProReferee, #television & #digitalmedia Executive, 15-time @TellyAwards winning #producer\/#director, soccer #referee, @SCUGradBiz EMBA '14","url":"http:\/\/t.co\/QCmDPcqoce","entities":{"url":{"urls":[{"url":"http:\/\/t.co\/QCmDPcqoce","expanded_url":"http:\/\/www.jasonsholl.com","display_url":"jasonsholl.com","indices":[0,22]}]},"description":{"urls":[]}},"protected":false,"followers_count":142,"friends_count":155,"listed_count":13,"created_at":"Fri Apr 11 20:13:57 +0000 2008","favourites_count":18,"utc_offset":-28800,"time_zone":"Pacific Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":2557,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"0E0D02","profile_background_image_url":"http:\/\/pbs.twimg.com\/profile_background_images\/378800000086223477\/d3337f4a0488aaada2c7e54b6bce1f14.jpeg","profile_background_image_url_https":"https:\/\/pbs.twimg.com\/profile_background_images\/378800000086223477\/d3337f4a0488aaada2c7e54b6bce1f14.jpeg","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/523598663172382720\/XtR_sa96_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/523598663172382720\/XtR_sa96_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/14364184\/1411005930","profile_link_color":"5681CC","profile_sidebar_border_color":"BEDFB6","profile_sidebar_fill_color":"0E0D02","profile_text_color":"39BD91","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":{"type":"Point","coordinates":[37.1432508,-121.6488283]},"coordinates":{"type":"Point","coordinates":[-121.6488283,37.1432508]},"place":{"id":"e872bcd2497287a7","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/e872bcd2497287a7.json","place_type":"city","name":"Morgan Hill","full_name":"Morgan Hill, CA","country_code":"US","country":"United States","contained_within":[],"bounding_box":{"type":"Polygon","coordinates":[[[-121.70014,37.088404],[-121.583333,37.088404],[-121.583333,37.16931],[-121.70014,37.16931]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[{"url":"http:\/\/t.co\/HfDCqKIu5p","expanded_url":"http:\/\/instagram.com\/p\/wuToh7LM0C\/","display_url":"instagram.com\/p\/wuToh7LM0C\/","indices":[64,86]}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"en"},{"metadata":{"iso_language_code":"en","result_type":"recent"},"created_at":"Wed Dec 17 15:53:20 +0000 2014","id":545245366191661056,"id_str":"545245366191661056","text":"Fun, brief video about #datacenter traffic. #CiscoGCI http:\/\/t.co\/z43TuYDws7 http:\/\/t.co\/gyOe7qQ95n #Cloud","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":809092,"id_str":"809092","name":"Mark Yolton","screen_name":"MarkYolton","location":"\u00dcT: 37.3920047,-122.1468241","profile_location":null,"description":"VP Digital (web, social, mobile, video) at Cisco. Views are mine, not necessarily my employer's. Also: husband, dad, Captain, bro ... http:\/\/t.co\/mTPHBPPB8g","url":"http:\/\/t.co\/mTPHBPPB8g","entities":{"url":{"urls":[{"url":"http:\/\/t.co\/mTPHBPPB8g","expanded_url":"http:\/\/about.me\/markyolton","display_url":"about.me\/markyolton","indices":[0,22]}]},"description":{"urls":[{"url":"http:\/\/t.co\/mTPHBPPB8g","expanded_url":"http:\/\/about.me\/markyolton","display_url":"about.me\/markyolton","indices":[134,156]}]}},"protected":false,"followers_count":7577,"friends_count":1317,"listed_count":456,"created_at":"Sat Mar 03 22:34:51 +0000 2007","favourites_count":471,"utc_offset":-28800,"time_zone":"Pacific Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":11041,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"022330","profile_background_image_url":"http:\/\/pbs.twimg.com\/profile_background_images\/743498470\/bd46012fd7157d5260cd1be994f250e0.jpeg","profile_background_image_url_https":"https:\/\/pbs.twimg.com\/profile_background_images\/743498470\/bd46012fd7157d5260cd1be994f250e0.jpeg","profile_background_tile":false,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/378800000478983905\/a7c469f9cea14cf932a221442bfda29b_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/378800000478983905\/a7c469f9cea14cf932a221442bfda29b_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/809092\/1389471588","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"C0DFEC","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":{"type":"Point","coordinates":[37.43316324,-121.89547864]},"coordinates":{"type":"Point","coordinates":[-121.89547864,37.43316324]},"place":{"id":"6ef29a7e78ca38a5","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/6ef29a7e78ca38a5.json","place_type":"city","name":"Milpitas","full_name":"Milpitas, CA","country_code":"US","country":"United States","contained_within":[],"bounding_box":{"type":"Polygon","coordinates":[[[-121.931305,37.39609],[-121.861066,37.39609],[-121.861066,37.466856],[-121.931305,37.466856]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"datacenter","indices":[23,34]},{"text":"CiscoGCI","indices":[44,53]},{"text":"Cloud","indices":[100,106]}],"symbols":[],"user_mentions":[],"urls":[{"url":"http:\/\/t.co\/z43TuYDws7","expanded_url":"http:\/\/cs.co\/6013v6SB","display_url":"cs.co\/6013v6SB","indices":[54,76]}],"media":[{"id":545237749906014208,"id_str":"545237749906014208","indices":[77,99],"media_url":"http:\/\/pbs.twimg.com\/media\/B5ES2EFIAAAt0rj.png","media_url_https":"https:\/\/pbs.twimg.com\/media\/B5ES2EFIAAAt0rj.png","url":"http:\/\/t.co\/gyOe7qQ95n","display_url":"pic.twitter.com\/gyOe7qQ95n","expanded_url":"http:\/\/twitter.com\/CiscoSP360\/status\/545237750300311552\/photo\/1","type":"photo","sizes":{"medium":{"w":600,"h":598,"resize":"fit"},"small":{"w":340,"h":339,"resize":"fit"},"large":{"w":846,"h":844,"resize":"fit"},"thumb":{"w":150,"h":150,"resize":"crop"}},"source_status_id":545237750300311552,"source_status_id_str":"545237750300311552","source_user_id":14616081,"source_user_id_str":"14616081"}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"en"},{"metadata":{"iso_language_code":"en","result_type":"recent"},"created_at":"Thu Dec 11 05:14:34 +0000 2014","id":542910291139248129,"id_str":"542910291139248129","text":"Good thing about being a datacenter nerd: I have several ups batteries laying around. #hellastorm","source":"\u003ca href=\"http:\/\/twitter.com\/download\/iphone\" rel=\"nofollow\"\u003eTwitter for iPhone\u003c\/a\u003e","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":17216994,"id_str":"17216994","name":"Andrew","screen_name":"AndrewKuhlmann","location":"North Oakland \/\/ SF","profile_location":null,"description":"Robot Commander @Piston, dad of a human cat. Opinions are like, my opinions, man.","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":53,"friends_count":160,"listed_count":0,"created_at":"Thu Nov 06 19:12:12 +0000 2008","favourites_count":34,"utc_offset":-28800,"time_zone":"Tijuana","geo_enabled":true,"verified":false,"statuses_count":511,"lang":"en","contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"131516","profile_background_image_url":"http:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_image_url_https":"https:\/\/abs.twimg.com\/images\/themes\/theme14\/bg.gif","profile_background_tile":true,"profile_image_url":"http:\/\/pbs.twimg.com\/profile_images\/541750475008122880\/0NFt-3Gq_normal.jpeg","profile_image_url_https":"https:\/\/pbs.twimg.com\/profile_images\/541750475008122880\/0NFt-3Gq_normal.jpeg","profile_banner_url":"https:\/\/pbs.twimg.com\/profile_banners\/17216994\/1417848103","profile_link_color":"009999","profile_sidebar_border_color":"EEEEEE","profile_sidebar_fill_color":"EFEFEF","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":false,"follow_request_sent":false,"notifications":false},"geo":{"type":"Point","coordinates":[37.85049224,-122.25722061]},"coordinates":{"type":"Point","coordinates":[-122.25722061,37.85049224]},"place":{"id":"ab2f2fac83aa388d","url":"https:\/\/api.twitter.com\/1.1\/geo\/id\/ab2f2fac83aa388d.json","place_type":"city","name":"Oakland","full_name":"Oakland, CA","country_code":"US","country":"United States","contained_within":[],"bounding_box":{"type":"Polygon","coordinates":[[[-122.34266,37.699279],[-122.114711,37.699279],[-122.114711,37.8847092],[-122.34266,37.8847092]]]},"attributes":{}},"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"hellastorm","indices":[86,97]}],"symbols":[],"user_mentions":[],"urls":[]},"favorited":false,"retweeted":false,"lang":"en"}],"search_metadata":{"completed_in":0.089,"max_id":545335269185294337,"max_id_str":"545335269185294337","query":"%22backup+software%22+OR+%22disaster+recovery%22+OR+datacenter","refresh_url":"?since_id=545335269185294337&q=%22backup%20software%22%20OR%20%22disaster%20recovery%22%20OR%20datacenter&geocode=37.7749%2C-122.4194%2C100mi&lang=en&result_type=recent&include_entities=1","count":15,"since_id":0,"since_id_str":"0"}}

wcpreston commented 9 years ago

Wait... Apparently updating to the new twurl DID fix the issue.

But I do still have a problem with my json2csv script. Anyone have a good command line json2csv converter?