search

watson-developer-cloud / java-sdk

:1st_place_medal: Java SDK to use the IBM Watson services.
http://watson-developer-cloud.github.io/java-sdk/
Apache License 2.0
590 stars 533 forks source link

Parse Date Error - Data News in Core #474

Closed remkohdev closed 7 years ago

remkohdev commented 7 years ago

When using the Java SDK for the Alchemy Data News https://github.com/watson-developer-cloud/java-sdk

on Liberty Localhost http://localhost:9080/JavaCloudantDBApp/api/watson/news?startdate=2016-10-01&enddate=2016-10-30&searchterm=elections&count=5

String[] fields =
        new String[] { "enriched.url.title", "enriched.url.url", "enriched.url.author", "enriched.url.publicationDate",
            "enriched.url.enrichedTitle.entities", "enriched.url.enrichedTitle.docSentiment"};
    params.put(AlchemyDataNews.RETURN, StringUtils.join(fields, ","));
    params.put(AlchemyDataNews.START, startdate);
    params.put(AlchemyDataNews.END, enddate);
    params.put(AlchemyDataNews.COUNT, count);

    // Query on adjacent nested fields:
    params.put("q.enriched.url.enrichedTitle.keywords.keyword.text", searchTerm);
    //params.put("q.enriched.url.enrichedTitle.entities.entity", "|text=IBM,type=company|");
    //params.put("q.enriched.url.enrichedTitle.docSentiment.type", "positive");
    //params.put("q.enriched.url.enrichedTitle.taxonomy.taxonomy_.label", "technology and computing");

    DocumentsResult result = service.getNewsDocuments(params).execute();

All goes well except I get a parse error on date [ERROR ] Error parsing: 1477799489 Unparseable date: "1477799489" [ERROR ] Error parsing: 1477798314 Unparseable date: "1477798314" [ERROR ] Error parsing: 1477797798 Unparseable date: "1477797798" [ERROR ] Error parsing: 1477797337 Unparseable date: "1477797337" [ERROR ] Error parsing: 1477796981 Unparseable date: "1477796981"

This error is thrown in the console in the DocumentsResult result = service.getNewsDocuments(params).execute(); line

when I change count to 5 to 10 the errors increase equal to count, so coming from the result json.

[{"docs":"{\n \"docs\": [\n {\n \"id\": \"MzUxODg3Mzk1NHwxNDc3Nzk5NDg5\",\n \"source\": {\n \"enriched\": {\n \"url\": {\n \"author\": \"\",\n \"enrichedTitle\": {\n \"entities\": [\n {\n \"count\": 1,\n \"disambiguated\": {},\n \"knowledgeGraph\": {},\n \"quotations\": [],\n \"relevance\": 0.00999999978,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n },\n \"text\": \"80%\",\n \"type\": \"Quantity\"\n }\n ],\n \"docSentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n }\n },\n \"publicationDate\": {\n \"confident\": false,\n \"date\": \"2016-10-30T00:00:00.000\"\n },\n \"title\": \"With 80% of Ballots Counted Independence Party Leads Icelandic Elections - USweekly\",\n \"url\": \"http://usweekly.com/news/390/67814/With_80_of_Ballots_Counted_Independence_Party_Leads_Icelandic_Elections.html\"\n }\n }\n }\n },\n {\n \"id\": \"MzUxNjIxMTcxNnwxNDc3Nzk4MzE0\",\n \"source\": {\n \"enriched\": {\n \"url\": {\n \"author\": \"Blasting News\",\n \"enrichedTitle\": {\n \"entities\": [\n {\n \"count\": 1,\n \"disambiguated\": {\n \"dbpedia\": \"http://dbpedia.org/resource/FBi_Radio\",\n \"geo\": \"\",\n \"name\": \"FBi Radio\",\n \"subType\": [\n \"Broadcast\",\n \"RadioStation\"\n ],\n \"website\": \"http://www.fbiradio.com/\"\n },\n \"knowledgeGraph\": {\n \"typeHierarchy\": \"/organizations/agencies/fbi\"\n },\n \"quotations\": [],\n \"relevance\": 0.330000013,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n },\n \"text\": \"FBI\",\n \"type\": \"Organization\"\n },\n {\n \"count\": 1,\n \"disambiguated\": {},\n \"knowledgeGraph\": {\n \"typeHierarchy\": \"/people/democrats/hillary\"\n },\n \"quotations\": [],\n \"relevance\": 0.330000013,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n },\n \"text\": \"Hillary\",\n \"type\": \"Person\"\n },\n {\n \"count\": 1,\n \"disambiguated\": {},\n \"knowledgeGraph\": {},\n \"quotations\": [],\n \"relevance\": 0.330000013,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.391602993,\n \"type\": \"positive\"\n },\n \"text\": \"Trump\",\n \"type\": \"Company\"\n }\n ],\n \"docSentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.374644995,\n \"type\": \"positive\"\n }\n },\n \"publicationDate\": {\n \"confident\": false,\n \"date\": \"2016-10-30T00:00:00.000\"\n },\n \"title\": \"New FBI Investigation on Hillary’s ‘lost emails’ can put elections in Trump’s favour\",\n \"url\": \"http://uk.blastingnews.com/politics/2016/10/new-fbi-investigation-on-hillary-s-lost-emails-can-put-elections-in-trump-s-favour-001219069.html\"\n }\n }\n }\n },\n {\n \"id\": \"MzUwMzk4OTc3NXwxNDc3Nzk3Nzk4\",\n \"source\": {\n \"enriched\": {\n \"url\": {\n \"author\": \"\",\n \"enrichedTitle\": {\n \"entities\": [],\n \"docSentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n }\n },\n \"publicationDate\": {\n \"confident\": false,\n \"date\": \"2016-10-28T00:00:00.000\"\n },\n \"title\": \"Nov. 8 elections update\",\n \"url\": \"http://www.reporternews.com/news/local/nov-8-elections-update-3fc6643d-399e-19a4-e053-0100007ff62b-399192141.html\"\n }\n }\n }\n },\n {\n \"id\": \"MTY5NDkxNDExMjF8MTQ3Nzc5NzMzNw\",\n \"source\": {\n \"enriched\": {\n \"url\": {\n \"author\": \"Lebanese authorities crack terror network\",\n \"enrichedTitle\": {\n \"entities\": [\n {\n \"count\": 1,\n \"disambiguated\": {},\n \"knowledgeGraph\": {\n \"typeHierarchy\": \"/organizations/agencies/council/fifa council\"\n },\n \"quotations\": [],\n \"relevance\": 0.330000013,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n },\n \"text\": \"FIFA council\",\n \"type\": \"Organization\"\n }\n ],\n \"docSentiment\": {\n \"mixed\": \"0\",\n \"score\": 0.0,\n \"type\": \"neutral\"\n }\n },\n \"publicationDate\": {\n \"confident\": false,\n \"date\": \"2016-10-30T00:00:00.000\"\n },\n \"title\": \"Soccer-Asian elections for FIFA council to be held in February | Zawya\",\n \"url\": \"http://www.zawya.com/mena/en/story/Asian_elections_for_FIFA_council_to_be_held_in_February-TR20161029nL4N1CZ05BX2/\"\n }\n }\n }\n },\n {\n \"id\": \"MzUwODkyNTkyNnwxNDc3Nzk2OTgx\",\n \"source\": {\n \"enriched\": {\n \"url\": {\n \"author\": \"Post Publishing PCL.\",\n \"enrichedTitle\": {\n \"entities\": [\n {\n \"count\": 1,\n \"disambiguated\": {\n \"dbpedia\": \"http://dbpedia.org/resource/United_States\",\n \"geo\": \"\",\n \"name\": \"United States\",\n \"subType\": [\n \"Location\",\n \"Region\",\n \"AdministrativeDivision\",\n \"GovernmentalJurisdiction\",\n \"FilmEditor\"\n ],\n \"website\": \"http://www.usa.gov/\"\n },\n \"knowledgeGraph\": {\n \"typeHierarchy\": \"/people/us\"\n },\n \"quotations\": [],\n \"relevance\": 0.330000013,\n \"sentiment\": {\n \"mixed\": \"0\",\n \"score\": -0.604476988,\n \"type\": \"negative\"\n },\n \"text\": \"US\",\n \"type\": \"Country\"\n }\n ],\n \"docSentiment\": {\n \"mixed\": \"0\",\n \"score\": -0.612100005,\n \"type\": \"negative\"\n }\n },\n \"publicationDate\": {\n \"confident\": false,\n \"date\": \"2016-10-30T09:45:00.000\"\n },\n \"title\": \"Elections or not, US Fed likely to stand pat on rates\",\n \"url\": \"http://www.bangkokpost.com/news/world/1122785/elections-or-not-us-fed-likely-to-stand-pat-on-rates\"\n }\n }\n }\n }\n ],\n \"next\": \"Blablablabla\"\n}","startdate":"2016-10-01","enddate":"2016-10-30","searchterm":"elections","count":"5"}]

package com.remkohde.dev.liberty;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.HashMap;
import java.util.Map;

import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;

import org.apache.commons.lang3.StringUtils;

import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.ibm.watson.developer_cloud.alchemy.v1.AlchemyDataNews;
import com.ibm.watson.developer_cloud.alchemy.v1.model.Documents;
import com.ibm.watson.developer_cloud.alchemy.v1.model.DocumentsResult;

@Path("/watson/news")
/**
 * REST service to search AlchemyData News API.
 * @see http://docs.alchemyapi.com/
 */
public class WatsonNewsServlet {
private JsonObject bluemixConf = null;
private String alchemyApikey = null;

public WatsonNewsServlet() {
    loadBluemixProperties();
    loadAlchemyApikey();
}

@GET
@Produces(MediaType.APPLICATION_JSON)
public Response get(@QueryParam("startdate") String startdate, @QueryParam("enddate") String enddate, 
        @QueryParam("searchterm") String searchterm, @QueryParam("count") String count) 
throws Exception {      
    JsonArray jsonArrayResponse = new JsonArray();

    SimpleDateFormat format1 = new SimpleDateFormat("yyyy-MM-dd");

    Calendar startdate1 = Calendar.getInstance();
    startdate1.setTime(format1.parse(startdate));
    long startdate2 = startdate1.getTimeInMillis()/1000;
    System.out.println("startdate2: "+startdate2);

    Calendar enddate1 = Calendar.getInstance();
    enddate1.setTime(format1.parse(enddate));
    long enddate2 = enddate1.getTimeInMillis()/1000;
    System.out.println("1");
    int cnt = Integer.valueOf(count).intValue();

    DocumentsResult result =  this.getAlchemyDataNews(startdate2, enddate2, searchterm, cnt) ;
    System.out.println("2");
    Documents docs = result.getDocuments();

    JsonObject jsonObject = new JsonObject();
    jsonObject.addProperty("docs", docs.toString());
    jsonObject.addProperty("startdate", startdate);
    jsonObject.addProperty("enddate", enddate);
    jsonObject.addProperty("searchterm", searchterm);
    jsonObject.addProperty("count", count);
    jsonArrayResponse.add(jsonObject);

    return Response.ok(jsonArrayResponse.toString()).build();
}

/**
 * 
 * @param startdate
 * @param enddate
 * @param searchTerm
 * @param entityType
 * @param count
 * @return
 */
public DocumentsResult getAlchemyDataNews(long startdate, long enddate, String searchTerm, int count) {     
    /**
    // 1day = 24 hours * 60 minutes * 60 seconds = 86400
    long oneday = new Long(86400).longValue();  
    Calendar now = Calendar.getInstance();
    long nowdays = (now.getTimeInMillis()/1000) / oneday;   
    long startdays = nowdays - (startdate / oneday);
    long enddays = nowdays - (enddate / oneday);
    String startdateindays = "now-"+startdays+"d";
    String enddateindays = "now-"+enddays+"d";
    */ 
    // Configure the AlchemyAPI Data News
    AlchemyDataNews service = new AlchemyDataNews(alchemyApikey);

    Map<String, Object> params = new HashMap<String, Object>();

    String[] fields =
        new String[] { "enriched.url.title", "enriched.url.url", "enriched.url.author", "enriched.url.publicationDate",
            "enriched.url.enrichedTitle.entities", "enriched.url.enrichedTitle.docSentiment"};
    params.put(AlchemyDataNews.RETURN, StringUtils.join(fields, ","));
    params.put(AlchemyDataNews.START, startdate);
    params.put(AlchemyDataNews.END, enddate);
    params.put(AlchemyDataNews.COUNT, count);

    // Query on adjacent nested fields:
    params.put("q.enriched.url.enrichedTitle.keywords.keyword.text", searchTerm);
    //params.put("q.enriched.url.enrichedTitle.entities.entity", "|text=IBM,type=company|");
    //params.put("q.enriched.url.enrichedTitle.docSentiment.type", "positive");
    //params.put("q.enriched.url.enrichedTitle.taxonomy.taxonomy_.label", "technology and computing");

    DocumentsResult result = service.getNewsDocuments(params).execute();

    return result;
}

/**
 * Method to load the Bluemix configuration from either the Bluemix system 
 * variable VCAP_SERVICES and if not available from Bluemix, for instance
 * because running on localhost, load the configuration from a local 
 * properties file ~/WebContent/WEB-INF/classes/bluemix.json
 */
private void loadBluemixProperties(){
    try {
        String vcapServices = System.getenv("VCAP_SERVICES");
        if (vcapServices != null) {
            this.bluemixConf = (JsonObject) new JsonParser().parse(vcapServices);
        }else{
            InputStream in = getClass().getClassLoader().getResourceAsStream("bluemix.json");               
            BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8"));       
            StringBuilder sb = new StringBuilder();
            String inputStr;
            while((inputStr = streamReader.readLine()) != null){
                sb.append(inputStr);
            }           
            this.bluemixConf = (JsonObject) new JsonParser().parse(sb.toString());
        }
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    } catch (IOException ioe) {
        ioe.printStackTrace();
    } 
}

/**
 * Load the AlchemyAPI apikey from the Bluemix configuration.
 */
private void loadAlchemyApikey() {      
    JsonArray alchemyAPIConfig = bluemixConf.getAsJsonArray("alchemy_api");
    JsonObject alchemyAPIConfig1 = (JsonObject) alchemyAPIConfig.get(0); 
    JsonObject alchemyAPICredentials = alchemyAPIConfig1.getAsJsonObject("credentials");
    this.alchemyApikey = alchemyAPICredentials.get("apikey").getAsString();
    System.out.println("AlchemyApikey: "+alchemyApikey);        
}

}
kognate commented 7 years ago

Sorry to hear you are having problems with this.

I just pushed a PR that (I think) fixes this. If you could try that branch (#475) and let me know if it fixes your issue that would be great.

germanattanasio commented 7 years ago

@remkohdev can you test if the changes now in master fix your issue ?