ohmage / server

The ohmage server application.
37 stars 25 forks source link

observer stream_schema upload failure #844

Closed stevenolen closed 9 years ago

stevenolen commented 9 years ago

When updating my existing 2.16 server to the latest 2.16 commit (or any commit after ac38d0d) I am greeted with this server error upon attempted probe upload.

2014-10-08 00:00:02,469 [http-bio-8080-exec-2] [eeda95e3-66d5-41c9-8ba5-cd78530b9551 client=mobilize-android] [ERROR] [org.ohmage.request.observer.StreamUploadRequest] - org.ohmage.exception.ServiceException: org.ohmage.exception.DataAccessException: Error executing SQL 'SELECT os.stream_id, os.version, os.name, os.description, os.with_id, os.with_timestamp, os.with_location, os.stream_schema FROM observer_stream os, observer_stream_link osl WHERE osl.observer_id = ? AND osl.observer_stream_id = os.id' with parameter: 4
org.ohmage.exception.ServiceException: org.ohmage.exception.DataAccessException: Error executing SQL 'SELECT os.stream_id, os.version, os.name, os.description, os.with_id, os.with_timestamp, os.with_location, os.stream_schema FROM observer_stream os, observer_stream_link osl WHERE osl.observer_id = ? AND osl.observer_stream_id = os.id' with parameter: 4
        at org.ohmage.service.ObserverServices.getObservers(ObserverServices.java:473)
        at org.ohmage.request.observer.StreamUploadRequest.service(StreamUploadRequest.java:286)
        at org.ohmage.jee.servlet.RequestServlet.processRequest(RequestServlet.java:447)
        at org.ohmage.jee.servlet.RequestServlet.doPost(RequestServlet.java:413)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
        at org.ohmage.jee.servlet.RequestServlet.service(RequestServlet.java:307)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.ohmage.jee.filter.ClientFilter.doFilter(ClientFilter.java:167)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.ohmage.jee.filter.GzipFilter.doFilter(GzipFilter.java:116)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.ohmage.jee.filter.Log4jNdcFilter.doFilter(Log4jNdcFilter.java:76)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at com.thetransactioncompany.cors.CORSFilter.doFilter(Unknown Source)
        at com.thetransactioncompany.cors.CORSFilter.doFilter(Unknown Source)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.ohmage.jee.filter.Utf8RequestEncodingFilter.doFilter(Utf8RequestEncodingFilter.java:69)
        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
        at com.googlecode.psiprobe.Tomcat70AgentValve.invoke(Tomcat70AgentValve.java:38)
        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
        at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
        at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1040)
        at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:607)
        at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:313)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:744)
Caused by: org.ohmage.exception.DataAccessException: Error executing SQL 'SELECT os.stream_id, os.version, os.name, os.description, os.with_id, os.with_timestamp, os.with_location, os.stream_schema FROM observer_stream os, observer_stream_link osl WHERE osl.observer_id = ? AND osl.observer_stream_id = os.id' with parameter: 4
        at org.ohmage.query.impl.ObserverQueries.getObservers(ObserverQueries.java:531)
        at org.ohmage.service.ObserverServices.getObservers(ObserverServices.java:468)
        ... 39 more
Caused by: org.springframework.jdbc.UncategorizedSQLException: PreparedStatementCallback; uncategorized SQLException for SQL [SELECT os.stream_id, os.version, os.name, os.description, os.with_id, os.with_timestamp, os.with_location, os.stream_schema FROM observer_stream os, observer_stream_link osl WHERE osl.observer_id = ? AND osl.observer_stream_id = os.id]; SQL state [null]; error code [0]; org.ohmage.exception.DomainException: The schema is invalid: The 'fields' field is missing: {"type":"object","doc":"Service interaction.","schema":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"},{"name":"device_id","doc":"A unique id to identify a device, usually the imei.","type":"string","optional":true}]} (Concordia.js#422); nested exception is java.sql.SQLException: org.ohmage.exception.DomainException: The schema is invalid: The 'fields' field is missing: {"type":"object","doc":"Service interaction.","schema":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"},{"name":"device_id","doc":"A unique id to identify a device, usually the imei.","type":"string","optional":true}]} (Concordia.js#422)
        at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:83)
        at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:80)
        at org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:80)
        at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:603)
        at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:637)
        at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:666)
        at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:674)
        at org.springframework.jdbc.core.JdbcTemplate.query(JdbcTemplate.java:714)
        at org.ohmage.query.impl.ObserverQueries.getObservers(ObserverQueries.java:477)
        ... 40 more
Caused by: java.sql.SQLException: org.ohmage.exception.DomainException: The schema is invalid: The 'fields' field is missing: {"type":"object","doc":"Service interaction.","schema":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"},{"name":"device_id","doc":"A unique id to identify a device, usually the imei.","type":"string","optional":true}]} (Concordia.js#422)
        at org.ohmage.query.impl.ObserverQueries$4.mapRow(ObserverQueries.java:523)
        at org.ohmage.query.impl.ObserverQueries$4.mapRow(ObserverQueries.java:483)
        at org.springframework.jdbc.core.RowMapperResultSetExtractor.extractData(RowMapperResultSetExtractor.java:92)
        at org.springframework.jdbc.core.RowMapperResultSetExtractor.extractData(RowMapperResultSetExtractor.java:1)
        at org.springframework.jdbc.core.JdbcTemplate$1.doInPreparedStatement(JdbcTemplate.java:649)
        at org.springframework.jdbc.core.JdbcTemplate.execute(JdbcTemplate.java:587)
        ... 45 more
Caused by: org.ohmage.exception.DomainException: The schema is invalid: The 'fields' field is missing: {"type":"object","doc":"Service interaction.","schema":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"},{"name":"device_id","doc":"A unique id to identify a device, usually the imei.","type":"string","optional":true}]} (Concordia.js#422)
        at org.ohmage.domain.Observer$Stream.validateSchema(Observer.java:668)
        at org.ohmage.domain.Observer$Stream.<init>(Observer.java:248)
        at org.ohmage.query.impl.ObserverQueries$4.mapRow(ObserverQueries.java:512)
        ... 50 more
Caused by: org.mozilla.javascript.JavaScriptException: The 'fields' field is missing: {"type":"object","doc":"Service interaction.","schema":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"},{"name":"device_id","doc":"A unique id to identify a device, usually the imei.","type":"string","optional":true}]} (Concordia.js#422)
        at org.mozilla.javascript.gen.Concordia_js_2783._c_validateSchemaObject_16(Concordia.js:422)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.optimizer.OptRuntime.callName(OptRuntime.java:63)
        at org.mozilla.javascript.gen.Concordia_js_2783._c_validateSchemaInternal_29(Concordia.js:1180)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.optimizer.OptRuntime.callName(OptRuntime.java:63)
        at org.mozilla.javascript.gen.Concordia_js_2783._c_validateSchema_33(Concordia.js:1335)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.optimizer.OptRuntime.callName(OptRuntime.java:63)
        at org.mozilla.javascript.gen.Concordia_js_2783._c_anonymous_1(Concordia.js:1436)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.optimizer.OptRuntime.call1(OptRuntime.java:32)
        at org.mozilla.javascript.gen.Concordia_js_2783._c_Concordia_0(Concordia.js:32)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.ContextFactory.doTopCall(ContextFactory.java:394)
        at org.mozilla.javascript.ScriptRuntime.doTopCall(ScriptRuntime.java:3091)
        at org.mozilla.javascript.gen.Concordia_js_2783.call(Concordia.js)
        at org.mozilla.javascript.BaseFunction.construct(BaseFunction.java:338)
        at org.ohmage.domain.Observer$Stream.validateSchema(Observer.java:662)
        ... 52 more
stevenolen commented 9 years ago

This error is the result of dropping in the updated version of Concordia.js. The existing default observer stream_schemas in the database following an older schema syntax which is not compatible with the concordia parser now in use in later commits of 2.16. The fix is a bit awkward, but simple enough. If you are running only the default observers: LogProbe and Mobility you need only execute the sql statements below in order to fix. If you have added additional concordia streams or versioned the default streams while adhering to older Concordia schema rules, you will need to revise those stream_schemas as well in order to resolve this issue completely. Please refer to the Concordia definition documentation regarding the small changes made. Please note the where clause in these statements and adjust accordingly if you've added streams that may have namespace collisions or versioned the default streams.

Here are the sql statements needed in order to fix existing default observer streams in your db:

update observer_stream set stream_schema = '{"type":"object","doc":"Only requires the user\'s mode, which must be "error".","fields":[{"name":"mode","type":"string","doc":"The user\'s mode."}]}' where stream_id like 'error' and version = 2012061300;
update observer_stream set stream_schema = '{"type":"object","doc":"Only requires the user\'s mode.","fields":[{"name":"mode","type":"string","doc":"The user\'s mode."}]}' where stream_id like 'regular' and version = 2012050700;
update observer_stream set stream_schema = '{"type":"object","doc":"Contains the user\'s mode as well as the sensor data.","fields":[{"name":"mode","type":"string","doc":"The user\'s mode."},{"name":"speed","type":"number","doc":"The user\'s speed in the last minute.","optional":true},{"name":"accel_data","type":"array","constType":{"type":"object","doc":"A single accelerometer reading.","fields":[{"name":"x","type":"number","doc":"The x-component of the accelerometer reading."},{"name":"y","type":"number","doc":"The y-component of the accelerometer reading."},{"name":"z","type":"number","doc":"The z-component of the accelerometer reading."}]},"doc":"An array of the accelerometer readings over the last minute."},{"name":"wifi_data","type":"object","doc":"A WiFi reading from the last minute.","fields":[{"name":"time","type":"number","doc":"The time this data was recorded."},{"name":"timezone","type":"string","doc":"The time zone of the device when this data was recorded."},{"name":"scan","type":"array","constType":{"type":"object","doc":"A single access point\'s information.","fields":[{"name":"ssid","type":"string","doc":"The access point\'s SSID."},{"name":"strength","type":"number","doc":"The strength of the signal from the access point."}]},"doc":"The scan of WiFi information."}]}]}' where stream_id like 'extended' and version = 2012050700;
update observer_stream set stream_schema = '{"type":"object","doc":"Service interaction.","fields":[{"name":"service","doc":"The class name of the service.","type":"string"},{"name":"status","doc":"Status information about the service. Either ON or OFF.","type":"string"}]}' where stream_id like 'service' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"Widget interaction.","fields":[{"name":"id","doc":"The id of the view which produced this event.","type":"number"},{"name":"name","doc":"The human readable name of the view which produced this event. Will read the getContentDescription() of the view.","type":"string"},{"name":"extra","doc":"Extra information supplied for this interaction.","optional":true,"type":"string"}]}' where stream_id like 'widget' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"log message.","fields":[{"name":"level","doc":"Log level. One of error, warning, info, debug, or verbose.","type":"string"},{"name":"tag","doc":"Used to identify the source of a log message. It usually identifies the class or activity where the log call occurs.","type":"string"},{"name":"message","doc":"The message you would like logged.","type":"string"}]}' where stream_id like 'log' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"Network information.","fields":[{"name":"context","doc":"Class name of the context which started the call to the network.","type":"string"},{"name":"resource","doc":"Name of the resource requested","type":"string"},{"name":"network_state","doc":"upload or download","type":"string"},{"name":"length","doc":"Number of bytes transmitted","type":"number"}]}' where stream_id like 'network' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"Activity interaction.","fields":[{"name":"activity","doc":"The class name of the activity.","type":"string"},{"name":"status","doc":"Status information about the activity. Either ON or OFF.","type":"string"}]}' where stream_id like 'activity' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"Prompt interaction.","fields":[{"name":"id","doc":"The promptId.","type":"string"},{"name":"type","doc":"The prompt type.","type":"string"},{"name":"status","doc":"Status information about the activity. Either ON or OFF.","type":"string"}]}' where stream_id like 'prompt' and version = 1;
update observer_stream set stream_schema = '{"type":"object","doc":"Trigger interaction.","fields":[{"name":"action","doc":"The action that was currently applied to the trigger. One of delete, add, trigger, update, or ignore.","type":"string"},{"name":"type","doc":"Trigger type.","type":"string"},{"name":"count","doc":"Current number of triggers for this campaign and trigger type set up in the system.","type":"number"},{"name":"campaign","doc":"The campaign urn to which this trigger belongs.","type":"string"}]}' where stream_id like 'trigger' and version = 1;