GTFS-Realtime message verification failed with protobuf

[dancesWithCycles]

Hi folks, I hope this finds you well. I appreciate your work on this project very much and want to say thank you again. Today I tried to use protobuf.js to verify, encode and decode a GTFS-Realtime messages. Already the verification failed. Does anyone has a little advice for me how to handle this error?

protobuf.js version: ^6.10.2

I expected the java script to verify, encode, decode the message and output the respective debug messages. However, verification failed with an error I do not understand. Java script:

const protobuf = require("protobufjs");
const debug = require('debug')('protobuf');

protobuf.load("gtfs-realtime.proto", function(err, root) {
    debug('gtfs-realtime.proto loaded')
    if (err)
        throw err;

    // Obtain a message type
    var FeedMessage = root.lookupType("transit_realtime.FeedMessage");

    // Exemplary payload
    const v="2.0"
    let msgFeedHdr={
        gtfs_realtime_version:{v}
      }

      let msgFeedMsg={
        header:{msgFeedHdr}
      }

    // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
    var errMsg = FeedMessage.verify(msgFeedMsg);
    if (errMsg){
        debug('msg invalid')
        throw Error(errMsg);
    }else{
        debug('msg valid')
    }

    // Create a new message
    var message = FeedMessage.create(msgFeedMsg); // or use .fromObject if conversion is necessary
    debug('msg created')
    debug('msg: %s',message)

    // Encode a message to an Uint8Array (browser) or Buffer (node)
    var buffer = FeedMessage.encode(message).finish();
    // ... do something with buffer
    debug('msg encoded')
    debug('buffer: %s',buffer)

    // Decode an Uint8Array (browser) or Buffer (node) to a message
    var message = FeedMessage.decode(buffer);
    // ... do something with message
    debug('msg decoded')
    debug('msg: %s',message)

    // If the application uses length-delimited buffers, there is also encodeDelimited and decodeDelimited.

    // Maybe convert the message back to a plain object
    var object = FeedMessage.toObject(message, {
        longs: String,
        enums: String,
        bytes: String,
        // see ConversionOptions
    });
    debug('plain obj')
    debug('obj: %s',object)
});

Stack trace:

~/Desktop/sandbox/gtfsRtProtobuf$ node index.js 
  protobuf gtfs-realtime.proto loaded +0ms
  protobuf msg invalid +13ms
/home/user/Desktop/sandbox/gtfsRtProtobuf/index.js:26
        throw Error(errMsg);
        ^

Error: header.gtfsRealtimeVersion: string expected
    at /home/user/Desktop/sandbox/gtfsRtProtobuf/index.js:26:15
    at finish (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:105:9)
    at process (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:143:13)
    at /home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:194:17
    at fetchReadFileCallback (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/@protobufjs/fetch/index.js:51:19)
    at FSReqWrap.readFileAfterClose [as oncomplete] (internal/fs/read_file_context.js:53:3)

gtfs-realtime.proto:

...
// This protocol is published at:
// https://github.com/google/transit/tree/master/gtfs-realtime

syntax = "proto2";

package transit_realtime;

option java_package = "com.google.transit.realtime";

// The contents of a feed message.
// A feed is a continuous stream of feed messages. Each message in the stream is
// obtained as a response to an appropriate HTTP GET request.
// A realtime feed is always defined with relation to an existing GTFS feed.
// All the entity ids are resolved with respect to the GTFS feed.
// Note that "required" and "optional" as stated in this file refer to Protocol
// Buffer cardinality, not semantic cardinality.  See reference.md at
// https://github.com/google/transit/tree/master/gtfs-realtime for field
// semantic cardinality.
message FeedMessage {
  // Metadata about this feed and feed message.
  required FeedHeader header = 1;

  // Contents of the feed.
  repeated FeedEntity entity = 2;

  // The extensions namespace allows 3rd-party developers to extend the
  // GTFS Realtime specification in order to add and evaluate new features and
  // modifications to the spec.
  extensions 1000 to 1999;
}

// Metadata about a feed, included in feed messages.
message FeedHeader {
  // Version of the feed specification.
  // The current version is 2.0.
  required string gtfs_realtime_version = 1;

  // Determines whether the current fetch is incremental.  Currently,
  // DIFFERENTIAL mode is unsupported and behavior is unspecified for feeds
  // that use this mode.  There are discussions on the GTFS Realtime mailing
  // list around fully specifying the behavior of DIFFERENTIAL mode and the
  // documentation will be updated when those discussions are finalized.
  enum Incrementality {
    FULL_DATASET = 0;
    DIFFERENTIAL = 1;
  }
  optional Incrementality incrementality = 2 [default = FULL_DATASET];

  // This timestamp identifies the moment when the content of this feed has been
  // created (in server time). In POSIX time (i.e., number of seconds since
  // January 1st 1970 00:00:00 UTC).
  optional uint64 timestamp = 3;

  // The extensions namespace allows 3rd-party developers to extend the
  // GTFS Realtime specification in order to add and evaluate new features and
  // modifications to the spec.
  extensions 1000 to 1999;
}
...

Cheers!

[dancesWithCycles]

Hi folks, how are you doing? I did some investigation in the meantime. I am able now to create some messages and also verify them. However, I do not know how to check if encoding finished with success. Does anyone know some good error handling for this case? After encoding, decoding and logging message fields I got the impression that still something goes wrong. The version value seems to be undefined. Does anyone know why? Here is the code:

const Protobuf = require("protobufjs");
const ProtoFile=__dirname+'/gtfs-realtime.proto';
const debug = require('debug')('protobuf');

Protobuf.load(ProtoFile, function(err, root) {
    debug('%s loaded',ProtoFile)
    if (err)
        throw err;

    // Obtain a message type
    const FeedMessage = root.lookupType("transit_realtime.FeedMessage");
    const FeedHeader = root.lookupType("transit_realtime.FeedHeader");
    const FeedEntity = root.lookupType("transit_realtime.FeedEntity");
    const VehiclePosition = root.lookupType("transit_realtime.VehiclePosition");
    const Position = root.lookupType("transit_realtime.Position");

    var pos = Position.create({latitude:36,
    longitude:-79});
    debug('pos created')

    var vehiclePosition = VehiclePosition.create({postion:pos});
    debug('vehiclePosition created')

    var errMsg = VehiclePosition.verify(vehiclePosition);
    if (errMsg){
        debug('vehiclePosition invalid')
        throw Error(errMsg);
    }else{
        debug('vehiclePosition valid')
    }

    var feedEntity = FeedEntity.create({id:'uuid-foo-bar',
    vehicle:vehiclePosition});
    debug('feedEntity created')

    var errMsg = FeedEntity.verify(feedEntity);
    if (errMsg){
        debug('feedEntity invalid')
        throw Error(errMsg);
    }else{
        debug('feedEntity valid')
    }

    var feedHeader = FeedHeader.create({gtfs_realtime_version:'2.0'});
    debug('feedHeader created')

    var feedMessage = FeedMessage.create({header:feedHeader,
    entity:[feedEntity]});
    debug('feedMessage created')

    // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
    var errMsg = FeedMessage.verify(feedMessage);
    if (errMsg){
        debug('feedMessage invalid')
        throw Error(errMsg);
    }else{
        debug('feedMessage valid')
    }

    var encodedFeedMsg = FeedMessage.encode(feedMessage).finish();
    debug('feedMessage encoded')
    var decodedFeedMsg = FeedMessage.decode(encodedFeedMsg);
    debug('feedMessage decoded')
    // console.log('version: %s',decodedFeedMsg.header.gtfs_realtime_version);
    console.log(decodedFeedMsg.header.gtfs_realtime_version.toString('utf8'));

CLI output:

[nodemon] starting `node index.js`
  protobuf /home/user/Desktop/sandbox/gtfsRtProtobuf/gtfs-realtime.proto loaded +0ms
  protobuf pos created +7ms
  protobuf vehiclePosition created +1ms
  protobuf vehiclePosition valid +6ms
  protobuf feedEntity created +1ms
  protobuf feedEntity valid +4ms
  protobuf feedHeader created +1ms
  protobuf feedMessage created +0ms
  protobuf feedMessage valid +3ms
  protobuf feedMessage encoded +2ms
  protobuf feedMessage decoded +2ms
/home/user/Desktop/sandbox/gtfsRtProtobuf/index.js:65
    console.log(decodedFeedMsg.header.gtfs_realtime_version.toString('utf8'));
                                                            ^

TypeError: Cannot read property 'toString' of undefined
    at /home/user/Desktop/sandbox/gtfsRtProtobuf/index.js:65:61
    at finish (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:105:9)
    at process (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:143:13)
    at /home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/protobufjs/src/root.js:194:17
    at fetchReadFileCallback (/home/user/Desktop/sandbox/gtfsRtProtobuf/node_modules/@protobufjs/fetch/index.js:51:19)
    at FSReqWrap.readFileAfterClose [as oncomplete] (internal/fs/read_file_context.js:53:3)
[nodemon] app crashed - waiting for file changes before starting...

Cheers!

[dancesWithCycles]

Hi folks, I found the error and the solution myself. I still do not understand the solution. In the file gtfs-realtime.proto a header field is defined as gtfs_realtime_version. However, you have to use the field as gtfsRealtimeVersion to encode and decode a valid message. Does anyone knows the reason?

Here is the example code:

const Protobuf = require("protobufjs");
const ProtoFile=__dirname+'/gtfs-realtime.proto';
const debug = require('debug')('protobuf');

Protobuf.load(ProtoFile, function(err, root) {
    debug('%s loaded',ProtoFile)
    if (err)
        throw err;

    // Obtain a message type
    const FeedMessage = root.lookupType("transit_realtime.FeedMessage");
    const FeedHeader = root.lookupType("transit_realtime.FeedHeader");
    const FeedEntity = root.lookupType("transit_realtime.FeedEntity");
    const VehiclePosition = root.lookupType("transit_realtime.VehiclePosition");
    const Position = root.lookupType("transit_realtime.Position");

    var pos = Position.create({latitude:36,
    longitude:-79});
    debug('pos created')

    var vehiclePosition = VehiclePosition.create({position:pos});
    debug('vehiclePosition created')

    var feedEntity = FeedEntity.create({id:'uuid-foo-bar',
    vehicle:vehiclePosition});
    debug('feedEntity created')

    var feedHeader = FeedHeader.create({gtfsRealtimeVersion:'2.0',
    incrementality:0});
    debug('feedHeader created')

    var feedMessage = FeedMessage.create({header:feedHeader,
    entity:[feedEntity]});
    debug('feedMessage created')
    debug("JSON: %s", JSON.stringify(feedMessage));

    // Verify the payload if necessary (i.e. when possibly incomplete or invalid)
    var errMsg = FeedMessage.verify(feedMessage);
    if (errMsg){
        debug('feedMessage invalid')
        throw Error(errMsg);
    }else{
        debug('feedMessage valid')
    }

    var encodedFeedMsg = FeedMessage.encode(feedMessage).finish();
    debug('feedMessage encoded')
    var decodedFeedMsg = FeedMessage.decode(encodedFeedMsg);
    debug('feedMessage decoded')
    debug("JSON: %s", JSON.stringify(decodedFeedMsg));
});

And the respective output:

[nodemon] starting `node index.js`
  protobuf /home/user/Desktop/sandbox/gtfsRtProtobuf/gtfs-realtime.proto loaded +0ms
  protobuf pos created +7ms
  protobuf vehiclePosition created +1ms
  protobuf feedEntity created +1ms
  protobuf feedHeader created +0ms
  protobuf feedMessage created +1ms
  protobuf JSON: {"header":{"gtfsRealtimeVersion":"2.0","incrementality":"FULL_DATASET"},"entity":[{"id":"uuid-foo-bar","vehicle":{"position":{"latitude":36,"longitude":-79}}}]} +13ms
  protobuf feedMessage valid +1ms
  protobuf feedMessage encoded +3ms
  protobuf feedMessage decoded +2ms
  protobuf JSON: {"header":{"gtfsRealtimeVersion":"2.0","incrementality":"FULL_DATASET"},"entity":[{"id":"uuid-foo-bar","vehicle":{"position":{"latitude":36,"longitude":-79}}}]} +0ms
[nodemon] clean exit - waiting for changes before restart

Cheers!