Not getting email notifications: gmailpush.getEmailAddress parse issues

[shelomito12]

@byeokim @aivantg Hey guys,

I've recently installed Gmailpush API and so far I'm unable to get email push notifications. The code is very identical to the request sample code from your readme section:

require('dotenv').config();
const jsonToken = require('./token.json');
const Gmailpush = require('gmailpush');
const express = require('express');
const app = express();

// Initialize with OAuth2 config and Pub/Sub topic
const gmailpush = new Gmailpush({
  clientId: process.env.CLIENT_ID,
  clientSecret: process.env.CLIENT_SECRET,
  pubsubTopic: process.env.TOPIC_URL
});

const users = [
  {
    email: process.env.EMAIL,
    token: {
      access_token: jsonToken.access_token,
      refresh_token: jsonToken.refresh_token,
      scope: jsonToken.scope,
      token_type: jsonToken.token_type,
      expiry_date: jsonToken.expiry_date
    }
  }
];

app.post(
  // Use URL set as Pub/Sub Subscription endpoint
  '/pubsub',
  // Parse JSON request payload
  express.json(),
  (req, res) => {
    // Acknowledge Gmail push notification webhook
    res.sendStatus(200); // added .end()
    console.log('request', req.body); // added this
    // Get Email address contained in the push notification
    const email = gmailpush.getEmailAddress(req.body);
    console.log('email', email); //added this
    // Get access token for the Email address
    const token = users.find((user) => user.email === email).token;

    gmailpush
      .getMessages({
        notification: req.body,
        token,
        withLabelIds: ['UNREAD']
      })
      .then((messages) => {
        console.log('messages', messages);
         fs.readFile('./gmailpush_history.json')
          .then((result) => {
            const prevHistories = JSON.parse(result);
            const prevHistory = prevHistories.find((prevHistory) => prevHistory.emailAddress === email);
            schedule.scheduleJob(new Date(prevHistory.watchExpiration - 1000 * 60 * 30), async () => {
              prevHistory.watchExpiration = await gmailpush._refreshWatch();
              fs.writeFile('./gmailpush_history.json', JSON.stringify(prevHistories));
            });
          });
      })
      .catch((err) => {
        console.log(err);
      });
  }
);

app.listen(3002, () => {
  console.log('Server listening on port 3002...');
});

I tried sending a test email from my dashboard (https://console.cloud.google.com/cloudpubsub/topic/) and noticed the following errors in my nodejs server console log:

It seems these errors are caused by: const email = gmailpush.getEmailAddress(req.body);

Error

ubuntu@vps1:~/gmailpush$ node index.js
Server listening on port 3002...
request {
  message: {
    data: 'SGVsbG8gV09STEQ=',
    messageId: '6340826340304395',
    message_id: '6340826340304395',
    publishTime: '2022-11-22T14:55:21.828Z',
    publish_time: '2022-11-22T14:55:21.828Z'
  },
  subscription: 'projects/{hidden}/subscriptions/GmailAPIPush-sub'
}
SyntaxError: Unexpected token H in JSON at position 0
    at JSON.parse (<anonymous>)
    at Gmailpush._parseNotificationPayload (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:43:17)
    at Gmailpush.getEmailAddress (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:59:17)
    at /home/ubuntu/gmailpush/index.js:39:29
    at Layer.handle [as handle_request] (/home/ubuntu/gmailpush/node_modules/express/lib/router/layer.js:95:5)
    at next (/home/ubuntu/gmailpush/node_modules/express/lib/router/route.js:144:13)
    at /home/ubuntu/gmailpush/node_modules/body-parser/lib/read.js:137:5
    at AsyncResource.runInAsyncScope (node:async_hooks:203:9)
    at invokeCallback (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:231:16)
    at done (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:220:7)

[shelomito12]

OK I've sent some random message in JSON format and another issue displayed: const token = users.find((user) => user.email === email).token;

request {
  message: {
    data: 'W3sKICAiaWQiOiAiVW5pdGVkIFN0YXRlcyIsCiAgImZpcnN0X25hbWUiOiAiRW15bGUiLAogICJsYXN0X25hbWUiOiAiRGlubmVnZXMiLAogICJlbWFpbCI6ICJlZGlubmVnZXMwQHlhbmRleC5ydSIsCiAgImdlbmRlciI6ICJGZW1hbGUiLAogICJpcF9hZGRyZXNzIjogIjE1MC44Ny4yMzQuMjA1Igp9LCB7CiAgImlkIjogIlJ1c3NpYSIsCiAgImZpcnN0X25hbWUiOiAiQ2F0aGEiLAogICJsYXN0X25hbWUiOiAiTWlsbGVuIiwKICAiZW1haWwiOiAiY21pbGxlbjFAc2l0ZW1ldGVyLmNvbSIsCiAgImdlbmRlciI6ICJGZW1hbGUiLAogICJpcF9hZGRyZXNzIjogIjE3NC4yMDkuNTIuMTA0Igp9XQ==',
    messageId: '6342237161442824',
    message_id: '6342237161442824',
    publishTime: '2022-11-22T18:37:27.705Z',
    publish_time: '2022-11-22T18:37:27.705Z'
  },
  subscription: 'projects/{hidden}/subscriptions/GmailAPIPush-sub'
}
email {
  message: {
    data: 'W3sKICAiaWQiOiAiVW5pdGVkIFN0YXRlcyIsCiAgImZpcnN0X25hbWUiOiAiRW15bGUiLAogICJsYXN0X25hbWUiOiAiRGlubmVnZXMiLAogICJlbWFpbCI6ICJlZGlubmVnZXMwQHlhbmRleC5ydSIsCiAgImdlbmRlciI6ICJGZW1hbGUiLAogICJpcF9hZGRyZXNzIjogIjE1MC44Ny4yMzQuMjA1Igp9LCB7CiAgImlkIjogIlJ1c3NpYSIsCiAgImZpcnN0X25hbWUiOiAiQ2F0aGEiLAogICJsYXN0X25hbWUiOiAiTWlsbGVuIiwKICAiZW1haWwiOiAiY21pbGxlbjFAc2l0ZW1ldGVyLmNvbSIsCiAgImdlbmRlciI6ICJGZW1hbGUiLAogICJpcF9hZGRyZXNzIjogIjE3NC4yMDkuNTIuMTA0Igp9XQ==',
    messageId: '6342237161442824',
    message_id: '6342237161442824',
    publishTime: '2022-11-22T18:37:27.705Z',
    publish_time: '2022-11-22T18:37:27.705Z'
  },
  subscription: 'projects/{hidden}/subscriptions/GmailAPIPush-sub'
}
TypeError: Cannot read properties of undefined (reading 'token')
    at /home/ubuntu/gmailpush/index.js:42:61
    at Layer.handle [as handle_request] (/home/ubuntu/gmailpush/node_modules/express/lib/router/layer.js:95:5)
    at next (/home/ubuntu/gmailpush/node_modules/express/lib/router/route.js:144:13)
    at /home/ubuntu/gmailpush/node_modules/body-parser/lib/read.js:137:5
    at AsyncResource.runInAsyncScope (node:async_hooks:203:9)
    at invokeCallback (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:231:16)
    at done (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:220:7)
    at IncomingMessage.onEnd (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:280:7)
    at IncomingMessage.emit (node:events:513:28)
    at endReadableNT (node:internal/streams/readable:1359:12)

[shelomito12]

I had to send a watch request using this script https://github.com/byeokim/gmailpush/issues/6#issuecomment-660764392 and I'm now able to receive some output when receiving an email which is nice, but noticed this Unexpected end of JSON input error. Maybe since I added this https://github.com/byeokim/gmailpush/issues/11#issuecomment-751632889 extra code ??:

$ node index.js
Server listening on port 3002...
request {
  message: {
    data: 'eyJlbWFpbEFkZHJlc3MiOiJoYXR6bGFjaGEuaG9tZS5jbGVhbmluZ0BnbWFpbC5jb20iLCJoaXN0b3J5SWQiOjExNjYzfQ==',
    messageId: '6353849924728870',
    message_id: '6353849924728870',
    publishTime: '2022-11-22T20:12:17.257Z',
    publish_time: '2022-11-22T20:12:17.257Z'
  },
  subscription: 'projects/{hidden}/subscriptions/GmailAPIPush-sub'
}
email test123@gmail.com

SyntaxError: Unexpected end of JSON input
    at JSON.parse (<anonymous>)
    at Gmailpush._initialize (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:94:38)
    at async Gmailpush.getMessagesWithoutAttachment (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:477:27)
    at async Gmailpush.getMessages (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:547:22)

[shelomito12]

@byeokim @aivantg I'm stuck at this point. Can you guys shed some light? - Thanks

[shelomito12]

I had to delete the gmailpush_history.json file and then I was able to get the email message but at the end I see another error: TypeError [ERR_INVALID_ARG_TYPE]: The "cb" argument must be of type function. Received undefined

$ node index.js
Server listening on port 3002...
request {
  message: {
    data: 'eyJlbWFpbEFkZHJlc3MiOiJoYXR6bGFjaGEuaG9tZS5jbGVhbmluZ0BnbWFpbC5jb20iLCJoaXN0b3J5SWQiOjEyNDg3fQ==',
    messageId: '6355147275576378',
    message_id: '6355147275576378',
    publishTime: '2022-11-23T01:42:04.991Z',
    publish_time: '2022-11-23T01:42:04.991Z'
  },
  subscription: 'projects/{hidden}/subscriptions/GmailAPIPush-sub'
}
email test@gmail.com
messages [
  {
    id: '184a2263122fc7be',
    threadId: '1849ff8ac229a344',
    labelIds: [
      'UNREAD',
      'IMPORTANT',
      'CATEGORY_PERSONAL',
      'Label_2437551251247852083',
      'INBOX'
    ],
    snippet: 'Google Voice test YOUR ACCOUNT HELP CENTER HELP FORUM This email was sent to you because you indicated that you'd like to receive email notifications for text messages. If you don't want to',
    payload: {
      partId: '',
      mimeType: 'multipart/alternative',
      filename: '',
      headers: [Array],
      body: [Object],
      parts: [Array]
    },
    sizeEstimate: 9902,
    historyId: '12446',
    internalDate: '1669167722000',
    historyType: 'messageAdded',
    from: {
      name: '"(917) 367-0969"',
      address: 'xxxxxx.xxxxxxx.SH35ZRBW2r@txt.voice.google.com'
    },
    to: [ [Object] ],
    subject: 'New text message from (917) 367-0969',
    date: 'Wed, 23 Nov 2022 01:42:02 +0000',
    attachments: [],
    bodyText: ...
    bodyHtml: '<html>...</html>`
  }
]
TypeError [ERR_INVALID_ARG_TYPE]: The "cb" argument must be of type function. Received undefined
    at maybeCallback (node:fs:186:3)
    at Object.readFile (node:fs:389:14)
    at /home/ubuntu/gmailpush/index.js:53:13
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
  code: 'ERR_INVALID_ARG_TYPE'
}

[byeokim]

Per _TypeError [ERR_INVALID_ARGTYPE]: The "cb" argument must be of type function. Received undefined, could you show me some code snippet near line 53 in index.js you are currently running? If index.js still has below you can remove it:

fs.readFile('./gmailpush_history.json')
  .then((result) => {
    const prevHistories = JSON.parse(result);
    const prevHistory = prevHistories.find((prevHistory) => prevHistory.emailAddress === email);
    schedule.scheduleJob(new Date(prevHistory.watchExpiration - 1000 * 60 * 30), async () => {
      prevHistory.watchExpiration = await gmailpush._refreshWatch();
      fs.writeFile('./gmailpush_history.json', JSON.stringify(prevHistories));
    });
  });

[shelomito12]

Thanks @byeokim! I removed it and everything looks good now... Just a couple of things:

1. By removing the readFile/schedule.scheduleJob code snippet, does it mean that node-schedule was already implemented into Gmailpush to call watch() before watchExpiration?
2. I just need to extract subject within messages from the response. Can you please provide a simple code sample? =)

Thanks

[byeokim]

1. Schedulers like node-schedule are not implemented into Gmailpush. However Gmailpush calls watch() whenever gmailpush.getMessages() is invoked. It means that watchExpiration is extended by seven days from the time when gmailpush.getMessages() is called. If you expect your Gmail inbox to receive no emails for seven days, you might have to re-implement the code you have removed from index.js.

2. Here's sample code:

require('dotenv').config();
const jsonToken = require('./token.json');
const Gmailpush = require('gmailpush');
const express = require('express');
const app = express();

const gmailpush = new Gmailpush({
  clientId: process.env.CLIENT_ID,
  clientSecret: process.env.CLIENT_SECRET,
  pubsubTopic: process.env.TOPIC_URL
});

app.post(
  '/pubsub',
  express.json(),
  async (req, res) => {
    res.sendStatus(200);
    const {subject} = await gmailpush.getNewMessage({
      notification: req.body,
      token: jsonToken,
    });
    console.log(subject);
  }
);

app.listen(3000);

[shelomito12]

Ah destructuring! 🙂

but sadly getting errors. Are you sure it's not the gmailpush.getMessages() function?

$ node index.js 
Server listening on port 3002...
undefined:1

SyntaxError: Unexpected end of JSON input
    at JSON.parse (<anonymous>)
    at Gmailpush._initialize (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:94:38)
    at async Gmailpush.getMessagesWithoutAttachment (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:477:27)
    at async Gmailpush.getNewMessage (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:581:22)
    at async /home/ubuntu/gmailpush/index.js:44:23

Node.js v18.12.1

...
async (req, res) => { 
...
 const {subject} = await gmailpush.getNewMessage({
        notification: req.body,
        token: jsonToken,
      });
      console.log(subject);
}
...

Note: I've also tried token,, (with & without comma) token: jsonToken.access_token, (with & without comma) but same errors

[byeokim]

getNewMessage() fetches only a new inbox message while getMessages() gets an array of messages related to a Gmail event, e.g. messages being added/deleted/moved, etc.

The error doesn't have to do with token. Open gmailpush_history.json and if it doesn't look like JSON like below, please remove the file and run node index.js again:

[
  {
    "emailAddress": "user1@gmail.com",
    "prevHistoryId": 9876543210,
    "watchExpiration": 1576543210
  }
]

[shelomito12]

Thanks for the clarification. The gmailpush_history.json file was already created but was empty. Deleted it and then ran node index.js again:

$ node index.js 
Server listening on port 3002...
/home/ubuntu/gmailpush/index.js:44
    const {subject} = await gmailpush.getNewMessage({
           ^

TypeError: Cannot destructure property 'subject' of '(intermediate value)' as it is null.
    at /home/ubuntu/gmailpush/index.js:44:12
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5)

[shelomito12]

Fixed it by removing the gmailpush_history.json file and then added || {}:

    const {subject} = await gmailpush.getNewMessage({
        notification: req.body,
        token
      }) || {};

finally ran node index.js again, and the email subject was there! - Thanks!

[shelomito12]

I'd like to filter only by User Label:

    const {subject} = await gmailpush.getNewMessage({
        notification: req.body,
        token,
        withLabelIds: ['Label_2437551251247852083']
      }) || {};

The getNewMessage() options lists withLabelIds but when I add it I get this error:

$ node index.js 
Server listening on port 3002...
/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:620
      throw new Error(
            ^

Error: Options may only contain the following: notification, token
    at Gmailpush._getPropsFromOptions (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:620:13)
    at Gmailpush.getNewMessage (/home/ubuntu/gmailpush/node_modules/gmailpush/lib/gmailpush.js:571:24)
    at /home/ubuntu/gmailpush/index.js:44:39
    at Layer.handle [as handle_request] (/home/ubuntu/gmailpush/node_modules/express/lib/router/layer.js:95:5)
    at next (/home/ubuntu/gmailpush/node_modules/express/lib/router/route.js:144:13)
    at /home/ubuntu/gmailpush/node_modules/body-parser/lib/read.js:137:5
    at AsyncResource.runInAsyncScope (node:async_hooks:203:9)
    at invokeCallback (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:231:16)
    at done (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:220:7)
    at IncomingMessage.onEnd (/home/ubuntu/gmailpush/node_modules/raw-body/index.js:280:7)

[byeokim]

For now you might need to add a label filter manually because getNewMessage() accepts only notification and token as the options object 🙃

const {subject} = await gmailpush
  .getNewMessage({
    notification: req.body,
    token
  })
  .then((message) => {
    if (message === null) {
      return {};
    }

    if (!message.labelIds.includes('Label_2437551251247852083')) {
      return {};
    }

    return message;
  });

[shelomito12]

Thank you very much for the help @byeokim! Hopefully this great API can be here for a long time! 👍