request-promise doesn't run the same way in konnector

[nicolaspernoud]

If I run the following code with node ., it works flawlessly

const rp = require('request-promise');
const qs = require('querystring');
const moment = require('moment');
const cookiejar = rp.jar();

//require('request-debug')(rp);

const username = '*******';
const password = '*******';
const startDate = moment().subtract(1, 'day').format('DD/MM/YYYY');
const endDate = moment().format('DD/MM/YYYY');

main();

async function main() {
    try {
        await authenticate(username, password);
        await getData();
    } catch (error) {
        console.error(error);
    }
}

function authenticate(username, password) {

    const authRequest = {
        method: 'POST',
        uri: 'https://espace-client-connexion.enedis.fr/auth/UI/Login',
        jar: cookiejar,
        headers: {
            'Host': 'espace-client-connexion.enedis.fr',
            'Content-Type': 'application/x-www-form-urlencoded',
        },
        form: {
            IDToken1: username,
            IDToken2: password,
            goto: 'aHR0cHM6Ly9lc3BhY2UtY2xpZW50LXBhcnRpY3VsaWVycy5lbmVkaXMuZnIv',
            SunQueryParamsString: 'cmVhbG09cGFydGljdWxpZXJz',
            encoded: true,
        },
        followAllRedirects: true
    };

    // Reset Content-Length header since Enedis auth wants Title-Cased Headers
    const authRequestLength = Buffer.byteLength(qs.stringify(authRequest.form));
    authRequest.headers['Content-Length'] = authRequestLength;
    return rp(authRequest);
}

async function getData() {
    const dataRequest = {
        method: 'POST',
        uri: 'https://espace-client-particuliers.enedis.fr/group/espace-particuliers/suivi-de-consommation?p_p_id=lincspartdisplaycdc_WAR_lincspartcdcportlet&p_p_lifecycle=2&p_p_resource_id=urlCdcHeure',
        jar: cookiejar,
        headers: {
            'Referer': 'https://espace-client-particuliers.enedis.fr/group/espace-particuliers/suivi-de-consommation',
        },
        form: {
            _lincspartdisplaycdc_WAR_lincspartcdcportlet_dateDebut: startDate,
            _lincspartdisplaycdc_WAR_lincspartcdcportlet_dateFin: endDate,
        },
        json: true
    }
    try {
        const response = await rp(dataRequest);
        const start = response.graphe.periode.dateDebut;
        const loadProfile = response.graphe.data.map(value => { return { load: value.valeur, time: moment(start, 'DD/MM/YYYY').add((value.ordre - 1) * 0.5, 'hour').format() }; });
        console.dir(loadProfile, { depth: null });
        return loadProfile;
    } catch (error) {
        console.error(error);
    }
}

response is

[ { load: -2, time: '2018-11-18T00:00:00+01:00' },
  ...,
  { load: 0.656, time: '2018-11-18T13:00:00+01:00' },
  { load: 0.946, time: '2018-11-18T13:30:00+01:00' },
  { load: 0.436, time: '2018-11-18T14:00:00+01:00' },
  { load: 0.55, time: '2018-11-18T14:30:00+01:00' },
  { load: 0.322, time: '2018-11-18T15:00:00+01:00' },
  { load: 0.734, time: '2018-11-18T15:30:00+01:00' },
  { load: 0.404, time: '2018-11-18T16:00:00+01:00' },
  { load: 0.226, time: '2018-11-18T16:30:00+01:00' },
  { load: 0.156, time: '2018-11-18T17:00:00+01:00' },
 ...,
  { load: 0.122, time: '2018-11-18T23:30:00+01:00' } ]

But if I try to put it in a konnector start function and launch it with yarn standalone, the auth fails (redirection loop), event if the logged request is exactly the same : Code :

const { BaseKonnector, log } = require('cozy-konnector-libs')

const rp = require('request-promise')
const qs = require('querystring')
const moment = require('moment')
const cookiejar = rp.jar()

require('request-debug')(rp)

module.exports = new BaseKonnector(start)

async function start(fields) {
  log('info', 'Authenticating ...')
  try {
    await authenticate(fields.username, fields.password)
    log('info', 'Successfully logged in')
    log('info', 'Getting data')
    await getData()
  } catch (error) {
    console.error(error)
  }
  log('info', 'Saving data to Cozy')
}

const startDate = moment()
  .subtract(1, 'day')
  .format('DD/MM/YYYY')
const endDate = moment().format('DD/MM/YYYY')

function authenticate(username, password) {
  const authRequest = {
    method: 'POST',
    uri: 'https://espace-client-connexion.enedis.fr/auth/UI/Login',
    jar: cookiejar,
    headers: {
      Host: 'espace-client-connexion.enedis.fr',
      'Content-Type': 'application/x-www-form-urlencoded'
    },
    form: {
      IDToken1: username,
      IDToken2: password,
      goto: 'aHR0cHM6Ly9lc3BhY2UtY2xpZW50LXBhcnRpY3VsaWVycy5lbmVkaXMuZnIv',
      SunQueryParamsString: 'cmVhbG09cGFydGljdWxpZXJz',
      encoded: true
    },
    followAllRedirects: true
  }

  // Reset Content-Length header since Enedis auth wants Title-Cased Headers
  const authRequestLength = Buffer.byteLength(qs.stringify(authRequest.form))
  authRequest.headers['Content-Length'] = authRequestLength
  return rp(authRequest)
}

async function getData() {
  const dataRequest = {
    method: 'POST',
    uri:
      'https://espace-client-particuliers.enedis.fr/group/espace-particuliers/suivi-de-consommation?p_p_id=lincspartdisplaycdc_WAR_lincspartcdcportlet&p_p_lifecycle=2&p_p_resource_id=urlCdcHeure',
    jar: cookiejar,
    headers: {
      Referer:
        'https://espace-client-particuliers.enedis.fr/group/espace-particuliers/suivi-de-consommation'
    },
    form: {
      _lincspartdisplaycdc_WAR_lincspartcdcportlet_dateDebut: startDate,
      _lincspartdisplaycdc_WAR_lincspartcdcportlet_dateFin: endDate
    },
    json: true
  }
  try {
    const response = await rp(dataRequest)
    const start = response.graphe.periode.dateDebut
    const loadProfile = response.graphe.data.map(value => {
      return {
        load: value.valeur,
        time: moment(start, 'DD/MM/YYYY')
          .add((value.ordre - 1) * 0.5, 'hour')
          .format()
      }
    })
    console.dir(loadProfile, { depth: null })
    return loadProfile
  } catch (error) {
    console.error(error)
  }
}

Logged request :

 request:
   { debugId: 1,
     uri: 'https://espace-client-connexion.enedis.fr/auth/UI/Login',
     method: 'POST',
     headers:
      { Host: 'espace-client-connexion.enedis.fr',
        'Content-Type': 'application/x-www-form-urlencoded',
        'Content-Length': 181 },
     body: 'IDToken1=***username***&IDToken2=***password***&goto=aHR0cHM6Ly9lc3BhY2UtY2xpZW50LXBhcnRpY3VsaWVycy5lbmVkaXMuZnIv&SunQueryParamsString=cmVhbG09cGFydGljdWxpZXJz&encoded=true' } }

Does konnector alter the working of request-promise in any way ?

PS : The SunQueryParamsString: 'cmVhbG09cGFydGljdWxpZXJz' bit reference a hidden field <input type="hidden" name="SunQueryParamsString" value="cmVhbG09cGFydGljdWxpZXJz" /> in the authentication page. It would be better to fetch the value of this field with cheerio in case it changes. I'm writing this just in case someone would be interested in writting a connector based on my code...

[nicolaspernoud]

Just in case someone would want to have a look, I could provide him/her with some credentials to test the access.

[nicolaspernoud]

Found it : it seems that the replay library strip the Content-Type and Content-Length header that I need. It works better with the altered initReplay function in standalone.js altered as to not load replay at all when replay option is not set :

function initReplay() {
  const replayOption = ['record', 'replay'].find(opt => program[opt] === true)
  if (replayOption) process.env.REPLAY = replayOption

  if (process.env.REPLAY) require('replay')

}

[doubleface]

@nicolaspernoud It is weird because the replay module is required only if --replay or --record options are given to the yarn standalone command. Did you use these options ?

[nicolaspernoud]

No, I didn't. But the replay module is required anyway (albeit turned into "bloody" mode which should not do anything) : here the original code which require replay (https://github.com/konnectors/libs/blob/master/packages/cozy-jobs-cli/src/standalone.js) :

function initReplay() {
  const replayOption = ['record', 'replay'].find(opt => program[opt] === true)
  if (replayOption) process.env.REPLAY = replayOption

  process.env.REPLAY =
    replayOption || (process.env.REPLAY ? process.env.REPLAY : 'bloody')

  require('replay')
}

My alteration disable requiring replay instead of turning bloody mode. I didn't do a pull request because I am not sure of possible side effects, but if you want, I can do it... Or you could alter the code directly.

[doubleface]

I do not see any side effect. Here is the PR

[doubleface]

Thanks @nicolaspernoud :+1:

[nicolaspernoud]

It would seem that the require('replay') line is still called anyway in your code. Line 69 should be removed !?.

[doubleface]

Sorry. Now this should be ok

[nicolaspernoud]

Thanks !