Found CONNX: MQCC = MQCC_FAILED [2] MQRC = MQRC_HOST_NOT_AVAILABLE [2538] ERROR in our Production only.

[JonnyyJ]

Please include the following information in your ticket.

• mq-mqi-nodejs version(s) that are affected by this issue. 0.9.23

  Desc: We found this issue after July 3rd in our production environment, but the latest deployment in a production environment is in May, from May to July 3rd, we don't see any errors.

  We restart the service, then the connection back to work, but after about 2 hours, we start seeing the error again. Also, there are NO issues in all our lower environments (QA/STG). We sent a support ticket to IBMMQ support, and they mentioned that it is related to the network, and then we reached out to our network team and found some RST from the MQ response, other than this, we are not seeing firewall issues or other blockers.

  Config We are connecting to the MQ server through TCP. We have two queues set up in the same QM, one is for listening to the messages, and the other is used for publishing messages. The error happens all the time while we are listening to the listener queue, and happens most of the time(few events processed successfully) while we publish the message to the publish queue. The traffic is about 70 messages per hour for each lisenerQueue and publish queue.

  Here's our API looks like: Listener:

  
  const mq = require('ibmmq');
  const { Subject } = require('rxjs');
  const { getProperty } = require('@xxx/utils/properties');
  const { getConsoleLogger } = require('@xxx/utils/logger');

// const { mqGet } = require('./utils/promise-wrappers');

// eslint-disable-next-line prefer-destructuring const MQC = mq.MQC; const logger = getConsoleLogger(); /*

• listener execution
• @param {MQQueueManager} queueManager
• @param {string} queueName
• @param {EventEmitter} emitter */ // flag for keeping process alive var ok = true var connectionHandle var queueHandle var msgId = null; var waitInterval = 300 // max seconds to wait for a new message

const listenerSubject = new Subject();

// assign param for listener const mqc = mq.MQC; const connectionOptions = new mq.MQCNO(); const connectionDetails = new mq.MQCD(); const connectionSecurity = new mq.MQCSP();

connectionDetails.ConnectionName = getProperty('IBMMQ.connectionName'); connectionDetails.ChannelName = getProperty('IBMMQ.channelName');

if (getProperty('IBMMQ.password')) { connectionSecurity.UserId = getProperty('IBMMQ.userId'); connectionSecurity.Password = null || getProperty('IBMMQ.password'); }

connectionOptions.SecurityParms = connectionSecurity; connectionOptions.Options |= mqc.MQCNO_CLIENT_BINDING; connectionOptions.ClientConn = connectionDetails;

/*

• This function is the async callback. Parameters
• include the message descriptor and the buffer containing
• the message data. */

function getCB(err, hObj, gmo, messageDetails, buf, hConn) { // If there is an error, prepare to exit by setting the ok flag to false. if (err) { if (err.mqrc == MQC.MQRC_NO_MSG_AVAILABLE) { logger.info("No more messages available."); } else { logger.info(err); } ok = false; // We don't need any more messages delivered, so cause the // callback to be deleted after this one has completed. mq.GetDone(hObj); } else { if (messageDetails.Format == "MQSTR") { logger.info(Received message: ${buf}); listenerSubject.next(buf.toString()); } else { logger.info("binary message: " + buf); } } } function hexToBytes(hex) { for (var bytes = [], c = 0; c < hex.length; c += 2) bytes.push(parseInt(hex.substr(c, 2), 16)); return bytes; } /**

• Define which messages we want to get and how. / function getMessages() { const messageDetails = new mq.MQMD(); const getMessageOptions = new mq.MQGMO(); messageDetails.Persistence = MQC.MQPER_PERSISTENT // Enable message persistence getMessageOptions.WaitInterval = waitInterval 1000; getMessageOptions.MatchOptions = MQC.MQMO_NONE; getMessageOptions.Options = MQC.MQGMO_NO_SYNCPOINT | MQC.MQGMO_WAIT | MQC.MQGMO_CONVERT | MQC.MQGMO_FAIL_IF_QUIESCING; if (msgId != null) { logger.info("Setting Match Option for MsgId"); getMessageOptions.MatchOptions = MQC.MQMO_MATCH_MSG_ID; md.MsgId = hexToBytes(msgId); } mq.Get(queueHandle, messageDetails, getMessageOptions, getCB) } /*

• When we're done, close any queues and connections. */ function cleanup(hConn, hObj, queueName) { mq.Close(hObj, 0, function (err) { if (err) { logger.error(err); } else { logger.info("listener: MQ Close successful"); } mq.Disc(hConn, function (err) { if (err) { logger.error(err); } else { logger.info("listener: MQ Disconnected successful"); } }); setTimeout(()=>{ logger.info("Reconnecting..."); initConnection(queueName) }, 5000) }); } async function initConnection(queueName) { ok = true // Get command line parameters var myArgs = process.argv.slice(2); // Remove redundant parms if (myArgs[2]) { msgId = myArgs[2]; }

  mq.setTuningParameters({ syncMQICompat: true }); mq.Connx(getProperty('IBMMQ.queueManager'), connectionOptions, function (err, hConn) { if (err) { logger.error("MQ call failed in " + err.message) ok = false } else { logger.info(connection status: ${JSON.stringify(hConn)}) connectionHandle = hConn // Define how to open the queue here const mqObjectDefinition = new mq.MQOD(); mqObjectDefinition.ObjectName = queueName; mqObjectDefinition.ObjectType = MQC.MQOT_Q; const listenerOptions = getProperty('IBMMQ.listenerOpenOptions') || MQC.MQOO_INPUT_SHARED; mq.Open(hConn, mqObjectDefinition, listenerOptions, (err, hObj) => { queueHandle = hObj if (err) { logger.error("Error occurs while trying to open the queue", err) } else { logger.info("MQ openned succesfully") // Start asking messages to be delivered getMessages() } }); } }) } async function executeListener(queueName) { initConnection(queueName) // We need to keep the program active so that the callbacks from the // message handler are processed. This is one way to do it. // If not OK, then close everything down cleanly. setInterval(function () { if (!ok) { logger.info("Exiting ..."); cleanup(connectionHandle, queueHandle, queueName); } }, 10000); }

/*

• @returns {function} listener function */ function listenerClosure() { /*
  • @param {string} queueName name of queue to listen to
  • @return {EventEmitter} data emitter for queue */ return function listener(queueName) { executeListener(queueName); return listenerSubject.asObservable(); }; }

module.exports = listenerClosure;

**Publisher:**

const { getConsoleLogger } = require('@xxx/utils/logger'); const mq = require('ibmmq'); const { getProperty } = require('@xxx/utils/properties'); const { mqPut } = require('./utils/promise-wrappers'); const { cleanUp } = require('./utils/clean-up')

const logger = getConsoleLogger(); // eslint-disable-next-line prefer-destructuring const MQC = mq.MQC; /*

• @returns {function} publisher function */ function publisherClosure() { /*

  • @param {string} queueName queue manager object
  • @param {string|object} data data to publish

  • @returns {null} no return value */ return async function publisher(queueName, data) { try { const mqObjectDefinition = new mq.MQOD(); mqObjectDefinition.ObjectName = queueName; mqObjectDefinition.ObjectType = MQC.MQOT_Q; const putOptions = getProperty('IBMMQ.putOpenOptions') || MQC.MQOO_OUTPUT;

    const messageDetails = new mq.MQMD(); const putMessageOptions = new mq.MQPMO();

    messageDetails.Format = MQC.MQFMT_STRING; messageDetails.Persistence = MQC.MQPER_PERSISTENT // Enable message persistence putMessageOptions.Options = MQC.MQPMO_NO_SYNCPOINT | MQC.MQPMO_NEW_MSG_ID | MQC.MQPMO_NEW_CORREL_ID;

    // Establish connection here const mqc = mq.MQC; const connectionOptions = new mq.MQCNO(); const connectionDetails = new mq.MQCD(); const connectionSecurity = new mq.MQCSP();

    connectionDetails.ConnectionName = getProperty('IBMMQ.connectionName'); connectionDetails.ChannelName = getProperty('IBMMQ.channelName');

    if (getProperty('IBMMQ.password')) { connectionSecurity.UserId = getProperty('IBMMQ.userId'); connectionSecurity.Password = null || getProperty('IBMMQ.password'); }

    connectionOptions.SecurityParms = connectionSecurity; connectionOptions.Options |= mqc.MQCNO_CLIENT_BINDING; connectionOptions.ClientConn = connectionDetails; mq.Connx(getProperty('IBMMQ.queueManager'), connectionOptions, function(err, hconn){ if(err) { logger.error("MQ call failed while establishing the connection" + err) } else{ mq.Open(hconn, mqObjectDefinition, putOptions, function(err, hObj){ if (err) { logger.error("MQ call failed while opening the connection " + err); } else { let dataBuffer; if (typeof data === 'object') dataBuffer = Buffer.from(JSON.stringify(data)); else dataBuffer = Buffer.from(data); mqPut(hObj, messageDetails, putMessageOptions, dataBuffer); logger.info("MQOPEN successful"); } cleanUp(hconn, hObj); }); logger.info(connection status: ${JSON.stringify(hconn)}) } }) return "Successfully published message"; } catch (err) { logger.error(Error publishing message to Queue ${queueName}, err); throw err; } }; }

module.exports = publisherClosure;

We want to locate whether this is an environment issue or a code issue, any thoughts?

[ibmmqmet]

Can't see anything obviously wrong. There might be additional diagnostics in the client and qmgr error logs (and perhaps even FDC files). There are many reasons for a 2538, mostly to do with network failures or timeouts in waiting for responses. Though I worry when I see a comment like "recursively calls itself" as it can imply all kinds of resources being exhausted if they are not being cleaned up. for example, reaching max number of sockets in a process.

[JonnyyJ]

Can't see anything obviously wrong. There might be additional diagnostics in the client and qmgr error logs (and perhaps even FDC files). There are many reasons for a 2538, mostly to do with network failures or timeouts in waiting for responses. Though I worry when I see a comment like "recursively calls itself" as it can imply all kinds of resources being exhausted if they are not being cleaned up. for example, reaching max number of sockets in a process.

Thanks for reaching out, the comment of "recursively calls" is a miss type, I just deleted that, sorry for the confusion. Also 2538 is the only error we are seeing, do you know where can we check the FDC files?

[ibmmqmet]

Error logs and FDCs can usually be found under /var/mqm/errors or $HOME/IBM/MQ/data/errors

[JonnyyJ]

Hi @ibmmqmet We cannot find any errors folder that you mentioned on our client side and we are using Linux. The service is deployed in a container, should we enable a sort of debug mode to generate the error logs? Also, our network guy is telling us this is not a network issue itself, since we didn't find any connection issue besides the RST packet, and they are saying it could be more like a session timeout or reaching to max session issue because each time we restart the service, the connection will come back about 1 hour. Is it possible that upgrading to the latest version could solve this issue? BTW we are still using Node 14 which is EOL and we are planning to upgrade to 18 in our next release.

Also, we are using the same channel for our Prod and non-prod QM, I see the docs mentioned that the default available sessions are 100, is it possible we already reached the max sessions and refuse to do the incoming request?

[ibmmqmet]

The MQ client error log directories ought to be SOMEWHERE in your environment. Perhaps under the root directory / in some container setups. There's no debug option - error logs are always generated by the client when needed.

And you also need to check the queue manager error logs to see if anything is mentioned there. Monitoring the queue manager channel status would also show if you are building up connections.

[JonnyyJ]

Thanks for the suggestion, we also did an update for the ibmmq sdk to the latest one and also update the node version from 14 to 18, after that the network error is magically gone, I will continue monitoring for few days, for now I will close this ticket.