GwilymNewton / node-red-contrib-discovery-insert

A node for inserting documents into the watson discovery service.
Other
0 stars 1 forks source link

Handle binary files #1

Closed sauraba closed 6 years ago

sauraba commented 6 years ago

There is a flow that reads pdf attachment from email adapter and updates WDS. the current WDS insert/update node has to be enhanced to handle the binary pdf file recvd from the email adapter.

[{"id":"a96b1a38.a60638","type":"debug","z":"6daff5e1.878cf4","name":"","active":true,"console":"false","complete":"payload","x":757.100118637085,"y":106.20001983642578,"wires":[]},{"id":"34132941.86f286","type":"e-mail in","z":"6daff5e1.878cf4","name":"","protocol":"IMAP","server":"imap.gmail.com","useSSL":true,"port":"993","box":"INBOX","disposition":"Read","repeat":"60","x":55,"y":38.400001525878906,"wires":[["f54702a1.cc60c","a462720c.f4fdc"]]},{"id":"f54702a1.cc60c","type":"debug","z":"6daff5e1.878cf4","name":"email","active":true,"console":"false","complete":"true","x":246.10000610351562,"y":101.60000991821289,"wires":[]},{"id":"945d3dd8.aea1f","type":"http request","z":"6daff5e1.878cf4","name":"update document","method":"PUT","ret":"txt","url":"","tls":"","x":591.0000686645508,"y":73.40002059936523,"wires":[["a96b1a38.a60638"]]},{"id":"360be0ab.8a5c8","type":"inject","z":"6daff5e1.878cf4","name":"","topic":"","payload":"","payloadType":"date","repeat":"","crontab":"","once":false,"x":112.10000610351562,"y":251.40001106262207,"wires":[["b0455a5a.8ce838"]]},{"id":"a462720c.f4fdc","type":"function","z":"6daff5e1.878cf4","name":"payload","func":"var attachment = msg.attachments;\n\nfor (i=0;i<attachment.length;i++)\n{\n if (attachment[i].contentType==\"application/pdf\")\n var doc = attachment[i].content;\n}\ndelete msg.header;\ndelete msg.attachments;\nmsg.attachment=doc;\nvar subject = msg.topic\n//var subject=\"[ Ticket: 4386 ] Feedback required_2fe8f923-d77b-47d5-827d-18b94b9dde8e\";\nmsg.payload=subject.substring(subject.lastIndexOf(\"_\")+1);\nreturn msg;","outputs":1,"noerr":0,"x":238.00000381469727,"y":153.40000438690186,"wires":[["9e56292b.c96518"]]},{"id":"9e56292b.c96518","type":"cloudant in","z":"6daff5e1.878cf4","name":"lookup","cloudant":"","database":"answerunits","service":"Cloudant NoSQL DB-WEA","search":"_id_","design":"","index":"","x":498.8999671936035,"y":165.79999351501465,"wires":[["88195ae.6ebeba8"]]},{"id":"88195ae.6ebeba8","type":"function","z":"6daff5e1.878cf4","name":"update doc","func":"msg.headers = {'content-type':'application/pdf'};\nmsg.query=msg.cloudant._rev;\nmsg.url=\"https://92dbebf7-c4c4-4ac5-92d3-ac0080aa8abb-bluemix.cloudant.com/answerunits/\"+msg.cloudant._id+\"/\"+msg.cloudant.title+\".pdf?rev=\"+msg.cloudant._rev;\nmsg.payload=msg.attachment;\nreturn msg;","outputs":1,"noerr":0,"x":649.0000228881836,"y":151.40002918243408,"wires":[["945d3dd8.aea1f"]]},{"id":"b0455a5a.8ce838","type":"function","z":"6daff5e1.878cf4","name":"","func":"msg={\"topic\":\"[ Ticket: 4345 ] Feedback required_2fe8f923-d77b-47d5-827d-18b94b9dde8e\",\"date\":\"2017-09-19T15:01:31.000Z\",\"header\":{\"delivered-to\":\"cea.watson@gmail.com\",\"received\":[\"by 10.237.36.179 with SMTP id t48csp4570572qtc; Tue, 19 Sep 2017 08:01:42 -0700 (PDT)\",\"from mxout29.expurgate.net (mxout29.expurgate.net. [91.198.224.29]) by mx.google.com with ESMTPS id z129si1449487wmc.29.2017.09.19.08.01.41 for <cea.watson@gmail.com> (version=TLS1_2 cipher=AES128-SHA bits=128/128); Tue, 19 Sep 2017 08:01:41 -0700 (PDT)\",\"from [127.0.0.1] (helo=localhost) by relay.expurgate.net with smtp (Exim 4.80.1) (envelope-from <dsn@my333936.mail.crm.ondemand.com>) id 1duK1s-0001L2-P6 for cea.watson@gmail.com; Tue, 19 Sep 2017 17:01:40 +0200\",\"from [155.56.208.68] (helo=smtp.sapbydesign.com) by relay.expurgate.net with esmtps (TLS1.2:DHE_RSA_AES_256_CBC_SHA256:256) (Exim 4.80.1) (envelope-from <dsn@my333936.mail.crm.ondemand.com>) id 1duK1s-0004qj-1j for cea.watson@gmail.com; Tue, 19 Sep 2017 17:01:40 +0200\",\"from [10.110.65.204] (port=26789 helo=my333936.mail.crm.ondemand.com) by smtp.sapbydesign.com with esmtps (TLSv1.2:AES128-GCM-SHA256:128) (Exim 4.82_1-5b7a7c0-XX) (envelope-from <dsn@my333936.mail.crm.ondemand.com>) id 1duK1m-0004NS-1W for cea.watson@gmail.com; Tue, 19 Sep 2017 15:01:37 +0000\"],\"x-google-smtp-source\":\"AOwi7QDCmobCF/KRJGcInavO51fAI0XdRPLagESLwO6lpV+M4YKCO5PDj90uilR2b5XXLW4Rd+m6\",\"x-received\":\"by 10.223.170.137 with SMTP id h9mr1715187wrc.91.1505833301724; Tue, 19 Sep 2017 08:01:41 -0700 (PDT)\",\"arc-seal\":\"i=1; a=rsa-sha256; t=1505833301; cv=none; d=google.com; s=arc-20160816; b=Kw5UtmPJ0GStU+tRZRVsQckOclEpIhzK6A38XNcnIacceDNKasSURlYfFF5atB0JWd zcJ3Ikn+dSDVTKthyJ/ui8cXTCQr7QFsKTQ8OJ6pKVHhEC1zJgsGfkrQ18xMqG/66IDY 0hX+GZwDyKpHIDXwmwGopLK6eXMxX8VZAD82tA71eSgPKduJKUrdfgusYwzzSEypkAJM g0N/vihxYrRIiV0kmCyGFa87O99NeXwYopzNM7l4KCfnl5aal1zaadyIwzNlWB3iUzTO GlBHi6+bsl9rxrzuakABtmoniwrNywBVIe44cH1fptwd2oIftb+Mteibs/4Ex5sTjzL0 rUlw==\",\"arc-message-signature\":\"i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=mime-version:message-id:reply-to:to:subject:from:date :arc-authentication-results; bh=+5CVFYhcNXsc014ZxiCZLh2LiwSNelHpPNi2y3J9fTg=; b=bDEoM7KL1S7qPDXq1vJHKVGlaIRhmygqaSRgBuvlFazYTLa7DRdG+IDcy+DNs6wbfL MDBuNI0my0YIKUZ8JQGZt3Gx9Sprmvfr58L8i5l9p2eDKCswMMzKOocya9p40RnAOAF4 harmhFYlXlqfO5WPBPeIWrqGO2huOLk894RjyytlcnKdIJsK1m2kF1fkt6/TQSRCpVN3 qCPcozmjMyxlMnUJdwd9bFoGdPL98A0lUzm25KqS90VHA5++e2n+kaHb8oeSyLFXcXR9 M4912Ox1AsWyd9kAVkWrMTrS2uPRu32N6Yaij4q86CUw7x4VNVQLSkkBzVSyv/iyp6t1 w5wg==\",\"arc-authentication-results\":\"i=1; mx.google.com; spf=neutral (google.com: 91.198.224.29 is neither permitted nor denied by best guess record for domain of dsn@my333936.mail.crm.ondemand.com) smtp.mailfrom=dsn@my333936.mail.crm.ondemand.com\",\"return-path\":\"<dsn@my333936.mail.crm.ondemand.com>\",\"received-spf\":\"neutral (google.com: 91.198.224.29 is neither permitted nor denied by best guess record for domain of dsn@my333936.mail.crm.ondemand.com) client-ip=91.198.224.29;\",\"authentication-results\":\"mx.google.com; spf=neutral (google.com: 91.198.224.29 is neither permitted nor denied by best guess record for domain of dsn@my333936.mail.crm.ondemand.com) smtp.mailfrom=dsn@my333936.mail.crm.ondemand.com\",\"date\":\"Tue, 19 Sep 2017 15:01:31 +0000 (UTC)\",\"from\":\"<donotreply@my333936.mail.crm.ondemand.com>\",\"subject\":\"[ Ticket: 4345 ] Feedback required_2fe8f923-d77b-47d5-827d-18b94b9dde8e\",\"to\":\"<cea.watson@gmail.com>\",\"reply-to\":\"\\\"DoNotReply@my333936.mail.crm.ondemand.com\\\" <donotreply@my333936.mail.crm.ondemand.com>\",\"message-id\":\"<00163E317F841EE7A7A96D69A4686384E16SERVICE_ONDEMAND00163E317F841EE7A7A96D69A4628384@my333936.mail.crm.ondemand.com>\",\"mime-version\":\"1.0\",\"x-mailer\":\"SAPBCS804\",\"content-type\":\"multipart/signed; protocol=\\\"application/pkcs7-signature\\\"; micalg=sha-1; boundary=\\\"=_SIG00163E317F841EE7A7A96D69A468A384\\\"\",\"x-purgate-relay-fid\":\"relay-b62ce8\",\"x-purgate-sourceid\":\"1duK1s-0004qj-1j\",\"x-purgate-ad\":\"Checked for spam and viruses by eXpurgate(R), see www.eleven.de for details.\",\"x-purgate-id\":\"151534::1505833300-00002A4C-41FF312B/0/0\",\"x-purgate\":\"clean\",\"x-purgate-type\":\"clean\",\"x-purgate-relay-bid\":\"relay-9b77e0\"},\"html\":\"<p>2fe8f923-d77b-47d5-827d-18b94b9dde8e</p>\",\"from\":\"donotreply@my333936.mail.crm.ondemand.com\",\"attachments\":[{\"contentType\":\"application/pdf\",\"fileName\":\"KUKA_SunriseServoing_Feedback_UpdateV1.pdf\",\"transferEncoding\":\"base64\",\"generatedFileName\":\"KUKA_SunriseServoing_Feedback_UpdateV1.pdf\",\"contentId\":\"499f9a6e788e934289d5c94f13f4fe6c@mailparser\",\"checksum\":\"c6cc617f94da5138827c2e3b093b0a60\",\"length\":221294,\"content\":{\"type\":\"Buffer\",\"data\":[37,80,68,70,45,49,46,53,13,10,37,181,181,181,181,13,10,49,32,48,32,111,98,106,13,10,60,60,47,84,121,112,101,47,67,97,116,97,108,111,103,47,80,97,103,101,115,32,50,32,48,32,82,47,76,97,110,103,40,101,110,45,85,83,41,32,47,83,116,114,117,99,116,84,114,101,101,82,111,111,116,32,50,57,32,48,32,82,47,77,97,114,107,73,110,102,111,60,60,47,77,97,114,107,101,100,32,116,114,117,101,62,62,47,77,101,116,97,100,97,116,97,32,56,55,32,48,32,82,47,86,105,101,119,101,114,80,114,101,102,101,114,101,110,99,101,115,32,56,56,32,48,32,82,62,62,13,10,101,110,100,111,98,106,13,10,50,32,48,32,111,98,106,13,10,60,60,47,84,121,112,101,47,80,97,103,101,115,47,67,111,117,110,116,32,49,47,75,105,100,115,91,32,51,32,48,32,82,93,32,62,62,13,10,101,110,100,111,98,106,13,10,51,32,48,32,111,98,106,13,10,60,60,47,84,121,112,101,47,80,97,103,101,47,80,97,114,101,110,116,32,50,32,48,32,82,47,82,101,115,111,117,114,99,101,115,60,60,47,70,111,110,116,60,60,47,70,49,32,53,32,48,32,82,47,70,50,32,57,32,48,32,82,47,70,51,32,49,49,32,48,32,82,47,70,52,32,49,54,32,48,32,82,47,70,53,32,49,56,32,48,32,82,47,70,54,32,50,51,32,48,32,82,62,62,47,69,120,116,71,83,116,97,116,101,60,60,47,71,83,55,32,55,32,48,32,82,47,71,83,56,32,56,32,48,32,82,62,62,47,88,79,98,106,101,99,116,60,60,47,77,101,116,97,50,53,32,50,53,32,48,32,82,47,73,109,97,103,101,50,55,32,50,55,32,48,32,82,62,62,47,80,114,111,99,83,101,116,91,47,80,68,70,47,84,101,120,116,47,73,109,97,103,101,66,47,73,109,97,103,101,67,47,73,109,97,103,101,73,93,32,62,62,47,77,101,100,105,97,66,111,120,91,32,48,32,48,32,54,49,50,32,55,57,50,93,32,47,67,111,110,116,101,110,116,115,32,52,32,48,32,82,47,71,114,111,117,112,60,60,47,84,121,112,101,47,71,114,111,117,112,47,83,47,84,114,97,110,115,112,97,114,101,110,99,121,47,67,83,47,68,101,118,105,99,101,82,71,66,62,62,47,84,97,98,115,47,83,47,83,116,114,117,99,116,80,97,114,101,110,116,115,32,48,62,62,13,10,101,110,100,111,98,106,13,10,52,32,48,32,111,98,106,13,10,60,60,47,70,105,108,116,101,114,47,70,108,97,116,101,68,101,99,111,100,101,47,76,101,110,103,116,104,32,49,55,56,56,62,62,13,10,115,116,114,101,97,109,13,10,120,156,189,90,109,111,156,56,16,254,190,210,254,7,127,132,83,215,245,43,134,42,90,41,47,77,149,74,213,93,219,72,253,208,222,7,178,33,201,234,146,101,75,72,174,189,95,127,51,99,216,37,1,178,33,71,175,82,89,176,141,231,241,204,227,241,204,16,246,250,15,182,183,247,250,195,225,201,17,19,243,57,59,56,58,100,223,167,19,193,5,254,75,164,98,130,69,112,117,137,98,69,54,157,124,249,141,173,166,147,131,211,233,228,245,177,100,82,114,97,216,233,197,116,34,97,156,96,146,57,197,133,50,204,137,132,195,27,167,55,48,238,221,103,199,46,111,97,78,118,73,79,113,245,244,110,58,97,51,16,164,44,76,177,248,26,28,103,217,249,89,186,248,43,156,169,128,133,179,40,200,46,66,19,92,132,51,27,100,139,50,252,147,157,190,159,78,222,130,228,143,211,201,127,129,40,173,229,50,110,66,36,100,4,232,43,72,126,32,136,189,253,112,200,88,67,73,114,176,146,64,76,212,165,163,40,137,185,138,90,0,142,178,219,69,56,115,65,17,234,96,185,46,81,17,203,124,133,90,96,47,213,65,11,129,84,9,143,163,62,4,187,84,160,6,170,64,245,171,32,142,120,212,182,193,151,112,166,101,112,21,74,19,100,43,224,0,187,13,103,112,91,220,131,74,114,118,19,206,100,18,228,229,50,140,131,28,187,111,73,57,105,145,1,107,164,8,238,224,89,7,217,57,244,188,98,160,191,56,184,202,112,146,2,137,149,159,161,110,243,146,65,203,58,156,193,60,52,122,25,218,160,164,183,241,14,213,237,2,182]}},{\"contentType\":\"application/pkcs7-signature\",\"fileName\":\"smime.p7s\",\"transferEncoding\":\"base64\",\"generatedFileName\":\"smime.p7s\",\"contentId\":\"451c979ee06edf6738e69f39a699a1ae@mailparser\",\"checksum\":\"8fa016d32e0f29b5d1369321132388d2\",\"length\":1469,\"content\":{\"type\":\"Buffer\",\"data\":[48,130,5,185,6,9,42,134,72,134,247,13,1,7,2,160,130,5,170,48,130,5,166,2,1,1,49,11,48,9,6,5,43,14,3,2,26,5,0,48,11,6,9,42,134,72,134,247,13,1,7,1,160,130,3,164,48,130,3,160,48,130,3,9,160,3,2,1,2,2,14,35,136,50,192,234,167,83,73,16,1,4,12,157,235,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,48,69,49,11,48,9,6,3,85,4,6,19,2,68,69,49,28,48,26,6,3,85,4,10,19,19,83,65,80,32,84,114,117,115,116,32,67,111,109,109,117,110,105,116,121,49,24,48,22,6,3,85,4,3,19,15,83,65,80,32,80,97,115,115,112,111,114,116,32,67,65,48,30,23,13,49,55,48,56,50,50,49,53,52,53,48,56,90,23,13,49,56,48,52,48,49,49,48,48,48,48,48,90,48,129,202,49,11,48,9,6,3,85,4,6,19,2,68,69,49,28,48,26,6,3,85,4,10,19,19,83,65,80,32,84,114,117,115,116,32,67,111,109,109,117,110,105,116,121,49,47,48,45,6,3,85,4,11,19,38,73,48,48,48,48,48,48,48,48,48,55,52,48,49,52,55,53,48,48,32,45,32,66,117,115,105,110,101,115,115,66,121,68,101,115,105,103,110,49,56,48,54,6,9,42,134,72,134,247,13,1,9,1,22,41,100,111,110,111,116,114,101,112,108,121,64,109,121,51,51,51,57,51,54,46,109,97,105,108,46,99,114,109,46,111,110,100,101,109,97,110,100,46,99,111,109,49,50,48,48,6,3,85,4,3,20,41,100,111,110,111,116,114,101,112,108,121,64,109,121,51,51,51,57,51,54,46,109,97,105,108,46,99,114,109,46,111,110,100,101,109,97,110,100,46,99,111,109,48,130,1,34,48,13,6,9,42,134,72,134,247,13,1,1,1,5,0,3,130,1,15,0,48,130,1,10,2,130,1,1,0,168,196,109,117,144,188,234,211,168,241,57,40,195,110,156,170,165,233,193,28,46,123,41,193,59,74,34,211,34,32,57,30,113,30,70,140,159,96,197,163,136,80,72,149,99,218,153,61,153,69,55,137,34,111,120,55,223,11,148,223,63,80,62,160,240,129,128,27,149,112,42,222,182,133,130,137,119,119,177,39,64,89,3,108,99,174,69,63,136,75,71,114,93,16,135,88,103,64,83,158,135,189,6,181,117,12,214,221,92,107,209,230,51,105,25,245,253,13,176,79,180,95,139,78,164,186,15,206,198,118,254,118,133,54,191,19,130,80,29,166,51,107,61,66,20,159,44,28,172,189,101,76,7,198,72,181,196,10,189,158,63,41,9,168,184,197,196,146,46,63,21,198,134,43,231,185,145,162,239,44,153,81,212,1,208,210,35,87,115,225,133,95,88,199,29,64,9,226,107,70,40,227,211,72,183,170,246,188,190,207,174,9,49,49,57,241,249,35,38,120,184,14,91,113,216,19,60,219,128,103,74,119,237,171,251,23,80,215,196,2,155,218,130,148,179,21,50,105,129,143,178,81,11,234,124,191,2,3,1,0,1,163,129,136,48,129,133,48,12,6,3,85,29,19,1,1,255,4,2,48,0,48,37,6,3,85,29,18,4,30,48,28,134,26,104,116,116,112,58,47,47,115,101,114,118,105,99,101,46,115,97,112,46,99,111,109,47,84,67,83,48,14,6,3,85,29,15,1,1,255,4,4,3,2,4,240,48,29,6,3,85,29,14,4,22,4,20,46,171,117,215,21,153,44,58,35,5,87,104,70,160,68,104,247,182,78,48,48,31,6,3,85,29,35,4,24,48,22,128,20,26,68,69,166,87,113,112,86,1,28,120,36,122,81,54,76,192,53,208,94,48,13,6,9,42,134,72,134,247,13,1,1,5,5,0,3,129,129,0,136,220,55,95,31,173,53,137,122,174,46,4,139,154,26,102,235,140,131,250,241,221,50,129,90,222,149,94,241,175,207,164,254,164,14,208,233,176,186,188,160,242,0,240,39,30,215,92,184,201,82,65,143,217,47,213,211,177,219,13,32,84,201,13,102,123,171,18,221,62,209,134,104,51,140,154,155,67,27,201,222,13,250,96,18,134,50,168,161,125,241,186,247,7,185,209,4,255,237,123,48,224,35,226,187,33,153,78,120,201,239,140,59,224,37,35,147,242,87,230,49,139,223,235,1,172,102,15,49,130,1,221,48,130,1,217,2,1,1,48]}}],\"_msgid\":\"e26707e4.6a9968\"}\nreturn msg;","outputs":1,"noerr":0,"x":105,"y":200.39999389648438,"wires":[["a462720c.f4fdc"]]}]

GwilymNewton commented 6 years ago

Fix in #ebb6352

Example flow:

[ { "id": "18d19352.338acd", "type": "tab", "label": "Flow 1" }, { "id": "6e28a6b9.60e568", "type": "debug", "z": "18d19352.338acd", "name": "", "active": false, "console": "false", "complete": "false", "x": 1046, "y": 102, "wires": [] }, { "id": "e61ec743.057fd8", "type": "e-mail in", "z": "18d19352.338acd", "name": "", "protocol": "IMAP", "server": "imap.gmail.com", "useSSL": true, "port": "993", "box": "INBOX", "disposition": "None", "repeat": "20", "x": 177.5, "y": 132, "wires": [ [ "2e2720c3.70a04" ] ] }, { "id": "2e2720c3.70a04", "type": "function", "z": "18d19352.338acd", "name": "", "func": "\n\nif(msg.attachments.length > 0)\n{msg.datatype = \"BIN\"\nmsg.payload = {};\nmsg.payload.content = msg.attachments[0].content\nmsg.payload.fileName = msg.attachments[0].fileName\nmsg.document_id = \"6052b729-f180-4772-88a7-f775d3594ead\"\n\nreturn msg;}", "outputs": 1, "noerr": 0, "x": 339.5, "y": 128, "wires": [ [ "d94ca1f6.bbfc", "2b91b508.e1085a" ] ] }, { "id": "d94ca1f6.bbfc", "type": "debug", "z": "18d19352.338acd", "name": "", "active": true, "console": "false", "complete": "true", "x": 493.5, "y": 88, "wires": [] }, { "id": "ba0c8e3d.fa45e", "type": "debug", "z": "18d19352.338acd", "name": "", "active": true, "console": "false", "complete": "false", "x": 740.5, "y": 130, "wires": [] }, { "id": "2b91b508.e1085a", "type": "discovery-update", "z": "18d19352.338acd", "name": "", "environment": "", "collection": "", "delay": 1000, "max_Q_size": 10000, "x": 533.5, "y": 133, "wires": [ [ "ba0c8e3d.fa45e" ] ] }, { "id": "63d64141.eab1c", "type": "discovery-insert", "z": "18d19352.338acd", "name": "", "environment": "", "collection": "", "delay": 1000, "max_Q_size": 10000, "x": 417.5, "y": 292, "wires": [ [] ] } ]

GwilymNewton commented 6 years ago

Releasing fix in version 0.1.5

sauraba commented 6 years ago

Appreciate all your help and time to fix the code base for node-red Discovery node to insert/update binary documents in near real time. This helps to capture the updated / relevant corpus of information (service / repair manuals) from the field engg / OEMs. Thanks a lot!