Open omkar-tenkale opened 4 years ago
This should not be slow. The most obvious reasons why it may be slow:
Considering your question, I suspect that the problem is the item 1.
Trying properly indexing your collection.
@omkar-tenkale You can look at the logs of your database to find out how long the query takes on the DB side.
With mongo profiling i can see 58 queries are run for 58000 rows
The last 2 blocks of log
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 39369748207,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(1288),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 671635,
"millis" : 1,
"execStats" : {
},
"ts" : ISODate("2020-03-30T10:53:02.885Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 39369748207,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(1552),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 682352,
"millis" : 1,
"execStats" : {
},
"ts" : ISODate("2020-03-30T10:53:02.918Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
Type "it" for more
>
Parse is still taking upto 10 seconds for 58000 items without any query constraints
The query i simple,this is the cloud codee
async function indexTest(req,res){ console.time("timelog");
var Product = Parse.Object.extend("Product"); var query = new Parse.Query(Product); query.limit(1000000) var res = await query.find(); console.timeEnd("timelog"); console.time("timelog"); console.log("resp size=="+res.length) console.timeEnd("timelog"); return res.length; }
This is the log (pm2 logs 0)
0|app1 | timelog: 10.097s 0|app1 | resp size==58110 0|app1 | timelog: 0.145ms 0|app1 | info: Ran cloud function indexTest for user undefined with: 0|app1 | Input: {} 0|app1 | Result: 58110 {"functionName":"indexTest","params":{}}
In your mongodb logs I don't see any excessive execution times.
Can you turn on profiling to only log queries that take longer than 100ms?
Actually, the execStats
are empty in the logs you posted here. The stats should give you more hints about the execution time and whether an index was used for the scan, etc.
How can i include execStats?
Here's what i could find Cloud code request from parse-dashboard console
> db.setProfilingLevel(2)
{ "was" : 0, "slowms" : -1, "ok" : 1 }
> db.system.profile.find().pretty()
{
"op" : "query",
"ns" : "dev.system.indexes",
"query" : {
"expireAfterSeconds" : {
"$exists" : true
}
},
"ntoreturn" : 0,
"ntoskip" : 0,
"nscanned" : 29,
"nscannedObjects" : 29,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(92),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(2)
}
},
"nreturned" : 0,
"responseLength" : 20,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 31,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 30,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 29,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:01:25.597Z"),
"client" : "0.0.0.0",
"allUsers" : [
{
"user" : "__system",
"db" : "local"
}
],
"user" : "__system@local"
}
>
>
>
>
> db.system.profile.find().pretty()
{
"op" : "query",
"ns" : "dev.system.indexes",
"query" : {
"expireAfterSeconds" : {
"$exists" : true
}
},
"ntoreturn" : 0,
"ntoskip" : 0,
"nscanned" : 29,
"nscannedObjects" : 29,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(92),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(2)
}
},
"nreturned" : 0,
"responseLength" : 20,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 31,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 30,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 29,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:01:25.597Z"),
"client" : "0.0.0.0",
"allUsers" : [
{
"user" : "__system",
"db" : "local"
}
],
"user" : "__system@local"
}
{
"op" : "query",
"ns" : "dev.system.profile",
"query" : {
},
"ntoreturn" : 0,
"ntoskip" : 0,
"nscanned" : 1,
"nscannedObjects" : 1,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(96),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(4)
}
},
"nreturned" : 1,
"responseLength" : 649,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 3,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 1,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 1,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:01:28.613Z"),
"client" : "157.33.212.99",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._SCHEMA",
"query" : {
"$query" : {
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 21,
"nscannedObjects" : 21,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(132),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(3)
}
},
"nreturned" : 21,
"responseLength" : 6562,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 23,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 21,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 21,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.054Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._Session",
"query" : {
"$orderby" : {
},
"$query" : {
"_session_token" : "r:69bf2af111cc47a67c6d3119f282e5a1"
}
},
"ntoreturn" : 1,
"ntoskip" : 0,
"nscanned" : 125,
"nscannedObjects" : 125,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(176),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(2)
}
},
"nreturned" : 1,
"responseLength" : 324,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 126,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 1,
"needTime" : 125,
"needFetch" : 0,
"isEOF" : 0,
"docsTested" : 125,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.056Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._User",
"query" : {
"$orderby" : {
},
"$query" : {
"_id" : "MoEjAOXpF4"
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 1,
"nscannedObjects" : 1,
"idhack" : true,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(42),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(1),
"w" : NumberLong(2)
}
},
"nreturned" : 1,
"responseLength" : 419,
"millis" : 0,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:05.057Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._Session",
"query" : {
"$orderby" : {
},
"$query" : {
"_session_token" : "r:69bf2af111cc47a67c6d3119f282e5a1"
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 219,
"nscannedObjects" : 219,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(281),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(4),
"w" : NumberLong(4)
}
},
"nreturned" : 1,
"responseLength" : 324,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 221,
"yields" : 1,
"unyields" : 1,
"invalidates" : 0,
"advanced" : 1,
"needTime" : 219,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 219,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.066Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._SCHEMA",
"query" : {
"$query" : {
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 21,
"nscannedObjects" : 21,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(80),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(2)
}
},
"nreturned" : 21,
"responseLength" : 6562,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 23,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 21,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 21,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.070Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._Join:users:_Role",
"query" : {
"$orderby" : {
},
"$query" : {
"relatedId" : {
"$in" : [
"MoEjAOXpF4"
]
}
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 0,
"nscannedObjects" : 0,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(151),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(3)
}
},
"nreturned" : 0,
"responseLength" : 20,
"millis" : 0,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:05.074Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._Role",
"query" : {
"$orderby" : {
"_id" : 1
},
"$query" : {
"_id" : {
"$in" : [ ]
}
}
},
"ntoreturn" : 100,
"ntoskip" : 0,
"nscanned" : 0,
"nscannedObjects" : 0,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(183),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(2)
}
},
"nreturned" : 0,
"responseLength" : 20,
"millis" : 0,
"execStats" : {
"type" : "FETCH",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"alreadyHasObj" : 0,
"forcedFetches" : 0,
"matchTested" : 0,
"children" : [
{
"type" : "IXSCAN",
"works" : 1,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 0,
"needTime" : 0,
"needFetch" : 0,
"isEOF" : 1,
"keyPattern" : "{ _id: 1 }",
"isMultiKey" : 0,
"boundsVerbose" : "field #0['_id']: ",
"yieldMovedCursor" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0,
"matchTested" : 0,
"keysExamined" : 0,
"children" : [ ]
}
]
},
"ts" : ISODate("2020-03-31T08:02:05.075Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev.userMaster_Test",
"query" : {
"$orderby" : {
},
"$query" : {
"_rperm" : {
"$in" : [
null,
"*",
"*",
"MoEjAOXpF4"
]
}
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 1,
"nscannedObjects" : 1,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(90),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(1)
}
},
"nreturned" : 1,
"responseLength" : 169,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 3,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 1,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 1,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.076Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._SCHEMA",
"query" : {
"$query" : {
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 21,
"nscannedObjects" : 21,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(91),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(4)
}
},
"nreturned" : 21,
"responseLength" : 6562,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 23,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 21,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 21,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:05.151Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "command",
"ns" : "dev.$cmd",
"command" : {
"findAndModify" : "_Session",
"query" : {
"_id" : "SJhEylfnb3",
"_wperm" : {
"$in" : [
null,
"*",
"MoEjAOXpF4"
]
}
},
"new" : true,
"remove" : false,
"upsert" : false,
"update" : {
"$set" : {
"_updated_at" : ISODate("2020-03-31T08:02:05.150Z")
}
}
},
"updateobj" : {
"$set" : {
"_updated_at" : ISODate("2020-03-31T08:02:05.150Z")
}
},
"nscanned" : 1,
"nscannedObjects" : 1,
"nMatched" : 1,
"nModified" : 1,
"fastmod" : true,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(0),
"w" : NumberLong(278)
},
"timeAcquiringMicros" : {
"r" : NumberLong(0),
"w" : NumberLong(5)
}
},
"responseLength" : 395,
"millis" : 0,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:05.153Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev._SCHEMA",
"query" : {
"$query" : {
}
},
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 21,
"nscannedObjects" : 21,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(240),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(5),
"w" : NumberLong(3)
}
},
"nreturned" : 21,
"responseLength" : 6562,
"millis" : 0,
"execStats" : {
"type" : "COLLSCAN",
"works" : 23,
"yields" : 0,
"unyields" : 0,
"invalidates" : 0,
"advanced" : 21,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 1,
"docsTested" : 21,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:16.592Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "query",
"ns" : "dev.Product",
"query" : {
"$orderby" : {
},
"$query" : {
"_rperm" : {
"$in" : [
null,
"*",
"*"
]
}
}
},
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"ntoskip" : 0,
"nscanned" : 1000,
"nscannedObjects" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(2778),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 687471,
"millis" : 2,
"execStats" : {
"type" : "COLLSCAN",
"works" : 1001,
"yields" : 7,
"unyields" : 7,
"invalidates" : 0,
"advanced" : 1000,
"needTime" : 1,
"needFetch" : 0,
"isEOF" : 0,
"docsTested" : 1000,
"children" : [ ]
},
"ts" : ISODate("2020-03-31T08:02:16.598Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(2151),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(5),
"w" : NumberLong(5)
}
},
"nreturned" : 1000,
"responseLength" : 689544,
"millis" : 2,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.654Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(2144),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(2),
"w" : NumberLong(4)
}
},
"nreturned" : 1000,
"responseLength" : 697344,
"millis" : 2,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.690Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(1619),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(4)
}
},
"nreturned" : 1000,
"responseLength" : 694691,
"millis" : 1,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.763Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(1525),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 682413,
"millis" : 1,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.797Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(2501),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(4),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 681485,
"millis" : 2,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.837Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
{
"op" : "getmore",
"ns" : "dev.Product",
"cursorid" : 82313479072,
"ntoreturn" : 1000,
"keyUpdates" : 0,
"numYield" : 0,
"lockStats" : {
"timeLockedMicros" : {
"r" : NumberLong(1579),
"w" : NumberLong(0)
},
"timeAcquiringMicros" : {
"r" : NumberLong(3),
"w" : NumberLong(3)
}
},
"nreturned" : 1000,
"responseLength" : 691719,
"millis" : 1,
"execStats" : {
},
"ts" : ISODate("2020-03-31T08:02:16.883Z"),
"client" : "127.0.0.1",
"allUsers" : [ ],
"user" : ""
}
Type "it" for more
>
it looks like the getmores are occurring in just a few milis.
there is a bottleneck of some sort.
If your run a query against the database form the server the iterates through all the objects in you parouct tables, how long does it take?
what does a product document 'look like' (i.e. how big is the json text).
what is the network topology between the database server and the parse-server? Are they on the same network? What is latency and bandwidth between the two?
If your run a query against the database form the server the iterates through all the objects in you parouct tables, how long does it take?
In mongoshell its quiet fast,1-2 secs
what does a product document 'look like' (i.e. how big is the json text).
Closely similar to { "_id": "y2IEREsCJR", "_p_createdBy": "Employee$nqzq73IJZn", "_p_company": "Company$othercmpny", "_p_brand": "Brand$5KUm1qj624", "name": "SOME NAM NAME NAME", "parentName": "SOME NAM NAME NAME", "ERPName": "SOME NAM NAME NAME", "description": "SOME NAM NAME NAME", "quantity": "28", "season": "SOME", "sleeve": "", "basicRate": 1899, "ERPCode": "2423423523532", "pattern": "", "subCategory2": "SLIM FIT", "HSCode": 0, "content": "SOME NAM NAME NAME", "source": "", "category": "JEANS", "exciseItem": false, "subCategory": "SLIM FIT", "EANCode": 24332543232532, "neck": "", "unitOfMeasurement": "SOME NAM NAME NAME", "SellingPrice": 1899, "_created_at": "2020-03-13T23:30:32.098Z", "_updated_at": "2020-03-27T11:36:35.814Z", }
what is the network topology between the database server and the parse-server? Are they on the same network? What is latency and bandwidth between the two?
Parse server specs npm list parse@2.11.0
Server version 3.9.0 (From Parse dashboard app list)
db.version() 2.6.10 <<Can be an issue?
System Ubuntu 16.04 Mongodb at localhost:27017 same server Mongouri (parse default)
Tried accessing db directly
Included in index.js
const Config = require('./node_modules/parse-server/lib/Config');
const config = Config.get(Parse.applicationId);
const mongoAdapter = config.database.adapter;
app.get('/mongotest',async function(req, res) {
console.time("connect")
await mongoAdapter.connect();
const collection = await mongoAdapter.database.collection("Product");
console.timeEnd("connect")
console.time("find")
var response = await collection.find().toArray();
console.timeEnd("find")
console.log("totals="+response.length)
res.status(200).send(response);
});
Processed in under 3 seconds, Console log
0|app1 | connect: 0.375ms
0|app1 | find: 1.701s
0|app1 | totals=58110
So the problem lies not in db but the server
On the same Ubuntu 16.04 VPS server
Monitor your server resources during the slow request. Do CPU, IOPS, network connections, etc max out?
Are there any other apps running on Node that could be blocking the Node thread?
Profile Node app. For example, https://nodejs.org/en/docs/guides/simple-profiling/
Is the response also slow when you deploy the latest version of parse server and mongodb?
Debug parse server while running the request and inspect the code to see at which point it is slow.
Sent with GitHawk
Monitor your server resources during the slow request. Do CPU, IOPS, network connections, etc max out?
CPU > There's a visible spike but cpu usage doesn't become 100% IOPS > Ram usage is increased but still no 100% usage. RAM normally uses 1.1gb when api is hit it goes to around 400mb then gradually increases till 1.9gb then drops to original 1.1gb. Strange?Please refer image
There may be a memory leak somewhere but this doesn't constitute to our problem as even a freshly restarted server with 86 mb usage gives slow response pm2 restart > 86mb > cloudcodehit > 1.9gb gradual > response send > 1.2 gb
network connections > No idea on this,doesnt seem an issue as response is slow when no other client interacting with server
4 times api hit usage>
Are there any other apps running on Node that could be blocking the Node thread
There are other pm2 parse instances but even stopping them doesn't solve the issue
Working on other checks.Does this info gives any clues?
db.version() 2.6.10 <<Can be an issue?
Is it possible to upgrade the database server? That is definitely an unsupported db version.
Can you test with >= Mongo 3.6?
Is the response also slow when you deploy the latest version of parse server and mongodb?
After updating
Parse server specs npm list parse@2.12.0 parse-server@4.2.0
Server version 4.2.0 (From Parse dashboard app list)
db.version() 4.2.5
Updating has not solved the issue
This is the transaction details
Seems to be related with mongoObjectToParseObject ,
Generated using node --prof index.js ...
This is the full log processlog.txt
If you set a breakpoint in that function of parse server and debug step-by-step you should be able to find the line that causes the delay.
Sent with GitHawk
Please share more info and tools on how i can do that as i never done this before
@flovilmart @dplewis @acinader @davimacedo @mtrezza I've tried many approaches and it finally seems the issue is in parse server code, I'm unable to debug the server being unfamiliar with its internal workings.
Can anyone shed some light on this issue? I'll research more in it if there's still scope for the issue being caused by outside code or my mistakes
You already got very close to identifying the issue, since you nailed it down to mongoObjectToParseObject
.
It is possible that the issue is specific to your architecture or data set. It is possible that the transformation of raw DB result to a Parse object may be inefficient for your specific data set. Maybe a recursion issue.
Debugging parse server is easy, for example with Visual Studio code. Instructions here. It involves setting up parse server locally, then launching it via VisualStudio Code to be able to inspect the code while running.
I closed due to the unsupported dB. Reopening since you are still encountering with a supported setup.
I've set up a repo to test my understanding of @omkar-tenkale's issue.
I am running the data load now which will take a few hours and then I'll benchmark paging through the records directly with Mongo js vs. through parse server and I'll attempt to profile if needed.
Ok, so far I have replicated @omkar-tenkale's issue.
It took a little shy of 3 hours to load 150k test products with roughly the same schema. I took a worst-case where each product had its own employee, brand, and company.
I was a little surprised that a session was also created for each user I created, but that's ok!
Next up I'll try and make an equivalent query bypassing parse-server and going direct to parse to see what the best case could be.
forgot to put in last comment that I also replicated the indexTest
function verbatim and it took me 16.5 seconds to query all 150k Product
objects and put 'em into a javascript array. So the time is roughly in line with what @omkar-tenkale is getting. So we've got a good test case!
I added querying with the mongo native driver, two different ways.
https://github.com/acinader/parse-playground
In short: via parse-server: ~ 16.5 s direct with native driver: ~ 4s
So is that bad? I'm not sure. As time permits, I'm going to use the test as a place to improve my profiling chops on parse-server...but how good would be good enough? Given what parse-server is and what it does, we shouldn't expect it to do a raw query at the same speed as the native driver, should we?
In your case @omkar-tenkale what is the use case for getting all of the records?
@acinader Thank you so much for putting such efforts,I really appreciate it.
In our use case scenario, we need to maintain a dynamic catalog of products on server and sync it with Android app for offline use (Fetch from api,create a sqlite database for local use) when the user logs in. With targeted 250000 product items to load offline, total time consumed is
It takes more then a minute, Response download, create db can be optimized but i wasn't able to figure out the reason for high processing time.
I used a memory cache strategy (https://www.npmjs.com/package/memory-cache) to skip the query on every request,it was working well too but then comes the main issue as in the title.
I wanted to generate reports on the data,so i wrote a cloud code for this, Fetching all the rows,grouping,sorting,filtering to create reports on it in the cloud code. After noticing the reports taking 30+ seconds,I put some log statements and found that processing data i.e. grouping,sorting,filtering was not time consuming but it was the find query,just fetching the data to process it was taking about 95% of total api response time. And i can't use a cache strategy here as reports are different for different users, new data is also added in real time and managing cache will be very complex.
That's why i wanted to find the issue,was it from my side or my initial doubt
I assume this is fastest parse can work.
Is there a scope of improvement here in future?
@acinader Did you find out what causes the extra processing time in Parse Server?
@omkar-tenkale profiled Nodejs and found out that most of the time is consumed by mongoObiectToParseObject
. If that’s the case, maybe there is a specific optimization that can be done there?
Sent with GitHawk
@omkar-tenkale would it be possible to share a small sample of your data and also your class schema?
Unfortunately I'm not allowed to do that, I think @acinader is quite successful in reproducing the issue at https://github.com/acinader/parse-playground
one thing you may want to look at for the reports is using an aggregation pipeline so you don't need to get all the data, but do the grouping, sorting and filtering on the dataserver instead of the parse server.
i will profile the test, but i don't think that'll be a solution for you.
One thing to try is adding bson-ext
to your dependencies. This is a C++ addon for Node that performs the BSON deserialization, and the Mongo node driver should automatically start using it once it is installed. It can make a big difference for some workloads.
It looks like there's already an optionalDependencies
section in package.json for bcrypt
(another performance enhancing replacement). Maybe we could add bson-ext
there?
so I ran the profile and I'm trying to work my way through it, but one thing I noticed immediately is that
mongoObjectToParseObject
is only taking up 2.4%
The real culprit is: 10045 67.4% 69.1% t __ZN2v88internal15TranslatedStateD1Ev
which is just shy of 70% of the time
if we take the top 3 offenders, we get to 3/4 of our time.
10045 67.4% 69.1% t __ZN2v88internal15TranslatedStateD1Ev
538 3.6% 3.7% T _task_policy_set
434 2.9% 3.0% T node::contextify::ContextifyContext::CompileFunction(v8::FunctionCallbackInfo<v8::Value> const&)
@acinader Did you profile without bson-ext
dependency? It would be interesting to see if adding the dependency improves the execution time.
I did! It did not improve execution time (it actually made it marginally worse, but I think I'd have to do 100 tests and average to see for sure. Definitely changes the profile.)
Overall, I do not see any obvious bottlenecks.
converting from mongo doc to parse object is intensive enough that doing 150k take some time.
@acinader
mongoObjectToParseObject is only taking up 2.4% The real culprit is: 10045 67.4% 69.1% t __ZN2v88internal15TranslatedStateD1Ev
Do we know why this is invoked, what the stack trace is?
Here's an excerpt of the process file:
(if anyone wants to play along https://nodejs.org/en/docs/guides/simple-profiling/ and clone https://github.com/acinader/parse-playground and you can easily run this yourself -- would love a second set of 👀)
[JavaScript]:
ticks total nonlib name
351 2.4% 2.4% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
162 1.1% 1.1% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
119 0.8% 0.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
74 0.5% 0.5% LazyCompile: *decode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/decode.js:40:16
...
[C++]:
ticks total nonlib name
10045 67.4% 69.1% t __ZN2v88internal15TranslatedStateD1Ev
538 3.6% 3.7% T _task_policy_set
434 2.9% 3.0% T node::contextify::ContextifyContext::CompileFunction(v8::FunctionCallbackInfo<v8::Value> const&)
260 1.7% 1.8% T _read$NOCANCEL
179 1.2% 1.2% T ___ulock_wait
132 0.9% 0.9% T __kernelrpc_thread_policy_set
...
[Summary]:
ticks total nonlib name
1198 8.0% 8.2% JavaScript
13282 89.2% 91.3% C++
2705 18.2% 18.6% GC
354 2.4% Shared libraries
61 0.4% Unaccounted
...
[Bottom up (heavy) profile]:
Note: percentage shows a share of a particular caller in the total
amount of its parent calls.
Callers occupying less than 1.0% are not shown.
ticks parent name
10045 67.4% t __ZN2v88internal15TranslatedStateD1Ev
5256 52.3% t __ZN2v88internal15TranslatedStateD1Ev
1098 20.9% t __ZN2v88internal15TranslatedStateD1Ev
393 35.8% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
387 98.5% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
387 100.0% t __ZN2v88internal15TranslatedStateD1Ev
6 1.5% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
6 100.0% t __ZN2v88internal15TranslatedStateD1Ev
280 25.5% LazyCompile: *initializeState /Users/arthur/code/parse-playground/node_modules/parse/lib/node/UniqueInstanceStateController.js:108:25
279 99.6% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
272 97.5% LazyCompile: *decode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/decode.js:40:16
7 2.5% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
152 13.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:106
152 100.0% t __ZN2v88internal15TranslatedStateD1Ev
152 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:83
96 8.7% LazyCompile: *parseDate /Users/arthur/code/parse-playground/node_modules/parse/lib/node/parseDate.js:18:19
94 97.9% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
94 100.0% t __ZN2v88internal15TranslatedStateD1Ev
2 2.1% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
2 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
95 8.7% LazyCompile: *encode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/encode.js:41:16
49 51.6% LazyCompile: *commitServerChanges /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ObjectStateMutations.js:207:29
44 89.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
5 10.2% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
46 48.4% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
46 100.0% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
39 3.6% LazyCompile: *_clearServerData /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:309:19
39 100.0% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
39 100.0% t __ZN2v88internal15TranslatedStateD1Ev
809 15.4% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
792 97.9% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
792 100.0% t __ZN2v88internal15TranslatedStateD1Ev
792 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:523:14
17 2.1% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
17 100.0% t __ZN2v88internal15TranslatedStateD1Ev
17 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:523:14
776 14.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
776 100.0% t __ZN2v88internal15TranslatedStateD1Ev
776 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
776 100.0% t __ZN2v88internal15TranslatedStateD1Ev
593 11.3% LazyCompile: *commitServerChanges /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ObjectStateMutations.js:207:29
574 96.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
574 100.0% t __ZN2v88internal15TranslatedStateD1Ev
574 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
17 2.9% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
17 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
17 100.0% t __ZN2v88internal15TranslatedStateD1Ev
378 7.2% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
376 99.5% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
376 100.0% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
372 98.9% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
4 1.1% LazyCompile: ~deserialize /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:19:27
375 7.1% LazyCompile: *encode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/encode.js:41:16
227 60.5% LazyCompile: *commitServerChanges /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ObjectStateMutations.js:207:29
218 96.0% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
218 100.0% t __ZN2v88internal15TranslatedStateD1Ev
9 4.0% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
9 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
147 39.2% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
146 99.3% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
146 100.0% t __ZN2v88internal15TranslatedStateD1Ev
296 5.6% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:106
296 100.0% t __ZN2v88internal15TranslatedStateD1Ev
296 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:83
296 100.0% t __ZN2v88internal15TranslatedStateD1Ev
232 4.4% LazyCompile: *parseDate /Users/arthur/code/parse-playground/node_modules/parse/lib/node/parseDate.js:18:19
227 97.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
227 100.0% t __ZN2v88internal15TranslatedStateD1Ev
227 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
5 2.2% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
5 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
5 100.0% t __ZN2v88internal15TranslatedStateD1Ev
131 2.5% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/FilesController.js:66:18
131 100.0% t __ZN2v88internal15TranslatedStateD1Ev
131 100.0% LazyCompile: ~expandFilesInObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/FilesController.js:64:22
131 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/RestQuery.js:611:97
83 1.6% LazyCompile: *initializeState /Users/arthur/code/parse-playground/node_modules/parse/lib/node/UniqueInstanceStateController.js:108:25
83 100.0% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
80 96.4% LazyCompile: *decode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/decode.js:40:16
80 100.0% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
3 3.6% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
3 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
78 1.5% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
49 62.8% LazyCompile: *decode /Users/arthur/code/parse-playground/node_modules/parse/lib/node/decode.js:40:16
49 100.0% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
49 100.0% t __ZN2v88internal15TranslatedStateD1Ev
26 33.3% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
26 100.0% t __ZN2v88internal15TranslatedStateD1Ev
26 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
3 3.8% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
3 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
3 100.0% t __ZN2v88internal15TranslatedStateD1Ev
842 8.4% LazyCompile: *mongoObjectToParseObject /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoTransform.js:1431:34
826 98.1% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
826 100.0% t __ZN2v88internal15TranslatedStateD1Ev
826 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:523:14
826 100.0% t __ZN2v88internal15TranslatedStateD1Ev
16 1.9% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
16 100.0% t __ZN2v88internal15TranslatedStateD1Ev
16 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:523:14
16 100.0% t __ZN2v88internal15TranslatedStateD1Ev
618 6.2% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
618 100.0% t __ZN2v88internal15TranslatedStateD1Ev
618 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
618 100.0% t __ZN2v88internal15TranslatedStateD1Ev
618 100.0% LazyCompile: *processTicksAndRejections internal/process/task_queues.js:69:35
538 5.4% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
535 99.4% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
535 100.0% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
483 90.3% LazyCompile: *deserializeObject /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:43:33
483 100.0% LazyCompile: ~deserialize /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:19:27
52 9.7% LazyCompile: ~deserialize /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/parser/deserializer.js:19:27
52 100.0% LazyCompile: ~BSON.deserialize /Users/arthur/code/parse-playground/node_modules/bson/lib/bson/bson.js:134:38
282 2.8% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:106
282 100.0% t __ZN2v88internal15TranslatedStateD1Ev
282 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Controllers/DatabaseController.js:1162:83
282 100.0% t __ZN2v88internal15TranslatedStateD1Ev
282 100.0% LazyCompile: *processTicksAndRejections internal/process/task_queues.js:69:35
240 2.4% LazyCompile: *commitServerChanges /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ObjectStateMutations.js:207:29
233 97.1% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
233 100.0% t __ZN2v88internal15TranslatedStateD1Ev
233 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
233 100.0% t __ZN2v88internal15TranslatedStateD1Ev
7 2.9% LazyCompile: *fromJSON /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseObject.js:2203:18
7 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:759:44
7 100.0% t __ZN2v88internal15TranslatedStateD1Ev
7 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse/lib/node/ParseQuery.js:753:77
154 1.5% LazyCompile: *<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:528:26
154 100.0% t __ZN2v88internal15TranslatedStateD1Ev
154 100.0% LazyCompile: ~<anonymous> /Users/arthur/code/parse-playground/node_modules/parse-server/lib/Adapters/Storage/Mongo/MongoStorageAdapter.js:523:14
154 100.0% t __ZN2v88internal15TranslatedStateD1Ev
154 100.0% LazyCompile: *processTicksAndRejections internal/process/task_queues.js:69:35
This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
I'm trying to fetch more than 100,000 rows for report generation. I tried these approaches,
Both the calls are slow,
For cloud code query
query = //new query query.equalTo("SOMEKEY","SOMEVALUE")
For total of 1,500,000 rows In cloud code,
In recursive call approach, there is approx 1 second waiting time per 1000 rows.
In query.each() approach this query takes approx 60 seconds to process, http waiting time
Is there a better approach for processing such huge data.
One alternative I can think of is parse aggregate but this approach doesn't respect parse authentication, security, acl etc. and doesn't work with existing query logic. Will this show any performance improvement and worth a try?