aravindnc / mongoose-aggregate-paginate-v2

A cursor based custom aggregate pagination library for Mongoose with customizable labels.
MIT License
131 stars 23 forks source link

Sorted result in mongoose-aggregate-paginate-v2 is changing #18

Open asmafakhfakh opened 4 years ago

asmafakhfakh commented 4 years ago

I'm sending a query with postman to an apollo server to get a list of users. This is my options object:

const options = {
            page: parseInt(page),
            limit: 15,
            lean: true,
            pagination: true,
            sort:{ lastUpdate: "descending", creationDateTime:"descending"}
        }

Sometimes when I resend the request with exactly the same parameters, the result changes even though my data is the same. These are some samples of different results I got: sample 1:

{
    "listOfUsers": {
        "list": [
            {
                "lastUpdate": "2020-03-27T06:00:47.689Z",
                "creationDateTime": "2019-10-09T11:20:13.004Z"
            },
            {
                "lastUpdate": "2020-03-27T05:54:00.105Z",
                "creationDateTime": "2020-01-06T12:54:16.699Z"
            },
            {
                "lastUpdate": "2020-03-19T18:36:29.163Z",
                "creationDateTime": "2019-10-21T11:16:30.428Z"
            },
            {
                "lastUpdate": "2020-03-19T17:46:36.799Z",
                "creationDateTime": "2020-01-06T09:43:41.844Z"
            },
            {
                "lastUpdate": null,
                "creationDateTime": "2020-01-06T10:55:47.714Z"
            }
        ],
        "totalDocs": 5,
        "limit": 15,
        "totalPages": 1,
        "page": 1,
        "pagingCounter": 1,
        "hasPrevPage": false,
        "hasNextPage": false,
        "prevPage": null,
        "nextPage": null
    }
}

sample 2:

{
    "listOfUsers": {
        "list": [
            {
                "lastUpdate": "2020-03-27T06:00:47.689Z",
                "creationDateTime": "2019-10-09T11:20:13.004Z"
            },
            {
                "lastUpdate": "2020-03-27T05:54:00.105Z",
                "creationDateTime": "2020-01-06T12:54:16.699Z"
            },
            {
                "lastUpdate": "2020-03-19T17:46:36.799Z",
                "creationDateTime": "2020-01-06T09:43:41.844Z"
            },
            {
                "lastUpdate": null,
                "creationDateTime": "2020-01-06T10:55:47.714Z"
            },
            {
                "lastUpdate": "2020-03-19T18:36:29.163Z",
                "creationDateTime": "2019-10-21T11:16:30.428Z"
            }
        ],
        "totalDocs": 5,
        "limit": 15,
        "totalPages": 1,
        "page": 1,
        "pagingCounter": 1,
        "hasPrevPage": false,
        "hasNextPage": false,
        "prevPage": null,
        "nextPage": null
    }
}

sample 3:

{
    "listOfUsers": {
        "list": [
            {
                "lastUpdate": "2020-03-19T18:36:29.163Z",
                "creationDateTime": "2019-10-21T11:16:30.428Z"
            },
            {
                "lastUpdate": "2020-03-27T06:00:47.689Z",
                "creationDateTime": "2019-10-09T11:20:13.004Z"
            },
            {
                "lastUpdate": "2020-03-19T17:46:36.799Z",
                "creationDateTime": "2020-01-06T09:43:41.844Z"
            },
            {
                "lastUpdate": "2020-03-27T05:54:00.105Z",
                "creationDateTime": "2020-01-06T12:54:16.699Z"
            },
            {
                "lastUpdate": null,
                "creationDateTime": "2020-01-06T10:55:47.714Z"
            }
        ],
        "totalDocs": 5,
        "limit": 15,
        "totalPages": 1,
        "page": 1,
        "pagingCounter": 1,
        "hasPrevPage": false,
        "hasNextPage": false,
        "prevPage": null,
        "nextPage": null
    }
}
aravindnc commented 4 years ago

@asmafakhfakh Can you share the query. Also try adding an index for creationDateTime field and try again.

asmafakhfakh commented 4 years ago

@aravindnc I tried indexing both fields i'm using for sort, doesn't resolve the problem. So this is my query PS: I tried passing directly -1 , "descending" and "desc" to sort parameter, and also a string like mentioned in documentation, still the problem persists

module.exports.listOfUsers = (root, { page, name, lastname, sort }) => {
    return new Promise((resolve, reject) => {

        const options = {
            page: parseInt(page),
            limit: 15,
            lean: true,
            pagination: true,
            sort: { lastUpdate: sort, creationDateTime: sort }
        }
        var aggUser = user.aggregate([
            {
                $lookup: {
                    from: "profiles",
                    localField: "profile",
                    foreignField: "_id",
                    as: "profile"
                }
            }, {
                $match: {
                    "profile.0.name":name,
                    "profile.0.lastname":lastname,
                }
            }, {
                $lookup: {
                    from: "books",
                    localField: "favoritebook",
                    foreignField: "_id",
                    as: "favoritebook"
                }
            }, {
                $lookup: {
                    from: "movies",
                    localField: "favoritemovie",
                    foreignField: "_id",
                    as: "favoritemovie"
                }
            }
        ]);
        user.aggregatePaginate(aggUser, options).then(
            async res => {
                if (res.totalDocs == 0) {
                    resolve({
                        list: res.docs,
                        totalDocs: res.totalDocs,
                        limit: res.limit,
                        totalPages: res.totalPages,
                        page: res.page,
                        pagingCounter: res.pagingCounter,
                        hasPrevPage: res.hasPrevPage,
                        hasNextPage: res.hasNextPage,
                        prevPage: res.prevPage,
                        nextPage: res.nextPage
                    })
                }
                let FinalTab = []
                await res.docs.map(ell => {
                    axios.get(
                        `${ENDPOINT}/${ell.profile.state}`
                    ).then(async ress => {
                        let Objec = ell
                        Objec.profile.state = ress.data.state
                        await FinalTab.push(Objec)
                        if (FinalTab.length == res.docs.length) {
                            await resolve({
                                list: FinalTab,
                                totalDocs: res.totalDocs,
                                limit: res.limit,
                                totalPages: res.totalPages,
                                page: res.page,
                                pagingCounter: res.pagingCounter,
                                hasPrevPage: res.hasPrevPage,
                                hasNextPage: res.hasNextPage,
                                prevPage: res.prevPage,
                                nextPage: res.nextPage
                            })
                        }
                        return ell
                    }).catch(err => reject(new Error(err)))
                })

            }
        ).catch(
            err => { return err }
        )
    }).catch((e) => { return e });
}
aravindnc commented 4 years ago

Can you try sort inside aggregate itself.

asmafakhfakh commented 4 years ago

@aravindnc I added a sort stage and the problem persists

{ 
   $sort : { lastUpdate : -1 } 
}
aravindnc commented 4 years ago

@asmafakhfakh If you don't mind, can you share your sample dateset with all required collections(minimal) to run the aggregate query, so I can test it at my end.

asmafakhfakh commented 4 years ago

Archive.zip this is a cropped dataset of the two collections users and profiles, you need to drop the last two stages of lookup for favoritebook and favoritemovie, as well as the query for state at the end

yacineBR commented 4 years ago

has this problem been resolved ?

czystyl commented 4 years ago

I can also confirm that the order is changing even after adding indexes. Let me know how I can help.

czystyl commented 4 years ago

I fixed the problem by adding the _id field to sort statement. The problem can be reproduced even without this plugin using $skip and $limit on aggregate.

@aravindnc I think now issue could be closed.

asmafakhfakh commented 4 years ago

@czystyl I dont think that solves it because the problem is not in documents that has equivalent dates for the sorted field that changes order, I get a totally wrong order, you can see that in the examples I put above. But I will try your solution anyway and give you feedback

ChavaMS commented 3 years ago

@aravindnc im having the same issue without sorting, i receive different responses. As you can se on the examples bellow, every array is a different request result, and they are all different:

[
  { _id: 5fa04d3480595823a06288cf },
  { _id: 5fc27783e415b54f64729eca },
  { _id: 5fa04dfb80595823a06288d1 },
  { _id: 5fc277bce415b54f64729ecb },
  { _id: 5fc27834e415b54f64729ecd }
]
[
  { _id: 5fc277f5e415b54f64729ecc },
  { _id: 5fc277bce415b54f64729ecb },
  { _id: 5fc27834e415b54f64729ecd },
  { _id: 5fa04dfb80595823a06288d1 },
  { _id: 5fc27867e415b54f64729ece }
]
[
  { _id: 5fc277bce415b54f64729ecb },
  { _id: 5fa04dfb80595823a06288d1 },
  { _id: 5fc27783e415b54f64729eca },
  { _id: 5fc27834e415b54f64729ecd },
  { _id: 5fa04d3480595823a06288cf }
]

This is the code of my method:

var myAggregate = UserProducts.aggregate([{ $group: { _id: "$user" } }]);

        const options = {
            page: page,
            limit: itemsPerPage
        };

        //Pagina lo anterior
        UserProducts.aggregatePaginate(myAggregate, options).then(async function (results) {

            console.log(results.docs);
            //retorna los productos por usuario y su usuario
            for (let i = 0; i < results.docs.length; i++) {
                await getUser(results.docs[i]).then((value) => {
                    userProductsArray[i] = value;

                    //console.log(userProductsArray);
                });

            }

            return res.status(200).send({
                userProductsArray,
                total: results.totalPages
            });
KristinaHus commented 3 years ago

Hi, is there any solution to this or a workaround? Cuz I'm having the same problem.

My aggregation with the $sort stage works just fine, but when I use aggregatePaginate I'm getting a different result.

I've tried to add a sort to options, still no luck.

doanthan commented 3 years ago

Hello, is there a fix for this problem yet? I tried putting the _id in the sort object but it doesn't work- it only sorts by _id when i do that.

trancephorm commented 2 years ago

I see this bug may not be fixed yet, so I assume it's too much to ask if it's possible to sort aggregation result's by full-text search textScore metadata?

jhaaken commented 2 years ago

So I am on v1.0.42 and noticed a sort issue. what I found was the fields I am sorting on need to be in the result set ($project) or it won't sort the records properly