Automattic / mongoose

MongoDB object modeling designed to work in an asynchronous environment.
https://mongoosejs.com
MIT License
26.88k stars 3.83k forks source link

Validation of deeply nested documents broken as of v6.2.8 #11610

Closed chaeron closed 2 years ago

chaeron commented 2 years ago

We have some very complicated documents, that can nest very deeply.

With v6.2.7 validation of correctly formatted documents worked fine. This broke as of v6.2.8 and is still broken in v6.2.9 which is causing validation exceptions to the thrown for documents that are perfectly fine.

Samples.zip

I have attached a zip file that has a very deeply nested lookup table and the mongoose definition code.

This document validates find in v6.2.7. In v6.2.8/9 if fails with an incorrect validation error:

Error: Lookup validation failed: sub_tables.1.sub_tables.0.updated_by: Path updated_by is required., sub_tables.1.sub_tables.0.updated_at: Path updated_at is required., sub_tables.1.sub_tables.0.created_by: Path created_by is required., sub_tables.1.sub_tables.0.created_at: Path created_at is required., sub_tables.1.sub_tables.0.lookup_id: Path lookup_id is required.

Even though lookup_id, created_at, created_by, updated_at and updated_by fields are present in the referenced path.

IARKI commented 2 years ago

Same here, deep populate doesn't work for me starting with version 6.2.9

IslandRhythms commented 2 years ago
const mongoose = require('mongoose');

const values = new mongoose.Schema({

}, {_id:false, strict: false })

const sub_table_attributes = new mongoose.Schema({
    lookup_id:                           { type: String, required: true },
    version:                             { type: String, match: /^\d+\.\d+\.\d+$/ },
    description:                         { type: String, maxlength: 1000 },
    restricted_permissions:              { type: Boolean },
    created_at:                          { type: Date, required: true },
    created_by:                          { type: String, required: true },
    updated_at:                          { type: Date, required: true },
    updated_by:                          { type: String, required: true }
  },
  {  
      _id: false,
  });

var table_entry = new mongoose.Schema({
    code:                                { type: {} },
    value:                               { type: String, required: true, maxlength: 100 },
    values:                              values,
    effective_date:                      { type: Date },
    expired_date:                        { type: Date },
    sub_tables:                          { type: {} },
    created_at:                          { type: Date, required: true },
    created_by:                          { type: String, required: true },
    updated_at:                          { type: Date, required: true },
    updated_by:                          { type: String, required: true }
  },
  {  
      _id: false,
      strict: false,
  });

const lookup = new mongoose.Schema({
    _id:                                 { type: mongoose.Schema.Types.ObjectId, required: true },
    lookup_id:                           { type: String, required: true },
    version:                             { type: String, match: /^\d+\.\d+\.\d+$/ },
    description:                         { type: String, maxlength: 1000 },
    restricted_permissions:              { type: Boolean },
    sub_tables:                          { type: [ { type: sub_table_attributes } ], default: void 0 },
    entries:                             [ table_entry ],
    created_at:                          { type: Date, required: true },
    created_by:                          { type: String, required: true },
    updated_at:                          { type: Date, required: true },
    updated_by:                          { type: String, required: true }
  },
  {  
  });

const Test = mongoose.model('Test', lookup);

async function run() {
    await mongoose.connect('mongodb://localhost:27017');
    await mongoose.connection.dropDatabase();

    await Test.create({
        _id: mongoose.Types.ObjectId(),
        lookup_id: '1',
        verison: '1.0.2',
        description: 'description',
        restricted_permissions: true,
        sub_tables: [{
            lookup_id: '1',
            verison: '1.0.2',
            description: 'description',
            restricted_permissions: true,
            created_at: new Date(),
            created_by: 'Test',
            updated_at: new Date(),
            updated_by: 'Test'
        }],
        entries: [{
            "code": "A",
            "value": "Accept",
            "values": {
              "en": "Accept",
              "fr": "J'accepte",
              "es": "Aceptar"
            },
            "effective_date": "2022-04-01T15:57:34.667Z",
            "expired_date": null,
            "created_at": "2022-04-01T15:57:34.667Z",
            "created_by": "shore migration",
            "updated_at": "2022-04-01T15:57:34.667Z",
            "updated_by": "shore migration",
            "sub_tables": {
                "offer_reason_categories": [
                  {
                    "code": 1,
                    "value": "Accepting pending further diagnostic imaging",
                    "sub_tables": {
                      "offer_reasons": [
                        {
                          "code": 1,
                          "value": "Echo",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 2,
                          "value": "Repeat Echo",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 3,
                          "value": "Angio",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 4,
                          "value": "CT Scan",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 5,
                          "value": "Ultrasound",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 6,
                          "value": "Chest xray",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 7,
                          "value": "Repeat Chest xray",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 8,
                          "value": "Bronchoscopy",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 9,
                          "value": "Other",
                          "other_selected": true,
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        }
                      ]
                    },
                    "created_at": "2022-04-01T15:57:34.667Z",
                    "created_by": "shore migration",
                    "updated_at": "2022-04-01T15:57:34.667Z",
                    "updated_by": "shore migration"
                  },
                  {
                    "code": 2,
                    "value": "Accept pending additional donor information",
                    "sub_tables": {
                      "offer_reasons": [
                        {
                          "code": 1,
                          "value": "Repeat Labs",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 2,
                          "value": "Albumin Creatinine ratio",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 3,
                          "value": "Urinalysis",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 4,
                          "value": "Historic lab values (i.e Cr)",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 5,
                          "value": "Repeat ABGs",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 6,
                          "value": "Serology and/or NAT testing",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 7,
                          "value": "HLA (including stat crossmatch)",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 8,
                          "value": "Donor history details (i.e. cancer)",
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        },
                        {
                          "code": 9,
                          "value": "Other",
                          "other_selected": true,
                          "created_at": "2022-04-01T15:57:34.667Z",
                          "created_by": "shore migration",
                          "updated_at": "2022-04-01T15:57:34.667Z",
                          "updated_by": "shore migration"
                        }
                      ]
                    },
                    "created_at": "2022-04-01T15:57:34.667Z",
                    "created_by": "shore migration",
                    "updated_at": "2022-04-01T15:57:34.667Z",
                    "updated_by": "shore migration"
                  }
                ]
              }
          }],
        created_at: new Date(),
        created_by: 'Test',
        updated_at: new Date(),
        updated_by: 'Test'

    })

    console.log('done');
}

run();
IslandRhythms commented 2 years ago

Just looking at the json file you gave, on line 9835 you insert a sub_tables array and put a nested sub_tables array that is not defined in the schema. It would be odd if that was creating the problem but It's the only thing that doesn't look right in what you gave. Try adding strict: false in the sub_tables schema definition and see if that does anything.

chaeron commented 2 years ago

Why was this closed? AFAIK the bug still exists and hasn't been fixed.

Uzlopak commented 2 years ago

It was tagged as "can't reproduce". So the bug could not be reproduced. It was open for about a month without a reaction regarding the question of IslandRhythms.

If you provide a mvce, we could actually investigate the issue.