Mongo4j creates duplicates every time I save my project (nodemon)

[niktverd]

What am I doing wrong? I use mongoose and add mongo4j as a plugin. There are 2 instances: carMake:

const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const mongoosastic = require("mongoosastic");
const mongo4j = require("./plugins/mongo4j");

mongoose.set("debug", false);

const CarMakeSchema = Schema(
    {
        name: {
            type: String,
            required: true,
            es_indexed: true,
            neo_prop: true,
        },
        syns: {
            type: Array,
            es_indexed: true,
            es_boost: 1.0,
            es_type: "string",
        },
    },
    { collection: "carMake" }
);

CarMakeSchema.plugin(mongoosastic, {
    index: "makes",
    hosts: ["localhost:9200"],
    hydrate: true,
    hydrateOptions: { lean: true },
});
CarMakeSchema.plugin(mongo4j.plugin());

module.exports = mongoose.model("carMake", CarMakeSchema);

// _id  name    partNumber  CarMakeSections catalogs    analogsGroup

and carModel:

const mongoose = require("mongoose");
const Schema = mongoose.Schema;
const mongoosastic = require("mongoosastic");
const mongo4j = require("./plugins/mongo4j");
// const mongo4j = require("mongo4j");

mongoose.set("debug", false);
// mongo4j.init("neo4j://localhost", { user: "neo4j", pass: "123" });

const CarModelSchema = Schema(
    {
        name: {
            type: String,
            required: true,
            es_indexed: true,
            neo_prop: true,
        },
        syns: {
            type: Array,
            es_indexed: true,
            es_boost: 1.0,
            es_type: "string",
        },
        makes: {
            makeId: {
                type: Schema.Types.ObjectId,
                ref: "carMake",
                es_indexed: true,
                neo_rel_name: "MK_MD",
            },
            score: Number,
        },
        // makeId: {
        //     type: String,
        //     es_indexed: true,
        // },
    },
    { collection: "carModel" }
);

CarModelSchema.plugin(mongoosastic, {
    index: "models",
    hosts: ["localhost:9200"],
    hydrate: true,
    hydrateOptions: { lean: true },
});
CarModelSchema.plugin(mongo4j.plugin());

module.exports = mongoose.model("carModel", CarModelSchema);

// _id  name    partNumber  CarModelSections    catalogs    analogsGroup

And every time I save the project, my server restarts and I get new nodes and relationships to my Neo4j database.

"dependencies": {
        "axios": "^0.20.0",
        "axios-https-proxy-fix": "^0.17.1",
        "body-parser": "^1.19.0",
        "cheerio": "^1.0.0-rc.3",
        "cors": "^2.8.5",
        "csvtojson": "^2.0.10",
        "elasticsearch": "^16.7.1",
        "express": "^4.17.1",
        "fetch": "^1.1.0",
        "http-proxy-agent": "^4.0.1",
        "https-proxy-agent": "^5.0.0",
        "mongo4j": "^3.0.3",
        "mongoosastic": "^4.6.0",
        "mongoose": "^5.10.7",
        "node-fetch": "^2.6.1",
        "node-fetch-with-proxy": "^0.1.2",
        "nodemon": "^2.0.4",
        "puppeteer": "^5.3.1",
        "request": "^2.88.2",
        "request-promise-native": "^1.0.9",
        "selenium-webdriver": "^4.0.0-alpha.7",
        "socks-proxy-agent": "^5.0.0"
    }

What should I change to avoid creating nodes that are already in the database?

Or it is a feature with which I must live?)

video how it works

thanks)

[SvenWesterlaken]

First of all, thanks for the video, that helps a lot for understanding the problem.

From the top of my head, I think I don't have a check for unique ID's so I guess your use case triggers the save hook every time on startup which causes the node to be created every time. I can have a look for a feature to implement that it looks for unique items and doesn't create them if they already exist. Unfortunately, I don't have that much time on my hand at the moment so I think the fix will be after or during this weekend.

For now, a temporary workaround could be to use a cypher query (static function) to check if the nodes already exist and/or delete them manually after creation.

Lastly, to be sure, could you provide me with your ./plugins/mongo4j file so I can rule that out from causing the issue. (you can leave out the credentials of course.)

[niktverd]

Thank you for your quick response.

the file you are asking for is quiet stupid

const mongo4j = require('mongo4j');
console.log( "mongo4j is executiong...." )
mongo4j.init('neo4j://localhost', {user: 'neo4j', pass: '123'});
module.exports = mongo4j;

[SvenWesterlaken]

Makes sense. Just wanted to be sure :). Rewatched the video and I indeed think checking for ID's for creating new nodes would fix the issue. Will make this into a new feature as soon as possible (like I said, probably around the end of this weekend) and you'll be notified when I have a pull request ready that fixes this issue.

[niktverd]

So, I look into your files and have tried to update a file 'save.js'

one(schema, doc, next, driver) {
    const neo_doc = node.convertToNeoModel(schema, doc);

    let properties = ``;
    let counter = 0;
    for( let i in neo_doc.props ){
      if( counter !== 0 ) properties +=", ";
      counter ++;
      properties += `doc.${i} = ${helper.valueToPlainString(neo_doc.props[i])}`
    }

    console.log( "one(schema, doc, next, driver) {", helper.getLabel(doc), helper.toPlainString(neo_doc.props.name), properties);
    let neo_query = 
    `MERGE (doc:${helper.getLabel(doc)} { m_id: '${neo_doc.props.m_id}' })     
    ON CREATE SET ${properties}
    ON MATCH SET ${properties} `;

    // let neo_query = `CREATE (doc:${helper.getLabel(doc)} ${helper.toPlainString(neo_doc.props)}) `;
    neo_query += this.writeSubDocumentsToCypher(neo_doc.subdocs, '');
    neo_query += this.writeRelationshipsToCypher(neo_doc.rels, '');

    const session = driver.session();
    session.run(neo_query).catch((err) => {
      helper.endMiddleware(next, session, err);
    }).then(() => helper.endMiddleware(next, session));
  },

now it creates only one entity of each node and add relations to each on every save))) The solution is ugly, but I hope it will help you to dive into the right one quicker

and I am not sure about subDocs and how it will work

[SvenWesterlaken]

That will for sure help to get solving the problem quicker. Might reserve some time this evening to try and fix this!

You can create a pull request so I can see the diffs a bit better, but don't feel obliged to do so. I will then use that as a base.

[niktverd]

Dear SvenWesterlaken!

I am not sure I can pull a request because I am absolutely new to github. Sorry for that

If it is so important i can dive into the question and learn how to do that.

And while.... I have changed relationship saver. The massive change I created is relationships will not be created if only properties are different. If only properties are different in relationships but label, direction and nodes are the same, props will be updated and that is all

code bellow

 writeRelationshipsToCypher(relationships, doc_identifier) {
    console.log(  "writeRelationshipsToCypher(relationships, doc_identifier) {", relationships);
    let neo_query = '';
    _.forEach(relationships, (rel, index) => {
        const identifier = helper.romanize(index+1);
        const properties = rel.rel_props ? ` ${helper.toPlainString(rel.rel_props)}` : '';

        let properties2 = ``;
        let counter = 0;
        for( let i in rel.rel_props ){
          if( counter !== 0 ) properties2 +=", ";
          counter ++;
          properties2 += `r.${i} = ${helper.valueToPlainString(rel.rel_props[i])}`
        }

        neo_query += `WITH doc${doc_identifier} `;
        neo_query += `MATCH (${identifier}:${rel.rel_label}) WHERE ${identifier}.m_id = '${rel.m_id}' `;
        neo_query += `MERGE (doc${doc_identifier})-[r:${rel.rel_name}]->(${identifier}) `;
        neo_query += ` ON CREATE SET  ${properties2}`;
        neo_query += ` ON MATCH SET  ${properties2}`;
        // neo_query += `ON MATCH (doc${doc_identifier})-[:${rel.rel_name}${properties}]->(${identifier}) `;
    });

    return neo_query;
  }

[SvenWesterlaken]

Ah, don't worry in that case.

Unfortunately, as I said, I don't have time until this evening. If you think that might be too long, you can always try and learn to create a pull-request in the meantime to speed things up. (It never hurts to learn these features in any case, as they are applicable to services like Gitlab or Bitbucket as well).

[SvenWesterlaken]

Looks like Travis is not responsive on updating the build. Will try to switch to GitHub actions in order for the package to be published to npm.

[SvenWesterlaken]

:tada: This issue has been resolved in version 3.1.0 :tada:

The release is available on:

• npm package (@latest dist-tag)
• GitHub release

Your semantic-release bot :package::rocket: