agmen-hu / node-datapumps

Node.js ETL (Extract, Transform, Load) toolkit for easy data import, export or transfer between systems.
MIT License
291 stars 38 forks source link

Read from 2 collections and write #41

Closed rafaneri closed 7 years ago

rafaneri commented 7 years ago

Hello,

Can I read data from 2 collections and write in an another?

var
    clientPump = new Pump(),
    previousDiagnosisPump = new Pump(),
    oncologyOutputPump = new Pump();

clientPump
    .mixin(MongodbMixin(mongoInputURL))
    .useCollection('client')
    .from(clientPump.find()); // ??? is this way?

previousDiagnosisPump
    .mixin(MongodbMixin(mongoInputURL))
    .useCollection('previousDiagnosis')
    .from(previousDiagnosisPump.find()); // ??? is this way?

oncologyOutputPump
    .mixin(MongodbMixin(mongoOutputURL))
    .useCollection('oncology')
    .from(...) // HOW TO USE TWO COLLECTIONS
    .process(function (client, previousDiagnosis) { // ???
        return oncologyOutputPump.insert(...);
    })
    .logErrorsToConsole();

Promise.all([
    clientPump.run(),
    previousDiagnosisPump.run(),
    oncologyOutputPump.run()
])
    .then(function () {
        console.log("Done");
    });

Is possible use a pump to did a findOne inner a process method?

oncologyOutputPump
    .mixin(MongodbMixin(mongoOutputURL))
    .useCollection('oncology')
    .from(oncologyPump.buffer())
    .process(function (data) {
        var client = clientPump.findOne({_id:data.client}); // is possible??
        return oncologyOutputPump.insert(data);
    })
    .logErrorsToConsole();
novaki commented 7 years ago

Hello @rafaneri

Use the MergeMixin for that: http://agmen-hu.github.io/node-datapumps/docs/mixin/MergeMixin.html

rafaneri commented 7 years ago

Hi @novaki,

I'm sorry. I've tried, but doesn't work yet. Can you help me please?

var
    coll1 = new Pump(),
    coll2 = new Pump(),
    destination = new Pump();

coll1 
    .mixin(MongodbMixin(mongoInputURL))
    .useCollection('coll1')
    .from(coll1.find());

coll2 
    .mixin(MongodbMixin(mongoInputURL))
    .useCollection('coll2')
    .from(coll2.find());

var merge = new Pump();
merge
    .mixin(MergeMixin)
    .from(coll1)
    .from(coll2);

destination  = new Pump();
destination
    .mixin(MongodbMixin(mongoOutputURL))
    .useCollection('coll3')
    .from(merge)
    .process(function (data) {
        return destination.insert(data);
    })
    .logErrorsToConsole()
    .run()
    .then(function () {
        console.log("Done");
    });
rafaneri commented 7 years ago

Hello @novaki,

I forgot to run the pumps!