Closed nguyentiennghiep closed 5 years ago
juanjoDiaz
commented
6 years ago Hi @nguyentiennghiep ,
Quick googling tells me that the error is produced by lodash (https://github.com/Automattic/mongoose/issues/6507).
Unfortunately, Lodash is no maintaining it's individuals package up to date. I created an issue in their repo for that. If they don't respond, I'll remove our lodash dependency to resolve this.
Could you provide some data that I can put in Mongo to test your code?
nguyentiennghiep
commented
6 years ago @juanjoDiaz Thanks for your help . this is one document in my book collection
{"_id":"5bd93e9a6353e1110ca67532","bookTitle":{"extends":{"mddc":{"r18Rating":3,"dataId":"mddcdataid"},"ccc":{"genreCode":{"1":"TSUTAYAGENRECODE1","2":"TSUTAYAGENRECODE2","3":"TSUTAYAGENRECODE3"},"productUrl":{"stockSearchUrl":"http://www.storesearchurl.com","relatedbookSearchUrl":"http://www.storesearchurl.com"},"publisherId":"TSUTAYAPUBLISHERID"}},"description":"TESTDISCRIPTION","targetRoles":["tfree-user","tmagazine-user"],"category":"female-lifestyle","title":"AnAnAA","kana":"アンアン","imagePath":"/mock-images/cover-1@2x.png","publishDate":"2013-12-31T15:00:00.000Z","updateDate":"2013-12-31T15:00:00.000Z"},"author":{"extends":{"mddc":{"authorId":"Author-46623"}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.094Z","isActive":true,"_id":"5bd93e9a6353e1110ca6751b","name":"鳥山明","kana":"トリヤマアキラ","imagePath":"http://www.hoge.com"},"publisher":{"extends":{"mddc":{"publisherId":"Publisher-41207","reportName":"集英社"}},"createAt":"2018-10-31T05:32:47.094Z","updateAt":"2018-10-31T05:32:47.094Z","isActive":true,"_id":"5bd93e9a6353e1110ca6751c","name":"集英社","kana":"シュウエイシャ"},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"sourceType":"test","syncedAt":"2018-08-29T16:00:00.000Z","books":[{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":0,"downloadLimit":1000,"fileSize":10000,"master":0,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160901}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2010-12-18T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tpremium-user","tmagazine-user"],"lastestTargetRoles":["guest"],"title":"1月号","kana":"アンアンゴガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67537"},{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":1,"downloadLimit":1000,"fileSize":10000,"master":0,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160901}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2018-06-02T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tpremium-user","tmagazine-user"],"lastestTargetRoles":[],"title":"2月号","kana":"アンアンゴガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67538"},{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":0,"downloadLimit":1000,"fileSize":10000,"master":1,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160910}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2011-08-31T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tfree-user","tmagazine-user"],"lastestTargetRoles":["tfree-user"],"title":"1月号","kana":"アンアンシチガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67535"},{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":1,"downloadLimit":1000,"fileSize":10000,"master":1,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160911}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2014-11-26T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tmagazine-user"],"lastestTargetRoles":[],"title":"2月号","kana":"アンアンシチガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67534"},{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":2,"downloadLimit":1000,"fileSize":10000,"master":1,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160911}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2019-06-21T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tmagazine-user"],"lastestTargetRoles":["tmagazine-user"],"title":"3月号","kana":"アンアンシチガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67533"},{"extends":{"mddc":{"price":100,"version":"","jdcn":"JDCN","iban":"IBAN","publisherContentCode":"PUBLISHERCONTENTSCODE","rowIndex":3,"downloadLimit":1000,"fileSize":10000,"master":1,"permitId":"PERMITID","format":"formatstring","permitStart":"2014/8/1","permitEnd":"2015/8/1","jan":"JAN","bookIndex":20160901}},"createAt":"2018-10-31T05:32:47.092Z","updateAt":"2018-10-31T05:32:47.092Z","isActive":true,"deliveryDate":"2019-11-30T15:00:00.000Z","expirationDate":"2019-12-29T15:00:00.000Z","description":"ほげほげほげほげおｈｇへおｇほえｇほえｈｇ","targetRoles":["tpremium-user","tmagazine-user"],"lastestTargetRoles":["tpremium-user"],"title":"4月号","kana":"アンアンロクガツゴウ","imagePath":"/mock-images/cover-2@2x.png","_id":"5bd93e9a6353e1110ca67536"}],"__v":0}This is the full schema
import * as mongoose from 'mongoose';
const Schema = mongoose.Schema;
const ObjectId = mongoose.Schema.Types.ObjectId;
const BookCollection = new Schema({
_id: { type: ObjectId, required: true, default: () => new mongoose.Types.ObjectId() },
sourceType: { type: String, required: true },
createAt: { type: Date, required: true, default: new Date() },
updateAt: { type: Date, required: true, default: new Date() },
syncedAt: { type: Date, required: true },
isActive: { type: Boolean, required: true, default: false },
bookTitle: {
title: { type: String, required: true },
kana: { type: String, required: true },
description: { type: String, required: false, default: '' },
imagePath: { type: String, required: true },
targetRoles: { type: [String], required: true, default: [] },
category: { type: String, required: false, default: '' },
publishDate: { type: Date, required: true },
updateDate: { type: Date, required: true },
extends: {
mddc: {
dataId: { type: String, required: true },
r18Rating: { type: Number, required: true, default: 0 },
},
ccc: {
genreCode: {
1: { type: String, required: false, default: '' },
2: { type: String, required: false, default: '' },
3: { type: String, required: false, default: '' },
},
productUrl: {
stockSearchUrl: { type: String, required: false, default: '' },
relatedbookSearchUrl: { type: String, required: false, default: '' },
},
publisherId: { type: String, required: false, default: '' }
}
},
},
books: [
{
_id: { type: ObjectId, required: true, default: () => new mongoose.Types.ObjectId() },
createAt: { type: Date, required: true, default: new Date() },
updateAt: { type: Date, required: true, default: new Date() },
isActive: { type: Boolean, required: true, default: false },
title: { type: String, required: true },
kana: { type: String, required: true },
deliveryDate: { type: Date, required: true, default: new Date() },
expirationDate: { type: Date, required: true, default: new Date() },
description: { type: String, required: false, default: '' },
imagePath: { type: String, required: true },
targetRoles: { type: [String], required: true },
lastestTargetRoles: { type: [String], required: true, default: [] },
extends: {
mddc: {
price: { type: Number, required: true, default: 0 },
downloadLimit: { type: Number, required: true },
fileSize: { type: Number, required: true },
master: { type: Number, required: true },
permitId: { type: String, required: true },
version: { type: String, default: '' },
format: { type: String, required: true },
permitStart: { type: String, required: true },
permitEnd: { type: String, required: true },
jan: { type: String, required: true },
jdcn: { type: String, required: false, default: '' },
iban: { type: String, required: false, default: '' },
publisherContentCode: { type: String, required: false, default: '' },
bookIndex: { type: Number, required: true },
rowIndex: { type: Number, default: 0 },
}
}
}
],
author: {
_id: ObjectId,
createAt: { type: Date, default: new Date(), required: true },
updateAt: { type: Date, default: new Date() },
isActive: { type: Boolean, default: true, required: true },
name: { type: String, index: true, required: true },
kana: { type: String, index: true, required: true },
imagePath: { type: String },
extends: {
mddc: {
authorId: { type: String, index: true, required: true, unique: true }
}
}
},
publisher: {
_id: ObjectId,
createAt: { type: Date, default: new Date(), required: true },
updateAt: { type: Date, default: new Date() },
isActive: { type: Boolean, default: true, required: true },
name: { type: String, index: true, required: true },
kana: { type: String, index: true, required: true },
extends: {
mddc: {
publisherId: { type: String, index: true, required: true, unique: true },
reportName: { type: String }
}
}
}
});
export default mongoose.model('books', BookCollection);
Full my export code
import BookCollection from '../models/book-collection';
import * as fs from 'fs';
import * as moment from 'moment';
import * as path from 'path';
import jsonTocsv from 'json2csv';
const Json2csvParser = jsonTocsv.Parser;
const fields = [
'_id',
'sourceType',
'createAt',
'updateAt',
'syncedAt',
'isActive',
// 'bookTitle.title', 'bookTitle.kana', 'bookTitle.description',
// 'bookTitle.imagePath', 'bookTitle.targetRoles', 'bookTitle.category',
// 'bookTitle.publishDate', 'bookTitle.updateDate',
// 'bookTitle.extends.mddc.dataId', 'bookTitle.extends.mddc.r18Rating',
// 'bookTitle.extends.ccc.genreCode.1', 'bookTitle.extends.ccc.genreCode.2', 'bookTitle.extends.ccc.genreCode.3',
// 'bookTitle.extends.ccc.productUrl.stockSearchUrl', 'bookTitle.extends.ccc.productUrl.relatedbookSearchUrl',
// 'bookTitle.extends.ccc.publisherId',
'books._id', 'books.createAt', 'books.updateAt', 'books.isActive', 'book.title',
'books.kana', 'books.deliveryDate', 'books.expirationDate', 'books.description',
'books.imagePath', 'books.targetRoles', 'books.lastestTargetRoles',
'books.extends.mddc.price', 'books.extends.mddc.downloadLimit',
'books.extends.mddc.fileSize', 'books.extends.mddc.master', 'books.extends.mddc.permitId',
'books.extends.mddc.version', 'books.extends.mddc.format',
'books.extends.mddc.permitStart', 'books.extends.mddc.permitEnd',
'books.extends.mddc.jan', 'books.extends.mddc.jdcn', 'books.extends.mddc.iban',
'books.extends.mddc.publisherContentCode', 'books.extends.mddc.bookIndex', 'books.extends.mddc.rowIndex',
// 'author._id', 'author.createAt', 'author.updateAt', 'author.isActive', 'author.name', 'author.', 'author.kana',
// 'author.imagePath',
// 'author.extends.mddc.authorId',
// 'publisher._id', 'publisher.createAt', 'publisher.updateAt', 'publisher.isActive',
// 'publisher.name', 'publisher.kana',
// 'publisher.extends.mddc.publisherId', 'publisher.extends.mddc.reportName'
];
export default async function bookCollectionExport() {
try {
const data = await BookCollection.find({ _id: '5bd93e9a6353e1110ca67532' }).exec();
let csv;
const json2csvParser = new Json2csvParser({ fields, unwind: 'books' });
try {
csv = json2csvParser.parse(data);
} catch (e) {
console.log(e);
}
const dateTime = moment().format('YYYYMMDDhhmmss');
const filePath = path.join(__dirname, '..', '..', 'export', 'csv-books-' + dateTime + '.csv');
fs.writeFile(filePath, csv, (err) => {
if (err) {
console.log(err);
}
});
} catch (err) {
console.log(err);
}
}@juanjoDiaz sr for long question . but when i change unwind:'bookTitle.targetRoles' to unwind:'books' to get books unwind data , dont get bookTitle.targetRoles anymore . the error is gone but output csv data look like this
i dont set unwind_blank options
nguyentiennghiep
commented
6 years ago @juanjoDiaz i don't know why but when i change code from csv = json2csvParser.parse(data); to csv = json2csvParser.parse(JSON.parse(JSON.stringify(data))); it work :D
juanjoDiaz
commented
6 years ago Yes.
The problem is that lodash.deepClone try to clone Mongo's internal classes wrongly.
JSON.parse(JSON.stringify(...)) removes custom classes and normalize everything to plain JSON so it solves the problem.
lodash maintainer had close my issue without resolving it so, for now, you have to use that workaround. In the future, I might get rid of lodash completely to avoid this kind of issue.
I'll keep this open in case that someone else has this issue!
Hi guys , i have a collection from mongDB like that , when i add options 'unwind ' always fails .. error in the image below . Can u guys help me ?thank you very much.
my export func
I recevice this error