How can i combine multiple collection into one collection using with $lookup mongodb or nodejs mongodb?

i have below collections with document count division - 30,

category - 248

productgroup - 1402

product - 60000

for example i given below data division

{

    "divisionid": "1",
    "divisioncode": "0",
    "divisionname": "ELECTRONICS/HOME APPLIANCE",
    "divisionpoint": "2"
},
{

    "divisionid": "2",
    "divisioncode": "1",
    "divisionname": "FOODS",
    "divisionpoint": "8"
}

category collection data's

{

    "categoryid": "1",
    "divisionid": "1",
    "categorycode": "34",
    "categoryname": "AUDIO SYSTEM",
    "categorypoint": "Null"
},
{

    "categoryid": "2",
    "divisionid": "1",
    "categorycode": "348",
    "categoryname": "DVD/VCD",
    "categorypoint": "8"
}

productgroup collection data's

{

    "productgroupid": "1",
    "divisionid": "1",
    "categoryid": "1",
    "productgroupname": "ADAPTOR",
    "productgroupcode": "6765",
    "productgrouppoint": "7"
},
{

    "productgroupid": "2",
    "divisionid": "1",
    "categoryid": "2",
    "productgroupname": "WALKMAN",
    "productgroupcode": "7659",
    "productgrouppoint": "Null"
}

product collection data's

{

    "productid": "1",
    "divisionid": "1",
    "categoryid": "1",
    "productgroupid":"1",
    "productname":"UNIVERSAL AC DC ADAPTER-PCS",
    "productcode": "1000054",
    "productpoint": "1"
},
{

    "productid": "2",
    "divisionid": "1",
    "categoryid": "2",
    "productgroupid":"2",
    "productname":"WALKMAN WM#M470-PCS",
    "productcode": "1000089",
    "productpoint": "2"
}

i have 4 collections like

division,

category,

productgroup,

product

these all are my data's which is stored in collection my division collecion data's

{

"divisionid": "1",
"divisioncode": "0",
"divisionname": "ELECTRONICS/HOME APPLIANCE",
"divisionpoint": "2"

}, {

"divisionid": "2",
"divisioncode": "1",
"divisionname": "FOODS",
"divisionpoint": "8"

} category collection data's

{

"categoryid": "1",
"divisionid": "1",
"categorycode": "34",
"categoryname": "AUDIO SYSTEM",
"categorypoint": "Null"

}, {

"categoryid": "2",
"divisionid": "1",
"categorycode": "348",
"categoryname": "DVD/VCD",
"categorypoint": "8"

} productgroup collection data's

{

"productgroupid": "1",
"divisionid": "1",
"categoryid": "1",
"productgroupname": "ADAPTOR",
"productgroupcode": "6765",
"productgrouppoint": "7"

}, {

"productgroupid": "2",
"divisionid": "1",
"categoryid": "2",
"productgroupname": "WALKMAN",
"productgroupcode": "7659",
"productgrouppoint": "Null"

} product collection data's

{

"productid": "1",
"divisionid": "1",
"categoryid": "1",
"productgroupid":"1",
"productname":"UNIVERSAL AC DC ADAPTER-PCS",
"productcode": "1000054",
"productpoint": "1"

}, {

"productid": "2",
"divisionid": "1",
"categoryid": "2",
"productgroupid":"2",
"productname":"WALKMAN WM#M470-PCS",
"productcode": "1000089",
"productpoint": "2"

} i want to combine these 4 collection into one collection. i should get summary like this productsummary

{


        "productpoint": "1",
        "productname": "UNIVERSAL AC DC ADAPTER-PCS",
        "productcode": "10000054",
        "productid"  :"1"

    "group": {
        "point": "7",
        "name": "ADAPTOR",
        "id"  :"1"
    },
    "category": {
        "point": "0",
        "name": "AUDIO SYSTEM",
        "id"  :"1"
    },
    "division": {
        "point": "2",
        "name": "ELECTRONICS/HOME APPLIANCE",
        "id"  :"1"
    }
},
{


        "productpoint": "2",
        "productname": "WALKMAN WM#M470-PCS",
        "productcode": "1000089",
        "productid"  :"2"

    "group": {
        "point": "7",
        "name": "WALKMAN",
        "id"  :"Null"
    },
    "category": {
        "point": "8",
        "name": "DVD/VCD",
        "id"  :"2"
    },
    "division": {
        "point": "2",
        "name": "ELECTRONICS/HOME APPLIANCE",
        "id"  :"1"
    }
}

i run this code

db.products.aggregate([ 
    {$lookup:{from:"productgroups", localField:"productgroupid", foreignField:"productgroupid", as:"group"}},
    {$lookup:{from:"category", localField:"categoryid", foreignField:"categoryid", as:"category"}},
    {$lookup:{from:"divisions", localField:"divisionid", foreignField:"divisionid", as:"division"}},
    {$project: {productpoint:1,productname:1,productcode:1,productid:1,group: { $arrayElemAt: [ "$group", 0 ]},
                category: { $arrayElemAt: [ "$category", 0 ]}, division: { $arrayElemAt: [ "$division", 0 ]} }} 
  ]);

its working but i had 60 k products that time its not working and getting error like this

and data get stored like this in productpointallocations

{

    "productpoint": "2",
        "productname": "WALKMAN WM#M470-PCS",
        "productcode": "1000089",
        "productid"  :"2"
}

is there any other way to do this and give some other solutions for this

is there any other option in nodejs mongodb?

You should use $skip and $limit to avoid this error. why getting this error ? you have to read @Alex comment link.

db.products.aggregate([ 
    {$skip:0},{$limit: 1000},
    {$lookup:{from:"productgroups", localField:"productgroupid", foreignField:"productgroupid", as:"group"}},
    {$lookup:{from:"category", localField:"categoryid", foreignField:"categoryid", as:"category"}},
    {$lookup:{from:"divisions", localField:"divisionid", foreignField:"divisionid", as:"division"}},
    {$project: {productpoint:1,productname:1,productcode:1,productid:1,group: { $arrayElemAt: [ "$group", 0 ]},
                category: { $arrayElemAt: [ "$category", 0 ]}, division: { $arrayElemAt: [ "$division", 0 ]} }} 
  ]);

{$skip:0},{$limit: 1000}// for initial and then for next time should skip: 1000, limit:1000 and so on. you may get it from request or by using loop or as you implement.

来源：https://stackoverflow.com/questions/43542270/how-can-i-combine-multiple-collection-into-one-collection-using-with-lookup-mon

标签

node.js

mongodb

express

lookup