createNode making only 20 nodes (totalcount in grapql) out of 2480 from the object array data

[cymkd]

Summary

Second day I can't seem to fix this issue at all. Graphql spits only 20 nodes, while it is supposed to be 2480 which are created during sourceNode / createNode from an array of objects.

on SourceNodes step:

1. pulling data from API, 2 objects arrays: Products and Prices
2. Creating Nodes.

Console logging shows 2480 objects in the arrays. I couldn't find any callbacks for successful createNode, so I just console logged for-of with count. Fires 2480 times, yet in grapql the totalcount is 20, for both objects (Products and Prices)

Relevant information

This is the code that ought to do the job:

source-plugin:

const axios = require('axios');

exports.onPreInit = () => console.log("Loaded gatsby-starter-plugin")

exports.sourceNodes = async ({ actions, createNodeId, createContentDigest }) => {
    const {
        createNode,
        createTypes,
        schema } = actions

    //
    // Fetching access token
    //
    const token = await axios({
        method: 'POST',
        url: `${process.env.COMMERCE_LAYER_API}/oauth/token`,
        headers: {
            'Content-Type': 'application/json',
        },
        data: {
            grant_type: "client_credentials",
            client_id: `${process.env.CLIENTID}`,
            client_secret: null,
            scope: `${process.env.MARKET}`
        }
    })

    //
    // Fetching and paginating
    //
    const fetchCL = async (url) => {
        let x = await axios({
            method: 'GET',
            url: url,
            headers: {
                'Accept': 'application/vnd.api+json',
                'Authorization': `Bearer ${token.data.access_token}`
            }
        })
        return x.data
    }

    let productsPageCount = 1
    let currentPage = 1

    const handleFetchCL = async url => {
        let productsData = []
        const x = await fetchCL(url)
        productsData.push(...x.data)
        console.log(productsData)
        productsPageCount = x.meta.page_count
        while (currentPage < productsPageCount) {
            let y = await fetchCL(x.links.next)
            productsData.push(...y.data)
            currentPage++
            console.log('Page', currentPage, productsData.length)
        }
        console.log('Done', currentPage, productsData.length)
        // resetting the current Page for future fetches
        currentPage = 1
        return productsData
    }

    //
    // Fetching products and Prices
    //
    const products = await handleFetchCL(`${process.env.COMMERCE_LAYER_API}/api/skus?include=prices`)
    const prices = await handleFetchCL(`${process.env.COMMERCE_LAYER_API}/api/prices`)
    console.log('JEL SAM SE POMAKO?')

    //
    // Creating Nodes 
    // 
    let productCount = 0
    for await (const productAttributes of products) {
        await createNode({
            // Data for the node.
            code: productAttributes.attributes.code,
            reference: productAttributes.attributes.reference,
            // ...arbitraryOtherData,
            ...productAttributes,

            // Required fields.
            id: createNodeId(productAttributes.id),
            parent: null, // or null if it's a source node without a parent
            children: [],
            internal: {
                type: 'productCommerceLayer',
                mediaType: `text/html`,
                content: JSON.stringify(productAttributes),
                contentDigest: createContentDigest(productAttributes)
            }
        })
        productCount++
        console.log('+_+_+_+', productCount)
    }

    for await (const productPrices of prices) {
        await createNode({
            // Data for the node.
            skuCode: productPrices.attributes.sku_code,
            amountFloat: productPrices.attributes.amount_float,
            compareAt: productPrices.attributes.compare_at_amount_float,
            // ...arbitraryOtherData,

            // Required fields.
            id: createNodeId(productPrices.id),
            parent: null, // or null if it's a source node without a parent
            children: [],
            internal: {
                type: 'pricesCommerceLayer',
                contentDigest: createContentDigest(productPrices.attributes),
            },
        })
    }

    return
};

Initially code was located in gatsby-node. After spending 7 hours hawking and trying to implement a solution, I've made a plugin thinking (in a superstitious manner) that it might just start working.

Environment (if relevant)

System:
    OS: Windows 10 10.0.19041
    CPU: (16) x64 AMD Ryzen 7 3700X 8-Core Processor
  Binaries:
    Node: 12.18.3 - C:\Program Files\nodejs\node.EXE
    Yarn: 1.22.4 - C:\Program Files (x86)\Yarn\bin\yarn.CMD
    npm: 6.14.6 - C:\Program Files\nodejs\npm.CMD
  Languages:
    Python: 3.7.7
  Browsers:
    Chrome: 86.0.4240.75
    Edge: Spartan (44.19041.423.0)
  npmPackages:
    gatsby: ^2.24.47 => 2.24.47
    gatsby-image: ^2.2.40 => 2.4.16
    gatsby-plugin-algolia: ^0.5.0 => 0.5.0
    gatsby-plugin-breadcrumb: ^7.0.0 => 7.0.0
    gatsby-plugin-google-analytics: ^2.1.35 => 2.3.13
    gatsby-plugin-layout: ^1.1.22 => 1.3.10
    gatsby-plugin-load-script: ^1.0.6 => 1.1.0
    gatsby-plugin-manifest: ^2.2.41 => 2.4.23
    gatsby-plugin-netlify: ^2.1.33 => 2.3.13
    gatsby-plugin-nprogress: ^2.1.19 => 2.3.10
    gatsby-plugin-offline: ^3.0.34 => 3.2.23
    gatsby-plugin-react-helmet: ^3.1.22 => 3.3.10
    gatsby-plugin-react-svg: ^3.0.0 => 3.0.0
    gatsby-plugin-recaptcha: ^1.0.5 => 1.0.5
    gatsby-plugin-robots-txt: ^1.5.0 => 1.5.1
    gatsby-plugin-sharp: ^2.4.5 => 2.6.27
    gatsby-plugin-sitemap: ^2.2.27 => 2.4.11
    gatsby-plugin-theme-ui: ^0.3.0 => 0.3.0
    gatsby-plugin-typography: ^2.5.10 => 2.5.10
    gatsby-source-datocms: ^2.1.23 => 2.3.0
    gatsby-source-filesystem: ^2.1.48 => 2.3.24
    gatsby-source-instagram: ^0.7.0 => 0.7.2
    gatsby-source-yotpo: ^0.8.3 => 0.8.7
    gatsby-transformer-remark: ^2.6.51 => 2.8.28
    gatsby-transformer-remark-frontmatter: ^0.1.0 => 0.1.0
    gatsby-transformer-sharp: ^2.3.14 => 2.5.13
  npmGlobalPackages:
    gatsby-cli: 2.12.87

File contents (if changed)

gatsby-config:

require('dotenv').config({
  path: `.env.${process.env.NODE_ENV}`,
})

const website = require('./config/website')

const queries = require('./src/utils/algolia')

const pathPrefix = website.pathPrefix === `/` ? `` : website.pathPrefix

// Default title description favicon, twitte,r facebook done via DatoCMS.
module.exports = {
  siteMetadata: {
    siteUrl: "https://XXXXX.rs", // For gatsby-plugin-sitemap
    // pathPrefix,
    title: website.title,
    titleAlt: website.titleAlt,
    description: website.description,
    headline: website.headline,
    siteLanguage: website.siteLanguage,
    ogLanguage: website.ogLanguage,
    author: website.author,

    pathPrefix: `/`,
    // url: `https://biospajz.rs`,
    // title: 'Bio Spajz doo',
    // titleTemplate: `%s — Bio Spajz doo`,
    // description: `Internet prodavnice zdrave hrane, organske hrane, dijabetskih proizvoda, bez glutena, celijakije itd.`,
    // image: `/images/meta-image-default.png`, // Path to the default meta image in `static` folder
    // siteLanguage: `sr-Latn-RS`, // Language tag on <html> element
    // ogLanguage: 'sr_RS', // Facebook Language
    // author: 'Bio Špajz', // Author for schemaORGJSONLD
    // favicon: `src/images/favicon.png`, // Used for manifest favicon generation
    // shortName: `Starter Styled`, // shortname for manifest. *Must* be shorter than 12 characters
    // themeColor: `#253688`,
    // backgroundColor: `#253688`,
  },
  plugins: [
    `gatsby-plugin-recaptcha`,
    `gatsby-plugin-netlify`,  
    {
      resolve: `gatsby-source-yotpo`,
      options: {
        appKey: `${process.env.YOTPO_APPKEY}`,
        appSecret: `${process.env.YOTPO_APPSECRET}`
      }
    },
    {
      resolve: require.resolve(`./src/plugins/source-plugin`),
    },
    {
      resolve: `gatsby-source-datocms`,
      options: {
        // You can find your read-only API token under the Settings > API tokens
        // section of your administrative area:
        apiToken: `${process.env.DATO_APITOKEN}`,
        // If you are working on development/staging environment, you might want to
        // preview the latest version of records instead of the published one:
        // previewMode: false,
        // Disable automatic reloading of content when some change occurs on DatoCMS:
        // disableLiveReload: false,
        // Custom API base URL
        // apiUrl: 'https://bio-spajz.admin.datocms.com/',
        // Setup locale fallbacks
        // In this example, if some field value is missing in Italian, fall back to English
        // localeFallbacks: {
        //   it: ['en'],
        // },
      },
    },
    {
      resolve: `gatsby-plugin-nprogress`,
      options: {
        // Setting a color is optional.
        color: `orange`,
        // Disable the loading spinner.
        showSpinner: true,
      },
    },
    {
      resolve: `gatsby-plugin-algolia`,
      options: {
        appId: `${process.env.ALGOLIA_APPID}`,
        apiKey: ` ${process.env.ALGOLIA_APIKEY}`,
        queries, // TODO: Uncomment once live - deactivated to not waste operations
        chunkSize: 10000, // default: 1000
      },
    },
    {
      resolve: 'gatsby-plugin-robots-txt',
      options: {
        host: 'https://vibrant-saha-9ee3d0.netlify.com/',
        sitemap: 'https://vibrant-saha-9ee3d0.netlify.com/sitemap.xml',
        policy: [{ userAgent: '*', disallow: '/' }]
      }
    },
    `gatsby-plugin-sitemap`,
    `gatsby-plugin-react-helmet`,
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `images`,
        path: `${__dirname}/src/images`,
      },
    },
    {
      resolve: `gatsby-source-filesystem`,
      options: {
        name: `fonts`,
        path: `${__dirname}/src/fonts`,
      },
    },
    {
      resolve: `gatsby-transformer-remark`,
      options: {
        // CommonMark mode (default: true)
        commonmark: true,
        // Footnotes mode (default: true)
        footnotes: true,
        // Pedantic mode (default: true)
        pedantic: true,
        // GitHub Flavored Markdown mode (default: true)
        gfm: true,
        // Plugins configs
        plugins: [],
      },
    },
    `gatsby-transformer-sharp`,
    `gatsby-plugin-sharp`,
    `gatsby-plugin-theme-ui`,
    // {
    //   resolve: `gatsby-plugin-typography`,
    //   options: {
    //     pathToConfigModule: `./config/typography.js`,
    //   },
    // },
    {
      resolve: `gatsby-plugin-manifest`,
      options: {
        name: `gatsby-starter-default`,
        short_name: `starter`,
        start_url: `/`,
        background_color: `#663399`,
        theme_color: `#663399`,
        display: `minimal-ui`,
        icon: `src/images/gatsby-icon.png`, // This path is relative to the root of the site.
      },
    },
    {
      resolve: `gatsby-plugin-layout`,
      options: {
        component: require.resolve(`./src/layouts/index.js`),
      },
    },
    {
      resolve: "gatsby-plugin-react-svg",
      options: {
        rule: {
          include: /assets/
        }
      }
    }
    // this (optional) plugin enables Progressive Web App + Offline functionality
    // To learn more, visit: https://gatsby.dev/offline
    // `gatsby-plugin-offline`,
  ],
}

Else didn't think it relevant will provide if needed!

[cymkd]

A very, very dumb mistake in data collection (repeating data).