[v2] stability issue: www randomly fails to build. how to improve debuggability?

this is a placeholder issue while i try to figure out what causes this as I have absolutely no idea and this is a very bad stability issue

Description

i was resuming work on code that was previously working - and then it suddenly fails to build. :( i know that's not a very good bug report, but i dont have much else to debug this error message with (i copy out the error message below). Clearing .cache and reinstalling npm modules doesnt help. This is forcing me to abandon this gatsby instance and just git clone from scratch in the hopes that it will work.

Steps to reproduce

i know i write this a lot but literally git clone, then git checkout v2 && cd www && yarn && yarn run develop.

Expected result

www should build

Actual result

the error I see:

 ✝  StarterShowcase/gatsby/www   StarterShowcase±  yarn run develop
yarn run v1.6.0
$ gatsby develop
success delete html files from previous builds — 0.411 s
success open and validate gatsby-config — 0.005 s
success copy gatsby files — 0.032 s
success onPreBootstrap — 1.152 s
success source and transform nodes — 3.858 s
success building schema — 0.937 s
error gatsby-node.js returned an error

  TypeError: Cannot read property 'allMarkdownRemark' of undefined

  - gatsby-node.js:133 graphql.then.result
    /Users/swyx/gatsby/StarterShowcase/gatsby/www/gatsby-node.js:133:23

  - es6-promise.js:409 tryCatch
    [www]/[es6-promise]/dist/es6-promise.js:409:12

  - es6-promise.js:424 invokeCallback
    [www]/[es6-promise]/dist/es6-promise.js:424:13

  - es6-promise.js:176
    [www]/[es6-promise]/dist/es6-promise.js:176:14

  - es6-promise.js:128 flush
    [www]/[es6-promise]/dist/es6-promise.js:128:5

  - next_tick.js:150 process._tickCallback
    internal/process/next_tick.js:150:11

line 133 is code to do with the blog posts - which I never even touched. :(

Environment


  System:
    OS: macOS High Sierra 10.13.3
    CPU: x64 Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
    Shell: 5.3 - /bin/zsh
  Binaries:
    Node: 9.4.0 - /usr/local/bin/node
    Yarn: 1.6.0 - /usr/local/bin/yarn
    npm: 6.0.1 - /usr/local/bin/npm
  Browsers:
    Chrome: 67.0.3396.87
    Firefox: 60.0.1
    Safari: 11.0.3
  npmPackages:
    gatsby: next => 2.0.0-alpha.59 
    gatsby-image: next => 2.0.0-alpha.9 
    gatsby-plugin-canonical-urls: next => 2.0.0-alpha.2 
    gatsby-plugin-catch-links: next => 2.0.2-alpha.2 
    gatsby-plugin-feed: next => 2.0.0-alpha.2 
    gatsby-plugin-fullstory: ^1.0.4-5 => 1.0.4-9 
    gatsby-plugin-glamor: next => 2.0.0-alpha.16 
    gatsby-plugin-google-analytics: next => 2.0.0-alpha.2 
    gatsby-plugin-lodash: next => 3.0.1-alpha.2 
    gatsby-plugin-mailchimp: ^2.2.3 => 2.2.3 
    gatsby-plugin-manifest: next => 2.0.2-alpha.2 
    gatsby-plugin-netlify: next => 2.0.0-alpha.2 
    gatsby-plugin-nprogress: next => 2.0.0-alpha.15 
    gatsby-plugin-offline: next => 2.0.0-alpha.19 
    gatsby-plugin-react-helmet: next => 3.0.0-alpha.2 
    gatsby-plugin-sharp: next => 2.0.0-alpha.23 
    gatsby-plugin-sitemap: next => 2.0.0-alpha.2 
    gatsby-plugin-twitter: next => 2.0.0-alpha.2 
    gatsby-plugin-typography: next => 2.2.0-alpha.2 
    gatsby-remark-autolink-headers: next => 2.0.0-alpha.2 
    gatsby-remark-copy-linked-files: next => 2.0.0-alpha.2 
    gatsby-remark-images: next => 2.0.1-alpha.2 
    gatsby-remark-prismjs: next => 3.0.0-alpha.2 
    gatsby-remark-responsive-iframe: next => 2.0.0-alpha.2 
    gatsby-remark-smartypants: next => 2.0.0-alpha.2 
    gatsby-source-filesystem: next => 2.0.1-alpha.2 
    gatsby-source-npm-package-search: ^1.0.8 => 1.0.8 
    gatsby-transformer-csv: next => 2.0.0-alpha.2 
    gatsby-transformer-documentationjs: next => 2.0.0-alpha.2 
    gatsby-transformer-remark: next => 2.1.1-alpha.2 
    gatsby-transformer-sharp: next => 2.1.1-alpha.2 
    gatsby-transformer-yaml: next => 2.1.1-alpha.2 
  npmGlobalPackages:
    gatsby-cli: 1.1.58
    gatsby: 1.9.231

File contents (if changed)

gatsby-config.js: N/A package.json: N/A gatsby-node.js:

const _ = require(`lodash`)
const Promise = require(`bluebird`)
const path = require(`path`)
const parseFilepath = require(`parse-filepath`)
const fs = require(`fs-extra`)
const slash = require(`slash`)
const slugify = require(`limax`)

const localPackages = `../packages`
const localPackagesArr = []
fs.readdirSync(localPackages).forEach(file => {
  localPackagesArr.push(file)
})
// convert a string like `/some/long/path/name-of-docs/` to `name-of-docs`
const slugToAnchor = slug =>
  slug
    .split(`/`) // split on dir separators
    .filter(item => item !== ``) // remove empty values
    .pop() // take last item

exports.createPages = ({ graphql, actions }) => {
  const { createPage, createRedirect } = actions

  // Random redirects
  createRedirect({
    fromPath: `/blog/2018-02-26-documentation-project/`, // Tweeted this link out then switched it
    toPath: `/blog/2018-02-28-documentation-project/`,
    isPermanent: true,
  })

  createRedirect({
    fromPath: `/community/`, // Moved "Community" page from /community to /docs/community
    toPath: `/docs/community/`,
    isPermanent: true,
  })

  createRedirect({
    fromPath: `/packages/`, // Moved "Plugins" page from /packages to /plugins
    toPath: `/plugins/`,
    isPermanent: true,
  })

  return new Promise((resolve, reject) => {
    const docsTemplate = path.resolve(`src/templates/template-docs-markdown.js`)
    const blogPostTemplate = path.resolve(`src/templates/template-blog-post.js`)
    const tagTemplate = path.resolve(`src/templates/tags.js`)
    const contributorPageTemplate = path.resolve(
      `src/templates/template-contributor-page.js`
    )
    const localPackageTemplate = path.resolve(
      `src/templates/template-docs-local-packages.js`
    )
    const remotePackageTemplate = path.resolve(
      `src/templates/template-docs-remote-packages.js`
    )

    createRedirect({
      fromPath: `/docs/bound-action-creators/`,
      isPermanent: true,
      redirectInBrowser: true,
      toPath: `/docs/actions/`,
    })

    createRedirect({
      fromPath: `/docs/bound-action-creators`,
      isPermanent: true,
      redirectInBrowser: true,
      toPath: `/docs/actions`,
    })

    // Query for markdown nodes to use in creating pages.
    resolve(
      graphql(
        `
          query {
            allMarkdownRemark(
              sort: { order: DESC, fields: [frontmatter___date] }
              limit: 1000
              filter: { fileAbsolutePath: { ne: null } }
            ) {
              edges {
                node {
                  fields {
                    slug
                    package
                    starterShowcase {
                      slug
                    }
                  }
                  frontmatter {
                    title
                    draft
                    canonicalLink
                    publishedAt
                    tags
                  }
                }
              }
            }
            allAuthorYaml {
              edges {
                node {
                  fields {
                    slug
                  }
                }
              }
            }
            allNpmPackage {
              edges {
                node {
                  id
                  title
                  slug
                  readme {
                    id
                    childMarkdownRemark {
                      id
                      html
                    }
                  }
                }
              }
            }
          }
        `
      ).then(result => {
        if (result.errors) {
          reject(result.errors)
        }
        // here is line 133 - where the error is supposedly at
        const blogPosts = _.filter(
          result.data.allMarkdownRemark.edges,
          edge => {
            const slug = _.get(edge, `node.fields.slug`)
            const draft = _.get(edge, `node.frontmatter.draft`)
            if (!slug) return

            if (_.includes(slug, `/blog/`) && !draft) {
              return edge
            }
          }
        )

        // Create blog pages.
        blogPosts.forEach((edge, index) => {
          const next = index === 0 ? null : blogPosts[index - 1].node
          const prev =
            index === blogPosts.length - 1 ? null : blogPosts[index + 1].node

          createPage({
            path: `${edge.node.fields.slug}`, // required
            component: slash(blogPostTemplate),
            context: {
              slug: edge.node.fields.slug,
              prev,
              next,
            },
          })
        })

        const tagLists = blogPosts
          .filter(post => _.get(post, `node.frontmatter.tags`))
          .map(post => _.get(post, `node.frontmatter.tags`))

        _.uniq(_.flatten(tagLists)).forEach(tag => {
          createPage({
            path: `/blog/tags/${_.kebabCase(tag)}/`,
            component: tagTemplate,
            context: {
              tag,
            },
          })
        })

        // Create contributor pages.
        result.data.allAuthorYaml.edges.forEach(edge => {
          createPage({
            path: `${edge.node.fields.slug}`,
            component: slash(contributorPageTemplate),
            context: {
              slug: edge.node.fields.slug,
            },
          })
        })

        // Create starters.
        const starters = _.filter(
          result.data.allMarkdownRemark.edges,
          edge => {
            const slug = _.get(edge, `node.fields.starterShowcase.slug`)
            if (!slug) return null
            else return edge
          }
        )
        const starterTemplate = path.resolve(`src/templates/template-starter-showcase.js`)

        starters.forEach((edge, index) => {
          createPage({
            path: `/starters${edge.node.fields.starterShowcase.slug}`, // required
            component: slash(starterTemplate),
            context: {
              slug: edge.node.fields.starterShowcase.slug,
            },
          })
        })
        // END Create starters.

        // Create docs pages.
        result.data.allMarkdownRemark.edges.forEach(edge => {
          const slug = _.get(edge, `node.fields.slug`)
          if (!slug) return

          if (!_.includes(slug, `/blog/`)) {
            createPage({
              path: `${edge.node.fields.slug}`, // required
              component: slash(
                edge.node.fields.package ? localPackageTemplate : docsTemplate
              ),
              context: {
                slug: edge.node.fields.slug,
              },
            })
          }
        })

        const allPackages = result.data.allNpmPackage.edges
        // Create package readme
        allPackages.forEach(edge => {
          if (_.includes(localPackagesArr, edge.node.title)) {
            createPage({
              path: edge.node.slug,
              component: slash(localPackageTemplate),
              context: {
                slug: edge.node.slug,
                id: edge.node.id,
              },
            })
          } else {
            createPage({
              path: edge.node.slug,
              component: slash(remotePackageTemplate),
              context: {
                slug: edge.node.slug,
                id: edge.node.id,
              },
            })
          }
        })

        return
      })
    )
  })
}

// Create slugs for files.
exports.onCreateNode = ({ node, actions, getNode, boundActionCreators }) => {
  const { createNodeField } = actions
  let slug
  if (node.internal.type === `File`) {
    const parsedFilePath = parseFilepath(node.relativePath)
    if (node.sourceInstanceName === `docs`) {
      if (parsedFilePath.name !== `index` && parsedFilePath.dir !== ``) {
        slug = `/${parsedFilePath.dir}/${parsedFilePath.name}/`
      } else if (parsedFilePath.dir === ``) {
        slug = `/${parsedFilePath.name}/`
      } else {
        slug = `/${parsedFilePath.dir}/`
      }
    }
    if (slug) {
      createNodeField({ node, name: `slug`, value: slug })
    }
  } else if (
    node.internal.type === `MarkdownRemark` &&
    getNode(node.parent).internal.type === `File`
  ) {
    const fileNode = getNode(node.parent)
    const parsedFilePath = parseFilepath(fileNode.relativePath)
    // Add slugs for docs pages
    if (fileNode.sourceInstanceName === `docs`) {
      if (parsedFilePath.name !== `index` && parsedFilePath.dir !== ``) {
        slug = `/${parsedFilePath.dir}/${parsedFilePath.name}/`
      } else if (parsedFilePath.dir === ``) {
        slug = `/${parsedFilePath.name}/`
      } else {
        slug = `/${parsedFilePath.dir}/`
      }
    }
    // Add slugs for package READMEs.
    if (
      fileNode.sourceInstanceName === `packages` &&
      parsedFilePath.name === `README`
    ) {
      slug = `/packages/${parsedFilePath.dir}/`
      createNodeField({
        node,
        name: `title`,
        value: parsedFilePath.dir,
      })
      createNodeField({ node, name: `package`, value: true })
    }
    if (
      fileNode.sourceInstanceName === `StarterShowcaseData` &&
      parsedFilePath.name !== `README`
    ) {
      createNodesForStarterShowcase({ node, getNode, createNodeField })
    }
    if (slug) {
      createNodeField({ node, name: `anchor`, value: slugToAnchor(slug) })
      createNodeField({ node, name: `slug`, value: slug })
    }

  } else if (node.internal.type === `AuthorYaml`) {
    slug = `/contributors/${slugify(node.id)}/`
    createNodeField({ node, name: `slug`, value: slug })
  }

}

exports.onPostBuild = () => {
  fs.copySync(
    `../docs/blog/2017-02-21-1-0-progress-update-where-came-from-where-going/gatsbygram.mp4`,
    `./public/gatsbygram.mp4`
  )
}

// Starter Showcase related code
const { createFilePath } = require(`gatsby-source-filesystem`)
const gitFolder = './src/data/StarterShowcase/generatedGithubData'
function createNodesForStarterShowcase({ node, getNode, createNodeField }) {
  if (node.internal.type === `MarkdownRemark`) {
    const slug = createFilePath({
      node,
      getNode,
      basePath: `startersData`,
    })
    // preprocessing
    const stub = slug.replace(/\//gi, '')
    var fromPath = path.join(gitFolder, `${stub}.json`)
    var data = fs.readFileSync(fromPath, 'utf8')
    const ghdata = JSON.parse(data)
    const { repoMetadata, dependencies = [], devDependencies = [] } = ghdata
    const allDependencies = Object.entries(dependencies).concat(
      Object.entries(devDependencies)
    )
    // make an object to stick into a Field
    const starterShowcaseFields = {
      slug,
      stub,
      date: new Date(node.frontmatter.date),
      githubData: ghdata,
      // nice-to-have destructures of githubData
      description: ghdata.description,
      stars: repoMetadata.stargazers_count,
      lastUpdated: repoMetadata.created_at,
      owner: repoMetadata.owner,
      githubFullName: repoMetadata.full_name,
      allDependencies,
      gatsbyDependencies: allDependencies
        .filter(
          ([key, _]) => !['gatsby', 'gatsby-cli', 'gatsby-link'].includes(key)
        )
        .filter(([key, _]) => key.includes('gatsby')),
      miscDependencies: allDependencies.filter(([key, _]) => !key.includes('gatsby')),
    }
    createNodeField({ node, name: `starterShowcase`, value: starterShowcaseFields })
  }
}
// End Starter Showcase related code

gatsby-browser.js: N/A gatsby-ssr.js: N/A

gatsbyjs / gatsby