Gatsby always fetch cached data

[AndrejGajdos]

I use gatsby-source-git to pull files from Github repository. It worked well, but I added some files into the repository and I am not able to pull new files into my Gatsby project.

If I run this query in http://localhost:8000/___graphql:

query MyQuery {
  allMarkdownRemark {
    edges {
      node {
        frontmatter {
          title
        }
        fileAbsolutePath
      }
    }
  }
}

I get this result:

{
  "data": {
    "allMarkdownRemark": {
      "edges": [
        {
          "node": {
            "frontmatter": {
              "title": "Introduction"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Conceptual Guide"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tau-conceptual-guide.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tau & Agoras Overview"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/what-is-tauchain-tau.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "FAQs"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tauchain-agoras-faqs.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tutorials"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Analysis of TauBot.TML"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/analysis-of-taubot-tml.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "TML Bot Tutorial"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/tml-Bot-tutorial.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Understanding TML"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/understanding-tml-prolog-datalog-tau.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Quick Start"
            },
            "fileAbsolutePath": "dev/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/quick-start.md"
          }
        }
      ]
    }
  }
}

However, Tutorials folder contains 6 files, which aren't included in query result. I don't know why.

gatsby-config.js

require("dotenv").config({
  path: `.env.${process.env.NODE_ENV}`
});

module.exports = {
  siteMetadata: {
    title: "Fan Site"
  },
  plugins: [
    "gatsby-plugin-react-helmet",
    "svgo",
    "gatsby-plugin-sass",
    "gatsby-plugin-postcss",
    {
      resolve: "gatsby-source-filesystem",
      options: {
        name: "data",
        path: `${__dirname}/src/data/`
      }
    },
    {
      resolve: "gatsby-source-filesystem",
      options: {
        path: `${__dirname}/src/pages`,
        name: "pages"
      }
    },
    {
      resolve: `gatsby-transformer-remark`,
      options: {
        plugins: [`gatsby-remark-autolink-headers`]
      }
    },
    "gatsby-transformer-json",
    "gatsby-plugin-dark-mode",
    {
      resolve: "gatsby-source-prismic-graphql",
      options: {
        repositoryName: "funsite",
        accessToken: `${process.env.API_KEY}`
      }
    },
    {
      resolve: `gatsby-source-git`,
      options: {
        name: `tau-guide-documents`,
        remote: `https://github.com/TauGuide/tau-guide-documents.git`,
        branch: `master`,
        // Only import the docs folder from a codebase.
        patterns: `docs/**`
      }
    }
  ]
};

I tried to add local: "/dev/tauguide/tau-guide-documents", as gatsby-source-git option, but it didn't help.

I tried to run gatsby clean which removed .cache folder and gatsby develop, but I got the same result.

I tried to remove the project and pull it from git again, but it didn't help.

package.json

{
  "name": "fan-site",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "develop": "gatsby develop",
    "build": "gatsby build",
    "serve": "gatsby serve",
    "start": "node server.js",
    "gh-pages": "gatsby build --prefix-paths && gh-pages -d public",
    "lint": "eslint src --fix",
    "dev": "(shx --silent rm -rf public .cache || shx true) && gatsby develop",
    "server": "cross-env NODE_ENV=development DEBUG=api nodemon server.js",
    "postinstall": "npm rebuild node-sass"
  },
  "author": "Prototype Interactive",
  "license": "MIT",
  "dependencies": {
    "@fortawesome/fontawesome-svg-core": "^1.2.25",
    "@fortawesome/free-brands-svg-icons": "^5.12.0",
    "@fortawesome/free-solid-svg-icons": "^5.11.2",
    "@fortawesome/react-fontawesome": "^0.1.7",
    "@kunukn/react-collapse": "1",
    "@material-ui/core": "^4.9.5",
    "@popperjs/core": "^2.3.3",
    "add": "^2.0.6",
    "axios": "^0.19.0",
    "basic-auth": "^2.0.1",
    "bootstrap": "4.2.1",
    "chart.js": "^2.9.3",
    "d3-node": "^2.2.1",
    "debug": "^4.1.1",
    "font-awesome": "4.7.0",
    "framer-motion": "^1.10.3",
    "gatsby": "^2.18.12",
    "gatsby-plugin-canonical-urls": "^2.3.0",
    "gatsby-plugin-dark-mode": "^1.1.0",
    "gatsby-remark-autolink-headers": "^2.3.3",
    "gatsby-source-git": "^1.0.2",
    "gatsby-source-prismic-graphql": "3.6.2",
    "gatsby-transformer-remark": "^2.6.53",
    "lodash.get": "^4.4.2",
    "lodash.groupby": "^4.6.0",
    "lodash.pickby": "^4.6.0",
    "lodash.set": "^4.3.2",
    "lodash.update": "^4.10.2",
    "marked": "^0.8.0",
    "moment": "^2.24.0",
    "prismic-reactjs": "^1.3.1",
    "prop-types": "^15.7.2",
    "react-chartjs-2": "^2.8.0",
    "react-collapsible": "^2.6.3",
    "react-d3-components": "^0.9.1",
    "react-d3-library": "^1.1.8",
    "react-headroom": "^3.0.0",
    "react-helmet": "^5.2.0",
    "react-lazyload": "^2.6.5",
    "react-moment": "^0.9.6",
    "react-onclickout": "^2.0.8",
    "react-popper": "^2.2.3",
    "react-popper-tooltip": "^2.11.1",
    "react-responsive": "^8.0.1",
    "react-scroll-to": "^3.0.0-beta.3",
    "react-select": "^3.1.0",
    "react-sidebar": "^3.0.2",
    "react-slick": "^0.25.2",
    "react-svg-donuts": "^1.0.0",
    "react-telegram-embed": "^0.0.10",
    "react-toastify": "^5.4.1",
    "react-twitter-embed": "^3.0.3",
    "react-window": "^1.8.5",
    "reactstrap": "^8.4.1",
    "slick-carousel": "^1.8.1",
    "underscore": "^1.9.1",
    "yarn": "^1.21.1"
  },
  "devDependencies": {
    "@prototype-interactive/eslint-config": "^0.1.1",
    "autoprefixer": "^9.4.4",
    "cross-env": "^5.2.0",
    "dotenv": "^8.2.0",
    "eslint": "^5.12.0",
    "gatsby-plugin-google-analytics": "^2.3.0",
    "gatsby-plugin-postcss": "^2.0.2",
    "gatsby-plugin-postcss-sass": "^1.0.22",
    "gatsby-plugin-react-helmet": "^3.0.5",
    "gatsby-plugin-sass": "^2.0.7",
    "gatsby-source-filesystem": "^2.2.2",
    "gatsby-transformer-json": "^2.1.7",
    "gh-pages": "^2.0.1",
    "husky": "^1.3.1",
    "prettier": "^1.15.3",
    "pretty-quick": "^1.8.0",
    "react": "^16.7.0",
    "react-dom": "^16.7.0",
    "shx": "^0.3.2",
    "svg-sprite-loader": "^4.1.3"
  },
  "resolutions": {
    "gatsby-source-graphql-universal": "3.3.0"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/PrototypeInteractive/gatsby-react-boilerplate.git"
  },
  "bugs": {
    "url": "https://github.com/PrototypeInteractive/gatsby-react-boilerplate/issues"
  },
  "homepage": "https://github.com/PrototypeInteractive/gatsby-react-boilerplate#readme"
}

gatsby-node.js

const path = require(`path`);
const { createFilePath } = require(`gatsby-source-git`);

exports.onCreateNode = ({ node, getNode, actions }) => {
  const { createNodeField } = actions;
  if (node.internal.type === `MarkdownRemark`) {
    const slug = createFilePath({ node, getNode, basePath: `pages` });
    createNodeField({
      node,
      name: `slug`,
      value: slug
    });
  }
};

exports.createPages = async ({ graphql, actions }) => {
  const { createPage } = actions;
  const result = await graphql(`
    query {
      allMarkdownRemark {
        edges {
          node {
            fields {
              slug
            }
          }
        }
      }
    }
  `);
  result.data.allMarkdownRemark.edges.forEach(({ node }) => {
    createPage({
      path: node.fields.slug,
      component: path.resolve(`src/templates/tau-document.js`),
      context: {
        // Data passed to context is available
        // in page queries as GraphQL variables.
        slug: node.fields.slug
      }
    });
  });
};

[stevetweeddale]

Hmm, interesting. Your query results include 4 of the 6 "Tutorial" files in that repo… so it must be something about the other 2 files (the "step-by-step" ones). I can't say for sure what the problem is without setting up your site locally, but here's probably what I'd check first:

• Manually examine the .cache directory to confirm the 2 files are actually there in the git repo that gets checked out. If they're not, there's the problem.
• If they are there, but just not being imported, then I'd try manually cloning the repo somewhere and try sourcing the files with gatsby-source-filesystem, to see if that works.

Let me know how you get on!

[AndrejGajdos]

@stevetweeddale I can confirm all files are actually in .cache folder. I tried to pull files using gatsby-source-filesystem like this:

{
      resolve: "gatsby-source-filesystem",
      options: {
        path: `${__dirname}/.cache/gatsby-source-git/tau-guide-documents/docs`,
        name: "docs"
      }
    }

and this is the result:

{
  "data": {
    "allMarkdownRemark": {
      "edges": [
        {
          "node": {
            "frontmatter": {
              "title": "Introduction"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Conceptual Guide"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tau-conceptual-guide.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "FAQs"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tauchain-agoras-faqs.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tau & Agoras Overview"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/what-is-tauchain-tau.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Analysis of TauBot.TML"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/analysis-of-taubot-tml.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tutorials"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "TML Bot Tutorial"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/tml-Bot-tutorial.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Understanding TML"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/understanding-tml-prolog-datalog-tau.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Quick Start"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/quick-start.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Introduction"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Quick Start"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/quick-start.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Conceptual Guide"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tau-conceptual-guide.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "FAQs"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/tauchain-agoras-faqs.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tau & Agoras Overview"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/what-is-tauchain-tau.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Analysis of TauBot.TML"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/analysis-of-taubot-tml.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Tutorials"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/index.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "TML Bot Tutorial"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/tml-Bot-tutorial.md"
          }
        },
        {
          "node": {
            "frontmatter": {
              "title": "Understanding TML"
            },
            "fileAbsolutePath": "dev/tauguide/tau-guide-website/.cache/gatsby-source-git/tau-guide-documents/docs/Tutorials/understanding-tml-prolog-datalog-tau.md"
          }
        }
      ]
    }
  }
}

the files are still missing there.

[AndrejGajdos]

There was an issue with formatting of the frontmatter in 2 files. They both have colon inside a value:

title: Agoras: How to Store
description: Agoras: How to Buy

Adding quotes solved this issue.

title: "Agoras: How to Store"
description: "Agoras: How to Buy"

[stevetweeddale]

Yeah, I assumed it must have been something funky with those particular files. Glad you worked it out!