Data file data disappearing from template

[smth]

I have tried to create a data file that fetches an API, loops through the responses, then fetches another API for each response. Everything worked as expected when I was just fetching the initial API, but when adding the loop I found that anything prompting a page refresh would cause all the data to disappear from the page. Stopping and starting the dev server would cause the page to render correctly (with data in place), until another change is made to a template.

To Reproduce

Here is an example using the GitHub API. I'm actually using the Flickr API, which requires something like this to get the data I want. If there's a flaw in my approach / code though (quite possible) let me know :)

// github.js

const fetch = require("node-fetch");

module.exports = async function() {
  let repoList;
  let repos = {};

  return fetch("https://api.github.com/orgs/11ty/repos")
    .then(res => res.json()) // node-fetch option to transform to json
    .then(json => {
      repoList = json;
    })
    .then(json => {
      let key;
      let value;
      for (i = 0; i < repoList.length; i++) {

        fetch("https://api.github.com/repos/11ty/" + repoList[i].name)
        .then(res => res.json()) // node-fetch option to transform to json
        .then(json => {
          key = "repo" + json.id;
          value = json.stargazers_count;
          repos[key] = value;
        })
      }
    })
    .then(json => {
      return {
        repos
      };
    });
};

Steps to reproduce the behavior:

1. Add the above file to your _data folder.
2. Output the data in a template My data: {{ github | dump }}.
3. Run dev server npx @11ty/eleventy --serve (everything looks good).
4. Edit the template to trigger a page rebuild/refresh. Still my data: {{ github | dump }}.

Expected behavior I would expect nothing but the edited words in the template to change. Instead the github object appears empty.

Environment:

• OS and Version: macOS 10.14.6
• Eleventy Version: 0.7.1 (also got the same result with 0.9.0)

[Ryuno-Ki]

Personally, I went to loading the data first and store it in a JSON file on disk. Then, my _data file merely reads the file and processes it as if it were a network request.

The nice advantage of this approach is that I don't hammer other people's API, because I have decoupled network traffic from local development. (Oh, and I can sit in a library or so without Internet and continue working on my page :-) Yes, Germany has many white spots when it comes to broadband coverage …)

[smth]

That's sort of what I was hoping to achieve with this feature, once there was some caching in place (though I couldn't get caching to work with that code either). Are you automating the fetching from an API in another way?

[Ryuno-Ki]

I'm using the SDK provided by Octokit:

const fs = require('fs')
const path = require('path')

const dotenv = require('dotenv')
const { promisify } = require('es6-promisify')
const Octokit = require('@octokit/rest')

const log = require('../helpers/logger')

const logger = log.getLogger('data:github_repo')
const writeFile = promisify(fs.writeFile)

dotenv.config()
let octokit
if (process.env.GITHUB_AUTH_TOKEN) {
  octokit = new Octokit({
    auth: process.env.GITHUB_AUTH_TOKEN
  })
} else {
  octokit = new Octokit()
}

async function getGitHubRepo () {
  let response
  try {
    response = await doFetch()
  } catch (error) {
    logger.error(`Could not fetch GitHub repo: ${error.errno} - ${JSON.stringify(error.response)}`)
    return []
  }
  await saveResponse(response)
  return response
}

function doFetch () {
  const username = 'Ryuno-Ki'
  const payload = {
    username,
    per_page: 100
  }
  const repos = octokit.repos.listForUser(payload)
  return repos
}

async function saveResponse (response) {
  return Promise.all(response.data
    .map((repo) => {
      const id = repo.name.toLowerCase().replace(/\W/g, '_')

      const fileName = `${id}.json`
      return {
        fileName: fileName,
        data: repo
      }
    })
    .map((pair) => {
      const fileName = pair.fileName
      const dirPath = path.resolve(__dirname, '..', 'downloads', 'github', 'repo')
      const filePath = path.join(dirPath, fileName)
      return {
        filePath: filePath,
        data: pair.data
      }
    })
    .map((pair) => {
      return writeFile(pair.filePath, JSON.stringify(pair.data, null, 2))
    })
  )
}

module.exports = getGitHubRepo

This is abstracting away the communication with the repositories endpoint.

I defined a npm run-script for its invocation: "github:repo": "node -e 'var gh=require(\"./scripts/github-repo\");gh().then(()=>{})'"

[Ryuno-Ki]

The invocation gets on push on my server as a post-receive git hook. Works good enough for me :-)

[zachleat]

I am very sorry for showing up embarrassingly late to this issue but this problem is solved by the official Fetch plugin: https://www.11ty.dev/docs/plugins/fetch/

This is an automated message to let you know that a helpful response was posted to your issue and for the health of the repository issue tracker the issue will be closed. This is to help alleviate issues hanging open waiting for a response from the original poster.

If the response works to solve your problem—great! But if you’re still having problems, do not let the issue’s closing deter you if you have additional questions! Post another comment and we will reopen the issue. Thanks!