algolia / gatsby-plugin-algolia

A plugin to push to Algolia based on graphQl queries
https://yarn.pm/gatsby-plugin-algolia
Apache License 2.0
177 stars 45 forks source link

Not finished onPostBuild #89

Closed h1eutran closed 3 years ago

h1eutran commented 4 years ago

Quite similar with this issues: Issues 55

My Queries Settings:


const escapeStringRegexp = require("escape-string-regexp")

const pageQuery = `
  query LandingQuery {
    allMarkdownRemark(sort: { fields: [fields___date], order: DESC }) {
      edges {
        node {
          fields {
            slug
            date
          }
          excerpt
          timeToRead
          frontmatter {
            title
            tags
            cover
            date
          }
        }
      }
    }
  }
`;
function pageToAlgoliaRecord({ node: { id, frontmatter, fields, ...rest } }) {
    return {
      ...frontmatter,
      ...fields,
      ...rest,
    }
  }

const queries = [
    {
        query: pageQuery,
        transformer: ({ data }) => data.allMarkdownRemark.edges.map(pageToAlgoliaRecord), // optional
        indexName: 'prod_GuillouTran', // overrides main index name, optional
    },
    ];

    module.exports = queries

And my Gatsby config:

const urljoin = require("url-join");
const path = require("path");
const config = require("./data/SiteConfig");
require('dotenv').config({
  path: `.env.${process.env.NODE_ENV}`,
})

module.exports = {
  pathPrefix: config.pathPrefix === "" ? "/" : config.pathPrefix,
  siteMetadata: {
    siteUrl: urljoin(config.siteUrl, config.pathPrefix),
    rssMetadata: {
      site_url: urljoin(config.siteUrl, config.pathPrefix),
      feed_url: urljoin(config.siteUrl, config.pathPrefix, config.siteRss),
      title: config.siteTitle,
      description: config.siteDescription,
      image_url: `${urljoin(
        config.siteUrl,
        config.pathPrefix
      )}/logos/logo-512.png`,
      copyright: config.copyright
    }
  },
  plugins: [
    "gatsby-plugin-react-helmet",
    "gatsby-plugin-lodash",
    {
      resolve: "gatsby-source-filesystem",
      options: {
        name: "assets",
        path: `${__dirname}/static/`
      }
    },
    {
      resolve: "gatsby-source-filesystem",
      options: {
        name: "posts",
        path: `${__dirname}/content/`
      }
    },
    {
    resolve: 'gatsby-plugin-google-fonts',
    options: {
        fonts: [
          'limelight',
          'ubuntu mono\:400,400i' // you can also specify font weights and styles
        ]
    }},
    {
      resolve: "gatsby-transformer-remark",
      options: {
        plugins: [
          {
            resolve: `gatsby-remark-relative-images`
          },
          {
            resolve: "gatsby-remark-images",
            options: {
              maxWidth: 690
            }
          },
          {
            resolve: "gatsby-remark-responsive-iframe"
          },
          "gatsby-remark-copy-linked-files",
          "gatsby-remark-autolink-headers",
          "gatsby-remark-prismjs"
        ]
      }
    },
    {
      resolve: "gatsby-plugin-google-analytics",
      options: {
        trackingId: config.googleAnalyticsID
      }
    },
    {
      resolve: "gatsby-plugin-nprogress",
      options: {
        color: config.themeColor
      }
    },
    "gatsby-plugin-sharp",
    "gatsby-transformer-sharp",
    "gatsby-plugin-catch-links",
    "gatsby-plugin-twitter",
    "gatsby-plugin-sitemap",
    {
      resolve: "gatsby-plugin-manifest",
      options: {
        name: config.siteTitle,
        short_name: config.siteTitleShort,
        description: config.siteDescription,
        start_url: config.pathPrefix,
        background_color: config.backgroundColor,
        theme_color: config.themeColor,
        display: "minimal-ui",
        icons: [
          {
            src: "/logos/logo-192.png",
            sizes: "192x192",
            type: "image/png"
          },
          {
            src: "/logos/logo-512.png",
            sizes: "512x512",
            type: "image/png"
          }
        ]
      }
    },
    "gatsby-plugin-offline",
    {
      resolve: "gatsby-plugin-netlify-cms",
      options: {
        modulePath: path.resolve("src/netlifycms/index.js"), // default: undefined
        enableIdentityWidget: true,
        publicPath: "admin",
        htmlTitle: "Content Manager",
        includeRobots: false
      }
    },
    {
      resolve: "gatsby-plugin-feed",
      options: {
        setup(ref) {
          const ret = ref.query.site.siteMetadata.rssMetadata;
          ret.allMarkdownRemark = ref.query.allMarkdownRemark;
          ret.generator = "GatsbyJS Advanced Starter";
          return ret;
        },
        query: `
        {
          site {
            siteMetadata {
              rssMetadata {
                site_url
                feed_url
                title
                description
                image_url
                copyright
              }
            }
          }
        }
      `,
        feeds: [
          {
            serialize(ctx) {
              const { rssMetadata } = ctx.query.site.siteMetadata;
              return ctx.query.allMarkdownRemark.edges.map(edge => ({
                categories: edge.node.frontmatter.tags,
                date: edge.node.fields.date,
                title: edge.node.frontmatter.title,
                description: edge.node.excerpt,
                url: rssMetadata.site_url + edge.node.fields.slug,
                guid: rssMetadata.site_url + edge.node.fields.slug,
                custom_elements: [
                  { "content:encoded": edge.node.html },
                  { author: config.userEmail }
                ]
              }));
            },
            query: `
            {
              allMarkdownRemark(
                limit: 1000,
                sort: { order: DESC, fields: [fields___date] },
              ) {
                edges {
                  node {
                    excerpt
                    html
                    timeToRead
                    fields {
                      slug
                      date
                    }
                    frontmatter {
                      title
                      cover
                      date
                      category
                      tags
                    }
                  }
                }
              }
            }
          `,
            output: config.siteRss,
            title: config.siteRssTitle
          }
        ]
      }
    },
    {
      // This plugin must be placed last in your list of plugins to ensure that it can query all the GraphQL data
      resolve: `gatsby-plugin-algolia`,
      options: {
        appId: process.env.ALGOLIA_APP_ID,
        // Use Admin API key without GATSBY_ prefix, so that the key isn't exposed in the application
        // Tip: use Search API key with GATSBY_ prefix to access the service from within components
        apiKey: process.env.ALGOLIA_API_KEY,
        indexName: process.env.ALGOLIA_INDEX_NAME, // for all queries
        queries: require("./src/utils/algolia-queries"),
        chunkSize: 10000, // default: 1000
        settings: {
          // optional, any index settings
        },
        enablePartialUpdates: true, // default: false
        matchFields: ['slug', 'modified'], // Array<String> default: ['modified']
      },
    },
  ]
};

This is the link to the branch that been created for this issue: https://github.com/GuillouTran/guilloutran.com/commits/algolia

Haroenv commented 4 years ago

Looks like in this line you explicitly remove the id/objectID field: https://github.com/GuillouTran/guilloutran.com/blob/algolia/src/utils/algolia-queries.js#L27

An id is required for this plugin. Normally we throw a readable error in that case though, do you have logs of the failure?

h1eutran commented 4 years ago

@Haroenv oh, j'ai oublié! Merci Beaucoup! Ca marche maintenant. Ma nouvelle configuration:

const escapeStringRegexp = require("escape-string-regexp")

const pageQuery = `
  query LandingQuery {
    allMarkdownRemark(sort: { fields: [fields___date], order: DESC }) {
      edges {
        node {
          id
          fields {
            slug
            date
          }
          excerpt
          timeToRead
          frontmatter {
            title
            tags
            cover
            date
          }
        }
      }
    }
  }
`;
function pageToAlgoliaRecord({ node: { id, frontmatter, fields, ...rest } }) {
    return {
      objectID: id,
      ...frontmatter,
      ...fields,
      ...rest,
    }
  }

const queries = [
    {
        query: pageQuery,
        transformer: ({ data }) => data.allMarkdownRemark.edges.map(pageToAlgoliaRecord), // optional
        indexName: 'prod_GuillouTran', // overrides main index name, optional
    },
    ];

    module.exports = queries
Haroenv commented 4 years ago

Thanks for confirming that it works, but I wonder what error you got before this, did it say anything about a missing objectID?

h1eutran commented 4 years ago

Nope! It doesn't said anything! I tried to replace stuff and swapping things but it just creating more errors. Haven't seen it mention missing objectID

Haroenv commented 4 years ago

I'll leave this open as a marker to check why the error did not catch this

h1eutran commented 4 years ago

Okay! It's strange though because I see that this is the only error I ever stumble across that doesn't have a specifically error message.

richhiggins commented 4 years ago

I'm currently experiencing this for local builds on OSX (High Sierra) - it rarely completes.

We deploy our production app to Heroku and the problem does not occur on the Heroku build servers.

On Heroku it usually completes indexing in around 13 seconds.

h1eutran commented 3 years ago

Have you try and remove dependencies, clear cache and try again

Haroenv commented 3 years ago

I think this individual issue is solved, if you have a similar behaviour, please open an issue with reproduction :)