Empty RSS Feed

[jackson-ua]

• podcastify-dir version: 1.7.1
• node version: 12.22.6
• npm version: 6.14.15

Relevant code or config

What you did: Implemented the app similarly to the given example code

What happened: When accessing the xml file directly (after Pocket Casts stated no episodes were in the feed) I looked through and found this as the only item in the rss feed:

<item>
<guid isPermaLink="false">undefined</guid>
<description>
<![CDATA[ ]]>
</description>
<pubDate>Invalid Date</pubDate>
<content:encoded>
<![CDATA[ ]]>
</content:encoded>
<enclosure length="0" type="audio/mpeg" url="http://74.208.35.188:7780/audiobook/undefined/audio.mp3"/>
<itunes:image href="http://74.208.35.188:7780/audiobook/undefined/image"/>
<itunes:summary/>
<itunes:subtitle/>
<itunes:explicit>no</itunes:explicit>
<itunes:episodeType>full</itunes:episodeType>
</item>

Reproduction repository: Here is the full index.js code I am using, port 7780 is open on server:

const path = require('path')
const {startServer} = require('@kentcdodds/podcastify-dir')

startServer({
  title: 'Jackson\'s Audiobooks',
  description: 'The audiobook library of Jackson',
  image: {
    url: 'https://is5-ssl.mzstatic.com/image/thumb/Purple125/v4/85/1c/50/851c505c-692a-b511-ab91-7b2b33e141f1/source/256x256bb.jpg',
    title: 'Jackson Audiobooks',
    link: 'https://is5-ssl.mzstatic.com',
    height: 256,
    width: 256,
  },
  port: 7780,
  directory: '/audiobooks/files',
  users: {jnh: 'moatie34'},
  modifyXmlJs(xmlJs) {
    xmlJs.rss.channel['itunes:author'] = 'Jackson H.'
    xmlJs.rss.channel['itunes:summary'] = 'My Audiobooks!'
    return xmlJs
  },
})

Problem description: No 'episodes' (audiobooks) being added to the feed

[kentcdodds]

Hi @jackson-ua,

I'm afraid I don't have the bandwidth to do much support on this project. What I would do if I were you is I would add console.logs in your node_modules directory so you can get an idea of where the issue happens. For example, right above this line I would add: console.log({files, directory}). If that files is empty ... wait...

Actually I think I know what's wrong. All files are identified by their asin value in the ID3 tags. I've got a local rewrite of this project that I worked on a while ago and I changed their ID to the filepath which works much better.

While you're waiting for me to finish my rewrite, you'll want to edit the ID3 tags to add an asin value that's unique across all files.

[jackson-ua]

Have new information, finally got a debug window to appear, seeing this error:

root@localhost:/audiobooks/files# RangeError [ERR_OUT_OF_RANGE]: The value of "end" is out of range. It mus                                      t be >= 0 && <= 9007199254740991. Received -1
    at new ReadStream (internal/fs/streams.js:107:5)
    at Object.createReadStream (fs.js:1919:10)
    at audio (/home/audiobook_lib/node_modules/@kentcdodds/podcastify-dir/dist/podcast-controller.js:330:25                                      )
    at processTicksAndRejections (internal/process/task_queues.js:97:5)
RangeError [ERR_OUT_OF_RANGE]: The value of "end" is out of range. It must be >= 0 && <= 9007199254740991.                                       Received -1
    at new ReadStream (internal/fs/streams.js:107:5)
    at Object.createReadStream (fs.js:1919:10)
    at audio (/home/audiobook_lib/node_modules/@kentcdodds/podcastify-dir/dist/podcast-controller.js:330:25                                      )
    at processTicksAndRejections (internal/process/task_queues.js:97:5)
RangeError [ERR_OUT_OF_RANGE]: The value of "end" is out of range. It must be >= 0 && <= 9007199254740991.                                       Received -1
    at new ReadStream (internal/fs/streams.js:107:5)
    at Object.createReadStream (fs.js:1919:10)
    at audio (/home/audiobook_lib/node_modules/@kentcdodds/podcastify-dir/dist/podcast-controller.js:330:25                                      )
    at processTicksAndRejections (internal/process/task_queues.js:97:5)
[ERR_OUT_OF_RANGE]: The value of "end" is out of range. It must be >= 0 && <= 9007199254740991. Received -1
[ERR_OUT_OF_RANGE]:: command not found

I believe this is appearing from the .createReadStream call on line 320 of podcast-controller.js

I think the issue may be propagating their through the ternary op on line 301 of the same file, which would imply the size is being listed as '0'.

Still hunting down where the issue is coming from, have doubled checked the metadata of all mp3 files to make sure there isn't any bad data or critical missing fields, none I could find thus far

[jackson-ua]

Thanks for the reply! Was writing the above while yours came through, I am very much a neophyte with nodejs so not sure what an ID3 tag is or asin, if you have any recommendations for self-edification, please send them my way! If not, thanks for the fast reply regardless

[jackson-ua]

The suggested fix seems to have worked!

If anyone else has this issue, simple use the linked Kid3 tag editor and add an asin tag to the Tag 2 section in the ID3 editior by manually typing asin into the text field when clicking the add button. Then just fill in any old unique identifier and should work!

Still having issure with pocket casts, but the rss feed seems to be generating correctly

[jackson-ua]

Using the site https://castfeedvalidator.com/ to debug some of my issues

currently producing the error Found zero items with a valid enclosure tag. Cannot validate. Did not finish feed tests.

[kentcdodds]

What's the feed look like now?

[jackson-ua]

http://jnh:moatie34@74.208.35.188/audiobook/feed.xml

<rss xmlns:atom="http://www.w3.org/2005/Atom" xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:googleplay="http://www.google.com/schemas/play-podcasts/1.0" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" version="2.0">
<channel>
<atom:link href="http://74.208.35.188/audiobook/feed.xml" rel="self" title="MP3 Audio" type="application/rss+xml"/>
<atom:link xmlns="http://www.w3.org/2005/Atom" rel="hub" href="https://pubsubhubbub.appspot.com/"/>
<title>Jackson's Audiobooks</title>
<link>http://74.208.35.188/audiobook/</link>
<description>
<![CDATA[ The audiobook library of Jackson ]]>
</description>
<lastBuildDate>Thu, 16 Sep 2021 22:41:18 GMT</lastBuildDate>
<image>
<url>https://is5-ssl.mzstatic.com/image/thumb/Purple125/v4/85/1c/50/851c505c-692a-b511-ab91-7b2b33e141f1/source/256x256bb.jpg</url>
<title>Jackson Audiobooks</title>
<link>https://is5-ssl.mzstatic.com</link>
<height>256</height>
<width>256</width>
</image>
<generator>http://74.208.35.188/audiobook/</generator>
<itunes:author>Jackson H.</itunes:author>
<itunes:summary>My Audiobooks!</itunes:summary>
</channel>
<item>
<guid isPermaLink="false">cradle1</guid>
<title>Unsouled: Cradle, Volume 1</title>
<description>
<![CDATA[ Section 23 ]]>
</description>
<pubDate>Invalid Date</pubDate>
<author>Will Wight</author>
<content:encoded>
<![CDATA[ Section 23 ]]>
</content:encoded>
<enclosure length="443242324" type="audio/mpeg" url="http://74.208.35.188/audiobook/cradle1/audio.mp3"/>
<itunes:title>Unsouled: Cradle, Volume 1</itunes:title>
<itunes:author>Will Wight</itunes:author>
<itunes:duration>29688.737959183673</itunes:duration>
<itunes:image href="http://74.208.35.188/audiobook/cradle1/image"/>
<itunes:summary>Section 23</itunes:summary>
<itunes:subtitle>Section 23</itunes:subtitle>
<itunes:explicit>no</itunes:explicit>
<itunes:episodeType>full</itunes:episodeType>
</item>
<item>
<guid isPermaLink="false">cradle2</guid>
<title>Soulsmith : Cradle, Book 2</title>
<description>
<![CDATA[ Chapter 22 ]]>
</description>
<pubDate>Invalid Date</pubDate>
<author>Will Wight</author>
<content:encoded>
<![CDATA[ Chapter 22 ]]>
</content:encoded>
<enclosure length="427055950" type="audio/mpeg" url="http://74.208.35.188/audiobook/cradle2/audio.mp3"/>
<itunes:title>Soulsmith : Cradle, Book 2</itunes:title>
<itunes:author>Will Wight</itunes:author>
<itunes:duration>28650.135510204083</itunes:duration>
<itunes:image href="http://74.208.35.188/audiobook/cradle2/image"/>
<itunes:summary>Chapter 22</itunes:summary>
<itunes:subtitle>Chapter 22</itunes:subtitle>
<itunes:explicit>no</itunes:explicit>
<itunes:episodeType>full</itunes:episodeType>
</item>
</rss>

[jackson-ua]

I'm not an expert in xml by any measure but does the enclosure url field need to come first?

[kentcdodds]

Hmmm...... That looks like it's working to me. Have you tried loading it up in a podcast player?

[jackson-ua]

Just tried uploading to the pocket casts submission page

[kentcdodds]

Did it work?

[jackson-ua]

Says unable to find podcast at this URL

[jackson-ua]

In the submission page it says Invalid Feed as it doesn't contain any episodes

[kentcdodds]

It could be a caching issue. Try changing the URL slightly. For example, you could add a query parameter that doesn't impact the functionality: http://jnh:moatie34@74.208.35.188/audiobook/feed.xml?v=2

Your RSS feed looks a lot like mine that I have running through PocketCasts just fine.

[kentcdodds]

It's also possible that it's not happy with the IP address I suppose (I have a domain name for mine). But that seems strange.

[jackson-ua]

Same error! For whatever reason it cannot seem to find any items with a valid enclosure tag, no idea why

[jackson-ua]

I have a domain ready, but have to wait 48 hours for google domains to pass it through

[kentcdodds]

Yeah, at this point I really have no idea. Sorry :(

[jackson-ua]

All good! Is your enclosure tag led with url? That seems to be the only difference between mine and other valid rss feeds

[kentcdodds]

This is what mine looks like

<enclosure length="282557445" type="audio/mpeg" url="http://my-domain/audiobook/B08HKS6EFJK/audio.mp3"/>

[jackson-ua]

gah! no clue why it isn't working then, I've posted a stack overflow question, hopefully someone sees something I don't!

[odi89]

Did you find a fix? I have the exact same problem=)

[odi89]

Sorry for not comming back to you i found the solution. The xml was not beeing parsed right by the modifyXmlJs function. I cant push code to this repo, will show you a very hacky and not polished way that i managed to get this working when i tried it a couple weeks back

Hope this helps=)

podcast-controller.js

import * as fs from 'fs'
import path from 'path'
import logger from 'loglevel'
import {sort} from 'fast-sort'
import * as mm from 'music-metadata'
import convert from 'xml-js'

const atob = data => Buffer.from(data, 'base64').toString()
const arrayify = val => (Array.isArray(val) ? val : [val].filter(Boolean))

function getPodcastMiddleware({
  title: podcastTitle,
  image: podcastImage,
  description: podcastDescription,
  modifyXmlJs = xmlJs => xmlJs,
  directory,
}) {
  let cache = {}
  async function getFileMetadata(id) {
    if (!cache[id]) {
      await loadFileMetadataCache()
    }
    return cache[id]
  }

  async function getFilesMetadata() {
    if (!Object.keys(cache).length) {
      await loadFileMetadataCache()
    }
    return cache
  }

  async function loadFileMetadataCache() {
    const files = await fs.promises.readdir(directory)
    const items = await Promise.all(
      files
        .filter(file => file.endsWith('.mp3'))
        .map(async file => {
          try {
            const filepath = path.join(directory, file)
            const stat = await fs.promises.stat(filepath)
            let metadata
            try {
              metadata = await mm.parseFile(filepath)
            } catch (error) {
              error.stack = `This error means that we couldn't parse the metadata for ${filepath}:\n${error.stack}`
              throw error
            }

            function getNativeValue(nativeId) {
              for (const nativeMetadata of Object.values(metadata.native)) {
                const foundItem = nativeMetadata.find(
                  item => item.id.toLowerCase() === nativeId.toLowerCase(),
                )
                if (foundItem) {
                  if (foundItem.value.text) {
                    return foundItem.value.text
                  } else {
                    return foundItem.value
                  }
                }
              }
              // the value probably doesn't exist...
              return ''
            }

            const json64 = getNativeValue('TXXX:json64')
            let audibleMetadata = {}
            if (json64) {
              try {
                audibleMetadata = JSON.parse(atob(json64))
              } catch {
                // sometimes the json64 data is incomplete for some reason
              }
            }
            console.log(metadata.format.duration)
            const durationInSeconds = metadata.format.duration || 2

            const {
              title = metadata.common.title,
              summary: description = getNativeValue('TXXX:comment') ||
                getNativeValue('COMM:comment'),
              asin: id = metadata.common.asin,
              // author = metadata.common.artist,
              copyright = metadata.common.copyright,
              // duration = metadata.format.duration,
              duration = Math.floor(durationInSeconds / 60),
              narrated_by: narrators = getNativeValue('TXXX:narrated_by'),
              genre: category = getNativeValue('TXXX:book_genre') ||
                getNativeValue('TXXX:genre'),
              // release_date: date = getNativeValue('TXXX:year'),
            } = audibleMetadata

            const {picture: [picture = getNativeValue('APIC')] = []} =
              metadata.common
            console.log(new Date())
            return {
              id,
              title,
              // author,
              pubDate: new Date(),

              description,
              content: description,
              category: category
                ?.split?.(':')
                .map(c => c.trim())
                .filter(Boolean),

              guid: id,

              size: stat.size,
              duration,
              type: `audio/${(
                metadata.format.container || 'mpeg'
              ).toLowerCase()}`,
              picture,
              contributor: narrators
                .split(',')
                .map(name => ({name: name.trim()})),

              copyright,
              filepath,
            }
          } catch (error) {
            logger.error(`Trouble getting metadata for "${file}"`)
            logger.error(error.stack)
            return null
          }
        }),
    )

    cache = {}
    for (const item of items) {
      if (item) {
        cache[item.id] = item
      }
    }
  }

  // eslint-disable-next-line complexity
  async function feed(req, res) {
    const items = Object.values(await getFilesMetadata())

    // filter
    const filteredItems = filterItems({items, query: req.query})

    // sort
    const sortOptions = (req.query.sort ?? 'desc:pubDate')
      .split(',')
      .map(set => {
        const [dir, prop] = set.split(':')
        return {[dir]: i => i[prop]}
      })
    const sortedItems = sort([...filteredItems]).by(sortOptions)

    const xmlObj = {
      _declaration: {_attributes: {version: '1.0', encoding: 'utf-8'}},
      rss: {
        _attributes: {
          version: '2.0',
          'xmlns:atom': 'http://www.w3.org/2005/Atom',
          'xmlns:content': 'http://purl.org/rss/1.0/modules/content/',
          'xmlns:googleplay': 'http://www.google.com/schemas/play-podcasts/1.0',
          'xmlns:itunes': 'http://www.itunes.com/dtds/podcast-1.0.dtd',
        },
        channel: {
          'atom:link': [
            {
              _attributes: {
                href: getResourceUrl('feed.xml'),
                rel: 'self',
                title: 'MP3 Audio',
                type: 'application/rss+xml',
              },
            },
            {
              _attributes: {
                rel: 'hub',
                xmlns: 'http://www.w3.org/2005/Atom',
                href: 'https://pubsubhubbub.appspot.com/',
              },
            },
          ],
          title: req.query.title || podcastTitle,
          link: getResourceUrl(),
          description: {
            _cdata: req._parsedUrl.query
              ? `<p>${podcastDescription}</p>\n\n<p>query: ${req._parsedUrl.query}</p>`
              : podcastDescription,
          },
          lastBuildDate: new Date().toUTCString(),
          image: removeEmpty(
            req.query['image.url']
              ? {
                  link: req.query['image.link'],
                  title: req.query['image.title'],
                  description: req.query['image.description'],
                  height: req.query['image.height'],
                  width: req.query['image.width'],
                  url: req.query['image.url'],
                }
              : podcastImage,
          ),
          generator: getResourceUrl(),

          item: sortedItems.map(item => {
            const {
              id,
              title,
              description,
              pubDate,
              category,
              // author,
              duration,
              size,
              type,
            } = item

            return removeEmpty({
              guid: {_attributes: {isPermaLink: false}, _text: id},
              title,
              description: {_cdata: description},
              pubDate: pubDate.toUTCString(),
              // author,
              category: category.length ? category : null,
              'content:encoded': {_cdata: description},
              enclosure: {
                _attributes: {
                  length: size,
                  type,
                  url: getResourceUrl(`${id}/audio.mp3`),
                },
              },
              'itunes:title': title,
              // 'itunes:author': author,
              'itunes:duration': duration,
              'itunes:image': {
                _attributes: {href: getResourceUrl(`${id}/image`)},
              },
              'itunes:summary': description,
              'itunes:subtitle': description,
              'itunes:explicit': 'no',
              'itunes:episodeType': 'full',
            })
          }),
        },
      },
    }

    res.set('Content-Type', 'text/xml')
    const finalObj = modifyXmlJs(xmlObj)
    // console.log(finalObj)
    res.send(
      convert.js2xml(finalObj, {
        compact: true,
        ignoreComment: true,
        spaces: 2,
      }),
    )

    function getResourceUrl(id = '') {
      const baseUrl = new URL(
        [
          req.secure ? 'https' : 'http',
          '://',
          req.get('host'),
          req.baseUrl,
        ].join(''),
      )

      const resourceUrl = new URL(baseUrl.toString())
      if (!resourceUrl.pathname.endsWith('/')) {
        resourceUrl.pathname = `${resourceUrl.pathname}/`
      }
      if (id.startsWith('/')) {
        id = id.slice(1)
      }
      resourceUrl.pathname = resourceUrl.pathname + id
      return resourceUrl.toString()
    }
  }

  async function image(req, res) {
    const item = await getFileMetadata(req.params.id)
    if (!item) return res.status(404).end()

    const {
      picture: {format, data},
    } = item
    res.set('content-type', format)
    res.end(data, 'binary')
  }

  async function audio(req, res) {
    const item = await getFileMetadata(req.params.id)
    if (!item) return res.status(404).end()

    const {filepath, size} = item

    const range = req.headers.range

    let options
    if (range) {
      const positions = range.replace(/bytes=/, '').split('-')
      const start = parseInt(positions[0], 10)
      const end = positions[1] ? parseInt(positions[1], 10) : size - 1
      const chunksize = end - start + 1

      res.writeHead(206, {
        'Content-Range': `bytes ${start}-${end}/${size}`,
        'Accept-Ranges': 'bytes',
        'Content-Length': chunksize,
        'Content-Type': 'audio/mp3',
      })

      options = {start, end}
    } else {
      res.writeHead(200, {
        'Content-Length': size,
        'Content-Type': 'audio/mp3',
      })
    }

    const stream = fs
      .createReadStream(filepath, options)
      .on('open', () => stream.pipe(res))
      .on('error', err => res.end(err))
      .on('end', () => res.end())
  }

  async function bustCache(req, res) {
    await loadFileMetadataCache()
    res.send('success 🎉')
  }

  return {feed, image, audio, bustCache}
}

function filterItems({items, query}) {
  // filter
  let filteredItems = []
  const filterIns = arrayify(query.filterIn)
  const filterOuts = arrayify(query.filterOut)

  if (filterIns.length) {
    for (const filterIn of filterIns) {
      const filterInOptions = filterIn
        .split(',')
        .filter(Boolean)
        .map(set => {
          const [regexString, prop] = set.split(':')
          return {regex: new RegExp(regexString, 'im'), prop}
        })
      for (const item of items) {
        const matches = filterInOptions.every(({regex, prop}) => {
          let value = item[prop]
          value = typeof value === 'string' ? value : JSON.stringify(value)
          return regex.test(value)
        })
        if (matches) {
          filteredItems.push(item)
        }
      }
    }
  } else {
    filteredItems = items
  }

  for (const filterOut of filterOuts) {
    const filterOutOptions = filterOut
      .split(',')
      .filter(Boolean)
      .map(set => {
        const [regexString, prop] = set.split(':')
        return {regex: new RegExp(regexString, 'im'), prop}
      })
    for (const item of items) {
      const matches = filterOutOptions.every(({regex, prop}) => {
        let value = item[prop]
        value = typeof value === 'string' ? value : JSON.stringify(value)
        return regex.test(value)
      })
      if (matches) {
        filteredItems.splice(filteredItems.indexOf(item), 1)
      }
    }
  }

  return filteredItems
}

function removeEmpty(obj) {
  if (!obj) {
    return obj
  }
  const newObj = {}
  for (const [key, value] of Object.entries(obj)) {
    if (value != null) {
      newObj[key] = value
    }
  }
  return newObj
}

export {getPodcastMiddleware}

/*
eslint
  max-lines-per-function: "off",
  no-inner-declarations: "off",
  consistent-return: "off",
*/