Feed streams not aborting properly

[clehner]

I converted git-ssb-web to run as a scuttlebot plugin, and found that certain pages would not fully load. The page for the global feed (which calls sbot.createFeedStream) and for user feeds (sbot.createUserStream) appear to load their items but then they hang. I did some experimenting with the render pipeline and found that having pull.take() prevents the sink from reading the end || error stream event. I made a test plugin here which demonstrates the issue with some ssb stream operations:

~/.ssb-test/node_modules/test/index.js:

var pull = require('pull-stream')

exports.name = 'test'
exports.version = '1.0.0'
exports.manifest = {}
exports.init = function (sbot) {

  pull(
    sbot.createLogStream({
    }),
    pull.take(1),
    pull.drain(function (msg) {
      console.error('createLogStream/take msg', msg.key)
    }, function (err) {
      console.error('createLogStream/take end', err)
    })
  )

  pull(
    sbot.createLogStream({
      limit: 1
    }),
    pull.drain(function (msg) {
      console.error('createLogStream/limit msg', msg.key)
    }, function (err) {
      console.error('createLogStream/limit end', err)
    })
  )

  pull(
    sbot.createFeedStream({
    }),
    pull.take(1),
    pull.drain(function (msg) {
      console.error('createFeedStream/take msg', msg.key)
    }, function (err) {
      console.error('createFeedStream/take end', err)
    })
  )

  pull(
    sbot.createFeedStream({
      limit: 1
    }),
    pull.drain(function (msg) {
      console.error('createFeedStream/limit msg', msg.key)
    }, function (err) {
      console.error('createFeedStream/limit end', err)
    })
  )

  sbot.whoami(function (err, feed) {
    pull(
      sbot.createUserStream({
        id: feed.id
      }),
      pull.take(1),
      pull.drain(function (link) {
        console.error('createUserStream/take msg', link.key)
      }, function (err) {
        console.error('createUserStream/take end', err)
      })
    )
  })

  sbot.whoami(function (err, feed) {
    pull(
      sbot.createUserStream({
        id: feed.id,
        limit: 1
      }),
      pull.drain(function (link) {
        console.error('createUserStream/limit msg', link.key)
      }, function (err) {
        console.error('createUserStream/limit end', err)
      })
    )
  })
}

$ ssb_appname=ssb-test sbot server
Loading plugins from /home/cel/.ssb-test/node_modules
Loading plugin "test"
Log level: notice
createLogStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createLogStream/limit end null
createLogStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createFeedStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createFeedStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createFeedStream/limit end null
createUserStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createUserStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createUserStream/limit end null

So we see that for the methods createLogStream, createFeedStream, and createUserStream, when the stream output is limited using the limit option, we get the end event, but when pull.take() is in the pipeline, the end event does not get read.

When connecting via ssb-client instead of as a plugin, the end event is read correctly. ~/.ssb-test/node_modules/test/server.js:

var config = require('ssb-config/inject')(process.env.ssb_appname)
var keys = require('ssb-keys')
  .loadOrCreateSync(require('path').join(config.path, 'secret'))
require('ssb-client')(keys, config, function (err, sbot) {
  if (err) throw err
  require('.').init(sbot)
})

$ ssb_appname=ssb-test node server.js
createLogStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createLogStream/take end null
createLogStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createLogStream/limit end null
createFeedStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createFeedStream/take end null
createFeedStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createFeedStream/limit end null
createUserStream/take msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createUserStream/take end null
createUserStream/limit msg %h40Zux4TVPfzHO7lmsOb0MPENVVGOl+xw2cqk75UI+M=.sha256
createUserStream/limit end null

@dominictarr @pfraze any ideas of what might be causing this?

[pfrazee]

Nice writeup. Interesting situation. I have to punt to @dominictarr, maybe it's a bug in pull.take()?

[clehner]

I found the problem in pull-paramap. I will make a test case and PR there.

[pfrazee]

Nice dude

[dominictarr]

@clehner's fix is merged into pull-paramap@1.1.6 (a patch) npm update to get the fix.