Invalid feeds

[d4tocchini]

After setting up a few sanity testing demos, I often get Invalid Feed errs when more than one authorized peer puts in the same key. Should i tightly manage blocking write access? I understand this is a quickly evolving proj with known dragons, is this a said dragon?

What is the cause & nature of invalidation? Ways to remedy?

Before hyperdb is rolled into hyperdrive, is there a proper strategy to use hyperdb with hyperdrive that could help or at least prepare for the impending future?

Any suggested best-practices, forking/replicating streams vs put/gets?

@mafintosh, massive kudos on all this!

[mafintosh]

Are you reusing the same secret key across multiple peers? You shouldn't do that.

[d4tocchini]

@mafintosh, indeed! I don't believe I am... Included below is one of my sanity test setups, roughly inspired by the first few lines of https://github.com/mafintosh/dat-npm:

Two different node servers ping-pong message via a hyperdb created by owner.js who authorizes the peer running author.js (see source below)

• run node owner on one machine and use the logged key in the other machine that runs node author
• They ping-pong just fine after connecting
• Invalid Feed Errors come up only after stopping & rerunning a few times in a somewhat random matter, for example:
  • run the owner, then the author, after they ping-pong kill both. then run just the owner & then kill it. then run just the owner & then kill it and then it will throw Error: Invalid feed
  • run the owner & then the author. you can kill & rerun the author as long as the owner is running.

/owner.js:

var hyperdb = require('hyperdb')
var hyperdiscovery = require('hyperdiscovery')

function setup (cb) {
  var db = hyperdb('./db', {
    //sparse: true, 
    valueEncoding: 'json'
  })
  db.on('ready', function () {    
    db.swarm = hyperdiscovery(db, {live: true})
    cb(null, db)
  })      
  db.on('error', function(err){
    console.log('db ERR',err)
  })
}

setup( function (err, db) {

  const FEED_KEY = db.key.toString('hex')
  var pongCount = 0
  var activePeerKeys = {}    

  console.log('share this with author:', FEED_KEY)

  db.put('/whiteboard', 'hello world', function (err) {

    db.swarm.on('connection', function (peer, info) {          
      console.log('peer connected')
      peer.on('close', function () { console.log('peer disconnected') })

      if (!peer.remoteUserData) return        
      var peerKey = peer.remoteUserData.toString('hex')    
      if (peerKey === FEED_KEY) return

      if (peerKey && !activePeerKeys[peerKey]) {
        activePeerKeys[peerKey] = true
        db.get('/peer-state/' + peerKey, function (err, node) {
          if (err) {
            console.log('get /peer-state/ ERR', err)
            return
          }
          if (!node) {
            db.authorize(peerKey, function() {
              console.log('authorized peer', peerKey)
              db.put('/peer-state/' + peerKey, 'authorized', function(err) {
                if (err) {
                  console.log('put peer-state ERR', err)
                  return
                }
              })
            })
          }
        })
      }
    })

    startPonging()    

    function startPonging() {
      onceWhiteboardChange(function() {
        pongIfPing(startPonging)
      })
    }
    function pongIfPing(cb) {
      db.get('/whiteboard', function(err, node) {        
        if (err) {
          console.log('get /whiteboard ERR', err)
          return
        }
        if (node && node[0].value.includes('ping')) {
          console.log('whiteboard pong')
          db.put('/whiteboard', 'pong-' + pongCount++, cb) 
        } 
        else {
          cb()
        }          
      })       
    }
    function onceWhiteboardChange(cb) {
      let unwatch = db.watch('/whiteboard', function () {      
        unwatch()
        cb()          
      })
    }         
  })  
})

/author.js:

var hyperdb = require('hyperdb')
var hyperdiscovery = require('hyperdiscovery')

const FEED_KEY = {{INSERT OWNER KEY HERE}}

function setup (cb) {

  var db = hyperdb('./db', FEED_KEY, {
    //sparse: true, 
    valueEncoding: 'json'
  })

  db.on('ready', function () {

    db.swarm = hyperdiscovery(db, {
      id: db.local.key.toString('hex'),
      live: true, upload: true, download: true,
      stream: function (peer) {
        return db.replicate({
          live: true,
          upload: true,
          download: true,
          // testing ability to auto authorize peers via exposed key in userData
          userData: db.local.key
        })
      }
    })    
    cb(null, db)
  })    

  db.on('error', function(err){
    console.log('db ERR',err)
  })
}

setup( function (err, db) {
  let localKey = db.local.key.toString('hex')
  var pingCount = 0
  var isPinging = false

  db.swarm.on('connection', function (peer, info) {
    console.log('peer connected')
    peer.on('close', function () { console.log('peer disconnected') })
    onAuthorize( startPinging )
  })

  // TODO: deauthorize
  var isWatchingAuth = false
  function onAuthorize (cb) {
    if (isWatchingAuth) return
    isWatchingAuth = true        
    db.watch('/peer-state/' + localKey, function(err) {
      if (err) {
        console.log('watch /peer-state/ ERR', err)
        return
      }
      db.get('/peer-state/' + localKey, function(err, node) {
        if (err) {
          console.log('get /peer-state/ ERR', err)
          return
        }
        if (node && node[0] && node[0].value === 'authorized') {
          console.log('WATCH authorized')
          cb()
        }
      })
    })
    // not DRY...
    db.get('/peer-state/' + localKey, function(err, node) {
      console.log('this peer-state =', node && node[0].value)
      if (err) {
        console.log('get /peer-state/ ERR', err)
        return
      }
      if (node && node[0].value === 'authorized') {
        cb()
      }
    })
  }
  function startPinging () {
    if (isPinging) return
    console.log('startPinging')
    isPinging = true
    putPing( function () {
      onPong( function () {
        putPing( function () {
          console.log('pinged a pong')
        })
      })
    })
  }
  function putPing(cb) {
    db.put('/whiteboard', 'ping-' + pingCount++, function (err) {
      console.log('ping', pingCount)
      if (err) {
        console.log('ping ERR', err)
        return
      }
      cb()
    })
  }
  function onPong(cb) {
    db.watch('/whiteboard', function (err) {
      console.log('onPong did change')
      if (err) {
        console.log('watch whiteboard ERR', err)
        return
      }
      db.get('/whiteboard', function(err, node) {
        console.log('==>', node && node[0].value)
        if (node && node[0].value.includes('pong')) cb()
      })
    })
  }
})

[d4tocchini]

On that note, it hasn't been easy navigating the layers of hyperlib nuisances, I'm down to PR some elaborate examples or expanded docs, whatever helps

[d4tocchini]

Commenting out the line that throws in lib/writer makes the demo work as I expected 😬

Not sure about the implications, but when writer.js would have thrown because node.clock.length > self._decodeMap.length , and instead you wait until the authorized peer reconnects then it corrects itself and node.clock.length === self._decodeMap.length and the feed appears to be valid & functional.

Is this a bug where some remote state associated with an authorized writing peer is assumed to be localized?

[mafintosh]

Ah, I see now that we have two errors named "Invalid Feeds" - that was the confusing part for me, sorry. Will try and reproduce.

Sure, any added docs etc is great appreciated!

[scriptjs]

I have confirmed this issue with the above code.

[mafintosh]

Yep, can confirm as well and know what is causing it. Working on a fix.

[scriptjs]

@mafintosh Could you explain what is happening with this. Are there commits towards fixing this?

[mafintosh]

fixed in master