Couchapp adds %5C to the start of every file

[mwmwmw]

After creating a new couchapp boiler, and pushing it to the server, I kept getting "file not found" errors.

I figured (and still do) that this is a virgin couch problem.. but I was able to sort out that the reason why the files were not found was because there was an escaped HTML "\" in the filename.

so if I typed in host:5984\myapp_design\myapp\index.html // file not founderror

http://dev.thumperdaffodil.com/couch/2.jpg

if I added a %5c to the url, I'd get the file

host:5984\myapp_design\myapp\%5cindex.html // works

http://dev.thumperdaffodil.com/couch/1.jpg

I'm using the latest version of Node (0.6.7) and the most recent version of Couchapp (0.9.0) and CouchDB version 1.1.1

Any thoughts?

[mwmwmw]

{
   "_id": "_design/intra",
   "_rev": "1-8b3291c3106f0aa8b0ebbaacdf974a4b",
   "rewrites": [
       {
           "from": "/",
           "to": "index.html"
       },
       {
           "from": "/api",
           "to": "../../"
       },
       {
           "from": "/api/*",
           "to": "../../*"
       },
       {
           "from": "/*",
           "to": "*"
       }
   ],
   "views": {
   },
   "validate_doc_update": "function (newDoc, oldDoc, userCtx) {   \n  if (newDoc._deleted === true && userCtx.roles.indexOf('_admin') === -1) {\n    throw \"Only admin can delete documents on this database.\";\n  } \n}",
   "attachments_md5": {
       "": {
           "revpos": 1,
           "md5": "d41d8cd98f00b204e9800998ecf8427e"
       },
       "\\sammy": {
           "revpos": 1,
           "md5": "d41d8cd98f00b204e9800998ecf8427e"
       },
       "\\layout.css": {
           "revpos": 1,
           "md5": "d41d8cd98f00b204e9800998ecf8427e"
       },
       "\\sammy\\plugins": {
           "revpos": 1,
           "md5": "d41d8cd98f00b204e9800998ecf8427e"
       },
       "\\index.html": {
           "revpos": 1,
           "md5": "1c10f75aa6ed81e4018c9d5952404a84"
       },
       "\\site.js": {
           "revpos": 1,
           "md5": "2297fd331f355b1d3d9f7d6d0dd47bcf"
       },
       "\\sammy\\plugins\\sammy.cache.js": {
           "revpos": 1,
           "md5": "a5efb3f6502d5b92a318cb4cfdcc8ded"
       },
       "\\sammy\\plugins\\sammy.data_location_proxy.js": {
           "revpos": 1,
           "md5": "c0a63ef0f53fdb99db36279065ab659c"
       },
       "\\sammy\\plugins\\sammy.ejs.js": {
           "revpos": 1,
           "md5": "b8fb0aa404893fc3c4251553fd9fc134"
       },
       "\\sammy\\plugins\\sammy.form.js": {
           "revpos": 1,
           "md5": "cf989e817d60ea28bc09d7295326e25b"
       },
       "\\sammy\\plugins\\sammy.haml.js": {
           "revpos": 1,
           "md5": "5da014e7fd1a6c9c37e4fe04113b6f19"
       },
       "\\sammy\\plugins\\sammy.json.js": {
           "revpos": 1,
           "md5": "ad0025eebda6b01c4f512610b5a7b5f1"
       },
       "\\sammy\\plugins\\sammy.meld.js": {
           "revpos": 1,
           "md5": "4b57aa753265ad1d7f6d5e8e42a98806"
       },
       "\\sammy\\plugins\\sammy.mustache.js": {
           "revpos": 1,
           "md5": "7860637317f506763a48abae53e0fb78"
       },
       "\\sammy\\plugins\\sammy.path_location_proxy.js": {
           "revpos": 1,
           "md5": "5b2586d7b05490634549a9b06a681a39"
       },
       "\\sammy\\plugins\\sammy.nested_params.js": {
           "revpos": 1,
           "md5": "9dba27635b4762c94fb110cbc112940f"
       },
       "\\sammy\\plugins\\sammy.pure.js": {
           "revpos": 1,
           "md5": "3f1efe82b6732fb1e10bfd1c5cc67549"
       },
       "\\sammy\\plugins\\sammy.storage.js": {
           "revpos": 1,
           "md5": "e7f6a9c824955371d00baef68c84ec88"
       },
       "\\sammy\\plugins\\sammy.template.js": {
           "revpos": 1,
           "md5": "7c15f08d4f545fd42b7cb9e4ba0b2d64"
       },
       "\\sammy\\plugins\\sammy.title.js": {
           "revpos": 1,
           "md5": "f922abe81d6aa903d6d8dfd9c26e54dd"
       },
       "\\jquery-1.4.4.min.js": {
           "revpos": 1,
           "md5": "b98dc4a66b80cce329d9127809a2ff2a"
       },
       "\\sammy\\sammy.js": {
           "revpos": 1,
           "md5": "c5a398f8a0cc398aa3b5dfe286faf96b"
       }
   },
   "_attachments": {
       "": {
           "content_type": "application/octet-stream",
           "revpos": 1,
           "digest": "md5-1B2M2Y8AsgTpgAmY7PhCfg==",
           "length": 0,
           "stub": true
       },
       "\\sammy": {
           "content_type": "application/octet-stream",
           "revpos": 1,
           "digest": "md5-1B2M2Y8AsgTpgAmY7PhCfg==",
           "length": 0,
           "stub": true
       },
       "\\layout.css": {
           "content_type": "text/css",
           "revpos": 1,
           "digest": "md5-3gIs+o2eJiHrXZqziQZqBA==",
           "length": 0,
           "stub": true
       },
       "\\sammy\\plugins": {
           "content_type": "application/octet-stream",
           "revpos": 1,
           "digest": "md5-1B2M2Y8AsgTpgAmY7PhCfg==",
           "length": 0,
           "stub": true
       },
       "\\index.html": {
           "content_type": "text/html",
           "revpos": 1,
           "digest": "md5-b05o/Kyr6cG718qsddvmJA==",
           "length": 815,
           "stub": true
       },
       "\\site.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-z4xMXW0jv57jvQJlYMHD6w==",
           "length": 1876,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.cache.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-2ZEEFXQiN1PcnKE9rOhORw==",
           "length": 3546,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.data_location_proxy.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-uSWLyy+QI5unWuiDYX7QvQ==",
           "length": 2775,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.ejs.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-RzvewaUgpgRnD5iS3r5/Kw==",
           "length": 23304,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.form.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-79Co92c56YUGO1NdK2106w==",
           "length": 9974,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.haml.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-BUieG9FUHadHr7UrVm8XsQ==",
           "length": 16127,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.json.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-GuX/QpElwOt8rUeJlCbyvg==",
           "length": 12582,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.meld.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-GjKyZB+idrk7kXvOJf3U6w==",
           "length": 4394,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.mustache.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-hk3CzdunPvwpoKAn6iq9Ww==",
           "length": 13681,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.path_location_proxy.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-/K6n022CSxpj9uhaIN5nmQ==",
           "length": 795,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.nested_params.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-6f40zJyYANU0pG+yEOZWJA==",
           "length": 3578,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.pure.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-aRFBwEQPX4w2lejWw00RzA==",
           "length": 23731,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.storage.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-3A8xMuA4pdySxSRVA8eLYw==",
           "length": 20830,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.template.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-xyOndYf4jwO7/uXrqwrJyg==",
           "length": 4143,
           "stub": true
       },
       "\\sammy\\plugins\\sammy.title.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-sai1mHGFrfRxoWsELlLaZA==",
           "length": 1704,
           "stub": true
       },
       "\\jquery-1.4.4.min.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-IWzL+gkbAtl/lCwkMzUuiQ==",
           "length": 78601,
           "stub": true
       },
       "\\sammy\\sammy.js": {
           "content_type": "text/javascript",
           "revpos": 1,
           "digest": "md5-DXy4ptGJo3NhU/MGipbfEg==",
           "length": 61422,
           "stub": true
       }
   }
}

[mwmwmw]

Note the double slash in every URL.

[mwmwmw]

Uploading files via Futon works fine. I can build an app in Futon the very very scenic route this way and it works as it should.

[mwmwmw]

So, Iriscouch did the same thing as my local machine.. so it's not a localcouch db problem

http://mwmwmw.iriscouch.com/test/_design/intra/index.html

http://mwmwmw.iriscouch.com/test/_design/intra/%5Cindex.html

I have no idea. I looked through the sourcecode for couchapps and I couldn't determine where these would get added.

[mwmwmw]

Fixed it.

Definitely a windows filepath issue.

I edited Main.Js of Couchapp and added this

f = f.replace(/\\/g, "/");

after this line

f = f.replace(att.root, att.prefix || '');

in the app.push section around Line 185

not sure what is the best way to actually fix this permanently. But it completely works.

[mwmwmw]

Here is my main.js in case someone needs a quick fix for this

var path = require('path')
  , sys = require('util')
  , fs = require('fs')
  , watch = require('watch')
  , request = require('request')
  , crypto = require('crypto')
  , mimetypes = require('./mimetypes')
  , spawn = require('child_process').spawn
  ;

var h = {'content-type':'application/json', 'accept-type':'application/json'}

/**
 * Recursively load directory contents into ddoc
 *
 * It's really convenient to see the main couchapp code in single file,
 * rather than mapped into little files in lots of directories like
 * the python couchapp. But there are definitely cases where we might want 
 * to use some module or another on the server side. This addition
 * loads file contents from a given directory (recursively) into a js 
 * object that can be added to a design document and require()'d in 
 * lists, shows, etc. 
 *
 * Use couchapp.loadFiles() in app.js like this:
 *
 *    ddoc = {
 *        _id: '_design/app'
 *      , views: {}
 *      , ...
 *      , lib: couchapp.loadFiles('./lib')
 *      , vendor: couchapp.loadFiles('./vendor')
 *    }
 *
 * Optionally, pass in operators to process file contents. For example, 
 * generate mustache templates from jade templates.
 *
 * In yourapp/templates/index.jade
 *  
 * !!!5
 * html
 *   head
 *     //- jade locals.title
 *     title!= title
 *   body
 *     .item
 *       //- mustache variable for server-side rendering
 *       h1 {{ heading }}
 *
 * in yourapp/app.js
 * var couchapp = require('couchapp')
 *   , jade = require('jade')
 *   , options = {
 *       , operators: [
 *           function renderJade (content, options) {
 *             var compiler = jade.compile(content);
 *             return compiler(options.locals || {});
 *           }
 *         ]
 *       , locals: { title: 'Now we\'re cookin with gas!' }
 *   };
 *
 * ddoc = { ... };
 * 
 * ddoc.templates = loadFiles(dir, options);
 */

function loadFiles(dir, options) {
  var listings = fs.readdirSync(dir)
    , options = options || { replace: function(data){ return data.replace(/\\\\/g, "/")} }
    , obj = {};

  listings.forEach(function (listing) {
    var file = path.join(dir, listing)
      , prop = listing.split('.')[0] // probably want regexp or something more robust
      , stat = fs.statSync(file);

      if (stat.isFile()) { 
        var content = fs.readFileSync(file).toString();
            console.log(content);
        if (options.operators) {
          options.operators.forEach(function (op) {
            content = op(content, options);
          });
        }
        obj[prop] = content;
      } else if (stat.isDirectory()) {
        obj[listing] = loadFiles(file, options);
      }
  });

  return obj;
}

/**
 * End of patch (also see exports and end of file)
 */

function loadAttachments (doc, root, prefix) {
  doc.__attachments = doc.__attachments || []
  try {
    fs.statSync(root)
  } catch(e) {
    throw e
    throw new Error("Cannot stat file "+root)
  }
  doc.__attachments.push({root:root, prefix:prefix});
}

function copy (obj) {
  var n = {}
  for (i in obj) n[i] = obj[i];
  return n
}

function playSound () {
  spawn("/usr/bin/afplay", ["/System/Library/Sounds/Blow.aiff"]);
}

function createApp (doc, url, cb) {
  var app = {doc:doc}

  app.fds = {};

  app.prepare = function () {
    var p = function (x) {
      for (i in x) {
        if (i[0] != '_') {
          if (typeof x[i] == 'function') {
            x[i] = x[i].toString()
            x[i] = 'function '+x[i].slice(x[i].indexOf('('))
          }
          if (typeof x[i] == 'object') {
            p(x[i])
          }
        }
      }
    }
    p(app.doc);
    app.doc.__attachments = app.doc.__attachments || []
    app.doc.attachments_md5 = app.doc.attachments_md5 || {}
    app.doc._attachments = app.doc._attachments || {}
  }

  var push = function (callback) {
    console.log('Serializing.')
    var doc = copy(app.doc);
    console.log("attachments", doc._attachments);
    doc._attachments = copy(app.doc._attachments)
    delete doc.__attachments;
    var body = JSON.stringify(doc);

    console.log('PUT '+url.replace(/^(https?:\/\/[^@:]+):[^@]+@/, '$1:******@'));
    request({uri:url, method:'PUT', body:body, headers:h}, function (err, resp, body) {
      if (err) throw err;
      if (resp.statusCode !== 201) throw new Error("Could not push document\n"+body)
      app.doc._rev = JSON.parse(body).rev
      console.log('Finished push. '+app.doc._rev)
      playSound();
      request({uri:url, headers:h}, function (err, resp, body) {
        body = JSON.parse(body);
        app.doc._attachments = body._attachments;
        if (callback) callback()
      })
    })
  }

  app.push = function (callback) {
    var revpos
      , pending_dirs = 0
      ;

    console.log('Preparing.')
    var doc = app.current;
    for (i in app.doc) {
      if (i !== '_rev') doc[i] = app.doc[i]
    }
    app.doc = doc;
    app.prepare();
    revpos = app.doc._rev ? parseInt(app.doc._rev.slice(0,app.doc._rev.indexOf('-'))) : 0;

    pending_dirs = app.doc.__attachments.length;
    app.doc.__attachments.forEach(function (att) {
      watch.walk(att.root, {ignoreDotFiles:true}, function (err, files) {
        var pending_files = Object.keys(files).length;
        for (i in files) { (function (f) {
          fs.readFile(f, function (err, data) {

            f = f.replace(att.root, att.prefix || '');
            f = f.replace(/\\/g, "/");

            if (f[0] == '/') f = f.slice(1)
            if (!err) {
              var d = data.toString('base64')
                , md5 = crypto.createHash('md5')
                , mime = mimetypes.lookup(path.extname(f).slice(1))
                ;
              md5.update(d)
              md5 = md5.digest('hex')
              if (app.doc.attachments_md5[f] && app.doc._attachments[f]) {
                if (app.doc._attachments[f].revpos === app.doc.attachments_md5[f].revpos &&
                    app.doc.attachments_md5[f].md5 === md5) {
                  pending_files -= 1;
                  if(pending_files === 0){
                    pending_dirs -= 1;
                    if(pending_dirs === 0){
                      push(callback);
                    }
                  }
                  return; // Does not need to be updated.
                }
              }
              app.doc._attachments[f] = {data:d, content_type:mime};
              app.doc.attachments_md5[f] = {revpos:revpos + 1, md5:md5};
            }
            pending_files -= 1
            if(pending_files === 0){
              pending_dirs -= 1;
              if(pending_dirs === 0){
                push(callback);
              }
            }
          })
        })(i)}
      })
    })
    if (!app.doc.__attachments || app.doc.__attachments.length == 0) push(callback);
  }  

  app.sync = function (callback) {
    // A few notes.
    //   File change events are stored in an array and bundled up in to one write call., 
    // this reduces the amount of unnecessary processing as we get a lof of change events.
    //   The file descriptors are stored and re-used because it cuts down on the number of bad change events.
    //   And finally, we check the md5 and only push when the document is actually been changed.
    //   A lot of crazy workarounds for the fact that we basically get an event every time someone
    // looks funny at the underlying files and even reading and opening fds to check on the file trigger
    // more events.

    app.push(function () {
      var changes = [];
      console.log('Watching files for changes...')
      app.doc.__attachments.forEach(function (att) {
        var pre = att.root
        if (pre[pre.length - 1] !== '/') pre += '/';
        watch.createMonitor(att.root, {ignoreDotFiles:true}, function (monitor) {
          monitor.on("removed", function (f, stat) {
            f = f.replace(pre, '');
            changes.push([null, f]);
          })
          monitor.on("created", function (f, stat) {
            changes.push([f, f.replace(pre, ''), stat]);
          })
          monitor.on("changed", function (f, curr, prev) {
            changes.push([f, f.replace(pre, ''), curr]);
          })
        })
      })
      var check = function () {
        var pending = 0
          , revpos = parseInt(app.doc._rev.slice(0,app.doc._rev.indexOf('-')))
          , dirty = false
          ;
        if (changes.length > 0) {
          changes.forEach(function (change) {
            if (!change[0]) {
              delete app.doc._attachments[change[1]];
              dirty = true;
              console.log("Removed "+change[1]);
            } else {
              pending += 1

              fs.readFile(change[0], function (err, data) {
                var f = change[1]
                  , d = data.toString('base64')
                  , md5 = crypto.createHash('md5')
                  , mime = mimetypes.lookup(path.extname(f).slice(1))
                  ;

                md5.update(d)
                md5 = md5.digest('hex')
                pending -= 1
                if (!app.doc.attachments_md5[f] || (md5 !== app.doc.attachments_md5[f].md5) ) {
                  app.doc._attachments[f] = {data:d, content_type:mime};
                  app.doc.attachments_md5[f] = {revpos:revpos + 1, md5:md5};
                  dirty = true;
                  console.log("Changed "+change[0]);
                }
                if (pending == 0 && dirty) push(function () {dirty = false; setTimeout(check, 50)})
                else if (pending == 0 && !dirty) setTimeout(check, 50)

              })
            }

          })
          changes = []
          if (pending == 0 && dirty) push(function () {dirty = false; setTimeout(check, 50)})
          else if (pending == 0 && !dirty) setTimeout(check, 50)
        } else {
          setTimeout(check, 50);
        }
      }
      setTimeout(check, 50)
    })
  }
  var _id = doc.app ? doc.app._id : doc._id

  if (url.slice(url.length - _id.length) !== _id) url += '/' + _id;

  request({uri:url, headers:h}, function (err, resp, body) {
    if (err) throw err;
    if (resp.statusCode == 404) app.current = {};
    else if (resp.statusCode !== 200) throw new Error("Failed to get doc\n"+body)
    else app.current = JSON.parse(body)
    cb(app)
  })
}

exports.createApp = createApp
exports.loadAttachments = loadAttachments
exports.bin = require('./bin')
exports.loadFiles = loadFiles

[yblee85]

at line of 185, where next line of f = f.replace(att.root, att.prefix || '');

if (f[0] == '/') f = f.slice(1)

==> if (f[0] === '/' || f[0] === '\\') f = f.slice(1)

for some machines f[0] is '\\'