Issue with gulp-sftp uploading files to wrong target directory

[u01jmg3]

When files are uploading, instead of doing:

[gulp] gulp-sftp: Uploaded: gulp\dist\scripts\jquery.waypoints.js => public_html/dist/scripts/jquery.waypoints.js

it's doing (missing out parent dist folder on transfer)

[gulp] gulp-sftp: Uploaded: gulp\dist\scripts\jquery.waypoints.js => public_html/scripts/jquery.waypoints.js

Is there a way to fix this?

Here is my code

var gulp = require('gulp');

var clean = require('gulp-clean'),
    plumber = require('gulp-plumber'),
    changed = require('gulp-changed'),
    urladjuster = require('gulp-css-url-adjuster'),
    minifycss = require('gulp-minify-css'),
    uglifyjs = require('gulp-uglify'),
    imagemin = require('gulp-imagemin'),
    soften = require('gulp-soften'),
    sftp = require('gulp-sftp'),
    gutil = require('gulp-util')
;

var htmlPhpFiles = './**/*.{html,php}';

var src = './src/',
    dist = './dist/';

var srcStyles = src + '**/*.css',
    srcScripts = src + '**/*.js',
    srcImages = src + '**/*.{gif,png}';

var distStyles = dist + '**/*.css',
    distScripts = dist + '**/*.js',
    distImages = dist + '**/*.{gif,png}';

var projectBase = 'gulp';

var host = 'ftp.xxxx.xx.xx',
    auth = 'privateKeyCustom',
    remotePath = 'public_html';

var ignoredDirs = '{secure,src}';

/*------------------------------------------------*/

var onError = function(error){
    gutil.beep();
    console.log(error);
};

/*------------------------------------------------*/

gulp.task('cleancss', function(){
    return gulp.src([dist + '**/*.css'], {read: false})
        .pipe(clean())
    ;
});

gulp.task('cleanjs', function(){
    return gulp.src([dist + '**/*.js'], {read: false})
        .pipe(clean())
    ;
});

gulp.task('cleanimages', function(){
    return gulp.src([dist + '**/*.{gif,png}'], {read: false})
        .pipe(clean())
    ;
});

gulp.task('compilecssremote', function(){
    return gulp.src(srcStyles)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(urladjuster({
            prepend: '/' + project + '/dist/' //based on location of CSS files
        }))
        .pipe(minifycss({keepBreaks: true}))
        .pipe(gulp.dest(dist))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('compilejsremote', function(){
    return gulp.src(srcScripts)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(uglifyjs({outSourceMap: false}))
        .pipe(gulp.dest(dist))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('tabsto4spaces', function(){
    return gulp.src(htmlPhpFiles)
        .pipe(soften(4)) //4 spaces
        .pipe(gulp.dest('./'))
    ;
})

gulp.task('optimages', function(){
    return gulp.src(srcImages)
        .pipe(imagemin({optimizationLevel: 5}))
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('moveotherfiles', function(){
    return gulp.src(['./src/**/*', '!./src/**/*.{css,js,gif,png}'])
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('sftp', function(){
    return gulp.src(['./**/*.{css,js,gif,png,php,eot,svg,ttf,woff}', '!./' + ignoredDirs + '/**/*', '!./gulpfile.js'])
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('mainremote', function(){
    return gulp.start('tabsto4spaces', 'cleancss', 'cleanjs', 'compilecssremote', 'compilejsremote', 'optimages', 'moveotherfiles');
});

/*------------------------------------------------*/

gulp.task('default', ['mainremote', 'sftp']);

[gtg092x]

u01jmg3 - was this addressed previously? I can take a look at this tomorrow.

Edit: Just an initial glance, gulp runs each task asynchronously, so you may have a race condition here. The errors I might have on my hands could be directory or stream based errors, but "mainremote" has no guarantee to finish before "sftp".

If you want to have a task with "mainremote" as a dependency, check out the dependency syntax used along with "The default task" on http://markgoodyear.com/2014/01/getting-started-with-gulp/

[u01jmg3]

gtg092x - no, this issue is still present for me

Okay I've re-evaluated my code based on your advice and am now using run-sequence to ensure minification finishes before my uploading task begins - thank you. This is working out a lot better (the order of tasks firing in my gulp terminal look a lot more logical).

However, I am still facing the issue for which I opened this ticket.

My code has changed so see below (I've tried to remove as much of the bloat as possible).

1. My first task (default) is to clean, minify and finally upload all code to my web server - this works flawlessly.
2. My second task (watchremote) is to watch for code changes - after taking advice from contra on the issue I raised with the gulp GitHub project, I can now successfully upload only the file that has changed once that file has been re-minified. My problem is during this second task instead of uploading from my dist folder to the dist folder on my web server, gulp-sftp is missing out uploading to my web server's dist folder. Any reason why that could be?

e.g. [gulp] gulp-sftp: Uploaded: gulp\dist\scripts\jquery.waypoints.js => public_html/scripts/jquery.waypoints.js

---

var gulp = require('gulp');

var plumber = require('gulp-plumber'),
    changed = require('gulp-changed'),
    urladjuster = require('gulp-css-url-adjuster'),
    minifycss = require('gulp-minify-css'),
    uglifyjs = require('gulp-uglify'),
    imagemin = require('gulp-imagemin'),
    soften = require('gulp-soften'),    
    sftp = require('gulp-sftp'),
    gutil = require('gulp-util'),
    rimraf = require('gulp-rimraf'),
    runSequence = require('run-sequence')
;

var htmlPhpFiles = './**/*.{html,php}';

var src = './src/',
    dist = './dist/';

var srcStyles = src + '**/*.css',
    srcScripts = src + '**/*.js',
    srcImages = src + '**/*.{gif,png}';

var distStyles = dist + '**/*.css',
    distScripts = dist + '**/*.js',
    distImages = dist + '**/*.{gif,png}';

var projectBase = 'gulp';

var host = 'ftp.xxxx.xx.xx',
    auth = 'privateKeyCustom',
    remotePath = 'public_html';

var ignoredDirs = '{secure,src}';

/*------------------------------------------------*/

var onError = function(error){
    gutil.beep();
    console.log(error);
};

/*------------------------------------------------*/

gulp.task('cleancss', function(){
    return gulp.src([dist + '**/*.css'], {read: false})
        .pipe(rimraf())
    ;
});

gulp.task('cleanjs', function(){
    return gulp.src([dist + '**/*.js'], {read: false})
        .pipe(rimraf())
    ;
});

gulp.task('cleanimages', function(){
    return gulp.src([dist + '**/*.{gif,png}'], {read: false})
        .pipe(rimraf())
    ;
});

gulp.task('tabsto4spaces', function(){
    return gulp.src(htmlPhpFiles)
        .pipe(soften(4)) //4 spaces
        .pipe(gulp.dest('./'))
    ;
});

gulp.task('cleanall', function(){
    return gulp.start('tabsto4spaces', 'cleancss', 'cleanjs', 'cleanimages');
});

/*------------------------------------------------*/

gulp.task('compilecsslocal', function(){
    return gulp.src(srcStyles)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(urladjuster({
            prepend: '/' + project'/dist/' //based on location of CSS files
        }))
        .pipe(minifycss({keepBreaks: true}))
        .pipe(livereload(server))
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('compilejslocal', function(){
    return gulp.src(srcScripts)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(uglifyjs({outSourceMap: false}))
        .pipe(livereload(server))
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('compilecssremote', function(){
    return gulp.src(srcStyles)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(urladjuster({
            prepend: '/' + project'/dist/' //based on location of CSS files
        }))
        .pipe(minifycss({keepBreaks: true}))
        .pipe(gulp.dest(dist))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('compilejsremote', function(){
    return gulp.src(srcScripts)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(dist)) //must be dist
        .pipe(uglifyjs({outSourceMap: false}))
        .pipe(gulp.dest(dist))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('reloadhtmlphpandupload', function(){
    return gulp.src(htmlPhpFiles)
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(changed(htmlPhpFiles))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('optimages', function(){
    return gulp.src(srcImages)
        .pipe(imagemin({optimizationLevel: 5}))
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('moveotherfiles', function(){
    return gulp.src(['./src/**/*', '!./src/**/*.{css,js,gif,png}'])
        .pipe(gulp.dest(dist))
    ;
});

gulp.task('sftp', function(){
    return gulp.src(['./**/*.{css,js,gif,png,php,eot,svg,ttf,woff}', '!./' + ignoredDirs + '/**/*', '!./gulpfile.js'])
        .pipe(plumber({
            errorHandler: onError
        }))
        .pipe(sftp({
            host: host,
            auth: auth,
            remotePath: remotePath
        }))
    ;
});

gulp.task('watchremote', function(){
    gulp.watch(htmlPhpFiles, ['reloadhtmlphpandupload']);
    gulp.watch(srcStyles, ['compilecssremote']);
    gulp.watch(srcScripts, ['compilejsremote']);
});

/*------------------------------------------------*/

gulp.task('default', ['cleanall'], function(callback){
    runSequence(['compilecsslocal', 'compilejslocal', 'optimages', 'moveotherfiles'], 'sftp', callback);
});

[u01jmg3]

Let me know if you need any clarification or if you are unsure of how to fix my issue

Thanks

[gtg092x]

u01jmg3 - This issue should be addressed in my latest push and package update. However, I'm not really crazy about the solution used (index.js line 189). If you have any insight on vinyl read streams, I'd prefer to not be creating artificial read streams when some have probably already been spawned.

[gtg092x]

https://github.com/gtg092x/gulp-sftp#optionsprependbase

Edit: I'm removing this functionality in favor of pushing you towards mastering the base option with gulp.src - the usage is:

gulp.src("pattern/**",{base:"base_dir"});

This makes pathing identical to gulp-ftp

[u01jmg3]

gtg092x - thanks for all your work but I am still struggling to get things configured properly

If I pick my compilecssremote task as an example the only addition I can make that works is:

return gulp.src('**/*.css', {base: './src/'})

However, my base directory for this particular task should be my dist folder (it could be different for other tasks) but gulp-sftp comes back as No files uploaded. I think it's the workflow of changing a source CSS file, waiting for it to minify, then uploading the outcome of that which is in a different folder.

And apologies, I have no experience of vinyl read streams!

[gtg092x]

Jonathan, can you put your code up on a repo? (Minus any authentication) I can try and run it to give you feedback.

[u01jmg3]

Appreciate your continued help - see https://github.com/u01jmg3/gulp-sftp-gtg092x

[gtg092x]

One request - can you get a package.json file together so I can download all your dependencies? Run npm init from your project directory and walk through the prompt. Be sure to add existing modules to your dependencies list.

[gtg092x]

Nevermind - looks like you're not using npm in your local directory. I'll give you a package.json for this afterwards.

[u01jmg3]

Nope, I am choosing to run gulp and all dependencies globally as each project I have, will be built using the same gulpfile.

I've just pushed a package.json up as I had already started

[gtg092x]

Ok - I'm getting some unexpected behavior from the node ssh2 plugin when it comes to large files or high memory usage. What ends up happening is unexpected closures of the sftp connections. Which might be affecting your project.

Two things you can do:

• Use streams instead of buffers (reduces memory usage). You can call it like this: gulp.src(['./**/*.{css,js,gif,png,php,eot,svg,ttf,woff}', '!./' + ignoredDirs + '/**/*', '!./gulpfile.js'],{buffer:false}) you can also check out this article if you need to mix streams and buffers: https://medium.com/web-code-junk/a2010c13d3d5
• Be more specific with your directories. Right now your globs load in a ton of files! Use gulp.src(['./dist/**/*.{css,js,gif,png,php,eot,svg,ttf,woff}', '!./' + ignoredDirs + '/**/*', '!./gulpfile.js']) instead of gulp.src(['./**/*.{css,js,gif,png,php,eot,svg,ttf,woff}', '!./' + ignoredDirs + '/**/*', '!./gulpfile.js'])

Finally - make sure you use an absolute path in your remote directory setting. /var/www/public_html or something like that. I'm adding this experience to known issues and will be encouraging people to use streams for large files.

[u01jmg3]

Weird - all the files in my template repo are small so I've not faced an issue with ssh2 struggling with large files.

1. Swapped to use {buffer: false}
2. I'm purposely being generic because this is a template repo but eventually I will be using gulp for a much larger project with many more directories so I don't wish to limit the glob to a specific folder. Is this a bad move?

None of this fixes my original issue when using my watchremote task and getting compilecssremote and compilejsremote to correctly upload changed files to the correct folder (dist) on my web server. The sftp task works fine.

I've tried using an absolute path (/home/x/y/public_html/) but gulp errors out so I simply use public_html as my remotePath:

[gulp] Authenticating with private key.
[gulp] SFTP error or directory exists: Error: No such file home/x/y/public_html/dist
[gulp] SFTP error or directory exists: Error: No such file home/x/y/public_html/dist/fonts

events.js:72
        throw er; // Unhandled 'error' event

[gtg092x]

The error for existing directories isn't a breaking error - it's just a message from the server, your upload will work in that case. I don't know what to expect if you use relative files. YMMV

As for the target directory, how about just changing the remote path? Gulp has a lot of path management built into it, so I shouldn't be overwriting that.

Finally, your issue isn't from large files, I think it's from the huge buffers that result from your generic globs, try to pare them down the best you can. Using your code, I was able to get it to work by having more focused globs.

You're welcome to take a pull request if you want to investigate this further. Also, be sure to bring ssh2 related issues to the author of that plugin, he did great work, but I can't support issues there that are out of my control.

Sent with AquaMail for Android http://www.aqua-mail.com

On May 24, 2014 1:10:30 PM Jonathan Goode notifications@github.com wrote:

Weird - all the files in my template repo are small so I've not faced an issue with ssh2 struggling with large files.

1. Swapped to use {buffer: false}
2. I'm purposely being generic because this is a template repo but eventually I will be using gulp for a much larger project with many more directories so I don't wish to limit the glob to a specific folder. Is this a bad move?

None of this fixes my original issue when using watchremote and getting compilecssremote and compilejsremote to correctly upload changed files to the correct folder (dist) on my web server. The sftp task works fine.

I've tried using an absolute path (/home/x/y/public_html/) but gulp errors out so I simply use public_html as my remotePath:

[gulp] Authenticating with private key.
[gulp] SFTP error or directory exists: Error: No such file 
home/x/x/public_html/dist
[gulp] SFTP error or directory exists: Error: No such file 
home/x/x/public_html/dist/fonts

events.js:72
        throw er; // Unhandled 'error' event

---

Reply to this email directly or view it on GitHub: https://github.com/gtg092x/gulp-sftp/issues/7#issuecomment-44098522

[u01jmg3]

Just to re-confirm - I can upload files without issue so the current config I'm using for gulp-sftp is working for me. I don't have issues with my remotePath being wrong or issues with ssh2 and my globs.

My issue is purely based on getting my watchremote task to correctly upload files to the right target folder on my server. i.e.:

• https://github.com/u01jmg3/gulp-sftp-gtg092x/blob/master/gulpfile.js#L195-199

I can't simply force my target folder to be public_html/dist via remotePath because files won't always be coming from the dist folder.

I'm expecting gulp to push the file from my local dist folder to the corresponding dist folder on my web server but this isn't working in my watchremote task.

[gtg092x]

You'll probably have to change the variable to reflect the new location. Unless that kind of directory management also shows up in gulp ftp, I feel like it's out of scope for this plugin. If I'm not getting it, let me know.

Sent with AquaMail for Android http://www.aqua-mail.com

On May 24, 2014 1:39:02 PM Jonathan Goode notifications@github.com wrote:

Just to re-confirm - I can upload files without issue so the current config I'm using for gulp-sftp is working for me. I don't have issues with my remotePath being wrong or issues with ssh2 and my globs.

My issue is purely based on getting my watchremote task to correctly upload files to the right target folder on my server. i.e.:

• https://github.com/u01jmg3/gulp-sftp-gtg092x/blob/master/gulpfile.js#L195-199

I can't simply force my target folder to be public_html/dist via remotePath because the dist folder won't always be correct. I would expect gulp to push the file from my local dist folder to the corresponding dist folder on my web server but this isn't working in my watchremote task.

---

Reply to this email directly or view it on GitHub: https://github.com/gtg092x/gulp-sftp/issues/7#issuecomment-44099231

[u01jmg3]

Which variable are you talking about changing? remotePath in these to be public_html/dist?:

• https://github.com/u01jmg3/gulp-sftp-gtg092x/blob/master/gulpfile.js#L198
• https://github.com/u01jmg3/gulp-sftp-gtg092x/blob/master/gulpfile.js#L214

I don't understand why when running my sftp task (to upload all required files using my glob pattern) how everything works fine and you get:

>gulp
...
[gulp] gulp-sftp: Uploaded: dist\styles\screen.css => public_html/dist/styles/screen.css

But when running watchremote it uploads to the wrong target dir, e.g.

>gulp watchremote
[gulp] Using gulpfile ~\gulp\gulpfile.js
[gulp] Starting 'watchremote'...
[gulp] Finished 'watchremote' after 221 ms
[gulp] Starting 'compilecssremote'...
[gulp] Authenticating with private key.
[gulp] SFTP Created: public_html/styles
[gulp] gulp-sftp: Uploaded: styles\screen.css => public_html/styles/screen.css
[gulp] gulp-sftp: 1 file uploaded successfully
[gulp] Finished 'compilecssremote' after 1.93 s
[gulp] Connection :: end
[gulp] SFTP :: SFTP session closed
[gulp] Connection :: close

[u01jmg3]

Sorry to bug but could you advise on my previous post? Thanks

[u01jmg3]

I'd really like closure on this - looking to roll this out at my University

[gtg092x]

Sorry for the delay, working a new job at the mo. I'll give you some time tonight, but my free time is zero right now.

Sent with AquaMail for Android http://www.aqua-mail.com

On June 2, 2014 6:13:22 AM Jonathan Goode notifications@github.com wrote:

I'd really like closure on this - looking to roll this out at my University

---

Reply to this email directly or view it on GitHub: https://github.com/gtg092x/gulp-sftp/issues/7#issuecomment-44834963

[yathi]

I have a similar issue with gulp putting all my images in "img/" instead of putting them in "dist/images" like I have configured it to. I am attaching my gulpfile.js Archive.zip

[maartenhunink]

Same issue here. Everything that is in a directory is automatically uploaded in the root directory.