tusharmath / Multi-threaded-downloader

A http file downloader made in nodejs
MIT License
195 stars 90 forks source link

Calling DownloadFromMTDFile on a file already complete results in error #157

Open Apteryx0 opened 7 years ago

Apteryx0 commented 7 years ago

I have code that works fine except when resuming a download where all the file is downloaded. I.e. I see something like this returned from the observable returned by DownloadFromMTDFile:

[2016-12-31 16:54:20:0311] [debug] x [ 'fdR$', 10 ]
[2016-12-31 16:54:20:0314] [debug] x [ 'localFileSize$', 263753530 ]
[2016-12-31 16:54:20:0314] [debug] x [ 'metaPosition$', 263749434 ]
[2016-12-31 16:54:20:0316] [debug] x [ 'meta$',
  { range: 3,
    metaWrite: 300,
    mtdPath: 'C:\\Users\\RussellMora\\AppData\\Roaming\\aboutgolf-update-center\\pkgs\\agcourse_pebblebeach2015.1.0.20160708.nupkg.mtd',
    url: 'https://shed.aboutgolf.com/users/unq888999320524/agcourse_pebblebeach2015.1.0.20160708.nupkg',
    path: 'C:\\Users\\RussellMora\\AppData\\Roaming\\aboutgolf-update-center\\pkgs\\agcourse_pebblebeach2015.1.0.20160708.nupkg',
    auth: 
     { user: 'unq888999320524',
       pass: '320524',
       sendImmediately: false },
    totalBytes: 263728954,
    threads: [ [Object], [Object], [Object] ],
    offsets: [ 87909651, 175819302, 263728954 ] } ]
[2016-12-31 16:54:20:0317] [error] x :EmptyError: Sequence contains no elements.

My code is (I think) relatively simple:

            var dloads = mtdfiles.map((mtd) => {
                // See comment above for files.txt download
                return mtd.flatMap(function (x,idx) {
                    common.log.debug("Starting download for " + x);
                    return downloader.DownloadFromMTDFile(x);}).share();
            }).share();
            // For each download stream emitted from dloads....
            var dloadsFin = dloads.flatMap((x,idx) => {
                common.log.debug("Starting dloadsFin for " + idx + "....");
                x.subscribe(
                        function (fd) { common.log.debug('x ' + util.inspect(fd)); },
                        function (err) { common.log.error('x :' + err); },
                        function () { common.log.debug('Completed x '); });
                return Rx.Observable.empty();
            }).share();

I assume that the problem is that the DownloadFromMTDFile tries to return/process HTTP responses, however, there are none (and should be none). I'll take a further look at this to see if I can work out what the root cause is, but my Rx/ramda skills are limits.

Apteryx0 commented 7 years ago

This seems to fix the issue:

diff --git a/src/Utils.js b/src/Utils.js
index 61002f1..4c1b7a1 100644
--- a/src/Utils.js
+++ b/src/Utils.js
@@ -177,7 +177,7 @@ export const FlattenMeta$ = Rx.flatMap((meta) => {
   return MergeMeta(IsValid(TimesCount(GetThreadCount(meta))))
 })
 export const RxThrottleComplete = (window$, $, sh) => {
-  const selector = window => O.merge($.throttle(window, sh), $.last())
+  const selector = window => O.merge($.throttle(window, sh), $.takeLast(1))
   return window$.first().flatMap(selector)
 }
 export const IsCompleted$ = (meta$) => {