File length defer from sum of chunk length

[botallen]

I am currently trying to create google drive support using NewResourcePieces I am using NewReader to read sequentially from torrent file.

func continueUpload(f *File, srv *ServiceAccount) {
    go func() {
        startOff := 0
        buf := make([]byte, 16*1024*1024)
        for {
            n, err := f.reader.Read(buf)
            if err != nil {
                if err != io.EOF {
                    log.Fatal(err)
                }
                uploadChunk(f, srv, bytes.NewBuffer(buf[:n]), startOff, startOff+n-1)
                f.reader.Close()
                break
            }
            go uploadChunk(f, srv, bytes.NewBuffer(buf), startOff, startOff+n-1)
            startOff += n
        }
        f.done <- true
    }()
}

But the at the end of reading the calculated length startOff+n is not matching original file length given by torrent.File.Length() Tell me if i am doing something wrong

In google drive api, I am giving them file size (torrent.File.Length()) in first metadata request and uploading chunk with Content-Range

func uploadChunk(f *File, srv *ServiceAccount, chunk *bytes.Buffer, startOff int, endOff int) {
    req, _ := http.NewRequest("PUT", f.resumableURI, chunk)
    crange := fmt.Sprintf("bytes %d-%d/%s", startOff, endOff, f.length)
    req.Header.Add("Content-Range", crange)
    log.Println("Uploading", crange)
    resp, err := srv.authClient.Do(req)
    if err != nil {
        log.Fatal(err)
    }
    if !(resp.StatusCode == 200 || resp.StatusCode == 201 || resp.StatusCode == 308) {
        defer resp.Body.Close()
        body, _ := ioutil.ReadAll(resp.Body)
        log.Println("Response code", resp.Status)
        log.Fatalf(string(body))
    }
    log.Println("Response range", resp.Header.Get("Range"))
}

[anacrolix]

Thanks for the report. Are there non-zero length files after the one you're reading above? It looks like Reads on File Readers aren't limited to the end of the file, that logic is missing in anacrolix/torrent.

[anacrolix]

I think something like the following would fix the problem:

diff --git a/reader.go b/reader.go
index b418f5f..85d23d3 100644
--- a/reader.go
+++ b/reader.go
@@ -200,6 +197,9 @@ func (r *reader) readOnceAt(b []byte, pos int64, ctxErr *error) (n int, err erro
        err = io.EOF
        return
    }
+   if int64(len(b)) > r.length-pos {
+       b = b[:r.length-pos]
+   }
    for {
        avail := r.waitAvailable(pos, int64(len(b)), ctxErr)
        if avail == 0 {

[botallen]

I think something like the following would fix the problem:

+ if int64(len(b)) > r.length-pos {
+     b = b[:r.length-pos]
+ }

I tried this but it still differs. But now this time i got less number of bytes from reader (Got total 1139 bytes at EOF from 1652 length file).

Currently i found a workaround that providing file length in initial request for google drive api is optional. So I'm relying on file reader by calculating length based on EOF

Not closing this issue for your reference To reproduce -

package main

import (
    "io"
    "log"
    "sync"

    "github.com/anacrolix/torrent/storage"

    "github.com/anacrolix/missinggo/v2/filecache"
    "github.com/anacrolix/missinggo/v2/resource"
    "github.com/anacrolix/missinggo/x"
    "github.com/anacrolix/torrent"
)

func getStorageProvider() resource.Provider {
    fc, err := filecache.NewCache("torrentcache")
    x.Pie(err)

    fc.SetCapacity(int64(100 << 20))
    return fc.AsResourceProvider()
}

func main() {

    config := torrent.NewDefaultClientConfig()
    config.DefaultStorage = storage.NewResourcePieces(getStorageProvider())

    client, err := torrent.NewClient(config)
    if err != nil {
        log.Fatal(err)
    }

    t, err := client.AddMagnet("magnet:?xt=urn:btih:08ada5a7a6183aae1e09d831df6748d566095a10&dn=Sintel&tr=udp%3A%2F%2Fexplodie.org%3A6969&tr=udp%3A%2F%2Ftracker.coppersurfer.tk%3A6969&tr=udp%3A%2F%2Ftracker.empire-js.us%3A1337&tr=udp%3A%2F%2Ftracker.leechers-paradise.org%3A6969&tr=udp%3A%2F%2Ftracker.opentrackr.org%3A1337&tr=wss%3A%2F%2Ftracker.btorrent.xyz&tr=wss%3A%2F%2Ftracker.fastcast.nz&tr=wss%3A%2F%2Ftracker.openwebtorrent.com&ws=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2F&xs=https%3A%2F%2Fwebtorrent.io%2Ftorrents%2Fsintel.torrent")
    if err != nil {
        log.Fatal(err)
    }

    <-t.GotInfo()
    file := t.Files()[0]
    var wa sync.WaitGroup
    reader := file.NewReader()
    wa.Add(1)
    log.Printf("Assuming %d bytes", file.Length())
    go func() {
        startOff := 0
        buf := make([]byte, 16*1024*1024)
        for {
            n, err := reader.Read(buf)
            if err != nil {
                if err != io.EOF {
                    log.Fatal(err)
                }
                log.Printf("Total %d bytes read", startOff+n)
                reader.Close()
                break
            }
            startOff += n
        }
        wa.Done()
    }()
    wa.Wait()
}

[botallen]

Sorry it was my mistake. I was reading first 512 bytes to sniff the mime-type of file and forgot to seek to 0. This is totally dumb mistake.

[Zerolzj]

I think something like the following would fix the problem:

diff --git a/reader.go b/reader.go
index b418f5f..85d23d3 100644
--- a/reader.go
+++ b/reader.go
@@ -200,6 +197,9 @@ func (r *reader) readOnceAt(b []byte, pos int64, ctxErr *error) (n int, err erro
      err = io.EOF
      return
  }
+ if int64(len(b)) > r.length-pos {
+     b = b[:r.length-pos]
+ }
  for {
      avail := r.waitAvailable(pos, int64(len(b)), ctxErr)
      if avail == 0 {

I want to know why not fix this code logic?