Warehouse: Test File Code

[Kleissner]

Tracking code that is not making it into master. Turns out virtual test files come with their own set of problems. If they are not cached in memory, generating them may be slower than sending, stalling and distorting the speed measurement. If they are cached in memory, an attacker could abuse them to potentially cause memory exhaustion.

It is better to deal with real files, and implement a real speed test separately with empty data (zeros).

Test File.go:

/*
File Name:  Test File.go
Copyright:  2021 Peernet Foundation s.r.o.
Author:     Peter Kleissner

Generates test files that can be used for file transfer tests.
Each 8-bytes contain the offset within the file, making any data in the file deterministic.
*/

package warehouse

import (
    "encoding/binary"
    "io"
)

func isTestFile(hashA string) (fileSize uint64, valid bool) {
    switch hashA {
    case "721b983b7cade0c27ea4f3a87f9877c0d7bf1e13bbaf493f3cf886cedb2c0121": // 10 MB test file
        return 10 * 1024 * 1024, true
    case "c2f454cb9ccddfb805f6b53c89ef85f9d7591e7508e8fe317c012ed88f417bf4": // 100 MB test file
        return 100 * 1024 * 1024, true
    }

    return 0, false
}

func readTestFile(offset, limit uint64, writer io.Writer) (status int, bytesRead int64, err error) {
    if limit == 0 {
        return StatusOK, 0, nil
    }

    data := make([]byte, 8)

    // pre
    if offset%8 > 0 {
        binary.LittleEndian.PutUint64(data[0:8], (offset/8)*8)

        start := offset % 8
        end := uint64(8)
        if end-start > limit {
            end = start + limit
        }

        writer.Write(data[start:end])

        size := end - start
        limit -= size
        bytesRead += int64(size)
        offset += size
    }

    bufferLength := 4 * 1024 * 1024
    buffer := make([]byte, bufferLength)
    var offsetB uint64

    for ; offset < offset+limit && limit >= 8; offset += 8 {
        binary.LittleEndian.PutUint64(buffer[offsetB:offsetB+8], offset)
        offsetB += 8

        if offsetB == uint64(len(buffer)) {
            writer.Write(buffer[:offsetB])
            offsetB = 0
        }

        limit -= 8
        bytesRead += 8
    }

    if offsetB > 0 {
        writer.Write(buffer[:offsetB])
    }

    // post
    if limit > 0 {
        binary.LittleEndian.PutUint64(data[0:8], offset)
        start := 0
        end := limit

        writer.Write(data[start:end])

        bytesRead += int64(limit)
    }

    return StatusOK, bytesRead, nil
}

Test File_test.go:

package warehouse

import (
    "bytes"
    "encoding/binary"
    "encoding/hex"
    "fmt"
    "testing"

    "lukechampine.com/blake3"
)

func TestCreateFiles(t *testing.T) {
    targetSize := uint64(10 * 1024 * 1024)
    data := make([]byte, targetSize)

    for offset := uint64(0); offset < targetSize; offset += 8 {
        binary.LittleEndian.PutUint64(data[offset:offset+8], offset)
    }

    hashWriter := blake3.New(hashSize, nil)
    hashWriter.Write(data)
    hash := hashWriter.Sum(nil)

    fmt.Printf("%s\n", hex.EncodeToString(hash))
}

func TestRead(t *testing.T) {
    targetSize := uint64(100)
    buffer := bytes.NewBuffer(make([]byte, 0, targetSize))

    readTestFile(1024, targetSize, buffer)

    fmt.Printf("%s\n", hex.Dump(buffer.Bytes()))
}

The code was referenced here:

func (wh *Warehouse) ReadFile(hash []byte, offset, limit int64, writer io.Writer) (status int, bytesRead int64, err error) {
    // validate the hash and build the path
    // 17.01.2022: This code previously used wh.FileExists which is not performant when used frequently. It is faster to instead catch the file-not-exist error on os.Open.
    hashA, err := ValidateHash(hash)
    if err != nil {
        return StatusInvalidHash, 0, err
    }

    if _, valid := isTestFile(hashA); valid {
        return readTestFile(uint64(offset), uint64(limit), writer)
    }

...

func (wh *Warehouse) FileExists(hash []byte) (path string, fileSize uint64, status int, err error) {
    hashA, err := ValidateHash(hash)
    if err != nil {
        return "", 0, StatusInvalidHash, err
    }

    if fileSize, valid := isTestFile(hashA); valid {
        return "", fileSize, StatusOK, nil
    }