🤗 [Question]: How to increase ReadBufferSize & WriteBufferSize

[klikpaspor]

Question Description

When make a post request i'm have problem error error when reading response headers: small read buffer. Increase ReadBufferSize. Buffer size=4096, ...

Thanks

Code Snippet (optional)

No response

Checklist:

• [X] I agree to follow Fiber's Code of Conduct.
• [X] I have checked for existing issues that describe my questions prior to opening this one.
• [X] I understand that improperly formatted questions may be closed without explanation.

[gaby]

@klikpaspor That can be achieved by specifying ReadBufferSize in your Fiber Config. Docs here: https://docs.gofiber.io/api/fiber#config

Example:

// Custom config
app := fiber.New(fiber.Config{
    ReadBufferSize: 8192
})

[klikpaspor]

@klikpaspor That can be achieved by specifying ReadBufferSize in your Fiber Config. Docs here: https://docs.gofiber.io/api/fiber#config

Example:

// Custom config
app := fiber.New(fiber.Config{
    ReadBufferSize: 8192
})

package http

import (
    "bufio"
    "bytes"
    "crypto/tls"
    "io"

    "github.com/gofiber/fiber/v2"
    "github.com/sourcegraph/conc/pool"
)

type Http struct {
    Tls       *tls.Config
    Url       string
    Headers   map[string]string
    Useragent string
    Cookies   string
}

func ReadBytes(data []byte) (response []byte, Error error) {
    var dst bytes.Buffer
    pool := pool.New().WithMaxGoroutines(100)
    pool.Go(func() {
        reader := bytes.NewReader(data)
        writer := bufio.NewWriter(&dst)
        _, error := io.Copy(writer, reader)
        if error != nil {
            Error = error
        }

        response = dst.Bytes()
    })
    pool.Wait()

    return
}

func (http *Http) Get() (data, cookie []byte, Error error) {
    client := fiber.AcquireAgent()
    if http.Tls != nil {
        client.TLSConfig(http.Tls)
    }

    if http.Headers != nil {
        for key, value := range http.Headers {
            client.Add(key, value)
        }
    }

    if http.Useragent != "" {
        client.UserAgent(http.Useragent)
    }

    if http.Cookies != "" {
        client.Cookies(http.Cookies)
    }

    request := client.Request()
    request.Header.SetMethod(fiber.MethodGet)
    request.SetRequestURI(http.Url)
    error := client.Parse()
    if error != nil {
        Error = error
    }

    response := fiber.AcquireResponse()
    error = client.Do(request, response)
    if error != nil {
        Error = error
    }

    data, error = ReadBytes(response.Body())
    if error != nil {
        Error = error
    }

    cookie = response.Header.Peek("Set-Cookie")

    return
}

func (http *Http) Post(Body []byte) (data []byte, Error error) {
    client := fiber.AcquireAgent()
    if http.Tls != nil {
        client.TLSConfig(http.Tls)
    }

    if http.Headers != nil {
        for key, value := range http.Headers {
            client.Add(key, value)
        }
    }

    if http.Useragent != "" {
        client.UserAgent(http.Useragent)
    }

    if http.Cookies != "" {
        client.Cookies(http.Cookies)
    }

    request := client.Request()
    request.Header.SetMethod(fiber.MethodPost)
    request.SetRequestURI(http.Url)
    request.SetBody(Body)
    error := client.Parse()
    if error != nil {
        Error = error
    }

    response := fiber.AcquireResponse()
    error = client.Do(request, response)
    if error != nil {
        Error = error
    }

    data, error = ReadBytes(response.Body())
    if error != nil {
        Error = error
    }

    return
}

[ReneWerner87]

you can directly change it on the client

[klikpaspor]

you can directly change it on the client

okey i'm try

[klikpaspor]

after i'm chage like this :

client := fiber.AcquireAgent()
client.ReadBufferSize = 4096 * 2

got error panic: runtime error: invalid memory address or nil pointer dereference [signal 0xc0000005 code=0x1 addr=0x60 pc=0xa7620e]

[ReneWerner87]

from which line?

this is the error you get when you do not make a copy of the content and reuse it

because we use string references https://docs.gofiber.io/#zero-allocation

[klikpaspor]

After I'm add this client.ReadBufferSize = 4096 * 2

[ReneWerner87]

https://docs.gofiber.io/api/client#parse

yes i know, but i suspect that the previous error only masked the current one what are you doing there with the DO method ? the parse method sends the request already and returns the parts of the response

[klikpaspor]

Parse function is only for check connection the url? And DO method is for sending the post request?

[ReneWerner87]

yes right my mistake

can you give me a small reproducible and complete example, then i will try to find the error

[klikpaspor]

This my full custom package http :

package http

import (
    "bufio"
    "bytes"
    "crypto/tls"
    "io"

    "github.com/gofiber/fiber/v2"
    "github.com/sourcegraph/conc/pool"
)

type Http struct {
    Tls       *tls.Config
    Url       string
    Headers   map[string]string
    Useragent string
    Cookies   string
}

func ReadBytes(data []byte) (response []byte, Error error) {
    var dst bytes.Buffer
    pool := pool.New().WithMaxGoroutines(100)
    pool.Go(func() {
        reader := bytes.NewReader(data)
        writer := bufio.NewWriter(&dst)
        _, error := io.Copy(writer, reader)
        if error != nil {
            Error = error
        }

        response = dst.Bytes()
    })
    pool.Wait()

    return
}

func (http *Http) Get() (data, cookie []byte, code int, Error error) {
    client := fiber.AcquireAgent()
    client.ReadBufferSize = 4096 * 2
    if http.Tls != nil {
        client.TLSConfig(http.Tls)
    }

    if http.Headers != nil {
        for key, value := range http.Headers {
            client.Add(key, value)
        }
    }

    if http.Useragent != "" {
        client.UserAgent(http.Useragent)
    }

    if http.Cookies != "" {
        client.Cookies(http.Cookies)
    }

    request := client.Request()
    request.Header.SetMethod(fiber.MethodGet)
    request.SetRequestURI(http.Url)
    error := client.Parse()
    if error != nil {
        Error = error
    }

    response := fiber.AcquireResponse()
    error = client.Do(request, response)
    if error != nil {
        Error = error
    }

    data, error = ReadBytes(response.Body())
    if error != nil {
        Error = error
    }

    cookie = response.Header.Peek("Set-Cookie")
    code = response.StatusCode()

    return
}

func (http *Http) Post(Body []byte) (data []byte, code int, Error error) {
    client := fiber.AcquireAgent()
    client.ReadBufferSize = 4096 * 2
    if http.Tls != nil {
        client.TLSConfig(http.Tls)
    }

    if http.Headers != nil {
        for key, value := range http.Headers {
            client.Add(key, value)
        }
    }

    if http.Useragent != "" {
        client.UserAgent(http.Useragent)
    }

    if http.Cookies != "" {
        client.Cookies(http.Cookies)
    }

    request := client.Request()
    request.Header.SetMethod(fiber.MethodPost)
    request.SetRequestURI(http.Url)
    request.SetBodyRaw(Body)
    error := client.Parse()
    if error != nil {
        Error = error
    }

    response := fiber.AcquireResponse()
    error = client.Do(request, response)
    if error != nil {
        Error = error
    }

    data, error = ReadBytes(response.Body())
    if error != nil {
        Error = error
    }

    code = response.StatusCode()

    return
}

got error if i'm add client.ReadBufferSize = 4096 * 2 but when not add got error error when reading response headers: small read buffer. Increase ReadBufferSize. Buffer size=4096

[klikpaspor]

yes right my mistake

can you give me a small reproducible and complete example, then i will try to find the error

i'm already make changes to fix this problem please check it https://github.com/klikpaspor/fiber/commit/11d4cfa5de9578be65596ebc06197646cc79affd

[erikdubbelboer]

Couple of suggestions.

Your ReadBytes function is way over complicated and should just be just this:

func ReadBytes(data []byte) (response []byte, Error error) {
    response = make([]byte, len(data))
    copy(response, data)
    return
}

You Acquire all kinds of resources but never release them again completely negating any performance benefits.

The panic: runtime error: invalid memory address or nil pointer dereference clearly indicates that you are doing something else wrong like reusing resources you shouldn't.

To be honest if I were you I would just stick with a framework that uses net/http that makes things much simpler. net/http is very fast. You probably don't need the small performance gain from fasthttp/fiber.

[gaby]

@ReneWerner87 This can be closed