search

phuslu / log

Fastest structured logging
MIT License
672 stars 44 forks source link

Panic on using for aws s3 #26

Closed pankajsoni19 closed 3 years ago

pankajsoni19 commented 3 years ago

project code here

http://github.com/pankajsoni19/go-kafka-to-s3


admin@ip-41-2-0-28:~/go-kafka-to-s3$ panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x20 pc=0x7891ed]

goroutine 140 [running]:
github.com/phuslu/log.(*FileWriter).Write(0x0, 0xc000f78000, 0x99, 0xa0, 0x98, 0xc000f78000, 0x18)
    /home/admin/go/pkg/mod/github.com/phuslu/log@v1.0.60/file.go:92 +0x2d
log.(*Logger).Output(0xc000082cd0, 0x2, 0xc0000ae300, 0x80, 0x0, 0x0)
    /usr/local/go/src/log/log.go:181 +0x284
log.(*Logger).Printf(0xc000082cd0, 0xc0000ac1b0, 0x2e, 0xc000150320, 0x1, 0x1)
    /usr/local/go/src/log/log.go:188 +0x7e
github.com/aws/smithy-go/logging.StandardLogger.Logf(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/logging/logger.go:74
github.com/aws/aws-sdk-go-v2/aws/retry.Attempt.logf(...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/retry/middleware.go:59
github.com/aws/aws-sdk-go-v2/aws/retry.Attempt.handleAttempt(0xc7b101, 0xc7bde0, 0xc0000cc420, 0xc04c90, 0xc7b1e0, 0xc000138b40, 0xbd4620, 0xc000138a50, 0xc728a0, 0xc00000e560, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/retry/middleware.go:150 +0x9af
github.com/aws/aws-sdk-go-v2/aws/retry.Attempt.HandleFinalize(0xed7b58701, 0xc7bde0, 0xc0000cc420, 0xc04c90, 0xc7b1e0, 0xc000138a20, 0xbd4620, 0xc000138960, 0xc728a0, 0xc00000e560, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/retry/middleware.go:86 +0x305
github.com/aws/smithy-go/middleware.decoratedFinalizeHandler.HandleFinalize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_finalize.go:200
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding.(*DisableGzip).HandleFinalize(0x121ae80, 0xc7b1e0, 0xc000138a20, 0xbd4620, 0xc000138960, 0xc728a0, 0xc00000e580, 0xc00000e5a0, 0xc000f577d0, 0xc00000e5a0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding@v1.0.0/accept_encoding_gzip.go:67 +0x167
github.com/aws/smithy-go/middleware.decoratedFinalizeHandler.HandleFinalize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_finalize.go:200
github.com/aws/smithy-go/middleware.(*FinalizeStep).HandleMiddleware(0xc0000b2048, 0xc7b1e0, 0xc000138a20, 0xbd4620, 0xc000138960, 0xc728c0, 0xc00000e2a0, 0xa81a664ba2bfe8a1, 0xc76c51a3c24b8b70, 0xb8cb68db0a658c3c, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_finalize.go:114 +0x1b7
github.com/aws/smithy-go/middleware.decoratedHandler.Handle(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/middleware.go:57
github.com/aws/smithy-go/middleware.buildWrapHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:184
github.com/aws/aws-sdk-go-v2/aws/middleware.(*requestUserAgent).HandleBuild(0xc0001500d0, 0xc7b1e0, 0xc000138a20, 0xbd4620, 0xc000138960, 0xc72840, 0xc0001501d0, 0x4461900b0dca02ef, 0x10, 0xb51260, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/middleware/user_agent.go:232 +0x3b7
github.com/aws/smithy-go/middleware.decoratedBuildHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:200
github.com/aws/aws-sdk-go-v2/aws/signer/v4.(*contentSHA256Header).HandleBuild(0x121ae80, 0xc7b1e0, 0xc000138a20, 0xbd4620, 0xc000138960, 0xc72860, 0xc00000e4a0, 0x121ae80, 0xb6e4c0, 0xc0001389f0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/signer/v4/middleware.go:195 +0x1dd
github.com/aws/smithy-go/middleware.decoratedBuildHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:200
github.com/aws/aws-sdk-go-v2/aws/signer/v4.(*computePayloadSHA256).HandleBuild(0x121ae80, 0xc7b1e0, 0xc000138870, 0xbd4620, 0xc000138960, 0xc72860, 0xc00000e4c0, 0x0, 0x0, 0x7f65e4342500, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/signer/v4/middleware.go:156 +0x27c
github.com/aws/smithy-go/middleware.decoratedBuildHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:200
github.com/aws/smithy-go/transport/http.(*ComputeContentLength).HandleBuild(0x121ae80, 0xc7b1e0, 0xc000138870, 0xbd4620, 0xc000138960, 0xc72860, 0xc00000e4e0, 0x237b1360e7940a64, 0xc000f5a160, 0xe91575e0efd69d15, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/transport/http/middleware_content_length.go:55 +0x17f
github.com/aws/smithy-go/middleware.decoratedBuildHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:200
github.com/aws/aws-sdk-go-v2/aws/middleware.ClientRequestID.HandleBuild(0xc7b1e0, 0xc000138870, 0xbd4620, 0xc000138960, 0xc72860, 0xc00000e500, 0x20, 0x20, 0x7f66176217d0, 0xc00000e520, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/middleware/middleware.go:42 +0x1ed
github.com/aws/smithy-go/middleware.decoratedBuildHandler.HandleBuild(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:200
github.com/aws/smithy-go/middleware.(*BuildStep).HandleMiddleware(0xc0000b2040, 0xc7b1e0, 0xc000138870, 0xbd4620, 0xc000138960, 0xc728c0, 0xc00000e2c0, 0xc000f580a0, 0x0, 0x0, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_build.go:114 +0x1b7
github.com/aws/smithy-go/middleware.decoratedHandler.Handle(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/middleware.go:57
github.com/aws/smithy-go/middleware.serializeWrapHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:192
github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations.(*removeBucketFromPathMiddleware).HandleSerialize(0x121ae80, 0xc7b1e0, 0xc000138870, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138960, 0xc72980, 0xc000150140, 0xc0001161b0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/internal/customizations/remove_bucket_middleware.go:27 +0x27a
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations.(*updateEndpoint).HandleSerialize(0xc00000e220, 0xc7b1e0, 0xc000138870, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138960, 0xc72900, 0xc00000e3c0, 0xc000138840, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/internal/customizations/update_endpoint.go:175 +0x213
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/aws-sdk-go-v2/service/internal/s3shared.(*EnableDualstack).HandleSerialize(0xc00000e1e0, 0xc7b1e0, 0xc000138870, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138960, 0xc72900, 0xc00000e3e0, 0x1, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/internal/s3shared@v1.0.1/update_endpoint.go:75 +0x1b1
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/aws-sdk-go-v2/service/s3.(*awsRestxml_serializeOpPutObject).HandleSerialize(0x121ae80, 0xc7b1e0, 0xc000138870, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138720, 0xc72900, 0xc00000e400, 0xc000138840, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/serializers.go:6936 +0x3a9
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/aws-sdk-go-v2/service/s3/internal/customizations.(*processARNResource).HandleSerialize(0xc00000e1c0, 0xc7b1e0, 0xc000138870, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138720, 0xc72900, 0xc00000e420, 0x121ae80, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/internal/customizations/process_arn_resource.go:48 +0x1382
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/aws-sdk-go-v2/service/s3.(*ResolveEndpoint).HandleSerialize(0xc00000e140, 0xc7b1e0, 0xc0001386f0, 0xb3d4e0, 0xc0000aa140, 0xbd4620, 0xc000138720, 0xc72900, 0xc00000e440, 0x7a0b25, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/endpoints.go:111 +0x5c7
github.com/aws/smithy-go/middleware.decoratedSerializeHandler.HandleSerialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:208
github.com/aws/smithy-go/middleware.(*SerializeStep).HandleMiddleware(0xc000150080, 0xc7b1e0, 0xc0001386f0, 0xb3d4e0, 0xc0000aa140, 0xc728c0, 0xc00000e2e0, 0x3e8000081a4, 0x3e8, 0x203000, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_serialize.go:122 +0x1fb
github.com/aws/smithy-go/middleware.decoratedHandler.Handle(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/middleware.go:57
github.com/aws/smithy-go/middleware.initializeWrapHandler.HandleInitialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:184
github.com/aws/aws-sdk-go-v2/service/s3.(*validateOpPutObject).HandleInitialize(0x121ae80, 0xc7b1e0, 0xc0001386f0, 0xb3d4e0, 0xc0000aa140, 0xc72960, 0xc0001500f0, 0x549046, 0xbafa40, 0xc0001386f0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/validators.go:1630 +0xdb
github.com/aws/smithy-go/middleware.decoratedInitializeHandler.HandleInitialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:200
github.com/aws/smithy-go/middleware.(*setLogger).HandleInitialize(0xc0001500c0, 0xc7b1e0, 0xc0001386c0, 0xb3d4e0, 0xc0000aa140, 0xc728e0, 0xc00000e340, 0x121ae80, 0xb6e4c0, 0xc000138690, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/logging.go:45 +0xb5
github.com/aws/smithy-go/middleware.decoratedInitializeHandler.HandleInitialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:200
github.com/aws/aws-sdk-go-v2/aws/middleware.RegisterServiceMetadata.HandleInitialize(0xbe1fde, 0x2, 0xbe20a8, 0x2, 0xc0000b1ee6, 0x9, 0xbe4466, 0x9, 0xc7b1e0, 0xc0001382a0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2@v1.2.0/aws/middleware/metadata.go:40 +0xe7
github.com/aws/smithy-go/middleware.decoratedInitializeHandler.HandleInitialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:200
github.com/aws/aws-sdk-go-v2/service/internal/s3shared.(*ARNLookup).HandleInitialize(0xc0000b2058, 0xc7b1e0, 0xc0001382a0, 0xb3d4e0, 0xc0000aa140, 0xc728e0, 0xc00000e380, 0xc00000e3a0, 0xc000f497c0, 0xc00000e3a0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/internal/s3shared@v1.0.1/arn_lookup.go:39 +0xca
github.com/aws/smithy-go/middleware.decoratedInitializeHandler.HandleInitialize(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:200
github.com/aws/smithy-go/middleware.(*InitializeStep).HandleMiddleware(0xc0000b2038, 0xc7b1e0, 0xc0001382a0, 0xb3d4e0, 0xc0000aa140, 0xc728c0, 0xc00000e300, 0x0, 0xc728c0, 0xc00000e300, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/step_initialize.go:114 +0x1b7
github.com/aws/smithy-go/middleware.decoratedHandler.Handle(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/middleware.go:57
github.com/aws/smithy-go/middleware.(*Stack).HandleMiddleware(0xc0000c8000, 0xc7b1e0, 0xc0001382a0, 0xb3d4e0, 0xc0000aa140, 0xc729a0, 0xc0001500e0, 0x0, 0xc729a0, 0xc0001500e0, ...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/stack.go:109 +0x151
github.com/aws/smithy-go/middleware.decoratedHandler.Handle(...)
    /home/admin/go/pkg/mod/github.com/aws/smithy-go@v1.1.0/middleware/middleware.go:57
github.com/aws/aws-sdk-go-v2/service/s3.(*Client).invokeOperation(0xc000130dc0, 0xc7b160, 0xc0000b0018, 0xbe4466, 0x9, 0xb3d4e0, 0xc0000aa140, 0x0, 0x0, 0x0, ...)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/api_client.go:165 +0x533
github.com/aws/aws-sdk-go-v2/service/s3.(*Client).PutObject(0xc000130dc0, 0xc7b160, 0xc0000b0018, 0xc0000aa140, 0x0, 0x0, 0x0, 0x12, 0xc00045fea0, 0xc00045fed8)
    /home/admin/go/pkg/mod/github.com/aws/aws-sdk-go-v2/service/s3@v1.1.0/api_op_PutObject.go:82 +0xfb
main.upload(0xc000143860, 0xc0000b1eb0, 0xb, 0xc0000ac0f0, 0x27, 0x0, 0x0)
    /home/admin/go-kafka-to-s3/main.go:235 +0x34c
main.setupRotation.func2.1(0xc0000ac060, 0x24, 0xc000143860, 0xc0000b1eb0, 0xb)
    /home/admin/go-kafka-to-s3/main.go:192 +0x172
created by main.setupRotation.func2
    /home/admin/go-kafka-to-s3/main.go:182 +0x1cd
^C
[1]+  Exit 2                  ./go-kafka-to-s3
pankajsoni19 commented 3 years ago

Issue occurs when I upload a file to aws s3. It tries to log something and goes kaput.

I am setting up logger like this

var logWriter *log.FileWriter

func setupAWS(appConfig *Config) {
    clientLogMode := aws.LogRetries | aws.LogRequest | aws.LogResponse

    s3Cfg, err := config.LoadDefaultConfig(context.TODO(),
        config.WithRegion(appConfig.AWSS3.Region),
        config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(appConfig.AWSS3.AccessKey, appConfig.AWSS3.AccessSecret, "")),
        config.WithLogConfigurationWarnings(true),
        config.WithClientLogMode(clientLogMode),
        config.WithLogger(logging.NewStandardLogger(logWriter)))

    s3Client = s3.NewFromConfig(s3Cfg)
}

logWriter confirms to io.Writer so ideally it should work

pankajsoni19 commented 3 years ago

bad syntax from my side. do ignore