search

Giorgi / DuckDB.NET

Bindings and ADO.NET Provider for DuckDB
https://duckdb.net
MIT License
339 stars 61 forks source link

Memory leak #113

Closed inacionery closed 1 year ago

inacionery commented 1 year ago

A memory leak occurs when running this code with a lambda function on AWS.

using Amazon.Lambda.Core;
using DuckDB.NET.Data;
using System;
using System.Collections.Generic;
using System.IO;

[assembly: LambdaSerializer(typeof(Amazon.Lambda.Serialization.SystemTextJson.DefaultLambdaJsonSerializer))]

public class Function
{
    [STAThread]
    static void Main()
    {
    }
    public object FunctionHandler(object input, ILambdaContext context)
    {
        try
        {
            string duckDBFilePath = "/tmp/data.db";
            int numberOfRecords = 3000000;
            if (!File.Exists(duckDBFilePath))
            {
                using (var connection = new DuckDBConnection($"Data Source={duckDBFilePath}"))
                {
                    connection.Open();
                    using (var createTableCommand = connection.CreateCommand())
                    {
                        createTableCommand.CommandText = "CREATE TABLE DummyData (Field1 VARCHAR, Field2 VARCHAR, Field3 VARCHAR, Field4 VARCHAR, Field5 VARCHAR, Field6 VARCHAR, Field7 VARCHAR, Field8 VARCHAR, Field9 VARCHAR, Field10 VARCHAR)";
                        createTableCommand.ExecuteNonQuery();
                    }
                    using (var transaction = connection.BeginTransaction())
                    {
                        try
                        {
                            using (var insertCommand = connection.CreateCommand())
                            {
                                insertCommand.CommandText = "INSERT INTO DummyData (Field1, Field2, Field3, Field4, Field5, Field6, Field7, Field8, Field9, Field10) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
                                for (int i = 0; i < 10; i++)
                                {
                                    insertCommand.Parameters.Add(new DuckDBParameter());
                                }
                                insertCommand.Prepare();
                                for (int i = 0; i < numberOfRecords; i++)
                                {
                                    for (int j = 0; j < 10; j++)
                                    {
                                        insertCommand.Parameters[j].Value = $"Field_{j + 1}{i}";
                                    }
                                    insertCommand.ExecuteNonQuery();
                                }
                                transaction.Commit();
                            }
                        }
                        catch
                        {
                            transaction.Rollback();
                            throw;
                        }
                    }
                }
            }
            using (var connection = new DuckDBConnection($"Data Source={duckDBFilePath}"))
            {
                connection.Open();
                using (var command = connection.CreateCommand())
                {
                    var list = new List<dynamic>();
                    command.CommandText = "SELECT 'DUPLICATES' as duplicateType, Field5, COUNT(1) FROM DummyData GROUP BY duplicateType, Field5 HAVING COUNT(Field5) > 1";
                    using (var reader = command.ExecuteReader())
                    {
                        while (reader.Read())
                        {
                            list.Add(reader.GetString(0));
                        }
                    }

                    return new
                    {
                        message = $"Successfully inserted {numberOfRecords} dummy records and the duplicate count is {list.Count}"
                    };
                }
            }
        }
        catch (Exception e)
        {
            return new
            {
                message = $"An error occurred: {e.Message}"
            };
        }
    }
}

the output on cloudwatch


INIT_START Runtime Version: dotnet:6.v17    Runtime Version ARN: arn:aws:lambda:us-east-1::runtime:
START RequestId: 21be4e5c-dab7-4e68-a899-07106b8af7ca Version: 114
REPORT RequestId: 21be4e5c-dab7-4e68-a899-07106b8af7ca  Duration: 491851.59 ms  Billed Duration: 491852 ms  Memory Size: 1532 MB    Max Memory Used: 923 MB Init Duration: 332.49 ms    
START RequestId: 807a1bb1-b5e8-485a-9286-5d765a2ca71e Version: 114
REPORT RequestId: 807a1bb1-b5e8-485a-9286-5d765a2ca71e  Duration: 3151.67 ms    Billed Duration: 3152 ms    Memory Size: 1532 MB    Max Memory Used: 1131 MB    
START RequestId: b20f3d0c-0fba-4dab-be10-15bb4a73c399 Version: 114
REPORT RequestId: b20f3d0c-0fba-4dab-be10-15bb4a73c399  Duration: 2607.03 ms    Billed Duration: 2608 ms    Memory Size: 1532 MB    Max Memory Used: 1139 MB    
START RequestId: 84ed71cd-60d8-43fa-a53f-efe7acf0143c Version: 114
REPORT RequestId: 84ed71cd-60d8-43fa-a53f-efe7acf0143c  Duration: 2534.38 ms    Billed Duration: 2535 ms    Memory Size: 1532 MB    Max Memory Used: 1139 MB    
START RequestId: 1fb5f162-166d-4fb5-8989-bfd4a585a156 Version: 114
REPORT RequestId: 1fb5f162-166d-4fb5-8989-bfd4a585a156  Duration: 2522.53 ms    Billed Duration: 2523 ms    Memory Size: 1532 MB    Max Memory Used: 1139 MB    
START RequestId: aa71bf03-126e-4674-873b-5e20d7393dfd Version: 114
REPORT RequestId: aa71bf03-126e-4674-873b-5e20d7393dfd  Duration: 2974.45 ms    Billed Duration: 2975 ms    Memory Size: 1532 MB    Max Memory Used: 1144 MB    
START RequestId: ce1b3bbc-25e0-41e8-884e-8bfa4ac7c814 Version: 114
REPORT RequestId: ce1b3bbc-25e0-41e8-884e-8bfa4ac7c814  Duration: 3970.48 ms    Billed Duration: 3971 ms    Memory Size: 1532 MB    Max Memory Used: 1220 MB    
START RequestId: bd16deb2-90be-4810-8867-53ab6052bdef Version: 114
REPORT RequestId: bd16deb2-90be-4810-8867-53ab6052bdef  Duration: 2667.78 ms    Billed Duration: 2668 ms    Memory Size: 1532 MB    Max Memory Used: 1220 MB    
START RequestId: 26ff836d-d9b7-43f8-ac45-447c4a90ed05 Version: 114
REPORT RequestId: 26ff836d-d9b7-43f8-ac45-447c4a90ed05  Duration: 3101.63 ms    Billed Duration: 3102 ms    Memory Size: 1532 MB    Max Memory Used: 1220 MB    
START RequestId: 915b1872-5307-4487-a3a3-4fd18a3b345e Version: 114
REPORT RequestId: 915b1872-5307-4487-a3a3-4fd18a3b345e  Duration: 2695.14 ms    Billed Duration: 2696 ms    Memory Size: 1532 MB    Max Memory Used: 1220 MB    
START RequestId: 60c5ff27-646e-40f3-88f7-47ecce99cbf1 Version: 114
REPORT RequestId: 60c5ff27-646e-40f3-88f7-47ecce99cbf1  Duration: 2834.73 ms    Billed Duration: 2835 ms    Memory Size: 1532 MB    Max Memory Used: 1220 MB    
START RequestId: 751c1496-b182-460a-8d1d-49e521aaf3d6 Version: 114
REPORT RequestId: 751c1496-b182-460a-8d1d-49e521aaf3d6  Duration: 2893.63 ms    Billed Duration: 2894 ms    Memory Size: 1532 MB    Max Memory Used: 1309 MB    
START RequestId: d8409179-3d0a-4917-ac0c-0cd8082bae8e Version: 114
REPORT RequestId: d8409179-3d0a-4917-ac0c-0cd8082bae8e  Duration: 3253.93 ms    Billed Duration: 3254 ms    Memory Size: 1532 MB    Max Memory Used: 1494 MB    
START RequestId: 60c3475a-c354-4c0c-8e01-f3e9f41f6e13 Version: 114
REPORT RequestId: 60c3475a-c354-4c0c-8e01-f3e9f41f6e13  Duration: 2709.59 ms    Billed Duration: 2710 ms    Memory Size: 1532 MB    Max Memory Used: 1494 MB    
START RequestId: d51ca47b-b9aa-4936-a3c9-31411d0d96f6 Version: 114
REPORT RequestId: d51ca47b-b9aa-4936-a3c9-31411d0d96f6  Duration: 3085.24 ms    Billed Duration: 3086 ms    Memory Size: 1532 MB    Max Memory Used: 1532 MB    
START RequestId: a706751d-9440-428d-a425-1c1db7b46ebe Version: 114
REPORT RequestId: a706751d-9440-428d-a425-1c1db7b46ebe  Duration: 2711.09 ms    Billed Duration: 2712 ms    Memory Size: 1532 MB    Max Memory Used: 1532 MB    
START RequestId: 4ccf486f-ec74-4ad7-ac06-569ed6286ec0 Version: 114
REPORT RequestId: 4ccf486f-ec74-4ad7-ac06-569ed6286ec0  Duration: 2525.58 ms    Billed Duration: 2526 ms    Memory Size: 1532 MB    Max Memory Used: 1532 MB    
START RequestId: eb4cdfd4-7b36-4346-b76f-57f14abda551 Version: 114
RequestId: eb4cdfd4-7b36-4346-b76f-57f14abda551 Error: Runtime exited with error: signal: killed
Runtime.ExitError
Giorgi commented 1 year ago

@inacionery Can you try the latest preview version of the package? https://github.com/Giorgi/DuckDB.NET/pkgs/nuget/DuckDB.NET.Data.Full

inacionery commented 1 year ago

Hey @Giorgi,

I had to remove the insertCommand.Prepare(); and reduce the memory limit, but the error persists. I believe the issue is in the query execution, something is getting stuck in memory and the garbage collector is unable to clean it.

INIT_START Runtime Version: dotnet:6.v17    Runtime Version ARN: arn:aws:lambda:us-east-1::runtime:c3ade6bc354f465cf7a9bb7193500f1ec1c23b10c575d100dd086311be8854c1
START RequestId: c02c9ab8-3fe5-4b5e-afa5-ae46dbd4badc Version: 163
END RequestId: c02c9ab8-3fe5-4b5e-afa5-ae46dbd4badc
REPORT RequestId: c02c9ab8-3fe5-4b5e-afa5-ae46dbd4badc  Duration: 8029.28 ms    Billed Duration: 8030 ms    Memory Size: 1024 MB    Max Memory Used: 731 MB Init Duration: 313.37 ms    
START RequestId: 0f96cb5f-8ba7-431a-b471-bbe31d064ff3 Version: 163
END RequestId: 0f96cb5f-8ba7-431a-b471-bbe31d064ff3
REPORT RequestId: 0f96cb5f-8ba7-431a-b471-bbe31d064ff3  Duration: 3980.03 ms    Billed Duration: 3981 ms    Memory Size: 1024 MB    Max Memory Used: 895 MB 
START RequestId: 2c52edf7-21db-44ac-8b11-9d1933b5f27d Version: 163
END RequestId: 2c52edf7-21db-44ac-8b11-9d1933b5f27d
REPORT RequestId: 2c52edf7-21db-44ac-8b11-9d1933b5f27d  Duration: 3600.75 ms    Billed Duration: 3601 ms    Memory Size: 1024 MB    Max Memory Used: 895 MB 
START RequestId: 84153678-7ef5-45af-b01a-029782c5ca83 Version: 163
END RequestId: 84153678-7ef5-45af-b01a-029782c5ca83
REPORT RequestId: 84153678-7ef5-45af-b01a-029782c5ca83  Duration: 3970.58 ms    Billed Duration: 3971 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: 5b8f6b1b-a08d-47f5-a75a-a1c8eba72e68 Version: 163
END RequestId: 5b8f6b1b-a08d-47f5-a75a-a1c8eba72e68
REPORT RequestId: 5b8f6b1b-a08d-47f5-a75a-a1c8eba72e68  Duration: 3709.19 ms    Billed Duration: 3710 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: f5f2a774-0ff1-4a24-b13b-4e28a31268e9 Version: 163
END RequestId: f5f2a774-0ff1-4a24-b13b-4e28a31268e9
REPORT RequestId: f5f2a774-0ff1-4a24-b13b-4e28a31268e9  Duration: 3586.26 ms    Billed Duration: 3587 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: bfd82831-5b2a-49d5-ab2d-5d5a4228496f Version: 163
END RequestId: bfd82831-5b2a-49d5-ab2d-5d5a4228496f
REPORT RequestId: bfd82831-5b2a-49d5-ab2d-5d5a4228496f  Duration: 3680.81 ms    Billed Duration: 3681 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: 81324391-d8cf-4a6d-b2ae-6eadeb1fbb31 Version: 163
END RequestId: 81324391-d8cf-4a6d-b2ae-6eadeb1fbb31
REPORT RequestId: 81324391-d8cf-4a6d-b2ae-6eadeb1fbb31  Duration: 3626.74 ms    Billed Duration: 3627 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: d5eef24d-7ed2-4800-a368-5fcc3ea22e57 Version: 163
END RequestId: d5eef24d-7ed2-4800-a368-5fcc3ea22e57
REPORT RequestId: d5eef24d-7ed2-4800-a368-5fcc3ea22e57  Duration: 3732.62 ms    Billed Duration: 3733 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: f6e59f4e-00a9-467e-98e5-fadb1e508726 Version: 163
END RequestId: f6e59f4e-00a9-467e-98e5-fadb1e508726
REPORT RequestId: f6e59f4e-00a9-467e-98e5-fadb1e508726  Duration: 3526.59 ms    Billed Duration: 3527 ms    Memory Size: 1024 MB    Max Memory Used: 903 MB 
START RequestId: bc213089-51d9-4165-b54e-e9fdbcbe44e0 Version: 163
END RequestId: bc213089-51d9-4165-b54e-e9fdbcbe44e0
REPORT RequestId: bc213089-51d9-4165-b54e-e9fdbcbe44e0  Duration: 4001.20 ms    Billed Duration: 4002 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 49f63748-9523-4ae8-9ac4-278e98115a19 Version: 163
END RequestId: 49f63748-9523-4ae8-9ac4-278e98115a19
REPORT RequestId: 49f63748-9523-4ae8-9ac4-278e98115a19  Duration: 3611.09 ms    Billed Duration: 3612 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 995b51d2-c4ac-4a80-b5ee-772aa2cf966c Version: 163
END RequestId: 995b51d2-c4ac-4a80-b5ee-772aa2cf966c
REPORT RequestId: 995b51d2-c4ac-4a80-b5ee-772aa2cf966c  Duration: 3645.31 ms    Billed Duration: 3646 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 0803e270-8dbc-464c-bae3-59f7c5ff3951 Version: 163
END RequestId: 0803e270-8dbc-464c-bae3-59f7c5ff3951
REPORT RequestId: 0803e270-8dbc-464c-bae3-59f7c5ff3951  Duration: 3744.64 ms    Billed Duration: 3745 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: bb474391-baca-4de0-90e7-f131c37e2f00 Version: 163
END RequestId: bb474391-baca-4de0-90e7-f131c37e2f00
REPORT RequestId: bb474391-baca-4de0-90e7-f131c37e2f00  Duration: 3704.37 ms    Billed Duration: 3705 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 3c902642-52aa-4078-a928-e1a6eb151362 Version: 163
END RequestId: 3c902642-52aa-4078-a928-e1a6eb151362
REPORT RequestId: 3c902642-52aa-4078-a928-e1a6eb151362  Duration: 3750.01 ms    Billed Duration: 3751 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: f73423ba-017f-4c76-b9d4-6f71ee9f507a Version: 163
END RequestId: f73423ba-017f-4c76-b9d4-6f71ee9f507a
REPORT RequestId: f73423ba-017f-4c76-b9d4-6f71ee9f507a  Duration: 4008.25 ms    Billed Duration: 4009 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 7c47095c-772a-4f1e-b6c3-ba634238a807 Version: 163
END RequestId: 7c47095c-772a-4f1e-b6c3-ba634238a807
REPORT RequestId: 7c47095c-772a-4f1e-b6c3-ba634238a807  Duration: 3909.80 ms    Billed Duration: 3910 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 73d09ebd-75d6-4de9-bc19-ef932f14d95e Version: 163
END RequestId: 73d09ebd-75d6-4de9-bc19-ef932f14d95e
REPORT RequestId: 73d09ebd-75d6-4de9-bc19-ef932f14d95e  Duration: 3563.89 ms    Billed Duration: 3564 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: d1c9fccd-58fe-49da-9efa-a67f1412155a Version: 163
END RequestId: d1c9fccd-58fe-49da-9efa-a67f1412155a
REPORT RequestId: d1c9fccd-58fe-49da-9efa-a67f1412155a  Duration: 4123.47 ms    Billed Duration: 4124 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 1c74520b-f9b8-4e35-9144-13999fcabc08 Version: 163
END RequestId: 1c74520b-f9b8-4e35-9144-13999fcabc08
REPORT RequestId: 1c74520b-f9b8-4e35-9144-13999fcabc08  Duration: 34534.57 ms   Billed Duration: 34535 ms   Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: d8dc1f63-2534-4ade-b1fd-6dda028389d4 Version: 163
END RequestId: d8dc1f63-2534-4ade-b1fd-6dda028389d4
REPORT RequestId: d8dc1f63-2534-4ade-b1fd-6dda028389d4  Duration: 4160.39 ms    Billed Duration: 4161 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: e1a894a3-d98a-40dd-8b6f-ebaae288ecb0 Version: 163
END RequestId: e1a894a3-d98a-40dd-8b6f-ebaae288ecb0
REPORT RequestId: e1a894a3-d98a-40dd-8b6f-ebaae288ecb0  Duration: 3586.53 ms    Billed Duration: 3587 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 3ef05a79-cf7e-44e4-8b81-5810d5976ea8 Version: 163
END RequestId: 3ef05a79-cf7e-44e4-8b81-5810d5976ea8
REPORT RequestId: 3ef05a79-cf7e-44e4-8b81-5810d5976ea8  Duration: 3741.85 ms    Billed Duration: 3742 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 26439556-c300-4ef6-af70-b46990773e1b Version: 163
END RequestId: 26439556-c300-4ef6-af70-b46990773e1b
REPORT RequestId: 26439556-c300-4ef6-af70-b46990773e1b  Duration: 4337.59 ms    Billed Duration: 4338 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 94430207-4c64-4216-9add-f5e6c5695f80 Version: 163
END RequestId: 94430207-4c64-4216-9add-f5e6c5695f80
REPORT RequestId: 94430207-4c64-4216-9add-f5e6c5695f80  Duration: 4230.03 ms    Billed Duration: 4231 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 5d1eee5c-7f64-4633-806c-499ad916f1bd Version: 163
RequestId: 5d1eee5c-7f64-4633-806c-499ad916f1bd Error: Runtime exited with error: signal: killed
Runtime.ExitError
END RequestId: 5d1eee5c-7f64-4633-806c-499ad916f1bd
REPORT RequestId: 5d1eee5c-7f64-4633-806c-499ad916f1bd  Duration: 3527.95 ms    Billed Duration: 3528 ms    Memory Size: 1024 MB    Max Memory Used: 1025 MB

I saved the database file in S3 to speed up the execution, but the code to create it is the same as in the example.

string duckDBFilePath = "/tmp/data.db";
if (!File.Exists(duckDBFilePath))
{
    using (var s3Client = new AmazonS3Client())
    {
        using (var fileTransferUtility = new TransferUtility(s3Client))
        {
            fileTransferUtility.Download(duckDBFilePath, "my-s3-bucket", "data.db");
        }
    }
}
Giorgi commented 1 year ago

I found one object that wasn't disposed of but it shouldn't have taken so much memory. I'm pretty confident that I'm not leaking a native object because when I forget to release some native item my tests fail because deleting the db file fails.

Can you try 0.8.1-alpha.19?

inacionery commented 1 year ago

Hey @Giorgi, same behavior

INIT_START Runtime Version: dotnet:6.v17    Runtime Version ARN: arn:aws:lambda:us-east-1::runtime:c3ade6bc354f465cf7a9bb7193500f1ec1c23b10c575d100dd086311be8854c1
START RequestId: 5a8781c6-37fd-4afb-90c1-e53bc515ed07 Version: 164
END RequestId: 5a8781c6-37fd-4afb-90c1-e53bc515ed07
REPORT RequestId: 5a8781c6-37fd-4afb-90c1-e53bc515ed07  Duration: 8095.59 ms    Billed Duration: 8096 ms    Memory Size: 1024 MB    Max Memory Used: 727 MB Init Duration: 310.49 ms    
START RequestId: c4f1d691-5429-40d4-9399-9ca3761ea351 Version: 164
END RequestId: c4f1d691-5429-40d4-9399-9ca3761ea351
REPORT RequestId: c4f1d691-5429-40d4-9399-9ca3761ea351  Duration: 4054.97 ms    Billed Duration: 4055 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 84852c4e-5b10-4683-8a16-fd6665160ef8 Version: 164
END RequestId: 84852c4e-5b10-4683-8a16-fd6665160ef8
REPORT RequestId: 84852c4e-5b10-4683-8a16-fd6665160ef8  Duration: 3564.35 ms    Billed Duration: 3565 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 71ba295e-ed21-4ac0-b8d4-06dfb952ef46 Version: 164
END RequestId: 71ba295e-ed21-4ac0-b8d4-06dfb952ef46
REPORT RequestId: 71ba295e-ed21-4ac0-b8d4-06dfb952ef46  Duration: 3627.61 ms    Billed Duration: 3628 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 8a5d9784-e67b-4fcb-96d3-8f91b555ef75 Version: 164
END RequestId: 8a5d9784-e67b-4fcb-96d3-8f91b555ef75
REPORT RequestId: 8a5d9784-e67b-4fcb-96d3-8f91b555ef75  Duration: 3795.66 ms    Billed Duration: 3796 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 00bd47cc-9852-49d5-8227-10abab03c4ea Version: 164
END RequestId: 00bd47cc-9852-49d5-8227-10abab03c4ea
REPORT RequestId: 00bd47cc-9852-49d5-8227-10abab03c4ea  Duration: 3505.47 ms    Billed Duration: 3506 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 14c70942-4952-42c8-9ef2-cc32a2b55cd4 Version: 164
END RequestId: 14c70942-4952-42c8-9ef2-cc32a2b55cd4
REPORT RequestId: 14c70942-4952-42c8-9ef2-cc32a2b55cd4  Duration: 3601.55 ms    Billed Duration: 3602 ms    Memory Size: 1024 MB    Max Memory Used: 896 MB 
START RequestId: 76219c45-34ef-4d19-aa7a-dc52547d6699 Version: 164
END RequestId: 76219c45-34ef-4d19-aa7a-dc52547d6699
REPORT RequestId: 76219c45-34ef-4d19-aa7a-dc52547d6699  Duration: 3998.82 ms    Billed Duration: 3999 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: c1cff540-c4cf-41e4-a617-88a416648cf9 Version: 164
END RequestId: c1cff540-c4cf-41e4-a617-88a416648cf9
REPORT RequestId: c1cff540-c4cf-41e4-a617-88a416648cf9  Duration: 3594.33 ms    Billed Duration: 3595 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 75096780-6396-4888-a644-c0641668e4c2 Version: 164
END RequestId: 75096780-6396-4888-a644-c0641668e4c2
REPORT RequestId: 75096780-6396-4888-a644-c0641668e4c2  Duration: 3643.83 ms    Billed Duration: 3644 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 804477fc-dc74-4af0-8a05-bc33482abd57 Version: 164
END RequestId: 804477fc-dc74-4af0-8a05-bc33482abd57
REPORT RequestId: 804477fc-dc74-4af0-8a05-bc33482abd57  Duration: 3693.11 ms    Billed Duration: 3694 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 77e1e84e-0b93-47a1-9536-3ee98b06b4d3 Version: 164
END RequestId: 77e1e84e-0b93-47a1-9536-3ee98b06b4d3
REPORT RequestId: 77e1e84e-0b93-47a1-9536-3ee98b06b4d3  Duration: 3782.43 ms    Billed Duration: 3783 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 337221f9-6d83-4656-86aa-98ab949f88ce Version: 164
END RequestId: 337221f9-6d83-4656-86aa-98ab949f88ce
REPORT RequestId: 337221f9-6d83-4656-86aa-98ab949f88ce  Duration: 3560.14 ms    Billed Duration: 3561 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 358ae92f-0894-4b7b-9790-a2bcc6c46211 Version: 164
END RequestId: 358ae92f-0894-4b7b-9790-a2bcc6c46211
REPORT RequestId: 358ae92f-0894-4b7b-9790-a2bcc6c46211  Duration: 22181.15 ms   Billed Duration: 22182 ms   Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: bf5e04d9-1803-4cec-b257-34f989996f74 Version: 164
END RequestId: bf5e04d9-1803-4cec-b257-34f989996f74
REPORT RequestId: bf5e04d9-1803-4cec-b257-34f989996f74  Duration: 4295.69 ms    Billed Duration: 4296 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 37ff5238-e44c-462b-917a-8162fabb5de4 Version: 164
END RequestId: 37ff5238-e44c-462b-917a-8162fabb5de4
REPORT RequestId: 37ff5238-e44c-462b-917a-8162fabb5de4  Duration: 4313.28 ms    Billed Duration: 4314 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 375d50a0-f5e1-42a6-a1ad-40d48e5867b5 Version: 164
END RequestId: 375d50a0-f5e1-42a6-a1ad-40d48e5867b5
REPORT RequestId: 375d50a0-f5e1-42a6-a1ad-40d48e5867b5  Duration: 3525.13 ms    Billed Duration: 3526 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: c566cf77-8c06-44e8-846e-9821a0320eb5 Version: 164
END RequestId: c566cf77-8c06-44e8-846e-9821a0320eb5
REPORT RequestId: c566cf77-8c06-44e8-846e-9821a0320eb5  Duration: 3557.98 ms    Billed Duration: 3558 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 17068c7a-e117-436e-8f09-60f3fd182598 Version: 164
END RequestId: 17068c7a-e117-436e-8f09-60f3fd182598
REPORT RequestId: 17068c7a-e117-436e-8f09-60f3fd182598  Duration: 3756.15 ms    Billed Duration: 3757 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 6694c278-6d3a-4163-870b-ec14f71c3cab Version: 164
END RequestId: 6694c278-6d3a-4163-870b-ec14f71c3cab
REPORT RequestId: 6694c278-6d3a-4163-870b-ec14f71c3cab  Duration: 3992.56 ms    Billed Duration: 3993 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 933a8f76-9c85-4a2b-9011-a55cddabd386 Version: 164
END RequestId: 933a8f76-9c85-4a2b-9011-a55cddabd386
REPORT RequestId: 933a8f76-9c85-4a2b-9011-a55cddabd386  Duration: 3794.28 ms    Billed Duration: 3795 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 7686d69c-75a0-4f3c-85f7-57e765fee933 Version: 164
END RequestId: 7686d69c-75a0-4f3c-85f7-57e765fee933
REPORT RequestId: 7686d69c-75a0-4f3c-85f7-57e765fee933  Duration: 3736.15 ms    Billed Duration: 3737 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: e1ffad18-c7a3-4e2f-b382-ccc188d7eddc Version: 164
END RequestId: e1ffad18-c7a3-4e2f-b382-ccc188d7eddc
REPORT RequestId: e1ffad18-c7a3-4e2f-b382-ccc188d7eddc  Duration: 3778.08 ms    Billed Duration: 3779 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 70aa0f4f-0715-4b87-91ec-220dec244b16 Version: 164
END RequestId: 70aa0f4f-0715-4b87-91ec-220dec244b16
REPORT RequestId: 70aa0f4f-0715-4b87-91ec-220dec244b16  Duration: 3654.28 ms    Billed Duration: 3655 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 0489b835-76ec-4fdc-a818-73d84be8f780 Version: 164
END RequestId: 0489b835-76ec-4fdc-a818-73d84be8f780
REPORT RequestId: 0489b835-76ec-4fdc-a818-73d84be8f780  Duration: 3894.31 ms    Billed Duration: 3895 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: d263420b-62cb-4f83-8644-f982bcd43422 Version: 164
END RequestId: d263420b-62cb-4f83-8644-f982bcd43422
REPORT RequestId: d263420b-62cb-4f83-8644-f982bcd43422  Duration: 3556.85 ms    Billed Duration: 3557 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: f654a7f2-cfe2-4cba-b74c-228f874bcc51 Version: 164
END RequestId: f654a7f2-cfe2-4cba-b74c-228f874bcc51
REPORT RequestId: f654a7f2-cfe2-4cba-b74c-228f874bcc51  Duration: 3760.56 ms    Billed Duration: 3761 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 3f220979-bffa-426b-b230-e0f166a8b202 Version: 164
END RequestId: 3f220979-bffa-426b-b230-e0f166a8b202
REPORT RequestId: 3f220979-bffa-426b-b230-e0f166a8b202  Duration: 3796.07 ms    Billed Duration: 3797 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: c7dfcdfb-4cc2-4001-a6ae-5a8ae51fbad4 Version: 164
END RequestId: c7dfcdfb-4cc2-4001-a6ae-5a8ae51fbad4
REPORT RequestId: c7dfcdfb-4cc2-4001-a6ae-5a8ae51fbad4  Duration: 3816.74 ms    Billed Duration: 3817 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 58718d38-09c4-4f1d-a20b-9f80e47490a2 Version: 164
END RequestId: 58718d38-09c4-4f1d-a20b-9f80e47490a2
REPORT RequestId: 58718d38-09c4-4f1d-a20b-9f80e47490a2  Duration: 4085.78 ms    Billed Duration: 4086 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: 0e63e89f-0613-437b-b0ab-3ed33aaa9c0c Version: 164
END RequestId: 0e63e89f-0613-437b-b0ab-3ed33aaa9c0c
REPORT RequestId: 0e63e89f-0613-437b-b0ab-3ed33aaa9c0c  Duration: 4238.66 ms    Billed Duration: 4239 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: efce155b-4bd1-48bd-832b-a8ba6b9ac09e Version: 164
END RequestId: efce155b-4bd1-48bd-832b-a8ba6b9ac09e
REPORT RequestId: efce155b-4bd1-48bd-832b-a8ba6b9ac09e  Duration: 3808.44 ms    Billed Duration: 3809 ms    Memory Size: 1024 MB    Max Memory Used: 1024 MB    
START RequestId: b1d2f9f3-e509-4ce1-9d47-5d703538502f Version: 164
RequestId: b1d2f9f3-e509-4ce1-9d47-5d703538502f Error: Runtime exited with error: signal: killed
Runtime.ExitError
END RequestId: b1d2f9f3-e509-4ce1-9d47-5d703538502f
REPORT RequestId: b1d2f9f3-e509-4ce1-9d47-5d703538502f  Duration: 4902.21 ms    Billed Duration: 4903 ms    Memory Size: 1024 MB    Max Memory Used: 1025 MB
Giorgi commented 1 year ago

Are you able to reproduce it locally?

inacionery commented 1 year ago

Yes, memory grows locally as well.

using DuckDB.NET.Data;
using System;
using System.Diagnostics;
public class Function
{
    static void Main(string[] args)
    {
        var function = new Function();
        for (int i = 0; i < 100; i++)
        {
            function.FunctionHandler();
            Process currentProcess = Process.GetCurrentProcess();
            Console.WriteLine("Total Memory: {0}MB", currentProcess.WorkingSet64 / (1024 * 1024));
        }
    }
    public void FunctionHandler()
    {
        Console.WriteLine("START");
        try
        {
            string duckDBFilePath = "/tmp/data.db";
            if (!File.Exists(duckDBFilePath))
            {
                using (var connection = new DuckDBConnection($"Data Source={duckDBFilePath}"))
                {
                    connection.Open();
                    using (var createTableCommand = connection.CreateCommand())
                    {
                        createTableCommand.CommandText = "CREATE TABLE DummyData (Field1 VARCHAR, Field2 VARCHAR, Field3 VARCHAR, Field4 VARCHAR, Field5 VARCHAR, Field6 VARCHAR, Field7 VARCHAR, Field8 VARCHAR, Field9 VARCHAR, Field10 VARCHAR)";
                        createTableCommand.ExecuteNonQuery();
                    }
                    using (var transaction = connection.BeginTransaction())
                    {
                        try
                        {
                            using (var insertCommand = connection.CreateCommand())
                            {
                                insertCommand.CommandText = "INSERT INTO DummyData (Field1, Field2, Field3, Field4, Field5, Field6, Field7, Field8, Field9, Field10) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
                                for (int i = 0; i < 10; i++)
                                {
                                    insertCommand.Parameters.Add(new DuckDBParameter());
                                }
                                for (int i = 0; i < 3000000; i++)
                                {
                                    for (int j = 0; j < 10; j++)
                                    {
                                        insertCommand.Parameters[j].Value = $"Field_{j + 1}{i}";
                                    }
                                    insertCommand.ExecuteNonQuery();
                                }
                                transaction.Commit();
                            }
                        }
                        catch
                        {
                            transaction.Rollback();
                            throw;
                        }
                    }
                }
            }
            using (var connection = new DuckDBConnection($"Data Source={duckDBFilePath}"))
            {
                connection.Open();
                var list = new List<dynamic>();
                using (var command = connection.CreateCommand())
                {
                    command.CommandText = "SELECT 'DUPLICATES' as duplicateType, Field5, COUNT(1) FROM DummyData GROUP BY duplicateType, Field5 HAVING COUNT(Field5) > 1";
                    using (var reader = command.ExecuteReader())
                    {
                        while (reader.Read())
                        {
                            list.Add(reader.GetString(0));
                        }
                    }
                }
                var numberOfRecords = "";
                using (var command = connection.CreateCommand())
                {
                    command.CommandText = "SELECT COUNT(1) FROM DummyData";
                    using (var reader = command.ExecuteReader())
                    {
                        while (reader.Read())
                        {
                            numberOfRecords = reader.GetString(0);
                        }
                    }
                }
                Console.WriteLine("END");
            }
        }
        catch (Exception e)
        {
            Console.WriteLine($"An error occurred: {e}");
        }
    }
}

.csproj


<Project Sdk="Microsoft.NET.Sdk">

  <PropertyGroup>
    <OutputType>Exe</OutputType>
    <TargetFramework>net6.0</TargetFramework>
    <ImplicitUsings>enable</ImplicitUsings>
    <Nullable>enable</Nullable>
    <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
  </PropertyGroup>

  <ItemGroup>
    <PackageReference Include="DuckDB.NET.Data.Full" Version="0.8.1-alpha.19" />
  </ItemGroup>

</Project>

Outuput

START
END
Total Memory: 450MB
START
END
Total Memory: 508MB
START
END
Total Memory: 522MB
START
END
Total Memory: 530MB
START
END
Total Memory: 545MB
START
END
Total Memory: 553MB
START
END
Total Memory: 560MB
Giorgi commented 1 year ago

I did some profiling and the memory allocation comes from native code. .Net allocation stays at about 2MB throughout the app.

The native memory allocation could be related to https://github.com/duckdb/duckdb/pull/7697 The PR says:

DuckDB's threads do not exit, but stay idle when no queries are run, so, currently, some of jemalloc's allocations are never cleaned up after queries are done.

This would explain why memory isn't released once the query ends and all resources are disposed.

To validate you could build duckdb from the feature branch or grab from Windows CI artifacts and try running the app with that build.

inacionery commented 1 year ago

Hi @Giorgi,

I apologize for the delay, but I was having difficulties building with this branch. I tested both this code and another application and found that the problem does not occur when using the branch feature 🙌

Now the question is, do you know when this fix will be officially released? 🤔

Thank you for helping me with this issue 😄 

REPORT RequestId: 0e6f8d29-c9fa-4a57-a01d-0981565ed658  Duration: 10176.28 ms   Billed Duration: 10177 ms   Memory Size: 800 MB Max Memory Used: 653 MB Init Duration: 237.47 ms
REPORT RequestId: e334c3c0-414e-4846-b12d-9334957d7987  Duration: 4164.91 ms    Billed Duration: 4165 ms    Memory Size: 800 MB Max Memory Used: 663 MB
REPORT RequestId: 96cdb2d2-29ae-464b-8e8c-99e6779629f5  Duration: 4351.47 ms    Billed Duration: 4352 ms    Memory Size: 800 MB Max Memory Used: 663 MB
REPORT RequestId: 21db640b-e869-4c72-b1f6-15b92c91b3fa  Duration: 4489.60 ms    Billed Duration: 4490 ms    Memory Size: 800 MB Max Memory Used: 664 MB
REPORT RequestId: f778c02a-47a7-490d-a216-dab9d8b894b8  Duration: 4571.88 ms    Billed Duration: 4572 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 1853c47b-4f9c-4ac8-a70f-29006113afa4  Duration: 4054.66 ms    Billed Duration: 4055 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 06c65b2e-fad5-4c87-9333-6328dbe5f9e3  Duration: 4253.71 ms    Billed Duration: 4254 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 07fb102c-7589-46f5-bc2b-e4d3b693f791  Duration: 4171.10 ms    Billed Duration: 4172 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 66e9cda3-3fa5-4309-9b87-f331758a5739  Duration: 4590.70 ms    Billed Duration: 4591 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 9d48373e-a1fd-435a-8f17-24d5650ac95e  Duration: 4068.82 ms    Billed Duration: 4069 ms    Memory Size: 800 MB Max Memory Used: 758 MB
REPORT RequestId: 7ba2dbf0-52d5-458d-8dec-440b60a0390b  Duration: 4595.27 ms    Billed Duration: 4596 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 7850dc4c-0b02-42c4-8de5-44ddd8ddefce  Duration: 4130.49 ms    Billed Duration: 4131 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: b59ce5eb-ff4c-4b04-864a-c6f93d46010f  Duration: 4135.23 ms    Billed Duration: 4136 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 66272c41-4ef4-4d06-b684-e99a10962101  Duration: 3991.72 ms    Billed Duration: 3992 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: bce09771-419b-427f-b3bc-2b7c900f6802  Duration: 4131.18 ms    Billed Duration: 4132 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 0511e064-3c9b-4feb-bd91-055ff1c356d6  Duration: 4451.45 ms    Billed Duration: 4452 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 15950e5e-1b02-4528-a7f2-59be35a9db78  Duration: 4472.21 ms    Billed Duration: 4473 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: d3f5c680-96ab-49fb-9392-8cae40109633  Duration: 4174.37 ms    Billed Duration: 4175 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: c761b289-6eb7-486c-b9a0-484716c731fd  Duration: 4369.84 ms    Billed Duration: 4370 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 1bb1aa5e-d648-4a47-bc6f-32bc5bf62e27  Duration: 4509.40 ms    Billed Duration: 4510 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: a81bb0bd-0394-4b65-9e59-4327e64427fa  Duration: 4392.50 ms    Billed Duration: 4393 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: bf76db86-5d8d-485d-9a5e-b9d96e6075cb  Duration: 4188.42 ms    Billed Duration: 4189 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 47e87d48-3419-4c09-aba4-e29647e7ff22  Duration: 4273.93 ms    Billed Duration: 4274 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: ffb16519-c141-4c42-a8a0-29c6c41fda8a  Duration: 4672.95 ms    Billed Duration: 4673 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 6b1a5e45-16df-4161-b1ac-f11ad19a2d1f  Duration: 4329.29 ms    Billed Duration: 4330 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 8fdc287f-5cba-4716-8573-d48a54beeeb9  Duration: 4157.62 ms    Billed Duration: 4158 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 60af58f2-b4e7-4301-8dee-b69dd51094c6  Duration: 4151.41 ms    Billed Duration: 4152 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 42e8b999-97d3-429a-992b-5483dcbdfe45  Duration: 4472.25 ms    Billed Duration: 4473 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: 46806b2b-6bdd-4c60-9384-5f58ceea450d  Duration: 4355.32 ms    Billed Duration: 4356 ms    Memory Size: 800 MB Max Memory Used: 797 MB
REPORT RequestId: d27e4156-1cbb-4236-84d9-7ead9e4a7389  Duration: 4153.83 ms    Billed Duration: 4154 ms    Memory Size: 800 MB Max Memory Used: 797 MB
Giorgi commented 1 year ago

@inacionery It's nice to hear that you no longer have the memory leak.

I don't know when a new version will be released, probably in couple of month. But the feature branch was branched with master today so you can download build from artifacts sections. Check the GitHub master tab.

If you find this library helpful, consider making a donation to support the library.