search

constellation-rs / constellation

Distributed programming for Rust.
Apache License 2.0
563 stars 24 forks source link

Add tests for resource limits #61

Closed alecmocatta closed 4 years ago

alecmocatta commented 4 years ago

This adds a very strong test, resource-limit-stress, which concurrently spawns and drops 471 communicating processes. It takes ages, so the CI test iterations have been reduced. I've doubled the number of linux CI runners however.

Also add parsing of fabric output to the test harness; this catches a very rare (~1 every 5 cpu hours running the output-data test in a tight loop) hang, see pid 109965 not exiting here:

2020-02-11T19:53:52.9035098Z Memory: 781.1 MB used / 7.3 GB (7284568064 bytes) total (PlatformMemory { meminfo: {"Active": 3.4 GB, "Active(anon)": 415.6 MB, "Active(file)": 2.9 GB, "AnonHugePages": 245.4 MB, "AnonPages": 447.5 MB, "Bounce": 0 B, "Buffers": 162.2 MB, "Cached": 5.5 GB, "CmaFree": 0 B, "CmaTotal": 0 B, "CommitLimit": 12.2 GB, "Committed_AS": 2.8 GB, "DirectMap1G": 6.4 GB, "DirectMap2M": 3.1 GB, "DirectMap4k": 127.9 MB, "Dirty": 41.0 KB, "HardwareCorrupted": 0 B, "Hugepagesize": 2.1 MB, "Inactive": 2.8 GB, "Inactive(anon)": 102.1 MB, "Inactive(file)": 2.7 GB, "KernelStack": 6.8 MB, "Mapped": 254.5 MB, "MemAvailable": 6.2 GB, "MemFree": 636.5 MB, "MemTotal": 7.3 GB, "Mlocked": 3.8 MB, "NFS_Unstable": 0 B, "PageTables": 8.1 MB, "SReclaimable": 226.2 MB, "SUnreclaim": 77.2 MB, "Shmem": 69.7 MB, "ShmemHugePages": 0 B, "ShmemPmdMapped": 0 B, "Slab": 303.3 MB, "SwapCached": 8.2 KB, "SwapFree": 8.6 GB, "SwapTotal": 8.6 GB, "Unevictable": 3.8 MB, "VmallocChunk": 0 B, "VmallocTotal": 35.2 TB, "VmallocUsed": 0 B, "Writeback": 0 B, "WritebackTmp": 0 B} })
2020-02-11T19:53:52.9036163Z Load average: 2.3427734 3.8081055 3.6811523
2020-02-11T19:53:52.9036359Z CPU load: 0% user, 0% nice, 0% system, 0% intr, 100% idle
2020-02-11T19:53:52.9036531Z Processes: 131, threads: 284
2020-02-11T19:53:52.9036672Z File descriptors: 1472 open, 706991 max
2020-02-11T19:53:52.9037421Z tests/output-data.rs
2020-02-11T19:53:52.9037616Z     0
2020-02-11T19:53:53.0454193Z fab stdout: "{\"init\":{\"pid\":{\"key\":274773370605286673489937816620515070915,\"ip\":\"127.0.0.1\",\"port\":42413},\"system_pid\":109892}}\n"
2020-02-11T19:53:53.0578561Z fab stdout: "{\"exit\":{\"pid\":{\"key\":274773370605286673489937816620515070915,\"ip\":\"127.0.0.1\",\"port\":42413},\"system_pid\":109892}}\n"
2020-02-11T19:53:53.0962008Z     1
2020-02-11T19:53:53.2338325Z fab stdout: "{\"init\":{\"pid\":{\"key\":198082233292506167084671543068278489464,\"ip\":\"127.0.0.1\",\"port\":34983},\"system_pid\":109916}}\n"
2020-02-11T19:53:53.2447520Z fab stdout: "{\"exit\":{\"pid\":{\"key\":198082233292506167084671543068278489464,\"ip\":\"127.0.0.1\",\"port\":34983},\"system_pid\":109916}}\n"
2020-02-11T19:53:53.2840786Z     2
2020-02-11T19:53:53.4194760Z fab stdout: "{\"init\":{\"pid\":{\"key\":26910306650878490903501923730859136574,\"ip\":\"127.0.0.1\",\"port\":36845},\"system_pid\":109940}}\n"
2020-02-11T19:53:53.4322153Z fab stdout: "{\"exit\":{\"pid\":{\"key\":26910306650878490903501923730859136574,\"ip\":\"127.0.0.1\",\"port\":36845},\"system_pid\":109940}}\n"
2020-02-11T19:53:53.4721202Z     3
2020-02-11T19:53:53.6167364Z fab stdout: "{\"init\":{\"pid\":{\"key\":1612076484495532056923727983250612711,\"ip\":\"127.0.0.1\",\"port\":34465},\"system_pid\":109965}}\n"
2020-02-11T19:53:53.6640465Z     4
2020-02-11T19:53:53.8037926Z fab stdout: "{\"init\":{\"pid\":{\"key\":293338737214668155455319182865647767841,\"ip\":\"127.0.0.1\",\"port\":43981},\"system_pid\":109993}}\n"
2020-02-11T19:53:53.8145987Z fab stdout: "{\"exit\":{\"pid\":{\"key\":293338737214668155455319182865647767841,\"ip\":\"127.0.0.1\",\"port\":43981},\"system_pid\":109993}}\n"
2020-02-11T19:53:53.8600468Z     5
2020-02-11T19:53:54.0017557Z fab stdout: "{\"init\":{\"pid\":{\"key\":15118717843972193183638946127534517648,\"ip\":\"127.0.0.1\",\"port\":32769},\"system_pid\":110017}}\n"
2020-02-11T19:53:54.0156548Z fab stdout: "{\"exit\":{\"pid\":{\"key\":15118717843972193183638946127534517648,\"ip\":\"127.0.0.1\",\"port\":32769},\"system_pid\":110017}}\n"
2020-02-11T19:53:54.0554074Z     6
2020-02-11T19:53:54.2055258Z fab stdout: "{\"init\":{\"pid\":{\"key\":289869825600804344587524749851988246995,\"ip\":\"127.0.0.1\",\"port\":46361},\"system_pid\":110042}}\n"
2020-02-11T19:53:54.2166899Z fab stdout: "{\"exit\":{\"pid\":{\"key\":289869825600804344587524749851988246995,\"ip\":\"127.0.0.1\",\"port\":46361},\"system_pid\":110042}}\n"
2020-02-11T19:53:54.2561509Z     7
2020-02-11T19:53:54.3960407Z fab stdout: "{\"init\":{\"pid\":{\"key\":202314181534811222481601923695096653730,\"ip\":\"127.0.0.1\",\"port\":41803},\"system_pid\":110066}}\n"
2020-02-11T19:53:54.4080228Z fab stdout: "{\"exit\":{\"pid\":{\"key\":202314181534811222481601923695096653730,\"ip\":\"127.0.0.1\",\"port\":41803},\"system_pid\":110066}}\n"
2020-02-11T19:53:54.4442655Z     8
2020-02-11T19:53:54.5896521Z fab stdout: "{\"init\":{\"pid\":{\"key\":287275833841488667447350661600443489574,\"ip\":\"127.0.0.1\",\"port\":38497},\"system_pid\":110091}}\n"
2020-02-11T19:53:54.6029687Z fab stdout: "{\"exit\":{\"pid\":{\"key\":287275833841488667447350661600443489574,\"ip\":\"127.0.0.1\",\"port\":38497},\"system_pid\":110091}}\n"
2020-02-11T19:53:54.6440777Z     9
2020-02-11T19:53:54.7799018Z fab stdout: "{\"init\":{\"pid\":{\"key\":54734315715571197327329834059527787409,\"ip\":\"127.0.0.1\",\"port\":43535},\"system_pid\":110115}}\n"
2020-02-11T19:53:54.7911026Z fab stdout: "{\"exit\":{\"pid\":{\"key\":54734315715571197327329834059527787409,\"ip\":\"127.0.0.1\",\"port\":43535},\"system_pid\":110115}}\n"
2020-02-11T19:53:55.3581124Z Memory: 817.6 MB used / 7.3 GB (7284568064 bytes) total (PlatformMemory { meminfo: {"Active": 3.4 GB, "Active(anon)": 433.5 MB, "Active(file)": 3.0 GB, "AnonHugePages": 245.4 MB, "AnonPages": 449.6 MB, "Bounce": 0 B, "Buffers": 162.2 MB, "Cached": 5.6 GB, "CmaFree": 0 B, "CmaTotal": 0 B, "CommitLimit": 12.2 GB, "Committed_AS": 3.0 GB, "DirectMap1G": 6.4 GB, "DirectMap2M": 3.1 GB, "DirectMap4k": 127.9 MB, "Dirty": 53.2 KB, "HardwareCorrupted": 0 B, "Hugepagesize": 2.1 MB, "Inactive": 2.8 GB, "Inactive(anon)": 119.9 MB, "Inactive(file)": 2.7 GB, "KernelStack": 7.1 MB, "Mapped": 257.7 MB, "MemAvailable": 6.2 GB, "MemFree": 600.0 MB, "MemTotal": 7.3 GB, "Mlocked": 3.8 MB, "NFS_Unstable": 0 B, "PageTables": 8.7 MB, "SReclaimable": 226.2 MB, "SUnreclaim": 76.9 MB, "Shmem": 103.0 MB, "ShmemHugePages": 0 B, "ShmemPmdMapped": 0 B, "Slab": 303.1 MB, "SwapCached": 8.2 KB, "SwapFree": 8.6 GB, "SwapTotal": 8.6 GB, "Unevictable": 3.8 MB, "VmallocChunk": 0 B, "VmallocTotal": 35.2 TB, "VmallocUsed": 0 B, "Writeback": 0 B, "WritebackTmp": 0 B} })
2020-02-11T19:53:55.3583021Z Load average: 2.3427734 3.8081055 3.6811523
2020-02-11T19:53:55.3583653Z CPU load: 0% user, 0% nice, 0.9803922% system, 0% intr, 99.01961% idle
2020-02-11T19:53:55.3584105Z Processes: 143, threads: 308
2020-02-11T19:53:55.3584327Z File descriptors: 1504 open, 706991 max

Also seen with success and success-return.