philipp-baumann
commented
1 year ago r$> # Set up -----------------------------------------------------------------------
library("opusreader2")
path <- file.path("data", "spectra", "2018-NABO")
opus_files <- list.files(
path = path, pattern = "\\.\\d+$", full.names = TRUE, recursive = TRUE
)
# number of OPUS files
length(opus_files)
# Benchmark single-threaded reading (standard) ---------------------------------
bnch_seq <- bench::mark(
data <- opusreader2::read_opus(dsn = path)
)
bnch_seq
# A tibble: 1 × 13
expression min median itr/s…¹ mem_a…² gc/se…³ n_itr n_gc total…⁴ result memory
<bch:expr> <bch> <bch:> <dbl> <bch:b> <dbl> <int> <dbl> <bch:t> <list> <list>
1 data <- opusreader2::read_opus(dsn = path) 21.1s 21.1s 0.0475 8.32GB 4.32 1 91 21.1s <list> <Rprofmem>
# … with 2 more variables: time <list>, gc <list>, and abbreviated variable names ¹`itr/sec`, ²mem_alloc, ³`gc/sec`,
# ⁴total_time
# ℹ Use `colnames()` to see all variable names
Warning message:
Some expressions had a GC in every iteration; so filtering is disabled.
r$> # memory sizes on disk for files, memory size
total_filesize_mb <- sum(sapply(opus_files, file.size)) / 1000^2
parsed_data_size <- unclass(pryr::object_size(data)) / 1000^2
read_alloc_memory <- unclass(bnch_seq$mem_alloc) / 1000^2
# comparson of file size and parsed size
mem_comp_seq <- tibble::tribble(
~total_file_size_MB, ~read_alloc_memory_in_MB, ~parsed_size_in_MB,
total_filesize_mb, read_alloc_memory, parsed_data_size
)
mem_comp_seq
# A tibble: 1 × 3
total_file_size_MB read_alloc_memory_in_MB parsed_size_in_MB
<dbl> <dbl> <dbl>
1 490. 8930. 2104.
r$> # Benchmark asynchronous (multisession) reading --------------------------------
library("future")
availableCores() # 8 threads := 4 cores
plan(multisession(workers = 4))
# does chunk the file list according to dsn into number of `workers` registered
# in future
bnch_multis4 <- bench::mark(
data_multis4 <- opusreader2::read_opus(dsn = path, parallel = TRUE)
)
Loading required package: future.apply
Warning message:
Some expressions had a GC in every iteration; so filtering is disabled.
r$> bnch_multis4
# A tibble: 1 × 13
expression min median itr/s…¹ mem_a…² gc/se…³ n_itr n_gc
<bch:expr> <bch:> <bch:> <dbl> <bch:b> <dbl> <int> <dbl>
1 data_multis4 <- opusreader2::read_opus(dsn = path, parallel = TRUE) 8.92s 8.92s 0.112 2.65GB 0.112 1 1
# … with 5 more variables: total_time <bch:tm>, result <list>, memory <list>, time <list>, gc <list>, and abbreviated
# variable names ¹`itr/sec`, ²mem_alloc, ³`gc/sec`
# ℹ Use `colnames()` to see all variable names
r$> parsed_data_size_multis4 <- unclass(pryr::object_size(data_multis4)) / 1000^2
read_alloc_memory_multis4 <- unclass(bnch_multis4$mem_alloc) / 1000^2
mem_comp_multis4 <- tibble::tribble(
~total_file_size_MB, ~read_alloc_memory_in_MB, ~parsed_size_in_MB,
total_filesize_mb, read_alloc_memory_multis4, parsed_data_size_multis4
)
mem_comp_multis4
# A tibble: 1 × 3
total_file_size_MB read_alloc_memory_in_MB parsed_size_in_MB
<dbl> <dbl> <dbl>
1 490. 2846. 3076.
r$> # Benchmark asynchronous (multicore) reading -----------------------------------
# uses forking of processes; only supported on Linux and MacOS
# also not recommended in RStudio (other the terminal or another editor like
# VSCode)
library("future")
plan(multicore(workers = 4))
# does chunk the file list according to dsn into number of `workers` registered
# in future
bnch_multic4 <- bench::mark(
data_multic4 <- opusreader2::read_opus(dsn = path, parallel = TRUE),
memory = FALSE # If you are benchmarking parallel code you must set `memory = FALSE`.
)
bnch_multic4
# A tibble: 1 × 13
expression min median itr/s…¹ mem_a…² gc/se…³ n_itr n_gc
<bch:expr> <bch:> <bch:> <dbl> <bch:b> <dbl> <int> <dbl>
1 data_multic4 <- opusreader2::read_opus(dsn = path, parallel = TRUE) 11.7s 11.7s 0.0852 NA 2.98 1 35
# … with 5 more variables: total_time <bch:tm>, result <list>, memory <list>, time <list>, gc <list>, and abbreviated
# variable names ¹`itr/sec`, ²mem_alloc, ³`gc/sec`
# ℹ Use `colnames()` to see all variable names
Warning message:
Some expressions had a GC in every iteration; so filtering is disabled.
r$> (parsed_data_size_multic4 <- unclass(pryr::object_size(data_multis4)) / 1000^2)
[1] 3076.439
r$> (read_alloc_memory_multic4 <- unclass(bnch_multis4$mem_alloc) / 1000^2)
[1] 2845.69
=> suggested fix for the moment: use
htopto observe/screenshot max memory.