Goal
Compare yaml12 and yaml for reading and
writing YAML over a range of document sizes.
We intentionally avoid testing semantic differences between YAML 1.1 and 1.2, as well as less-used YAML features like tags and streams. Instead, we focus on the simple default usage of both.
Operations:
- read: YAML file → R object
-
write: R object → YAML (written to
nullfile())
Setup
Generate inputs
To benchmark, we create YAML files consisting of a repeating sequence
of a small, fixed “mixed” node, repeated 2^(1:15) times.
This node is designed to exercise the full YAML 1.2 core schema (every
core type is represented). We generate these files with
yaml12::write_yaml().
Each file is written to a temporary file and then we measure read
time. Note that on macOS and Linux, tempfile() paths often
live on a tmpfs in RAM, so this typically won’t trigger a
write to disk under most circumstances.
# A YAML node that exercises every core schema type:
# seq, map, str, bool, int, float, null
mixed_node <- list(
str = c(
"Lorem ipsum dolor sit amet, vel accumsan vitae faucibus ultrices leo",
"neque? Et cursus lacinia, ut, sit donec facilisi eu interdum. Dui",
"ipsum, vitae ligula commodo convallis ac sed nunc. Ipsum at nec lacus",
"eros suscipit vitae."
),
block_str = "lorem \n ipsum \n dolor\n",
bools = c(TRUE, FALSE),
ints = c(123L, -123L),
floats = c(123.456, -123.456),
null = NULL
)
make_yaml_doc <- function(size = 1) {
path <- tempfile(fileext = ".yaml")
yaml12::write_yaml(rep(list(mixed_node), size), path)
path
}
docs <- sapply(2^(1:15), make_yaml_doc)Read performance
read_results <- lapply(docs, function(doc) {
result <- bench::mark(
yaml12::read_yaml(doc),
yaml::read_yaml(doc),
check = FALSE
)
result$file_size <- file.info(doc)$size
result
})
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.Read results summary
read_results_df <- bind_rows(read_results) |>
mutate(expression = as.factor(sapply(expression, deparse1)))
read_results_df |>
ggplot(aes(x = file_size, y = median, color = expression)) +
labs(
y = "Median Run Time",
x = "File Size",
title = "Read Performance"
) +
geom_point() +
geom_smooth(linewidth = .5) +
scale_x_log10(labels = scales::label_bytes()) +
scale_y_continuous(trans = bench::bench_time_trans())
#> `geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Read results (detailed)
invisible(lapply(read_results, \(result) {
print(plot(result) + ggplot2::ggtitle(scales::label_bytes()(result$file_size[1])))
print(summary(result, filter_gc = FALSE))
}))
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::re… 840 39.2µs 49.4µs 18986. 2.75KB 0 9487 0
#> 2 yaml::read… 840 133.7µs 199.9µs 4923. 34.83KB 2.00 2461 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:t> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::rea… 1676 76.1µs 106µs 9272. 0B 0 4635 0
#> 2 yaml::read_… 1676 179.6µs 267µs 3656. 10.8KB 4.00 1828 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read_… 3348 148µs 205µs 4790. 0B 0 2395 0
#> 2 yaml::read_ya… 3348 311µs 423µs 2348. 13.7KB 2.00 1174 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read_… 6692 315µs 381µs 2567. 0B 0 1284 0
#> 2 yaml::read_ya… 6692 611µs 758µs 1286. 19.3KB 3.99 644 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::read_y… 13380 593.42µs 729.35µs 1402. 304B 0 701
#> 2 yaml::read_yam… 13380 1.09ms 1.39ms 721. 30.9KB 2.00 361
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read… 26756 1.27ms 1.47ms 661. 560B 2.00 331 1
#> 2 yaml::read_y… 26756 2.34ms 2.65ms 369. 69.4KB 1.99 185 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read… 53508 2.7ms 2.94ms 338. 1.05KB 0 169 0
#> 2 yaml::read_y… 53508 4.83ms 5.38ms 182. 146.29KB 1.99 91 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::rea… 107012 5.6ms 5.92ms 166. 2.05KB 1.98 84 1
#> 2 yaml::read_… 107012 11ms 11.48ms 84.0 300.09KB 2.00 42 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read… 214020 10.1ms 12.1ms 83.5 4.05KB 0 42 0
#> 2 yaml::read_y… 214020 22ms 24.9ms 40.4 607.63KB 1.92 21 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read… 428036 22.5ms 25.9ms 39.4 8.05KB 1.97 20 1
#> 2 yaml::read_y… 428036 63.2ms 64.7ms 15.5 1.19MB 0 8 0
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::re… 856068 49.6ms 50.4ms 19.4 16KB 1.94 10 1
#> 2 yaml::read… 856068 178ms 180.2ms 5.56 2.4MB 0 3 0
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::read… 1712132 91ms 96.1ms 10.5 32KB 0 6 0
#> 2 yaml::read_y… 1712132 896ms 896.5ms 1.12 4.8MB 1.12 1 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::read_y… 3424260 173.08ms 178.28ms 5.63 64KB 0 3
#> 2 yaml::read_yam… 3424260 5.05s 5.05s 0.198 9.6MB 0.198 1
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <dbl> <bch:t> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::re… 6848516 367.4ms 394.6ms 2.53 128KB 1.27 2 1
#> 2 yaml::read… 6848516 22.8s 22.8s 0.0439 19.2MB 0.132 1 3
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression file_size min median `itr/sec` mem_alloc `gc/sec` n_itr
#> <bch:expr> <dbl> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl> <int>
#> 1 yaml12::read_y… 13697028 749.66ms 749.66ms 1.33 256KB 2.67 1
#> 2 yaml::read_yam… 13697028 1.18m 1.18m 0.0141 38.4MB 0.0849 1
#> # ℹ 6 more variables: n_gc <dbl>, total_time <bch:tm>, result <list>,
#> # memory <list>, time <list>, gc <list>Write performance
objs <- lapply(2^(1:15), function(n) rep(list(mixed_node), n))
write_results <- lapply(objs, function(obj) {
result <- bench::mark(
yaml12::write_yaml(obj, nullfile()),
yaml::write_yaml(obj, nullfile()),
check = FALSE
)
result$obj_size <- object.size(obj)
result
})
#> Warning: Some expressions had a GC in every iteration; so filtering is
#> disabled.Write results summary:
write_results_df <- bind_rows(write_results) |>
mutate(expression = as.factor(sapply(expression, deparse1)))
write_results_df |>
ggplot(aes(x = obj_size, y = median, color = expression)) +
labs(
y = "Median Run Time",
x = "Object Size",
title = "Write Performance"
) +
geom_point() +
geom_smooth(linewidth = .5) +
scale_x_log10(labels = scales::label_bytes()) +
scale_y_continuous(trans = bench::bench_time_trans())
#> `geom_smooth()` using method = 'loess' and formula = 'y ~ x'
Write results (detailed)
invisible(lapply(write_results, \(result) {
print(plot(result) + ggplot2::ggtitle(scales::label_bytes()(as.numeric(result$obj_size[1]))))
print(summary(result, filter_gc = FALSE))
}))
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 3360 by… 18.8µs 19.2µs 50743. 3.02KB 0 10000 0
#> 2 yaml::write_y… 3360 by… 55.4µs 60.3µs 15608. 33.74KB 4.00 7800 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 6672 by… 31µs 31.9µs 30692. 0B 0 10000 0
#> 2 yaml::write_y… 6672 by… 77.7µs 84µs 11180. 1.56KB 4.00 5588 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:t> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::wri… 13296 b… 56.9µs 57.9µs 16912. 0B 0 8451 0
#> 2 yaml::write… 13296 b… 122.1µs 131.6µs 7226. 3.07KB 4.00 3613 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write_… 26544 b… 106µs 108µs 9064. 0B 0 4531 0
#> 2 yaml::write_ya… 26544 b… 210µs 226µs 4224. 6.09KB 4.00 2112 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write_… 53040 b… 204µs 210µs 4619. 0B 0 2310 0
#> 2 yaml::write_ya… 53040 b… 398µs 428µs 2282. 12.1KB 2.00 1141 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write_… 106032 … 403µs 422µs 2288. 0B 0 1144 0
#> 2 yaml::write_ya… 106032 … 778µs 828µs 1059. 24.2KB 4.00 530 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:tm> <bch:tm> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::w… 212016 … 812.07µs 845.44µs 1169. 0B 0 585 0
#> 2 yaml::wri… 212016 … 1.51ms 1.62ms 594. 48.3KB 2.00 297 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 423984 … 1.68ms 1.71ms 575. 0B 0 288 0
#> 2 yaml::write_y… 423984 … 3.03ms 3.2ms 296. 96.6KB 3.97 149 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 847920 … 3.35ms 3.41ms 290. 0B 0 146 0
#> 2 yaml::write_y… 847920 … 5.97ms 6.3ms 154. 193KB 2.00 77 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:t> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::wri… 1695792… 6.77ms 6.92ms 143. 0B 0 72 0
#> 2 yaml::write… 1695792… 12.01ms 12.54ms 75.5 386KB 3.97 38 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 3391536… 14.2ms 14.8ms 66.8 0B 0 34 0
#> 2 yaml::write_y… 3391536… 24.1ms 25.1ms 38.8 772KB 1.94 20 1
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write… 6783024… 29.7ms 30.6ms 32.6 0B 0 17 0
#> 2 yaml::write_y… 6783024… 47.9ms 49.6ms 19.1 1.51MB 3.82 10 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch:> <bch:t> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::writ… 1356600… 58.4ms 58.9ms 17.0 0B 0 9 0
#> 2 yaml::write_… 1356600… 98ms 101.3ms 9.64 3.02MB 3.86 5 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write_… 2713195… 116ms 116ms 8.59 0B 0 5 0
#> 2 yaml::write_ya… 2713195… 198ms 211ms 4.84 6.03MB 3.23 3 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>
#> # A tibble: 2 × 14
#> expression obj_size min median `itr/sec` mem_alloc `gc/sec` n_itr n_gc
#> <bch:expr> <objct_> <bch> <bch:> <dbl> <bch:byt> <dbl> <int> <dbl>
#> 1 yaml12::write_… 5426385… 253ms 254ms 3.93 0B 0 2 0
#> 2 yaml::write_ya… 5426385… 407ms 526ms 1.90 12.1MB 1.90 2 2
#> # ℹ 5 more variables: total_time <bch:tm>, result <list>, memory <list>,
#> # time <list>, gc <list>