## ----include = FALSE----------------------------------------------------------
knitr::opts_chunk$set(
  collapse = TRUE,
  comment = "#>",
  fig.width = 7,
  fig.height = 4
)

## ----setup--------------------------------------------------------------------
library(arrow)
library(dplyr)
library(summarisebig)

## ----data---------------------------------------------------------------------
d <- data.frame(
  grp = rep(c("A", "B", "C"), each = 4),
  row_id = rep(1:4, 3),
  x = c(1, 2, 4, 8, 10, 12, 14, 18, 3, 6, 9, 15),
  y = c(2, 3, 5, 9, 9, 13, 15, 20, 5, 5, 11, 14),
  txt = c("alpha", "bravo", "charlie", "delta",
          "echo", "foxtrot", "golf", "hotel",
          "india", "juliet", "kilo", "lima")
)

path <- tempfile("summarisebig-vignette-")
arrow::write_dataset(d, path, format = "parquet")
ds <- arrow::open_dataset(path)

## ----arrow-fast-path----------------------------------------------------------
summarise_big(
  ds,
  result = mean(x),
  .by = grp
) |>
  arrange(grp)

## ----map-reduce---------------------------------------------------------------
summarise_big(
  ds,
  .by = grp,
  .strategy = "map_reduce",
  .map_reduce = list(
    n = ~ dplyr::n(),
    sum_x = ~ sum(x),
    sum_y = ~ sum(y),
    sum_x2 = ~ sum(x * x),
    sum_y2 = ~ sum(y * y),
    sum_xy = ~ sum(x * y)
  ),
  .finalize = function(state) {
    state |>
      mutate(
        centered_xx = sum_x2 - sum_x^2 / n,
        centered_yy = sum_y2 - sum_y^2 / n,
        centered_xy = sum_xy - sum_x * sum_y / n,
        slope = centered_xy / centered_xx,
        residual_ss = centered_yy - slope * centered_xy,
        slope_se = sqrt((residual_ss / (n - 2)) / centered_xx),
        t_value = slope / slope_se,
        p_value = 2 * stats::pt(-abs(t_value), df = n - 2)
      ) |>
      select(grp, slope, slope_se, p_value)
  }
) |>
  arrange(grp)

## ----custom-function----------------------------------------------------------
interquartile_span <- function(x) {
  q <- stats::quantile(
    x,
    probs = c(0.25, 0.75),
    names = FALSE,
    type = 7
  )
  q[[2]] - q[[1]]
}

## ----parallel-chunks----------------------------------------------------------
summarise_big(
  ds,
  result = interquartile_span(x),
  .by = grp,
  .strategy = "parallel_chunks",
  .workers = 2,
  .chunk_rows = 4,
  .try_arrow = FALSE
) |>
  arrange(grp)

## ----shared-chunk-------------------------------------------------------------
initials_in_order <- function(txt) {
  paste0(substr(txt, 1, 1), collapse = "")
}

summarise_big(
  ds,
  result = initials_in_order(txt),
  .by = grp,
  .order_by = "row_id",
  .strategy = "shared_chunk",
  .workers = 1,
  .chunk_rows = 8,
  .task_rows = 4,
  .try_arrow = FALSE
) |>
  arrange(grp)

## ----shared-chunk-parallel, eval=FALSE----------------------------------------
# summarise_big(
#   ds,
#   result = initials_in_order(txt),
#   .by = grp,
#   .order_by = "row_id",
#   .strategy = "shared_chunk",
#   .workers = 2,
#   .chunk_rows = 8,
#   .task_rows = 4,
#   .try_arrow = FALSE
# )

## ----cleanup, include=FALSE---------------------------------------------------
unlink(path, recursive = TRUE, force = TRUE)