miniextendr_api/dataframe_builder.rs
1//! Closure-per-column [`DataFrame`](crate::dataframe::DataFrame) builder.
2//!
3//! [`RDataFrameBuilder`] assembles an R `data.frame` from a set of typed columns,
4//! each with its own element type and fill closure. It is the heterogeneous-column
5//! analogue of `with_r_matrix`: you declare columns, the builder allocates the
6//! backing storage with strict PROTECT discipline, fills it, and assembles the
7//! `data.frame` on the R thread.
8//!
9//! # Fill strategy (parallel with `rayon`, serial otherwise)
10//!
11//! The builder type and its surface are **available regardless of the `rayon`
12//! feature** (#1055). Only the fill pass differs:
13//!
14//! - **With `rayon`** — the whole job is flattened into a single work-list of
15//! `(column_index, row-range)` items and filled in one work-stealing
16//! `par_iter` pass (see [`RDataFrameBuilder`] docs for the scheduling argument).
17//! - **Without `rayon`** — each column is filled in one full-range serial pass.
18//!
19//! Both paths produce an identical `data.frame`. Nothing else in the builder
20//! depends on rayon: `with_r_thread`, `RNativeType`, the PROTECT
21//! discipline, and the assembly step are all rayon-independent. The `Send + Sync`
22//! bounds on the fill closures are simply unused when rayon is absent.
23
24use crate::{RNativeType, SEXP, SexpExt, with_r_thread};
25
26// region: type-erased column fill machinery
27
28/// Type-erased "fill this row-range of this column" closure.
29///
30/// Called with `(offset, len)` describing a contiguous half-open row range
31/// `[offset, offset + len)` of one column. The concrete closure (captured at
32/// `.column::<T>()` / `.column_str()` registration time) knows the column's
33/// element type and destination buffer, and writes exactly that range — no
34/// other thread touches it (see the safety argument on [`RDataFrameBuilder`]).
35type RangeFiller = Box<dyn Fn(usize, usize) + Send + Sync>;
36
37/// Send+Sync wrapper for a raw column-data pointer carried across the flatten
38/// boundary. The pointer addresses R-owned memory for native columns or a
39/// `Vec<Option<String>>` backing buffer for character columns. Disjointness
40/// (per column and per row-range within a column) is the caller's invariant —
41/// see the safety argument on [`RDataFrameBuilder`].
42#[derive(Clone, Copy)]
43struct ColPtr(*mut ());
44
45unsafe impl Send for ColPtr {}
46unsafe impl Sync for ColPtr {}
47
48impl ColPtr {
49 /// Reinterpret the erased base pointer as `*mut T`.
50 ///
51 /// Taking `&self` (a method call on the whole struct) makes a capturing
52 /// closure capture the `Send + Sync` `ColPtr` as a whole rather than its
53 /// raw `*mut ()` field (which is neither), keeping the closure `Send + Sync`.
54 #[inline]
55 fn cast<T>(&self) -> *mut T {
56 self.0 as *mut T
57 }
58}
59
60/// How a column's R storage is materialized after the fill.
61enum ColumnKind {
62 /// Native column: the fill wrote directly into R memory; the SEXP is already
63 /// complete. Holds the allocated (and currently protected) SEXP.
64 Native(SEXP),
65 /// Character column: the fill computed `Option<String>` values into a `Vec`
66 /// (no R API on fill threads). The `CHARSXP`s are set serially on the R
67 /// thread during assembly. `None` becomes `NA_character_`.
68 Str(Vec<Option<String>>),
69}
70
71/// One registered column: a serial allocation step plus the range filler that
72/// the flattened work-list (or serial loop) dispatches.
73struct ColumnReg {
74 /// Allocates the column's backing storage (R SEXP for native columns, an
75 /// owned `Vec` for character columns) for `nrow` rows. Runs serially on
76 /// R's main thread. Returns the [`ColumnKind`] (carrying the protected SEXP
77 /// or the owned buffer) and the raw data pointer the range filler writes
78 /// through during the fill phase.
79 #[allow(clippy::type_complexity)]
80 alloc: Box<dyn FnOnce(usize) -> (ColumnKind, ColPtr) + Send>,
81 /// Builds the type-erased range filler once the data pointer is known.
82 make_filler: Box<dyn FnOnce(ColPtr, usize) -> RangeFiller + Send>,
83}
84
85// endregion
86
87// region: RDataFrameBuilder
88
89/// Builder for assembling an R `data.frame` from per-column fill closures.
90///
91/// This is the heterogeneous-column analogue of `with_r_matrix`: instead of one
92/// homogeneous matrix, you declare a set of typed columns (each with its own
93/// element type and fill closure) and the builder fills them all, then assembles
94/// the `data.frame`.
95///
96/// # Fill strategy (parallel with `rayon`, serial otherwise)
97///
98/// The builder is available regardless of the `rayon` feature; only the fill
99/// pass differs (see the [module docs](crate::dataframe_builder)). The rest of
100/// this section describes the parallel pass that `rayon` enables.
101///
102/// ## Two axes of parallelism, one work-stealing pass
103///
104/// There are two ways to parallelise a column fill:
105///
106/// - **Column-granular** — one task per column. Fan-out width equals the column
107/// count, so a 3-column × 10M-row frame only ever uses 3 threads.
108/// - **Row-slice-granular** — split *one* column into contiguous row ranges.
109/// Great for one long column, but on its own it serialises across columns.
110///
111/// `RDataFrameBuilder` does **not** choose. With `rayon`,
112/// [`build`][RDataFrameBuilder::build] flattens the entire job into a single
113/// work-list of `(column_index, row-range)` items — each native/character column
114/// is split into `chunk_size = max(1, nrow / (current_num_threads() * 4))`-row
115/// chunks (with 4× oversubscription) — then runs **one** `par_iter` over that
116/// flat list. Rayon's work-stealing balances both axes automatically:
117///
118/// - **wide** (100 cols × short) → ~100+ items, column-dominated.
119/// - **tall** (3 cols × 10M rows) → each column shatters into `~nthreads*4`
120/// chunks → hundreds of items, saturated even with 3 columns.
121/// - **skewed** (1 huge col + many tiny) → the huge column's chunks get stolen
122/// by threads idle after finishing the tiny columns.
123///
124/// This also avoids the per-column barrier and repeated pool spin-up that the
125/// naive "fill each column, each internally parallel" (nested `par_iter`) shape
126/// would cause. Without `rayon`, each column is filled in one full-range pass.
127///
128/// # Phases
129///
130/// 1. Allocate each column's backing storage **serially on R's main thread**
131/// (native columns get a protected R vector; character columns get an owned
132/// `Vec<Option<String>>`). Strict PROTECT discipline — the dangerous part.
133/// 2. Fill all columns (parallel flat pass with `rayon`, serial otherwise). No R
134/// API calls happen inside the parallel region.
135/// 3. Set character `CHARSXP`s serially on the R thread (CHARSXP allocation is
136/// forbidden on rayon threads), then assemble the `VECSXP`, `names`, compact
137/// `row.names` (`c(NA_integer_, -nrow)`), and `class = "data.frame"`.
138///
139/// # Column kinds
140///
141/// - [`column::<T>`][RDataFrameBuilder::column] — a native-typed column
142/// (`f64`/`i32`/`RLogical`/`u8`/`Rcomplex`). The fill closure receives a
143/// mutable chunk and its offset. The buffer is R memory, filled directly with
144/// zero intermediate allocation.
145/// - [`column_str`][RDataFrameBuilder::column_str] — a character (`STRSXP`)
146/// column. The per-row `Option<String>` values are computed during the fill
147/// pass, but the `CHARSXP`s are set **serially** afterward. `None` becomes
148/// `NA_character_`.
149///
150/// # Example
151///
152/// ```ignore
153/// use miniextendr_api::dataframe_builder::RDataFrameBuilder;
154///
155/// let df = RDataFrameBuilder::new(1000)
156/// .column::<f64>("x", |chunk, offset| {
157/// for (i, slot) in chunk.iter_mut().enumerate() {
158/// *slot = ((offset + i) as f64).sqrt();
159/// }
160/// })
161/// .column::<i32>("y", |chunk, offset| {
162/// for (i, slot) in chunk.iter_mut().enumerate() {
163/// *slot = (offset + i) as i32;
164/// }
165/// })
166/// .column_str("label", |i| Some(format!("row_{i}")))
167/// .build();
168/// ```
169///
170/// # Safety argument (disjoint mutation, no aliasing)
171///
172/// Neither fill path ever produces two items that overlap:
173///
174/// - Different columns address **different** backing buffers (distinct R vectors
175/// / distinct `Vec`s), so cross-column items are trivially disjoint.
176/// - Within a column, the row ranges are a partition of `[0, nrow)`. The serial
177/// path uses the single full range; the parallel path chunks `nrow` into
178/// fixed-size, non-overlapping spans. Each `(offset, len)` item therefore owns
179/// a unique slice of that column's buffer.
180///
181/// Each `RangeFiller` reconstitutes its slice via
182/// `slice::from_raw_parts_mut(base.add(offset), len)` and writes only that span.
183/// Because the spans are disjoint, no two threads ever form overlapping `&mut`
184/// references — there is no aliasing UB even though the work-list shares the raw
185/// base pointers (`ColPtr`, `Send + Sync`).
186///
187/// # Protection
188///
189/// Every native column SEXP is PROTECTed from allocation through insertion into
190/// the `VECSXP`; the `names` / `row.names` / class transients are likewise
191/// protected across each subsequent allocation. Before those temporary guards
192/// drop, [`build`][RDataFrameBuilder::build] transfers the assembled frame into
193/// a GC-rooted [`BuiltDataFrame`](crate::dataframe::BuiltDataFrame). The root is
194/// released when the handle is returned to R via `IntoR` or dropped in Rust.
195pub struct RDataFrameBuilder {
196 nrow: usize,
197 names: Vec<String>,
198 columns: Vec<ColumnReg>,
199}
200
201impl RDataFrameBuilder {
202 /// Start building a data.frame with `nrow` rows.
203 pub fn new(nrow: usize) -> Self {
204 // Compact row.names uses i32, so nrow must fit in i32; this also implies
205 // it fits in R_xlen_t on all supported pointer widths.
206 assert!(
207 nrow <= i32::MAX as usize,
208 "RDataFrameBuilder: nrow {} exceeds i32 maximum (compact row.names)",
209 nrow
210 );
211 Self {
212 nrow,
213 names: Vec::new(),
214 columns: Vec::new(),
215 }
216 }
217
218 /// Add a native-typed column (`f64`/`i32`/`RLogical`/`u8`/`Rcomplex`).
219 ///
220 /// The fill closure `f(chunk, offset)` is dispatched over chunks of the
221 /// (already-allocated) R column buffer — in parallel with `rayon`, or in one
222 /// full-range pass otherwise. Chunk boundaries are deterministic for a given
223 /// `nrow` and thread count.
224 pub fn column<T>(
225 mut self,
226 name: impl Into<String>,
227 f: impl Fn(&mut [T], usize) + Send + Sync + 'static,
228 ) -> Self
229 where
230 T: RNativeType + Send + Sync,
231 {
232 self.names.push(name.into());
233 self.columns.push(ColumnReg {
234 alloc: Box::new(|nrow| {
235 // Allocate + protect the R vector serially on the R thread, then
236 // hand back its data pointer for the fill. The protection is
237 // balanced during assembly in `build`.
238 let (sexp, ptr) = with_r_thread(move || unsafe {
239 let sexp =
240 crate::sys::Rf_allocVector_unchecked(T::SEXP_TYPE, nrow as crate::R_xlen_t);
241 crate::sys::Rf_protect_unchecked(sexp);
242 (sexp, T::dataptr_mut(sexp))
243 });
244 (ColumnKind::Native(sexp), ColPtr(ptr as *mut ()))
245 }),
246 make_filler: Box::new(move |base: ColPtr, nrow: usize| {
247 Box::new(move |offset: usize, len: usize| {
248 debug_assert!(offset + len <= nrow);
249 // Safety: this range `[offset, offset+len)` is a disjoint
250 // partition of this column's buffer (see the safety argument
251 // on `RDataFrameBuilder`); no other thread writes it. `base`
252 // is the `Send + Sync` `ColPtr`; cast inside the closure so
253 // the closure stays `Send + Sync`.
254 let ptr = base.cast::<T>();
255 let slice = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), len) };
256 f(slice, offset);
257 })
258 }),
259 });
260 self
261 }
262
263 /// Add a character (`STRSXP`) column.
264 ///
265 /// The fill closure `f(i)` returns the value for row `i` as `Option<String>`,
266 /// where `None` maps to `NA_character_`. Values are computed during the fill
267 /// pass (parallel with `rayon`, serial otherwise), then set into the R
268 /// `STRSXP` serially on the R thread (CHARSXP allocation cannot happen on
269 /// rayon threads).
270 pub fn column_str(
271 mut self,
272 name: impl Into<String>,
273 f: impl Fn(usize) -> Option<String> + Send + Sync + 'static,
274 ) -> Self {
275 self.names.push(name.into());
276 self.columns.push(ColumnReg {
277 alloc: Box::new(|nrow| {
278 // No R allocation here: the fill phase fills an owned Vec.
279 let mut buf: Vec<Option<String>> = (0..nrow).map(|_| None).collect();
280 let ptr = buf.as_mut_ptr();
281 (ColumnKind::Str(buf), ColPtr(ptr as *mut ()))
282 }),
283 make_filler: Box::new(move |base: ColPtr, nrow: usize| {
284 Box::new(move |offset: usize, len: usize| {
285 debug_assert!(offset + len <= nrow);
286 // Safety: disjoint partition of this column's Vec buffer.
287 // Cast `base` (Send + Sync `ColPtr`) inside the closure.
288 let ptr = base.cast::<Option<String>>();
289 let slice = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), len) };
290 for (i, slot) in slice.iter_mut().enumerate() {
291 *slot = f(offset + i);
292 }
293 })
294 }),
295 });
296 self
297 }
298
299 /// Allocate, fill, and assemble the data.frame, returning an owned,
300 /// GC-rooted [`BuiltDataFrame`](crate::dataframe::BuiltDataFrame) (it
301 /// `Deref`s to [`DataFrame`](crate::dataframe::DataFrame)).
302 ///
303 /// With `rayon`, flattens every column into a single `(column_index,
304 /// row-range)` work-list and runs one parallel pass over it (see the
305 /// type-level docs for the scheduling argument); without `rayon`, fills each
306 /// column serially. Then assembles the `data.frame` on the R thread.
307 pub fn build(self) -> crate::dataframe::BuiltDataFrame {
308 // SAFETY: `build_sexp` returns a well-formed data.frame VECSXP; root it
309 // immediately (no allocation between assembly and adopt).
310 unsafe { crate::dataframe::BuiltDataFrame::adopt_sexp(self.build_sexp()) }
311 }
312
313 /// Assemble and return the raw `VECSXP` SEXP (internal; prefer [`build`](Self::build)).
314 fn build_sexp(self) -> SEXP {
315 let RDataFrameBuilder {
316 nrow,
317 names,
318 columns,
319 } = self;
320 let ncol = columns.len();
321 assert_eq!(
322 names.len(),
323 ncol,
324 "RDataFrameBuilder: names/columns length mismatch"
325 );
326 // Compact row.names `c(NA, -nrow)` are emitted as INTSXP, so `nrow` must
327 // fit in `i32`. Validate up front (panic → R error) rather than letting
328 // `-(nrow as i32)` below silently wrap for >2^31-row frames.
329 assert!(
330 nrow <= i32::MAX as usize,
331 "RDataFrameBuilder: nrow {nrow} exceeds i32 maximum for compact row.names"
332 );
333
334 // Phase 1: allocate every column's backing storage serially. Native
335 // columns return a freshly-protected R SEXP and its data pointer;
336 // character columns return an owned `Vec<Option<String>>` and its
337 // pointer. We re-protect native columns *as they are allocated* (inside
338 // `alloc`), so the per-column allocation in the next iteration cannot GC
339 // an earlier column. These protections are balanced during assembly.
340 let mut kinds: Vec<ColumnKind> = Vec::with_capacity(ncol);
341 let mut fillers: Vec<RangeFiller> = Vec::with_capacity(ncol);
342 for col in columns {
343 let ColumnReg { alloc, make_filler } = col;
344 let (kind, ptr) = alloc(nrow);
345 kinds.push(kind);
346 fillers.push(make_filler(ptr, nrow));
347 }
348
349 // Phase 2: fill every column. No R API calls happen here.
350 if nrow > 0 && ncol > 0 {
351 #[cfg(feature = "rayon")]
352 {
353 use rayon::prelude::*;
354 crate::optionals::parallel::ensure_pool();
355 // Flatten to ONE (column, row-range) work-list and run a single
356 // parallel pass. Each item is `(column_index, offset, len)`; the
357 // column's type-erased range filler writes exactly that disjoint
358 // span. Rayon's work-stealing balances the column axis and the
359 // row-slice axis together.
360 let chunk_size = std::cmp::max(1, nrow / (rayon::current_num_threads() * 4));
361 let work: Vec<(usize, usize, usize)> = (0..ncol)
362 .flat_map(|col_idx| {
363 (0..nrow).step_by(chunk_size).map(move |offset| {
364 let len = std::cmp::min(chunk_size, nrow - offset);
365 (col_idx, offset, len)
366 })
367 })
368 .collect();
369
370 work.par_iter().for_each(|&(col_idx, offset, len)| {
371 (fillers[col_idx])(offset, len);
372 });
373 }
374 #[cfg(not(feature = "rayon"))]
375 {
376 // Serial fill: each column filled in one full-range pass.
377 // ponytail: the chunked work-list only buys parallelism, which is
378 // exactly what `rayon` adds; the result is identical either way.
379 for filler in &fillers {
380 filler(0, nrow);
381 }
382 }
383 }
384 // Fillers are no longer needed; drop them before assembly so any captured
385 // closures release before we touch R.
386 drop(fillers);
387
388 // Phase 3: assemble on the R thread with strict PROTECT discipline. We
389 // are inside `with_r_thread`, a known-safe context, so `_unchecked` FFI
390 // is correct here (MXL301).
391 //
392 // PROTECT-stack invariant on entry: phase 1 left one protection per
393 // *native* column (character columns hold no SEXP yet). Track that exact
394 // count and balance it precisely.
395 with_r_thread(move || unsafe {
396 use crate::SEXPTYPE::{INTSXP, STRSXP, VECSXP};
397
398 // Materialize character columns into protected STRSXPs now (CHARSXP
399 // allocation must be serial on the R thread). Each freshly allocated
400 // STRSXP is protected immediately and stays protected until rooted in
401 // the parent VECSXP, exactly like the native columns.
402 //
403 // `native_protected` counts the protections phase 1 left on the
404 // stack; we add one per character column we protect here.
405 let mut native_protected = 0i32;
406 let mut col_sexps: Vec<SEXP> = Vec::with_capacity(ncol);
407 for kind in kinds {
408 match kind {
409 ColumnKind::Native(sexp) => {
410 native_protected += 1;
411 col_sexps.push(sexp);
412 }
413 ColumnKind::Str(values) => {
414 let sexp =
415 crate::sys::Rf_allocVector_unchecked(STRSXP, nrow as crate::R_xlen_t);
416 crate::sys::Rf_protect_unchecked(sexp);
417 native_protected += 1;
418 for (i, v) in values.iter().enumerate() {
419 match v {
420 Some(s) => {
421 sexp.set_string_elt_unchecked(i as isize, SEXP::charsxp(s))
422 }
423 None => {
424 sexp.set_string_elt_unchecked(i as isize, SEXP::na_string())
425 }
426 }
427 }
428 col_sexps.push(sexp);
429 }
430 }
431 }
432 // SAFETY: `native_protected` is a non-negative running count, so the
433 // sign cast to `usize` cannot lose data.
434 #[allow(clippy::cast_sign_loss)]
435 let native_protected_usize = native_protected as usize;
436 debug_assert_eq!(native_protected_usize, ncol);
437
438 // Allocate the parent list and protect it.
439 let df = crate::sys::Rf_allocVector_unchecked(VECSXP, ncol as crate::R_xlen_t);
440 crate::sys::Rf_protect_unchecked(df);
441
442 // Root every column in the parent (SET_VECTOR_ELT does not allocate).
443 for (i, col) in col_sexps.into_iter().enumerate() {
444 df.set_vector_elt_unchecked(i as isize, col);
445 }
446
447 // The columns are now reachable from `df`, so their individual
448 // protections are no longer needed. Drop all `ncol + 1` protections
449 // (the columns and `df`) and immediately re-protect `df` — no
450 // allocation happens between the two calls, so `df` cannot be
451 // collected in the gap.
452 crate::sys::Rf_unprotect_unchecked(native_protected + 1);
453 crate::sys::Rf_protect_unchecked(df);
454
455 // names: STRSXP of column names. Protect across CHARSXP allocations.
456 let names_sexp = crate::sys::Rf_allocVector_unchecked(STRSXP, ncol as crate::R_xlen_t);
457 crate::sys::Rf_protect_unchecked(names_sexp);
458 for (i, name) in names.iter().enumerate() {
459 let charsxp = SEXP::charsxp(name);
460 names_sexp.set_string_elt_unchecked(i as isize, charsxp);
461 }
462 df.set_names(names_sexp);
463 crate::sys::Rf_unprotect_unchecked(1); // names_sexp now reachable via df
464
465 // Compact row.names: c(NA_integer_, -nrow).
466 let row_names = crate::sys::Rf_allocVector_unchecked(INTSXP, 2);
467 crate::sys::Rf_protect_unchecked(row_names);
468 row_names.set_integer_elt(0, i32::MIN); // NA_integer_
469 // SAFETY: `nrow <= i32::MAX` asserted in `build_sexp`, so the
470 // narrowing cast cannot truncate the compact row.names count.
471 #[allow(clippy::cast_possible_truncation)]
472 let neg_nrow = -(nrow as i32);
473 row_names.set_integer_elt(1, neg_nrow);
474 df.set_row_names(row_names);
475 crate::sys::Rf_unprotect_unchecked(1); // row_names now reachable via df
476
477 // class = "data.frame" (cached STRSXP — no fresh allocation).
478 df.set_class(crate::cached_class::data_frame_class_sexp());
479
480 // Balance the remaining `df` protection. No allocation follows, so
481 // `df` survives until the caller takes ownership.
482 crate::sys::Rf_unprotect_unchecked(1);
483 df
484 })
485 }
486}
487
488// endregion
489
490#[cfg(test)]
491mod tests;