Skip to main content

miniextendr_api/
dataframe_builder.rs

1//! Closure-per-column [`DataFrame`](crate::dataframe::DataFrame) builder.
2//!
3//! [`RDataFrameBuilder`] assembles an R `data.frame` from a set of typed columns,
4//! each with its own element type and fill closure. It is the heterogeneous-column
5//! analogue of `with_r_matrix`: you declare columns, the builder allocates the
6//! backing storage with strict PROTECT discipline, fills it, and assembles the
7//! `data.frame` on the R thread.
8//!
9//! # Fill strategy (parallel with `rayon`, serial otherwise)
10//!
11//! The builder type and its surface are **available regardless of the `rayon`
12//! feature** (#1055). Only the fill pass differs:
13//!
14//! - **With `rayon`** — the whole job is flattened into a single work-list of
15//!   `(column_index, row-range)` items and filled in one work-stealing
16//!   `par_iter` pass (see [`RDataFrameBuilder`] docs for the scheduling argument).
17//! - **Without `rayon`** — each column is filled in one full-range serial pass.
18//!
19//! Both paths produce an identical `data.frame`. Nothing else in the builder
20//! depends on rayon: `with_r_thread`, `RNativeType`, the PROTECT
21//! discipline, and the assembly step are all rayon-independent. The `Send + Sync`
22//! bounds on the fill closures are simply unused when rayon is absent.
23
24use crate::{RNativeType, SEXP, SexpExt, with_r_thread};
25
26// region: type-erased column fill machinery
27
28/// Type-erased "fill this row-range of this column" closure.
29///
30/// Called with `(offset, len)` describing a contiguous half-open row range
31/// `[offset, offset + len)` of one column. The concrete closure (captured at
32/// `.column::<T>()` / `.column_str()` registration time) knows the column's
33/// element type and destination buffer, and writes exactly that range — no
34/// other thread touches it (see the safety argument on [`RDataFrameBuilder`]).
35type RangeFiller = Box<dyn Fn(usize, usize) + Send + Sync>;
36
37/// Send+Sync wrapper for a raw column-data pointer carried across the flatten
38/// boundary. The pointer addresses R-owned memory for native columns or a
39/// `Vec<Option<String>>` backing buffer for character columns. Disjointness
40/// (per column and per row-range within a column) is the caller's invariant —
41/// see the safety argument on [`RDataFrameBuilder`].
42#[derive(#[automatically_derived]#[doc(hidden)]unsafe impl ::core::clone::TrivialClone
    for ColPtr {
}
#[automatically_derived]impl ::core::clone::Clone for ColPtr {
    #[inline]
    fn clone(&self) -> Self {
        let _: ::core::clone::AssertParamIsClone<*mut ()>;
        *self
    }
}Clone, #[automatically_derived]impl ::core::marker::Copy for ColPtr { }Copy)]
43struct ColPtr(*mut ());
44
45unsafe impl Send for ColPtr {}
46unsafe impl Sync for ColPtr {}
47
48impl ColPtr {
49    /// Reinterpret the erased base pointer as `*mut T`.
50    ///
51    /// Taking `&self` (a method call on the whole struct) makes a capturing
52    /// closure capture the `Send + Sync` `ColPtr` as a whole rather than its
53    /// raw `*mut ()` field (which is neither), keeping the closure `Send + Sync`.
54    #[inline]
55    fn cast<T>(&self) -> *mut T {
56        self.0 as *mut T
57    }
58}
59
60/// How a column's R storage is materialized after the fill.
61enum ColumnKind {
62    /// Native column: the fill wrote directly into R memory; the SEXP is already
63    /// complete. Holds the allocated (and currently protected) SEXP.
64    Native(SEXP),
65    /// Character column: the fill computed `Option<String>` values into a `Vec`
66    /// (no R API on fill threads). The `CHARSXP`s are set serially on the R
67    /// thread during assembly. `None` becomes `NA_character_`.
68    Str(Vec<Option<String>>),
69}
70
71/// One registered column: a serial allocation step plus the range filler that
72/// the flattened work-list (or serial loop) dispatches.
73struct ColumnReg {
74    /// Allocates the column's backing storage (R SEXP for native columns, an
75    /// owned `Vec` for character columns) for `nrow` rows. Runs serially on
76    /// R's main thread. Returns the [`ColumnKind`] (carrying the protected SEXP
77    /// or the owned buffer) and the raw data pointer the range filler writes
78    /// through during the fill phase.
79    #[allow(clippy::type_complexity)]
80    alloc: Box<dyn FnOnce(usize) -> (ColumnKind, ColPtr) + Send>,
81    /// Builds the type-erased range filler once the data pointer is known.
82    make_filler: Box<dyn FnOnce(ColPtr, usize) -> RangeFiller + Send>,
83}
84
85// endregion
86
87// region: RDataFrameBuilder
88
89/// Builder for assembling an R `data.frame` from per-column fill closures.
90///
91/// This is the heterogeneous-column analogue of `with_r_matrix`: instead of one
92/// homogeneous matrix, you declare a set of typed columns (each with its own
93/// element type and fill closure) and the builder fills them all, then assembles
94/// the `data.frame`.
95///
96/// # Fill strategy (parallel with `rayon`, serial otherwise)
97///
98/// The builder is available regardless of the `rayon` feature; only the fill
99/// pass differs (see the [module docs](crate::dataframe_builder)). The rest of
100/// this section describes the parallel pass that `rayon` enables.
101///
102/// ## Two axes of parallelism, one work-stealing pass
103///
104/// There are two ways to parallelise a column fill:
105///
106/// - **Column-granular** — one task per column. Fan-out width equals the column
107///   count, so a 3-column × 10M-row frame only ever uses 3 threads.
108/// - **Row-slice-granular** — split *one* column into contiguous row ranges.
109///   Great for one long column, but on its own it serialises across columns.
110///
111/// `RDataFrameBuilder` does **not** choose. With `rayon`,
112/// [`build`][RDataFrameBuilder::build] flattens the entire job into a single
113/// work-list of `(column_index, row-range)` items — each native/character column
114/// is split into `chunk_size = max(1, nrow / (current_num_threads() * 4))`-row
115/// chunks (with 4× oversubscription) — then runs **one** `par_iter` over that
116/// flat list. Rayon's work-stealing balances both axes automatically:
117///
118/// - **wide** (100 cols × short) → ~100+ items, column-dominated.
119/// - **tall** (3 cols × 10M rows) → each column shatters into `~nthreads*4`
120///   chunks → hundreds of items, saturated even with 3 columns.
121/// - **skewed** (1 huge col + many tiny) → the huge column's chunks get stolen
122///   by threads idle after finishing the tiny columns.
123///
124/// This also avoids the per-column barrier and repeated pool spin-up that the
125/// naive "fill each column, each internally parallel" (nested `par_iter`) shape
126/// would cause. Without `rayon`, each column is filled in one full-range pass.
127///
128/// # Phases
129///
130/// 1. Allocate each column's backing storage **serially on R's main thread**
131///    (native columns get a protected R vector; character columns get an owned
132///    `Vec<Option<String>>`). Strict PROTECT discipline — the dangerous part.
133/// 2. Fill all columns (parallel flat pass with `rayon`, serial otherwise). No R
134///    API calls happen inside the parallel region.
135/// 3. Set character `CHARSXP`s serially on the R thread (CHARSXP allocation is
136///    forbidden on rayon threads), then assemble the `VECSXP`, `names`, compact
137///    `row.names` (`c(NA_integer_, -nrow)`), and `class = "data.frame"`.
138///
139/// # Column kinds
140///
141/// - [`column::<T>`][RDataFrameBuilder::column] — a native-typed column
142///   (`f64`/`i32`/`RLogical`/`u8`/`Rcomplex`). The fill closure receives a
143///   mutable chunk and its offset. The buffer is R memory, filled directly with
144///   zero intermediate allocation.
145/// - [`column_str`][RDataFrameBuilder::column_str] — a character (`STRSXP`)
146///   column. The per-row `Option<String>` values are computed during the fill
147///   pass, but the `CHARSXP`s are set **serially** afterward. `None` becomes
148///   `NA_character_`.
149///
150/// # Example
151///
152/// ```ignore
153/// use miniextendr_api::dataframe_builder::RDataFrameBuilder;
154///
155/// let df = RDataFrameBuilder::new(1000)
156///     .column::<f64>("x", |chunk, offset| {
157///         for (i, slot) in chunk.iter_mut().enumerate() {
158///             *slot = ((offset + i) as f64).sqrt();
159///         }
160///     })
161///     .column::<i32>("y", |chunk, offset| {
162///         for (i, slot) in chunk.iter_mut().enumerate() {
163///             *slot = (offset + i) as i32;
164///         }
165///     })
166///     .column_str("label", |i| Some(format!("row_{i}")))
167///     .build();
168/// ```
169///
170/// # Safety argument (disjoint mutation, no aliasing)
171///
172/// Neither fill path ever produces two items that overlap:
173///
174/// - Different columns address **different** backing buffers (distinct R vectors
175///   / distinct `Vec`s), so cross-column items are trivially disjoint.
176/// - Within a column, the row ranges are a partition of `[0, nrow)`. The serial
177///   path uses the single full range; the parallel path chunks `nrow` into
178///   fixed-size, non-overlapping spans. Each `(offset, len)` item therefore owns
179///   a unique slice of that column's buffer.
180///
181/// Each `RangeFiller` reconstitutes its slice via
182/// `slice::from_raw_parts_mut(base.add(offset), len)` and writes only that span.
183/// Because the spans are disjoint, no two threads ever form overlapping `&mut`
184/// references — there is no aliasing UB even though the work-list shares the raw
185/// base pointers (`ColPtr`, `Send + Sync`).
186///
187/// # Protection
188///
189/// Every native column SEXP is PROTECTed from allocation through insertion into
190/// the `VECSXP`; the `names` / `row.names` / class transients are likewise
191/// protected across each subsequent allocation. Before those temporary guards
192/// drop, [`build`][RDataFrameBuilder::build] transfers the assembled frame into
193/// a GC-rooted [`BuiltDataFrame`](crate::dataframe::BuiltDataFrame). The root is
194/// released when the handle is returned to R via `IntoR` or dropped in Rust.
195pub struct RDataFrameBuilder {
196    nrow: usize,
197    names: Vec<String>,
198    columns: Vec<ColumnReg>,
199}
200
201impl RDataFrameBuilder {
202    /// Start building a data.frame with `nrow` rows.
203    pub fn new(nrow: usize) -> Self {
204        // Compact row.names uses i32, so nrow must fit in i32; this also implies
205        // it fits in R_xlen_t on all supported pointer widths.
206        if !(nrow <= i32::MAX as usize) {
    {
        ::core::panicking::panic_fmt(format_args!("RDataFrameBuilder: nrow {0} exceeds i32 maximum (compact row.names)",
                nrow));
    }
};assert!(
207            nrow <= i32::MAX as usize,
208            "RDataFrameBuilder: nrow {} exceeds i32 maximum (compact row.names)",
209            nrow
210        );
211        Self {
212            nrow,
213            names: Vec::new(),
214            columns: Vec::new(),
215        }
216    }
217
218    /// Add a native-typed column (`f64`/`i32`/`RLogical`/`u8`/`Rcomplex`).
219    ///
220    /// The fill closure `f(chunk, offset)` is dispatched over chunks of the
221    /// (already-allocated) R column buffer — in parallel with `rayon`, or in one
222    /// full-range pass otherwise. Chunk boundaries are deterministic for a given
223    /// `nrow` and thread count.
224    pub fn column<T>(
225        mut self,
226        name: impl Into<String>,
227        f: impl Fn(&mut [T], usize) + Send + Sync + 'static,
228    ) -> Self
229    where
230        T: RNativeType + Send + Sync,
231    {
232        self.names.push(name.into());
233        self.columns.push(ColumnReg {
234            alloc: Box::new(|nrow| {
235                // Allocate + protect the R vector serially on the R thread, then
236                // hand back its data pointer for the fill. The protection is
237                // balanced during assembly in `build`.
238                let (sexp, ptr) = with_r_thread(move || unsafe {
239                    let sexp =
240                        crate::sys::Rf_allocVector_unchecked(T::SEXP_TYPE, nrow as crate::R_xlen_t);
241                    crate::sys::Rf_protect_unchecked(sexp);
242                    (sexp, T::dataptr_mut(sexp))
243                });
244                (ColumnKind::Native(sexp), ColPtr(ptr as *mut ()))
245            }),
246            make_filler: Box::new(move |base: ColPtr, nrow: usize| {
247                Box::new(move |offset: usize, len: usize| {
248                    if true {
    if !(offset + len <= nrow) {
        ::core::panicking::panic("assertion failed: offset + len <= nrow")
    };
};debug_assert!(offset + len <= nrow);
249                    // Safety: this range `[offset, offset+len)` is a disjoint
250                    // partition of this column's buffer (see the safety argument
251                    // on `RDataFrameBuilder`); no other thread writes it. `base`
252                    // is the `Send + Sync` `ColPtr`; cast inside the closure so
253                    // the closure stays `Send + Sync`.
254                    let ptr = base.cast::<T>();
255                    let slice = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), len) };
256                    f(slice, offset);
257                })
258            }),
259        });
260        self
261    }
262
263    /// Add a character (`STRSXP`) column.
264    ///
265    /// The fill closure `f(i)` returns the value for row `i` as `Option<String>`,
266    /// where `None` maps to `NA_character_`. Values are computed during the fill
267    /// pass (parallel with `rayon`, serial otherwise), then set into the R
268    /// `STRSXP` serially on the R thread (CHARSXP allocation cannot happen on
269    /// rayon threads).
270    pub fn column_str(
271        mut self,
272        name: impl Into<String>,
273        f: impl Fn(usize) -> Option<String> + Send + Sync + 'static,
274    ) -> Self {
275        self.names.push(name.into());
276        self.columns.push(ColumnReg {
277            alloc: Box::new(|nrow| {
278                // No R allocation here: the fill phase fills an owned Vec.
279                let mut buf: Vec<Option<String>> = (0..nrow).map(|_| None).collect();
280                let ptr = buf.as_mut_ptr();
281                (ColumnKind::Str(buf), ColPtr(ptr as *mut ()))
282            }),
283            make_filler: Box::new(move |base: ColPtr, nrow: usize| {
284                Box::new(move |offset: usize, len: usize| {
285                    if true {
    if !(offset + len <= nrow) {
        ::core::panicking::panic("assertion failed: offset + len <= nrow")
    };
};debug_assert!(offset + len <= nrow);
286                    // Safety: disjoint partition of this column's Vec buffer.
287                    // Cast `base` (Send + Sync `ColPtr`) inside the closure.
288                    let ptr = base.cast::<Option<String>>();
289                    let slice = unsafe { std::slice::from_raw_parts_mut(ptr.add(offset), len) };
290                    for (i, slot) in slice.iter_mut().enumerate() {
291                        *slot = f(offset + i);
292                    }
293                })
294            }),
295        });
296        self
297    }
298
299    /// Allocate, fill, and assemble the data.frame, returning an owned,
300    /// GC-rooted [`BuiltDataFrame`](crate::dataframe::BuiltDataFrame) (it
301    /// `Deref`s to [`DataFrame`](crate::dataframe::DataFrame)).
302    ///
303    /// With `rayon`, flattens every column into a single `(column_index,
304    /// row-range)` work-list and runs one parallel pass over it (see the
305    /// type-level docs for the scheduling argument); without `rayon`, fills each
306    /// column serially. Then assembles the `data.frame` on the R thread.
307    pub fn build(self) -> crate::dataframe::BuiltDataFrame {
308        // SAFETY: `build_sexp` returns a well-formed data.frame VECSXP; root it
309        // immediately (no allocation between assembly and adopt).
310        unsafe { crate::dataframe::BuiltDataFrame::adopt_sexp(self.build_sexp()) }
311    }
312
313    /// Assemble and return the raw `VECSXP` SEXP (internal; prefer [`build`](Self::build)).
314    fn build_sexp(self) -> SEXP {
315        let RDataFrameBuilder {
316            nrow,
317            names,
318            columns,
319        } = self;
320        let ncol = columns.len();
321        {
    match (&names.len(), &ncol) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val,
                    ::core::option::Option::Some(format_args!("RDataFrameBuilder: names/columns length mismatch")));
            }
        }
    }
};assert_eq!(
322            names.len(),
323            ncol,
324            "RDataFrameBuilder: names/columns length mismatch"
325        );
326        // Compact row.names `c(NA, -nrow)` are emitted as INTSXP, so `nrow` must
327        // fit in `i32`. Validate up front (panic → R error) rather than letting
328        // `-(nrow as i32)` below silently wrap for >2^31-row frames.
329        if !(nrow <= i32::MAX as usize) {
    {
        ::core::panicking::panic_fmt(format_args!("RDataFrameBuilder: nrow {0} exceeds i32 maximum for compact row.names",
                nrow));
    }
};assert!(
330            nrow <= i32::MAX as usize,
331            "RDataFrameBuilder: nrow {nrow} exceeds i32 maximum for compact row.names"
332        );
333
334        // Phase 1: allocate every column's backing storage serially. Native
335        // columns return a freshly-protected R SEXP and its data pointer;
336        // character columns return an owned `Vec<Option<String>>` and its
337        // pointer. We re-protect native columns *as they are allocated* (inside
338        // `alloc`), so the per-column allocation in the next iteration cannot GC
339        // an earlier column. These protections are balanced during assembly.
340        let mut kinds: Vec<ColumnKind> = Vec::with_capacity(ncol);
341        let mut fillers: Vec<RangeFiller> = Vec::with_capacity(ncol);
342        for col in columns {
343            let ColumnReg { alloc, make_filler } = col;
344            let (kind, ptr) = alloc(nrow);
345            kinds.push(kind);
346            fillers.push(make_filler(ptr, nrow));
347        }
348
349        // Phase 2: fill every column. No R API calls happen here.
350        if nrow > 0 && ncol > 0 {
351            #[cfg(feature = "rayon")]
352            {
353                use rayon::prelude::*;
354                crate::optionals::parallel::ensure_pool();
355                // Flatten to ONE (column, row-range) work-list and run a single
356                // parallel pass. Each item is `(column_index, offset, len)`; the
357                // column's type-erased range filler writes exactly that disjoint
358                // span. Rayon's work-stealing balances the column axis and the
359                // row-slice axis together.
360                let chunk_size = std::cmp::max(1, nrow / (rayon::current_num_threads() * 4));
361                let work: Vec<(usize, usize, usize)> = (0..ncol)
362                    .flat_map(|col_idx| {
363                        (0..nrow).step_by(chunk_size).map(move |offset| {
364                            let len = std::cmp::min(chunk_size, nrow - offset);
365                            (col_idx, offset, len)
366                        })
367                    })
368                    .collect();
369
370                work.par_iter().for_each(|&(col_idx, offset, len)| {
371                    (fillers[col_idx])(offset, len);
372                });
373            }
374            #[cfg(not(feature = "rayon"))]
375            {
376                // Serial fill: each column filled in one full-range pass.
377                // ponytail: the chunked work-list only buys parallelism, which is
378                // exactly what `rayon` adds; the result is identical either way.
379                for filler in &fillers {
380                    filler(0, nrow);
381                }
382            }
383        }
384        // Fillers are no longer needed; drop them before assembly so any captured
385        // closures release before we touch R.
386        drop(fillers);
387
388        // Phase 3: assemble on the R thread with strict PROTECT discipline. We
389        // are inside `with_r_thread`, a known-safe context, so `_unchecked` FFI
390        // is correct here (MXL301).
391        //
392        // PROTECT-stack invariant on entry: phase 1 left one protection per
393        // *native* column (character columns hold no SEXP yet). Track that exact
394        // count and balance it precisely.
395        with_r_thread(move || unsafe {
396            use crate::SEXPTYPE::{INTSXP, STRSXP, VECSXP};
397
398            // Materialize character columns into protected STRSXPs now (CHARSXP
399            // allocation must be serial on the R thread). Each freshly allocated
400            // STRSXP is protected immediately and stays protected until rooted in
401            // the parent VECSXP, exactly like the native columns.
402            //
403            // `native_protected` counts the protections phase 1 left on the
404            // stack; we add one per character column we protect here.
405            let mut native_protected = 0i32;
406            let mut col_sexps: Vec<SEXP> = Vec::with_capacity(ncol);
407            for kind in kinds {
408                match kind {
409                    ColumnKind::Native(sexp) => {
410                        native_protected += 1;
411                        col_sexps.push(sexp);
412                    }
413                    ColumnKind::Str(values) => {
414                        let sexp =
415                            crate::sys::Rf_allocVector_unchecked(STRSXP, nrow as crate::R_xlen_t);
416                        crate::sys::Rf_protect_unchecked(sexp);
417                        native_protected += 1;
418                        for (i, v) in values.iter().enumerate() {
419                            match v {
420                                Some(s) => {
421                                    sexp.set_string_elt_unchecked(i as isize, SEXP::charsxp(s))
422                                }
423                                None => {
424                                    sexp.set_string_elt_unchecked(i as isize, SEXP::na_string())
425                                }
426                            }
427                        }
428                        col_sexps.push(sexp);
429                    }
430                }
431            }
432            // SAFETY: `native_protected` is a non-negative running count, so the
433            // sign cast to `usize` cannot lose data.
434            #[allow(clippy::cast_sign_loss)]
435            let native_protected_usize = native_protected as usize;
436            if true {
    {
        match (&native_protected_usize, &ncol) {
            (left_val, right_val) => {
                if !(*left_val == *right_val) {
                    let kind = ::core::panicking::AssertKind::Eq;
                    ::core::panicking::assert_failed(kind, &*left_val,
                        &*right_val, ::core::option::Option::None);
                }
            }
        }
    };
};debug_assert_eq!(native_protected_usize, ncol);
437
438            // Allocate the parent list and protect it.
439            let df = crate::sys::Rf_allocVector_unchecked(VECSXP, ncol as crate::R_xlen_t);
440            crate::sys::Rf_protect_unchecked(df);
441
442            // Root every column in the parent (SET_VECTOR_ELT does not allocate).
443            for (i, col) in col_sexps.into_iter().enumerate() {
444                df.set_vector_elt_unchecked(i as isize, col);
445            }
446
447            // The columns are now reachable from `df`, so their individual
448            // protections are no longer needed. Drop all `ncol + 1` protections
449            // (the columns and `df`) and immediately re-protect `df` — no
450            // allocation happens between the two calls, so `df` cannot be
451            // collected in the gap.
452            crate::sys::Rf_unprotect_unchecked(native_protected + 1);
453            crate::sys::Rf_protect_unchecked(df);
454
455            // names: STRSXP of column names. Protect across CHARSXP allocations.
456            let names_sexp = crate::sys::Rf_allocVector_unchecked(STRSXP, ncol as crate::R_xlen_t);
457            crate::sys::Rf_protect_unchecked(names_sexp);
458            for (i, name) in names.iter().enumerate() {
459                let charsxp = SEXP::charsxp(name);
460                names_sexp.set_string_elt_unchecked(i as isize, charsxp);
461            }
462            df.set_names(names_sexp);
463            crate::sys::Rf_unprotect_unchecked(1); // names_sexp now reachable via df
464
465            // Compact row.names: c(NA_integer_, -nrow).
466            let row_names = crate::sys::Rf_allocVector_unchecked(INTSXP, 2);
467            crate::sys::Rf_protect_unchecked(row_names);
468            row_names.set_integer_elt(0, i32::MIN); // NA_integer_
469            // SAFETY: `nrow <= i32::MAX` asserted in `build_sexp`, so the
470            // narrowing cast cannot truncate the compact row.names count.
471            #[allow(clippy::cast_possible_truncation)]
472            let neg_nrow = -(nrow as i32);
473            row_names.set_integer_elt(1, neg_nrow);
474            df.set_row_names(row_names);
475            crate::sys::Rf_unprotect_unchecked(1); // row_names now reachable via df
476
477            // class = "data.frame" (cached STRSXP — no fresh allocation).
478            df.set_class(crate::cached_class::data_frame_class_sexp());
479
480            // Balance the remaining `df` protection. No allocation follows, so
481            // `df` survives until the caller takes ownership.
482            crate::sys::Rf_unprotect_unchecked(1);
483            df
484        })
485    }
486}
487
488// endregion
489
490#[cfg(test)]
491mod tests;