The audit log
audit.log.jsonl format and querying
This page is implementation detail, beyond what normal use requires.
Every init and add is appended to audit.log.jsonl in the storage
directory. The log is append-only and newline-delimited JSON, with one operation
per line.
🔗Setup
options(width = 1000)
library(dvs)
library(fs)
library(here)source(here::here("R/mkdatasetfiles.R"))storage <- basename(tempfile(fileext = "_storage"))
new_project <- basename(tempfile(fileext = "_project"))
dir.create(here::here(storage))
dir.create(here::here(new_project))Run a sequence of dvs operations to populate the log:
setwd(here::here(new_project))
mkdatasetfiles(n_files = 3, size_mb = 2, prefix = "file_", dir = "data", show_progress = !nzchar(Sys.getenv("QUARTO_DOCUMENT_PATH")))
dvs_init(here::here(storage))# A tibble: 1 × 4
compression metadata_folder_name backend_path backend_group
<chr> <lgl> <chr> <chr>
1 zstd NA /Users/elea/Documents/a2ai_github/dvs2-demo-repo/file8150331eb0f2_storage staff setwd(here::here(new_project))
invisible(dvs_add("data/file_1.csv", message = "initial add of file 1"))
invisible(dvs_add("data/file_2.csv", message = "initial add of file 2"))
invisible(dvs_add("data/file_3.csv", message = "initial add of file 3"))Restore a file with get. get does not append to the audit log; only init
and add are recorded.
setwd(here::here(new_project))
unlink("data/file_1.csv")
dvs_get("data/file_1.csv")# Restored 1
# A tibble: 1 × 13
path local_path before action satisfied size target_hash local_hash stored_copy code message hint duration_ms
* <chr> <chr> <chr> <chr> <lgl> <bytes> <chr> <chr> <chr> <chr> <chr> <chr> <dbl>
1 data/file_1.csv data/file_1.csv absent restored TRUE 2.0 MB 716693ea8b20e5a73abf0e5e35247cc232f1ae66c573ed1bc013cab8199f2cae <NA> <NA> <NA> <NA> <NA> 6Modify file_2.csv and re-add:
setwd(here::here(new_project))
cat("99,0,0,0,0,0\n", file = "data/file_2.csv", append = TRUE)
invisible(dvs_add("data/file_2.csv", message = "updated file 2, added row 99"))🔗Reading the log
The audit log is a newline-delimited JSON file (audit.log.jsonl) in the
storage directory. Each line is one operation:
audit_path <- here::here(storage, "audit.log.jsonl")Raw file, one JSON object per line:
lines <- readLines(audit_path)
cat("```json\n")
```r
cat(lines, sep = "\n"){"operation_id":"584853c1-2f31-4173-8475-ae531e5ee54e","timestamp":1793358670,"user":"elea","action":{"init":{"settings":{"compression":"zstd","metadata_folder_name":null,"backend":{"path":"/Users/elea/Documents/a2ai_github/dvs2-demo-repo/file8150331eb0f2_storage","group":"staff"}},"project_path":"/Users/elea/Documents/a2ai_github/dvs2-demo-repo/file815023f2fa16_project"}}} {"operation_id":"7c489a9b-0048-4744-a023-54251aa579be","timestamp":1793358670,"user":"elea","action":{"add":{"file":{"path":"data/file_1.csv","hashes":{"blake3":"716693ea8b20e5a73abf0e5e35247cc232f1ae66c573ed1bc013cab8199f2cae"}},"compression":"zstd","message":"initial add of file 1"}}} {"operation_id":"03ee6e34-8da3-46ae-bcca-eb93225d1e3b","timestamp":1793358670,"user":"elea","action":{"add":{"file":{"path":"data/file_2.csv","hashes":{"blake3":"59548c554d27252f5f5cf9a11ba5df4de216602831b000a9105c56094c958e88"}},"compression":"zstd","message":"initial add of file 2"}}} {"operation_id":"1a507427-06f4-4858-badd-40a1b205a74b","timestamp":1793358670,"user":"elea","action":{"add":{"file":{"path":"data/file_3.csv","hashes":{"blake3":"2d9855e7f48b07cb280cfba4596893f9fc9b615ff96107d8b83446a12ee0451d"}},"compression":"zstd","message":"initial add of file 3"}}} {"operation_id":"5375b6ad-782c-4451-9e0e-05925d5bbe9c","timestamp":1793358670,"user":"elea","action":{"add":{"file":{"path":"data/file_2.csv","hashes":{"blake3":"db1eb2d8544f2c649b86a8058342273d221f9d9a0134227d03d36c924e9c85f9"}},"compression":"zstd","message":"updated file 2, added row 99"}}}
cat("\n```\n")
Same entries, pretty-printed:
```r
cat("```json\n")
```r
for (l in lines) {
cat(jsonlite::toJSON(jsonlite::fromJSON(l, simplifyVector = FALSE), pretty = TRUE, auto_unbox = TRUE))
cat("\n")
}{ "operation_id": "584853c1-2f31-4173-8475-ae531e5ee54e", "timestamp": 1793358670, "user": "elea", "action": { "init": { "settings": { "compression": "zstd", "metadata_folder_name": {}, "backend": { "path": "/Users/elea/Documents/a2ai_github/dvs2-demo-repo/file8150331eb0f2_storage", "group": "staff" } }, "project_path": "/Users/elea/Documents/a2ai_github/dvs2-demo-repo/file815023f2fa16_project" } } } { "operation_id": "7c489a9b-0048-4744-a023-54251aa579be", "timestamp": 1793358670, "user": "elea", "action": { "add": { "file": { "path": "data/file_1.csv", "hashes": { "blake3": "716693ea8b20e5a73abf0e5e35247cc232f1ae66c573ed1bc013cab8199f2cae" } }, "compression": "zstd", "message": "initial add of file 1" } } } { "operation_id": "03ee6e34-8da3-46ae-bcca-eb93225d1e3b", "timestamp": 1793358670, "user": "elea", "action": { "add": { "file": { "path": "data/file_2.csv", "hashes": { "blake3": "59548c554d27252f5f5cf9a11ba5df4de216602831b000a9105c56094c958e88" } }, "compression": "zstd", "message": "initial add of file 2" } } } { "operation_id": "1a507427-06f4-4858-badd-40a1b205a74b", "timestamp": 1793358670, "user": "elea", "action": { "add": { "file": { "path": "data/file_3.csv", "hashes": { "blake3": "2d9855e7f48b07cb280cfba4596893f9fc9b615ff96107d8b83446a12ee0451d" } }, "compression": "zstd", "message": "initial add of file 3" } } } { "operation_id": "5375b6ad-782c-4451-9e0e-05925d5bbe9c", "timestamp": 1793358670, "user": "elea", "action": { "add": { "file": { "path": "data/file_2.csv", "hashes": { "blake3": "db1eb2d8544f2c649b86a8058342273d221f9d9a0134227d03d36c924e9c85f9" } }, "compression": "zstd", "message": "updated file 2, added row 99" } } }
cat("```\n")
# Parsing the log
Each entry has the shape:
```json
{
"operation_id": "<uuid>",
"timestamp": <unix-seconds>,
"user": "<username>",
"action": { "<command>": { ... } }
}The action field is a tagged union. The single key is the command name,
either "init" or "add" (get is not logged). Parse it with purrr and
jsonlite:
parse_entry <- function(line) {
entry <- jsonlite::fromJSON(line)
cmd <- names(entry$action)[1]
# init records settings and project_path, not a file, so it has no path.
path <- switch(cmd,
add = entry$action$add$file$path,
init = NA_character_,
NA_character_
)
list(
operation_id = entry$operation_id,
timestamp = as.POSIXct(entry$timestamp, origin = "1970-01-01"),
user = entry$user,
command = cmd,
path = path
)
}
audit_tbl <- readLines(audit_path) |>
purrr::map(parse_entry) |>
dplyr::bind_rows()
audit_tbl# A tibble: 5 × 5
operation_id timestamp user command path
<chr> <dttm> <chr> <chr> <chr>
1 584853c1-2f31-4173-8475-ae531e5ee54e 2026-10-30 12:11:10 elea init <NA>
2 7c489a9b-0048-4744-a023-54251aa579be 2026-10-30 12:11:10 elea add data/file_1.csv
3 03ee6e34-8da3-46ae-bcca-eb93225d1e3b 2026-10-30 12:11:10 elea add data/file_2.csv
4 1a507427-06f4-4858-badd-40a1b205a74b 2026-10-30 12:11:10 elea add data/file_3.csv
5 5375b6ad-782c-4451-9e0e-05925d5bbe9c 2026-10-30 12:11:10 elea add data/file_2.csv🔗Querying the log
Operations by command type:
dplyr::count(audit_tbl, command, sort = TRUE)# A tibble: 2 × 2
command n
<chr> <int>
1 add 4
2 init 1Files touched per operation (excluding init):
audit_tbl |>
dplyr::filter(!is.na(path)) |>
dplyr::select(timestamp, command, path) |>
dplyr::arrange(timestamp)# A tibble: 4 × 3
timestamp command path
<dttm> <chr> <chr>
1 2026-10-30 12:11:10 add data/file_1.csv
2 2026-10-30 12:11:10 add data/file_2.csv
3 2026-10-30 12:11:10 add data/file_3.csv
4 2026-10-30 12:11:10 add data/file_2.csvHow many times was each file touched?
audit_tbl |>
dplyr::filter(!is.na(path)) |>
dplyr::count(path, command, sort = TRUE)# A tibble: 3 × 3
path command n
<chr> <chr> <int>
1 data/file_2.csv add 2
2 data/file_1.csv add 1
3 data/file_3.csv add 1🔗Cleanup
unlink(here::here(new_project), recursive = TRUE)
unlink(here::here(storage), recursive = TRUE)🔗See also
- Storage and meta files: the blob and meta layout.
- The dvs.toml project file: the project configuration.
- The command references: dvs_add() / dvs add.