From 8fc6da2bae832ed9ceebb521fb086cb273eabf2d Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Fri, 21 Aug 2026 17:13:46 +0100 Subject: [PATCH 01/20] Increment version number to 0.2.2.9000 --- DESCRIPTION | 2 +- NEWS.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 3fc07d9..b8e4be9 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: dsROCrate Title: 'DataSHIELD' RO-Crate Governance Functions -Version: 0.2.2 +Version: 0.2.2.9000 Authors@R: c( person(given = "Roberto", family = "Villegas-Diaz", diff --git a/NEWS.md b/NEWS.md index ee3e9ee..d1c4cef 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,5 @@ +# dsROCrate (development version) + # dsROCrate 0.2.2 ## Bug fixes From ade4258c224670a551a593b51a7bfbd187a8ddf2 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Fri, 28 Aug 2026 11:22:24 +0100 Subject: [PATCH 02/20] Update documentation for enrich_argument and add new cases to capture symbols when the arguments given are of the form SYMBOL$COLUMN --- R/safe-call-utils.R | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/R/safe-call-utils.R b/R/safe-call-utils.R index 553e296..dbcb5bd 100644 --- a/R/safe-call-utils.R +++ b/R/safe-call-utils.R @@ -5,7 +5,8 @@ #' @param timestamp Timestamp to map the symbol details. #' @param session String with session ID. #' -#' @returns `safe_reference` object. +#' @returns `safe_reference` object, or the original argument when no +#' registered symbol can be resolved. #' @keywords internal #' #' @noRd @@ -14,6 +15,7 @@ enrich_argument <- function(arg, registry, timestamp, session) { return(arg) } + # attempt solving the argument as a direct symbol reference symbol <- resolve_symbol( registry, symbol = arg, @@ -21,13 +23,42 @@ enrich_argument <- function(arg, registry, timestamp, session) { session = session ) + if (!is.null(symbol)) { + return(safe_reference(symbol = symbol$symbol, symbol_id = symbol$id)) + } + + # if the argument isn't itself a symbol, try interpreting it as an R expr. + refs <- tryCatch( + find_symbols(arg), + error = function(e) NULL + ) + + if (is.null(refs) || !length(refs)) { + return(arg) + } + + # if multiple symbols are found, return original obkect + if (length(refs) != 1L) { + return(arg) + } + + ref <- refs[[1]] + + symbol <- resolve_symbol( + registry, + symbol = ref$symbol, + timestamp = timestamp, + session = session + ) + if (is.null(symbol)) { return(arg) } safe_reference( symbol = symbol$symbol, - symbol_id = symbol$id + symbol_id = symbol$id, + column = ref$column ) } From d5c0ca95cbdbda45cca1edffd9d2aafea9171459 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Fri, 28 Aug 2026 11:22:59 +0100 Subject: [PATCH 03/20] Include column in the returned object by safe_output --- R/safe_output.R | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R/safe_output.R b/R/safe_output.R index cb183b8..1498a38 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -105,6 +105,7 @@ safe_output.opal <- function( ds_action <- ds_eval <- ds_id <- ds_function <- ds_symbol <- ds_table <- NULL asset <- action <- is_placeholder <- kind <- symbol_id <- timestamp <- NULL ds_profile <- expr <- fx <- log_id <- r_cmd <- session <- symbol <- NULL + column <- NULL if (is.null(profile)) { profile <- "default" @@ -419,6 +420,7 @@ safe_output.opal <- function( r_cmd, fx, symbol, + column, kind, asset, expr, From 4b5df2d833c19e6333d9ac852201c504e99d706f Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Wed, 2 Sep 2026 16:50:25 +0100 Subject: [PATCH 04/20] Add command level ID for the logs, this helps keeping track of commands that are split into multiple rows when mapping different symbols within a call --- R/safe-call.R | 4 ++++ R/safe-symbol.R | 2 +- R/safe_output.R | 7 ++++++- R/utils-safe_output.R | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/R/safe-call.R b/R/safe-call.R index 40e2d26..8f4e54f 100644 --- a/R/safe-call.R +++ b/R/safe-call.R @@ -1,6 +1,7 @@ #' @export as.data.frame.safe_call <- function(x, ...) { data.frame( + id = x$id, timestamp = format(x$created_at, '%Y-%m-%dT%H:%M:%S'), user = x$user, r_cmd = x$original, @@ -19,6 +20,7 @@ as.data.frame.safe_call <- function(x, ...) { #' @param namespace Function's namespace. #' @param fx Function's name. #' @param args List with arguments. +#' @param id Unique call ID. #' @param ... Additional arguments. #' #' @returns New `safe_call` object. @@ -31,6 +33,7 @@ new_safe_call <- function( namespace, fx, args, + id = uuid::UUIDgenerate(), ... ) { stopifnot(is.character(original)) @@ -39,6 +42,7 @@ new_safe_call <- function( structure( list( + id = id, original = original, package = package, namespace = namespace, diff --git a/R/safe-symbol.R b/R/safe-symbol.R index b3dc956..2058a62 100644 --- a/R/safe-symbol.R +++ b/R/safe-symbol.R @@ -48,7 +48,7 @@ new_safe_symbol <- function( session = NA_character_, action = NA_character_, metadata = list(), - id = paste0("symbol-", uuid::UUIDgenerate()) + id = uuid::UUIDgenerate() ) { stopifnot(is.character(symbol)) stopifnot(length(symbol) == 1) diff --git a/R/safe_output.R b/R/safe_output.R index 1498a38..45c5fcf 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -230,7 +230,8 @@ safe_output.opal <- function( dplyr::filter(`@timestamp` >= logs_from, `@timestamp` <= logs_to) |> dplyr::filter(logger_name == "datashield.user") |> dplyr::filter(ds_profile == profile) |> - dplyr::filter(username %in% user) + dplyr::filter(username %in% user) |> + (\(.) dplyr::mutate(., id = uuid::UUIDgenerate(n = nrow(.)), .before = 1))() userlogs <- NULL if (nrow(userlogs_tbl) > 0) { @@ -296,6 +297,7 @@ safe_output.opal <- function( symbols_tbl <- seq_len(nrow(userlogs_assign_tbl)) |> purrr::map(function(i) { # extract log components + id <- getElement(userlogs_assign_tbl[i, ], "id") ds_eval <- getElement(userlogs_assign_tbl[i, ], "ds_eval") ds_resource <- getElement(userlogs_assign_tbl[i, ], "ds_resource") ds_symbol <- getElement(userlogs_assign_tbl[i, ], "ds_symbol") @@ -307,6 +309,7 @@ safe_output.opal <- function( is_table <- !is.null(ds_table) && !is.na(ds_table) tibble::tibble( + id = id, symbol = ds_symbol, kind = ifelse( is_expr, @@ -414,6 +417,7 @@ safe_output.opal <- function( dplyr::mutate(backend = "OBiBa's Opal") |> # subset columns dplyr::select( + id, timestamp, action, user, @@ -442,6 +446,7 @@ safe_output.opal <- function( backend = "OBiBa's Opal" ) |> dplyr::select( + id, timestamp, action = ds_action, user = username, diff --git a/R/utils-safe_output.R b/R/utils-safe_output.R index 78f3578..ccde77e 100644 --- a/R/utils-safe_output.R +++ b/R/utils-safe_output.R @@ -11,6 +11,7 @@ calls_to_tbl <- function(calls, registry) { if (length(calls) == 0L) { return( tibble::tibble( + id = character(), timestamp = character(), action = character(), user = character(), @@ -27,6 +28,7 @@ calls_to_tbl <- function(calls, registry) { purrr::map(calls, \(x) { tibble::tibble( + id = x$id, timestamp = format(x$created_at, "%Y-%m-%dT%H:%M:%S"), action = "AGGREGATE", user = x$user, From ba405a80f0eb738d25b4d154d62c2da93744f55e Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 3 Sep 2026 13:50:08 +0100 Subject: [PATCH 05/20] Update code to include ASSIGN operations in the output logs --- R/safe_output.R | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/R/safe_output.R b/R/safe_output.R index 45c5fcf..bb6c00a 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -231,7 +231,7 @@ safe_output.opal <- function( dplyr::filter(logger_name == "datashield.user") |> dplyr::filter(ds_profile == profile) |> dplyr::filter(username %in% user) |> - (\(.) dplyr::mutate(., id = uuid::UUIDgenerate(n = nrow(.)), .before = 1))() + dplyr::mutate(id = paste0("", dplyr::row_number())) userlogs <- NULL if (nrow(userlogs_tbl) > 0) { @@ -456,13 +456,39 @@ safe_output.opal <- function( backend ) + # parse symbols to log entries + assign_tbl <- symbols_tbl |> + dplyr::transmute( + id, + timestamp = format(created_at, "%Y-%m-%dT%H:%M:%S"), + action = action, + user, + r_cmd = dplyr::case_when( + kind == "table" ~ paste0(symbol, " <- opal[", asset, "]"), + kind == "resource" ~ paste0(symbol, " <- opal[", asset, "]"), + kind == "expression" ~ expr, + TRUE ~ NA_character_ + ), + fx = created_by, + symbol, + column = NA_character_, + kind, + asset, + expr, + session, + backend = "OBiBa's Opal" + ) + # combine the logs - userlogs_tbl_maps_evals <- calls_symbols_tbl |> + userlogs_tbl_maps_evals <- dplyr::bind_rows( + assign_tbl, + calls_symbols_tbl, + session_tbl + ) |> dplyr::distinct() |> - dplyr::mutate(log_id = dplyr::row_number()) |> - dplyr::bind_rows(session_tbl) |> - dplyr::arrange(timestamp, log_id) |> - dplyr::select(-log_id) + dplyr::arrange(as.numeric(id), timestamp) |> + # regenerate unique operation IDs + (\(.) dplyr::mutate(., id = uuid::UUIDgenerate(n = nrow(.)), .before = 1))() log_maps_filename <- paste0( format(Sys.time(), "%Y%m%dT%H%M%S"), From 64e607910500feb1cb25f1e2e7822e174f6018f8 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 3 Sep 2026 15:37:05 +0100 Subject: [PATCH 06/20] Update code used to update unique command IDs --- R/safe_output.R | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/R/safe_output.R b/R/safe_output.R index bb6c00a..b2a56d2 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -488,7 +488,18 @@ safe_output.opal <- function( dplyr::distinct() |> dplyr::arrange(as.numeric(id), timestamp) |> # regenerate unique operation IDs - (\(.) dplyr::mutate(., id = uuid::UUIDgenerate(n = nrow(.)), .before = 1))() + (\(.) { + dplyr::left_join( + ., + tibble::tibble( + id = unique(.$id), + uuid = uuid::UUIDgenerate(n = length(id)) + ), + by = "id" + ) + })() |> + dplyr::mutate(id = uuid) |> + dplyr::select(-uuid) log_maps_filename <- paste0( format(Sys.time(), "%Y%m%dT%H%M%S"), From 6712624617d1c6a06cc0530a88100be9e5f44195 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 3 Sep 2026 17:36:07 +0100 Subject: [PATCH 07/20] Bump version number to next patch release --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index b8e4be9..ac93cb4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: dsROCrate Title: 'DataSHIELD' RO-Crate Governance Functions -Version: 0.2.2.9000 +Version: 0.2.3 Authors@R: c( person(given = "Roberto", family = "Villegas-Diaz", From 48cd32690e002a9d113c7996e2f6b4354818c93a Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Wed, 9 Sep 2026 16:55:46 +0100 Subject: [PATCH 08/20] Update local bindings --- R/safe_output.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/safe_output.R b/R/safe_output.R index b2a56d2..eb2dea6 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -105,7 +105,7 @@ safe_output.opal <- function( ds_action <- ds_eval <- ds_id <- ds_function <- ds_symbol <- ds_table <- NULL asset <- action <- is_placeholder <- kind <- symbol_id <- timestamp <- NULL ds_profile <- expr <- fx <- log_id <- r_cmd <- session <- symbol <- NULL - column <- NULL + column <- created_at <- created_by <- id <- uuid <- NULL if (is.null(profile)) { profile <- "default" From b62b03121bff626038477a5414951224972b2c58 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Wed, 9 Sep 2026 16:55:58 +0100 Subject: [PATCH 09/20] Update test expected output --- tests/testthat/test-safe-call.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testthat/test-safe-call.R b/tests/testthat/test-safe-call.R index 9ca87ca..d329630 100644 --- a/tests/testthat/test-safe-call.R +++ b/tests/testthat/test-safe-call.R @@ -29,7 +29,7 @@ test_that("as.data.frame.safe_call returns the expected columns", { expect_equal(nrow(res), 1) expect_setequal( names(res), - c("timestamp", "user", "r_cmd", "fx", "args", "session", "profile") + c("id", "timestamp", "user", "r_cmd", "fx", "args", "session", "profile") ) expect_equal(res$user, "alice") expect_equal(res$session, "session-1") From cdd3350d981f4bb5d892dbd2f9220bf1fdbd8de4 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Fri, 11 Sep 2026 11:57:15 +0100 Subject: [PATCH 10/20] Suppress warnings when checking minimum version of Opal, as instances like "6.0.0-SNAPSHOT" resulted in a NAs introduced by coercion, even when the check was successfully done --- R/backend-opal.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/backend-opal.R b/R/backend-opal.R index c8dec8c..71ee8ed 100644 --- a/R/backend-opal.R +++ b/R/backend-opal.R @@ -110,7 +110,7 @@ is_audit_con.opal <- function(x, ...) { #' @export validate_backend_version.opal <- function(x, ..., minimum = "5.7.2") { - if (utils::compareVersion(x$version, minimum) < 0) { + if (suppressWarnings(utils::compareVersion(x$version, minimum)) < 0) { stop( sprintf( "Opal >= %s is required, but server version is %s.", From 6d2d05e798d528df0f765e5d8bd37decf7ae8c8b Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Fri, 11 Sep 2026 11:57:44 +0100 Subject: [PATCH 11/20] Validate that symbol_id is not empty inside the resolve_provenance helper --- R/safe-symbol-utils.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/safe-symbol-utils.R b/R/safe-symbol-utils.R index dc26956..180a579 100644 --- a/R/safe-symbol-utils.R +++ b/R/safe-symbol-utils.R @@ -102,7 +102,7 @@ resolve_provenance <- function(symbol_id, registry, visited = character()) { # local bindings id <- NULL - if (symbol_id %in% visited) { + if (symbol_id %in% visited || is.na(symbol_id)) { return(tibble::tibble()) } From 4ec020b6051d7ceaabbd2ed2df1017667513451b Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 13:51:23 +0100 Subject: [PATCH 12/20] Capture RESOLVE operations in the logs --- R/safe_output.R | 80 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/R/safe_output.R b/R/safe_output.R index eb2dea6..ce95f34 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -222,7 +222,7 @@ safe_output.opal <- function( # parse logs userlogs_tbl <- backend_logs(x) |> tibble::as_tibble() |> - dplyr::bind_rows(tibble::tibble(ds_profile = "-999999")) |> + dplyr::bind_rows(tibble::tibble(ds_profile = character())) |> dplyr::mutate( `@timestamp` = as.POSIXct(`@timestamp`, format = "%Y-%m-%dT%H:%M:%S") ) |> @@ -289,6 +289,20 @@ safe_output.opal <- function( ) # update symbol registry + ## check if Opal 6.0+ is available + opal_6_available <- tryCatch( + { + validate_backend_version(x, minimum = "6.0") + TRUE + }, + error = function(e) FALSE + ) + ## extract 'RESOLVE' operations from the logs + userlogs_resolve_tbl <- NULL + if (opal_6_available) { + userlogs_resolve_tbl <- userlogs_tbl |> + dplyr::filter(ds_action %in% c("RESOLVE")) + } ## extract 'ASSIGN' operations from the logs userlogs_assign_tbl <- userlogs_tbl |> dplyr::filter(ds_action %in% c("ASSIGN")) @@ -345,6 +359,69 @@ safe_output.opal <- function( purrr::list_c() |> dplyr::distinct() + ## reshape RESOLVE operations into log entries + resolve_tbl <- tibble::tibble( + id = character(), + timestamp = character(), + action = character(), + user = character(), + r_cmd = character(), + fx = character(), + symbol = character(), + column = character(), + kind = character(), + asset = character(), + expr = character(), + session = character(), + backend = character() + ) + + if (!is.null(userlogs_resolve_tbl) && nrow(userlogs_resolve_tbl) > 0) { + resolve_tbl <- userlogs_resolve_tbl |> + # insert column placeholders for ds_resource and ds_table + dplyr::bind_rows(tibble::tibble( + ds_resource = character(), + ds_table = character() + )) |> + dplyr::transmute( + id, + timestamp = format(`@timestamp`, "%Y-%m-%dT%H:%M:%S"), + action = ds_action, + user = username, + r_cmd = dplyr::case_when( + !is.na(ds_resource) ~ paste0( + "RESOLVE ", + ds_resource, + " -> ", + ds_symbol + ), + !is.na(ds_table) ~ paste0( + "RESOLVE ", + ds_table, + " -> ", + ds_symbol + ), + TRUE ~ NA_character_ + ), + fx = NA_character_, + symbol = ds_symbol, + column = NA_character_, + kind = dplyr::case_when( + !is.na(ds_resource) ~ "resource", + !is.na(ds_table) ~ "table", + TRUE ~ NA_character_ + ), + asset = dplyr::case_when( + !is.na(ds_resource) ~ ds_resource, + !is.na(ds_table) ~ ds_table, + TRUE ~ NA_character_ + ), + expr = NA_character_, + session = ds_id, + backend = "OBiBa's Opal" + ) + } + ## add symbols to registry registry <- symbols_tbl |> purrr::pmap(safe_symbol) |> @@ -481,6 +558,7 @@ safe_output.opal <- function( # combine the logs userlogs_tbl_maps_evals <- dplyr::bind_rows( + resolve_tbl, assign_tbl, calls_symbols_tbl, session_tbl From b20596075fbec36ad83cb68a421552da47d5ae7b Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:09:16 +0100 Subject: [PATCH 13/20] Add filter to handle logs without AGGREGATE operations --- R/safe_output.R | 162 +++++++++++++++++++++++++++--------------------- 1 file changed, 90 insertions(+), 72 deletions(-) diff --git a/R/safe_output.R b/R/safe_output.R index ce95f34..ea8308f 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -436,79 +436,97 @@ safe_output.opal <- function( enrich_call(registry = registry) }) - # convert list of calls into tibble - calls_tbl <- calls_to_tbl(calls_lst, registry) - - # combine function calls with symbol's registry - calls_symbols_tbl <- calls_tbl |> - dplyr::select(-symbol) |> - dplyr::mutate( - args = purrr::map( - args, - ~ purrr::imap_dfr(.x, function(arg, nm) { - if (!inherits(arg, "safe_reference")) { - tibble::tibble( - argument = nm, - value = list(arg), - symbol_id = NA_character_, - symbol = NA_character_, - column = NA_character_ - ) - } else { - tibble::tibble( - argument = nm, - value = list(arg), - symbol_id = arg$symbol_id, - symbol = arg$symbol, - column = arg$column - ) - } - }) - ) - ) |> - (\(x) { - purrr::map2( - split(x |> dplyr::select(-args), seq_len(nrow(x))), - x$args, - dplyr::bind_cols - ) - })() |> - purrr::list_c() |> - dplyr::left_join( - registry$symbols, - by = c("symbol_id" = "id"), - suffix = c("", "_registry") - ) |> - dplyr::mutate( - asset = dplyr::if_else( - kind == "expression", - purrr::map_chr( - symbol_id, - resolve_symbol_asset, - registry = registry - ), - asset - ) - ) |> - # add column with backend - dplyr::mutate(backend = "OBiBa's Opal") |> - # subset columns - dplyr::select( - id, - timestamp, - action, - user, - r_cmd, - fx, - symbol, - column, - kind, - asset, - expr, - # table = ds_table, - session, - backend + if (length(calls_lst) == 0) { + calls_symbols_tbl <- tibble::tibble( + id = character(), + timestamp = character(), + action = character(), + user = character(), + r_cmd = character(), + fx = character(), + symbol = character(), + column = character(), + kind = character(), + asset = character(), + expr = character(), + session = character(), + backend = character() ) + } else { + # convert list of calls into tibble + calls_tbl <- calls_to_tbl(calls_lst, registry) + + # combine function calls with symbol's registry + calls_symbols_tbl <- calls_tbl |> + dplyr::select(-symbol) |> + dplyr::mutate( + args = purrr::map( + args, + ~ purrr::imap_dfr(.x, function(arg, nm) { + if (!inherits(arg, "safe_reference")) { + tibble::tibble( + argument = nm, + value = list(arg), + symbol_id = NA_character_, + symbol = NA_character_, + column = NA_character_ + ) + } else { + tibble::tibble( + argument = nm, + value = list(arg), + symbol_id = arg$symbol_id, + symbol = arg$symbol, + column = arg$column + ) + } + }) + ) + ) |> + (\(x) { + purrr::map2( + split(x |> dplyr::select(-args), seq_len(nrow(x))), + x$args, + dplyr::bind_cols + ) + })() |> + purrr::list_c() |> + dplyr::left_join( + registry$symbols, + by = c("symbol_id" = "id"), + suffix = c("", "_registry") + ) |> + dplyr::mutate( + asset = dplyr::if_else( + kind == "expression", + purrr::map_chr( + symbol_id, + resolve_symbol_asset, + registry = registry + ), + asset + ) + ) |> + # add column with backend + dplyr::mutate(backend = "OBiBa's Opal") |> + # subset columns + dplyr::select( + id, + timestamp, + action, + user, + r_cmd, + fx, + symbol, + column, + kind, + asset, + expr, + # table = ds_table, + session, + backend + ) + } # extract session details session_tbl <- userlogs_tbl |> From d039dc3bb375def7fc219cbc66f16fad6c1e7d08 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:09:38 +0100 Subject: [PATCH 14/20] Add filter to handle empty symbol registry --- R/safe-symbol-utils.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/safe-symbol-utils.R b/R/safe-symbol-utils.R index 180a579..b5d97a8 100644 --- a/R/safe-symbol-utils.R +++ b/R/safe-symbol-utils.R @@ -70,7 +70,7 @@ resolve_dependencies <- function(expr, registry) { # find symbols for the given expression refs <- find_symbols(expr) - if (is.null(refs)) { + if (is.null(refs) || !length(registry$symbols)) { return(tibble::tibble()) } From 74ef9bd5a5c7919db47fa7cedb297c9010b15f8e Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:35:53 +0100 Subject: [PATCH 15/20] Update processing of symbols_tbl to handle scenarios without ASSIGN operations in the logs --- R/safe_output.R | 112 +++++++++++++++++++++++++++--------------------- 1 file changed, 63 insertions(+), 49 deletions(-) diff --git a/R/safe_output.R b/R/safe_output.R index ea8308f..cfd870f 100644 --- a/R/safe_output.R +++ b/R/safe_output.R @@ -308,56 +308,70 @@ safe_output.opal <- function( dplyr::filter(ds_action %in% c("ASSIGN")) ## reshape the logs into a tibble of `symbols` - symbols_tbl <- seq_len(nrow(userlogs_assign_tbl)) |> - purrr::map(function(i) { - # extract log components - id <- getElement(userlogs_assign_tbl[i, ], "id") - ds_eval <- getElement(userlogs_assign_tbl[i, ], "ds_eval") - ds_resource <- getElement(userlogs_assign_tbl[i, ], "ds_resource") - ds_symbol <- getElement(userlogs_assign_tbl[i, ], "ds_symbol") - ds_table <- getElement(userlogs_assign_tbl[i, ], "ds_table") - - # evaluate which fields are populated - is_expr <- !is.null(ds_eval) && !is.na(ds_eval) - is_resource <- !is.null(ds_resource) && !is.na(ds_resource) - is_table <- !is.null(ds_table) && !is.na(ds_table) - - tibble::tibble( - id = id, - symbol = ds_symbol, - kind = ifelse( - is_expr, - 'expression', - ifelse( - is_resource, - 'resource', - ifelse(is_table, 'table', NA_character_) - ) - ), - asset = ifelse( - is_resource, - ds_resource, - ifelse(is_table, ds_table, NA_character_) - ), - expr = ifelse(is_expr, ds_eval, NA_character_), - # expr = if (is_expr) str2lang(ds_eval) else NULL, - created_by = ifelse( - is_expr, - 'DSI::datashield.assign.expr', - ifelse( + symbols_tbl <- if (nrow(userlogs_assign_tbl) == 0) { + tibble::tibble( + id = character(), + symbol = character(), + kind = character(), + asset = character(), + expr = character(), + created_by = character(), + created_at = as.POSIXct(character()), + user = character(), + action = character(), + session = character() + ) + } else { + seq_len(nrow(userlogs_assign_tbl)) |> + purrr::map(function(i) { + # extract log components + id <- getElement(userlogs_assign_tbl[i, ], "id") + ds_eval <- getElement(userlogs_assign_tbl[i, ], "ds_eval") + ds_resource <- getElement(userlogs_assign_tbl[i, ], "ds_resource") + ds_symbol <- getElement(userlogs_assign_tbl[i, ], "ds_symbol") + ds_table <- getElement(userlogs_assign_tbl[i, ], "ds_table") + + # evaluate which fields are populated + is_expr <- !is.null(ds_eval) && !is.na(ds_eval) + is_resource <- !is.null(ds_resource) && !is.na(ds_resource) + is_table <- !is.null(ds_table) && !is.na(ds_table) + + tibble::tibble( + id = id, + symbol = ds_symbol, + kind = ifelse( + is_expr, + 'expression', + ifelse( + is_resource, + 'resource', + ifelse(is_table, 'table', NA_character_) + ) + ), + asset = ifelse( is_resource, - 'DSI::datashield.assign.resource', - ifelse(is_table, 'DSI::datashield.assign.table', NA_character_) - ) - ), - created_at = userlogs_assign_tbl$`@timestamp`[[i]], - user = userlogs_assign_tbl$username[[i]], - action = userlogs_assign_tbl$ds_action[[i]], - session = userlogs_assign_tbl$ds_id[[i]] - ) - }) |> - purrr::list_c() |> - dplyr::distinct() + ds_resource, + ifelse(is_table, ds_table, NA_character_) + ), + expr = ifelse(is_expr, ds_eval, NA_character_), + created_by = ifelse( + is_expr, + 'DSI::datashield.assign.expr', + ifelse( + is_resource, + 'DSI::datashield.assign.resource', + ifelse(is_table, 'DSI::datashield.assign.table', NA_character_) + ) + ), + created_at = userlogs_assign_tbl$`@timestamp`[[i]], + user = userlogs_assign_tbl$username[[i]], + action = userlogs_assign_tbl$ds_action[[i]], + session = userlogs_assign_tbl$ds_id[[i]] + ) + }) |> + dplyr::bind_rows() |> + dplyr::distinct() + } ## reshape RESOLVE operations into log entries resolve_tbl <- tibble::tibble( From 735853fe1aecd5efe79f6d82427bea60a8916ec6 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:36:14 +0100 Subject: [PATCH 16/20] Add new tests to check the handle of RESOLVE operations --- tests/testthat/test-safe_output.R | 129 ++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) diff --git a/tests/testthat/test-safe_output.R b/tests/testthat/test-safe_output.R index 8d6f04d..e8cb59b 100644 --- a/tests/testthat/test-safe_output.R +++ b/tests/testthat/test-safe_output.R @@ -200,3 +200,132 @@ test_that("safe_output works", { # close connection to OBiBa's Opal demo server opalr::opal.logout(opal_con) }) + +test_that("safe_output captures RESOLVE and ASSIGN operations independently", { + local_mocked_bindings( + validate_backend = function(x, ...) invisible(TRUE), + validate_backend_version = function(x, ..., minimum = "5.7.2") { + invisible(TRUE) + }, + backend_logs = function(x, ...) { + tibble::tibble( + `@timestamp` = as.POSIXct( + c( + "2026-09-24 10:00:00", + "2026-09-24 10:00:01", + "2026-09-24 10:00:02" + ), + tz = "UTC" + ), + logger_name = "datashield.user", + level = "INFO", + message = c( + "Session opened", + "Resolved CNSIM.CNSIM1 to D", + "Assigned D from CNSIM.CNSIM1" + ), + ds_action = c("OPEN", "RESOLVE", "ASSIGN"), + ds_id = "session-1", + ds_profile = "default", + username = "test_user", + ds_symbol = c(NA_character_, "D", "D"), + ds_table = c(NA_character_, "CNSIM.CNSIM1", "CNSIM.CNSIM1"), + ds_resource = rep(NA_character_, 3), + ds_eval = rep(NA_character_, 3) + ) + }, + .package = "dsROCrate" + ) + + out <- suppressWarnings( + safe_output( + fake_opal_con(), + user = "test_user", + rocrate = rocrateR::rocrate_5s() + ) + ) + + files <- .get_entity(out, type = "File") + mapping <- purrr::keep(files, \(x) grepl("_mappings\\.csv$", x$`@id`))[[1]] + mappings <- mapping$content[[1]] + + expect_equal(mappings$action, c("OPEN", "RESOLVE", "ASSIGN")) + + expect_equal( + mappings |> + dplyr::filter(action == "RESOLVE") |> + dplyr::select(symbol, kind, asset), + tibble::tibble( + symbol = "D", + kind = "table", + asset = "CNSIM.CNSIM1" + ) + ) + + expect_equal( + mappings |> + dplyr::filter(action == "ASSIGN") |> + dplyr::select(symbol, kind, asset), + tibble::tibble( + symbol = "D", + kind = "table", + asset = "CNSIM.CNSIM1" + ) + ) +}) + +test_that("safe_output retains RESOLVE operations without ASSIGN", { + local_mocked_bindings( + validate_backend = function(x, ...) invisible(TRUE), + validate_backend_version = function(x, ..., minimum = "5.7.2") { + invisible(TRUE) + }, + backend_logs = function(x, ...) { + tibble::tibble( + `@timestamp` = as.POSIXct( + c( + "2026-09-24 10:00:00", + "2026-09-24 10:00:01" + ), + tz = "UTC" + ), + logger_name = "datashield.user", + level = "INFO", + message = c( + "Session opened", + "Unable to resolve CNSIM.MISSING to D" + ), + ds_action = c("OPEN", "RESOLVE"), + ds_id = "session-1", + ds_profile = "default", + username = "test_user", + ds_symbol = c(NA_character_, "D"), + ds_table = c(NA_character_, "CNSIM.MISSING"), + ds_resource = rep(NA_character_, 2), + ds_eval = rep(NA_character_, 2) + ) + }, + .package = "dsROCrate" + ) + + out <- suppressWarnings( + safe_output( + fake_opal_con(), + user = "test_user", + rocrate = rocrateR::rocrate_5s() + ) + ) + + files <- .get_entity(out, type = "File") + mapping <- purrr::keep(files, \(x) grepl("_mappings\\.csv$", x$`@id`))[[1]] + mappings <- mapping$content[[1]] + + resolve <- mappings |> + dplyr::filter(action == "RESOLVE") + + expect_equal(nrow(resolve), 1) + expect_equal(resolve$symbol, "D") + expect_equal(resolve$asset, "CNSIM.MISSING") + expect_equal(resolve$kind, "table") + expect_false("ASSIGN" %in% mappings$action) +}) From 8686eeee137511c74407e8c9204d99690167923b Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:36:29 +0100 Subject: [PATCH 17/20] Add comment for upcoming release --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index d1c4cef..ea40398 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # dsROCrate (development version) +## New features + +* `safe_output()` now preserves Opal 6.0 `RESOLVE` operations independently of `ASSIGN` operations, including the resolved symbol and source table/resource for provenance. + # dsROCrate 0.2.2 ## Bug fixes From 051afbcd822490ea33f80cd805ae9308d9e28c2c Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 15:36:40 +0100 Subject: [PATCH 18/20] Bump version of roxygen2 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index ac93cb4..6caa38c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -59,4 +59,4 @@ Depends: VignetteBuilder: knitr URL: https://github.com/FederatedMethods/dsROCrate BugReports: https://github.com/FederatedMethods/dsROCrate/issues -Config/roxygen2/version: 8.0.0 +Config/roxygen2/version: 8.1.0 From 71d76c6733aa8ab8ce42216d84dbbaf673b57bea Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 16:26:30 +0100 Subject: [PATCH 19/20] Update NEWS for release --- NEWS.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index ea40398..5d0dd09 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,29 @@ -# dsROCrate (development version) +# dsROCrate 0.2.3 -## New features +## New Features + +* `safe_output()` now preserves Opal 6.0 `RESOLVE` operations independently of `ASSIGN` operations, including the resolved symbol and source table/resource for provenance. `RESOLVE` entries are retained even when no matching `ASSIGN` follows (e.g., a failed resolution). This is only enabled when the server reports Opal >= 6.0. +* Every entry in the audit log tibble returned via `safe_output()` (and written to the `_mappings.csv` file) now has a unique `id` (UUID), and the tibble gains a `column` field recording the column referenced by a function call argument (e.g. `D$LAB_TSC`), where applicable. + +## Improvements + +* Function-call arguments supplied as R expressions (e.g. `D$LAB_TSC`) are now resolved back to the single registered symbol they reference, along with the referenced column. Expressions that reference several symbols, or none that are registered, are returned unchanged. +* Audit log entries are now ordered by their original log-entry order and then timestamp, and operation IDs are regenerated as UUIDs. +* Symbol IDs generated by `safe_symbol()` are now plain UUIDs, without the previous `"symbol-"` prefix. + +## Bug fixes + +* Fixed `safe_output()` failing when an audit session contains no `ASSIGN` operations: an empty, correctly typed symbols table is now used. +* Fixed `safe_output()` failing when no function calls could be reconstructed from the logs: an empty, correctly typed calls table is now used. +* `safe_output()` no longer injects a placeholder row (`ds_profile = "-999999"`) into the parsed logs; a zero-row typed tibble is used to guarantee the `ds_profile` column exists instead. +* Fixed `resolve_dependencies()` erroring on an empty symbol registry, and `resolve_provenance()` erroring when given an `NA` symbol ID. Both now return an empty tibble. +* `validate_backend_version()` (opal method) no longer emits warnings from `utils::compareVersion()`. + +## Internal Changes -* `safe_output()` now preserves Opal 6.0 `RESOLVE` operations independently of `ASSIGN` operations, including the resolved symbol and source table/resource for provenance. +* Added tests covering `safe_output()` with `RESOLVE` and `ASSIGN` operations, and `RESOLVE` operations without `ASSIGN`; updated `as.data.frame.safe_call()` test for the new `id` column. +* Declared additional local bindings in `safe_output()` to silence `R CMD check` notes about undefined global variables. +* Updated roxygen2 documentation for `enrich_argument()` and `safe_call()`; regenerated with roxygen2 8.1.0. # dsROCrate 0.2.2 From 091c2773626586caae6129b80277bf1323601df3 Mon Sep 17 00:00:00 2001 From: Roberto Villegas-Diaz Date: Thu, 24 Sep 2026 16:53:58 +0100 Subject: [PATCH 20/20] Update wording on NEWS --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 5d0dd09..957d8a8 100644 --- a/NEWS.md +++ b/NEWS.md @@ -16,7 +16,7 @@ * Fixed `safe_output()` failing when an audit session contains no `ASSIGN` operations: an empty, correctly typed symbols table is now used. * Fixed `safe_output()` failing when no function calls could be reconstructed from the logs: an empty, correctly typed calls table is now used. * `safe_output()` no longer injects a placeholder row (`ds_profile = "-999999"`) into the parsed logs; a zero-row typed tibble is used to guarantee the `ds_profile` column exists instead. -* Fixed `resolve_dependencies()` erroring on an empty symbol registry, and `resolve_provenance()` erroring when given an `NA` symbol ID. Both now return an empty tibble. +* Fixed `resolve_dependencies()` failing on an empty symbol registry, and `resolve_provenance()` failing when given an `NA` symbol ID. Both now return an empty tibble. * `validate_backend_version()` (opal method) no longer emits warnings from `utils::compareVersion()`. ## Internal Changes