From 9aa5ae990083a2b80d675ce76df114d76885cfb6 Mon Sep 17 00:00:00 2001 From: Evan Pierce Brenner <108823789+epbrenner@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:04:05 -0600 Subject: [PATCH 1/2] Updating exportProcessedData Allows exportProcessedData function to handle dynamic HMMER data using the manifest, removed old mention of domains, and added in a helper and exportProcessedData parameter to export a dyad feature table mapping all features linked to that dyad as a little bonus. --- R/data_processing.R | 139 ++++++++++++++-- R/helpers.R | 389 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 517 insertions(+), 11 deletions(-) diff --git a/R/data_processing.R b/R/data_processing.R index 12df346..f63431e 100644 --- a/R/data_processing.R +++ b/R/data_processing.R @@ -3192,6 +3192,7 @@ exportProcessedData <- function(duckdb_path, amr_phenotype_mode = c("separate", "append"), export_formats = c("csv"), export_sequences = FALSE, + export_dyads = FALSE, tables = NULL, export_tables = TRUE, verbose = TRUE) { @@ -3265,14 +3266,14 @@ exportProcessedData <- function(duckdb_path, df, file = file.path(output_path, paste0(stem, ".csv")), sep = ",", row.names = FALSE, col.names = TRUE, quote = TRUE, na = "", qmethod = "double", fileEncoding = "UTF-8" - ) + ) } if ("tsv" %in% export_formats) { utils::write.table( df, file = file.path(output_path, paste0(stem, ".tsv")), sep = "\t", row.names = FALSE, col.names = TRUE, quote = TRUE, na = "", qmethod = "double", fileEncoding = "UTF-8" - ) + ) } if ("parquet" %in% export_formats) { arrow::write_parquet(df, file.path(output_path, paste0(stem, ".parquet"))) @@ -3282,6 +3283,56 @@ exportProcessedData <- function(duckdb_path, } } + # Determine which HMMER databases were actually run from the latest + # successful `runDataProcessing()` manifest + manifest_path <- .manifest_find_latest(duckdb_path) + + hmmer_databases <- character() + + if (!is.null(manifest_path)) { + manifest <- jsonlite::read_json( + manifest_path, + simplifyVector = FALSE + ) + + successful_hmmer <- list() + + for (run in rev(manifest$runs %||% list())) { + stages <- run$stages %||% list() + + matches <- purrr::keep( + stages, + ~ identical(.x$name, "hmmer") && + identical(.x$status, "success") + ) + + if (length(matches)) { + successful_hmmer <- matches[[1]] + break + } + } + + if (length(successful_hmmer)) { + hmmer_databases <- unlist( + successful_hmmer$parameters$databases %||% character(), + use.names = FALSE + ) + hmmer_databases <- unique(as.character(hmmer_databases)) + } + } + + if (!length(hmmer_databases) && isTRUE(verbose)) { + message( + "No successful HMMER runs were found in the manifest. ", + "HMMER tables will not be selected automatically." + ) + } else if (isTRUE(verbose)) { + message( + "Successful HMMER runs were identified in the manifest: ", + paste(hmmer_databases, collapse = ", ") + ) + } + build_amr_wide <- function() { source_tbl <- if ("metadata" %in% available_tables) { "metadata" @@ -3326,35 +3377,75 @@ exportProcessedData <- function(duckdb_path, table_specs <- list( gene_count = list(source = "gene_count", stem = "gene_count", appendable = TRUE), protein_count = list(source = "protein_count", stem = "protein_count", appendable = TRUE), - domain_count = list(source = "domain_count", stem = "domain_count", appendable = TRUE), struct = list(source = "gene_struct", stem = "struct", appendable = TRUE), gene_names = list(source = "gene_names", stem = "gene_names", appendable = FALSE), protein_names = list(source = "protein_names", stem = "protein_names", appendable = FALSE), - domain_names = list(source = "domain_names", stem = "domain_names", appendable = FALSE), metadata = list(source = "metadata", stem = "metadata", appendable = FALSE), genome_data = list(source = "genome_data", stem = "genome_data", appendable = FALSE), amr_phenotype_wide = list(source = NULL, stem = "amr_phenotype_wide", appendable = FALSE) ) + # Add only the HMMER databases recorded in the manifest. + for (database in hmmer_databases) { + annotation_key <- paste0("protein_", database) + count_key <- paste0(annotation_key, "_count") + + table_specs[[annotation_key]] <- list( + source = annotation_key, + stem = annotation_key, + appendable = FALSE + ) + + table_specs[[count_key]] <- list( + source = count_key, + stem = count_key, + appendable = TRUE + ) + } + if (isTRUE(export_sequences)) { - table_specs$gene_seqs <- list(source = "gene_ref_seq", stem = "gene_seqs", appendable = FALSE) - table_specs$protein_seqs <- list(source = "protein_cluster_seq", stem = "protein_seqs", appendable = FALSE) - table_specs$genome_gene_protein <- list(source = "genome_gene_protein", stem = "genome_gene_protein", appendable = FALSE) + table_specs$gene_seqs <- list( + source = "gene_ref_seq", + stem = "gene_seqs", + appendable = FALSE + ) + + table_specs$protein_seqs <- list( + source = "protein_cluster_seq", + stem = "protein_seqs", + appendable = FALSE + ) + + table_specs$genome_gene_protein <- list( + source = "genome_gene_protein", + stem = "genome_gene_protein", + appendable = FALSE + ) } if (is.null(tables)) { selected_keys <- c( "gene_count", "protein_count", - "domain_count", "struct", "gene_names", "protein_names", - "domain_names", "metadata", "genome_data", - "amr_phenotype_wide" + "amr_phenotype_wide", + paste0( + "protein_", + hmmer_databases, + "_count" + ), + paste0( + "protein_", + hmmer_databases + ) ) + + selected_keys <- selected_keys[selected_keys %in% names(table_specs)] + if (isTRUE(export_sequences)) { selected_keys <- c(selected_keys, "gene_seqs", "protein_seqs", "genome_gene_protein") } @@ -3373,6 +3464,28 @@ exportProcessedData <- function(duckdb_path, } exported <- character(0) + # Optionally export a mapping table rooted on the protein-gene dyads + if (isTRUE(export_dyads)) { + dyad_tbl <- .exportDyadAnnotations( + duckdb_path = duckdb_path, + verbose = verbose + ) + + write_one( + dyad_tbl, + "dyad_annotations" + ) + + exported <- c( + exported, + "dyad_annotations" + ) + + if (isTRUE(verbose)) { + message("Exported: dyad_annotations") + } + } + for (key in selected_keys) { spec <- table_specs[[key]] @@ -3416,6 +3529,10 @@ exportProcessedData <- function(duckdb_path, tables = exported, amr_phenotype_mode = amr_phenotype_mode, export_formats = export_formats, - export_sequences = isTRUE(export_sequences) + export_sequences = isTRUE(export_sequences), + hmmer_databases = hmmer_databases, + manifest_path = manifest_path )) } + + diff --git a/R/helpers.R b/R/helpers.R index 76c3287..38b33ff 100644 --- a/R/helpers.R +++ b/R/helpers.R @@ -742,6 +742,395 @@ ) } +#' Export a dyad-centric feature table +#' +#' Builds a one-row-per-dyad table linking protein-gene dyads to structural +#' features and HMMER annotations recorded in the dataset provenance manifest. +#' Feature values are deduplicated and combined into semicolon-separated +#' character fields. +#' +#' @param duckdb_path Character. Path to the source dataset DuckDB. Associated +#' Parquet files and provenance manifest are expected there. +#' @param output_path Character or NULL. Directory where the output Parquet file +#' will be written. Defaults to the DuckDB directory. +#' @param output_stem Character. Output filename stem. Default +#' `"dyad_annotations"`. +#' @param feature_scales Character vector of optional feature types to include. +#' If NULL, includes `struct` plus all HMMER databases recorded in the +#' manifest. +#' @param verbose Logical. Print progress messages. +#' +#' @return Invisibly returns the path to the generated Parquet file. +#' +#' @keywords internal +.exportDyadAnnotations <- function( + duckdb_path, + feature_scales = NULL, + verbose = TRUE +) { + duckdb_path <- normalizePath(duckdb_path, mustWork = TRUE) + parquet_dir <- dirname(duckdb_path) + + manifest_path <- .manifest_find_latest(duckdb_path) + + if (is.null(manifest_path)) { + stop( + "No provenance manifest found for: ", + duckdb_path + ) + } + + manifest <- jsonlite::read_json( + manifest_path, + simplifyVector = FALSE + ) + + # Find latest successful HMMER stage + hmmer_stage <- NULL + + for (run in rev(manifest$runs %||% list())) { + stages <- run$stages %||% list() + + matches <- purrr::keep( + stages, + ~ identical(.x$name, "hmmer") && + identical(.x$status, "success") + ) + + if (length(matches)) { + hmmer_stage <- matches[[1]] + break + } + } + + if (is.null(hmmer_stage)) { + stop( + "No successful HMMER stage found in manifest: ", + manifest_path + ) + } + + hmmer_databases <- unique(as.character( + unlist( + hmmer_stage$parameters$databases %||% character(), + use.names = FALSE + ) + )) + + allowed_features <- c("struct", hmmer_databases) + + if (is.null(feature_scales)) { + feature_scales <- allowed_features + } else { + feature_scales <- unique(as.character(feature_scales)) + + unknown_features <- setdiff( + feature_scales, + allowed_features + ) + + if (length(unknown_features)) { + stop( + "Unsupported feature scale(s): ", + paste(unknown_features, collapse = ", "), + ". Available features: ", + paste(allowed_features, collapse = ", ") + ) + } + } + + con <- DBI::dbConnect( + duckdb::duckdb(), + dbdir = ":memory:" + ) + + duckdb_temp_dir <- file.path( + parquet_dir, + ".duckdb_temp" + ) + + dir.create( + duckdb_temp_dir, + recursive = TRUE, + showWarnings = FALSE + ) + + DBI::dbExecute( + con, + sprintf( + "SET temp_directory=%s", + DBI::dbQuoteString( + con, + normalizePath( + duckdb_temp_dir, + winslash = "/", + mustWork = TRUE + ) + ) + ) + ) + + on.exit( + { + try(DBI::dbDisconnect(con, shutdown = TRUE), silent = TRUE) + unlink( + duckdb_temp_dir, + recursive = TRUE, + force = TRUE + ) + }, + add = TRUE + ) + + parquet_sql <- function(dataset_name) { + path <- file.path( + parquet_dir, + paste0(dataset_name, ".parquet") + ) + + if (!file.exists(path)) { + return(NULL) + } + + normalizePath( + path, + winslash = "/", + mustWork = TRUE + ) + } + + # Initialize using protein-gene dyads + genome_gene_protein_path <- parquet_sql( + "genome_gene_protein" + ) + + if (is.null(genome_gene_protein_path)) { + stop( + "Required Parquet file not found: ", + file.path( + parquet_dir, + "genome_gene_protein.parquet" + ) + ) + } + + DBI::dbExecute( + con, + sprintf( + " + CREATE OR REPLACE VIEW protein_gene AS + SELECT DISTINCT + protein_ids AS protein, + REPLACE(Gene, '~', '.') AS gene + FROM read_parquet('%s') + WHERE protein_ids IS NOT NULL + AND Gene IS NOT NULL + ", + genome_gene_protein_path + ) + ) + + DBI::dbExecute( + con, + " + CREATE OR REPLACE VIEW protein_gene_dyad AS + SELECT DISTINCT + protein, + gene, + CONCAT(protein, '|', gene) AS dyad + FROM protein_gene + " + ) + + # Finish initializing with one row per dyad + feature_select <- character() + feature_joins <- character() + + # Pangenome graph structural variant ('struct') annotations + if ("struct" %in% feature_scales) { + struct_path <- parquet_sql("struct") + + if (is.null(struct_path)) { + if (isTRUE(verbose)) { + message( + "Skipping struct: struct.parquet was not found." + ) + } + } else { + DBI::dbExecute( + con, + sprintf( + " + CREATE OR REPLACE VIEW struct_genes AS + SELECT DISTINCT + s.struct, + t.gene + FROM read_parquet('%s') s + CROSS JOIN UNNEST( + string_split(s.struct, '.') + ) AS t(gene) + WHERE s.value = 1 + ", + struct_path + ) + ) + + DBI::dbExecute( + con, + " + CREATE OR REPLACE VIEW dyad_struct AS + SELECT + pgd.dyad, + string_agg( + DISTINCT sg.struct, + ';' + ORDER BY sg.struct + ) AS struct + FROM protein_gene_dyad pgd + JOIN struct_genes sg + ON pgd.gene = sg.gene + GROUP BY pgd.dyad + " + ) + + feature_select <- c( + feature_select, + "ds.struct" + ) + + feature_joins <- c( + feature_joins, + "LEFT JOIN dyad_struct ds ON b.dyad = ds.dyad" + ) + } + } + + # HMMER feature annotations + for (database in intersect( + hmmer_databases, + feature_scales + )) { + dataset_name <- paste0( + "protein_", + database + ) + + hmmer_path <- parquet_sql(dataset_name) + + if (is.null(hmmer_path)) { + if (isTRUE(verbose)) { + message( + "Skipping ", + database, + ": ", + dataset_name, + ".parquet was not found." + ) + } + next + } + + view_name <- paste0( + "dyad_", + make.names(database) + ) + + DBI::dbExecute( + con, + sprintf( + " + CREATE OR REPLACE VIEW %s AS + SELECT + pgd.dyad, + string_agg( + DISTINCT h.query_name, + ';' + ORDER BY h.query_name + ) AS feature + FROM protein_gene_dyad pgd + JOIN read_parquet('%s') h + ON pgd.protein = h.protein + WHERE h.query_name IS NOT NULL + GROUP BY pgd.dyad + ", + view_name, + hmmer_path + ) + ) + + # Give AMRFinder a better human-readable name (ARG, in this case) + output_column <- if (identical(database, "AMRFinder")) { + "ARG" + } else { + database + } + + alias <- paste0( + "d_", + make.names(database) + ) + + feature_select <- c( + feature_select, + sprintf( + '%s.feature AS "%s"', + alias, + output_column + ) + ) + + feature_joins <- c( + feature_joins, + sprintf( + "LEFT JOIN %s %s ON b.dyad = %s.dyad", + view_name, + alias, + alias + ) + ) + } + + select_features <- if (length(feature_select)) { + paste0( + ",\n ", + paste(feature_select, collapse = ",\n ") + ) + } else { + "" + } + + join_features <- if (length(feature_joins)) { + paste0( + "\n ", + paste(feature_joins, collapse = "\n ") + ) + } else { + "" + } + + result_sql <- paste0( + " + SELECT + b.dyad, + b.protein, + b.gene", + select_features, + " + FROM protein_gene_dyad b", + join_features + ) + + if (isTRUE(verbose)) { + message("Building dyad annotation table.") + } + + DBI::dbGetQuery( + con, + result_sql + ) |> + tibble::as_tibble() +} + ######################### # HMMER helpers # ######################### From 2742f643841f9ed8d25d4d927cab392a491fc782 Mon Sep 17 00:00:00 2001 From: Evan Pierce Brenner <108823789+epbrenner@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:55:58 -0600 Subject: [PATCH 2/2] Update exportProcessedData and documentation Updated stale documentation throughout, changed a parameter name (Total_proteins -> total_proteins), wired @AbhirupaGhosh's dyad exporter into `runDataProcessing()`, and added `exportProcessedData()` into `runDataProcessing()`. --- R/data_processing.R | 139 +++++++++++++++++++++++++++++++------------- 1 file changed, 99 insertions(+), 40 deletions(-) diff --git a/R/data_processing.R b/R/data_processing.R index f63431e..ba369d8 100644 --- a/R/data_processing.R +++ b/R/data_processing.R @@ -962,10 +962,17 @@ CDHIT2duckdb <- function(duckdb_path, #' Download and prepare HMMER databases for generating new file types. #' -#' @param hmmer_db_dir Directory to store HMMER databases -#' @param databases List of databases to prepare (default: c("Pfam", "COG", "AMRFinder")) -#' @param docker_image Docker image containing HMMER (default: "staphb/hmmer") -#' @param hmmer_db_url If the databases contain custom database(s), the url is required to download the database. +#' @param hmmer_db_dir Character. Directory where HMMER databases are cached. +#' @param databases Character vector of database names to prepare. Supported +#' built-in databases are `Pfam`, `COG`, and `AMRFinder`; `DefenseCas` is +#' prepared separately by the DefenseFinder/CasFinder workflow. +#' @param docker_image Character. Docker image containing HMMER tools used to +#' press the prepared databases. Default: `"staphb/hmmer"`. +#' @param hmmer_db_url NON-FUNCTIONAL. Character or `NULL`. URL used to download +#' a custom HMMER database when `databases` contains names not covered by the +#' built-in database definitions. This function is not currently active! +#' @param verbose Logical. Print status messages while checking, downloading, +#' combining, and pressing databases. Default: `TRUE`. #' #' @returns A list of paths to the database hmm files. #' @@ -1281,20 +1288,28 @@ CDHIT2duckdb <- function(duckdb_path, #' The function to run HMMER with docker #' -#' @param JOB_NAME -#' @param FASTA -#' @param DB -#' @param Total_proteins -#' @param output_path -#' @param db_paths -#' @param docker_image -#' @param threads -#' @param n_workers +##' @param JOB_NAME Character. Identifier used for the HMMER job and output +#' filename. +#' @param FASTA Character. File name of the protein FASTA chunk to search. +#' @param DB Character. Name of the HMMER database to search against. +#' @param total_proteins Integer. Total number of proteins in the full input +#' dataset, used to set HMMER's `-Z` and `--domZ` values. +#' @param output_path Character. Directory containing the FASTA input, HMMER +#' output, and final Parquet result. +#' @param db_paths List. Prepared HMMER database metadata indexed by database +#' name. +#' @param docker_image Character. Docker image containing HMMER. Default: +#' `"staphb/hmmer"`. +#' @param threads Integer. Total CPU budget used when calculating the number +#' of threads allocated to this job. Default: `8`. +#' @param n_workers Integer. Number of HMMER jobs being run in parallel. +#' Used to divide the CPU budget among jobs. Default: `8`. +#' @param verbose Logical. Print progress messages. Default: `TRUE`. #' #' @returns #' #' @keywords internal -.runHmmerJob <- function(JOB_NAME, FASTA, DB, Total_proteins, +.runHmmerJob <- function(JOB_NAME, FASTA, DB, total_proteins, output_path = NULL, db_paths, docker_image = "staphb/hmmer", threads = 8L, n_workers = 8L, @@ -1327,8 +1342,8 @@ CDHIT2duckdb <- function(duckdb_path, "hmmsearch", "--notextw", "--cpu", as.character(threads_per_job), - "-Z", Total_proteins, - "--domZ", Total_proteins, + "-Z", total_proteins, + "--domZ", total_proteins, "--domtblout", .to_container(hmmer_output, mount_host, mount_cont), db_cont_path, .to_container(hmmer_input, mount_host, mount_cont) @@ -1375,19 +1390,25 @@ CDHIT2duckdb <- function(duckdb_path, #' Wrapper for preparing HMM databases and running HMMER on protein sequences from duckdb and writing them. #' -#' @param duckdb_path -#' @param output_path -#' @param threads -#' @param hmmer_db_dir -#' @param databases -#' @param docker_image -#' @param num_of_splits -#' @param n_workers -#' +#' @param duckdb_path Character. Path to the DuckDB database containing +#' `protein_cluster_seq`, which provides the protein sequences to analyze. +#' @param output_path Character. Directory for HMMER intermediate and final +#' Parquet outputs. +#' @param threads Integer. Total CPU budget used by HMMER jobs. Default: `8`. +#' @param hmmer_db_dir Character. Directory containing the prepared HMMER +#' databases. If `NULL`, the default `amRdata` HMMER database cache is used. +#' @param databases Character vector of HMMER databases to run. +#' @param docker_image Character. Docker image containing HMMER. Default: +#' `"staphb/hmmer"`. +#' @param num_of_splits Integer. Number of chunks into which the protein +#' sequences should be divided. Must be a positive integer. The requested +#' value is automatically reduced when fewer protein sequences are available. +#' Default: `8`. +#' @param n_workers Integer. Number of parallel HMMER jobs to run. Default: `8`. +#' @param verbose Logical. Print progress messages. Default: `TRUE`.#' #' @returns #' #' @keywords internal -#' @examples .runHMMER <- function(duckdb_path, output_path, threads = 8L, @@ -1437,7 +1458,7 @@ CDHIT2duckdb <- function(duckdb_path, } # required to define the database size for hmmsearch --Z and --domZ parameters - Total_proteins <- nrow(prot_seqs) + total_proteins <- nrow(prot_seqs) if (is.null(hmmer_db_dir)) { hmmer_db_dir <- .defaultHmmerDbDir() @@ -1513,7 +1534,7 @@ CDHIT2duckdb <- function(duckdb_path, JOB_NAME = job_list$JOB_NAME[i], FASTA = job_list$FASTA[i], DB = job_list$DB[i], - Total_proteins = Total_proteins, + total_proteins = total_proteins, output_path = output_path, db_paths = db_paths, docker_image = docker_image, @@ -1612,11 +1633,13 @@ CDHIT2duckdb <- function(duckdb_path, #' counts per genome and annotation, and writes the result both as a Parquet file #' and as a new table in the DuckDB database. #' -#' @param annotated_parquet Path to the combined HMMER results Parquet file -#' (e.g. `"results/Ecoli/protein_COG.parquet"`). The filename stem is used as -#' the table name in DuckDB. -#' @param duckdb_path Path to the per-selection DuckDB database containing a -#' `protein_count` table (created by [CDHIT2duckdb()]). +#' @param duckdb_path Character. Path to the per-selection DuckDB database +#' containing the `protein_count` table created by [CDHIT2duckdb()]. +#' @param databases Character vector of HMMER database names to process. +#' Each database must correspond to a `protein_` annotation table +#' already present in the DuckDB. +#' @param output_path Character. Directory where the genome-by-annotation +#' Parquet files will be written. Defaults to `dirname(duckdb_path)`. #' #' @return Invisibly returns the path to the written count Parquet file. #' @@ -2056,7 +2079,7 @@ CDHIT2duckdb <- function(duckdb_path, ) # required to define the database size for hmmsearch --Z and --domZ parameters - Total_proteins <- nrow(prot_seqs) + total_proteins <- nrow(prot_seqs) readr::write_lines( paste0( @@ -2111,8 +2134,8 @@ CDHIT2duckdb <- function(duckdb_path, "--notextw", "--cpu", as.character(threads), - "-Z", Total_proteins, - "--domZ", Total_proteins, + "-Z", total_proteins, + "--domZ", total_proteins, "--domtblout", file.path( "/work", @@ -2629,6 +2652,9 @@ cleanData <- function(duckdb_path, path) { #' \item **Metadata cleaning + Parquet export** via [cleanData()] -> writes #' Parquet files to `output_path`, and builds a **Parquet-backed DuckDB** #' (`*_parquet.duckdb`) with views over those Parquets. +#' +#' \item **Dyad feature mapping** via [buildDyadFeatureMap()] -> writes the +#' protein-gene dyad feature network used for downstream graph analysis. #' } #' #' @param duckdb_path Character. Path to the **per-selection DuckDB** produced by @@ -2642,6 +2668,10 @@ cleanData <- function(duckdb_path, path) { #' @param threads Integer. Shared concurrency budget used across Panaroo, CD-HIT, #' and HMMER. Defaults to `8`. #' +#' @param export_tabular_data Logical. If TRUE, automatically call +#' [exportProcessedData()] after the processing pipeline completes to create +#' the default human-readable exports. Default: `FALSE`. +#' #' @param panaroo_split_jobs Logical. If `TRUE`, Panaroo runs in multiple batches #' that can be merged by [.mergePanaroo()]. If `FALSE`, Panaroo runs once on all #' isolates. Default: `FALSE`. @@ -2711,6 +2741,10 @@ cleanData <- function(duckdb_path, path) { #' over the generated Parquet files. #' * Records processing parameters, software versions, database selections, and #' other provenance information in the dataset manifest. +#' * Creates the protein-gene dyad feature map for downstream graph analysis. +#' * If `export_tabular_data = TRUE`, exports features in human-readable CSVs. +#' Additional export options are available by calling `exportProcessedData()` +#' after `runDataProcessing()` completes. #' #' **Threading** #' * `threads` provides the shared CPU budget for the major processing stages. @@ -2740,6 +2774,7 @@ runDataProcessing <- function( duckdb_path, output_path = NULL, threads = 8, + export_tabular_data = FALSE, # Panaroo panaroo_split_jobs = FALSE, @@ -3116,6 +3151,23 @@ runDataProcessing <- function( "_parquet.duckdb" ) + # With all features completed, create the dyad feature map + buildDyadFeatureMap(duckdb_path = duckdb_path, output_path = out_dir) + + # And if the user wants to export processed data + if (export_tabular_data == TRUE) { + if (isTRUE(verbose)) + message("\n============================================") + message("Exporting human-readable processed data tables.") + message("Additional export options are available through `exportProcessedData()`.") + message("\n============================================") + + # Export with default parameters + exportProcessedData(duckdb_path = duckdb_path, + output_path = out_dir, + verbose = verbose) + } + if (isTRUE(verbose)) { message("\n============================================") message("Completed data-processing workflow successfully.") @@ -3181,8 +3233,17 @@ runDataProcessing <- function( #' "separate" exports the AMR labels as a separate wide table. #' "append" joins those labels onto the main feature tables before export. #' @param export_formats Character vector. Any of "csv", "tsv", "parquet", "xlsx". -#' @param tables Character vector or NULL. Tables to export. If NULL, exports the -#' standard processed tables present in the database. +#' @param export_sequences Logical. If TRUE, also exports gene and protein +#' sequence tables and the genome-to-gene-to-protein mapping. Default FALSE. +#' @param export_dyads Logical. If TRUE, exports the optional dyad annotation +#' table. Each row represents a protein-gene dyad with semicolon-separated +#' mapped feature values for structural annotations and the HMMER databases +#' recorded in the dataset manifest. The export uses `export_formats`. +#' Default FALSE. +#' @param tables Character vector or NULL. Tables to export. If NULL, exports +#' the standard processed tables plus HMMER tables recorded in the manifest. +#' @param export_tables Logical. If TRUE, write the selected tables to disk. +#' Default TRUE. #' @param verbose Logical. If TRUE, prints progress messages. #' #' @return Invisibly returns a list containing the export path, table names, and mode. @@ -3534,5 +3595,3 @@ exportProcessedData <- function(duckdb_path, manifest_path = manifest_path )) } - -