Merge pull request #347 from USEPA/figupdates

Fixes figure bug

R/GenerateRefTables.R

@@ -411,7 +411,8 @@ TADA_GetActivityTypeRef <- function() {
     "Quality Control Field Replicate Msr/Obs",
     "Quality Control Field Replicate Portable Data Logger",
     "Quality Control Field Replicate Sample-Composite",
-    "Quality Control Sample-Field Replicate"
+    "Quality Control Sample-Field Replicate",
+    "Quality Control Field Replicate Sample-Field Subsample"
   )
   dup <- c(
     "Quality Control Alternative Measurement Sensitivity",
@@ -471,6 +472,24 @@ TADA_GetActivityTypeRef <- function() {
     "Sample-Positive Control"
   )
   other <- c("Quality Control Sample-Other")
+
+  nonQC <- c("Field Msr/Obs",
+             "Field Msr/Obs-Continuous Time Series",
+             "Field Msr/Obs-Habitat Assessment",
+             "Field Msr/Obs-Incidental",
+             "Field Msr/Obs-Portable Data Logger",
+             "Sample-Composite With Parents",
+             "Sample-Composite Without Parents",
+             "Sample-Field Split",
+             "Sample-Field Subsample",
+             "Sample-Integrated Cross-Sectional Profile",
+             "Sample-Integrated Flow Proportioned",
+             "Sample-Integrated Horizontal Profile",
+             "Sample-Integrated Horizontal and Vertical Composite Profile",
+             "Sample-Integrated Time Series",
+             "Sample-Integrated Vertical Profile",
+             "Sample-Other",
+             "Sample-Routine")
 
   WQXActivityTypeRef <- raw.data %>%
     dplyr::mutate(TADA.ActivityType.Flag = dplyr::case_when(
@@ -479,7 +498,10 @@ TADA_GetActivityTypeRef <- function() {
       Code %in% blank ~ "QC_blank",
       Code %in% cal ~ "QC_calibration",
       Code %in% other ~ "QC_other",
-      TRUE ~ as.character("Non_QC")
+      Code %in% nonQC ~ "Non_QC",
+      TRUE ~ as.character("Not Reviewed"),
+      Code %in% NA ~ "Not Reviewed"
+
     )) %>%
     dplyr::distinct()
 

R/ResultFlagsDependent.R

@@ -550,7 +550,7 @@ TADA_FindQCActivities <- function(.data, clean = FALSE, flaggedonly = FALSE) {
 #' prepare a dataframe for quantitative analyses. Ideally, this function should
 #' be run after other data cleaning, QA/QC, and harmonization steps are 
 #' completed using other TADA package functions, or manually. Specifically, .
-#' this function removes rows with "Text","Coerced to NA", and "ND or NA" 
+#' this function removes rows with "Text","Coerced to NA", and "Blank" 
 #' in the TADA.ResultMeasureValueDataTypes.Flag column, or NA in the 
 #' TADA.ResultMeasureValue column.
 #'
@@ -578,7 +578,7 @@ TADA_AutoFilter <- function(.data) {
     "ActivityTypeCode"
   ))
 
-  autofilter <- dplyr::filter(.data, TADA.ResultMeasureValueDataTypes.Flag != "ND or NA" &
+  autofilter <- dplyr::filter(.data, TADA.ResultMeasureValueDataTypes.Flag != "Blank" &
     TADA.ResultMeasureValueDataTypes.Flag != "Text" &
     TADA.ResultMeasureValueDataTypes.Flag != "Coerced to NA" &
     !is.na(TADA.ResultMeasureValue))# &

R/Utilities.R

@@ -150,11 +150,14 @@ TADA_AutoClean <- function(.data) {
   # Move detection limit value and unit to TADA Result Measure Value and Unit columns
   .data$TADA.ResultMeasureValue <- ifelse(is.na(.data$TADA.ResultMeasureValue) & !is.na(.data$TADA.DetectionQuantitationLimitMeasure.MeasureValue), .data$TADA.DetectionQuantitationLimitMeasure.MeasureValue, .data$TADA.ResultMeasureValue)
   .data$TADA.ResultMeasure.MeasureUnitCode <- ifelse(is.na(.data$TADA.ResultMeasure.MeasureUnitCode) & !is.na(.data$TADA.DetectionQuantitationLimitMeasure.MeasureUnitCode), .data$TADA.DetectionQuantitationLimitMeasure.MeasureUnitCode, .data$TADA.ResultMeasure.MeasureUnitCode)
-  .data$TADA.ResultMeasureValueDataTypes.Flag <- ifelse(.data$TADA.ResultMeasureValueDataTypes.Flag == "ND or NA" & !is.na(.data$TADA.DetectionQuantitationLimitMeasure.MeasureValue), "Result Value/Unit Copied from Detection Limit", .data$TADA.ResultMeasureValueDataTypes.Flag)
+  .data$TADA.ResultMeasureValueDataTypes.Flag <- ifelse(.data$TADA.ResultMeasureValueDataTypes.Flag == "Blank" & !is.na(.data$TADA.DetectionQuantitationLimitMeasure.MeasureValue), "Result Value/Unit Copied from Detection Limit", .data$TADA.ResultMeasureValueDataTypes.Flag)
 
   # Identify detection limit data
   print("TADA_Autoclean: identifying detection limit data.")
   .data <- TADA_IDCensoredData(.data)
+
+  # Identify QC data
+  .data <- TADA_FindQCActivities(.data, clean = FALSE, flaggedonly = FALSE)
 
   # change latitude and longitude measures to class numeric
   .data$TADA.LatitudeMeasure <- as.numeric(.data$LatitudeMeasure)
@@ -328,7 +331,7 @@ TADA_ConvertSpecialChars <- function(.data, col) {
   if (is.numeric(chars.data$orig)) {
     clean.data <- chars.data %>%
       dplyr::mutate(flag = dplyr::case_when(
-        is.na(masked) ~ as.character("ND or NA"),
+        is.na(masked) ~ as.character("Blank"),
         TRUE ~ as.character("Numeric")
       ))
   } else {
@@ -338,8 +341,8 @@ TADA_ConvertSpecialChars <- function(.data, col) {
     # of the specific type of character/data type
     clean.data <- chars.data %>%
       dplyr::mutate(flag = dplyr::case_when(
-        is.na(masked) ~ as.character("ND or NA"),
-        (masked == "ND") ~ as.character("ND or NA"),
+        is.na(masked) ~ as.character("Blank"),
+        (masked == "ND") ~ as.character("Blank"),
         (!is.na(suppressWarnings(as.numeric(masked)) == TRUE)) ~ as.character("Numeric"),
         (grepl("<", masked) == TRUE) ~ as.character("Less Than"),
         (grepl(">", masked) == TRUE) ~ as.character("Greater Than"),
@@ -1080,7 +1083,7 @@ TADA_UpdateExampleData <- function() {
     od_method = "as-is",
     od_multiplier = "null"
   )
-  y <- dplyr::filter(y, TADA.ResultMeasureValueDataTypes.Flag != "ND or NA" &
+  y <- dplyr::filter(y, TADA.ResultMeasureValueDataTypes.Flag != "Blank" &
     TADA.ResultMeasureValueDataTypes.Flag != "Text" &
     TADA.ResultMeasureValueDataTypes.Flag != "Coerced to NA" &
     !is.na(TADA.ResultMeasureValue))
@@ -1279,3 +1282,61 @@ TADA_CheckRequiredFields <- function(.data) {
     stop("The dataframe does not contain the required fields to use TADA Module 1.")
   }
 }
+
+
+#' AutoFilter
+#'
+#' This function removes rows where the result value is not numeric to 
+#' prepare a dataframe for quantitative analyses. Ideally, this function should
+#' be run after other data cleaning, QA/QC, and harmonization steps are 
+#' completed using other TADA package functions, or manually. Specifically, .
+#' this function removes rows with "Text","Coerced to NA", and "Blank" 
+#' in the TADA.ResultMeasureValueDataTypes.Flag column, or NA in the 
+#' TADA.ResultMeasureValue column.
+#'
+#' @param .data TADA dataframe OR TADA sites dataframe
+#'
+#' @return .data with rows removed where result values are not quantitative (NA or text),
+#' or the results have other issues that are not dealt with elsewhere.
+#'
+#' @export
+#'
+#' @examples
+#' # Load example dataset:
+#' data(Data_Nutrients_UT)
+#'
+#' # Remove all:
+#' TADA_filtered <- TADA_AutoFilter(Data_Nutrients_UT)
+#'
+TADA_AutoFilter <- function(.data) {
+
+  # check .data is data.frame
+  TADA_CheckType(.data, "data.frame", "Input object")
+
+  TADA_CheckColumns(.data, c(
+    "ActivityTypeCode", "MeasureQualifierCode",
+    "TADA.ResultMeasureValueDataTypes.Flag",
+    "TADA.ResultMeasureValue", "TADA.ActivityMediaName",
+    "ActivityTypeCode", "TADA.ActivityType.Flag"
+  ))
+
+  # keep track of starting and ending number of rows
+  start <- dim(.data)[1]
+
+  # remove text, NAs and QC results
+  .data <- dplyr::filter(.data, TADA.ResultMeasureValueDataTypes.Flag != "Blank" &
+                                TADA.ResultMeasureValueDataTypes.Flag != "Text" &
+                                TADA.ResultMeasureValueDataTypes.Flag != "Coerced to NA" &
+                                TADA.ActivityType.Flag == "Non_QC" & # filter out QA/QC ActivityTypeCode's
+                                !is.na(TADA.ResultMeasureValue))
+
+  end <- dim(.data)[1]
+
+  # print number of results removed
+  if (!start == end) {
+    net <- start - end
+    print(paste0("Function removed ", net, " results. These results are either text or NA and cannot be plotted or represent quality control activities (not routine samples or measurements)."))
+  }
+
+  return(.data)
+}

inst/extdata/WQXActivityTypeRef.csv

@@ -13,6 +13,7 @@
 "Activity Type(ActivityTypeCode)",16,"Quality Control Field Replicate Msr/Obs","A field measurement or observation taken to assess the reproducity of the sampling technique or analytical method.","N","Y","7/26/2006 10:57:46 AM","QRM","QC_replicate"
 "Activity Type(ActivityTypeCode)",17,"Quality Control Field Replicate Portable Data Logger","A measurement taken to verify the quality of data taken by a data logger.","N","Y","7/26/2006 10:57:46 AM","QRPDL","QC_replicate"
 "Activity Type(ActivityTypeCode)",53,"Quality Control Field Replicate Sample-Composite","A composite of either several discrete sampling events or a sample collected by a continuous process samples taken from various vertical and horizontal locations combined into one sample to assess the reproducity of the sampling technique or analytical method.","N","Y","8/29/2017 12:00:00 AM","QRSC","QC_replicate"
+"Activity Type(ActivityTypeCode)",61,"Quality Control Field Replicate Sample-Field Subsample","it’s a Field Replicate and a Field Subsample: A Subsample of either several discrete sampling events or a sample collected by a continuous process samples taken from various vertical and horizontal locations combined into one sample to assess the reproducity of the sampling technique or analytical method.","N","Y","10/17/2023 10:27:13 AM","QCFRFS","QC_replicate"
 "Activity Type(ActivityTypeCode)",18,"Quality Control Field Sample Equipment Rinsate Blank","A sample of analyte-free media that has been used to rinse the sampling equipment. It is collected after completion of decontamination and prior to sampling. This blank is useful in documenting adequate decontamination of sampling equipment.","N","N","5/3/2013 10:57:46 AM","QERB","QC_blank"
 "Activity Type(ActivityTypeCode)",116,"Quality Control Lab Sample Equipment Rinsate Blank","A sample of analyte-free media that has been used to rinse the sampling equipment in the laboratory. It is collected after completion of decontamination and prior to sampling. This blank is useful in documenting adequate decontamination of sampling equipment prior to the equipment being used in the field.","N","N","7/12/2017 1:58:07 PM","QLERB","QC_blank"
 "Activity Type(ActivityTypeCode)",33,"Quality Control Sample-Blind Duplicate","The duplicate samples are identical samples (collected from same source and same time) however blind duplicates are submitted to the laboratory by an identification scheme so that the laboratory does not know the samples are identical.   The purpose of blind duplicates is to test for laboratory bias.  ","Y","Y","7/26/2006 10:57:46 AM","QBD","QC_duplicate"