diff --git a/README.md b/README.md index fc9aa24..833ab73 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# 2018_smartdotsReport_template +# 2023_smartdotsReport_template This template is used to create reports from smartdots events. The settings for the report are contained in the [config.json](config.json) file. Currently, the code requires direct access to the smartdots database and so must be run by someone with the appropriate access. To run the template: diff --git a/data_checker.R b/data_checker.R index 3af8563..32b4516 100644 --- a/data_checker.R +++ b/data_checker.R @@ -5,10 +5,13 @@ library(jsonlite) library(tidyr) # # load configuration -config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +#config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +config <- read_json("bootstrap/initial/data/config.json", simplifyVector = TRUE) # get data from bootstrap folder ------------------------------- -ad <- read.taf("bootstrap/data/smartdots_db/ad.csv") +#ad <- read.taf("bootstrap/smartdots_db/ad.csv") +ad <- read.taf("bootstrap/ad.csv") + # tag some feilds as missing? diff --git a/data_processing.R b/data_processing.R index a7fed04..9ea38be 100644 --- a/data_processing.R +++ b/data_processing.R @@ -12,20 +12,25 @@ library(lubridate) library(plyr) library(dplyr) library(tidyverse) -taf.library(ragree) +#taf.library(ragree) +#devtools::install_github("raredd/ragree") +library(ragree) # create data directory -mkdir("data") +#mkdir("data") # get utility functions source("utilities.R") source("utilities_data.R") # load configuration -config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +#config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +config <- read_json("bootstrap/initial/data/config.json", simplifyVector = TRUE) + # get data from bootstrap folder ------------------------------- -ad <- read.taf("bootstrap/data/smartdots_db/ad.csv") +#ad <- read.taf("bootstrap/data/smartdots_db/ad.csv") +ad <- read.taf("bootstrap/ad.csv") # prepare data ------------------------------- @@ -42,6 +47,7 @@ ad <- month <- month(parse_date_time(catch_date, "%d/%m/%Y %H:%M:%S")) }) + # if variables are missing add "missing" ad$ices_area[is.na(ad$ices_area) | ad$ices_area == ""] <- "missing" ad$stock[is.na(ad$stock) | ad$stock == ""] <- "missing" @@ -76,9 +82,9 @@ for (i in 1:length(fishid)) yeshist <- yeshist[rep(row.names(yeshist), length(sampleid_nohist)), ] yeshist$SampleID <- sampleid_nohist } - + temp <- rbind(nohist, yeshist) - + if (i == 1) { result <- temp } else { @@ -88,13 +94,21 @@ for (i in 1:length(fishid)) ad <- result +ad$TypeAnnotation[ad$TypeAnnotation=="Delegate"]<-"eventOrganizer" +ad$TypeAnnotation[ad$TypeAnnotation=="Organizer"]<-"eventOrganizer" +ad$reader[ad$reader==""]<-"eventOrganizer" ## to add a name to the Reader column from the Event Organizer +ad$Sex[is.na(ad$Sex) | ad$Sex==""]<-"NI" +ad<-ad[ad$Sex!="NI",] +ad$Maturity[is.na(ad$Maturity) | ad$Maturity==""]<-"NI" +ad<-ad[ad$Maturity!="NI",] + # Calculate modal maturity stage and coefficient of unalikability of maturity stage ad_long <- ad %>% add_modal_trad(varmod = "Maturity", config$ma_method) %>% add_modal_linearweight(varmod = "Maturity", config$ma_method) %>% add_modal_negexpweight(varmod = "Maturity", config$ma_method) -ad_long_ex <- ad[ad$expertise == "Advanced", ] %>% +ad_long_adv <- ad[ad$expertise == "Advanced", ] %>% add_modal_trad(varmod = "Maturity", config$ma_method) %>% add_modal_linearweight(varmod = "Maturity", config$ma_method) %>% add_modal_negexpweight(varmod = "Maturity", config$ma_method) @@ -106,14 +120,14 @@ ad_long <- ad_long %>% add_modal_linearweight(varmod = "Sex", config$ma_method) %>% add_modal_negexpweight(varmod = "Sex", config$ma_method) -ad_long_ex <- ad_long_ex %>% +ad_long_adv <- ad_long_adv %>% add_modal_trad(varmod = "Sex", config$ma_method) %>% add_modal_linearweight(varmod = "Sex", config$ma_method) %>% add_modal_negexpweight(varmod = "Sex", config$ma_method) # Choose the final mode (traditional, readers linear weight or negative exponential linear weight) based in the existence of histological samples or not, and, in case there are no histological samples, depending if there is multimodality or not. ad_long <- select_mode(ad_long, config$ma_method, config$mode_definition) -ad_long_ex <- select_mode(ad_long_ex, config$ma_method, config$mode_definition) +ad_long_adv <- select_mode(ad_long_adv, config$ma_method, config$mode_definition) # prepare data in wbgr output format # IMAGE,1,2,3,4,5,6,7,8,9,10,11,12,13 @@ -150,6 +164,6 @@ head(webgr_sex) # write out input data tables for use later write.taf(ad4webgr, "data/data.csv", quote = TRUE) write.taf(ad_long, "data/ad_long.csv", quote = TRUE) -write.taf(ad_long_ex, "data/ad_long_ex.csv", quote = TRUE) +write.taf(ad_long_adv, "data/ad_long_adv.csv", quote = TRUE) write.taf(webgr_maturity, "data/WebGR_maturity_ages_all.csv", quote = TRUE) write.taf(webgr_sex, "data/WebGR_sex_ages_all.csv", quote = TRUE) diff --git a/model.R b/model.R index 9ee6de8..b7d9587 100644 --- a/model.R +++ b/model.R @@ -1,7 +1,7 @@ ## Run analysis, write model results -## Before: data/ad_long.csv, data/ad_long_ex.csv, -## data/ad_wide.csv, data/ad_wide_ex.csv +## Before: data/ad_long.csv, data/ad_long_adv.csv, +## data/ad_wide.csv, data/ad_wide_adv.csv ## After: library(icesTAF) @@ -16,14 +16,15 @@ library(ggplot2) library(scales) # rescale_none # library ragree contains the function unalike, that is used to estimate the coefficient of unalikeability -taf.library(ragree) +library(ragree) # # make model directory mkdir("model") # # load configuration -config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +#config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +config <- read_json("bootstrap/initial/data/config.json", simplifyVector = TRUE) # load utilities source("utilities.R") @@ -31,16 +32,16 @@ source("utilities_model.R") # read input data ad_long_all <- read.taf("data/ad_long.csv") -ad_long_ex <- read.taf("data/ad_long_ex.csv") +ad_long_adv <- read.taf("data/ad_long_adv.csv") # model maturity range modal_matur_unique_all <- sort(unique(ad_long_all$modal_maturity)) -modal_matur_unique_ex <- sort(unique(ad_long_ex$modal_maturity)) +modal_matur_unique_adv <- sort(unique(ad_long_adv$modal_maturity)) # model sex range modal_sex_unique_all <- sort(unique(ad_long_all$modal_sex)) -modal_sex_unique_ex <- sort(unique(ad_long_ex$modal_sex)) +modal_sex_unique_adv <- sort(unique(ad_long_adv$modal_sex)) # set strata to NULL if all are NA # if(length(setdiff("strata", names(config)))==0) {if(all(is.na(ad_long_all[["strata"]]))) config$strata <- NULL} @@ -53,15 +54,16 @@ write.taf(sample_data_overview, dir = "model") # Participants table stager_data <- reader_data_table(ad_long_all, strata=config$strata) +stager_data <- slice(stager_data, 1:(n() - 1)) write.taf(stager_data, dir = "model") # Results ############################################## # repeat for all and for experts only -for (group in c("all", "ex")) { +for (group in c("all", "adv")) { #group <- "all" - #group <- "ex" + #group <- "adv" # get the appropriate dataset ad_long <- get(vname("ad_long")) @@ -148,6 +150,23 @@ for (group in c("all", "ex")) { ) write.taf(vname("cu_tab_maturity"), dir = "model") + # CU table (coefficient of unalikeability) - Females + assign( + vname("cu_tab_maturity_females"), + cu_table(ad_long[ad_long$Sex=="F",], "Maturity", by = "reader") + ) + write.taf(vname("cu_tab_maturity_females"), dir = "model") + + + # CU table (coefficient of unalikeability) - Males + assign( + vname("cu_tab_maturity_males"), + cu_table(ad_long[ad_long$Sex=="M",], "Maturity", by = "reader") + ) + write.taf(vname("cu_tab_maturity_males"), dir = "model") + + + # CU table (coefficient of unalikeability) - by Sex assign( vname("cu_tab_sex"), cu_table(ad_long, "Sex", by = "reader") @@ -168,6 +187,22 @@ for (group in c("all", "ex")) { pa_table(ad_long, "Sex", by = "reader") ) write.taf(vname("pa_tab_sex"), dir = "model") + + ##females + assign( + vname("pa_tab_maturity_females"), + pa_table(ad_long[ad_long$Sex=="F",], "Maturity", by = "reader") + ) + write.taf(vname("pa_tab_maturity"), dir = "model") + + + ##males + assign( + vname("pa_tab_maturity_males"), + pa_table(ad_long[ad_long$Sex=="M",], "Maturity", by = "reader") + ) + write.taf(vname("pa_tab_maturity_males"), dir = "model") + ################################################################################################################################################## # Frequency table (Number for each maturity stage per modal_maturity for each ). This is the equivalent to the relative bias table in the ageing. @@ -225,6 +260,9 @@ for (group in c("all", "ex")) { maturity_composition_table(ad_long, by = "reader") ) write.taf(vname("maturity_composition_tab"), dir = "model") + + + #################### # sex composition @@ -243,6 +281,7 @@ for (group in c("all", "ex")) { saveRDS(get(vname("msem")), file = file.path("model", paste0(vname("msem"), ".rds"))) + ######################################################### # sex category error matrix (SSEM) only for advanced s assign( @@ -298,6 +337,21 @@ for (group in c("all", "ex")) { ) write.taf(vsname("cu_tab_sex_by"), dir = "model") + + # CU table (coefficient of unalikeability) - Males + assign( + vname("cu_tab_maturity_males"), + cu_table(ad_long[ad_long$Sex=="M",], "Maturity", by = "reader") + ) + write.taf(vname("cu_tab_maturity_males"), dir = "model") + + + # CU table (coefficient of unalikeability) - by Sex + assign( + vname("cu_tab_sex"), + cu_table(ad_long, "Sex", by = "reader") + ) + write.taf(vname("cu_tab_sex"), dir = "model") ############################################################################################################################## # Percent agreement between maturity stagings and modal maturity stage. @@ -312,6 +366,21 @@ for (group in c("all", "ex")) { pa_table(ad_long, "Sex", by = stratum) ) write.taf(vsname("pa_tab_sex_by"), dir = "model") + + ##females + assign( + vname("pa_tab_maturity_females"), + pa_table(ad_long[ad_long$Sex=="F",], "Maturity", by = "reader") + ) + write.taf(vname("pa_tab_maturity"), dir = "model") + + + ##males + assign( + vname("pa_tab_maturity_males"), + pa_table(ad_long[ad_long$Sex=="M",], "Maturity", by = "reader") + ) + write.taf(vname("pa_tab_maturity_males"), dir = "model") ################################## ## Annex tables ################# @@ -333,7 +402,8 @@ for (group in c("all", "ex")) { mat_stage_error_matrix(ad_long, by = stratum) ) saveRDS(get(vname("msem")), file = file.path("model", paste0(vname("msem"), ".rds"))) - + + ######################################################### # sex category error matrix (MSEM) only for advanced s assign( @@ -347,5 +417,4 @@ for (group in c("all", "ex")) { } - #config$strata=select_strata diff --git a/report.R b/report.R index dddbd4e..d147cbb 100644 --- a/report.R +++ b/report.R @@ -8,7 +8,7 @@ library(icesTAF) library(rmarkdown) library(jsonlite) library(knitr) -# +# library(pander) library(ggplot2) library(scales) @@ -24,11 +24,12 @@ source("utilities.R") source("utilities_report.R") # load configuration data -config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +#config <- read_json("bootstrap/data/config.json", simplifyVector = TRUE) +config <- read_json("bootstrap/initial/data/config.json", simplifyVector = TRUE) # load data for report ad_long_all <- read.taf("data/ad_long.csv") -ad_long_ex <- read.taf("data/ad_long_ex.csv") +ad_long_adv <- read.taf("data/ad_long_adv.csv") # # set strata to NULL is all are NA # if (all(is.na(ad_long_all[[config$strata]]))) config$strata <- NULL diff --git a/report_full.Rmd b/report_full.Rmd index a8e07f1..9ffacf6 100644 --- a/report_full.Rmd +++ b/report_full.Rmd @@ -4,7 +4,7 @@ output: fig_caption: true fig_height: 10 fig_width: 10 - reference_docx: bootstrap/data/reportTemplate.docx + reference_docx: bootstrap/initial/data/reportTemplate.docx toc: true keep_md: false params: @@ -62,7 +62,7 @@ and what to expect to read about in the report. # Methods -This report contains statistical analyses and comparisons of sex categorization and maturity stagings +This report contains statistical analyses and comparisons of sex categorization and maturity staging’s in the form of tables and graphical plots. First, an overview of participating maturity stagers and the samples are presented. @@ -76,12 +76,12 @@ be moved to the body of the report this is also possible. Only the plots cannot be changed. In the first part of analysis some of the tables and plots from the -Guus Eltink Excel sheet 'Age Reading Comparisons' **(Eltink, A.T.G.W. 2000)** are presented, that can be used for "maturity staging comparisons". Since the sex categories and maturity stages are categorical in comparison to ages that are quantitative, some of the statistics presented in the report of the age exchange events cannot be calculated here, like the average percentage error or the coefficient of variation. The later is being replaced by the coefficient of unlikeability (see bellow). +Guus Eltink Excel sheet 'Age Reading Comparisons' **(Eltink, A.T.G.W. 2000)** are presented, that can be used for "maturity staging comparisons". Since the sex categories and maturity stages are categorical in comparison to ages that are quantitative, some of the statistics presented in the report of the age exchange events cannot be calculated here, like the average percentage error or the coefficient of variation. The latter is being replaced by the coefficient of unlikeability (see below). -**Pecentage Agreement** +**Percentage Agreement** -The percentage agreement tells how large is the part of sex categorizations/maturity stagings that are equal to the modal sex/maturity. The percentage agreement is estimated by modal sex or maturity and stager as the proportion (as percentage) of times that the lectures of that stager agreed with the resulting modal sex or maturity.This percentage is estimated as the number of times that a stager agreed with the modal sex or maturity divided by the total number of gonads analyzed by a stager for that modal maturity stage. +The percentage agreement tells how large is the part of sex categorizations/maturity staging’s that are equal to the modal sex/maturity. The percentage agreement is estimated by modal sex or maturity and stager as the proportion (as percentage) of times that the lectures of that stager agreed with the resulting modal sex or maturity. This percentage is estimated as the number of times that a stager agreed with the modal sex or maturity divided by the total number of gonads analyzed by a stager for that modal maturity stage. $$PA = { \frac{ {number \,of \,readings \,that \,agree \,with \,modal \,maturity}} {total \,number \,of \,readings \,by \,modal \,maturity} } \cdot {100 \%} $$ @@ -113,6 +113,8 @@ is “advanced” then they are considered well trained and they provide maturit for stock assessment or similar purposes. When the MSEM is compiled for assessment purposes it uses only those stagers who provide maturity data for the stock assessment in that specific area. +In this report the MSEM presented represents is in the transpose format. +The frequency bias table represents the MSEM in the format proposed by WKSABCAL (2014). @@ -145,10 +147,10 @@ pander(style_table0(sample_data_overview), style = "simple") ```{r participants_overview} -# PARTICIPATANTS OVERVIEW ##################################################### +# PARTICIPANTS OVERVIEW ##################################################### # Table caption -set.caption('**Table X:** Overview of stagers participating in the event, with their overal expertise (Expertise), and their ranking position based in their experience (only valid if the experience weighting protocol has been applied).') +set.caption('**Table X:** Overview of stagers participating in the event, with their overall expertise (Expertise), and their ranking position based in their experience (only valid if the experience weighting protocol has been applied).') # Output table pander(style_table0(filter(stager_data))) @@ -178,7 +180,11 @@ If there were available histological samples for all the fish individuals in the ```{r summary_multiple_modes_sex_all} -histN=ad_long_all %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() +#histN=ad_long_all %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() +#allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() +#perc_not_Hist=100-round(100*histN[1]/allN[1], 0) + +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) @@ -206,7 +212,7 @@ If there were available histological samples for all the fish individuals in the ```{r summary_multiple_modes_maturity_all} -histN=ad_long_all %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) @@ -250,11 +256,10 @@ The average percentage agreement by modal sex category for all stagers was `r su Text? -```{r num_read_sex_all} +```{r sex_composition_tab_all} # Sex categorization TABLE - All stagers ################################################# -data=num_read_tab_modal_sex_by_reader_all -#data=sex_composition_tab_all +data=sex_composition_tab_all nstagers=length(unique(stager_data$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers @@ -272,7 +277,7 @@ for(i in 1:nfig) pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt } else { if(i -The percentage agreement per stager and modal maturity stage represent the proportion of the total number of stagings that are equal to the modal maturity stage. The weighted mean including at the bottom of the table is weighted according to number of maturity stagings. +The percentage agreement per stager and modal maturity stage represent the proportion of the total number of staging’s that are equal to the modal maturity stage. The weighted mean including at the bottom of the table is weighted according to number of maturity staging’s. ```{r percentage_agreement_sex_all} @@ -519,28 +503,11 @@ for(i in 1:nfig) pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt } else { selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):dim(data)[2])] - pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt + pander(style_table1(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt } } } -# nfig=round((dim(data)[2]-basicols)/10, digits=0) -# for(i in 1:nfig) -# { -# if(i==1){ -# # Table caption -# set.caption('**Table X:** Percentage agreement (PA) table: shows the PA per modal sex category and stager, the PA for all stagers combined and the weighted mean of the PA per stager.') -# selec=data[,c(1:basicols, (i+basicols+(9*(i-1))):((i+basicols+(9*(i-1)))+9))] -# pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt -# } else { if(i -The frequency bias is shown by sex category for each modal sex category per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a buble plot may be used to explore the bias in the sex determination by the stagers participating in this event. +The frequency bias is shown by sex category for each modal sex category per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a bubble plot may be used to explore the bias in the sex determination by the stagers participating in this event. ```{r freq_sex_all} @@ -707,25 +724,7 @@ for(i in 1:nfig) } } -# nfig=round((dim(data)[2]-basicols)/10, digits=0) -# for(i in 1:nfig) -# { -# if(i==1){ -# # Table caption -# set.caption('**Table X:** Frequency bias table: represents the frequency per -# modal sex category per stager and the frequency distribution of all stagers combined -# per modal sex category.') -# selec=data[,c(1:basicols, (i+basicols+(9*(i-1))):((i+basicols+(9*(i-1)))+9))] -# pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt -# } else { if(i -The frequency bias is shown by maturity stages for each modal maturity stage per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a buble plot may be used to explore the bias in the maturity determination by the stagers participating in this event. +The frequency bias is shown by maturity stages for each modal maturity stage per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a bubble plot may be used to explore the bias in the maturity determination by the stagers participating in this event. ```{r freq_maturity_all} @@ -766,25 +765,6 @@ for(i in 1:nfig) } } -# nfig=round((dim(data)[2]-basicols)/10, digits=0) -# for(i in 1:nfig) -# { -# if(i==1){ -# # Table caption -# set.caption('**Table X:** Frequency bias table: represents the frequency per -# modal maturity stage per stager and the frequency distribution of all stagers combined -# per modal maturity stage.') -# selec=data[,c(1:basicols, (i+basicols+(9*(i-1))):((i+basicols+(9*(i-1)))+9))] -# pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt -# } else { if(i -```{r summary_multiple_modes_sex_ex} - -histN=ad_long_ex %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() -allN=ad_long_ex %>% select(FishID) %>% unique() %>% dim() +```{r summary_multiple_modes_sex_adv} +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() +allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) -summary_multiple_modes_sex_ex <- - c(NSample = length(unique(ad_long_ex$FishID)), +summary_multiple_modes_sex_adv <- + c(NSample = length(unique(ad_long_adv$FishID)), Nhist=histN[1], Perc_not_Hist=perc_not_Hist, - PercMM_traditional=unique(ifelse(multimode_cases_tab_traditional_Sex_ex$NModes_trad=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_traditional_Sex_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_linear_weight=unique(ifelse(multimode_cases_tab_linear_Sex_ex$NModes_linear=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_linear_Sex_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_negexp_weight=unique(ifelse(multimode_cases_tab_negexp_Sex_ex$NModes_negexp=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_negexp_Sex_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_multistage=unique(ifelse(multimode_cases_tab_multistage_Sex_ex$NModes_multistage=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_multistage_Sex_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%")))) + PercMM_traditional=unique(ifelse(multimode_cases_tab_traditional_Sex_adv$NModes_trad=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_traditional_Sex_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_linear_weight=unique(ifelse(multimode_cases_tab_linear_Sex_adv$NModes_linear=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_linear_Sex_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_negexp_weight=unique(ifelse(multimode_cases_tab_negexp_Sex_adv$NModes_negexp=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_negexp_Sex_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_multistage=unique(ifelse(multimode_cases_tab_multistage_Sex_adv$NModes_multistage=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_multistage_Sex_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%")))) # Table caption set.caption('**Table X:** Summary of statistics for sex staging; Total number of fish individuals studied (NSample), number of fish individuals with histological samples (Nhist), percentage of fish individuals without histology (Perc_not_Hist). The percentage of cases (fish samples) with multiple modes depending on the approach to weight the experience of the stager which will be considered when defining the fish sex stage mode. PercMM_traditional shows the percentage of the total samples for which multiple modes are obtained when all the stagers are equally weighted. PercMM_linear_weight shows the percentage of the total samples for which multiple modes are obtained when the weight assigned to the different stagers decreases linearly with the experience, while in the PercMM_negexp the weight applied decreases with a negative exponential shape with the experience. The PercMM_multistage shows the percentage of multiple mode cases when a combination of the different methodologies is used, as explained in the material and methods section') # Output table -pander(style_table0(summary_multiple_modes_sex_ex), missing="") +pander(style_table0(summary_multiple_modes_sex_adv), missing="") ``` -If there were available histological samples for all the fish individuals in the exchange event, no multiple modes are expected, however, if there were no hitology samples for some of them, multiple maturity stage modes could be found. In the next table this information is presented for the maturity staging. +If there were available histological samples for all the fish individuals in the exchange event, no multiple modes are expected, however, if there were no histology samples for some of them, multiple maturity stage modes could be found. In the next table this information is presented for the maturity staging. -```{r summary_multiple_modes_maturity_ex} +```{r summary_multiple_modes_maturity_adv} -histN=ad_long_ex %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() -allN=ad_long_ex %>% select(FishID) %>% unique() %>% dim() +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() +allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) -summary_multiple_modes_maturity_ex <- - c(NSample = length(unique(ad_long_ex$FishID)), +summary_multiple_modes_maturity_adv <- + c(NSample = length(unique(ad_long_adv$FishID)), Nhist=histN[1], Perc_not_Hist=perc_not_Hist, - PercMM_traditional=unique(ifelse(multimode_cases_tab_traditional_Maturity_ex$NModes_trad=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_traditional_Maturity_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_linear_weight=unique(ifelse(multimode_cases_tab_linear_Maturity_ex$NModes_linear=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_linear_Maturity_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_negexp_weight=unique(ifelse(multimode_cases_tab_negexp_Maturity_ex$NModes_negexp=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_negexp_Maturity_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%"))), - PercMM_multistage=unique(ifelse(multimode_cases_tab_multistage_Maturity_ex$NModes_multistage=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_multistage_Maturity_ex)/length(unique(ad_long_ex$SampleID)))*100, digits=0),"%")))) + PercMM_traditional=unique(ifelse(multimode_cases_tab_traditional_Maturity_adv$NModes_trad=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_traditional_Maturity_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_linear_weight=unique(ifelse(multimode_cases_tab_linear_Maturity_adv$NModes_linear=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_linear_Maturity_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_negexp_weight=unique(ifelse(multimode_cases_tab_negexp_Maturity_adv$NModes_negexp=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_negexp_Maturity_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%"))), + PercMM_multistage=unique(ifelse(multimode_cases_tab_multistage_Maturity_adv$NModes_multistage=="zero", paste(0,"%"), paste(round((nrow(multimode_cases_tab_multistage_Maturity_adv)/length(unique(ad_long_adv$SampleID)))*100, digits=0),"%")))) # Table caption set.caption('**Table X:** Summary of statistics for maturity staging; Total number of fish individuals studied (NSample), number of fish individuals with histological samples (Nhist), percentage of fish individuals without histology (Perc_not_Hist). The percentage of cases (fish samples) with multiple modes depending on the approach to weight the experience of the stager which will be considered when defining the fish maturity stage mode. PercMM_traditional shows the percentage of the total samples for which multiple modes are obtained when all the stagers are equally weighted. PercMM_linear_weight shows the percentage of the total samples for which multiple modes are obtained when the weight assigned to the different stagers decreases linearly with the experience, while in the PercMM_negexp the weight applied decreases with a negative exponential shape with the experience. The PercMM_multistage shows the percentage of multiple mode cases when a combination of the different methodologies is used, as explained in the material and methods section') # Output table -pander(style_table0(summary_multiple_modes_maturity_ex), missing="") +pander(style_table0(summary_multiple_modes_maturity_adv), missing="") @@ -1122,35 +1101,35 @@ pander(style_table0(summary_multiple_modes_maturity_ex), missing="") -```{r summary_statistics_sex_ex} -summary_statistics_sex_ex <- - c(NSample = length(unique(ad_long_ex$SampleID)), - CU = cu_tab_sex_ex[nrow(cu_tab_sex_ex), "Total"], - PA = pa_tab_sex_ex[nrow(pa_tab_sex_ex), "Total"]) +```{r summary_statistics_sex_adv} +summary_statistics_sex_adv <- + c(NSample = length(unique(ad_long_adv$SampleID)), + CU = cu_tab_sex_adv[nrow(cu_tab_sex_adv), "Total"], + PA = pa_tab_sex_adv[nrow(pa_tab_sex_adv), "Total"]) -summary_statistics_maturity_ex <- - c(NSample = length(unique(ad_long_ex$SampleID)), - CU = cu_tab_maturity_ex[nrow(cu_tab_maturity_ex), "Total"], - PA = pa_tab_maturity_ex[nrow(pa_tab_maturity_ex), "Total"]) +summary_statistics_maturity_adv <- + c(NSample = length(unique(ad_long_adv$SampleID)), + CU = cu_tab_maturity_adv[nrow(cu_tab_maturity_adv), "Total"], + PA = pa_tab_maturity_adv[nrow(pa_tab_maturity_adv), "Total"]) ``` -The average percentage agreement by modal sex category for all stagers was `r summary_statistics_sex_ex[3]`, with a weighted average CU of `r summary_statistics_sex_ex[2]`. Regarding the maturation staging, the percentage agreement by modal maturation stage was `r summary_statistics_maturity_ex[3]`, and the weight average CU was `r summary_statistics_maturity_ex[2]` +The average percentage agreement by modal sex category for all stagers was `r summary_statistics_sex_adv[3]`, with a weighted average CU of `r summary_statistics_sex_adv[2]`. Regarding the maturation staging, the percentage agreement by modal maturation stage was `r summary_statistics_maturity_adv[3]`, and the weight average CU was `r summary_statistics_maturity_adv[2]` ***Sex categorization table*** -```{r num_read_sex_ex} +```{r sex_composition_tab_adv} -# Sex categorization TABLE - Expert stagers ################################################# +# Sex categorization TABLE - Advanced stagers ################################################# -data=num_read_tab_modal_sex_by_reader_ex -#data=sex_composition_tab_ex +data=sex_composition_tab_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers -z=8 +#z=8 +z=ncol(data)-2 nfig=ceiling((dim(data)[2]-basicols)/z) N=min(z,ncols) if(nfig==0){ @@ -1174,11 +1153,10 @@ if(nfig==0){ pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt } else { selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):dim(data)[2])] - pander(style_table0(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt + pander(style_table1(selec), missing = "-", style = "simple") #changed here to not appear the last column in bolt } } -} - + } } ``` @@ -1186,25 +1164,25 @@ if(nfig==0){ ***Maturity staging table*** -```{r num_read_maturity_ex} +```{r num_read_maturity_adv} -# maturity stagings TABLE - Expert stagers ################################################# +# maturity staging’s TABLE - Advanced stagers ################################################# #data=num_read_tab_modal_matur_by_reader_ex -data=maturity_composition_tab_ex +data=maturity_composition_tab_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers -z=8 +z=ncol(data)-1 nfig=round((dim(data)[2]-basicols)/z, digits=0) N=min(z,ncols) if(nfig==0){ # Table caption set.caption('**Table X:** Maturity staging table: presents the - number of stagings made per expert stager for each modal maturity stage.') + number of staging’s made per expert stager for each modal maturity stage.') selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):dim(data)[2])] pander(style_table0(selec), missing = "-") } else { @@ -1214,7 +1192,7 @@ for(i in 1:nfig) if(i==1){ # Table caption set.caption('**Table X:** Maturity staging table: presents the - number of stagings made per expert stager for each modal maturity stage.') + number of staging’s made per expert stager for each modal maturity stage.') selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):((i+basicols+((N-1)*(i-1)))+(N-1)))] pander(style_table0(selec), missing = "-") } else { if(i -The percentage agreement per stager and modal maturity stage represent the proportion of the total number of stagings that are equal to the modal maturity stage. The weighted mean including at the bottom of the table is weighted according to number of maturity stagings. +The percentage agreement per stager and modal maturity stage represent the proportion of the total number of staging’s that are equal to the modal maturity stage. The weighted mean including at the bottom of the table is weighted according to number of maturity staging’s. -```{r percentage_agreement_sex_ex} +```{r percentage_agreement_sex_adv} -# PERCENTAGE AGREEMENT TABLE by modal sex category - Expert stagers #################################### +# PERCENTAGE AGREEMENT TABLE by modal sex category - Advanced stagers #################################### -data=pa_tab_sex_ex +data=pa_tab_sex_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers -z=8 +z=ncol(data)-1 nfig=round((dim(data)[2]-basicols)/z, digits=0) N=min(z,ncols) @@ -1383,11 +1456,11 @@ for(i in 1:nfig) ***PA table by modal maturity stage*** -```{r percentage_agreement_maturity_ex} +```{r percentage_agreement_maturity_adv} # PERCENTAGE AGREEMENT TABLE by modal maturity stage - Expert stagers #################################### -data=pa_tab_maturity_ex +data=pa_tab_maturity_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] @@ -1424,23 +1497,104 @@ for(i in 1:nfig) ``` +```{r percentage_agreement_maturity_females_adv} + +# PERCENTAGE AGREEMENT TABLE by modal maturity stage (females) - Advanced stagers #################################### + +data=pa_tab_maturity_females_adv +stagers=stager_data[stager_data$Expertise=="Advanced",] +nstagers=length(unique(stagers$'Reader code')) +ncols=dim(data[,!colnames(data) %in% c("Total")])[2] +basicols=ncols-nstagers + +z=8 +nfig=round((dim(data)[2]-basicols)/z, digits=0) +N=min(z,ncols) + +if(nfig==0){ + # Table caption + set.caption('**Table X:** Percentage agreement (PA) table (only females): shows the PA per modal maturity and stager, the PA for all stagers combined and the weighted mean of the PA per stager.') + selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):dim(data)[2])] + pander(style_table1(selec), missing = "-") +} else { + +for(i in 1:nfig) +{ + if(i==1){ + # Table caption + set.caption('**Table X:** Percentage agreement (PA) table (only females): shows the PA per modal maturity and stager, the PA for all stagers combined and the weighted mean of the PA per stager.') + selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):((i+basicols+((N-1)*(i-1)))+(N-1)))] + pander(style_table1(selec), missing = "-") + } else { if(i -The frequency bias is shown by sex category for each modal sex category per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a buble plot may be used to explore the bias in the sex determination by the stagers participating in this event. +The frequency bias is shown by sex category for each modal sex category per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a bubble plot may be used to explore the bias in the sex determination by the stagers participating in this event. -```{r freq_sex_ex} +```{r freq_sex_adv} # frequency bias TABLE by sex category- Expert stagers ########################################### -data=stager_bias_freq_tab_sex_ex +data=stager_bias_freq_tab_sex_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers -z=ncol(data)-2 +#z=ncol(data)-2 +z=8 nfig=round((dim(data)[2]-basicols)/z, digits=0) N=min(z,ncols) @@ -1461,13 +1615,13 @@ for(i in 1:nfig) modal sex category per stager and the frequency distribution of all stagers combined per modal sex category.') selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):((i+basicols+((N-1)*(i-1)))+(N-1)))] - pander(style_table1(selec), missing = "-") + pander(style_table0(selec), missing = "-") } else { if(i -The frequency bias is shown by maturity stages for each modal maturity stage per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a buble plot may be used to explore the bias in the maturity determination by the stagers participating in this event. +The frequency bias is shown by maturity stages for each modal maturity stage per stager. As for the previous tables, a combined frequency distribution for all stagers are calculated. This frequency distribution presented in the form of a table or a bubble plot may be used to explore the bias in the maturity determination by the stagers participating in this event. -```{r freq_maturity_ex} +```{r freq_maturity_adv} # frequency bias TABLE by maturity stage - Expert stagers ########################################### -data=stager_bias_freq_tab_maturity_ex +data=stager_bias_freq_tab_maturity_adv stagers=stager_data[stager_data$Expertise=="Advanced",] nstagers=length(unique(stagers$'Reader code')) ncols=dim(data[,!colnames(data) %in% c("Total")])[2] basicols=ncols-nstagers -z=ncol(data)-2 +#z=ncol(data)-2 +z=8 nfig=round((dim(data)[2]-basicols)/z, digits=0) N=min(z,ncols) @@ -1501,7 +1656,7 @@ if(nfig==0){ modal maturity stage per stager and the frequency distribution of all stagers combined per modal maturity stage.') selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):dim(data)[2])] - pander(style_table1(selec), missing = "-") + pander(style_table0(selec), missing = "-") } else { for(i in 1:nfig) @@ -1512,13 +1667,13 @@ for(i in 1:nfig) modal maturity stage per stager and the frequency distribution of all stagers combined per modal maturity stage.') selec=data[,c(1:basicols, (i+basicols+((N-1)*(i-1))):((i+basicols+((N-1)*(i-1)))+(N-1)))] - pander(style_table1(selec), missing = "-") + pander(style_table0(selec), missing = "-") } else { if(i -The Sex Category Error Matrix is calculated per area, and only based on the maturity stagings of the advanced stagers. +The Sex Category Error Matrix is calculated per area, and only based on the maturity staging’s of the advanced stagers. ```{r sex_category_error_matrix_ex, results='asis'} # SSEM - Advanced stagers ################################################# # Loop through each area and output MSEM for that area -for (i in seq_along(ssem_ex)) { +for (i in seq_along(ssem_adv)) { # name - strata_name <- names(ssem_ex)[i] + strata_name <- names(ssem_adv)[i] # Table caption cap_i <- paste0("**Table X:** Sex Category Error Matrix (SSEM) for ", strata_name ,". @@ -1618,7 +1773,7 @@ for (i in seq_along(ssem_ex)) { some cases. Only advanced stagers are used for calculating the SSEM.") set.caption(cap_i) - pander(style_table3(ssem_ex[[i]]), missing = "-") + pander(style_table3(ssem_adv[[i]]), missing = "-") } ``` @@ -1627,26 +1782,26 @@ for (i in seq_along(ssem_ex)) { ***Maturity Stage Error Matrix (MSME)*** -The Maturity Stage Error Matrix is calculated per area, and only based on the maturity stagings of the advanced stagers. +The Maturity Stage Error Matrix is calculated per area, and only based on the maturity staging’s of the advanced stagers. ```{r maturity_stage_error_matrix_ex, results='asis'} # MSEM - Advanced stagers ################################################# # Loop through each area and output MSEM for that area -for (i in seq_along(msem_ex)) { +for (i in seq_along(msem_adv)) { # name - strata_name <- names(msem_ex)[i] + strata_name <- names(msem_adv)[i] # Table caption cap_i <- paste0("**Table X:** Maturity stage error matrix (MSEM) for ", strata_name ,". The MSEM shows the proportional distribution - of maturity stagings for each modal maturity stage. Stage column should sum to + of maturity staging’s for each modal maturity stage. Stage column should sum to one but due to rounding there might be small deviations in some cases. Only advanced stagers are used for calculating the MSEM.") set.caption(cap_i) - pander(style_table3(msem_ex[[i]]), missing = "-") + pander(style_table3(msem_adv[[i]]), missing = "-") } ``` @@ -1657,10 +1812,10 @@ for (i in seq_along(msem_ex)) { #### With samples split by strata -```{r set_strata_ex} +```{r set_strata_adv} # initialise strata loop istrata <- 0 -group <- "ex" +group <- "adv" print_strata <- function() length(params$strata) >= istrata ``` @@ -1984,14 +2139,14 @@ plot_bias_sex(get(vname("ad_long"))) # INDIVIDUAL Frequency PLOTS - All stagers ######################################### # Figure caption -cap_in <- '**Figure X:** Frequency plots per stager. The modal maturity stage is plotted against the different maturity stages. The frequency of stagings corresponging to each modal maturity/maturity combinations is represented together with the 1:1 equilibrium line (solid line).' +cap_in <- '**Figure X:** Frequency plots per stager. The modal maturity stage is plotted against the different maturity stages. The frequency of staging’s corresponging to each modal maturity/maturity combinations is represented together with the 1:1 equilibrium line (solid line).' # Output figures plot_bias_matur(get(vname("ad_long"))) ``` -***Plot with the CU AND PA statistichs for maturity stagings*** +***Plot with the CU AND PA statistichs for maturity staging’s*** ```{r stat_plot_maturity, fig.width = 7, fig.height = 4, fig.cap = cap_in} # PLOT STATISTICS, CU AND PA - All stagers ############################# @@ -2026,8 +2181,8 @@ plot_stat_sex(get(vname("ad_long"))) ## Results Advanced stagers -```{r set_annex_ex} -group <- "ex" +```{r set_annex_adv} +group <- "adv" ``` **All samples included** @@ -2050,14 +2205,14 @@ group <- "ex" ```{r ref.label='freq_bias_plots_maturity', fig.width = 5, fig.height = 4, fig.cap = cap_in} ``` -***Plot with the CU AND PA statistichs for maturity stagings*** +***Plot with the CU AND PA statistichs for maturity staging’s*** ```{r ref.label='stat_plot_maturity'} ``` ***Plot with the CU AND PA statistichs for sex categorizations*** -```{r ref.label='stat_plot_sex_ex'} +```{r ref.label='stat_plot_sex_adv'} ``` @@ -2092,7 +2247,7 @@ pander(style_table0(datos), missing="-", style = "simple") ***Multiple mode in the maturity stage*** ```{r multiple mode cases maturity advanced stagers} -datos=ad_long_ex[ad_long_ex$NModes_trad_Maturity>1, c(1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 16:26, 35, 36)] +datos=ad_long_adv[ad_long_adv$NModes_trad_Maturity>1, c(1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 16:26, 35, 36)] # Output table pander(style_table0(datos), missing="-", style = "simple") @@ -2103,7 +2258,7 @@ pander(style_table0(datos), missing="-", style = "simple") ***Multiple mode in the sex category*** ```{r multiple mode cases sex advanced stagers} -datos=ad_long_ex[ad_long_ex$NModes_trad_Sex>1, c(1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 16:26, 56, 57)] +datos=ad_long_adv[ad_long_adv$NModes_trad_Sex>1, c(1, 2, 4, 5, 8, 9, 10, 11, 12, 13, 16:26, 56, 57)] # Output table pander(style_table0(datos), missing="-", style = "simple") diff --git a/report_summary.Rmd b/report_summary.Rmd index 6f3746e..8451678 100644 --- a/report_summary.Rmd +++ b/report_summary.Rmd @@ -4,7 +4,7 @@ output: fig_caption: true fig_height: 10 fig_width: 10 - reference_docx: bootstrap/data/summaryTemplate.docx + reference_docx: bootstrap/initial/data/summaryTemplate.docx toc: false keep_md: false params: @@ -42,7 +42,7 @@ panderOptions('keep.trailing.zeros', TRUE) panderOptions('table.alignment.default', "center") panderOptions('knitr.auto.asis', FALSE) -group <- "ex" +group <- "adv" # from extrafont # font_import(pattern=c("Cali")) @@ -110,7 +110,7 @@ If there were available histological samples for all the fish individuals in the ```{r summary_multiple_modes_sex} -histN=ad_long_all %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) @@ -140,7 +140,7 @@ If there were available histological samples for all the fish individuals in the if(!isTRUE(config$maturity_hist)){ -histN=ad_long_all %>% subset(TypeAnnotation=="eventOrganizer") %>% select(FishID) %>% unique() %>% dim() +histN=ad %>% subset(DoesSampleHaveHistologyImage=="Yes") %>% select(FishID) %>% unique() %>% dim() allN=ad_long_all %>% select(FishID) %>% unique() %>% dim() perc_not_Hist=100-round(100*histN[1]/allN[1], 0) @@ -166,7 +166,7 @@ pander(style_table0(summary_multiple_modes_maturity_all), missing="") **Sex categorization table** -```{r num_read_sex_ex} +```{r sex_composition_tab_adv} # Sex categorization TABLE - Advanced stagers ################################################# @@ -175,14 +175,14 @@ set.caption('**Table X:** Sex categorization table: presents the number of categorizations made per expert for each modal sex category.') # Output table -pander(style_table0(sex_composition_tab_ex), missing = "-", style = "simple") +pander(style_table0(sex_composition_tab_adv), missing = "-", style = "simple") ``` **Maturity staging table** -```{r num_read_maturity_ex} +```{r num_read_maturity_adv} # maturity stagings TABLE - Advanced stagers ################################################# @@ -191,14 +191,14 @@ set.caption('**Table X:** Maturity staging table: presents the number of stagings made per expert stager for each modal maturity stage.') # Output table -pander(style_table0(maturity_composition_tab_ex), missing = "-", style = "simple") +pander(style_table0(maturity_composition_tab_adv), missing = "-", style = "simple") ``` **Coefficient of Unalikeability (CU) table by modal sex category** -```{r cv_sex_ex} +```{r cv_sex_adv} # CU TABLE by modal sex category - Advanced stagers ################################################# @@ -209,14 +209,14 @@ set.caption('**Table X:** Coefficient of unlikeability (CU) table by modal sex c per stager.') # Output table -pander(style_table1(cu_tab_sex_ex), missing = "-") +pander(style_table1(cu_tab_sex_adv), missing = "-") ``` **Coefficient of Unalikeability (CU) table by modal maturity stage** -```{r cv_maturity_ex} +```{r cv_maturity_adv} # CU TABLE by modal maturity stage - Advanced stagers ################################################# @@ -227,14 +227,14 @@ set.caption('**Table X:** Coefficient of unlikeability (CU) table by modal matur per stager.') # Output table -pander(style_table1(cu_tab_maturity_ex), missing = "-") +pander(style_table1(cu_tab_maturity_adv), missing = "-") ``` **PA table by modal sex category** -```{r percentage_agreement_sex_ex} +```{r percentage_agreement_sex_adv} # PERCENTAGE AGREEMENT TABLE by modal sex category - Advanced stagers ############################### @@ -244,14 +244,14 @@ set.caption('**Table X:** Percentage agreement (PA) table by modal sex category: combined and a weighted mean of the PA per stager.') # Output table -pander(style_table1(pa_tab_sex_ex), missing = "-") +pander(style_table1(pa_tab_sex_adv), missing = "-") ``` **PA table by modal maturity stage** -```{r percentage_agreement_maturity_ex} +```{r percentage_agreement_maturity_adv} # PERCENTAGE AGREEMENT TABLE by modal maturity stage - Advanced stagers ############################### @@ -261,7 +261,7 @@ set.caption('**Table X:** Percentage agreement (PA) table: presents the PA per combined and a weighted mean of the PA per stager.') # Output table -pander(style_table1(pa_tab_maturity_ex), missing = "-") +pander(style_table1(pa_tab_maturity_adv), missing = "-") ``` @@ -269,7 +269,7 @@ pander(style_table1(pa_tab_maturity_ex), missing = "-") **General frequency bias table by modal sex category ** -```{r rb_sex_ex,results='asis'} +```{r rb_sex_adv,results='asis'} # Frequency bias table by modal sex category - Advanced stagers ###################################### @@ -278,14 +278,14 @@ set.caption('**Table X:** Frequency bias table by modal sex category: presents t modal maturity for all the advanced stager together.') # Output table -pander(style_table0(general_bias_freq_tab_sex_ex), missing = "-") +pander(style_table0(general_bias_freq_tab_sex_adv), missing = "-") ``` **General frequency bias table by modal maturity stage** -```{r rb_maturity_ex,results='asis'} +```{r rb_maturity_adv,results='asis'} # Frequency bias table by modal maturity stage - Advanced stagers ###################################### @@ -294,7 +294,7 @@ set.caption('**Table X:** Frequency bias table by modal maturity stage: presents modal maturity stage for all the advanced stager together.') # Output table -pander(style_table0(general_bias_freq_tab_maturity_ex), missing = "-") +pander(style_table0(general_bias_freq_tab_maturity_adv), missing = "-") ``` @@ -310,7 +310,7 @@ cap_in <- '**Figure X:** Sex categorization bias plot by modal sex category for modal sex category and sex category for all the advanced stager together. ' # Output figure -plot_general_freq_sex(ad_long_ex, strata=NULL) +plot_general_freq_sex(ad_long_adv, strata=NULL) ``` @@ -327,7 +327,7 @@ cap_in <- '**Figure X:** Maturity staging bias plot by modal maturity stage for modal maturity and maturity stage for all the advanced stager together. ' # Output figure -plot_general_freq_matur(ad_long_ex, strata=NULL) +plot_general_freq_matur(ad_long_adv, strata=NULL) ``` @@ -592,5 +592,4 @@ pander(style_table2(get(vsname("pa_tab_maturity_by"))), missing = "-") ```{r ref.label='pa_matur_by', eval = print_strata()} ``` - # Conclusion diff --git a/utilities_data.R b/utilities_data.R index 51df208..0466bcd 100644 --- a/utilities_data.R +++ b/utilities_data.R @@ -1,13 +1,13 @@ - # Determine modal maturity and CV ################################################## # For each sample the cv and modal maturity are calculated. # If one maturity is more frequent than others, that maturity is chosen as modal maturity. -# If no maturity is more frequent, then the average of all ages are chosen or -# if two (or more) ages are equally frequent then the age read by the most -# expericed reader will be chosen as modal age. +# If no maturity is more frequent, then the average of all maturities are chosen or +# if two (or more) maturities are equally frequent then the maturity by the most +# EventOrganizer will be chosen as modal maturity. # WHich method to use is set in the ma_method variable. -# If the modal age is 0 the CV is set to 0 as well. +# If the modal maturity is 0 the CV is set to 0 as well. + add_modal_trad <- function(ad, varmod, ma_method) { @@ -156,7 +156,8 @@ add_modal_negexpweight <- function(ad, varmod, ma_method) { select_mode=function(ad, ma_method, mode_definition){ - dat1=ad[ad$TypeAnnotation=="eventOrganizer",] %>% + #dat1=ad[ad$TypeAnnotation=="eventOrganizer",] %>% + dat1=ad[ad$TypeAnnotation=="eventOrganizer" & ad$DoesSampleHaveHistologyImage=="Yes",] %>% mutate(modal_maturity=Maturity, modal_sex=Sex) %>% select(FishID, SampleID, modal_maturity, modal_sex) %>% distinct() diff --git a/utilities_model.R b/utilities_model.R index bce19b6..b30a26d 100644 --- a/utilities_model.R +++ b/utilities_model.R @@ -1,4 +1,3 @@ - format_table_modal_maturity <- function(tab, fmt = "%i", extra_rows = "Total", matur_unique = modal_matur_unique) { # tab[] <- lapply(tab, function(x) ifelse(is.na(x), "-", sprintf(fmt, x))) cbind(`Modal maturity` = c(matur_unique, extra_rows), tab) @@ -22,7 +21,6 @@ format_table_sex_stage <- function(tab, fmt = "%i", extra_rows = "Total", sex_un sample_data_overview_table <- function(dat, strata) { - #debug #dat<-ad_long_all #strata<-config$strata @@ -236,7 +234,7 @@ pa_table <- function(ad_long, varmod, by = "reader") { reader_freq_table <- function(ad_long, varmod, by=NULL) { if(is.null(by)) { - reader_freq_tab <- + reader_freq_tab <- ad_long %>% ddply(.(eval(parse(text=paste0("modal_", tolower(varmod)))), get(all_of(varmod)), reader), summarise, count=length(get(all_of(varmod)))) colnames(reader_freq_tab) <- c(paste0("modal_", tolower(varmod)), tolower(varmod), "reader","count") @@ -252,13 +250,13 @@ reader_freq_table <- function(ad_long, varmod, by=NULL) { reader_freq_tab <- reader_freq_tab %>% select(-count.x, -count.y) - reader_freq_tab <- + reader_freq_tab <- reader_freq_tab %>% spread(key=reader, value=freqrelat) #temp <- # expand.grid( - # unique(ad_long[, paste0("modal_", tolower(varmod))]), + # unique(ad_long[, paste0("modal_", tolower(varmod))]), # unique(ad_long[, varmod]) # ) temp <- data.frame(expand.grid(unique(eval(parse(text=paste0("ad_long$modal_", tolower(all_of(varmod)))))), unique(eval(parse(text=paste0("ad_long$", all_of(varmod))))))) @@ -275,6 +273,8 @@ reader_freq_table <- function(ad_long, varmod, by=NULL) { reader_freq_tab$Total <- round(rowMeans(as.data.frame(lapply(reader_freq_tab[,3:dim(reader_freq_tab)[2]],as.numeric)), na.rm=TRUE), digits=3) reader_freq_tab[is.na(reader_freq_tab)] <- "-" + reader_freq_tab2 <- slice(reader_freq_tab, 1:(n() - 1)) + } else { reader_freq_tab <- @@ -307,13 +307,11 @@ reader_freq_table <- function(ad_long, varmod, by=NULL) { reader_freq_tab=rbind(reader_freq_tab, c("all", "all", as.numeric(colMeans(reader_freq_tab[,4:dim(reader_freq_tab)[2]], na.rm=TRUE)))) reader_freq_tab[is.na(reader_freq_tab)]=0 - reader_freq_tab$Total=round(rowMeans(as.data.frame(lapply(reader_freq_tab[,4:dim(reader_freq_tab)[2]],as.numeric)), na.rm=TRUE), digits=3) - - reader_freq_tab[is.na(reader_freq_tab)]="-" - + reader_freq_tab[is.na(reader_freq_tab)]<-"-" + reader_freq_tab2 <- slice(reader_freq_tab, 1:(n() - 1)) } - return(reader_freq_tab) + return(reader_freq_tab2) } @@ -333,12 +331,12 @@ general_freq_table <- function(ad_long, varmod, by=NULL) { temp=data.frame(expand.grid(unique(eval(parse(text=paste0("ad_long$", varmod)))))) colnames(temp)=c("modal_maturity") - general_freq_tab=merge(general_freq_tab, temp, by.x=c("modes"), by.y=c("modal_maturity"), all.y=T) - general_freq_tab$modes=as.character(general_freq_tab$modes) - - general_freq_tab=general_freq_tab[with(general_freq_tab, order(modes)),] - general_freq_tab[is.na(general_freq_tab)]="-" - + general_freq_tab<-merge(general_freq_tab, temp, by.x=c("modes"), by.y=c("modal_maturity"), all.y=T) + general_freq_tab$modes<-as.character(general_freq_tab$modes) + general_freq_tab<-general_freq_tab[with(general_freq_tab, order(modes)),] + general_freq_tab[is.na(general_freq_tab)]<-"-" + general_freq_tab2 <- slice(general_freq_tab, 1:(n() - 1)) + colnames(general_freq_tab)=c(paste0("modal_", tolower(varmod), "/", tolower(varmod), "_stages"), colnames(general_freq_tab)[-1]) } else { general_freq_tab <- @@ -349,18 +347,21 @@ general_freq_table <- function(ad_long, varmod, by=NULL) { general_freq_tab %>% spread(key=get(all_of(varmod)), value=frequency) general_freq_tab=general_freq_tab[with(general_freq_tab, order(get(all_of(by)))),] - general_freq_tab[is.na(general_freq_tab)]="-" + general_freq_tab[is.na(general_freq_tab)]<-"-" + general_freq_tab2 <- slice(general_freq_tab, 1:(n() - 1)) } - return(general_freq_tab) + return(general_freq_tab2) } data_overview_table <- function(dat, varmod, report_token) { - if (any(dat$TypeAnnotation == "eventOrganizer")) { + #if (any(dat$TypeAnnotation == "eventOrganizer")) { + if (any(dat$DoesSampleHaveHistologyImage == "Yes")) { hist <- - dat[dat$TypeAnnotation == "eventOrganizer", ] %>% + #dat[dat$TypeAnnotation == "eventOrganizer", ] %>% + dat[dat$DoesSampleHaveHistologyImage == "Yes", ] %>% ddply(.(SampleID), summarise, Histology = "yes") %>% select(., c("SampleID", "Histology")) dat <- merge(dat, hist, by.x = "SampleID", by.y = "SampleID", all.x = TRUE) @@ -379,7 +380,7 @@ data_overview_table <- function(dat, varmod, report_token) { readings <- ad_wide %>% select(matches("R[0-9][0-9]*")) - + complete <- complete.cases(readings) ad_wide[c("Mode", "PA %", "CU %")] <- NA @@ -433,6 +434,7 @@ maturity_composition <- function(dat, by = "reader") { as.data.frame } + sex_composition <- function(dat, by = "reader") { # Number of gonads staged per reader and maturity stage dat %>%