Merge pull request #31 from ghga-de/27-rearrange-source-requirements

27 rearrange source requirements
ghga-de · Jan 2, 2024 · ed86d5c · ed86d5c
2 parents e42f66a + b7ba924
commit ed86d5c
Show file tree

Hide file tree

Showing 42 changed files with 219 additions and 255 deletions.
diff --git a/.DS_Store b/.DS_Store
diff --git a/README.md b/README.md
@@ -138,7 +138,7 @@ Note that some form of configuration will be needed so that Nextflow knows how t
 
 **control**: The path to the control file, if there is no control will be kept blank.
 
-**contro*index***: The path to the control index file, if there is no control will be kept blank.
+**control_index***: The path to the control index file, if there is no control will be kept blank.
 
 ## Data Requirements
 
@@ -178,6 +178,8 @@ All VCF and BED files need to be indexed with tabix and should be in the same fo
 - UCSC Phast Cons Elements (bed)
 - UCSC Encode TFBS (bed)
 
+NOTE: The reference set bundle which is used in PCAWG study can be found and downloaded [here](https://dcc.icgc.org/api/v1/download?fn=/PCAWG/reference_data/pcawg-dkfz/dkfz-workflow-dependencies_150318_0951.tar.gz). (NOTE: only in hg19)
+
 ## Documentation
 
 The nf-snvcalling pipeline comes with documentation about the pipeline [usage](https://github.com/ghga-de/nf-snvcalling/blob/main/docs/usage.md) and [output](https://github.com/ghga-de/nf-snvcalling/blob/main/docs/output.md).
@@ -195,7 +197,7 @@ The ICGC/TCGA Pan-Cancer Analysis of Whole Genomes Consortium.
 Nature volume 578, pages 82–93 (2020).
 DOI 10.1038/s41586-020-1969-6
 
-We tank the following people for their extensive assistance in the development of this pipeline:
+We thank the following people for their extensive assistance in the development of this pipeline:
 
 Nagarajan Paramasivam (@NagaComBio) [n.paramasivam@dkfz.de](mailto:n.paramasivam@dkfz.de)
 

diff --git a/assets/samplesheet_hg37_whole.csv b/assets/samplesheet_hg37_whole.csv
diff --git a/assets/samplesheet_hg38_WES.csv b/assets/samplesheet_hg38_WES.csv
@@ -0,0 +1,2 @@
+sample,tumor,tumor_index,control,control_index
+SEQC2_LL1,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/exon_sequencing/results_per_pid/SEQC2_LL1/alignment_hg38/tumor01_SEQC2_LL1_merged.mdup.bam,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/exon_sequencing/results_per_pid/SEQC2_LL1/alignment_hg38/tumor01_SEQC2_LL1_merged.mdup.bam.bai,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/exon_sequencing/results_per_pid/SEQC2_LL1/alignment_hg38/control01_SEQC2_LL1_merged.mdup.bam,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/exon_sequencing/results_per_pid/SEQC2_LL1/alignment_hg38/control01_SEQC2_LL1_merged.mdup.bam.bai
diff --git a/assets/samplesheet_hg38_WGS.csv b/assets/samplesheet_hg38_WGS.csv
@@ -0,0 +1,2 @@
+sample,tumor,tumor_index,control,control_index
+SEQC2_LL1,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/whole_genome_sequencing/results_per_pid/SEQC2_LL1/alignment/tumor01_SEQC2_LL1_merged.mdup.bam,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/whole_genome_sequencing/results_per_pid/SEQC2_LL1/alignment/tumor01_SEQC2_LL1_merged.mdup.bam.bai,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/whole_genome_sequencing/results_per_pid/SEQC2_LL1/alignment/control01_SEQC2_LL1_merged.mdup.bam,/omics/odcf/analysis/OE0526_projects/public_data_analyses/seqc2/sequencing/whole_genome_sequencing/results_per_pid/SEQC2_LL1/alignment/control01_SEQC2_LL1_merged.mdup.bam.bai
diff --git a/assets/samplesheet_val_WES.csv b/assets/samplesheet_val_WES.csv
@@ -0,0 +1,6 @@
+sample,tumor,tumor_index,control,control_index
+H021-1HBRFX,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-1HBRFX/alignment/metastasis_V_H021-1HBRFX_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-1HBRFX/alignment/metastasis_V_H021-1HBRFX_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-1HBRFX/alignment/blood_V_H021-1HBRFX_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-1HBRFX/alignment/blood_V_H021-1HBRFX_merged.mdup.bam.bai
+K26K-BF6J54,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_K26K-BF6J54/alignment/tumor12-01-p_V_K26K-BF6J54_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_K26K-BF6J54/alignment/tumor12-01-p_V_K26K-BF6J54_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_K26K-BF6J54/alignment/buffy-coat1-01-p_V_K26K-BF6J54_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_K26K-BF6J54/alignment/buffy-coat1-01-p_V_K26K-BF6J54_merged.mdup.bam.bai
+H021-XU49TH,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XU49TH/alignment/metastasis_V_H021-XU49TH_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XU49TH/alignment/metastasis_V_H021-XU49TH_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XU49TH/alignment/blood_V_H021-XU49TH_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XU49TH/alignment/blood_V_H021-XU49TH_merged.mdup.bam.bai
+H021-XCTR5S,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XCTR5S/alignment/metastasis_V_H021-XCTR5S_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XCTR5S/alignment/metastasis_V_H021-XCTR5S_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XCTR5S/alignment/blood_V_H021-XCTR5S_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-XCTR5S/alignment/blood_V_H021-XCTR5S_merged.mdup.bam.bai
+H021-HVSG6L,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-HVSG6L/alignment/tumor_V_H021-HVSG6L_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-HVSG6L/alignment/tumor_V_H021-HVSG6L_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-HVSG6L/alignment/blood_V_H021-HVSG6L_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/exon_sequencing/results_per_pid/V_H021-HVSG6L/alignment/blood_V_H021-HVSG6L_merged.mdup.bam.bai
diff --git a/assets/samplesheet_val_WGS.csv b/assets/samplesheet_val_WGS.csv
@@ -0,0 +1,8 @@
+sample,tumor,tumor_index,control,control_index
+H021-182ECT,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-182ECT/alignment/metastasis_V_H021-182ECT_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-182ECT/alignment/metastasis_V_H021-182ECT_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-182ECT/alignment/buffy_coat02_V_H021-182ECT_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-182ECT/alignment/buffy_coat02_V_H021-182ECT_merged.mdup.bam.bai 
+K26K-EVVHM1,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_K26K-EVVHM1/alignment/metastasis12_V_K26K-EVVHM1_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_K26K-EVVHM1/alignment/metastasis12_V_K26K-EVVHM1_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_K26K-EVVHM1/alignment/buffy_coat1_V_K26K-EVVHM1_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_K26K-EVVHM1/alignment/buffy_coat1_V_K26K-EVVHM1_merged.mdup.bam.bai
+H021-1K9WSG,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-1K9WSG/alignment/tumor_V_H021-1K9WSG_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-1K9WSG/alignment/tumor_V_H021-1K9WSG_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-1K9WSG/alignment/buffy_coat_V_H021-1K9WSG_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-1K9WSG/alignment/buffy_coat_V_H021-1K9WSG_merged.mdup.bam.bai
+H021-2HFQKW,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-2HFQKW/alignment/tumor03_V_H021-2HFQKW_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-2HFQKW/alignment/tumor03_V_H021-2HFQKW_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-2HFQKW/alignment/blood_V_H021-2HFQKW_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-2HFQKW/alignment/blood_V_H021-2HFQKW_merged.mdup.bam.bai
+H021-6ERRP7,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-6ERRP7/alignment/metastasis_V_H021-6ERRP7_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-6ERRP7/alignment/metastasis_V_H021-6ERRP7_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-6ERRP7/alignment/blood_V_H021-6ERRP7_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-6ERRP7/alignment/blood_V_H021-6ERRP7_merged.mdup.bam.bai
+H021-7FS1CB,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-7FS1CB/alignment/tumor_V_H021-7FS1CB_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-7FS1CB/alignment/tumor_V_H021-7FS1CB_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-7FS1CB/alignment/blood02_V_H021-7FS1CB_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-7FS1CB/alignment/blood02_V_H021-7FS1CB_merged.mdup.bam.bai
+H021-DX3UW1,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-DX3UW1/alignment/tumor02_V_H021-DX3UW1_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-DX3UW1/alignment/tumor02_V_H021-DX3UW1_merged.mdup.bam.bai,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-DX3UW1/alignment/buffy-coat04_V_H021-DX3UW1_merged.mdup.bam,/omics/odcf/analysis/evaluation_projects/workflow-validation/sequencing_full_test/whole_genome_sequencing/results_per_pid/V_H021-DX3UW1/alignment/buffy-coat04_V_H021-DX3UW1_merged.mdup.bam.bai
diff --git a/bin/vcfparser.pyc b/bin/vcfparser.pyc
diff --git a/conf/base.config b/conf/base.config
@@ -33,24 +33,24 @@ process {
     }
     withLabel:process_low {
         cpus   = { check_max( 2     * task.attempt, 'cpus'    ) }
-        memory = { check_max( 12.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 4.h   * task.attempt, 'time'    ) }
+        memory = { check_max( 6.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 6.h   * task.attempt, 'time'    ) }
     }
     withLabel:process_medium {
         cpus   = { check_max( 4     * task.attempt, 'cpus'    ) }
-        memory = { check_max( 16.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 6.h   * task.attempt, 'time'    ) }
+        memory = { check_max( 8.GB * task.attempt, 'memory'  ) }
+        time   = { check_max( 8.h   * task.attempt, 'time'    ) }
     }
     withLabel:process_intermediate {
         cpus   = { check_max( 6     * task.attempt, 'cpus'    ) }
         memory = { check_max( 36.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 8.h   * task.attempt, 'time'    ) }
+        time   = { check_max( 12.h   * task.attempt, 'time'    ) }
     }
 
     withLabel:process_high {
         cpus   = { check_max( 8    * task.attempt, 'cpus'    ) }
         memory = { check_max( 64.GB * task.attempt, 'memory'  ) }
-        time   = { check_max( 12.h  * task.attempt, 'time'    ) }
+        time   = { check_max( 16.h  * task.attempt, 'time'    ) }
     }
     withLabel:process_medium_high {
         cpus   = { check_max( 12    * task.attempt, 'cpus'    ) }
@@ -75,12 +75,20 @@ process {
         errorStrategy = 'retry'
         maxRetries    = 1
     }
-    withName:'CUSTOM_DUMPSOFTWAREVERSIONS|SET_CHR|CHECK_IF_CORRUPTED|GREP_SAMPLENAME|JSON_REPORT|MERGE_PLOTS' {
+    withName:'CUSTOM_DUMPSOFTWAREVERSIONS|SET_CHR|CHECK_IF_CORRUPTED|GREP_SAMPLENAME|JSON_REPORT|MERGE_PLOTS|SAMPLESHEET_CHECK' {
         cpus            = { check_max( 1 * task.attempt, 'cpus' ) }
         memory          = { check_max( 1.GB * task.attempt, 'memory' ) }
     }
     withName: 'MULTIQC' {
         cpus            = { check_max( 4 * task.attempt, 'cpus' ) }
         memory          = { check_max( 12.GB * task.attempt, 'memory' ) }
     }
+    withName: 'BCFTOOLS_MPILEUP' {
+        cpus            = { check_max( 8 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
+    }
+    withName: 'ANNOTATE_VCF|ANNOTATION_PIPES|FILTER_PEOVERLAP' {
+        cpus            = { check_max( 8 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 8.GB * task.attempt, 'memory' ) }
+    }
 }
diff --git a/conf/dkfz_cluster_hg37.config b/conf/dkfz_cluster_hg37.config
@@ -36,7 +36,8 @@ params {
     runCompareGermline         = true
 
     // Annovar
-    annovar_path               ="/omics/odcf/reference_data/legacy/ngs_share/annovar/annovar_Feb2016"
+    annotation_tool            = "annovar"
+    annovar_path               ="/omics/odcf/reference_data/legacy/ngs_share/annovar/annovar_Nov2014"
     buildver                   = "hg19"
     dbtype                     = "wgEncodeGencodeCompV19"
     segdupcol                  = "SEGDUP"
@@ -47,8 +48,8 @@ params {
 
     // Reference //
     data_path                  = "/omics/odcf/reference_data/legacy/ngs_share/assemblies/hg19_GRCh37_1000genomes"
-    fasta                      = "${params.data_path}/sequence/1KGRef_Phix/hs37d5_PhiX.fa"
-    fasta_fai                  = "${params.data_path}/sequence/1KGRef_Phix/hs37d5_PhiX.fa.fai"
+    fasta                      = "${params.data_path}/sequence/1KGRef/hs37d5.fa"
+    fasta_fai                  = "${params.data_path}/sequence/1KGRef/hs37d5.fa.fai"
     chrom_sizes                = "${params.data_path}/stats/hs37d5.fa.chrLenOnlyACGT_realChromosomes.tab"
     genome                     = null
     chr_prefix                 = ""

diff --git a/conf/dkfz_cluster_hg38.config b/conf/dkfz_cluster_hg38.config
@@ -38,6 +38,7 @@ params {
 
 
     // Annovar
+    annotation_tool            = "annovar"
     buildver                   = "hg38"
     dbtype                     = "wgEncodeGencodeCompV39"
     segdupcol                  = "SEGDUP"

diff --git a/conf/modules.config b/conf/modules.config
@@ -41,7 +41,7 @@ process {
     withName: BCFTOOLS_MPILEUP {
         ext.args   = "-EI -q 30 --ignore-RG --ff UNMAP,SECONDARY,QCFAIL,DUP,SUPPLEMENTARY -d 9999 -a AD -x"
         ext.args2  = "-A -EI -q 30 --ignore-RG --ff UNMAP,SECONDARY,QCFAIL,DUP -d 9999 -a AD -x"
-        ext.args3  = "-vc -p 2"
+        ext.args3  = "-vc -p 2.0"
         publishDir = [
             path:  {"${params.outdir}/${meta.id}"},
             pattern: "{noting}",
@@ -147,7 +147,7 @@ process {
         ext.args   = ""
          publishDir = [
             path: {"${params.outdir}/processed_ref"},
-            pattern: "*.{txt}",
+            pattern: "*.{txt,bed}",
             mode: params.publish_dir_mode
         ]
     } 

diff --git a/conf/test_full.config b/conf/test_full.config