Merge pull request #23 from ghga-de/main

syn dev
ghga-de · Dec 4, 2023 · 49916d0 · 49916d0
2 parents dda220c + 1645fec
commit 49916d0
Show file tree

Hide file tree

Showing 13 changed files with 318 additions and 64 deletions.
diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml
@@ -0,0 +1,52 @@
+name: Bug report
+description: Report something that is broken or incorrect
+labels: bug
+body:
+  - type: checkboxes
+    attributes:
+      label: Have you checked the docs?
+      description: I have checked the following places for my error
+      options:
+        - label: "[nf-core website: troubleshooting](https://nf-co.re/usage/troubleshooting)"
+          required: true
+        - label: "[nf-core modules documentation](https://nf-co.re/docs/contributing/modules)"
+          required: true
+
+  - type: textarea
+    id: description
+    attributes:
+      label: Description of the bug
+      description: A clear and concise description of what the bug is.
+    validations:
+      required: true
+
+  - type: textarea
+    id: command_used
+    attributes:
+      label: Command used and terminal output
+      description: Steps to reproduce the behaviour. Please paste the command you used to launch the pipeline and the output from your terminal.
+      render: console
+      placeholder: |
+        $ nextflow run ...
+
+        Some output where something broke
+
+  - type: textarea
+    id: files
+    attributes:
+      label: Relevant files
+      description: |
+        Please drag and drop the relevant files here. Create a `.zip` archive if the extension is not allowed.
+        Your verbose log file `.nextflow.log` is often useful _(this is a hidden file in the directory where you launched the pipeline)_ as well as custom Nextflow configuration files.
+
+  - type: textarea
+    id: system
+    attributes:
+      label: System information
+      description: |
+        * Nextflow version _(eg. 21.10.3)_
+        * Hardware _(eg. HPC, Desktop, Cloud)_
+        * Executor _(eg. slurm, local, awsbatch)_
+        * Container engine and version: _(e.g. Docker, Singularity)_
+        * OS and version: _(eg. CentOS Linux, macOS, Ubuntu 22.04)_
+        * Image tag: <!-- [e.g. biocontainers/platypus-variant:0.8.1.2] -->
diff --git a/.github/ISSUE_TEMPLATE/feature_reuest.yml b/.github/ISSUE_TEMPLATE/feature_reuest.yml
@@ -0,0 +1,32 @@
+name: Feature request
+description: Suggest an idea
+labels: feature
+title: "[FEATURE]"
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe
+      description: A clear and concise description of what the bug is.
+      placeholder: |
+        <!-- e.g. [I'm always frustrated when ...] -->
+    validations:
+      required: true
+
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of the solution you want to happen.
+
+  - type: textarea
+    id: alternatives
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+
+  - type: textarea
+    id: additional_context
+    attributes:
+      label: Additional context
+      description: Add any other context about the feature request here.
diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml
@@ -0,0 +1,45 @@
+name: branch protection
+# This workflow is triggered on PRs to master branch on the repository
+# It fails when someone tries to make a PR against the `master` branch instead of `dev`
+on:
+  pull_request_target:
+    branches: [master]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      # PRs to the master branch are only ok if coming from the repo `dev` or any `patch` branches
+      - name: Check PRs
+        if: github.repository == 'ghga-de/nf-snvcalling'
+        run: |
+          { [[ ${{github.event.pull_request.head.repo.full_name }} == ghga-de/nf-snvcalling ]] && [[ $GITHUB_HEAD_REF = "dev" ]]; } || [[ $GITHUB_HEAD_REF == "patch" ]]
+
+
+      # If the above check failed, post a comment on the PR explaining the failure
+      # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets
+      - name: Post PR comment
+        if: failure()
+        uses: mshick/add-pr-comment@v1
+        with:
+          message: |
+            ## This PR is against the `master` branch :x:
+
+            * Do not close this PR
+            * Click _Edit_ and change the `base` to `dev`
+            * This CI test will remain failed until you push a new commit
+
+            ---
+
+            Hi @${{ github.event.pull_request.user.login }},
+
+            It looks like this pull-request is has been made against the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `master` branch.
+            The `master` branch on nf-core repositories should always contain code from the latest release.
+            Because of this, PRs to `master` are only allowed if they come from the [${{github.event.pull_request.head.repo.full_name }}](https://github.com/${{github.event.pull_request.head.repo.full_name }}) `dev` branch.
+
+            You do not need to close this PR, you can change the target branch to `dev` by clicking the _"Edit"_ button at the top of this page.
+            Note that even after this, the test will continue to show as failing until you push a new commit.
+
+            Thanks again for your contribution!
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          allow-repeats: false
diff --git a/.github/workflows/code_linting.yml b/.github/workflows/code_linting.yml
@@ -0,0 +1,46 @@
+name: Code Linting
+on:
+  push:
+    branches:
+      - master
+      - dev
+      - 14-cicd-test-integration
+  pull_request:
+    branches: [master]
+  merge_group:
+    types: [checks_requested]
+    branches: [master]
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  Prettier:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Install NodeJS
+        uses: actions/setup-node@v2
+
+      - name: Install Prettier
+        run: npm install -g prettier
+
+      - name: Run Prettier --check
+        run: prettier --check .
+
+  EditorConfig:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v2
+
+      - name: Install editorconfig-checker
+        run: npm install -g editorconfig-checker
+
+      - name: Run ECLint check
+        run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,44 @@
+name: Main
+# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors
+on:
+  push:
+    branches:
+      - dev
+      - 14-cicd-test-integration
+  pull_request:
+  release:
+    types: [published]
+
+# Uncomment if we need an edge release of Nextflow again
+# env: NXF_EDGE: 1
+
+jobs:
+  test:
+    name: Run workflow tests
+    runs-on: ubuntu-latest
+    env:
+      NXF_VER: ${{ matrix.nxf_ver }}
+      NXF_ANSI_LOG: false
+    strategy:
+      matrix:
+        # Nextflow versions: check pipeline minimum and current latest
+        nxf_ver: ['22.10.6']
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v3
+
+      - name: Install Nextflow
+        env:
+          CAPSULE_LOG: none
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+      - name: HELPTEXT Run with the help flag
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} --help
+      - name: DELAY to try address some odd behaviour with what appears to be a conflict between parallel htslib jobs leading to CI hangs
+        run: | 
+          if [[ $NXF_VER = '' ]]; then sleep 1200; fi
+      - name: BASIC Run the basic pipeline only for SNV calling with docker
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile test,docker
diff --git a/README.md b/README.md
@@ -3,13 +3,8 @@
 [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/)
 [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/)
 
-|        |                     | | |
-| :-----------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------: |
-| [![National Genomics Infrastructure](docs/images/denbi.png )](https://ngisweden.scilifelab.se/)|              [![National Bioinformatics Infrastructure Sweden](docs/images/odcf_blue.png)](https://nbis.se)         |[![QBiC](docs/images/DKFZ_Logo.png)](https://www.qbic.uni-tuebingen.de)|[![GHGA](docs/images/GHGA_short_Logo_orange.png)](https://www.ghga.de/)|
-
-
 <p align="center">
-    <img title="nf-snvcalling workflow" src="docs/images/nf-snvcalling-2.png" width=70%>
+    <img title="nf-snvcalling workflow" src="docs/images/nf-snvcalling3.png" width=70%>
 </p>
 
 

diff --git a/conf/test.config b/conf/test.config
@@ -5,32 +5,79 @@
     Defines input files and everything required to run a fast and simple pipeline test.
 
     Use as follows:
-        nextflow run nf/snvcalling -profile test,<docker/singularity> --outdir <OUTDIR>
-
+        nextflow run main.nf -profile test,singularity
+        nextflow run main.nf -profile test,docker    
 ----------------------------------------------------------------------------------------
 */
 
 params {
-    config_profile_name        = 'Test profile'
-    config_profile_description = 'Minimal test dataset to check pipeline function'
+    config_profile_contact     = 'Kübra Narcı kuebra.narci@dkfz-heidelberg.de'
+    config_profile_name        = 'Test profile for CI/CD test'
+    config_profile_description = 'Use test dataset and test profile for pytest on github actions'
 
     // Limit resources so that this can run on GitHub Actions
-    max_cpus   = 2
-    max_memory = '6.GB'
-    max_time   = '6.h'
+    max_cpus   = 30
+    max_memory = '250.GB'
+    max_time   = '48.h'
 
     // Input data
-    // TODO nf-core: Specify the paths to your test data on nf-core/test-datasets
-    // TODO nf-core: Give any required params for the test so that command line flags are not needed
-    input  =  'assets/samplesheet.csv'
-
-    // Genome references
-    genome = 'R64-1-1'
-    fasta = '/Users/w620-admin/Desktop/Workflows/data/hs37d5_PhiX.fa'
-    ref_type = null
-
-    enchangers = 'enchanger'
-    cpgislands = 'cpgislands'
-    tfbscons = null
-    mirnas_snornas = null
+    input       = "${projectDir}/seq2_testdata_snv/samplesheet_test.csv"
+
+    // workflow parameters
+    outdir      = "${projectDir}/results"
+
+    // Post Processes
+    runSNVAnnotation           = false
+    runArtifactFilter          = false
+    generateExtendedQcPlots    = false 
+    runSNVDeepAnnotation       = false
+    runSNVVCFFilter            = false
+    skip_multiqc               = false
+    runCompareGermline         = false
+    runplots                   = false
+    runpurest                  = false
+    runcontigs                 = "NONE"
+
+    // Reference Files //
+    genome                     = "GRCh38"
+
 }
+
+// Enable container engines/virtualisation envs for CI testing
+// only works when specified with the profile ENV
+// otherwise tests can be done with the regular provided profiles
+if (System.getenv('PROFILE')) {
+    if ("$PROFILE" == "docker") {
+        conda.enabled                            = false
+        docker.enabled                           = true
+        docker.userEmulation                     = { params.use_gatk_spark ? false : true }.call()
+        charliecloud.enabled                     = false
+        podman.enabled                           = false
+        shifter.enabled                          = false
+        singularity.enabled                      = false
+    } else if ("$PROFILE" == "singularity") {
+        conda.enabled                            = false
+        params.singularity_pull_docker_container = false
+        singularity.autoMounts                   = true
+        singularity.enabled                      = true
+        charliecloud.enabled                     = false
+        docker.enabled                           = false
+        podman.enabled                           = false
+        shifter.enabled                          = false
+    }
+}
+process {
+   withName:'BCFTOOLS_MPILEUP'{
+        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 6.GB * task.attempt, 'memory' ) }
+   }
+   withName:'FILE_CONCATENATOR'{
+        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 6.GB * task.attempt, 'memory' ) }
+   }
+   withName:'MULTIQC'{
+        cpus            = { check_max( 2 * task.attempt, 'cpus' ) }
+        memory          = { check_max( 6.GB * task.attempt, 'memory' ) }
+   }
+}
+
diff --git a/docs/images/nf-snvcalling.png b/docs/images/nf-snvcalling.png
diff --git a/docs/images/nf-snvcalling3.png b/docs/images/nf-snvcalling3.png