Merge integration test changes

.github/workflows/dea-coastlines-image.yaml

@@ -0,0 +1,128 @@
+name: Image build and test
+
+env:
+  IMAGE_NAME: geoscienceaustralia/dea-coastlines
+
+on:
+  push:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  pull_request:
+    branches:
+      - develop
+    paths:
+      - 'coastlines/**'
+      - 'data/**'
+      - 'tests/**'
+      - '.github/workflows/dea-coastlines-image.yml'
+      - 'Dockerfile'
+      - 'requirements.in'
+      - 'requirements.txt'
+      - 'setup.py'
+      - 'codecov.yaml'
+
+  release:
+    types: [edited, published]
+
+permissions:
+  id-token: write  # Required for requesting Json web token
+  contents: write  # Required for actions/checkout
+  pull-requests: write  # Required for validation results comment bot
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::060378307146:role/github-actions-dea-notebooks--sandbox
+          aws-region: ap-southeast-2
+          role-duration-seconds: 7200  # 2 hours
+
+      - name: Get database credentials
+        run: |
+          username_password=$(aws ssm get-parameter --name /dea-sandbox-eks/sandbox_reader/db.creds --with-decryption  --query Parameter.Value --output text)
+          echo DATACUBE_DB_URL=postgresql://${username_password}@localhost:5432/odc >> $GITHUB_ENV
+
+      - name: Open port forward to RDS
+        run: |
+          npx basti connect \
+            --custom-target-vpc vpc-086904199e505c1f6 \
+            --custom-target-host db-aurora-dea-sandbox-eks-1.cos5zfpkso9m.ap-southeast-2.rds.amazonaws.com \
+            --custom-target-port 5432 \
+            --local-port 5432 &
+          # Wait until the connection comes up, but, if it doesn't, don't hang forever.
+          npx wait-on --timeout 120000 --interval 1000 tcp:127.0.0.1:5432
+          echo "PGPORT=5432" >> $GITHUB_ENV
+          echo "PGHOST=localhost" >> $GITHUB_ENV
+
+      - name: Build DEA Coastlines docker image 
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          docker build -t dea_coastlines .
+
+      - name: Run integration tests
+        run: |
+          # Download tide modelling files and unzip
+          wget --no-verbose https://www.dropbox.com/s/uemd8ib2vfw5nad/tide_models.zip?dl=1 -O tide_models.zip
+          unzip -q tide_models.zip
+
+          # Run integration tests using Docker
+          docker run \
+            --net=host \
+            --env DATACUBE_DB_URL \
+            --env AWS_SESSION_TOKEN \
+            --env AWS_REGION \
+            --env AWS_ACCESS_KEY_ID \
+            --env AWS_SECRET_ACCESS_KEY \
+            --env AWS_SESSION_TOKEN \
+            --volume ${GITHUB_WORKSPACE}:/code \
+            --volume ${GITHUB_WORKSPACE}/tide_models:/var/share/tide_models \
+            dea_coastlines pytest -v --cov=coastlines --cov-report=xml tests
+
+          ls -l .
+          ls -l ./data/validation/processed/
+          ls -l ./tests/
+          git status
+
+      # Commit validation results produced by integration tests back into repo
+      - name: Commit validation results into repository
+        uses: stefanzweifel/git-auto-commit-action@v4
+        if: github.event_name == 'pull_request'
+        continue-on-error: true
+        with:
+          commit_message: Automatically update integration test validation results
+          file_pattern: 'tests/stats_tests.png tests/stats_tests.csv tests/README.md'
+
+      # Post validation tesults as comment on PR
+      - name: Post validation results as comment
+        uses: mshick/add-pr-comment@v2
+        if: github.event_name == 'pull_request'
+        with:
+          message: |
+            ![](https://github.com/GeoscienceAustralia/dea-coastlines/blob/${{ github.head_ref }}/tests/stats_tests.png?raw=true")
+            
+            For full integration test results, refer to the [Tests directory README](https://github.com/GeoscienceAustralia/dea-coastlines/tree/${{ github.head_ref }}/tests).
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/docker.yaml

@@ -1,59 +1,59 @@
-name: Build and push image
-
-# Run action on both a push to the "develop" branch,
-# and when a Github release is created/edited/published
-on:
-  push:
-    branches:
-      - develop
-    paths:
-      - 'coastlines/**'
-      - '.github/workflows/docker.yaml'
-      - 'Dockerfile'
-
-  release:
-    types: [created, edited, published]
-
-permissions:
-  id-token: write   # This is required for requesting the JWT
-  contents: read    # This is required for actions/checkout
-
-env:
-  IMAGE_NAME: geoscienceaustralia/dea-coastlines
-
-jobs:
-  push_ecr:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
-      # If action is triggered by a release, push image to
-      # Dockerhub using custom image tag extracted from the release
-      - name: Get current version tag from release to use as image tag
-        if: github.event_name == 'release'
-        run: |
-          echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV
-
-      # If action is trigged by a push (not release), push a
-      # latest/unstable image to Dockerhub using image tag based
-      # on the most recent Github tag and commit hash
-      - name: Get git commit hash for push to branch to use as image tag
-        if: github.event_name != 'release'
-        run: |
-          git fetch --all --tags
-          echo "RELEASE=$(git describe --tags)" >> $GITHUB_ENV
-
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          role-to-assume: arn:aws:iam::538673716275:role/github-actions-role
-          aws-region: ap-southeast-2
-
-      - name: Push image to ECR
-        uses: whoan/docker-build-with-cache-action@master
-        with:
-          context: ./
-          registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com
-          image_name: ${{ env.IMAGE_NAME }}
-          image_tag: latest,${{ env.RELEASE }}
+# name: Build and push image
+
+# # Run action on both a push to the "develop" branch,
+# # and when a Github release is created/edited/published
+# on:
+#   push:
+#     branches:
+#       - develop
+#     paths:
+#       - 'coastlines/**'
+#       - '.github/workflows/docker.yaml'
+#       - 'Dockerfile'
+
+#   release:
+#     types: [created, edited, published]
+
+# permissions:
+#   id-token: write   # This is required for requesting the JWT
+#   contents: read    # This is required for actions/checkout
+
+# env:
+#   IMAGE_NAME: geoscienceaustralia/dea-coastlines
+
+# jobs:
+#   push_ecr:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout code
+#         uses: actions/checkout@v3
+
+#       # If action is triggered by a release, push image to
+#       # Dockerhub using custom image tag extracted from the release
+#       - name: Get current version tag from release to use as image tag
+#         if: github.event_name == 'release'
+#         run: |
+#           echo "RELEASE=${GITHUB_REF/refs\/tags\/}" >> $GITHUB_ENV
+
+#       # If action is trigged by a push (not release), push a
+#       # latest/unstable image to Dockerhub using image tag based
+#       # on the most recent Github tag and commit hash
+#       - name: Get git commit hash for push to branch to use as image tag
+#         if: github.event_name != 'release'
+#         run: |
+#           git fetch --all --tags
+#           echo "RELEASE=$(git describe --tags)" >> $GITHUB_ENV
+
+#       - name: Configure AWS credentials
+#         uses: aws-actions/configure-aws-credentials@v1
+#         with:
+#           role-to-assume: arn:aws:iam::538673716275:role/github-actions-role
+#           aws-region: ap-southeast-2
+
+#       - name: Push image to ECR
+#         uses: whoan/docker-build-with-cache-action@master
+#         with:
+#           context: ./
+#           registry: 538673716275.dkr.ecr.ap-southeast-2.amazonaws.com
+#           image_name: ${{ env.IMAGE_NAME }}
+#           image_tag: latest,${{ env.RELEASE }}