Add test for notification service with self-signed certs

Signed-off-by: Siddhesh Ghadi <sghadi1203@gmail.com>

tests/auxiliary/webhook-server/Dockerfile

@@ -0,0 +1,14 @@
+FROM docker.io/library/golang:1.20 as builder
+
+WORKDIR /go/src/github.com/adnanh/webhook
+RUN git clone --depth 1 -b 2.8.1 https://github.com/adnanh/webhook.git ./ && \
+    go build -o server github.com/adnanh/webhook
+
+FROM docker.io/library/ubuntu:22.04
+
+WORKDIR /var/webhook
+COPY hooks.yaml /var/webhook/hooks.yaml
+COPY --from=builder /go/src/github.com/adnanh/webhook/server /var/webhook/server
+RUN chmod +x /var/webhook/server
+
+CMD ["/var/webhook/server", "-hooks", "/var/webhook/hooks.yaml", "-verbose"]

tests/auxiliary/webhook-server/README.md

@@ -0,0 +1,19 @@
+# Webhook Server
+
+This directory contains source for webhook server used in argocd-operator testing. The server is built using [adnanh/webhook](https://github.com/adnanh/webhook).
+
+## Multiarch Container Image
+
+Use `docker-build-push.sh` shell script to build & push multiarch container image. It uses `docker buildx` to build image for amd64, arm64, ppc64le & s390x architecture. Before you run below script, ensure you have push access to the image registry referenced in script.
+
+```bash
+./docker-build-push.sh
+```
+
+## Local Build (for development)
+
+Use build command to build image for debugging/testing changes to this image.
+
+```bash
+docker build -t quay.io/svghadi/webhook-server:latest -f Dockerfile .
+```

tests/auxiliary/webhook-server/docker-build-push.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# https://docs.docker.com/build/building/multi-platform/#getting-started
+# create builder instance to build multiarch image
+BUIDLER=$(docker buildx create --use)
+
+# build & push multiarch image
+docker buildx build \
+        --push \
+        --tag quay.io/svghadi/webhook-server:latest \
+        --platform linux/amd64,linux/arm64,linux/ppc64le,linux/s390x \
+        -f Dockerfile .
+
+# remove builder instance
+docker buildx rm -f $BUIDLER

tests/auxiliary/webhook-server/hooks.yaml

@@ -0,0 +1,3 @@
+- id: example
+  execute-command: "/usr/bin/date"
+  command-working-directory: "/var/webhook"

tests/k8s/1-034_validate_webhook_notifications/00-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: webhook
+status:
+  availableReplicas: 1

tests/k8s/1-034_validate_webhook_notifications/00-setup-webhook-server.yaml

@@ -0,0 +1,168 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: webhook
+spec:
+  selector:
+    app: webhook
+  ports:
+    - name: https
+      port: 443
+      targetPort: 9000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: webhook
+  labels:
+    app: webhook
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: webhook
+  template:
+    metadata:
+      labels:
+        app: webhook
+    spec:
+      containers:
+        - name: server
+          image: quay.io/svghadi/webhook-server:latest
+          command: ["/var/webhook/server"]
+          args:
+            - -hooks
+            - /var/webhook/hooks.yaml
+            - -cert
+            - /var/webhook/tls.crt
+            - -key
+            - /var/webhook/tls.key
+            - -secure
+            - -verbose
+          ports:
+            - containerPort: 9000
+          volumeMounts:
+            - mountPath: "/var/webhook/tls.crt"
+              name: webhook-tls
+              subPath: tls.crt
+            - mountPath: "/var/webhook/tls.key"
+              name: webhook-tls
+              subPath: tls.key
+            - mountPath: "/var/webhook/hooks.yaml"
+              name: webhook-config
+              subPath: hooks.yaml
+      volumes: 
+        - name: webhook-tls
+          secret:
+            secretName: webhook-tls
+        - name: webhook-config
+          configMap:
+            name: webhook-config
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: webhook-config
+data:
+  hooks.yaml: |
+    - id: example
+      execute-command: "/usr/bin/date"
+      command-working-directory: "/var/webhook"
+      incoming-payload-content-type: "application/json"
+      pass-environment-to-command:
+        - source: entire-payload
+          envname: PAYLOAD
+---
+kind: Secret
+apiVersion: v1
+metadata:
+  name: webhook-tls
+stringData:
+  # certificate is valid till 2123
+  tls.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIFrjCCA5agAwIBAgIUbM9O0W6IdumLQodDCDqyckYDr2IwDQYJKoZIhvcNAQEL
+    BQAwTTELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAoMBFRlc3Qx
+    DTALBgNVBAsMBFRlc3QxETAPBgNVBAMMCHRlc3QuY29tMCAXDTIzMTEyNjIyMTg0
+    N1oYDzIxMjMxMTI3MjIxODQ3WjBNMQswCQYDVQQGEwJVUzENMAsGA1UECAwEVGVz
+    dDENMAsGA1UECgwEVGVzdDENMAsGA1UECwwEVGVzdDERMA8GA1UEAwwIdGVzdC5j
+    b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDbgAmnUjFux9u2Xzhi
+    mno5zjA/YsoXr3eFtK9XtByQMLLyT0hbXoa9gpTeafOs3IkCotPdN+omxm2tN9UA
+    ebAq+EamWyIF28EA3UbCWWULghveezrmAKSMcqQqby3knbcbGng+ZZjRdC3xc0uz
+    /sd4FqaLt0UHBDMlpxRskj/S3CDetfyIrKYQcZ5NQjx75aRN8At5OPC1NiWTmlsv
+    ppa4LLV0HR6AJzq+C6RAmJTcHQOFAq33wZEHHIpoQoGWHHPpT0ut54KIiVTRJ2o4
+    MEV4KlBBgL3ux4+v7R0RfVmzgaMEDG1fC9tX8pIofv7wP7WX/5XHTjyAiv8gbpUW
+    nLiU8FoTDZWxZN+MiCkUvZl8KqotbcUPjhnRdnq4anFwywY1lKILnCIayqzI7mPW
+    12h39fNwprFz9YFYbLLoQHekir2nLw8ZH83nNyD82YQ3EFm7UnOld6zw/8aURRuQ
+    C0oOEHyAXsvIyaWAb6lWvplDdCUGQWWr7MVp5YPPhWdtAv7B4QLDUNHGQMU/1Qrq
+    VBH22lcU7XrCh6GXrRVm+gF7kAuJzkuae0txvk9mHc+8Y0C4/i9C3xU2qHjWcElw
+    etcHbqOZjDtC8+n8mDD4hDYEMGV54VhXCKwoFLneT2no27S3SVPvNbMfyyNuUa2i
+    5azKnIf439Cmfww7ImxIpOR5nQIDAQABo4GDMIGAMB0GA1UdDgQWBBQfe95iWKlT
+    K6BGFov9JFXQTQN0ZjAfBgNVHSMEGDAWgBQfe95iWKlTK6BGFov9JFXQTQN0ZjAP
+    BgNVHRMBAf8EBTADAQH/MC0GA1UdEQQmMCSCB3dlYmhvb2uCDndlYmhvb2stc2Vy
+    dmVygglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggIBAH7Vv+Iar1UbF41c9I88
+    oIu8iWfLVnAfe/64tULy77x4nfEQiukBJDoZ9m19KEVdPqsFzT6lFB7Fu1oc9A28
+    5b1+PEynHcopNK41zF4n4FnIy9h8zJfaPYYCPPMT0v9LzuT5zyF5sXCz0o4KwQJ6
+    zrggZme8udl9sWyDxZyFoFPLWtnQFY7vJ9LSM2Gt+XUIuYNwDkvGFs6RfBYJGarX
+    qq7YHYj0H2x/us3KQCXGX5GzSmM9ewHvaScRpFcCdVwszKwWF0vMvdnh+3P72/Yy
+    dQvZXyfNiwqaIdznJn/AjzR9K4dHfbY7wMm83WHwWyjzV6CybHbtWpoUIlZtW3TT
+    gz6MP2z+BhOdMiQA33aO38J2TX/CMkEvkagEiZdS9t3xtpF2LOb5bRIdlENtZU0i
+    LnhgWEpJmswxBtuJ0d/zcyUlvK7FYoJZB7pT3YX/321HXZVCKyw+xrinwQoI3RnX
+    7u0TZ3MqtSKEwCyDWYRJDbs6XUX1G0q7jXBf1+3cd+lBdOZ4Kl5B4YSU9hcFxAuO
+    4a1eFXBdmT8PnwoTizFvag3IgBXkf8PqcKNvSMU6UKcD5LYTwRGK3JVl1L79gkrb
+    LmWEfOXFHgSlMIZkEs41TiopXy8p/LSera8NR86Q3mTZ7rRdEveOb6ZLJksRqaqr
+    UVwpFuaKz5vTCD36Gmmy/u8y
+    -----END CERTIFICATE-----
+  tls.key: |
+    -----BEGIN PRIVATE KEY-----
+    MIIJQAIBADANBgkqhkiG9w0BAQEFAASCCSowggkmAgEAAoICAQDbgAmnUjFux9u2
+    Xzhimno5zjA/YsoXr3eFtK9XtByQMLLyT0hbXoa9gpTeafOs3IkCotPdN+omxm2t
+    N9UAebAq+EamWyIF28EA3UbCWWULghveezrmAKSMcqQqby3knbcbGng+ZZjRdC3x
+    c0uz/sd4FqaLt0UHBDMlpxRskj/S3CDetfyIrKYQcZ5NQjx75aRN8At5OPC1NiWT
+    mlsvppa4LLV0HR6AJzq+C6RAmJTcHQOFAq33wZEHHIpoQoGWHHPpT0ut54KIiVTR
+    J2o4MEV4KlBBgL3ux4+v7R0RfVmzgaMEDG1fC9tX8pIofv7wP7WX/5XHTjyAiv8g
+    bpUWnLiU8FoTDZWxZN+MiCkUvZl8KqotbcUPjhnRdnq4anFwywY1lKILnCIayqzI
+    7mPW12h39fNwprFz9YFYbLLoQHekir2nLw8ZH83nNyD82YQ3EFm7UnOld6zw/8aU
+    RRuQC0oOEHyAXsvIyaWAb6lWvplDdCUGQWWr7MVp5YPPhWdtAv7B4QLDUNHGQMU/
+    1QrqVBH22lcU7XrCh6GXrRVm+gF7kAuJzkuae0txvk9mHc+8Y0C4/i9C3xU2qHjW
+    cElwetcHbqOZjDtC8+n8mDD4hDYEMGV54VhXCKwoFLneT2no27S3SVPvNbMfyyNu
+    Ua2i5azKnIf439Cmfww7ImxIpOR5nQIDAQABAoIB/2wImLfBvJLJy1n3g8kEPyQ0
+    V4rbFJyTwEAOrj58Z5KQZYLdgr91xtt/acYOX+C0qrqhaaV338c14sVetXeGbS65
+    BAzczeIURuol/q2pUhJX91+JR3Ps3RBDXImGLxBWj8jHPmd3mb99bx9nn9r3izWP
+    8GjTyyWo64OcuHC3irI9pe/3olOiphlx0ng0IZDZdgTmIL+JRu/ptpTvY/IQDB6Z
+    4rVDn79zj3X6RN2GO74aiaDtsLJAkyDs6zJliWJYnrQ2UwlE6PpKnXRT8fO1zntW
+    WCnlM5ZSomX0TlpNV9kB9ToI48vkChE/UrCb0N5ufPJS2WU/HIgn4WoVA0wd1rqO
+    OYfJB1IMY2RoWR9CXO0U51tCji+M83ATq+Fl0Xbxl8grn/q0PWlhmUvS9/Fe8aPA
+    yVTkEjT2j7MQGtqAO7L+xTUfVfGpFkDUn+QkM8BgNcygagN5ViOfWDFgMgjaFLrd
+    RZMh9kBi3Qjigj0NP4RaK4/ixURMT/FfwiRwEaH/1O1KXB3a0vanVuiXj5+oCrSE
+    gRBXdRt2+5FOtli8asre7NLk9unTDY1iEiIsVY8nIV+zmWhf2mR5MB34EoTEIunb
+    OaP9kbiJI6MctKoCsfsWNHfUDPsvriQevG65WETZ1/JKxxjxYlv/Xg702Cnk91Qv
+    DPrdZCbunMTP3pk5KMECggEBAO0W6hWye+r6e8aBX431Vhv78FDE/suE4iWeCCbA
+    to7gTnwWZfAB9ynp61bJDS7jXon7Vk0ExkB6nxNTIEj+Yn86M3+UjjuoadCL6hhL
+    h6xpkc1h1mj5A4IR/yi7RQgHmjKGHURgKyFIwAMYPXNVYD1Ozn9DyGmhG4LcGVQS
+    zfqclJu5oBCegAkf8EjIaDqMZGJZefxp8UYQy9FjAH1zzG/DXiEWgSPuwoeAu8Ep
+    SCKsc8EbmxLl9HvJCwvrVaqfuUygLESc/hZZoUFN6fAOQst2B5FS/ZklUECCGiiW
+    7/8nnL7wbILV+AcGYVQrUBij9CtUzBZpcMMkHREkmZeN6wkCggEBAO0B+C+kAoat
+    UCfFG5I2Ds4Cro71AEpuWvEl6wtp5WKiZYuHR4ssGDUOshD4uLb44y4mqTphTiU+
+    REV0RLQ/9mgFEmErK2glqkRKdskophbPTGQgwxgmfdQWe0Q42yuo47ljNZVEO201
+    SxgpOrHlRYzOQ9XGJmuduKxnrarOYfEXJu1WiGbsiEtY/mrMOov6rcbNsZqsWYqG
+    kmE5Msg1PsuFvlQ9ndVmE+pd3rEIhYxicD8pyFvonvi2uMmR8HmNShWKi1FZxq8e
+    OlIgdsY4BuqnNUrnQprhm0hG5cGwcl5auL2+Jc5Uagm/egvtwxPhx+pVYcimKOL9
+    CutpY7BeuvUCggEAC6UrfENXCNSizb4/Bkb9osQ+KolyhmaRgQ2BEv42OVBVKo0j
+    FqXSERH3SDz508rBMv/QXloUrsgXFijoFg3AosUmEGcokU+VWvP0XJshH9vTmIXs
+    tR0+Cd5+bO691kYhUcf6mggrNihPnhdLtWWFI53CUMfwiRertULAT7vYuC2Gsxtr
+    /ET8vvX9pGWLkQyiRZ5lenttqWZbzH4TYRYV/YtYDUIAt9YbYfJ1xmgTrfhQezSy
+    6ju3RXk7fKtjesz7mgLoCbq4VDq0y/NawTrCFyJF/uJXqHUHuxNo24OGaD722P4Q
+    JmECHL44e5zhA0TSUmqI17T4H+2fK99jV+lVmQKCAQB2nTi3pw54ln56GOSOjS1l
+    nuP7udQWbBppe7+ha7MYZQwLA34jwcKvsxYc9k2DjRYtf73L8OzqKLqERAcqaqSI
+    NJmZNcC4k7keCmJelFBjNAYYSmk5SfJJVaMFZqsRs6mcm3Eyrf5LzpMxmVi9tW/U
+    Y1qBv3R1AW9uIUlCJZ3QyfR6bYdAc3pWs0hI7MMUUTXtO/552W3KrUTPEZA/sJ4n
+    v1yczmWSak7nSWltEkW8F3vzsJaMoOQGt3PNtZMzUinUlAzbfuG3vJoVhhfLZjjX
+    8Szzur+Twfsz9f+Aqyzh2eeBVouXMpoLHOAY3jp2VdX2ihqxD6+AwoFXhdwVZaON
+    AoIBAF0/qvwsFThhB9a1wnXuGx1OBY+9owIoinIF2qNcHuqeontxfLWBg1izelJg
+    gxaATIMvpXgt7y5cBx6fLnylpLgl+TNXCrsrcLnXwJz0Neg/gcSZfcnqwhAhTio9
+    iYLVJiK8wnh0pXONutGSasgq3tJLyrzT2+1L5jYKUaFkojIR16sHjo3/MJMPTHvL
+    fF1DX7y6acz3JXrGJYQsqcrVodSfcGZK/RJQkdvrSdBRZYgWq+CBYViOxkN7cscr
+    ruQ/DZH/ZCIxVckbuVsAMqdCqAO0gX83eEp7elfAVlnLhvxPluxISuXaJmhJNafr
+    Xq+NinfrqOLJkIZ/u/PJu4KqN3M=
+    -----END PRIVATE KEY-----

tests/k8s/1-034_validate_webhook_notifications/01-assert.yaml

@@ -0,0 +1,9 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ArgoCD
+metadata:
+  name: argocd
+status:
+  phase: Available
+  notificationsController: Running 
+  applicationController: Running 
+  server: Running 

tests/k8s/1-034_validate_webhook_notifications/01-install-argocd.yaml

@@ -0,0 +1,44 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ArgoCD
+metadata:
+  name: argocd
+spec:
+  notifications:
+    enabled: true
+  tls: 
+    initialCerts:
+      # value is copied from tls.crt key in setup-webhook-server.yaml
+      webhook: |
+        -----BEGIN CERTIFICATE-----
+        MIIFrjCCA5agAwIBAgIUbM9O0W6IdumLQodDCDqyckYDr2IwDQYJKoZIhvcNAQEL
+        BQAwTTELMAkGA1UEBhMCVVMxDTALBgNVBAgMBFRlc3QxDTALBgNVBAoMBFRlc3Qx
+        DTALBgNVBAsMBFRlc3QxETAPBgNVBAMMCHRlc3QuY29tMCAXDTIzMTEyNjIyMTg0
+        N1oYDzIxMjMxMTI3MjIxODQ3WjBNMQswCQYDVQQGEwJVUzENMAsGA1UECAwEVGVz
+        dDENMAsGA1UECgwEVGVzdDENMAsGA1UECwwEVGVzdDERMA8GA1UEAwwIdGVzdC5j
+        b20wggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQDbgAmnUjFux9u2Xzhi
+        mno5zjA/YsoXr3eFtK9XtByQMLLyT0hbXoa9gpTeafOs3IkCotPdN+omxm2tN9UA
+        ebAq+EamWyIF28EA3UbCWWULghveezrmAKSMcqQqby3knbcbGng+ZZjRdC3xc0uz
+        /sd4FqaLt0UHBDMlpxRskj/S3CDetfyIrKYQcZ5NQjx75aRN8At5OPC1NiWTmlsv
+        ppa4LLV0HR6AJzq+C6RAmJTcHQOFAq33wZEHHIpoQoGWHHPpT0ut54KIiVTRJ2o4
+        MEV4KlBBgL3ux4+v7R0RfVmzgaMEDG1fC9tX8pIofv7wP7WX/5XHTjyAiv8gbpUW
+        nLiU8FoTDZWxZN+MiCkUvZl8KqotbcUPjhnRdnq4anFwywY1lKILnCIayqzI7mPW
+        12h39fNwprFz9YFYbLLoQHekir2nLw8ZH83nNyD82YQ3EFm7UnOld6zw/8aURRuQ
+        C0oOEHyAXsvIyaWAb6lWvplDdCUGQWWr7MVp5YPPhWdtAv7B4QLDUNHGQMU/1Qrq
+        VBH22lcU7XrCh6GXrRVm+gF7kAuJzkuae0txvk9mHc+8Y0C4/i9C3xU2qHjWcElw
+        etcHbqOZjDtC8+n8mDD4hDYEMGV54VhXCKwoFLneT2no27S3SVPvNbMfyyNuUa2i
+        5azKnIf439Cmfww7ImxIpOR5nQIDAQABo4GDMIGAMB0GA1UdDgQWBBQfe95iWKlT
+        K6BGFov9JFXQTQN0ZjAfBgNVHSMEGDAWgBQfe95iWKlTK6BGFov9JFXQTQN0ZjAP
+        BgNVHRMBAf8EBTADAQH/MC0GA1UdEQQmMCSCB3dlYmhvb2uCDndlYmhvb2stc2Vy
+        dmVygglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQADggIBAH7Vv+Iar1UbF41c9I88
+        oIu8iWfLVnAfe/64tULy77x4nfEQiukBJDoZ9m19KEVdPqsFzT6lFB7Fu1oc9A28
+        5b1+PEynHcopNK41zF4n4FnIy9h8zJfaPYYCPPMT0v9LzuT5zyF5sXCz0o4KwQJ6
+        zrggZme8udl9sWyDxZyFoFPLWtnQFY7vJ9LSM2Gt+XUIuYNwDkvGFs6RfBYJGarX
+        qq7YHYj0H2x/us3KQCXGX5GzSmM9ewHvaScRpFcCdVwszKwWF0vMvdnh+3P72/Yy
+        dQvZXyfNiwqaIdznJn/AjzR9K4dHfbY7wMm83WHwWyjzV6CybHbtWpoUIlZtW3TT
+        gz6MP2z+BhOdMiQA33aO38J2TX/CMkEvkagEiZdS9t3xtpF2LOb5bRIdlENtZU0i
+        LnhgWEpJmswxBtuJ0d/zcyUlvK7FYoJZB7pT3YX/321HXZVCKyw+xrinwQoI3RnX
+        7u0TZ3MqtSKEwCyDWYRJDbs6XUX1G0q7jXBf1+3cd+lBdOZ4Kl5B4YSU9hcFxAuO
+        4a1eFXBdmT8PnwoTizFvag3IgBXkf8PqcKNvSMU6UKcD5LYTwRGK3JVl1L79gkrb
+        LmWEfOXFHgSlMIZkEs41TiopXy8p/LSera8NR86Q3mTZ7rRdEveOb6ZLJksRqaqr
+        UVwpFuaKz5vTCD36Gmmy/u8y
+        -----END CERTIFICATE-----

tests/k8s/1-034_validate_webhook_notifications/02-update-notifications-cm.yaml

@@ -0,0 +1,19 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: argocd-notifications-cm
+data:
+  service.webhook.test-webhook: |
+    url: https://webhook/hooks/example
+  template.test-app-created: |
+    webhook:
+      test-webhook:
+        method: POST
+        body: |
+          {"created":"{{.app.metadata.name}}"}
+  trigger.test-on-created: |-
+    - description: Application is created.
+      send: ["test-app-created"]
+      when: "true"
+
+

tests/k8s/1-034_validate_webhook_notifications/03-sleep.yaml

@@ -0,0 +1,5 @@
+# let notifications controller pick up configmap changes.
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: sleep 20

tests/k8s/1-034_validate_webhook_notifications/04-create-app.yaml

@@ -0,0 +1,25 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    set -e
+
+    cat << EOF | kubectl apply -f -
+    apiVersion: argoproj.io/v1alpha1
+    kind: Application
+    metadata:
+      name: my-app-3
+      namespace: $NAMESPACE
+      annotations:
+        "notifications.argoproj.io/subscribe.test-on-created.test-webhook": ""
+    spec:
+      destination:
+        namespace: $NAMESPACE
+        server: https://kubernetes.default.svc
+      project: default
+      source:
+        repoURL: https://github.com/redhat-developer/gitops-operator
+        path: test/examples/nginx
+        targetRevision: HEAD
+    EOF
+- script: sleep 5

tests/k8s/1-034_validate_webhook_notifications/05-verify-webhook-notification.yaml

@@ -0,0 +1,25 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+# verify no x509 in notifications-controller logs
+- script: | 
+    #!/bin/bash
+    kubectl -n $NAMESPACE logs deployment.apps/argocd-notifications-controller | grep "x509"
+
+    PREV_CMD=$?
+    if [ $PREV_CMD -eq 1 ]; then
+        exit 0
+    else
+        exit 1
+    fi
+# verify notification delivery
+- script: | 
+    #!/bin/bash
+    kubectl -n $NAMESPACE logs deployment.apps/webhook | grep '{"created":"my-app-3"}'
+
+    PREV_CMD=$?
+    if [ $PREV_CMD -eq 0 ]; then
+        exit 0
+    else
+        exit 1
+    fi