From 90054b3b272b48f5b55850f920a6765bb018a9b8 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Wed, 17 Jul 2024 15:06:28 +0530 Subject: [PATCH 1/4] kuma: bump e2e version to 2.7.5 Signed-off-by: Sanskar Jaiswal --- .github/workflows/build.yaml | 1 + .github/workflows/e2e.yaml | 1 + .github/workflows/scan.yml | 3 +- pkg/metrics/observers/kuma.go | 6 +-- pkg/metrics/observers/kuma_test.go | 4 +- test/kuma/install.sh | 13 +++---- test/kuma/run.sh | 1 + test/kuma/test-canary.sh | 61 +++++++++++++++++++----------- test/workloads/init.sh | 2 +- 9 files changed, 55 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 88f175356..b2eaf715f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -16,6 +16,7 @@ jobs: build-flagger: runs-on: group: "Default Larger Runners" + labels: ubuntu-latest-16-cores steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 22f0b7cf8..fc18effc5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -16,6 +16,7 @@ jobs: e2e-test: runs-on: group: "Default Larger Runners" + labels: ubuntu-latest-16-cores strategy: fail-fast: false matrix: diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml index 57e998605..72b60d21f 100644 --- a/.github/workflows/scan.yml +++ b/.github/workflows/scan.yml @@ -25,8 +25,7 @@ jobs: fossa-api-key: 5ee8bf422db1471e0bcf2bcb289185de github-token: ${{ github.token }} scan-codeql: - runs-on: - group: "Default Larger Runners" + runs-on: ubuntu-latest permissions: security-events: write steps: diff --git a/pkg/metrics/observers/kuma.go b/pkg/metrics/observers/kuma.go index c652732d2..e32d7bfc6 100644 --- a/pkg/metrics/observers/kuma.go +++ b/pkg/metrics/observers/kuma.go @@ -33,7 +33,7 @@ var kumaQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq{ - envoy_cluster_name=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", + service=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", envoy_response_code!~"5.*" }[{{ interval }}] ) @@ -42,7 +42,7 @@ var kumaQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq{ - envoy_cluster_name=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", + service=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", }[{{ interval }}] ) ) @@ -53,7 +53,7 @@ var kumaQueries = map[string]string{ sum( rate( envoy_cluster_upstream_rq_time_bucket{ - envoy_cluster_name=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", + service=~"{{ target }}-canary_{{ namespace }}_svc_[0-9a-zA-Z-]+", }[{{ interval }}] ) ) by (le) diff --git a/pkg/metrics/observers/kuma_test.go b/pkg/metrics/observers/kuma_test.go index dda802a11..0178917b5 100644 --- a/pkg/metrics/observers/kuma_test.go +++ b/pkg/metrics/observers/kuma_test.go @@ -30,7 +30,7 @@ import ( ) func TestKumaObserver_GetRequestSuccessRate(t *testing.T) { - expected := ` sum( rate( envoy_cluster_upstream_rq{ envoy_cluster_name=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ envoy_cluster_name=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", }[1m] ) ) * 100` + expected := ` sum( rate( envoy_cluster_upstream_rq{ service=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ service=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", }[1m] ) ) * 100` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] @@ -65,7 +65,7 @@ func TestKumaObserver_GetRequestSuccessRate(t *testing.T) { } func TestKumaObserver_GetRequestDuration(t *testing.T) { - expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ envoy_cluster_name=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", }[1m] ) ) by (le) )` + expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ service=~"podinfo-canary_default_svc_[0-9a-zA-Z-]+", }[1m] ) ) by (le) )` ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { promql := r.URL.Query()["query"][0] diff --git a/test/kuma/install.sh b/test/kuma/install.sh index fcf0db8ae..e85f0886b 100755 --- a/test/kuma/install.sh +++ b/test/kuma/install.sh @@ -2,14 +2,13 @@ set -o errexit -KUMA_VER="2.1.0" +KUMA_VER="2.7.5" REPO_ROOT=$(git rev-parse --show-toplevel) mkdir -p ${REPO_ROOT}/bin echo ">>> Downloading Kuma ${KUMA_VER}" -curl -SsL https://download.konghq.com/mesh-alpine/kuma-${KUMA_VER}-ubuntu-amd64.tar.gz -o kuma-${KUMA_VER}.tar.gz -tar xvzf kuma-${KUMA_VER}.tar.gz -cp kuma-${KUMA_VER}/bin/kumactl ${REPO_ROOT}/bin/kumactl +curl -L https://docs.konghq.com/mesh/installer.sh | VERSION=${KUMA_VER} sh - +cp kong-mesh-${KUMA_VER}/bin/kumactl ${REPO_ROOT}/bin/kumactl chmod +x ${REPO_ROOT}/bin/kumactl echo ">>> Installing Kuma ${KUMA_VER}" @@ -17,7 +16,7 @@ ${REPO_ROOT}/bin/kumactl install control-plane | kubectl apply -f - echo ">>> Waiting for Kuma Control Plane to be ready" kubectl wait --for condition=established crd/meshes.kuma.io -kubectl -n kuma-system rollout status deployment/kuma-control-plane +kubectl -n kong-mesh-system rollout status deployment/kong-mesh-control-plane echo ">>> Installing Prometheus" ${REPO_ROOT}/bin/kumactl install observability --components "prometheus" | kubectl apply -f - @@ -26,5 +25,5 @@ kubectl -n mesh-observability rollout status deployment/prometheus-server echo '>>> Installing Flagger' kubectl apply -k ${REPO_ROOT}/kustomize/kuma -kubectl -n kuma-system set image deployment/flagger flagger=test/flagger:latest -kubectl -n kuma-system rollout status deployment/flagger +kubectl -n kong-mesh-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n kong-mesh-system rollout status deployment/flagger diff --git a/test/kuma/run.sh b/test/kuma/run.sh index 67153fa9f..d28c073f1 100755 --- a/test/kuma/run.sh +++ b/test/kuma/run.sh @@ -8,4 +8,5 @@ DIR="$(cd "$(dirname "$0")" && pwd)" "$DIR"/install.sh "$REPO_ROOT"/test/workloads/init.sh +kubectl label namespace test kuma.io/sidecar-injection=enabled "$DIR"/test-canary.sh diff --git a/test/kuma/test-canary.sh b/test/kuma/test-canary.sh index b2ec3969a..7030fb468 100755 --- a/test/kuma/test-canary.sh +++ b/test/kuma/test-canary.sh @@ -6,6 +6,21 @@ set -o errexit REPO_ROOT=$(git rev-parse --show-toplevel) +cat <>> Waiting for primary to be ready' @@ -68,7 +84,7 @@ until ${ok}; do sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then - kubectl -n kuma-system logs deployment/flagger + kubectl -n kong-mesh-system logs deployment/flagger echo "No more retries left" exit 1 fi @@ -94,10 +110,10 @@ ok=false until ${ok}; do kubectl -n test describe deployment/podinfo-primary | grep '6.0.1' && ok=true || ok=false sleep 10 - kubectl -n kuma-system logs deployment/flagger --tail 1 + kubectl -n kong-mesh-system logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then - kubectl -n kuma-system logs deployment/flagger + kubectl -n kong-mesh-system logs deployment/flagger echo "No more retries left" exit 1 fi @@ -112,7 +128,7 @@ until ${ok}; do sleep 5 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then - kubectl -n kuma-system logs deployment/flagger + kubectl -n kong-mesh-system logs deployment/flagger echo "No more retries left" exit 1 fi @@ -159,19 +175,20 @@ spec: threshold: 500 interval: 30s webhooks: - - name: acceptance-test - type: pre-rollout - url: http://flagger-loadtester.test/ - timeout: 30s - metadata: - type: bash - cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" + # temproarily disabled due to upstream issues + # - name: acceptance-test + # type: pre-rollout + # url: http://flagger-loadtester.test/ + # timeout: 30s + # metadata: + # type: bash + # cmd: "curl -sd 'test' http://podinfo-canary.test:9898/token | grep token" - name: load-test url: http://flagger-loadtester.test/ timeout: 5s metadata: type: cmd - cmd: "hey -z 2m -q 10 -c 2 http://podinfo-canary.test:9898/status/500" + cmd: "hey -z 2m -q 10 -c 2 http://podinfo.test:9898/status/500" EOF echo '>>> Triggering canary deployment rollback test' @@ -184,10 +201,10 @@ ok=false until ${ok}; do kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false sleep 10 - kubectl -n kuma-system logs deployment/flagger --tail 1 + kubectl -n kong-mesh-system logs deployment/flagger --tail 1 count=$(($count + 1)) if [[ ${count} -eq ${retries} ]]; then - kubectl -n kuma-system logs deployment/flagger + kubectl -n kong-mesh-system logs deployment/flagger echo "No more retries left" exit 1 fi diff --git a/test/workloads/init.sh b/test/workloads/init.sh index e76b55ee2..04d98efbc 100755 --- a/test/workloads/init.sh +++ b/test/workloads/init.sh @@ -13,7 +13,7 @@ echo '>>> Creating test namespace' kubectl create namespace test kubectl label namespace test istio-injection=enabled kubectl annotate namespace test linkerd.io/inject=enabled -kubectl annotate namespace test kuma.io/sidecar-injection=enabled +kubectl label namespace test kuma.io/sidecar-injection=enabled echo '>>> Installing the load tester' kubectl apply -k ${REPO_ROOT}/kustomize/tester From 01dfa0689146edbe002dac8d9fc94942219001b2 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Wed, 17 Jul 2024 18:07:10 +0530 Subject: [PATCH 2/4] kuma: update default namespace to kong-mesh-system Signed-off-by: Sanskar Jaiswal --- docs/gitbook/tutorials/kuma-progressive-delivery.md | 2 +- kustomize/kuma/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gitbook/tutorials/kuma-progressive-delivery.md b/docs/gitbook/tutorials/kuma-progressive-delivery.md index b09133c88..626e62a2f 100644 --- a/docs/gitbook/tutorials/kuma-progressive-delivery.md +++ b/docs/gitbook/tutorials/kuma-progressive-delivery.md @@ -15,7 +15,7 @@ kumactl install control-plane | kubectl apply -f - kumactl install observability --components "grafana,prometheus" | kubectl apply -f - ``` -Install Flagger in the `kuma-system` namespace: +Install Flagger in the `kong-mesh-system` namespace: ```bash kubectl apply -k github.com/fluxcd/flagger//kustomize/kuma diff --git a/kustomize/kuma/kustomization.yaml b/kustomize/kuma/kustomization.yaml index e17f64b06..21a72816c 100644 --- a/kustomize/kuma/kustomization.yaml +++ b/kustomize/kuma/kustomization.yaml @@ -1,4 +1,4 @@ -namespace: kuma-system +namespace: kong-mesh-system bases: - ../base/flagger/ patchesStrategicMerge: From 74b98dab006a17c93d74202f496990898fd66162 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Mon, 22 Jul 2024 18:57:26 +0530 Subject: [PATCH 3/4] kuma: add ingress annotaions as custom metadata Signed-off-by: Sanskar Jaiswal --- test/kuma/test-canary.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/kuma/test-canary.sh b/test/kuma/test-canary.sh index 7030fb468..67e6cef64 100755 --- a/test/kuma/test-canary.sh +++ b/test/kuma/test-canary.sh @@ -41,12 +41,18 @@ spec: apex: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" canary: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" primary: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" analysis: interval: 15s threshold: 15 @@ -156,15 +162,21 @@ spec: apex: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" canary: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" primary: annotations: 9898.service.kuma.io/protocol: "http" + ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/service-upstream: "true" analysis: interval: 15s - threshold: 15 + threshold: 5 maxWeight: 50 stepWeight: 10 metrics: From 4015103815512a94eeefec4737ea5a1cc9e496a6 Mon Sep 17 00:00:00 2001 From: Sanskar Jaiswal Date: Mon, 22 Jul 2024 19:30:36 +0530 Subject: [PATCH 4/4] kuma: disable daemonset for e2e Signed-off-by: Sanskar Jaiswal --- test/kuma/run.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/kuma/run.sh b/test/kuma/run.sh index d28c073f1..f8bf80806 100755 --- a/test/kuma/run.sh +++ b/test/kuma/run.sh @@ -9,4 +9,5 @@ DIR="$(cd "$(dirname "$0")" && pwd)" "$REPO_ROOT"/test/workloads/init.sh kubectl label namespace test kuma.io/sidecar-injection=enabled +kubectl delete -n test ds podinfo-ds "$DIR"/test-canary.sh