diff --git a/.github/actions/print-chainsaw-debug/action.yaml b/.github/actions/print-chainsaw-debug/action.yaml new file mode 100644 index 0000000000..be8138c4ba --- /dev/null +++ b/.github/actions/print-chainsaw-debug/action.yaml @@ -0,0 +1,79 @@ +name: Print Debug +description: 'Prints debug info from running clusters' + +inputs: + kubectl-context: + description: 'This will be passed to kubectl commands as --context argument' + required: false + default: 'k3d-test-gslb1' +runs: + using: "composite" + steps: + - name: Check verbose + shell: bash + run: | + if [[ "x${{ runner.debug }}" == "x1" ]]; then + echo "verbose=true" >> $GITHUB_ENV + else + echo "verbose=false" >> $GITHUB_ENV + fi + + - name: Env vars + shell: bash + run: | + echo "::group:: ☸☸☸ envs" + env | sort + echo --------------------------------------- + echo inputs.kubectl-context=${{ inputs.kubectl-context }} + echo verbose: ${{ env.verbose }} + echo "::endgroup::" + + - name: CoreDNS Hosts + shell: bash + if: env.verbose == 'true' + run: | + echo -e "\n\n\nVerbose is on, printing all the debug stuff:\n--------------------------------------------\n\n" + echo "::group:: ☸☸☸ cluster coredns - hosts (cluster 1)" + kubectl --context=${{ inputs.kubectl-context }} get cm coredns -n kube-system -o yaml | grep -A7 NodeHosts: + echo "::endgroup::" + + - name: K8s Events + shell: bash + if: env.verbose == 'true' + run: | + echo "::group:: ☸☸☸ k get events" + kubectl --context=${{ inputs.kubectl-context }} get events -A || true + echo "::endgroup::" + + - name: gslbs + shell: bash + if: env.verbose == 'true' + run: | + echo "::group:: ☸☸☸ gslbs" + kubectl --context=${{ inputs.kubectl-context }} get gslbs -owide -A || true + echo "::endgroup::" + + - name: Dnsendpoints + shell: bash + if: env.verbose == 'true' + run: | + echo "::group:: ☸☸☸ endpoints" + kubectl --context=${{ inputs.kubectl-context }} get dnsendpoints -A || true + echo "::endgroup::" + + - name: K8gb logs + shell: bash + if: env.verbose == 'true' + run: | + echo "::group:: ☸☸☸ k8gb logs" + kubectl --context=${{ inputs.kubectl-context }} logs -lname=k8gb -n k8gb --tail=-1 || true + echo "::endgroup::" + + - name: Metrics + shell: bash + if: env.verbose == 'true' + run: | + echo "::group:: ☸☸☸ k8gb metrics (cluster 1)" + _IP=$(kubectl --context=${{ inputs.kubectl-context }} get pods -lname=k8gb -n k8gb -o custom-columns='IP:status.podIP' --no-headers) + kubectl --context=${{ inputs.kubectl-context }} run -it --rm curl-metrics --restart=Never --image=curlimages/curl:7.82.0 -- $_IP:8080/metrics + echo "::endgroup::" diff --git a/.github/actions/print-debug/action.yaml b/.github/actions/print-terratest-debug/action.yaml similarity index 100% rename from .github/actions/print-debug/action.yaml rename to .github/actions/print-terratest-debug/action.yaml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dafef023e4..dc2bb11c2e 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,6 +10,6 @@ Add the [`heavy-tests`](/k8gb-io/k8gb/issues?q=is%3A*+label%3Aheavy-tests) label on this PR if you want full-blown tests that include more than 2-cluster scenarios. ### Debug tests - If the test suite is failing for you, you may want to try triggering `Re-run all jobs` (top right) with [debug logging](https://docs.github.com/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging) enabled. It will also make the [print debug](/k8gb-io/k8gb/blob/master/.github/actions/print-debug/action.yaml) action more verbose. + If the test suite is failing for you, you may want to try triggering `Re-run all jobs` (top right) with [debug logging](https://docs.github.com/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging) enabled. It will also make the [print debug](/k8gb-io/k8gb/blob/master/.github/actions/print-terratest-debug/action.yaml) action more verbose. diff --git a/.github/workflows/chainsaw.yaml b/.github/workflows/chainsaw.yaml new file mode 100644 index 0000000000..722ae2967c --- /dev/null +++ b/.github/workflows/chainsaw.yaml @@ -0,0 +1,110 @@ +name: Chainsaw + +on: + push: + paths-ignore: + - '**.md' + - '**.svg' + - '**.drawio' + - '.spelling' + pull_request: + branches: + - master + # The specific activity types are listed here to include "labeled" and "unlabeled" + # (which are not included by default for the "pull_request" trigger). + types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled] + paths-ignore: + - '**.md' + - '**.svg' + - '**.drawio' + - '.spelling' +permissions: + contents: read + +jobs: + skip-check: + runs-on: ubuntu-latest + name: Skip the job? + outputs: + should_skip: ${{ steps.skip_check.outputs.should_skip }} + steps: + - name: Harden Runner + uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1 + with: + egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs + + - id: skip_check + uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1 + with: + skip_after_successful_duplicate: 'true' + do_not_skip: '["workflow_dispatch", "schedule"]' + + chainsaw: + runs-on: ubuntu-24.04 + needs: skip-check + if: ${{ needs.skip-check.outputs.should_skip != 'true' }} && !contains( github.event.pull_request.labels.*.name, 'renovate') + steps: + - name: Checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + with: + fetch-depth: 0 + + - name: Setup Chainsaw + uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11 + with: + release: v0.2.11 + + - name: Check install + run: chainsaw version + + - name: Setup golang + uses: actions/setup-go@bfd2fb341f32be7281829126376a12a780ca79fc + with: + go-version: 1.22.3 + + - name: Build artifacts + uses: goreleaser/goreleaser-action@286f3b13b1b49da4ac219696163fb8c1c93e1200 # v6.0.0 + with: + version: v1.9.2 + args: release --rm-dist --skip-publish --skip-validate --snapshot --skip-sbom --skip-sign + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Create edgeDNS k3s Cluster + uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79 + with: + cluster-name: "edgedns" + args: -c k3d/edge-dns.yaml + + - name: Create 1st k3s Cluster + uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79 + with: + cluster-name: "test-gslb1" + args: -c k3d/test-gslb1.yaml + + - name: Create 2nd k3s Cluster + uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79 + with: + cluster-name: "test-gslb2" + args: -c k3d/test-gslb2.yaml + + - name: K8GB deployment + run: | + make deploy-test-version list-running-pods + echo "Cluster 1 (eu):" + kubectl get no -owide --context=k3d-test-gslb1 + echo "Cluster 2 (us):" + kubectl get no -owide --context=k3d-test-gslb2 + + - name: Run Chainsaw + run: make chainsaw + + - name: Print debug info + if: always() + uses: ./.github/actions/print-chainsaw-debug + + - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 + if: always() + with: + name: chainsaw-logs + path: ${{ github.workspace }}/tmp/chainsaw diff --git a/.github/workflows/terratest-istiov1beta1.yaml b/.github/workflows/terratest-istiov1beta1.yaml index b7cd31e732..333c98e889 100644 --- a/.github/workflows/terratest-istiov1beta1.yaml +++ b/.github/workflows/terratest-istiov1beta1.yaml @@ -71,7 +71,7 @@ jobs: - name: Print debug info if: always() - uses: ./.github/actions/print-debug + uses: ./.github/actions/print-terratest-debug - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: always() diff --git a/.github/workflows/terratest-more-clusters.yaml b/.github/workflows/terratest-more-clusters.yaml index 3930d7d3a1..85cddcb043 100644 --- a/.github/workflows/terratest-more-clusters.yaml +++ b/.github/workflows/terratest-more-clusters.yaml @@ -73,7 +73,7 @@ jobs: - name: Print debug info if: always() - uses: ./.github/actions/print-debug + uses: ./.github/actions/print-terratest-debug - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: always() diff --git a/.github/workflows/terratest.yaml b/.github/workflows/terratest.yaml index 39dddf1693..77b38c5494 100644 --- a/.github/workflows/terratest.yaml +++ b/.github/workflows/terratest.yaml @@ -98,7 +98,7 @@ jobs: - name: Print debug info if: always() - uses: ./.github/actions/print-debug + uses: ./.github/actions/print-terratest-debug - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: always() diff --git a/.github/workflows/upgrade-testing.yaml b/.github/workflows/upgrade-testing.yaml index 506f712780..e8de2701c7 100644 --- a/.github/workflows/upgrade-testing.yaml +++ b/.github/workflows/upgrade-testing.yaml @@ -93,7 +93,7 @@ jobs: - name: Print debug info if: always() - uses: ./.github/actions/print-debug + uses: ./.github/actions/print-terratest-debug - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0 if: always() diff --git a/Makefile b/Makefile index 3d062f4126..ff68c3ac0c 100644 --- a/Makefile +++ b/Makefile @@ -480,6 +480,20 @@ terratest: # Run terratest suite fi cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 25m -parallel=12 --tags=$(TEST_TAGS) +# executes chainsaw e2e tests +.PHONY: chainsaw +chainsaw: + mkdir -p chainsaw/kubeconfig + k3d kubeconfig get test-gslb1 > chainsaw/kubeconfig/eu.config + k3d kubeconfig get test-gslb2 > chainsaw/kubeconfig/us.config + @$(eval RUNNING_CLUSTERS := $(shell k3d cluster list --no-headers | grep $(CLUSTER_NAME) -c)) + @if [ "$(RUNNING_CLUSTERS)" -lt 2 ] ; then \ + echo -e "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\ + exit 1;\ + fi + cd chainsaw && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) chainsaw test --config ./config.yaml --values ./values.yaml + rm -r chainsaw/kubeconfig + .PHONY: website website: @if [ "$(CI)" = "true" ]; then\ diff --git a/chainsaw/config.yaml b/chainsaw/config.yaml new file mode 100644 index 0000000000..977c54d2c9 --- /dev/null +++ b/chainsaw/config.yaml @@ -0,0 +1,12 @@ +apiVersion: chainsaw.kyverno.io/v1alpha2 +kind: Configuration +metadata: + name: custom-config +spec: + clusters: + eu: + kubeconfig: kubeconfig/eu.config + context: k3d-test-gslb1 + us: + kubeconfig: kubeconfig/us.config + context: k3d-test-gslb2 diff --git a/chainsaw/step-templates/apply-podinfo.yaml b/chainsaw/step-templates/apply-podinfo.yaml new file mode 100644 index 0000000000..192f6d58ce --- /dev/null +++ b/chainsaw/step-templates/apply-podinfo.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: apply-podinfo +spec: + try: + - apply: + file: ../../testdata/podinfo.yaml + - wait: + apiVersion: v1 + kind: Pod + name: frontend-podinfo + for: + condition: + name: Ready + value: 'true' diff --git a/chainsaw/step-templates/assert-dns.yaml b/chainsaw/step-templates/assert-dns.yaml new file mode 100644 index 0000000000..a0def923ec --- /dev/null +++ b/chainsaw/step-templates/assert-dns.yaml @@ -0,0 +1,29 @@ +--- +# asserts that both clusters resolve the domain to the correct cluster +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: assert-dns +spec: + try: + - sleep: + duration: 30s # reconcile requeue seconds + coreDNS + - script: + env: + - name: NAME + value: ($test.metadata.name) + - name: DNS_PORT_EU + value: ($values.dns_port_eu) + - name: DNS_PORT_US + value: ($values.dns_port_us) + content: | + EXPECTED=$(kubectl get dnsendpoint $NAME -n $NAMESPACE -o json | jq -r '.spec.endpoints[] | select(.dnsName | contains("localtargets")).targets | join(",")') + ACTUAL_EU=$(dig @localhost -p $DNS_PORT_EU "$NAME.cloud.example.com" +short | paste -sd, -) + + EXPECTED_SORTED=$(echo "$EXPECTED" | tr ',' '\n' | sort | tr '\n' ',') + ACTUAL_EU_SORTED=$(echo "$ACTUAL_EU" | tr ',' '\n' | sort | tr '\n' ',') + [ "$EXPECTED_SORTED" = "$ACTUAL_EU_SORTED" ] || { echo "EU expected targets '$EXPECTED_SORTED' but found '$ACTUAL_EU_SORTED'"; exit 1; } + + ACTUAL_US=$(dig @localhost -p $DNS_PORT_US "$NAME.cloud.example.com" +short | paste -sd, -) + ACTUAL_US_SORTED=$(echo "$ACTUAL_US" | tr ',' '\n' | sort | tr '\n' ',') + [ "$EXPECTED_SORTED" = "$ACTUAL_US_SORTED" ] || { echo "US expected targets '$EXPECTED_SORTED' but found '$ACTUAL_US_SORTED'"; exit 1; } diff --git a/chainsaw/step-templates/init-istio.yaml b/chainsaw/step-templates/init-istio.yaml new file mode 100644 index 0000000000..50c3908b9d --- /dev/null +++ b/chainsaw/step-templates/init-istio.yaml @@ -0,0 +1,28 @@ +--- +# Creates a namespace, a service, a pod and a gslb +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: init-ingress +spec: + try: + - apply: + file: ../../testdata/namespace-istio.yaml + - apply: + file: ../../testdata/podinfo.yaml + - apply: + file: testdata/gslb.yaml + - assert: + resource: + apiVersion: externaldns.k8s.io/v1alpha1 + kind: DNSEndpoint + metadata: + name: ($test.metadata.name) + namespace: ($namespace) + spec: + endpoints: + - targets: + (length(@)): 2 + - targets: + (length(@)): 2 + timeout: 15s diff --git a/chainsaw/step-templates/init.yaml b/chainsaw/step-templates/init.yaml new file mode 100644 index 0000000000..70ec26fe52 --- /dev/null +++ b/chainsaw/step-templates/init.yaml @@ -0,0 +1,40 @@ +--- +# Creates a namespace, a service, a pod and a gslb +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: init-ingress +spec: + try: + - apply: + file: ../../testdata/namespace.yaml + - apply: + file: ../../testdata/podinfo.yaml + - apply: + file: ./testdata/gslb.yaml + - assert: + resource: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: ($test.metadata.name) + namespace: ($namespace) + status: + loadBalancer: + ingress: + (length(@)): 2 + timeout: 60s + - assert: + resource: + apiVersion: externaldns.k8s.io/v1alpha1 + kind: DNSEndpoint + metadata: + name: ($test.metadata.name) + namespace: ($namespace) + spec: + endpoints: + - dnsName: (join('', ['localtargets-', $test.metadata.name, '.cloud.example.com'])) + targets: + (length(@)): 2 + - dnsName: (join('', [$test.metadata.name, '.cloud.example.com'])) + timeout: 15s diff --git a/chainsaw/step-templates/namespace.yaml b/chainsaw/step-templates/namespace.yaml new file mode 100644 index 0000000000..17b7ddf228 --- /dev/null +++ b/chainsaw/step-templates/namespace.yaml @@ -0,0 +1,10 @@ +--- +# Creates a namespace, a service, a pod and a gslb +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: namespace +spec: + try: + - apply: + file: ../../testdata/namespace.yaml diff --git a/chainsaw/step-templates/stop-podinfo.yaml b/chainsaw/step-templates/stop-podinfo.yaml new file mode 100644 index 0000000000..6d8bfdc758 --- /dev/null +++ b/chainsaw/step-templates/stop-podinfo.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: stop-podinfo +spec: + try: + - delete: + ref: + apiVersion: v1 + kind: Pod + name: frontend-podinfo + - wait: + apiVersion: v1 + kind: Pod + name: frontend-podinfo + for: + deletion: {} diff --git a/chainsaw/testdata/namespace-istio.yaml b/chainsaw/testdata/namespace-istio.yaml new file mode 100644 index 0000000000..4fc233e982 --- /dev/null +++ b/chainsaw/testdata/namespace-istio.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ($namespace) + labels: + istio-inject: enabled diff --git a/chainsaw/testdata/namespace.yaml b/chainsaw/testdata/namespace.yaml new file mode 100644 index 0000000000..b0dca2d14d --- /dev/null +++ b/chainsaw/testdata/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: ($namespace) diff --git a/chainsaw/testdata/podinfo.yaml b/chainsaw/testdata/podinfo.yaml new file mode 100644 index 0000000000..1752e95593 --- /dev/null +++ b/chainsaw/testdata/podinfo.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-podinfo + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + type: ClusterIP + ports: + - port: 9898 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: frontend-podinfo +--- +apiVersion: v1 +kind: Pod +metadata: + name: frontend-podinfo + labels: + app.kubernetes.io/name: frontend-podinfo +spec: + containers: + - name: podinfo + image: "ghcr.io/stefanprodan/podinfo:5.1.1" + command: + - ./podinfo + - --port=9898 + ports: + - name: http + containerPort: 9898 + protocol: TCP + resources: + requests: + memory: 64Mi + cpu: 250m + limits: + memory: 128Mi + cpu: 500m diff --git a/chainsaw/tests/failover-playground-embedded/chainsaw-test.yaml b/chainsaw/tests/failover-playground-embedded/chainsaw-test.yaml new file mode 100644 index 0000000000..5952b91c5d --- /dev/null +++ b/chainsaw/tests/failover-playground-embedded/chainsaw-test.yaml @@ -0,0 +1,52 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: failover-playground-embedded +spec: + namespace: failover-playground-embedded + steps: + - name: namespace EU + description: create namespace on EU cluster + cluster: eu + use: + template: ../../step-templates/namespace.yaml + - name: namespace US + description: create namespace on US cluster + cluster: us + use: + template: ../../step-templates/namespace.yaml + - name: init EU + description: setup podinfo application and GSLB on EU cluster + cluster: eu + use: + template: ../../step-templates/init.yaml + - name: init US + description: setup podinfo application and GSLB on US cluster + cluster: us + use: + template: ../../step-templates/init.yaml + - name: DNS resolves to EU pre failover + description: verify that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml + - name: failover + description: stop app on EU cluster + cluster: eu + use: + template: ../../step-templates/stop-podinfo.yaml + - name: DNS resolves to US post failover + description: assert that coredns on both clusters resolves to US + cluster: us + use: + template: ../../step-templates/assert-dns.yaml + - name: failback + description: restore application on EU cluster + cluster: eu + use: + template: ../../step-templates/apply-podinfo.yaml + - name: DNS resolves to EU post failback + description: assert that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml diff --git a/chainsaw/tests/failover-playground-embedded/testdata/gslb.yaml b/chainsaw/tests/failover-playground-embedded/testdata/gslb.yaml new file mode 100644 index 0000000000..7ed745bcec --- /dev/null +++ b/chainsaw/tests/failover-playground-embedded/testdata/gslb.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-playground-embedded +spec: + ingress: + ingressClassName: nginx + rules: + - host: failover-playground-embedded.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo + port: + name: http + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/chainsaw/tests/failover-playground-istio/chainsaw-test.yaml b/chainsaw/tests/failover-playground-istio/chainsaw-test.yaml new file mode 100644 index 0000000000..ce01d6f802 --- /dev/null +++ b/chainsaw/tests/failover-playground-istio/chainsaw-test.yaml @@ -0,0 +1,42 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: failover-playground-istio +spec: + namespace: failover-playground-istio + steps: + - name: init EU + description: setup podinfo application and GSLB on EU cluster + cluster: eu + use: + template: ../../step-templates/init-istio.yaml + - name: init US + description: setup podinfo application and GSLB on US cluster + cluster: us + use: + template: ../../step-templates/init-istio.yaml + - name: DNS resolves to EU pre failover + description: verify that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml + - name: failover + description: stop app on EU cluster + cluster: eu + use: + template: ../../step-templates/stop-podinfo.yaml + - name: DNS resolves to US post failover + description: assert that coredns on both clusters resolves to US + cluster: us + use: + template: ../../step-templates/assert-dns.yaml + - name: failback + description: restore application on EU cluster + cluster: eu + use: + template: ../../step-templates/apply-podinfo.yaml + - name: DNS resolves to EU post failback + description: assert that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml diff --git a/chainsaw/tests/failover-playground-istio/testdata/gslb.yaml b/chainsaw/tests/failover-playground-istio/testdata/gslb.yaml new file mode 100644 index 0000000000..fb36c0fdbc --- /dev/null +++ b/chainsaw/tests/failover-playground-istio/testdata/gslb.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-playground-istio +spec: + resourceRef: + apiVersion: networking.istio.io/v1 + kind: VirtualService + matchLabels: + app: failover-playground-istio + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + name: failover-playground-istio + labels: + app: failover-playground-istio +spec: + gateways: + - istio-ingress/failover-playground-istio + hosts: + - failover-playground-istio.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1 +kind: Gateway +metadata: + name: failover-playground-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - failover-playground-istio.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/chainsaw/tests/failover-playground-referenced/chainsaw-test.yaml b/chainsaw/tests/failover-playground-referenced/chainsaw-test.yaml new file mode 100644 index 0000000000..823adf6676 --- /dev/null +++ b/chainsaw/tests/failover-playground-referenced/chainsaw-test.yaml @@ -0,0 +1,52 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: failover-playground-referenced +spec: + namespace: failover-playground-referenced + steps: + - name: namespace EU + description: create namespace on EU cluster + cluster: eu + use: + template: ../../step-templates/namespace.yaml + - name: namespace US + description: create namespace on US cluster + cluster: us + use: + template: ../../step-templates/namespace.yaml + - name: init EU + description: setup podinfo application and GSLB on EU cluster + cluster: eu + use: + template: ../../step-templates/init.yaml + - name: init US + description: setup podinfo application and GSLB on US cluster + cluster: us + use: + template: ../../step-templates/init.yaml + - name: DNS resolves to EU pre failover + description: verify that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml + - name: failover + description: stop app on EU cluster + cluster: eu + use: + template: ../../step-templates/stop-podinfo.yaml + - name: DNS resolves to US post failover + description: assert that coredns on both clusters resolves to US + cluster: us + use: + template: ../../step-templates/assert-dns.yaml + - name: failback + description: restore application on EU cluster + cluster: eu + use: + template: ../../step-templates/apply-podinfo.yaml + - name: DNS resolves to EU post failback + description: assert that coredns on both clusters resolves to EU + cluster: eu + use: + template: ../../step-templates/assert-dns.yaml diff --git a/chainsaw/tests/failover-playground-referenced/testdata/gslb.yaml b/chainsaw/tests/failover-playground-referenced/testdata/gslb.yaml new file mode 100644 index 0000000000..ec41e2444e --- /dev/null +++ b/chainsaw/tests/failover-playground-referenced/testdata/gslb.yaml @@ -0,0 +1,35 @@ +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-playground-referenced +spec: + resourceRef: + apiVersion: networking.k8s.io/v1 + kind: Ingress + matchLabels: + app: failover-playground-referenced + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: failover-playground-referenced + labels: + app: failover-playground-referenced +spec: + ingressClassName: nginx + rules: + - host: failover-playground-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo + port: + name: http diff --git a/chainsaw/values.yaml b/chainsaw/values.yaml new file mode 100644 index 0000000000..269eaaf25c --- /dev/null +++ b/chainsaw/values.yaml @@ -0,0 +1,2 @@ +dns_port_eu: "5053" +dns_port_us: "5054"