Introduce chainsaw as e2e testing framework

.github/actions/print-chainsaw-debug/action.yaml

@@ -0,0 +1,79 @@
+name: Print Debug
+description: 'Prints debug info from running clusters'
+
+inputs:
+  kubectl-context:
+    description: 'This will be passed to kubectl commands as --context argument'
+    required: false
+    default: 'k3d-test-gslb1'
+runs:
+  using: "composite"
+  steps:
+    - name: Check verbose
+      shell: bash
+      run: |
+        if [[ "x${{ runner.debug }}" == "x1" ]]; then
+          echo "verbose=true" >> $GITHUB_ENV
+        else
+          echo "verbose=false" >> $GITHUB_ENV
+        fi
+
+    - name: Env vars
+      shell: bash
+      run: |
+        echo "::group:: ☸☸☸ envs"
+        env | sort
+        echo ---------------------------------------
+        echo inputs.kubectl-context=${{ inputs.kubectl-context }}
+        echo verbose: ${{ env.verbose }}
+        echo "::endgroup::"
+
+    - name: CoreDNS Hosts
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo -e "\n\n\nVerbose is on, printing all the debug stuff:\n--------------------------------------------\n\n"
+        echo "::group:: ☸☸☸ cluster coredns - hosts (cluster 1)"
+        kubectl --context=${{ inputs.kubectl-context }} get cm coredns -n kube-system -o yaml | grep -A7 NodeHosts:
+        echo "::endgroup::"
+
+    - name: K8s Events
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo "::group:: ☸☸☸ k get events"
+        kubectl --context=${{ inputs.kubectl-context }} get events -A || true
+        echo "::endgroup::"
+
+    - name: gslbs
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo "::group:: ☸☸☸ gslbs"
+        kubectl --context=${{ inputs.kubectl-context }} get gslbs -owide -A || true
+        echo "::endgroup::"
+
+    - name: Dnsendpoints
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo "::group:: ☸☸☸ endpoints"
+        kubectl --context=${{ inputs.kubectl-context }} get dnsendpoints -A || true
+        echo "::endgroup::"
+
+    - name: K8gb logs
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo "::group:: ☸☸☸ k8gb logs"
+        kubectl --context=${{ inputs.kubectl-context }} logs -lname=k8gb -n k8gb --tail=-1 || true
+        echo "::endgroup::"
+
+    - name: Metrics
+      shell: bash
+      if: env.verbose == 'true'
+      run: |
+        echo "::group:: ☸☸☸ k8gb metrics (cluster 1)"
+        _IP=$(kubectl --context=${{ inputs.kubectl-context }} get pods -lname=k8gb -n k8gb -o custom-columns='IP:status.podIP' --no-headers)
+        kubectl --context=${{ inputs.kubectl-context }} run -it --rm curl-metrics --restart=Never --image=curlimages/curl:7.82.0 -- $_IP:8080/metrics
+        echo "::endgroup::"

.github/pull_request_template.md

@@ -10,6 +10,6 @@
   Add the [`heavy-tests`](/k8gb-io/k8gb/issues?q=is%3A*+label%3Aheavy-tests) label on this PR if you want full-blown tests that include more than 2-cluster scenarios.
 
   ### Debug tests
-  If the test suite is failing for you, you may want to try triggering `Re-run all jobs` (top right) with [debug logging](https://docs.github.com/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging) enabled. It will also make the [print debug](/k8gb-io/k8gb/blob/master/.github/actions/print-debug/action.yaml) action more verbose.
+  If the test suite is failing for you, you may want to try triggering `Re-run all jobs` (top right) with [debug logging](https://docs.github.com/actions/monitoring-and-troubleshooting-workflows/enabling-debug-logging) enabled. It will also make the [print debug](/k8gb-io/k8gb/blob/master/.github/actions/print-terratest-debug/action.yaml) action more verbose.
 
 </details>

.github/workflows/chainsaw.yaml

@@ -0,0 +1,110 @@
+name: Chainsaw
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+      - '**.svg'
+      - '**.drawio'
+      - '.spelling'
+  pull_request:
+    branches:
+      - master
+    # The specific activity types are listed here to include "labeled" and "unlabeled"
+    # (which are not included by default for the "pull_request" trigger).
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
+    paths-ignore:
+      - '**.md'
+      - '**.svg'
+      - '**.drawio'
+      - '.spelling'
+permissions:
+  contents: read
+
+jobs:
+  skip-check:
+    runs-on: ubuntu-latest
+    name: Skip the job?
+    outputs:
+          should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    steps:
+    - name: Harden Runner
+      uses: step-security/harden-runner@91182cccc01eb5e619899d80e4e971d6181294a7 # v2.10.1
+      with:
+        egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs
+
+    - id: skip_check
+      uses: fkirc/skip-duplicate-actions@f75f66ce1886f00957d99748a42c724f4330bdcf # v5.3.1
+      with:
+        skip_after_successful_duplicate: 'true'
+        do_not_skip: '["workflow_dispatch", "schedule"]'
+
+  chainsaw:
+    runs-on: ubuntu-24.04
+    needs: skip-check
+    if: ${{ needs.skip-check.outputs.should_skip != 'true' }} && !contains( github.event.pull_request.labels.*.name, 'renovate')
+    steps:
+      - name: Checkout
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
+        with:
+          fetch-depth: 0
+
+      - name: Setup Chainsaw
+        uses: kyverno/action-install-chainsaw@d311eacde764f806c9658574ff64c9c3b21f8397 # v0.2.11
+        with:
+          release: v0.2.11
+
+      - name: Check install
+        run: chainsaw version
+
+      - name: Setup golang
+        uses: actions/setup-go@bfd2fb341f32be7281829126376a12a780ca79fc
+        with:
+          go-version: 1.22.3
+
+      - name: Build artifacts
+        uses: goreleaser/goreleaser-action@286f3b13b1b49da4ac219696163fb8c1c93e1200 # v6.0.0
+        with:
+          version: v1.9.2
+          args: release --rm-dist --skip-publish --skip-validate --snapshot --skip-sbom --skip-sign
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create edgeDNS k3s Cluster
+        uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79
+        with:
+          cluster-name: "edgedns"
+          args: -c k3d/edge-dns.yaml
+
+      - name: Create 1st k3s Cluster
+        uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79
+        with:
+          cluster-name: "test-gslb1"
+          args: -c k3d/test-gslb1.yaml
+
+      - name: Create 2nd k3s Cluster
+        uses: AbsaOSS/k3d-action@4e8b3239042be1dc0aed6c5eb80c13b18200fc79
+        with:
+          cluster-name: "test-gslb2"
+          args: -c k3d/test-gslb2.yaml
+
+      - name: K8GB deployment
+        run: |
+          make deploy-test-version list-running-pods
+          echo "Cluster 1 (eu):"
+          kubectl get no -owide --context=k3d-test-gslb1
+          echo "Cluster 2 (us):"
+          kubectl get no -owide --context=k3d-test-gslb2
+
+      - name: Run Chainsaw
+        run: make chainsaw
+
+      - name: Print debug info
+        if: always()
+        uses: ./.github/actions/print-chainsaw-debug
+
+      - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        if: always()
+        with:
+          name: chainsaw-logs
+          path: ${{ github.workspace }}/tmp/chainsaw

.github/workflows/terratest-istiov1beta1.yaml

@@ -71,7 +71,7 @@ jobs:
 
       - name: Print debug info
         if: always()
-        uses: ./.github/actions/print-debug
+        uses: ./.github/actions/print-terratest-debug
 
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()

.github/workflows/terratest-more-clusters.yaml

@@ -73,7 +73,7 @@ jobs:
 
       - name: Print debug info
         if: always()
-        uses: ./.github/actions/print-debug
+        uses: ./.github/actions/print-terratest-debug
 
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()

.github/workflows/terratest.yaml

@@ -98,7 +98,7 @@ jobs:
 
       - name: Print debug info
         if: always()
-        uses: ./.github/actions/print-debug
+        uses: ./.github/actions/print-terratest-debug
 
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()

.github/workflows/upgrade-testing.yaml

@@ -93,7 +93,7 @@ jobs:
 
       - name: Print debug info
         if: always()
-        uses: ./.github/actions/print-debug
+        uses: ./.github/actions/print-terratest-debug
 
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()

Makefile

@@ -480,6 +480,20 @@ terratest: # Run terratest suite
 	fi
 	cd terratest/test/ && go mod download && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) go test -v -timeout 25m -parallel=12 --tags=$(TEST_TAGS)
 
+# executes chainsaw e2e tests
+.PHONY: chainsaw
+chainsaw:
+	mkdir -p chainsaw/kubeconfig
+	k3d kubeconfig get test-gslb1 > chainsaw/kubeconfig/eu.config
+	k3d kubeconfig get test-gslb2 > chainsaw/kubeconfig/us.config
+	@$(eval RUNNING_CLUSTERS := $(shell k3d cluster list --no-headers | grep $(CLUSTER_NAME) -c))
+	@if [ "$(RUNNING_CLUSTERS)" -lt 2 ] ; then \
+		echo -e "$(RED)Make sure you run the tests against at least two running clusters$(NC)" ;\
+		exit 1;\
+	fi
+	cd chainsaw && CLUSTERS_NUMBER=$(RUNNING_CLUSTERS) chainsaw test --config ./config.yaml --values ./values.yaml
+	rm -r chainsaw/kubeconfig
+
 .PHONY: website
 website:
 	@if [ "$(CI)" = "true" ]; then\

chainsaw/config.yaml

@@ -0,0 +1,12 @@
+apiVersion: chainsaw.kyverno.io/v1alpha2
+kind: Configuration
+metadata:
+  name: custom-config
+spec:
+  clusters:
+    eu:
+      kubeconfig: kubeconfig/eu.config
+      context: k3d-test-gslb1
+    us:
+      kubeconfig: kubeconfig/us.config
+      context: k3d-test-gslb2

chainsaw/step-templates/apply-podinfo.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: apply-podinfo
+spec:
+  try:
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - wait:
+      apiVersion: v1
+      kind: Pod
+      name: frontend-podinfo
+      for:
+        condition:
+          name: Ready
+          value: 'true'

chainsaw/step-templates/assert-dns.yaml

@@ -0,0 +1,29 @@
+---
+# asserts that both clusters resolve the domain to the correct cluster
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: assert-dns
+spec:
+  try:
+  - sleep: 
+      duration: 30s # reconcile requeue seconds + coreDNS
+  - script:
+      env:
+      - name: NAME
+        value: ($test.metadata.name)
+      - name: DNS_PORT_EU
+        value: ($values.dns_port_eu)
+      - name: DNS_PORT_US
+        value: ($values.dns_port_us)
+      content: |
+        EXPECTED=$(kubectl get dnsendpoint $NAME -n $NAMESPACE -o json | jq -r '.spec.endpoints[] | select(.dnsName | contains("localtargets")).targets | join(",")')
+        ACTUAL_EU=$(dig @localhost -p $DNS_PORT_EU "$NAME.cloud.example.com" +short | paste -sd, -)
+
+        EXPECTED_SORTED=$(echo "$EXPECTED" | tr ',' '\n' | sort | tr '\n' ',')
+        ACTUAL_EU_SORTED=$(echo "$ACTUAL_EU" | tr ',' '\n' | sort | tr '\n' ',')
+        [ "$EXPECTED_SORTED" = "$ACTUAL_EU_SORTED" ] || { echo "EU expected targets '$EXPECTED_SORTED' but found '$ACTUAL_EU_SORTED'"; exit 1; }
+
+        ACTUAL_US=$(dig @localhost -p $DNS_PORT_US "$NAME.cloud.example.com" +short | paste -sd, -)
+        ACTUAL_US_SORTED=$(echo "$ACTUAL_US" | tr ',' '\n' | sort | tr '\n' ',')
+        [ "$EXPECTED_SORTED" = "$ACTUAL_US_SORTED" ] || { echo "US expected targets '$EXPECTED_SORTED' but found '$ACTUAL_US_SORTED'"; exit 1; }

chainsaw/step-templates/init-istio.yaml

@@ -0,0 +1,28 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: init-ingress
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace-istio.yaml
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - apply:
+      file: testdata/gslb.yaml
+  - assert:
+      resource:
+        apiVersion: externaldns.k8s.io/v1alpha1
+        kind: DNSEndpoint
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        spec: 
+          endpoints:
+            - targets:
+                (length(@)): 2
+            - targets:
+                (length(@)): 2
+      timeout: 15s

chainsaw/step-templates/init.yaml

@@ -0,0 +1,40 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: init-ingress
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace.yaml
+  - apply:
+      file: ../../testdata/podinfo.yaml
+  - apply:
+      file: ./testdata/gslb.yaml
+  - assert:
+      resource:
+        apiVersion: networking.k8s.io/v1
+        kind: Ingress
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        status: 
+          loadBalancer:
+            ingress:
+              (length(@)): 2
+      timeout: 60s
+  - assert:
+      resource:
+        apiVersion: externaldns.k8s.io/v1alpha1
+        kind: DNSEndpoint
+        metadata:
+          name: ($test.metadata.name)
+          namespace: ($namespace)
+        spec: 
+          endpoints:
+          - dnsName: (join('', ['localtargets-', $test.metadata.name, '.cloud.example.com']))
+            targets:
+              (length(@)): 2
+          - dnsName: (join('', [$test.metadata.name, '.cloud.example.com']))
+      timeout: 15s

chainsaw/step-templates/namespace.yaml

@@ -0,0 +1,10 @@
+---
+# Creates a namespace, a service, a pod and a gslb
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: StepTemplate
+metadata:
+  name: namespace
+spec:
+  try:
+  - apply:
+      file: ../../testdata/namespace.yaml