force purging when all nodes are unhealthy (#21)

* added purge and rbac role for pod deletions #17

* bump image version #17

* added new rbac role to chart templates #17

* bump image versions #17

Makefile

@@ -52,7 +52,7 @@ OPERATOR_SDK_VERSION ?= v1.38.0
 # Image URL to use all building/pushing image targets
 DOCKER_HUB_NAME ?= $(shell docker info | sed '/Username:/!d;s/.* //')
 IMG_NAME ?= typesense-operator
-IMG_TAG ?= 0.2.5
+IMG_TAG ?= 0.2.6
 IMG ?= $(DOCKER_HUB_NAME)/$(IMG_NAME):$(IMG_TAG)
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.

charts/typesense-operator/Chart.yaml

@@ -13,9 +13,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.5
+version: 0.2.6
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.2.5"
+appVersion: "0.2.6"

charts/typesense-operator/templates/manager-rbac.yaml

@@ -22,6 +22,7 @@ rules:
   resources:
   - pods
   verbs:
+  - delete
   - get
   - list
   - watch

charts/typesense-operator/values.yaml

@@ -12,7 +12,7 @@ controllerManager:
         - ALL
     image:
       repository: akyriako78/typesense-operator
-      tag: 0.2.5
+      tag: 0.2.6
     resources:
       limits:
         cpu: 500m

config/rbac/role.yaml

@@ -21,6 +21,7 @@ rules:
   resources:
   - pods
   verbs:
+  - delete
   - get
   - list
   - watch

internal/controller/typesensecluster_controller.go

@@ -74,7 +74,7 @@ var (
 // +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete
-// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;delete
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch

internal/controller/typesensecluster_quorum.go

@@ -113,6 +113,14 @@ func (r *TypesenseClusterReconciler) getQuorumHealth(ctx context.Context, ts *ts
 			return ConditionReasonQuorumDowngraded, size, nil
 		}
 
+		if healthyNodes == 0 && minRequiredNodes == 1 {
+			r.logger.Info("purging quorum")
+			err := r.PurgeStatefulSetPods(ctx, sts)
+			if err != nil {
+				return ConditionReasonQuorumNotReady, 0, err
+			}
+		}
+
 		return ConditionReasonQuorumNotReady, healthyNodes, fmt.Errorf("quorum has %d healthy nodes, minimum required %d", healthyNodes, minRequiredNodes)
 	} else {
 		if sts.Status.ReadyReplicas < ts.Spec.Replicas {

internal/controller/typesensecluster_statefulset.go

@@ -9,6 +9,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -231,3 +232,26 @@ func (r *TypesenseClusterReconciler) ScaleStatefulSet(ctx context.Context, sts *
 
 	return nil
 }
+
+func (r *TypesenseClusterReconciler) PurgeStatefulSetPods(ctx context.Context, sts *appsv1.StatefulSet) error {
+	labelSelector := labels.SelectorFromSet(sts.Spec.Selector.MatchLabels)
+
+	var pods corev1.PodList
+	if err := r.List(ctx, &pods, &client.ListOptions{
+		Namespace:     sts.Namespace,
+		LabelSelector: labelSelector,
+	}); err != nil {
+		r.logger.Error(err, "failed to list pods", "statefulset", sts.Name)
+		return err
+	}
+
+	for _, pod := range pods.Items {
+		err := r.Delete(ctx, &pod)
+		if err != nil {
+			r.logger.Error(err, "failed to delete pod", "pod", pod.Name)
+			return err
+		}
+	}
+
+	return nil
+}