Added unready rule

README.md

@@ -164,6 +164,12 @@ Flags a pod for reaping based on the pods current run duration.
 
 Enabled and configured by setting the environment variable `MAX_DURATION` with a valid go-lang `time.duration` format (example: "1h15m30s"). If a pod has been running longer than the specified duration, the pod will be flagged for reaping.
 
+### Unready
+
+Flags a pod for reaping based on the time the pod has been unready.
+
+Enabled and configured by setting the environment variable `MAX_UNREADY` with a valid go-lang `time.duration` format (example: "10m"). If a pod has been unready longer than the specified duration, the pod will be flagged for reaping.
+
 ## Running Pod-Reapers
 
 ### Combining Rules

rules/rules.go

@@ -31,6 +31,7 @@ func LoadRules() (Rules, error) {
 		&chaos{},
 		&containerStatus{},
 		&duration{},
+		&unready{},
 		&podStatus{},
 	}
 	// return only the active rules

rules/unready.go

@@ -0,0 +1,53 @@
+package rules
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+const envMaxUnready = "MAX_UNREADY"
+
+var _ Rule = (*unready)(nil)
+
+type unready struct {
+	duration time.Duration
+}
+
+func (rule *unready) load() (bool, string, error) {
+	value, active := os.LookupEnv(envMaxUnready)
+	if !active {
+		return false, "", nil
+	}
+	duration, err := time.ParseDuration(value)
+	if err != nil {
+		return false, "", fmt.Errorf("invalid max unready duration: %s", err)
+	}
+	rule.duration = duration
+	return true, fmt.Sprintf("maximum unready %s", value), nil
+}
+
+func (rule *unready) ShouldReap(pod v1.Pod) (bool, string) {
+	condition := getCondition(pod, v1.PodReady)
+	if condition == nil || condition.Status == "True" {
+		return false, ""
+	}
+
+	startTime := time.Unix(condition.LastTransitionTime.Unix(), 0) // convert to standard go time
+	cutoffTime := time.Now().Add(-1 * rule.duration)
+	unreadyDuration := time.Now().Sub(startTime)
+	message := fmt.Sprintf("has been unready for %s", unreadyDuration.String())
+	return startTime.Before(cutoffTime), message
+}
+
+func getCondition(pod v1.Pod, conditionType v1.PodConditionType) *v1.PodCondition {
+	for _, condition := range pod.Status.Conditions {
+		if condition.Type == conditionType {
+			return &condition
+		}
+	}
+
+	return nil
+}

rules/unready_test.go

@@ -0,0 +1,86 @@
+package rules
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"k8s.io/client-go/pkg/api/unversioned"
+	"k8s.io/client-go/pkg/api/v1"
+)
+
+func testUnreadyPod(lastTransitionTime *time.Time) v1.Pod {
+	pod := v1.Pod{}
+	if lastTransitionTime != nil {
+		setTime := unversioned.NewTime(*lastTransitionTime)
+		pod.Status.Conditions = []v1.PodCondition{
+			v1.PodCondition{
+				Type:               v1.PodReady,
+				LastTransitionTime: setTime,
+				Reason:             "ContainersNotReady",
+				Status:             "False",
+			},
+		}
+	}
+	return pod
+}
+
+func TestUnreadyLoad(t *testing.T) {
+	t.Run("load", func(t *testing.T) {
+		os.Clearenv()
+		os.Setenv(envMaxUnready, "30m")
+		loaded, message, err := (&unready{}).load()
+		assert.NoError(t, err)
+		assert.Equal(t, "maximum unready 30m", message)
+		assert.True(t, loaded)
+	})
+	t.Run("invalid time", func(t *testing.T) {
+		os.Clearenv()
+		os.Setenv(envMaxUnready, "not-a-time")
+		loaded, message, err := (&unready{}).load()
+		assert.Error(t, err)
+		assert.Equal(t, "", message)
+		assert.False(t, loaded)
+	})
+	t.Run("no load", func(t *testing.T) {
+		os.Clearenv()
+		loaded, message, err := (&unready{}).load()
+		assert.NoError(t, err)
+		assert.Equal(t, "", message)
+		assert.False(t, loaded)
+	})
+}
+
+func TestUnreadyShouldReap(t *testing.T) {
+	t.Run("no ready time", func(t *testing.T) {
+		os.Clearenv()
+		os.Setenv(envMaxUnready, "10m")
+		unready := unready{}
+		unready.load()
+		pod := testUnreadyPod(nil)
+		shouldReap, _ := unready.ShouldReap(pod)
+		assert.False(t, shouldReap)
+	})
+	t.Run("reap", func(t *testing.T) {
+		os.Clearenv()
+		os.Setenv(envMaxUnready, "9m59s")
+		unready := unready{}
+		unready.load()
+		lastTransitionTime := time.Now().Add(-10 * time.Minute)
+		pod := testUnreadyPod(&lastTransitionTime)
+		shouldReap, reason := unready.ShouldReap(pod)
+		assert.True(t, shouldReap)
+		assert.Regexp(t, ".*has been unready.*", reason)
+	})
+	t.Run("no reap", func(t *testing.T) {
+		os.Clearenv()
+		os.Setenv(envMaxUnready, "10m1s")
+		unready := unready{}
+		unready.load()
+		lastTransitionTime := time.Now().Add(-10 * time.Minute)
+		pod := testUnreadyPod(&lastTransitionTime)
+		shouldReap, _ := unready.ShouldReap(pod)
+		assert.False(t, shouldReap)
+	})
+}