Skip to content

Commit 66a822f

Browse files
kitsuyaazumakitsuyaazuma
authored
Add memory maximum limit feature (#8)
* feat: set memory maxLimit * fix: do not include secret yaml * docs: update cofiguration * fix: comment * fix: review
1 parent fb30699 commit 66a822f

File tree

14 files changed

+343
-62
lines changed

14 files changed

+343
-62
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,6 @@ go.work
2626
*.swp
2727
*.swo
2828
*~
29+
30+
# sample config files
31+
config/samples/secret.yaml

‎README.md‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ spec:
4646
adjustment:
4747
type: Mul
4848
value: "2"
49-
restartPolicy: "OnOOM"
49+
maxLimit: 250Mi
50+
restartPolicy: OnSpecChanged
5051
slackWebhook:
5152
secret:
5253
namespace: default
@@ -55,13 +56,24 @@ spec:
5556
channel: "#alert"
5657
```
5758
58-
* **target** (optional): Specifies the target CronJobs to monitor for OOM events. Users can specify the target CronJobs using a combination of `name`, `namespace`, and `labels`. If not specified, the controller will monitor all CronJobs in the cluster.
59+
* `target` <small>(optional)</small> : Target CronJob conditions. If not specified, the controller will target all CronJobs in the cluster.
60+
* `namespace` <small>(optional)</small> : Namespace of the target CronJob.
61+
* `name` <small>(optional)</small> : Name of the target CronJob.
62+
* `labels` <small>(optional)</small> : Labels to filter the target CronJobs.
5963

60-
* **adjustment** (required): Specifies the method and value for adjusting memory limits. Users can choose between `Add` and `Mul` methods for increasing memory limits, along with the corresponding value.
64+
* `adjustment` <small>(<font color="red">required</font>)</small> : Memory limit adjustment parameters.
65+
* `type` <small>(<font color="red">required</font>)</small> : Adjustment method. Choose between `Add` and `Mul`.
66+
* `value` <small>(<font color="red">required</font>)</small> : Adjustment value. For `Add`, it is the value to add to the current limit (e.g., `100Mi`). For `Mul`, it is the multiplier to apply to the current limit (e.g., `"2"`, must be double-quoted).
67+
* `maxLimit` <small>(optional)</small> : Maximum memory limit to set.
6168

62-
* **restartPolicy** (required): Specifies the policy for restarting failed Jobs. Users can choose between `Never` and `OnOOM` policies.
69+
* `restartPolicy` <small>(<font color="red">required</font>)</small> : Policy for restarting failed Jobs. Choose among `Never` (no restart), `OnOOM` (restart when OOM CronJob is detected), and `OnSpecChanged` (restart only when OOM CronJob is detected and the memory limit is updated).
6370

64-
* **slackWebhook** (required): Specifies the Slack webhook integration for sending notifications. Users can provide the webhook URL using a Kubernetes Secret and specify the target channel (optional) for notifications.
71+
* `slackWebhook` <small>(<font color="red">required</font>)</small> : Slack notification integration.
72+
* `secret` <small>(<font color="red">required</font>)</small> : Kubernetes Secret containing the Slack webhook URL.
73+
* `namespace` <small>(<font color="red">required</font>)</small> : Namespace of the Secret.
74+
* `name` <small>(<font color="red">required</font>)</small> : Name of the Secret.
75+
* `key` <small>(<font color="red">required</font>)</small> : Key containing the webhook URL.
76+
* `channel` <small>(optional)</small> : Slack channel to send notifications. If not specified, the notification will be sent to the default channel.
6577

6678

6779
## Getting Started

‎api/v1alpha1/broom_types.go‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,16 @@ const (
4141
)
4242

4343
type BroomAdjustment struct {
44-
Type BroomAdjustmentType `json:"type"`
45-
Value string `json:"value"`
44+
// Adjustment type. `Add` or `Mul`.
45+
Type BroomAdjustmentType `json:"type"`
46+
// Adjustment value. For `Add` type, it is the value to be added to the current memory. For `Mul` type, it is the value to be multiplied with the current memory.
47+
Value string `json:"value"`
48+
// Maximum limit for the memory. If the memory after adjustment is greater than this value, the memory is set to this value.
49+
MaxLimit resource.Quantity `json:"maxLimit,omitempty"`
4650
}
4751

48-
func (adj BroomAdjustment) IncreaseMemory(m *resource.Quantity) error {
52+
// AdjustMemory adjusts the memory based on the adjustment type, value and maxLimit.
53+
func (adj BroomAdjustment) AdjustMemory(m *resource.Quantity) error {
4954
switch adj.Type {
5055
case AddAdjustment:
5156
y, err := resource.ParseQuantity(adj.Value)
@@ -60,14 +65,22 @@ func (adj BroomAdjustment) IncreaseMemory(m *resource.Quantity) error {
6065
}
6166
m.Mul(int64(y))
6267
}
68+
69+
if adj.MaxLimit.IsZero() { // maxLimit is not set
70+
return nil
71+
}
72+
if m.Cmp(adj.MaxLimit) == 1 { // m is greater than maxLimit
73+
*m = adj.MaxLimit
74+
}
6375
return nil
6476
}
6577

6678
type BroomRestartPolicy string
6779

6880
const (
69-
RestartOnOOMPolicy BroomRestartPolicy = "OnOOM"
70-
RestartNeverPolicy BroomRestartPolicy = "Never"
81+
RestartOnOOMPolicy BroomRestartPolicy = "OnOOM"
82+
RestartOnSpecChangedPolicy BroomRestartPolicy = "OnSpecChanged"
83+
RestartNeverPolicy BroomRestartPolicy = "Never"
7184
)
7285

7386
type BroomSlackWebhookSecret struct {

‎api/v1alpha1/broom_types_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package v1alpha1
2+
3+
import (
4+
"testing"
5+
6+
"github.com/stretchr/testify/assert"
7+
"k8s.io/apimachinery/pkg/api/resource"
8+
)
9+
10+
func TestAdjustMemory(t *testing.T) {
11+
tests := map[string]struct {
12+
beforeMemoryLimit string
13+
adjustment BroomAdjustment
14+
afterMemoryLimit string
15+
}{
16+
"add adjustment": {
17+
beforeMemoryLimit: "100Mi",
18+
adjustment: BroomAdjustment{Type: AddAdjustment, Value: "100Mi"},
19+
afterMemoryLimit: "200Mi",
20+
},
21+
"mul adjustment": {
22+
beforeMemoryLimit: "100Mi",
23+
adjustment: BroomAdjustment{Type: MulAdjustment, Value: "2"},
24+
afterMemoryLimit: "200Mi",
25+
},
26+
"max limit reached with add adjustment": {
27+
beforeMemoryLimit: "100Mi",
28+
adjustment: BroomAdjustment{Type: AddAdjustment, Value: "100Mi", MaxLimit: resource.MustParse("150Mi")},
29+
afterMemoryLimit: "150Mi",
30+
},
31+
"max limit reached with mul adjustment": {
32+
beforeMemoryLimit: "100Mi",
33+
adjustment: BroomAdjustment{Type: MulAdjustment, Value: "2", MaxLimit: resource.MustParse("150Mi")},
34+
afterMemoryLimit: "150Mi",
35+
},
36+
"already at max limit": {
37+
beforeMemoryLimit: "100Mi",
38+
adjustment: BroomAdjustment{Type: AddAdjustment, Value: "100Mi", MaxLimit: resource.MustParse("100Mi")},
39+
afterMemoryLimit: "100Mi",
40+
},
41+
}
42+
43+
for name, test := range tests {
44+
test := test
45+
t.Run(name, func(t *testing.T) {
46+
t.Parallel()
47+
m := resource.MustParse(test.beforeMemoryLimit)
48+
adj := test.adjustment
49+
err := adj.AdjustMemory(&m)
50+
assert.Nil(t, err)
51+
assert.True(t, m.Equal(resource.MustParse(test.afterMemoryLimit)))
52+
})
53+
}
54+
}

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/ai.m3.com_brooms.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,22 @@ spec:
4141
properties:
4242
adjustment:
4343
properties:
44+
maxLimit:
45+
anyOf:
46+
- type: integer
47+
- type: string
48+
description: Maximum limit for the memory. If the memory after
49+
adjustment is greater than this value, the memory is set to
50+
this value.
51+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
52+
x-kubernetes-int-or-string: true
4453
type:
54+
description: Adjustment type. `Add` or `Mul`.
4555
type: string
4656
value:
57+
description: Adjustment value. For `Add` type, it is the value
58+
to be added to the current memory. For `Mul` type, it is the
59+
value to be multiplied with the current memory.
4760
type: string
4861
required:
4962
- type

‎config/samples/ai_v1alpha1_broom.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ spec:
1717
adjustment:
1818
type: Mul
1919
value: "2"
20-
restartPolicy: "OnOOM"
20+
maxLimit: "250Mi"
21+
restartPolicy: "OnSpecChanged"
2122
slackWebhook:
2223
secret:
2324
namespace: default

‎config/samples/cronjob.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ metadata:
55
labels:
66
m3.com/use-broom: "true"
77
spec:
8-
schedule: "*/2 * * * *"
8+
schedule: "*/1 * * * *"
99
jobTemplate:
1010
spec:
1111
activeDeadlineSeconds: 60

config/samples/secret.yaml renamed to config/samples/secret.yaml.tmpl

File renamed without changes.

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ require (
66
github.com/onsi/ginkgo/v2 v2.14.0
77
github.com/onsi/gomega v1.30.0
88
github.com/slack-go/slack v0.12.5
9+
github.com/stretchr/testify v1.9.0
910
k8s.io/api v0.29.0
1011
k8s.io/apimachinery v0.29.0
1112
k8s.io/client-go v0.29.0
@@ -43,6 +44,7 @@ require (
4344
github.com/modern-go/reflect2 v1.0.2 // indirect
4445
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
4546
github.com/pkg/errors v0.9.1 // indirect
47+
github.com/pmezard/go-difflib v1.0.0 // indirect
4648
github.com/prometheus/client_golang v1.18.0 // indirect
4749
github.com/prometheus/client_model v0.5.0 // indirect
4850
github.com/prometheus/common v0.45.0 // indirect

0 commit comments

Comments
 (0)