Skip to content

Commit 6f6cee7

Browse files
ejsmithejsmith
committed
Add CPU based autoscaling, add signoz collector config
1 parent 08c97ea commit 6f6cee7

File tree

7 files changed

+129
-12
lines changed

7 files changed

+129
-12
lines changed

‎k8s/ex-prod-elasticsearch.yaml

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@ metadata:
66
spec:
77
version: 8.15.2
88
image: exceptionless/elasticsearch:8.15.2 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch
9-
monitoring:
10-
metrics:
11-
elasticsearchRefs:
12-
- name: elastic-monitor
13-
namespace: elastic-system
14-
logs:
15-
elasticsearchRefs:
16-
- name: elastic-monitor
17-
namespace: elastic-system
9+
# monitoring:
10+
# metrics:
11+
# elasticsearchRefs:
12+
# - name: elastic-monitor
13+
# namespace: elastic-system
14+
# logs:
15+
# elasticsearchRefs:
16+
# - name: elastic-monitor
17+
# namespace: elastic-system
1818
secureSettings:
1919
- secretName: ex-prod-snapshots
2020
http:

‎k8s/ex-prod-values.yaml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,17 @@ appMode: Production
22
appScope: prod
33
app:
44
replicaCount: 3
5+
minReplicaCount: 2
6+
maxReplicaCount: 5
57
defaultDomain: be.exceptionless.io
68
domains:
79
- be.exceptionless.io
810
- app.exceptionless.io
911
- app.exceptionless.com
1012
api:
1113
replicaCount: 5
14+
minReplicaCount: 2
15+
maxReplicaCount: 5
1216
defaultDomain: collector.exceptionless.io
1317
domains:
1418
- collector.exceptionless.io
@@ -21,7 +25,9 @@ jobs:
2125
eventNotifications:
2226
replicaCount: 2
2327
eventPosts:
24-
replicaCount: 10
28+
replicaCount: 2
29+
minReplicaCount: 2
30+
maxReplicaCount: 10
2531
eventUserDescriptions:
2632
replicaCount: 2
2733
mailMessage:
@@ -30,6 +36,8 @@ jobs:
3036
replicaCount: 2
3137
workitem:
3238
replicaCount: 4
39+
minReplicaCount: 2
40+
maxReplicaCount: 5
3341

3442
config:
3543
EX_EnableSnapshotJobs: "true"

‎k8s/ex-setup.ps1

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,10 @@ helm install ex-$ENV-redis bitnami/redis --values ex-$ENV-redis-values.yaml --na
147147
# upgrade redis server
148148
helm upgrade ex-$ENV-redis bitnami/redis --reset-values --values ex-$ENV-redis-values.yaml --namespace ex-$ENV
149149

150+
# install signoz otel collector
151+
helm repo add signoz https://charts.signoz.io
152+
helm install signoz-collector signoz/k8s-infra -f signoz.yaml --set "signozApiKey=$SIGNOZ_KEY"
153+
150154
# install exceptionless app
151155
$VERSION="8.0.0"
152156
helm install ex-$ENV .\exceptionless --namespace ex-$ENV --values ex-$ENV-values.yaml `

‎k8s/exceptionless/templates/api.yaml

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ metadata:
1010
release: {{ .Release.Name }}
1111
heritage: {{ .Release.Service }}
1212
spec:
13-
replicas: {{ .Values.api.replicaCount }}
13+
replicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }}{{ .Values.api.replicaCount }}{{ else }}{{ 2 }}{{ end }}
1414
selector:
1515
matchLabels:
1616
component: {{ template "exceptionless.fullname" . }}-api
@@ -90,6 +90,28 @@ spec:
9090
claimName: {{ template "exceptionless.fullname" . }}-storage
9191
{{- end }}
9292

93+
{{- if not (kindIs "invalid" ( .Values.api | default dict ).replicaCount) }}
94+
---
95+
apiVersion: autoscaling/v2
96+
kind: HorizontalPodAutoscaler
97+
metadata:
98+
name: {{ template "exceptionless.fullname" . }}-api-scaler
99+
spec:
100+
scaleTargetRef:
101+
apiVersion: apps/v1
102+
kind: Deployment
103+
name: {{ template "exceptionless.fullname" . }}-api
104+
minReplicas: {{ if not (kindIs "invalid" ( .Values.api | default dict ).minReplicaCount) }}{{ .Values.api.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
105+
maxReplicas: {{ .Values.api.maxReplicaCount }}
106+
metrics:
107+
- type: Resource
108+
resource:
109+
name: cpu
110+
target:
111+
type: Utilization
112+
averageUtilization: 50
113+
{{- end }}
114+
93115
---
94116
apiVersion: v1
95117
kind: Service

‎k8s/exceptionless/templates/app.yaml

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ metadata:
1010
release: {{ .Release.Name }}
1111
heritage: {{ .Release.Service }}
1212
spec:
13-
replicas: {{ .Values.app.replicaCount }}
13+
replicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }}{{ .Values.app.replicaCount }}{{ else }}{{ 2 }}{{ end }}
1414
selector:
1515
matchLabels:
1616
component: {{ template "exceptionless.fullname" . }}-app
@@ -88,6 +88,28 @@ spec:
8888
claimName: {{ template "exceptionless.fullname" . }}-storage
8989
{{- end }}
9090

91+
{{- if not (kindIs "invalid" ( .Values.app | default dict ).replicaCount) }}
92+
---
93+
apiVersion: autoscaling/v2
94+
kind: HorizontalPodAutoscaler
95+
metadata:
96+
name: {{ template "exceptionless.fullname" . }}-app-scaler
97+
spec:
98+
scaleTargetRef:
99+
apiVersion: apps/v1
100+
kind: Deployment
101+
name: {{ template "exceptionless.fullname" . }}-app
102+
minReplicas: {{ if not (kindIs "invalid" ( .Values.app | default dict ).minReplicaCount) }}{{ .Values.app.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
103+
maxReplicas: {{ .Values.app.maxReplicaCount }}
104+
metrics:
105+
- type: Resource
106+
resource:
107+
name: cpu
108+
target:
109+
type: Utilization
110+
averageUtilization: 50
111+
{{- end }}
112+
91113
---
92114
apiVersion: v1
93115
kind: Service

‎k8s/exceptionless/templates/jobs.yaml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,28 @@ spec:
343343
- name: EX_OTEL_RESOURCE_ATTRIBUTES
344344
value: k8s.pod.ip=$(K8S_POD_IP),k8s.pod.uid=$(K8S_POD_UID)
345345

346+
{{- if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).maxReplicaCount) }}
347+
---
348+
apiVersion: autoscaling/v2
349+
kind: HorizontalPodAutoscaler
350+
metadata:
351+
name: {{ template "exceptionless.fullname" . }}-jobs-event-posts-scaler
352+
spec:
353+
scaleTargetRef:
354+
apiVersion: apps/v1
355+
kind: Deployment
356+
name: {{ template "exceptionless.fullname" . }}-jobs-event-posts
357+
minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.eventPosts | default dict ).minReplicaCount) }}{{ .Values.jobs.eventPosts.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
358+
maxReplicas: {{ .Values.jobs.eventPosts.maxReplicaCount }}
359+
metrics:
360+
- type: Resource
361+
resource:
362+
name: cpu
363+
target:
364+
type: Utilization
365+
averageUtilization: 50
366+
{{- end }}
367+
346368
---
347369
apiVersion: batch/v1
348370
kind: CronJob
@@ -945,6 +967,28 @@ spec:
945967
claimName: {{ template "exceptionless.fullname" . }}-storage
946968
{{- end }}
947969

970+
{{- if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).maxReplicaCount) }}
971+
---
972+
apiVersion: autoscaling/v2
973+
kind: HorizontalPodAutoscaler
974+
metadata:
975+
name: {{ template "exceptionless.fullname" . }}-jobs-work-item-scaler
976+
spec:
977+
scaleTargetRef:
978+
apiVersion: apps/v1
979+
kind: Deployment
980+
name: {{ template "exceptionless.fullname" . }}-jobs-work-item
981+
minReplicas: {{ if not (kindIs "invalid" ( .Values.jobs.workitem | default dict ).minReplicaCount) }}{{ .Values.jobs.workitem.minReplicaCount }}{{ else }}{{ 2 }}{{ end }}
982+
maxReplicas: {{ .Values.jobs.workitem.maxReplicaCount }}
983+
metrics:
984+
- type: Resource
985+
resource:
986+
name: cpu
987+
target:
988+
type: Utilization
989+
averageUtilization: 50
990+
{{- end }}
991+
948992
---
949993
apiVersion: apps/v1
950994
kind: Deployment

‎k8s/signoz.yaml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
global:
2+
cloud: aks
3+
clusterName: ex-k8s-v6
4+
deploymentEnvironment: prod
5+
otelCollectorEndpoint: ingest.us.signoz.cloud:443
6+
otelInsecure: false
7+
presets:
8+
otlpExporter:
9+
enabled: true
10+
loggingExporter:
11+
enabled: false
12+
logsCollection:
13+
enabled: false
14+
resourceDetection:
15+
detectors:
16+
- azure
17+
- system

0 commit comments

Comments
 (0)