Skip to content

Commit 418ac5c

Browse files
ljupcovangelskiljupcovangelski
committed
Merge branch 'release/0.55.0' into main
2 parents a478a38 + 397e04b commit 418ac5c

File tree

137 files changed

+4455
-316
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

137 files changed

+4455
-316
lines changed

‎.github/workflows/main.yml

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,16 @@ jobs:
5454
run: |
5555
bazel test --test_tag_filters=-lint //...
5656
57+
- name: Cleanup space
58+
run: |
59+
df -h
60+
sudo apt-get autoremove -y
61+
sudo apt-get clean
62+
docker images prune -a
63+
sudo rm -rf /usr/local/share/powershell
64+
sudo rm -rf /opt/hostedtoolcache
65+
df -h
66+
5767
- name: Build all artifacts
5868
run: |
5969
bazel build //...
@@ -64,18 +74,14 @@ jobs:
6474
echo ${{ secrets.PAT }} | docker login ghcr.io -u airydevci --password-stdin
6575
./scripts/push-images.sh
6676
67-
- name: Install aws cli
68-
uses: chrislennon/action-aws-cli@v1.1
69-
env:
70-
ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
71-
7277
- name: Upload airy binary to S3
7378
if: ${{ github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release') || github.ref == 'refs/heads/main' }}
7479
run: |
7580
./scripts/upload-cli-binaries.sh
7681
env:
7782
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
7883
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
84+
AWS_REGION: ${{ secrets.AWS_REGION }}
7985
GITHUB_BRANCH: ${{ github.ref }}
8086

8187
- name: Publish helm charts

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
---
1717

18-
![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction-light.png)
18+
![Airy_Explainer_Highlevel_Readme](https://airy.co/docs/core/img/getting-started/introduction.png)
1919

2020
Airy Core is an is an open-source streaming app framework to train ML models and supply them with historical and real-time data. With Airy you can process data from a variety of
2121
sources:

‎VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.54.0
1+
0.55.0

‎backend/components/chat-plugin/helm/templates/backend/deployment.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ spec:
5151
initialDelaySeconds: 120
5252
periodSeconds: 10
5353
failureThreshold: 3
54+
{{ if .Values.global.kafkaCertAuth }}
55+
volumeMounts:
56+
- name: kafka-config-certs
57+
mountPath: /opt/kafka/certs
58+
{{ end }}
5459
resources:
5560
{{ toYaml .Values.backend.resources | indent 10 }}
5661
initContainers:
@@ -68,3 +73,8 @@ spec:
6873
- name: provisioning-scripts
6974
configMap:
7075
name: provisioning-scripts
76+
{{ if .Values.global.kafkaCertAuth }}
77+
- name: kafka-config-certs
78+
configMap:
79+
name: kafka-config-certs
80+
{{ end }}

‎backend/components/contacts/helm/templates/deployment.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ spec:
4545
initialDelaySeconds: 120
4646
periodSeconds: 10
4747
failureThreshold: 3
48+
{{ if .Values.global.kafkaCertAuth }}
49+
volumeMounts:
50+
- name: kafka-config-certs
51+
mountPath: /opt/kafka/certs
52+
{{ end }}
4853
resources:
4954
{{ toYaml .Values.resources | indent 10 }}
5055
initContainers:
@@ -62,3 +67,8 @@ spec:
6267
- name: provisioning-scripts
6368
configMap:
6469
name: provisioning-scripts
70+
{{ if .Values.global.kafkaCertAuth }}
71+
- name: kafka-config-certs
72+
configMap:
73+
name: kafka-config-certs
74+
{{ end }}

‎backend/components/facebook/helm/templates/deployments.yaml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ spec:
5959
- name: Health-Check
6060
value: health-check
6161
initialDelaySeconds: 120
62+
{{ if .Values.global.kafkaCertAuth }}
63+
volumeMounts:
64+
- name: kafka-config-certs
65+
mountPath: /opt/kafka/certs
66+
{{ end }}
6267
resources:
6368
{{ toYaml .Values.connector.resources | indent 12 }}
6469
initContainers:
@@ -76,6 +81,11 @@ spec:
7681
- name: provisioning-scripts
7782
configMap:
7883
name: provisioning-scripts
84+
{{ if .Values.global.kafkaCertAuth }}
85+
- name: kafka-config-certs
86+
configMap:
87+
name: kafka-config-certs
88+
{{ end }}
7989
---
8090
apiVersion: apps/v1
8191
kind: Deployment
@@ -124,6 +134,11 @@ spec:
124134
initialDelaySeconds: 120
125135
periodSeconds: 10
126136
failureThreshold: 3
137+
{{ if .Values.global.kafkaCertAuth }}
138+
volumeMounts:
139+
- name: kafka-config-certs
140+
mountPath: /opt/kafka/certs
141+
{{ end }}
127142
resources:
128143
{{ toYaml .Values.eventsRouter.resources | indent 10 }}
129144
initContainers:
@@ -141,3 +156,8 @@ spec:
141156
- name: provisioning-scripts
142157
configMap:
143158
name: provisioning-scripts
159+
{{ if .Values.global.kafkaCertAuth }}
160+
- name: kafka-config-certs
161+
configMap:
162+
name: kafka-config-certs
163+
{{ end }}

‎backend/components/flink-connector/Dockerfile.result-sender

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM golang:1.17
2+
3+
WORKDIR /app
4+
5+
COPY ./src/types.go ./src/tools.go ./src/result-sender.go ./
6+
7+
RUN go mod init main && \
8+
go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \
9+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \
10+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \
11+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \
12+
go get golang.org/x/net
13+
14+
RUN go build -o app
15+
16+
CMD ["./app"]

‎backend/components/flink-connector/Dockerfile.statements-executor

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
FROM golang:1.17
2+
3+
WORKDIR /app
4+
5+
COPY ./src/types.go ./src/tools.go ./src/statements-executor.go ./
6+
7+
RUN go mod init main && \
8+
go get github.com/confluentinc/confluent-kafka-go/v2/kafka && \
9+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry && \
10+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde && \
11+
go get github.com/confluentinc/confluent-kafka-go/v2/schemaregistry/serde/avro && \
12+
go get golang.org/x/net
13+
14+
RUN go build -o app
15+
16+
CMD ["./app"]

‎backend/components/flink-connector/Makefile

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
build-statements-executor:
2+
docker build -t flink-connector/statements-executor -f Dockerfile.statements-executor .
3+
4+
release-statements-executor: build-statements-executor
5+
docker tag flink-connector/statements-executor ghcr.io/airyhq/connectors/flink/statements-executor:release
6+
docker push ghcr.io/airyhq/connectors/flink/statements-executor:release
7+
8+
build-result-sender:
9+
docker build -t flink-connector/result-sender -f Dockerfile.result-sender .
10+
11+
release-result-sender: build-result-sender
12+
docker tag flink-connector/result-sender ghcr.io/airyhq/connectors/flink/result-sender:release
13+
docker push ghcr.io/airyhq/connectors/flink/result-sender:release

‎backend/components/flink-connector/helm/BUILD

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
load("//tools/build:helm.bzl", "helm_ruleset_core_version")
2+
3+
helm_ruleset_core_version()

‎backend/components/flink-connector/helm/Chart.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
2+
apiVersion: v2
3+
appVersion: "1.0"
4+
description: Flink connector
5+
name: flink-connector
6+
version: 1.0

‎backend/components/flink-connector/helm/templates/configmap.yaml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: {{ .Values.component }}
5+
labels:
6+
core.airy.co/managed: "true"
7+
core.airy.co/mandatory: "{{ .Values.mandatory }}"
8+
core.airy.co/component: "{{ .Values.component }}"
9+
annotations:
10+
core.airy.co/enabled: "{{ .Values.enabled }}"

‎backend/components/flink-connector/helm/templates/result-sender/deployment.yaml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ .Values.component }}-{{ .Values.resultSender.name }}
5+
labels:
6+
app: {{ .Values.component }}
7+
core.airy.co/managed: "true"
8+
core.airy.co/mandatory: "{{ .Values.mandatory }}"
9+
core.airy.co/component: {{ .Values.component }}
10+
spec:
11+
replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }}
12+
selector:
13+
matchLabels:
14+
app: {{ .Values.component }}-{{ .Values.resultSender.name }}
15+
strategy:
16+
rollingUpdate:
17+
maxSurge: 1
18+
maxUnavailable: 1
19+
type: RollingUpdate
20+
template:
21+
metadata:
22+
labels:
23+
app: {{ .Values.component }}-{{ .Values.resultSender.name }}
24+
spec:
25+
containers:
26+
- name: app
27+
image: "ghcr.io/airyhq/{{ .Values.resultSender.image }}:release"
28+
imagePullPolicy: Always
29+
envFrom:
30+
- configMapRef:
31+
name: security
32+
- configMapRef:
33+
name: kafka-config
34+
- configMapRef:
35+
name: {{ .Values.component }}
36+
env:
37+
- name: KAFKA_TOPIC_NAME
38+
value: {{ .Values.resultSender.topic }}
39+
- name: API_COMMUNICATION_URL
40+
value: {{ .Values.apiCommunicationUrl }}
41+
livenessProbe:
42+
httpGet:
43+
path: /actuator/health
44+
port: {{ .Values.port }}
45+
httpHeaders:
46+
- name: Health-Check
47+
value: health-check
48+
initialDelaySeconds: 43200
49+
periodSeconds: 10
50+
failureThreshold: 3

‎backend/components/flink-connector/helm/templates/result-sender/service.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
name: {{ .Values.component }}-{{ .Values.resultSender.name }}
5+
labels:
6+
app: {{ .Values.component }}-{{ .Values.resultSender.name }}
7+
spec:
8+
type: ClusterIP
9+
clusterIP: None
10+
ports:
11+
- name: {{ .Values.component }}-{{ .Values.resultSender.name }}
12+
port: 80
13+
targetPort: {{ .Values.port }}
14+
selector:
15+
app: {{ .Values.component }}-{{ .Values.resultSender.name }}

‎backend/components/flink-connector/helm/templates/statements-executor/deployment.yaml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: {{ .Values.component }}-{{ .Values.executor.name }}
5+
labels:
6+
app: {{ .Values.component }}
7+
core.airy.co/managed: "true"
8+
core.airy.co/mandatory: "{{ .Values.mandatory }}"
9+
core.airy.co/component: {{ .Values.component }}
10+
spec:
11+
replicas: {{ if .Values.enabled }} 1 {{ else }} 0 {{ end }}
12+
selector:
13+
matchLabels:
14+
app: {{ .Values.component }}-{{ .Values.executor.name }}
15+
strategy:
16+
rollingUpdate:
17+
maxSurge: 1
18+
maxUnavailable: 1
19+
type: RollingUpdate
20+
template:
21+
metadata:
22+
labels:
23+
app: {{ .Values.component }}-{{ .Values.executor.name }}
24+
spec:
25+
containers:
26+
- name: app
27+
image: "ghcr.io/airyhq/{{ .Values.executor.image }}:release"
28+
imagePullPolicy: Always
29+
envFrom:
30+
- configMapRef:
31+
name: security
32+
- configMapRef:
33+
name: kafka-config
34+
- configMapRef:
35+
name: {{ .Values.component }}
36+
env:
37+
- name: KAFKA_TOPIC_NAME
38+
value: {{ .Values.executor.topic }}
39+
- name: FLINK_GATEWAY_URL
40+
value: {{ .Values.gatewayUrl }}
41+
livenessProbe:
42+
httpGet:
43+
path: /actuator/health
44+
port: {{ .Values.port }}
45+
httpHeaders:
46+
- name: Health-Check
47+
value: health-check
48+
initialDelaySeconds: 43200
49+
periodSeconds: 10
50+
failureThreshold: 3

‎backend/components/flink-connector/helm/templates/statements-executor/service.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: {{ .Values.component }}-{{ .Values.executor.name }}
6+
labels:
7+
app: {{ .Values.component }}-{{ .Values.executor.name }}
8+
spec:
9+
type: ClusterIP
10+
clusterIP: None
11+
ports:
12+
- name: {{ .Values.component }}-{{ .Values.executor.name }}
13+
port: 80
14+
targetPort: {{ .Values.port }}
15+
selector:
16+
app: {{ .Values.component }}-{{ .Values.executor.name }}

‎backend/components/flink-connector/helm/values.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2+
component: flink-connector
3+
mandatory: false
4+
enabled: false
5+
port: 8080
6+
resources:
7+
gatewayUrl: "http://flink-jobmanager:8083"
8+
apiCommunicationUrl: "http://api-communication/messages.send"
9+
executor:
10+
name: statements-executor
11+
image: connectors/flink/statements-executor
12+
topic: flink.statements
13+
resultSender:
14+
name: result-sender
15+
image: connectors/flink/result-sender
16+
topic: flink.output

0 commit comments

Comments
 (0)