Skip to content

Commit 4f9b4f9

Browse files
committed
CTK: Improve transformations for MongoDB Table Loader
1 parent 398aad5 commit 4f9b4f9

File tree

3 files changed

+82
-8
lines changed

3 files changed

+82
-8
lines changed
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
cratedb-toolkit[influxdb,mongodb]==0.0.27
1+
cratedb-toolkit[influxdb,mongodb]==0.0.29

application/cratedb-toolkit/test_io.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,12 @@ def test_ctk_load_table_mongodb_json(drop_testing_tables):
134134
progress=GitProgressPrinter(),
135135
)
136136

137+
# The `countries-big.json` file contains bogus characters.
138+
countries_big_path = datasets_path / "countries-big.json"
139+
payload = countries_big_path.read_text()
140+
payload = payload.replace("\ufeff", "")
141+
countries_big_path.write_text(payload)
142+
137143
# Invoke data transfer.
138144
command = f"""
139145
ctk load table \

application/cratedb-toolkit/zyp-mongodb-json-files.yaml

Lines changed: 75 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,79 @@
3838
meta:
3939
type: zyp-project
4040
version: 1
41+
4142
collections:
42-
- address:
43-
container: datasets
44-
name: companies
45-
pre:
46-
rules:
47-
- expression: .[] |= del(.image.available_sizes, .screenshots[].available_sizes)
48-
type: jq
43+
44+
- address:
45+
container: datasets
46+
name: books
47+
pre:
48+
rules:
49+
- expression: .[] |= (._id |= tostring)
50+
type: jq
51+
52+
- address:
53+
container: datasets
54+
name: city_inspections
55+
pre:
56+
rules:
57+
- expression: |
58+
.[] |= (
59+
select(true)
60+
| .address.number |= numbers
61+
| .address.zip |= numbers
62+
| .certificate_number |= tostring
63+
)
64+
type: jq
65+
66+
- address:
67+
container: datasets
68+
name: companies
69+
pre:
70+
rules:
71+
- expression: |
72+
.[] |=
73+
del(
74+
.image.available_sizes,
75+
.screenshots[].available_sizes,
76+
.created_at
77+
)
78+
type: jq
79+
80+
- address:
81+
container: datasets
82+
name: countries-big
83+
pre:
84+
rules:
85+
- expression: .[] |= (.ISO |= tostring)
86+
type: jq
87+
88+
- address:
89+
container: datasets
90+
name: products
91+
pre:
92+
rules:
93+
- expression: |
94+
.[] |= (
95+
select(true)
96+
| if (.for) then .for |= to_array end
97+
| if (.type) then .type |= to_array end
98+
| if (.limits.data.n) then .limits.data.n |= tostring end
99+
| if (.limits.sms.n) then .limits.sms.n |= tostring end
100+
| if (.limits.voice.n) then .limits.voice.n |= tostring end
101+
| del(.additional_tarriffs)
102+
)
103+
type: jq
104+
105+
- address:
106+
container: datasets
107+
name: restaurant
108+
pre:
109+
rules:
110+
- expression: |
111+
.[] |= (
112+
select(true)
113+
| .rating |= tostring
114+
| .type |= to_array
115+
)
116+
type: jq

0 commit comments

Comments
 (0)