Skip to content

Commit 5d379e6

Browse files
owlleg6owlleg6
authored
Merge pull request #3 from data-platform-hq/add_python_project
microsoft demo wwi
2 parents 9f5a9ed + c5a4a35 commit 5d379e6

File tree

221 files changed

+10665
-21
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

221 files changed

+10665
-21
lines changed

‎.github/workflows/cicd.yml

Lines changed: 42 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,17 @@
1-
name: 'Build and Deploy to Databricks'
1+
name: 'Deploy Microsoft WWI packages to Databricks'
22

33
on:
44
workflow_dispatch:
55
inputs:
6-
projectFolders:
7-
description: 'List of folders to run'
8-
required: true
9-
type: choice
10-
options:
11-
- 01_raw
12-
- 02_data_product
13-
6+
cluster_id:
7+
description: 'Optionaly overwrite job cluster'
8+
required: false
9+
type: string
10+
default: ''
11+
1412
jobs:
1513
deploy:
16-
name: "Build and Deploy Bundle"
14+
name: "Build and Deploy Bundle to the Workspaces"
1715
runs-on: ubuntu-latest
1816

1917
steps:
@@ -25,17 +23,42 @@ jobs:
2523
# See https://github.com/databricks/setup-cli
2624
- uses: databricks/setup-cli@main
2725

28-
- run: databricks bundle validate
29-
working-directory: ./${{ github.event.inputs.projectFolders }}/
26+
- name: Configure demo auth profile
27+
run: |
28+
echo -e "[demo]\nhost=${{ secrets.DATABRICKS_HOST }}" > .databrickscfg
29+
30+
- name: Validation
31+
run: |
32+
databricks auth env --host ${{ secrets.DATABRICKS_HOST }}
33+
databricks bundle validate \
34+
-p demo
3035
env:
31-
DATABRICKS_TOKEN: ${{ secrets.SP_TOKEN }}
32-
DATABRICKS_BUNDLE_ENV: dev
36+
DATABRICKS_CONFIG_FILE: .databrickscfg
37+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
38+
DATABRICKS_BUNDLE_ENV: demo
3339

34-
# Deploy the bundle to the "qa" target as defined
40+
# Deploy the bundle to the "demo" target as defined
3541
# in the bundle's settings file.
36-
- run: databricks bundle deploy
37-
working-directory: ./${{ github.event.inputs.projectFolders }}/
42+
- name: Deploy with custom cluster
43+
if: ${{ github.event.inputs.cluster_id != '' }}
44+
run: |
45+
databricks bundle deploy \
46+
-c ${{ github.event.inputs.cluster_id }} \
47+
-p demo
48+
49+
env:
50+
DATABRICKS_CONFIG_FILE: .databrickscfg
51+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
52+
DATABRICKS_BUNDLE_ENV: demo
53+
54+
- name: Deploy with default cluster
55+
if: ${{ github.event.inputs.cluster_id == '' }}
56+
run: |
57+
databricks bundle deploy \
58+
-p demo
59+
3860
env:
39-
DATABRICKS_TOKEN: ${{ secrets.SP_TOKEN }}
40-
DATABRICKS_BUNDLE_ENV: dev
61+
DATABRICKS_CONFIG_FILE: .databrickscfg
62+
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
63+
DATABRICKS_BUNDLE_ENV: demo
4164

‎.gitignore

Lines changed: 311 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,311 @@
1+
.python-version
2+
# Byte-compiled / optimized / DLL files
3+
__pycache__/
4+
*.py[cod]
5+
*$py.class
6+
7+
# C extensions
8+
*.so
9+
10+
# Distribution / packaging
11+
.Python
12+
build/
13+
develop-eggs/
14+
dist/
15+
downloads/
16+
eggs/
17+
.eggs/
18+
lib/
19+
lib64/
20+
parts/
21+
sdist/
22+
var/
23+
wheels/
24+
share/python-wheels/
25+
*.egg-info/
26+
.installed.cfg
27+
*.egg
28+
MANIFEST
29+
30+
# PyInstaller
31+
# Usually these files are written by a python script from a template
32+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
33+
*.manifest
34+
*.spec
35+
36+
# Installer logs
37+
pip-log.txt
38+
pip-delete-this-directory.txt
39+
40+
# Unit test / coverage reports
41+
htmlcov/
42+
.tox/
43+
.nox/
44+
.coverage
45+
.coverage.*
46+
.cache
47+
nosetests.xml
48+
coverage.xml
49+
*.cover
50+
*.py,cover
51+
.hypothesis/
52+
.pytest_cache/
53+
cover/
54+
55+
# Translations
56+
*.mo
57+
*.pot
58+
59+
# Django stuff:
60+
*.log
61+
local_settings.py
62+
db.sqlite3
63+
db.sqlite3-journal
64+
65+
# Flask stuff:
66+
instance/
67+
.webassets-cache
68+
69+
# Scrapy stuff:
70+
.scrapy
71+
72+
# Sphinx documentation
73+
docs/_build/
74+
75+
# PyBuilder
76+
.pybuilder/
77+
target/
78+
79+
# Jupyter Notebook
80+
.ipynb_checkpoints
81+
82+
# IPython
83+
profile_default/
84+
ipython_config.py
85+
86+
# pyenv
87+
# For a library or package, you might want to ignore these files since the code is
88+
# intended to run in multiple environments; otherwise, check them in:
89+
# .python-version
90+
91+
# pipenv
92+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
93+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
94+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
95+
# install all needed dependencies.
96+
#Pipfile.lock
97+
98+
# poetry
99+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
100+
# This is especially recommended for binary packages to ensure reproducibility, and is more
101+
# commonly ignored for libraries.
102+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
103+
poetry.lock
104+
105+
# pdm
106+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
107+
#pdm.lock
108+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
109+
# in version control.
110+
# https://pdm.fming.dev/#use-with-ide
111+
.pdm.toml
112+
113+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
114+
__pypackages__/
115+
116+
# Celery stuff
117+
celerybeat-schedule
118+
celerybeat.pid
119+
120+
# SageMath parsed files
121+
*.sage.py
122+
123+
# Environments
124+
.env
125+
.venv
126+
env/
127+
venv/
128+
ENV/
129+
env.bak/
130+
venv.bak/
131+
132+
# Spyder project settings
133+
.spyderproject
134+
.spyproject
135+
136+
# Rope project settings
137+
.ropeproject
138+
139+
# mkdocs documentation
140+
/site
141+
142+
# mypy
143+
.mypy_cache/
144+
.dmypy.json
145+
dmypy.json
146+
147+
# Pyre type checker
148+
.pyre/
149+
150+
# pytype static type analyzer
151+
.pytype/
152+
153+
# Cython debug symbols
154+
cython_debug/
155+
156+
# PyCharm
157+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
158+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
159+
# and can be added to the global gitignore or merged into this file. For a more nuclear
160+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
161+
.idea/
162+
downloads/
163+
eggs/
164+
.eggs/
165+
lib/
166+
lib64/
167+
parts/
168+
sdist/
169+
var/
170+
wheels/
171+
share/python-wheels/
172+
*.egg-info/
173+
.installed.cfg
174+
*.egg
175+
MANIFEST
176+
177+
# PyInstaller
178+
# Usually these files are written by a python script from a template
179+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
180+
*.manifest
181+
*.spec
182+
183+
# Installer logs
184+
pip-log.txt
185+
pip-delete-this-directory.txt
186+
187+
# Unit test / coverage reports
188+
htmlcov/
189+
.tox/
190+
.nox/
191+
.coverage
192+
.coverage.*
193+
.cache
194+
nosetests.xml
195+
coverage.xml
196+
*.cover
197+
*.py,cover
198+
.hypothesis/
199+
.pytest_cache/
200+
cover/
201+
202+
# Translations
203+
*.mo
204+
*.pot
205+
206+
# Django stuff:
207+
*.log
208+
local_settings.py
209+
db.sqlite3
210+
db.sqlite3-journal
211+
212+
# Flask stuff:
213+
instance/
214+
.webassets-cache
215+
216+
# Scrapy stuff:
217+
.scrapy
218+
219+
# Sphinx documentation
220+
docs/_build/
221+
222+
# PyBuilder
223+
.pybuilder/
224+
target/
225+
226+
# Jupyter Notebook
227+
.ipynb_checkpoints
228+
229+
# IPython
230+
profile_default/
231+
ipython_config.py
232+
233+
# pyenv
234+
# For a library or package, you might want to ignore these files since the code is
235+
# intended to run in multiple environments; otherwise, check them in:
236+
# .python-version
237+
238+
# pipenv
239+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
240+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
241+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
242+
# install all needed dependencies.
243+
#Pipfile.lock
244+
245+
# poetry
246+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
247+
# This is especially recommended for binary packages to ensure reproducibility, and is more
248+
# commonly ignored for libraries.
249+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
250+
#poetry.lock
251+
252+
# pdm
253+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
254+
#pdm.lock
255+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
256+
# in version control.
257+
# https://pdm.fming.dev/#use-with-ide
258+
.pdm.toml
259+
260+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
261+
__pypackages__/
262+
263+
# Celery stuff
264+
celerybeat-schedule
265+
celerybeat.pid
266+
267+
# SageMath parsed files
268+
*.sage.py
269+
270+
# Environments
271+
.env
272+
.venv
273+
env/
274+
venv/
275+
ENV/
276+
env.bak/
277+
venv.bak/
278+
279+
# Spyder project settings
280+
.spyderproject
281+
.spyproject
282+
283+
# Rope project settings
284+
.ropeproject
285+
286+
# mkdocs documentation
287+
/site
288+
289+
# mypy
290+
.mypy_cache/
291+
.dmypy.json
292+
dmypy.json
293+
294+
# Pyre type checker
295+
.pyre/
296+
297+
# pytype static type analyzer
298+
.pytype/
299+
300+
# Cython debug symbols
301+
cython_debug/
302+
303+
# PyCharm
304+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
305+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
306+
# and can be added to the global gitignore or merged into this file. For a more nuclear
307+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
308+
.idea/
309+
incremental_insert_test
310+
delta_tables/
311+
.databricks

‎01_raw

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎02_data_product

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎data_product/.dockerignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Dockerfile
2+
.env

0 commit comments

Comments
 (0)