Skip to content

Commit 32c2e4d

Browse files
fahmizainal17fahmizainal17
committed
feat:added dockerfile airflow docker-compose prometheus grafana and test
1 parent 860af1e commit 32c2e4d

File tree

17 files changed

+1395
-148
lines changed

17 files changed

+1395
-148
lines changed

‎.github/workflows/ci-cd.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: CI/CD Pipeline
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
schedule:
9+
- cron: '0 0 * * *' # Run daily at midnight UTC
10+
11+
jobs:
12+
test:
13+
runs-on: ubuntu-latest
14+
15+
steps:
16+
- uses: actions/checkout@v2
17+
18+
- name: Set up Python 3.9
19+
uses: actions/setup-python@v2
20+
with:
21+
python-version: 3.9
22+
23+
- name: Install dependencies
24+
run: |
25+
python -m pip install --upgrade pip
26+
pip install flake8 pytest
27+
pip install -r requirements.txt
28+
29+
- name: Lint with flake8
30+
run: |
31+
# stop the build if there are Python syntax errors or undefined names
32+
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
33+
# exit-zero treats all errors as warnings
34+
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
35+
36+
- name: Test with pytest
37+
run: |
38+
# Create test directories
39+
mkdir -p data/raw data/processed data/output logs
40+
pytest -v
41+
42+
build:
43+
needs: test
44+
runs-on: ubuntu-latest
45+
46+
steps:
47+
- uses: actions/checkout@v2
48+
49+
- name: Build Docker image
50+
run: docker build -t weather-pipeline:${{ github.sha }} .
51+
52+
- name: Test Docker image
53+
run: |
54+
docker run --rm -e API_KEY=${{ secrets.OPENWEATHERMAP_API_KEY }} weather-pipeline:${{ github.sha }} --help
55+
56+
deploy:
57+
needs: build
58+
runs-on: ubuntu-latest
59+
if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request'
60+
61+
steps:
62+
- uses: actions/checkout@v2
63+
64+
- name: Deploy with Docker Compose
65+
env:
66+
API_KEY: ${{ secrets.OPENWEATHERMAP_API_KEY }}
67+
run: |
68+
echo "API_KEY=$API_KEY" > .env
69+
docker-compose up -d
70+
71+
- name: Wait for data processing
72+
run: |
73+
sleep 60 # Give it time to process data
74+
75+
- name: Create artifacts
76+
run: |
77+
mkdir -p artifacts
78+
cp -r data/output/* artifacts/
79+
cp logs/pipeline.log artifacts/
80+
81+
- name: Upload artifacts
82+
uses: actions/upload-artifact@v2
83+
with:
84+
name: weather-data-artifacts
85+
path: artifacts/

‎.gitignore

Lines changed: 31 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
1-
# Byte-compiled / optimized / DLL files
1+
# Python
22
__pycache__/
33
*.py[cod]
44
*$py.class
5-
6-
# C extensions
75
*.so
8-
9-
# Distribution / packaging
106
.Python
7+
env/
118
build/
129
develop-eggs/
1310
dist/
@@ -19,153 +16,40 @@ lib64/
1916
parts/
2017
sdist/
2118
var/
22-
wheels/
23-
share/python-wheels/
2419
*.egg-info/
2520
.installed.cfg
2621
*.egg
27-
MANIFEST
28-
29-
# PyInstaller
30-
# Usually these files are written by a python script from a template
31-
# before PyInstaller builds the exe, so as to inject date/other infos into it.
32-
*.manifest
33-
*.spec
34-
35-
# Installer logs
36-
pip-log.txt
37-
pip-delete-this-directory.txt
38-
39-
# Unit test / coverage reports
40-
htmlcov/
41-
.tox/
42-
.nox/
43-
.coverage
44-
.coverage.*
45-
.cache
46-
nosetests.xml
47-
coverage.xml
48-
*.cover
49-
*.py,cover
50-
.hypothesis/
51-
.pytest_cache/
52-
cover/
53-
54-
# Translations
55-
*.mo
56-
*.pot
57-
58-
# Django stuff:
59-
*.log
60-
local_settings.py
61-
db.sqlite3
62-
db.sqlite3-journal
63-
64-
# Flask stuff:
65-
instance/
66-
.webassets-cache
67-
68-
# Scrapy stuff:
69-
.scrapy
70-
71-
# Sphinx documentation
72-
docs/_build/
73-
74-
# PyBuilder
75-
.pybuilder/
76-
target/
77-
78-
# Jupyter Notebook
79-
.ipynb_checkpoints
80-
81-
# IPython
82-
profile_default/
83-
ipython_config.py
84-
85-
# pyenv
86-
# For a library or package, you might want to ignore these files since the code is
87-
# intended to run in multiple environments; otherwise, check them in:
88-
# .python-version
89-
90-
# pipenv
91-
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92-
# However, in case of collaboration, if having platform-specific dependencies or dependencies
93-
# having no cross-platform support, pipenv may install dependencies that don't work, or not
94-
# install all needed dependencies.
95-
#Pipfile.lock
9622

97-
# UV
98-
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99-
# This is especially recommended for binary packages to ensure reproducibility, and is more
100-
# commonly ignored for libraries.
101-
#uv.lock
102-
103-
# poetry
104-
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105-
# This is especially recommended for binary packages to ensure reproducibility, and is more
106-
# commonly ignored for libraries.
107-
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108-
#poetry.lock
109-
110-
# pdm
111-
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
112-
#pdm.lock
113-
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
114-
# in version control.
115-
# https://pdm.fming.dev/latest/usage/project/#working-with-version-control
116-
.pdm.toml
117-
.pdm-python
118-
.pdm-build/
119-
120-
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
121-
__pypackages__/
122-
123-
# Celery stuff
124-
celerybeat-schedule
125-
celerybeat.pid
126-
127-
# SageMath parsed files
128-
*.sage.py
129-
130-
# Environments
23+
# Environment variables
13124
.env
132-
.venv
133-
env/
25+
26+
# Virtual Environment
13427
venv/
13528
ENV/
136-
env.bak/
137-
venv.bak/
138-
139-
# Spyder project settings
140-
.spyderproject
141-
.spyproject
142-
143-
# Rope project settings
144-
.ropeproject
145-
146-
# mkdocs documentation
147-
/site
148-
149-
# mypy
150-
.mypy_cache/
151-
.dmypy.json
152-
dmypy.json
153-
154-
# Pyre type checker
155-
.pyre/
156-
157-
# pytype static type analyzer
158-
.pytype/
159-
160-
# Cython debug symbols
161-
cython_debug/
162-
163-
# PyCharm
164-
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
165-
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
166-
# and can be added to the global gitignore or merged into this file. For a more nuclear
167-
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
168-
#.idea/
16929

170-
# PyPI configuration file
171-
.pypirc
30+
# Data files
31+
data/raw/*
32+
data/processed/*
33+
data/output/*
34+
!data/raw/.gitkeep
35+
!data/processed/.gitkeep
36+
!data/output/.gitkeep
37+
38+
# Logs
39+
logs/*
40+
!logs/.gitkeep
41+
42+
# IDE
43+
.idea/
44+
.vscode/
45+
*.swp
46+
*.swo
47+
48+
# OS specific
49+
.DS_Store
50+
.DS_Store?
51+
._*
52+
.Spotlight-V100
53+
.Trashes
54+
ehthumbs.db
55+
Thumbs.db

‎Dockerfile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM python:3.9-slim
2+
3+
WORKDIR /app
4+
5+
# Copy requirements first to leverage Docker cache
6+
COPY requirements.txt .
7+
RUN pip install --no-cache-dir -r requirements.txt
8+
9+
# Copy the rest of the application
10+
COPY . .
11+
12+
# Create necessary directories
13+
RUN mkdir -p data/raw data/processed data/output logs
14+
15+
# Set environment variables
16+
ENV PYTHONPATH=/app
17+
ENV PYTHONUNBUFFERED=1
18+
19+
# Run the application
20+
ENTRYPOINT ["python", "main.py"]

0 commit comments

Comments
 (0)