Skip to content

Commit 7a9a38c

Browse files
BenGalewskyBenGalewsky
committed
Add code generator ABC and post operation
1 parent f9dfb6a commit 7a9a38c

File tree

9 files changed

+410
-4
lines changed

9 files changed

+410
-4
lines changed

‎.github/workflows/ci.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: CI/CD
2+
3+
on:
4+
push:
5+
branches:
6+
- "*"
7+
tags:
8+
- "*"
9+
10+
pull_request:
11+
12+
jobs:
13+
test:
14+
strategy:
15+
matrix:
16+
python-version: ['3.8', '3.9', '3.10']
17+
runs-on: ubuntu-latest
18+
19+
steps:
20+
- uses: actions/checkout@master
21+
- name: Set up Python ${{ matrix.python-version }}
22+
uses: actions/setup-python@v1
23+
with:
24+
python-version: ${{ matrix.python-version }}
25+
- name: Install dependencies
26+
run: |
27+
python -m pip install --upgrade pip setuptools wheel
28+
pip install --no-cache-dir ".[test]"
29+
pip list
30+
- name: Lint with Flake8
31+
run: |
32+
flake8
33+
- name: Test with pytest
34+
run: |
35+
python -m coverage run -m pytest -r sx
36+
- name: Report coverage with Codecov
37+
if: github.event_name == 'push'
38+
run: |
39+
codecov --token=${{ secrets.CODECOV_TOKEN }}

‎servicex_codegen/__init__.py

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,44 @@
3131
# Redistribution and use in source and binary forms, with or without
3232
# modification, are permitted provided that the following conditions are met:
3333
#
34-
#
35-
#
36-
#
34+
import os
35+
36+
from flask import Flask
37+
from flask_restful import Api
38+
39+
from servicex_codegen.post_operation import GeneratedCode
40+
41+
42+
def handle_invalid_usage(error: BaseException):
43+
from flask import jsonify
44+
response = jsonify({"message": str(error)})
45+
response.status_code = 400
46+
return response
47+
48+
49+
def create_app(test_config=None, provided_translator=None):
50+
"""Create and configure an instance of the Flask application."""
51+
app = Flask(__name__, instance_relative_config=True)
52+
53+
# ensure the instance folder exists
54+
try:
55+
os.makedirs(app.instance_path)
56+
except OSError:
57+
pass
58+
59+
if not test_config:
60+
app.config.from_envvar('APP_CONFIG_FILE')
61+
else:
62+
app.config.from_mapping(test_config)
63+
64+
with app.app_context():
65+
translator = provided_translator
66+
67+
api = Api(app)
68+
GeneratedCode.make_api(translator)
69+
70+
api.add_resource(GeneratedCode, '/servicex/generated-code')
71+
72+
app.errorhandler(Exception)(handle_invalid_usage)
73+
74+
return app

‎servicex_codegen/code_generator.py

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright (c) 2022 , IRIS-HEP
2+
# All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# * Redistributions of source code must retain the above copyright notice, this
8+
# list of conditions and the following disclaimer.
9+
#
10+
# * Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# * Neither the name of the copyright holder nor the names of its
15+
# contributors may be used to endorse or promote products derived from
16+
# this software without specific prior written permission.
17+
#
18+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#
29+
30+
#
31+
# Redistribution and use in source and binary forms, with or without
32+
# modification, are permitted provided that the following conditions are met:
33+
#
34+
from abc import ABC, abstractmethod
35+
import os
36+
import zipfile
37+
from collections import namedtuple
38+
from tempfile import TemporaryDirectory
39+
40+
GeneratedFileResult = namedtuple('GeneratedFileResult', 'hash output_dir')
41+
42+
43+
class GenerateCodeException(BaseException):
44+
"""Custom exception for top level code generation exceptions"""
45+
46+
def __init__(self, message: str):
47+
BaseException.__init__(self, message)
48+
49+
50+
class CodeGenerator(ABC):
51+
def zipdir(self, path: str, zip_handle: zipfile.ZipFile) -> None:
52+
"""Given a `path` to a directory, zip up its contents into a zip file.
53+
54+
Arguments:
55+
path Path to a local directory. The contents will be put into the zip file
56+
zip_handle The zip file handle to write into.
57+
"""
58+
for root, _, files in os.walk(path):
59+
for file in files:
60+
zip_handle.write(os.path.join(root, file), file)
61+
62+
@abstractmethod
63+
def generate_code(self, query, cache_path: str):
64+
pass
65+
66+
def translate_query_to_zip(self, query: str) -> bytes:
67+
"""Translate a text ast into a zip file as a memory stream
68+
69+
Arguments:
70+
code Text `qastle` version of the input ast generated by func_adl
71+
72+
Returns
73+
bytes Data that if written as a binary output would be a zip file.
74+
"""
75+
76+
# Generate the python code
77+
with TemporaryDirectory() as tempdir:
78+
r = self.generate_code(query, tempdir)
79+
80+
# Zip up everything in the directory - we are going to ship it as back as part
81+
# of the message.
82+
z_filename = os.path.join(str(tempdir), 'joined.zip')
83+
zip_h = zipfile.ZipFile(z_filename, 'w', zipfile.ZIP_DEFLATED)
84+
self.zipdir(r.output_dir, zip_h)
85+
zip_h.close()
86+
87+
with open(z_filename, 'rb') as b_in:
88+
return b_in.read()

‎servicex_codegen/post_operation.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
# Copyright (c) 2022 , IRIS-HEP
2+
# All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# * Redistributions of source code must retain the above copyright notice, this
8+
# list of conditions and the following disclaimer.
9+
#
10+
# * Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# * Neither the name of the copyright holder nor the names of its
15+
# contributors may be used to endorse or promote products derived from
16+
# this software without specific prior written permission.
17+
#
18+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#
29+
30+
#
31+
# Redistribution and use in source and binary forms, with or without
32+
# modification, are permitted provided that the following conditions are met:
33+
#
34+
from flask import request, Response
35+
from flask_restful import Resource
36+
from servicex_codegen.code_generator import CodeGenerator
37+
38+
39+
class GeneratedCode(Resource):
40+
@classmethod
41+
def make_api(cls, code_generator: CodeGenerator):
42+
cls.code_generator = code_generator
43+
return cls
44+
45+
def post(self):
46+
try:
47+
code = request.data.decode('utf8')
48+
zip_data = self.code_generator.generate_code(code)
49+
50+
# Send the response back to you-know-what.
51+
response = Response(
52+
response=zip_data,
53+
status=200, mimetype='application/octet-stream')
54+
return response
55+
except BaseException as e:
56+
print(str(e))
57+
import traceback
58+
import sys
59+
traceback.print_exc(file=sys.stdout)
60+
return {'Message': str(e)}, 500

‎setup.cfg

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[flake8]
2+
max-line-length = 99

‎setup.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,13 @@
6767
install_requires=[
6868
"Flask==1.1.2",
6969
"Flask-RESTful==0.3.8",
70-
"Flask-WTF==0.14.3"
70+
"Flask-WTF==0.14.3",
71+
# Incompatibility between flask and the latest itsdangerous
72+
"itsdangerous==2.0.1",
73+
# Avoid import error in Flask
74+
"werkzeug==2.0.3",
75+
"jinja2==3.0.3"
76+
7177
],
7278
extras_require={
7379
"test": [

‎tests/__init__.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Copyright (c) 2022 , IRIS-HEP
2+
# All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# * Redistributions of source code must retain the above copyright notice, this
8+
# list of conditions and the following disclaimer.
9+
#
10+
# * Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# * Neither the name of the copyright holder nor the names of its
15+
# contributors may be used to endorse or promote products derived from
16+
# this software without specific prior written permission.
17+
#
18+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#
29+
30+
#
31+
# Redistribution and use in source and binary forms, with or without
32+
# modification, are permitted provided that the following conditions are met:
33+
#
34+
#
35+
#
36+
#

‎tests/test_code_generator.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright (c) 2022 , IRIS-HEP
2+
# All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions are met:
6+
#
7+
# * Redistributions of source code must retain the above copyright notice, this
8+
# list of conditions and the following disclaimer.
9+
#
10+
# * Redistributions in binary form must reproduce the above copyright notice,
11+
# this list of conditions and the following disclaimer in the documentation
12+
# and/or other materials provided with the distribution.
13+
#
14+
# * Neither the name of the copyright holder nor the names of its
15+
# contributors may be used to endorse or promote products derived from
16+
# this software without specific prior written permission.
17+
#
18+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
22+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
24+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
25+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28+
#
29+
30+
#
31+
# Redistribution and use in source and binary forms, with or without
32+
# modification, are permitted provided that the following conditions are met:
33+
#
34+
import io
35+
import os
36+
import zipfile
37+
from tempfile import TemporaryDirectory
38+
39+
from servicex_codegen.code_generator import CodeGenerator, GeneratedFileResult
40+
41+
42+
def get_zipfile_data(zip_data: bytes):
43+
with zipfile.ZipFile(io.BytesIO(zip_data)) as thezip:
44+
for zipinfo in thezip.infolist():
45+
with thezip.open(zipinfo) as thefile:
46+
yield zipinfo.filename, thefile
47+
48+
49+
def check_zip_file(zip_data: bytes, expected_file_count):
50+
names = []
51+
for name, data in get_zipfile_data(zip_data):
52+
names.append(name)
53+
print(name)
54+
assert len(names) == expected_file_count
55+
56+
57+
class TestCodeGenerator:
58+
def test_translate_query_to_zip(self, mocker):
59+
mocker.patch.object(CodeGenerator, "__abstractmethods__", new_callable=set)
60+
code_gen = CodeGenerator()
61+
62+
with TemporaryDirectory() as tempdir, \
63+
open(os.path.join(tempdir, "baz.txt"), 'w'),\
64+
open(os.path.join(tempdir, "foo.txt"), 'w'):
65+
code_gen.generate_code = mocker.Mock(
66+
return_value=GeneratedFileResult(hash="31415", output_dir=tempdir)
67+
)
68+
69+
zip_bytes = code_gen.translate_query_to_zip("select * from foo")
70+
check_zip_file(zip_bytes, expected_file_count=2)

0 commit comments

Comments
 (0)