Skip to content

Commit 19a9f17

Browse files
pvanderknyffpvanderknyff
authored
Merge pull request #506 from tableau/dev-2020.2
Merge 2020.2 changes
2 parents f76c0a7 + 2f1e6d1 commit 19a9f17

20 files changed

+403
-10
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ The latest version of the SDK is always targeted towards the latest, non-beta ve
1212
| Connector Packager SDK (Beta) for Tableau 2019.3 | 12-11-2019 |
1313
| TDVT | 2.1.12 (04-30-2020) |
1414
| | 1.5.24 (04-13-2020) |
15-
| Connector Packager | 2.0.0 (04-08-2020) |
15+
| Connector Packager | 2.1.0 (05-08-2020) |
1616

1717
* [Why Connectors?](#why-connectors)
1818
* [Get started](#get-started)

‎connector-packager/CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
44
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
55
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
66

7+
## [2.1.0] - 5/8/2020
8+
- Add support for packaging connectors using connection dialogs v2
9+
710
## [2.0.0] - 4/7/2020
811
- Remove option to sign .taco file
912

‎connector-packager/connector_packager/jar_jdk_packager.py

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
from .connector_file import ConnectorFile
1111
from .helper import check_jdk_environ_variable
12-
from .version import __min_version_tableau__
12+
from .version import __default_min_version_tableau__
1313

1414
JAR_EXECUTABLE_NAME = "jar"
1515
if os.name == 'nt':
@@ -22,6 +22,28 @@
2222
MIN_TABLEAU_VERSION_ATTR = "min-version-tableau"
2323

2424

25+
def get_min_support_version(file_list: List[ConnectorFile]) -> str:
26+
"""
27+
Get the minimum support version based on features used in the connector
28+
29+
:param file_list: files need to be packaged
30+
:type file_list: list of ConnectorFile
31+
32+
:return: String
33+
"""
34+
35+
# set minimum tableau version to default, then check for features requiring later version
36+
min_version_tableau = __default_min_version_tableau__
37+
38+
# Check file types
39+
for connector_file in file_list:
40+
# if we have a connection-fields file, then we are using modular dialogs and need 2020.2+
41+
if connector_file.file_type == "connection-fields":
42+
min_version_tableau = "2020.2"
43+
44+
return min_version_tableau
45+
46+
2547
def stamp_min_support_version(input_dir: Path, file_list: List[ConnectorFile], jar_filename: str) -> bool:
2648
"""
2749
Stamp of minimum support version to the connector manifest in packaged jar file
@@ -52,12 +74,13 @@ def stamp_min_support_version(input_dir: Path, file_list: List[ConnectorFile], j
5274
shutil.copyfile(input_dir / manifest_file.file_name, input_dir / MANIFEST_FILE_COPY_NAME)
5375

5476
# stamp the original manifest file
77+
min_version_tableau = get_min_support_version(file_list)
5578
manifest = ET.parse(input_dir / manifest_file.file_name)
5679
plugin_elem = manifest.getroot()
5780
if plugin_elem.tag != MANIFEST_ROOT_ELEM:
5881
logger.info("Manifest's root element has been modified after xml validation")
5982
return False
60-
plugin_elem.set(MIN_TABLEAU_VERSION_ATTR, __min_version_tableau__)
83+
plugin_elem.set(MIN_TABLEAU_VERSION_ATTR, min_version_tableau)
6184
manifest.write(input_dir / manifest_file.file_name, encoding="utf-8", xml_declaration=True)
6285

6386
# update the connector manifest inside taco

‎connector-packager/connector_packager/version.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
__version__ = '2.0.0'
2-
__min_version_tableau__ = '2019.4'
1+
__version__ = '2.1.0'
2+
__default_min_version_tableau__ = '2019.4'

‎connector-packager/connector_packager/xml_parser.py

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,19 @@ def parse_file(self, file_to_parse: ConnectorFile) -> bool:
153153
# add new file to list
154154
self.file_list.append(new_file)
155155

156+
# If connection-metadata, make sure that connection-fields file exists
157+
if child.tag == 'connection-metadata':
158+
connection_fields_exists = False
159+
160+
for xml_file in self.file_list:
161+
if xml_file.file_type == 'connection-fields':
162+
connection_fields_exists = True
163+
break
164+
165+
if not connection_fields_exists:
166+
logger.debug("Error: connection-metadata file requires a connection-fields file")
167+
return False
168+
156169
# If not a script and not in list, parse the file for more files to include
157170
if child.tag != 'script' and not already_in_list:
158171
children_valid = self.parse_file(new_file)
@@ -164,7 +177,7 @@ def parse_file(self, file_to_parse: ConnectorFile) -> bool:
164177
url_link = child.attrib['url']
165178
# If URL does not start with https:// then do not package and return false
166179
if not url_link.startswith(HTTPS_STRING):
167-
logging.error("Error: Only HTTPS URL's are allowed. URL " + url_link +
180+
logging.error("Error: Only HTTPS URL's are allowed. URL " + url_link +
168181
" is a non-https link in file " + file_to_parse.file_name)
169182
return False
170183

‎connector-packager/connector_packager/xsd_validator.py

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,21 +6,28 @@
66

77
from xmlschema import XMLSchema
88

9+
from defusedxml.ElementTree import parse
10+
911
from .connector_file import ConnectorFile
1012

1113
logger = logging.getLogger('packager_logger')
1214

1315
MAX_FILE_SIZE = 1024 * 256 # This is based on the max file size we will load on the Tableau side
1416
PATH_TO_XSD_FILES = Path("../validation").absolute()
1517
VALID_XML_EXTENSIONS = ['tcd', 'tdr', 'tdd', 'xml'] # These are the file extensions that we will validate
18+
PLATFORM_FIELD_NAMES = ['server', 'port', 'sslmode', 'authentication', 'username', 'password']
19+
VENDOR_FIELD_NAME_PREFIX = 'v-'
1620

1721
# Holds the mapping between file type and XSD file name
1822
XSD_DICT = {
1923
"manifest": "connector_plugin_manifest_latest",
2024
"connection-dialog": "tcd_latest",
2125
"connection-resolver": "tdr_latest",
2226
"dialect": "tdd_latest",
23-
"resource": "connector_plugin_resources_latest"
27+
"resource": "connector_plugin_resources_latest",
28+
"connection-fields": "connection_fields",
29+
"connection-metadata": "connector_plugin_metadata"
30+
2431
}
2532

2633

@@ -115,6 +122,10 @@ def validate_single_file(file_to_test: ConnectorFile, path_to_file: Path, xml_vi
115122
logger.error("XML Validation failed for " + file_to_test.file_name)
116123
return False
117124

125+
if not validate_file_specific_rules(file_to_test, path_to_file, xml_violations_buffer):
126+
logger.error("XML Validation failed for " + file_to_test.file_name)
127+
return False
128+
118129
return True
119130

120131

@@ -132,3 +143,20 @@ def get_xsd_file(file_to_test: ConnectorFile) -> Optional[str]:
132143
return xsd_file + ".xsd"
133144
else:
134145
return None
146+
147+
148+
def validate_file_specific_rules(file_to_test: ConnectorFile, path_to_file: Path, xml_violations_buffer: List[str]) -> bool:
149+
150+
if file_to_test.file_type == 'connection-fields':
151+
xml_tree = parse(str(path_to_file))
152+
root = xml_tree.getroot()
153+
154+
for child in root.iter('field'):
155+
if 'name' in child.attrib:
156+
field_name = child.attrib['name']
157+
if not (field_name in PLATFORM_FIELD_NAMES or field_name.startswith(VENDOR_FIELD_NAME_PREFIX)):
158+
xml_violations_buffer.append("Element 'field', attribute 'name'='" + field_name +
159+
"' not an allowed value. See 'Connection Field Platform Integration' section of documentation for allowed values.")
160+
return False
161+
162+
return True

‎connector-packager/tests/test_jar_packager.py

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@
88
from .jar_packager import create_jar
99
from connector_packager.jar_jdk_packager import jdk_create_jar
1010
from connector_packager.connector_file import ConnectorFile
11-
from connector_packager.version import __min_version_tableau__
11+
from connector_packager.version import __default_min_version_tableau__
1212

1313
TEST_FOLDER = Path("tests/test_resources")
1414
MANIFEST_FILE_NAME = "manifest.xml"
1515

16+
VERSION_2020_2 = "2020.2"
17+
1618

1719
class TestJarPackager(unittest.TestCase):
1820

@@ -62,7 +64,39 @@ def test_jdk_create_jar(self):
6264

6365
manifest = ET.parse(path_to_extracted_manifest)
6466
self.assertEqual(manifest.getroot().get("min-version-tableau"),
65-
__min_version_tableau__, "wrong min-version-tableau attr or doesn't exist")
67+
__default_min_version_tableau__, "wrong min-version-tableau attr or doesn't exist")
68+
69+
if dest_dir.exists():
70+
shutil.rmtree(dest_dir)
71+
72+
def test_jdk_create_jar_mcd(self):
73+
files_list = [
74+
ConnectorFile("manifest.xml", "manifest"),
75+
ConnectorFile("connectionFields.xml", "connection-fields"),
76+
ConnectorFile("connectionMetadata.xml", "connection-metadata"),
77+
ConnectorFile("connectionBuilder.js", "script"),
78+
ConnectorFile("dialect.xml", "dialect"),
79+
ConnectorFile("connectionResolver.xml", "connection-resolver"),
80+
ConnectorFile("connectionProperties.js", "script")]
81+
source_dir = TEST_FOLDER / Path("modular_dialog_connector")
82+
dest_dir = TEST_FOLDER / Path("packaged-connector-by-jdk/")
83+
package_name = "test_mcd.taco"
84+
85+
jdk_create_jar(source_dir, files_list, package_name, dest_dir)
86+
87+
path_to_test_file = dest_dir / Path(package_name)
88+
self.assertTrue(os.path.isfile(path_to_test_file), "taco file doesn't exist")
89+
90+
# test min support tableau version is stamped
91+
args = ["jar", "xf", package_name, MANIFEST_FILE_NAME]
92+
p = subprocess.Popen(args, cwd=os.path.abspath(dest_dir))
93+
self.assertEqual(p.wait(), 0, "can not extract manfifest file from taco")
94+
path_to_extracted_manifest = dest_dir / MANIFEST_FILE_NAME
95+
self.assertTrue(os.path.isfile(path_to_extracted_manifest), "extracted manifest file doesn't exist")
96+
97+
manifest = ET.parse(path_to_extracted_manifest)
98+
self.assertEqual(manifest.getroot().get("min-version-tableau"),
99+
VERSION_2020_2, "wrong min-version-tableau attr or doesn't exist")
66100

67101
if dest_dir.exists():
68102
shutil.rmtree(dest_dir)

‎connector-packager/tests/test_resources/broken_xml/connectionFields.xml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
3+
<connection-fields>
4+
<field name="server" label="Server" value-type="string" category="endpoint" />
5+
6+
<field name="port" label="Port" value-type="string" category="endpoint" default-value="5432" />
7+
8+
<field name="warehouse" label="Warehouse" value-type="string" category="metadata" />
9+
10+
<field name="username" label="Username" value-type="string" category="authentication" />
11+
12+
<field name="password" label="Password" value-type="string" category="authentication" secure="true" />
13+
14+
</connection-fields>

‎connector-packager/tests/test_resources/modular_dialog_connector/connectionBuilder.js

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
(function dsbuilder(attr) {
2+
var urlBuilder = "jdbc:postgresql://" + attr[connectionHelper.attributeServer] + ":" + attr[connectionHelper.attributePort] + "/" + attr[connectionHelper.attributeDatabase] + "?";
3+
4+
return [urlBuilder];
5+
})

‎connector-packager/tests/test_resources/modular_dialog_connector/connectionFields.xml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
3+
<connection-fields>
4+
<field name="server" label="Server" value-type="string" category="endpoint" >
5+
<validation-rule reg-exp="^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$"/>
6+
</field>
7+
8+
<field name="port" label="Port" value-type="string" category="endpoint" default-value="5432" />
9+
10+
<field name="v-custom" label="Custom" value-type="string" category="endpoint" />
11+
12+
<field name="username" label="Username" value-type="string" category="authentication" />
13+
14+
<field name="password" label="Password" value-type="string" category="authentication" secure="true" />
15+
16+
</connection-fields>

‎connector-packager/tests/test_resources/modular_dialog_connector/connectionMetadata.xml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
3+
<connection-metadata>
4+
<database enabled='true' label='Database'>
5+
<field />
6+
</database>
7+
<schema enabled='false' />
8+
</connection-metadata>

‎connector-packager/tests/test_resources/modular_dialog_connector/connectionProperties.js

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
(function propertiesbuilder(attr) {
2+
var props = {};
3+
props["user"] = attr[connectionHelper.attributeUsername];
4+
props["password"] = attr[connectionHelper.attributePassword];
5+
6+
return props;
7+
})

‎connector-packager/tests/test_resources/modular_dialog_connector/connectionResolver.xml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
<?xml version='1.0' encoding='utf-8' ?>
2+
3+
<tdr class='postgres_mcd'>
4+
<connection-resolver>
5+
<connection-builder>
6+
<script file='connectionBuilder.js'/>
7+
</connection-builder>
8+
<connection-normalizer>
9+
<required-attributes>
10+
<attribute-list>
11+
<attr>server</attr>
12+
<attr>port</attr>
13+
<attr>dbname</attr>
14+
<attr>v-custom</attr>
15+
<attr>username</attr>
16+
<attr>password</attr>
17+
</attribute-list>
18+
</required-attributes>
19+
</connection-normalizer>
20+
<connection-properties>
21+
<script file='connectionProperties.js'/>
22+
</connection-properties>
23+
</connection-resolver>
24+
</tdr>

‎connector-packager/tests/test_resources/modular_dialog_connector/dialect.xml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
3+
<dialect name='PostgresMCD'
4+
class='postgres_mcd'
5+
base='PostgreSQL90Dialect'
6+
version='18.1'>
7+
</dialect>

‎connector-packager/tests/test_resources/modular_dialog_connector/manifest.xml

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
<?xml version='1.0' encoding='utf-8' ?>
2+
3+
<connector-plugin class='postgres_mcd' superclass='jdbc' plugin-version='0.0.1' name='PostgreSQL MCD' version='18.1'>
4+
<vendor-information>
5+
<company name="Connector SDK"/>
6+
<support-link url = "https://example.com"/>
7+
</vendor-information>
8+
<connection-customization class="postgres_mcd" enabled="true" version="10.0">
9+
<vendor name="vendor"/>
10+
<driver name="driver"/>
11+
<customizations>
12+
<customization name="CAP_SELECT_INTO" value="yes"/>
13+
<customization name="CAP_SELECT_TOP_INTO" value="yes"/>
14+
<customization name="CAP_CREATE_TEMP_TABLES" value="no"/>
15+
<customization name="CAP_QUERY_BOOLEXPR_TO_INTEXPR" value="no"/>
16+
<customization name="CAP_QUERY_GROUP_BY_BOOL" value="yes"/>
17+
<customization name="CAP_QUERY_GROUP_BY_DEGREE" value="yes"/>
18+
<customization name="CAP_QUERY_SORT_BY" value="yes"/>
19+
<customization name="CAP_QUERY_SUBQUERIES" value="yes"/>
20+
<customization name="CAP_QUERY_TOP_N" value="yes"/>
21+
<customization name="CAP_QUERY_TOP_SAMPLE" value="yes"/>
22+
<customization name="CAP_QUERY_TOP_SAMPLE_PERCENT" value="yes"/>
23+
<customization name="CAP_QUERY_WHERE_FALSE_METADATA" value="yes"/>
24+
<customization name="CAP_QUERY_SUBQUERIES_WITH_TOP" value="yes"/>
25+
<customization name="CAP_SUPPORTS_SPLIT_FROM_LEFT" value="yes"/>
26+
<customization name="CAP_SUPPORTS_SPLIT_FROM_RIGHT" value="yes"/>
27+
<customization name="CAP_SUPPORTS_UNION" value="yes"/>
28+
<customization name="CAP_QUERY_ALLOW_PARTIAL_AGGREGATION" value="no"/>
29+
<customization name="CAP_QUERY_TIME_REQUIRES_CAST" value="yes"/>
30+
</customizations>
31+
</connection-customization>
32+
<connection-fields file='connectionFields.xml'/>
33+
<connection-metadata file='connectionMetadata.xml'/>
34+
<connection-resolver file="connectionResolver.xml"/>
35+
<dialect file='dialect.xml'/>
36+
</connector-plugin>

‎connector-packager/tests/test_xml_parser.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,27 @@ def test_generate_file_list(self):
4545
expected_file_list, expected_class_name)
4646
self.assertFalse(actual_file_list, "Connector with non-https urls returned a file list when it shouldn't")
4747

48+
def test_genreate_file_list_mcd(self):
49+
# Test modular dialog connector
50+
expected_class_name = "postgres_mcd"
51+
expected_file_list = [
52+
ConnectorFile("manifest.xml", "manifest"),
53+
ConnectorFile("connectionFields.xml", "connection-fields"),
54+
ConnectorFile("connectionMetadata.xml", "connection-metadata"),
55+
ConnectorFile("connectionBuilder.js", "script"),
56+
ConnectorFile("dialect.xml", "dialect"),
57+
ConnectorFile("connectionResolver.xml", "connection-resolver"),
58+
ConnectorFile("connectionProperties.js", "script")]
59+
60+
actual_file_list, actual_class_name = self.parser_test_case(TEST_FOLDER / Path("modular_dialog_connector"),
61+
expected_file_list, expected_class_name)
62+
63+
self.assertTrue(actual_file_list, "Valid connector did not return a file list")
64+
self.assertTrue(sorted(actual_file_list) == sorted(expected_file_list),
65+
"Actual file list does not match expected for valid connector")
66+
self.assertTrue(actual_class_name == expected_class_name,
67+
"Actual class name does not match expected for valid connector")
68+
4869
def parser_test_case(self, test_folder, expected_file_list, expected_class_name):
4970

5071
xml_parser = XMLParser(test_folder)

0 commit comments

Comments
 (0)