Update CSV output for PerfLab (#1030)

This PR adds the following CLI flags:

--perf-run: takes no arguments; used for data sources that do not have RunAsPerf set in their .ini file.
--iteration: takes a single integer argument; if unspecified, defaults to 0.

Author: lukewrites

README.md

@@ -20,7 +20,7 @@ The latest version of the SDK is always targeted towards the latest, non-beta ve
 | Connector SDK for Tableau 2020.1                 | 5-08-2020           |
 | Connector SDK for Tableau 2019.4                 | 3-13-2020           |
 | Connector Packager SDK (Beta) for Tableau 2019.3 | 12-11-2019          |
-| TDVT                                             | 2.6.2 (03-22-2022)  |
+| TDVT                                             | 2.7.0 (10-28-2022)  |
 |                                                  | 1.5.24 (04-13-2020) |
 | Connector Packager                               | 2.1.0 (05-08-2020)  |
 

tdvt/CHANGELOG.md

@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.7.0]. - 2022-10-28
+- Add a `--perf-run` flag that changes the output of `tdvt_log_combined.csv`.
+
 ## [2.6.2] - 2022-03-22
 - Update file encoding of setup.string.regex.icu_fallback.txt to utf-8.
 

tdvt/tdvt/config_gen/datasource_list.py

@@ -7,6 +7,7 @@
 import glob
 import os.path
 import logging
+from typing import List, Optional
 
 from .gentests import list_configs, list_config
 from ..resources import *
@@ -407,12 +408,12 @@ def interpret_ds_list(self, ds_list, built_list=None):
     def add_test(self, test_config):
         self.dsnames[test_config.dsname] = test_config
 
-    def get_datasource_info(self, dsname):
+    def get_datasource_info(self, dsname) -> Optional[TestConfig]:
         if dsname in self.dsnames:
             return self.dsnames[dsname]
         return None
 
-    def get_datasources(self, suite):
+    def get_datasources(self, suite) -> Optional[List]:
         ds_to_run = []
         if not suite:
             return

tdvt/tdvt/config_gen/tdvtconfig.py

@@ -30,6 +30,7 @@ def __init__(self, from_args=None, from_json=None, test_config: TestConfig = Non
         self.tds = ''
         self.tested_run_time_config = None
         self.tabquery_path = ''
+        self.iteration: int = 0
 
         if from_args:
             self.init_from_args(from_args)
@@ -42,7 +43,7 @@ def __init__(self, from_args=None, from_json=None, test_config: TestConfig = Non
     def set_run_time_test_config(self, rtt: RunTimeTestConfig):
         self.timeout_seconds = rtt.timeout_seconds
         self.d_override = rtt.d_override
-        self.run_as_perf = rtt.run_as_perf
+        self.run_as_perf = rtt.run_as_perf or self.run_as_perf
         self.tested_run_time_config = rtt
         if rtt.tabquery_paths:
             self.tabquery_path = rtt.tabquery_paths.to_array()
@@ -60,6 +61,10 @@ def init_from_args(self, args):
             self.verbose = args.verbose
         if args.custom_output_dir:
             self.custom_output_dir = args.custom_output_dir
+        if args.perf_run:
+            self.run_as_perf = True
+        if args.perf_iteration:
+            self.iteration = args.perf_iteration
 
     def init_from_json(self, json):
         self.tested_sql = json.get('tested_sql', self.tested_sql)

tdvt/tdvt/constants.py

@@ -0,0 +1,41 @@
+DEFAULT_CSV_HEADERS = [
+    'Suite',
+    'Test Set',
+    'TDSName',
+    'TestName',
+    'TestPath',
+    'Passed',
+    'Closest Expected',
+    'Diff count',
+    'Test Case',
+    'Test Type',
+    'Priority',
+    'Categories',
+    'Functions',
+    'Process Output',
+    'Error Msg',
+    'Error Type',
+    'Query Time (ms)',
+    'Generated SQL',
+]
+
+PERFLAB_CSV_HEADERS = [
+    "TestGroup",
+    "TestSubGroup",
+    "Test",
+    "TestComment1",
+    "TestComment2",
+    "TestComment3",
+    "Iteration",
+    "IterationStartTime",
+    "IterationEndTime",
+    "ErrorString",
+    "IterationComment1",
+    "IterationComment2",
+    "IterationComment3",
+    "MetricResourceType",
+    "MetricResourceInstance",
+    "Result"
+]
+
+TUPLE_DISPLAY_LIMIT = 100

tdvt/tdvt/tabquery.py

@@ -1,5 +1,4 @@
 import configparser
-import os
 import sys
 
 from .resources import *
@@ -38,14 +37,15 @@ def get_max_process_level_of_parallelization(desired_threads):
         return 1
     return desired_threads
 
+
 def build_tabquery_command_line(work):
     try:
         sys.path.insert(0, get_extensions_dir())
         from extend_tabquery import TabqueryCommandLineExtension
         sys.path.pop(0)
         tb = TabqueryCommandLineExtension()
         logging.debug("Imported extension extend_tabquery")
-    except:
+    except ImportError:
         tb = TabqueryCommandLine()
 
     cmdline = tb.build_tabquery_command_line(work)

tdvt/tdvt/tdvt.py

@@ -2,6 +2,7 @@
     Test driver script for the Tableau Datasource Verification Tool
 
 """
+
 import sys
 
 if sys.version_info[0] < 3:
@@ -21,12 +22,12 @@
 from pathlib import Path
 from typing import List, Optional, Tuple, Union
 
-from .config_gen.datasource_list import print_ds, print_configurations, print_logical_configurations
+from .config_gen.datasource_list import TestRegistry, print_ds, print_configurations, print_logical_configurations
 from .config_gen.tdvtconfig import TdvtInvocation
 from .config_gen.test_config import TestSet, SingleLogicalTestSet, SingleExpressionTestSet, FileTestSet, TestConfig, RunTimeTestConfig
 from .setup_env import create_test_environment, add_datasource
 from .tabquery import *
-from .tdvt_core import generate_files, run_diff, run_tests, run_connectors_test_core
+from .tdvt_core import generate_files, run_diff, run_tests, run_connectors_test_core, return_csv_dialect
 from .version import __version__
 
 # This contains the dictionary of configs you can run.
@@ -45,43 +46,46 @@ class TestOutputFiles(object):
     combined_output = []
 
     @classmethod
-    def copy_output_file(c, src_name, src_dir):
+    def copy_output_file(cls, src_name, src_dir, csv_dialect):
         src = os.path.join(src_dir, src_name)
         logging.debug("Copying {0} to output".format(src))
         try:
             with open(src, 'r', encoding='utf8') as src_file:
-                reader = csv.DictReader(src_file, dialect='tdvt')
+                reader = csv.DictReader(src_file, dialect=csv_dialect)
                 for row in reader:
-                    c.combined_output.append(row)
+                    cls.combined_output.append(row)
 
         except IOError as e:
             logging.debug("Exception while copying files: " + str(e))
             return
 
     @classmethod
-    def write_test_results_csv(c, custom_output_dir: str=''):
-        if not c.combined_output:
+    def write_test_results_csv(cls, perf_run: bool, custom_output_dir: str = ''):
+        if not cls.combined_output:
             logging.debug("write_test_results_csv called with no test output")
             return
 
-        logging.debug("Copying output to {0}".format(c.output_csv))
+        logging.debug("Copying output to {0}".format(cls.output_csv))
         # Sort combined_output on the number of distinct functions (order of complexity)
         sort_by_complexity = lambda row: len(row['Functions'].split(','))
         try:
-            c.combined_output.sort(key=sort_by_complexity)
+            cls.combined_output.sort(key=sort_by_complexity)
         except KeyError as e:
             logging.debug("Tried to sort output on a key that doesn't exist. Leaving output unsorted.")
 
-        dst = os.path.join(os.getcwd(), c.output_csv)
+        dst = os.path.join(os.getcwd(), cls.output_csv)
         if custom_output_dir != '':
-            dst = os.path.join(Path(custom_output_dir), c.output_csv)
+            dst = os.path.join(Path(custom_output_dir), cls.output_csv)
         try:
-            dst_exists = os.path.isfile(dst)
             with open(dst, 'w', encoding='utf8') as dst_file:
-                writer = csv.DictWriter(dst_file, fieldnames=c.combined_output[0],
-                    dialect='tdvt', quoting=csv.QUOTE_MINIMAL)
+                writer = csv.DictWriter(
+                    dst_file,
+                    fieldnames=cls.combined_output[0],
+                    dialect=return_csv_dialect(perf_run),
+                    quoting=csv.QUOTE_MINIMAL
+                )
                 writer.writeheader()
-                for row in c.combined_output:
+                for row in cls.combined_output:
                     writer.writerow(row)
         except IOError as e:
             logging.debug("Exception while writing to file: " + str(e))
@@ -95,7 +99,7 @@ def do_test_queue_work(i, q):
     abort_test_run = False
     while True:
         # This blocks if the queue is empty.
-        work = q.get()
+        work: TestRunner = q.get()
 
         work.run()
 
@@ -135,8 +139,8 @@ def copy_files_to_zip(self, dst_file_name, src_dir, is_logs):
                 inner_output = os.path.join(optional_dir_name, file_to_be_zipped)
                 myzip.write(actual, inner_output)
 
-    def copy_output_files(self):
-        TestOutputFiles.copy_output_file("test_results.csv", self.temp_dir)
+    def copy_output_files(self, csv_dialect):
+        TestOutputFiles.copy_output_file("test_results.csv", self.temp_dir, csv_dialect)
 
     def copy_test_result_file(self):
         src = os.path.join(self.temp_dir, "tdvt_output.json")
@@ -179,10 +183,12 @@ def copy_test_result_file(self):
 
     def copy_files_and_cleanup(self):
         left_temp_dir = False
+        csv_dialect = 'perflab' if self.test_config.run_as_perf else 'tdvt'
+        # if
         try:
             self.copy_files_to_zip(TestOutputFiles.output_actuals, self.temp_dir, is_logs=False)
             self.copy_files_to_zip(TestOutputFiles.output_tabquery_log, self.temp_dir, is_logs=True)
-            self.copy_output_files()
+            self.copy_output_files(csv_dialect)
             self.copy_test_result_file()
         except Exception as e:
             print(e)
@@ -243,7 +249,9 @@ def get_datasource_registry(platform):
     return reg
 
 
-def enqueue_single_test(args, ds_info: TestConfig, suite) -> Union[Tuple[None, None], Tuple[Union[SingleLogicalTestSet, SingleExpressionTestSet], TdvtInvocation]]:  # noqa: E501
+def enqueue_single_test(
+        args, ds_info: TestConfig, suite
+) -> Union[Tuple[None, None], Tuple[Union[SingleLogicalTestSet, SingleExpressionTestSet], TdvtInvocation]]:
     if not args.command == 'run-pattern' or not args.tds_pattern or (args.logical_pattern and args.expression_pattern):
         return None, None
 
@@ -483,6 +491,8 @@ def create_parser():
     run_test_common_parser.add_argument('--output-dir', '-o', dest='custom_output_dir',
                                         help='Writes log files to a specified directory. The directory must exist.',
                                         required=False, default=None, const='*', nargs='?')
+    run_test_common_parser.add_argument('--perf-run', dest='perf_run', action='store_true', default=False)
+    run_test_common_parser.add_argument('--iteration', dest='perf_iteration', type=int)
     subparsers = parser.add_subparsers(help='commands', dest='command')
 
     #Get information.
@@ -507,7 +517,6 @@ def create_parser():
     run_test_parser.add_argument('--logical', '-q', dest='logical_only', help='Only run logical tests whose config file name matches the supplied string, or all if blank.', required=False, default=None, const='*', nargs='?')
     run_test_parser.add_argument('--expression', '-e', dest='expression_only', help='Only run expression tests whose config file name matches the suppled string, or all if blank.', required=False, default=None, const='*', nargs='?')
 
-
     #Run test pattern.
     run_test_pattern_parser = subparsers.add_parser('run-pattern', help='Run individual tests using a pattern.', parents=[run_test_common_parser], usage=run_pattern_usage_text)
     run_test_pattern_parser.add_argument('ds', help='Comma separated list of Datasource names or groups to test. See the \'list\' command.', nargs='+')
@@ -540,6 +549,14 @@ def register_tdvt_dialect():
     custom_dialect.skipinitialspace = True
     csv.register_dialect('tdvt', custom_dialect)
 
+def register_perflab_dialect():
+    custom_dialect = csv.excel
+    custom_dialect.lineterminator = '\n'
+    custom_dialect.delimiter = '|'
+    custom_dialect.strict = True
+    custom_dialect.skipinitialspace = True
+    csv.register_dialect('perflab', custom_dialect)
+
 def check_if_custom_output_dir_exists(custom_output_dir: str) -> bool:
     return Path(custom_output_dir).is_dir()
 
@@ -576,6 +593,7 @@ def init():
     ds_reg = get_datasource_registry(sys.platform)
     configure_tabquery_path()
     register_tdvt_dialect()
+    register_perflab_dialect()
 
     return parser, ds_reg, args
 
@@ -590,7 +608,7 @@ def active_thread_count(threads):
     return active
 
 
-def test_runner(all_tests, test_queue, max_threads):
+def test_runner(all_tests: List[TestRunner], test_queue: queue.Queue, max_threads: int) -> Tuple[int, int, int, int]:
     for i in range(0, max_threads):
         worker = threading.Thread(target=do_test_queue_work, args=(i, test_queue))
         worker.setDaemon(True)
@@ -607,11 +625,17 @@ def test_runner(all_tests, test_queue, max_threads):
         skipped_tests += work.skipped_tests if work.skipped_tests else 0
         disabled_tests += work.disabled_tests if work.disabled_tests else 0
         total_tests += work.total_tests if work.total_tests else 0
-    TestOutputFiles.write_test_results_csv(work.test_config.custom_output_dir)
+    is_perf_run = all_tests[0].test_config.run_as_perf
+    custom_output_dir = all_tests[0].test_config.custom_output_dir
+    TestOutputFiles.write_test_results_csv(is_perf_run, custom_output_dir)
     return failed_tests, skipped_tests, disabled_tests, total_tests
 
 
-def run_tests_impl(tests: List[Tuple[TestSet, TestConfig]], max_threads: int, args) -> Optional[Tuple[int, int, int, int]]:
+def run_tests_impl(
+    tests: List[Tuple[TestSet, TdvtInvocation]],
+    max_threads: int,
+    args
+) -> Optional[Tuple[int, int, int, int]]:
     if not tests:
         print("No tests found. Check arguments.")
         sys.exit()
@@ -661,7 +685,8 @@ def run_tests_impl(tests: List[Tuple[TestSet, TestConfig]], max_threads: int, ar
         print("Starting smoke tests. Creating", str(smoke_test_threads), "worker threads.\n")
 
         failed_smoke_tests, skipped_smoke_tests, disabled_smoke_tests, total_smoke_tests = test_runner(
-            smoke_tests, smoke_test_queue, smoke_test_threads)
+            smoke_tests, smoke_test_queue, smoke_test_threads
+        )
 
         smoke_tests_run = total_smoke_tests - disabled_smoke_tests
 
@@ -727,7 +752,7 @@ def get_ds_list(ds):
     ds_list = [x.strip() for x in ds_list]
     return ds_list
 
-def run_desired_tests(args, ds_registry):
+def run_desired_tests(args, ds_registry: TestRegistry):
     generate_files(ds_registry, False)
     ds_to_run = ds_registry.get_datasources(get_ds_list(args.ds))
     if not ds_to_run:
@@ -744,7 +769,7 @@ def run_desired_tests(args, ds_registry):
         sys.exit(0)
 
     max_threads = get_level_of_parallelization(args)
-    test_sets: List[TestSet] = []
+    test_sets: List[Tuple[TestSet, TestConfig], Tuple[TestSet, TdvtInvocation]] = []
 
     for ds in ds_to_run:
         ds_info = ds_registry.get_datasource_info(ds)