Merge remote-tracking branch 'origin/next'

Author: thomas-vogels

.github/dependabot.yaml

@@ -4,6 +4,8 @@ updates:
   - assignees:
       - "thomas-vogels"
     directory: "/"
+    ignore:
+      - dependency-name: "Sphinx"
     open-pull-requests-limit: 10
     package-ecosystem: "pip"
     reviewers:

.github/workflows/lint.yaml

@@ -18,11 +18,11 @@ jobs:
     - name: Checkout out code
       uses: actions/checkout@v3
     - name: Set up Python 3.7
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v3.0.2
+      uses: actions/cache@v3.0.8
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }}
@@ -47,11 +47,11 @@ jobs:
     - name: Checkout out code
       uses: actions/checkout@v3
     - name: Set up Python 3.7
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v3.0.2
+      uses: actions/cache@v3.0.8
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-linters.txt') }}

.github/workflows/unittest.yaml

@@ -16,11 +16,11 @@ jobs:
     - name: Checkout out code
       uses: actions/checkout@v3
     - name: Set up Python 3.7
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.7
     - name: Cache pip
-      uses: actions/cache@v3.0.2
+      uses: actions/cache@v3.0.8
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements-*.txt') }}

.pre-commit-config.yaml

@@ -39,29 +39,29 @@ repos:
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 22.8.0
     hooks:
       - id: black
   - repo: https://gitlab.com/pycqa/flake8
-    rev: 4.0.1
+    rev: 5.0.4
     hooks:
       - id: flake8
         additional_dependencies:
-          - "flake8-bugbear==22.4.25"
-          - "flake8-comprehensions==3.7.0"
+          - "flake8-bugbear==22.9.11"
+          - "flake8-comprehensions==3.10.0"
           - "flake8-docstrings==1.6.0"
           - "flake8-fixme==1.1.1"
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.950
+    rev: v0.971
     hooks:
       - id: mypy
         additional_dependencies:
-          - "mypy-boto3~=1.22"
-          - "types-PyYAML==6.0.7"
-          - "types-setuptools==57.4.14"
-          - "types-simplejson==3.17.6"
-          - "types-tabulate==0.8.8"
-          - "types-termcolor==1.1.4"
+          - "mypy-boto3~=1.24"
+          - "types-PyYAML==6.0.11"
+          - "types-setuptools==65.3.0"
+          - "types-simplejson==3.17.7"
+          - "types-tabulate==0.8.11"
+          - "types-termcolor==1.1.5"
   - repo: https://github.com/shellcheck-py/shellcheck-py
     rev: v0.7.2.1
     hooks:

bin/build_virtual_env

@@ -23,6 +23,7 @@ source "$ARTHUR_VENV/bin/activate"
 echo "Creating or updating packages using pip..."
 python3 -m pip install --upgrade pip==20.3.4 --disable-pip-version-check
 python3 -m pip install --upgrade --requirement ./requirements-all.txt --disable-pip-version-check
+python3 setup.py develop
 
 echo "To use this setup, you need to run:"
 echo "  source '$ARTHUR_VENV/bin/activate'"

bin/send_health_check.sh

@@ -29,6 +29,7 @@ set -o errexit
 FILENAME="/tmp/redshift_etl/config/environment.sh"
 echo "Attempting to read configuration environment variables from '$FILENAME'"
 if [ -r "$FILENAME" ]; then
+    # shellcheck disable=SC1090
     source "$FILENAME"
 else
     echo "Failed to find '$FILENAME'"

cloudformation/dw_cluster.yaml

@@ -74,6 +74,11 @@ Parameters:
         Type: String
         Default: ""
 
+    AdditionalClusterIAMRole6:
+        Description: (optional) ARN of an additional IAM role to associate with the Redshift cluster
+        Type: String
+        Default: ""
+
     PreferredMaintenanceWindow:
         Description: (optional) Preferred maintenance window for the Redshift cluster
         Type: String
@@ -102,6 +107,9 @@ Conditions:
     HasAdditionalRole5:
         !Not [ !Equals [ !Ref "AdditionalClusterIAMRole5", "" ] ]
 
+    HasAdditionalRole6:
+        !Not [ !Equals [ !Ref "AdditionalClusterIAMRole6", "" ] ]
+
 
 Resources:
 
@@ -121,12 +129,26 @@ Resources:
             Description: "Parameter group for Redshift cluster"
             ParameterGroupFamily: "redshift-1.0"
             Parameters:
+                - ParemeterName: "auto_analyze"
+                  ParemeterValue: "true"
+                - ParemeterName: "auto_mv"
+                  ParemeterValue: "true"
+                - ParemeterName: "datestyle	"
+                  ParemeterValue: "ISO, MDY"
                 - ParameterName: "require_ssl"
                   ParameterValue: "true"
+                - ParameterName: "enable_case_sensitive_identifier"
+                  ParameterValue: "false"
                 - ParameterName: "enable_user_activity_logging"
                   ParameterValue: "true"
+                - ParameterName: "extra_float_digits"
+                  ParameterValue: 0
+                - ParameterName: "max_concurrency_scaling_clusters"
+                  ParameterValue: 1
+                - ParameterName: "search_path"
+                  ParameterValue: "$user, public"
                 - ParameterName: "statement_timeout"
-                  ParameterValue: 7200000
+                  ParameterValue: 0
                 - ParameterName: "wlm_json_configuration"
                   ParameterValue: !Sub "[{ \"query_concurrency\": ${QueryConcurrency} }]"
             Tags:
@@ -162,6 +184,7 @@ Resources:
                 - !If [ HasAdditionalRole3, !Ref "AdditionalClusterIAMRole3", !Ref "AWS::NoValue" ]
                 - !If [ HasAdditionalRole4, !Ref "AdditionalClusterIAMRole4", !Ref "AWS::NoValue" ]
                 - !If [ HasAdditionalRole5, !Ref "AdditionalClusterIAMRole5", !Ref "AWS::NoValue" ]
+                - !If [ HasAdditionalRole6, !Ref "AdditionalClusterIAMRole6", !Ref "AWS::NoValue" ]
             MasterUsername:
                 !Ref MasterUsername
             MasterUserPassword:

docker/.bash_profile

@@ -1,12 +1,16 @@
+#!/bin/bash
+
 # This is the default .bash_profile in a container.
 
 # Get the aliases and functions
 if [ -f ~/.bashrc ]; then
-    . ~/.bashrc
+    # shellcheck disable=SC1090
+    source ~/.bashrc
 fi
 
 # Initial greeting
 cat /opt/src/arthur-redshift-etl/etc/motd
+
 echo -e "\nEnvironment settings:\n"
 arthur.py settings object_store.s3.* version
 echo

docker/.bashrc

@@ -1,8 +1,11 @@
+#!/bin/bash
+
 # This is the default .bashrc file inside a container.
 
 # Source global definitions
 if [ -f /etc/bashrc ]; then
-    . /etc/bashrc
+    # shellcheck disable=SC1091
+    source /etc/bashrc
 fi
 
 # Generally useful
@@ -15,8 +18,9 @@ alias develop="( \cd /opt/src/arthur-redshift-etl && python setup.py develop )"
 PS1='(aws:$AWS_PROFILE, prefix:$ARTHUR_DEFAULT_PREFIX) \$ '
 
 if [[ -z "$VIRTUAL_ENV" ]]; then
+    # shellcheck disable=SC1091
     source /opt/local/redshift_etl/venv/bin/activate
 fi
 
-# Commandline completion
+# shellcheck disable=SC1091
 source /opt/src/arthur-redshift-etl/etc/arthur_completion.sh

docker/entrypoint.sh

@@ -19,6 +19,7 @@ set -o errexit -o nounset
 export PATH="/opt/local/redshift_etl/bin:$PATH"
 
 if [[ -r "/opt/local/redshift_etl/venv/bin/activate" ]]; then
+    # shellcheck disable=SC1091
     source /opt/local/redshift_etl/venv/bin/activate
 
     # Using "--quiet" here to reduce the startup noise for "end users."

docs/source/conf.py

@@ -23,7 +23,7 @@
 author = "Data Engineering at Harry's"
 
 # TODO(tom): Extract from setup.py
-version = "1.59"
+version = "1.64"
 
 # -- General configuration ---------------------------------------------------
 

requirements-linters.txt

@@ -1,18 +1,18 @@
 # NOTE When changing versions here, be sure to keep .pre-commit-config.yaml consistent!
-black==22.3.0
-flake8==4.0.1
-flake8-bugbear==22.4.25
-flake8-comprehensions==3.7.0
+black==22.8.0
+flake8==5.0.4
+flake8-bugbear==22.9.11
+flake8-comprehensions==3.10.0
 flake8-docstrings==1.6.0
 flake8-fixme==1.1.1
 isort==5.10.1
-mypy==0.950
-mypy-boto3~=1.22
+mypy==0.971
+mypy-boto3~=1.24
 mypy-extensions==0.4.3
 pre-commit==2.17.0
 pyupgrade==2.31.0
-types-PyYAML==6.0.7
-types-setuptools==57.4.14
-types-simplejson==3.17.6
-types-tabulate==0.8.8
-types-termcolor==1.1.4
+types-PyYAML==6.0.11
+types-setuptools==65.3.0
+types-simplejson==3.17.7
+types-tabulate==0.8.11
+types-termcolor==1.1.5

requirements.txt

@@ -1,7 +1,7 @@
-# This is the set of packages need to run the code. All versions are pinned.
-arrow==1.2.2
-boto3==1.22.4
-botocore~=1.25
+# This is the set of packages needed to run the code. All versions are pinned.
+arrow==1.2.3
+boto3==1.23.10
+botocore~=1.26
 docker==5.0.3
 funcy==1.17
 jmespath==0.10.0
@@ -10,7 +10,7 @@ pgpasslib==1.1.0
 psycopg2-binary==2.9.3
 PyYAML==6.0
 simplejson==3.17.6
-tabulate==0.8.9
+tabulate==0.8.10
 termcolor==1.1.0
-tqdm==4.64.0
+tqdm==4.64.1
 watchtower==3.0.0

setup.py

@@ -1,6 +1,6 @@
 from setuptools import find_packages, setup
 
-ARTHUR_VERSION = "1.63.0"
+ARTHUR_VERSION = "1.64.0"
 
 
 setup(