Merge branch 'master' into ldap_integration

Author: hnanchahal

.circleci/config.yml

@@ -276,4 +276,4 @@ experimental:
 
 notify:
   webhooks:
-    - url: https://ci-webhooks.stackstorm.net/webhooks/build/events
+    - url: https://ci-webhooks.stackstorm.com/webhooks/build/events

.travis.yml

@@ -38,15 +38,15 @@ jobs:
     # scripts/travis/run-nightly-make-task-if-exists.sh
     - name: "Unit Tests (Python 2.7 MongoDB 4.0)"
       python: 2.7
-      env: TASK=ci-unit CACHE_NAME=py2 COMMAND_THRESHOLD=700
+      env: TASK=ci-unit CACHE_NAME=py2 PYTHON_VERSION=python2.7 COMMAND_THRESHOLD=700
 
     - name: "Integration Tests (Python 2.7)"
       python: 2.7
-      env: TASK=ci-integration CACHE_NAME=py2 COMMAND_THRESHOLD=700
+      env: TASK=ci-integration CACHE_NAME=py2 PYTHON_VERSION=python2.7 COMMAND_THRESHOLD=700
 
-    - name: "Lint Checks, Packs Tests (Python 2.7)"
-      python: 2.7
-      env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py2 COMMAND_THRESHOLD=430
+    - name: "Lint Checks, Packs Tests (Python 3.6)"
+      python: 3.6
+      env: TASK="ci-checks ci-packs-tests" CACHE_NAME=py3 PYTHON_VERSION=python3.6 COMMAND_THRESHOLD=430
 
     - name: "Unit Tests, Pack Tests (Python 3.6)"
       python: 3.6

CHANGELOG.rst

@@ -4,6 +4,14 @@ Changelog
 in development
 --------------
 
+Changed
+~~~~~~~~~
+* Improve the st2-self-check script to echo to stderr and exit if it isn't run with a
+  ST2_AUTH_TOKEN or ST2_API_KEY environment variable. (improvement) #5068
+
+3.3.0 - October 06, 2020
+------------------------
+
 Added
 ~~~~~
 * Added st2-auth-ldap pip requirements for Ldap integartion. (new feature) #5082
@@ -32,6 +40,9 @@ Added
   Python 2. (new feature) #5043
 
   Contributed by @amanda11
+* Added deprecation warning to st2ctl, if st2 python version is Python 2. (new feature) #5044 
+
+  Contributed by @amanda11
 
 
 Changed
@@ -47,6 +58,14 @@ Changed
   Contributed by Justin Sostre (@saucetray)
 * The built-in ``st2.action.file_writen`` trigger has been renamed to ``st2.action.file_written``
   to fix the typo (bug fix) #4992
+* Renamed reference to the RBAC backend/plugin from ``enterprise`` to ``default``. Updated st2api
+  validation to use the new value when checking RBAC configuration. Removed other references to
+  enterprise for RBAC related contents. (improvement)
+* Remove authentication headers ``St2-Api-Key``, ``X-Auth-Token`` and ``Cookie`` from webhook payloads to
+  prevent them from being stored in the database. (security bug fix) #4983
+
+  Contributed by @potato and @knagy
+* Updated orquesta to version v1.2.0.
 
 Fixed
 ~~~~~
@@ -89,6 +108,21 @@ Fixed
 * Fixed a regression in the ``linux.dig`` action on Python 3. (bug fix) #4993
 
   Contributed by @blag
+* Fixed a bug in pack installation logging code where unicode strings were not being
+  interpolated properly. (bug fix)
+
+  Contributed by @misterpah
+* Fixed a compatibility issue with the latest version of the ``logging`` library API
+  where the ``find_caller()`` function introduced some new variables. (bug fix) #4923
+
+  Contributed by @Dahfizz9897
+* Fixed another logging compatibility issue with the ``logging`` API in Python 3.
+  The return from the ``logging.findCaller()`` implementation now expects a 4-element
+  tuple. Also, in Python 3 there are new arguments that are passed in and needs to be
+  acted upon, specificall ``stack_info`` that determines the new 4th element in the returned
+  tuple. (bug fix) #5057
+
+  Contributed by Nick Maludy (@nmaludy Encore Technologies)
 
 Removed
 ~~~~~~~

Makefile

@@ -15,7 +15,7 @@ else
 	VIRTUALENV_COMPONENTS_DIR ?= virtualenv-components
 endif
 
-PYTHON_VERSION ?= python2.7
+PYTHON_VERSION ?= $(shell if [ -z "`which python3.6`" ]; then echo "python2.7"; else echo "python3.6"; fi)
 
 BINARIES := bin
 
@@ -50,7 +50,12 @@ COMPONENTS_TEST_MODULES_COMMA := $(subst $(space_char),$(comma),$(COMPONENTS_TES
 COVERAGE_GLOBS := .coverage.unit.* .coverage.integration.*
 COVERAGE_GLOBS_QUOTED := $(foreach glob,$(COVERAGE_GLOBS),'$(glob)')
 
-REQUIREMENTS := test-requirements.txt requirements.txt
+ifeq ($(PYTHON_VERSION),python2.7)
+	REQUIREMENTS := test-requirements-py27.txt requirements.txt
+else
+	REQUIREMENTS := test-requirements.txt requirements.txt
+endif
+
 # Pin common pip version here across all the targets
 # Note! Periodic maintenance pip upgrades are required to be up-to-date with the latest pip security fixes and updates
 PIP_VERSION ?= 20.0.2
@@ -107,6 +112,8 @@ all: requirements configgen check tests
 # Target for debugging Makefile variable assembly
 .PHONY: play
 play:
+	@echo PYTHON_VERSION=$(PYTHON_VERSION)
+	@echo
 	@echo COVERAGE_GLOBS=$(COVERAGE_GLOBS_QUOTED)
 	@echo
 	@echo COMPONENTS=$(COMPONENTS)
@@ -226,8 +233,7 @@ check-python-packages:
 	@echo ""
 	@echo "================== CHECK PYTHON PACKAGES ===================="
 	@echo ""
-
-	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download
+	test -f $(VIRTUALENV_COMPONENTS_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_COMPONENTS_DIR) --no-download --system-site-packages
 	@for component in $(COMPONENTS_WITHOUT_ST2TESTS); do \
 		echo "==========================================================="; \
 		echo "Checking component:" $$component; \
@@ -263,7 +269,7 @@ checklogs:
 	@echo
 	@echo "================== LOG WATCHER ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; ./tools/log_watcher.py 10
+	. $(VIRTUALENV_DIR)/bin/activate; python ./tools/log_watcher.py 10
 
 .PHONY: pylint
 pylint: requirements .pylint
@@ -329,7 +335,7 @@ lint-api-spec: requirements .lint-api-spec
 	@echo
 	@echo "================== Lint API spec ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf
+	. $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf
 
 .PHONY: generate-api-spec
 generate-api-spec: requirements .generate-api-spec
@@ -343,14 +349,14 @@ generate-api-spec: requirements .generate-api-spec
 	echo "# Edit st2common/st2common/openapi.yaml.j2 and then run" >> st2common/st2common/openapi.yaml
 	echo "# make .generate-api-spec" >> st2common/st2common/openapi.yaml
 	echo "# to generate the final spec file" >> st2common/st2common/openapi.yaml
-	. $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml
+	. $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-generate-api-spec --config-file conf/st2.dev.conf >> st2common/st2common/openapi.yaml
 
 .PHONY: circle-lint-api-spec
 circle-lint-api-spec:
 	@echo
 	@echo "================== Lint API spec ===================="
 	@echo
-	. $(VIRTUALENV_DIR)/bin/activate; st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf || echo "Open API spec lint failed."
+	. $(VIRTUALENV_DIR)/bin/activate; python st2common/bin/st2-validate-api-spec --config-file conf/st2.dev.conf || echo "Open API spec lint failed."
 
 .PHONY: flake8
 flake8: requirements .flake8
@@ -375,7 +381,7 @@ flake8: requirements .flake8
 	@echo
 	@echo "==================== st2client install check ===================="
 	@echo
-	test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --no-download
+	test -f $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_ST2CLIENT_DIR) --no-download --system-site-packages
 
 	# Setup PYTHONPATH in bash activate script...
 	# Delete existing entries (if any)
@@ -389,11 +395,12 @@ flake8: requirements .flake8
 	touch $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 	chmod +x $(VIRTUALENV_ST2CLIENT_DIR)/bin/activate
 
-	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
 	# NOTE We need to upgrade setuptools to avoid bug with dependency resolving in old versions
 	# Setuptools 42 added support for python_requires, which is used by the configparser package,
 	# which is required by the importlib-metadata package
+	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "pip==$(PIP_VERSION)"
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/pip install --upgrade "setuptools==44.1.0"
+
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/activate; cd st2client ; ../$(VIRTUALENV_ST2CLIENT_DIR)/bin/python setup.py install ; cd ..
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/st2 --version
 	$(VIRTUALENV_ST2CLIENT_DIR)/bin/python -c "import st2client"
@@ -541,10 +548,9 @@ requirements: virtualenv .requirements .sdist-requirements install-runners insta
 	fi
 
 	# Install requirements
-	#
 	for req in $(REQUIREMENTS); do \
-			echo "Installing $$req..." ; \
-			$(VIRTUALENV_DIR)/bin/pip install $(PIP_OPTIONS) -r $$req ; \
+		echo "Installing $$req..." ; \
+		$(VIRTUALENV_DIR)/bin/pip install $(PIP_OPTIONS) -r $$req ; \
 	done
 
 	# Install st2common package to load drivers defined in st2common setup.py
@@ -626,7 +632,7 @@ endif
 tests: pytests
 
 .PHONY: pytests
-pytests: compile requirements .flake8 .pylint .pytests-coverage
+pytests: compilepy3 requirements .flake8 .pylint .pytests-coverage
 
 .PHONY: .pytests
 .pytests: compile .configgen .generate-api-spec .unit-tests clean
@@ -963,7 +969,7 @@ debs:
 ci: ci-checks ci-unit ci-integration ci-packs-tests
 
 .PHONY: ci-checks
-ci-checks: compile .generated-files-check .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
+ci-checks: compilepy3 .generated-files-check .pylint .flake8 check-requirements check-sdist-requirements .st2client-dependencies-check .st2common-circular-dependencies-check circle-lint-api-spec .rst-check .st2client-install-check check-python-packages
 
 .PHONY: ci-py3-unit
 ci-py3-unit: