Merge branch 'master' into update-readme

Author: nmaludy

CHANGELOG.rst

@@ -19,6 +19,18 @@ Added
   default pass to screen when the script completes. (improvement) #5013
 
   Contributed by @punkrokk
+* Added deprecation warning if attempt to install or download a pack that only supports
+  Python 2. (new feature) #5037
+
+  Contributed by @amanda11
+* Added deprecation warning to each StackStorm service log, if service is running with
+  Python 2. (new feature) #5043
+
+  Contributed by @amanda11
+* Added deprecation warning to st2ctl, if st2 python version is Python 2. (new feature) #5044 
+
+  Contributed by @amanda11
+
 
 Changed
 ~~~~~~~
@@ -33,6 +45,13 @@ Changed
   Contributed by Justin Sostre (@saucetray)
 * The built-in ``st2.action.file_writen`` trigger has been renamed to ``st2.action.file_written``
   to fix the typo (bug fix) #4992
+* Renamed reference to the RBAC backend/plugin from ``enterprise`` to ``default``. Updated st2api
+  validation to use the new value when checking RBAC configuration. Removed other references to
+  enterprise for RBAC related contents. (improvement)
+* Remove authentication headers ``St2-Api-Key``, ``X-Auth-Token`` and ``Cookie`` from webhook payloads to
+  prevent them from being stored in the database. (security bug fix) #4983
+
+  Contributed by @potato and @knagy
 
 Fixed
 ~~~~~

contrib/packs/actions/get_pack_warnings.yaml

@@ -0,0 +1,13 @@
+---
+  name: "get_pack_warnings"
+  runner_type: "python-script"
+  description: "Get pack warnings from analysing pack.yaml"
+  enabled: true
+  pack: packs
+  entry_point: "pack_mgmt/get_pack_warnings.py"
+  parameters:
+    packs_status:
+      type: object
+      description: Dictionary of pack name to download status.
+      required: true
+      default: null

contrib/packs/actions/pack_mgmt/get_pack_warnings.py

@@ -0,0 +1,59 @@
+# Copyright 2020 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import six
+
+from st2common.content.utils import get_pack_base_path
+from st2common.runners.base_action import Action
+from st2common.util.pack import get_pack_metadata
+from st2common.util.pack import get_pack_warnings
+
+
+class GetPackWarnings(Action):
+    def run(self, packs_status):
+        """
+        :param packs_status: Name of the pack and download status.
+        :type: packs_status: ``dict``
+        """
+        result = {}
+        warning_list = []
+
+        if not packs_status:
+            return result
+
+        for pack, status in six.iteritems(packs_status):
+            if 'success' not in status.lower():
+                continue
+
+            warning = get_warnings(pack)
+
+            if warning:
+                warning_list.append(warning)
+
+        result['warning_list'] = warning_list
+
+        return result
+
+
+def get_warnings(pack=None):
+    result = None
+    pack_path = get_pack_base_path(pack)
+    try:
+        pack_metadata = get_pack_metadata(pack_dir=pack_path)
+        result = get_pack_warnings(pack_metadata)
+    except Exception:
+        print('Could not open pack.yaml at location %s' % pack_path)
+    finally:
+        return result

contrib/packs/actions/workflows/install.yaml

@@ -14,6 +14,7 @@ vars:
   - packs_list: null
   - dependency_list: null
   - conflict_list: null
+  - warning_list: null
   - nested: 10
   - message: ""
 
@@ -99,6 +100,16 @@ tasks:
       python3: <% ctx().python3 %>
     next:
       - when: <% succeeded() %>
+        do: get_pack_warnings
+
+  get_pack_warnings:
+    action: packs.get_pack_warnings
+    input:
+      packs_status: <% task(download_pack).result.result %>
+    next:
+      - when: <% succeeded() %>
+        publish:
+          - warning_list: <% result().result.warning_list %>
         do: register_pack
 
   register_pack:
@@ -114,3 +125,4 @@ output:
   - packs_list: <% ctx().packs_list %>
   - message: <% ctx().message %>
   - conflict_list: <% ctx().conflict_list %>
+  - warning_list: <% ctx().warning_list %>

contrib/packs/tests/test_get_pack_warnings.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+
+# Copyright 2020 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from st2tests.base import BaseActionTestCase
+from pack_mgmt.get_pack_warnings import GetPackWarnings
+
+PACK_METADATA = {
+    # Python 2 & 3
+    "py23": {
+        "version": "0.4.0",
+        "name": "py23",
+        "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-no_warnings",
+        "author": "st2-dev",
+        "keywords": ["some", "search", "another", "terms"],
+        "email": "info@stackstorm.com",
+        "description": "st2 pack to test package management pipeline",
+        "python_versions": ["2","3"],
+    },
+    # Python 3
+    "py3": {
+        "version": "0.4.0",
+        "name": "py3",
+        "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-no_warnings",
+        "author": "st2-dev",
+        "keywords": ["some", "search", "another", "terms"],
+        "email": "info@stackstorm.com",
+        "description": "st2 pack to test package management pipeline",
+        "python_versions": ["3"],
+    },
+    # Python missing
+    "pynone": {
+        "version": "0.4.0",
+        "name": "pynone",
+        "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-no_warnings",
+        "author": "st2-dev",
+        "keywords": ["some", "search", "another", "terms"],
+        "email": "info@stackstorm.com",
+        "description": "st2 pack to test package management pipeline",
+    },
+    # Python 2 only
+    "py2": {
+        "version": "0.5.0",
+        "name": "py2",
+        "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test2",
+        "author": "stanley",
+        "keywords": ["some", "special", "terms"],
+        "email": "info@stackstorm.com",
+        "description": "another st2 pack to test package management pipeline",
+        "python_versions": ["2"],
+    },
+    # Python 2 only
+    "py22": {
+        "version": "0.5.0",
+        "name": "py22",
+        "repo_url": "https://github.com/StackStorm-Exchange/stackstorm-test2",
+        "author": "stanley",
+        "keywords": ["some", "special", "terms"],
+        "email": "info@stackstorm.com",
+        "description": "another st2 pack to test package management pipeline",
+        "python_versions": ["2"]
+    }
+}
+
+def mock_get_pack_basepath(pack):
+    """
+    Mock get_pack_basepath function which just returns pack n ame
+    """
+    return pack
+
+
+def mock_get_pack_metadata(pack_dir):
+    """
+    Mock get_pack_version function which return mocked pack version
+    """
+    metadata = {}
+
+    if pack_dir in PACK_METADATA:
+        metadata = PACK_METADATA[pack_dir]
+    return metadata
+
+
+@mock.patch('pack_mgmt.get_pack_warnings.get_pack_base_path', mock_get_pack_basepath)
+@mock.patch('pack_mgmt.get_pack_warnings.get_pack_metadata', mock_get_pack_metadata)
+class GetPackWarningsTestCase(BaseActionTestCase):
+    action_cls = GetPackWarnings
+
+    def setUp(self):
+        super(GetPackWarningsTestCase, self).setUp()
+
+    def test_run_get_pack_warnings_py3_pack(self):
+        action = self.get_action_instance()
+        packs_status = {"py3": "Success."}
+
+        result = action.run(packs_status=packs_status)
+        self.assertEqual(result['warning_list'], [])
+
+    def test_run_get_pack_warnings_py2_pack(self):
+        action = self.get_action_instance()
+        packs_status = {"py2": "Success."}
+
+        result = action.run(packs_status=packs_status)
+        self.assertEqual(len(result['warning_list']), 1)
+        warning = result['warning_list'][0]
+        self.assertTrue("DEPRECATION WARNING" in warning)
+        self.assertTrue("Pack py2 only supports Python 2" in warning)
+
+    def test_run_get_pack_warnings_py23_pack(self):
+        action = self.get_action_instance()
+        packs_status = {"py23": "Success."}
+
+        result = action.run(packs_status=packs_status)
+        self.assertEqual(result['warning_list'], [])
+
+    def test_run_get_pack_warnings_pynone_pack(self):
+        action = self.get_action_instance()
+        packs_status = {"pynone": "Success."}
+
+        result = action.run(packs_status=packs_status)
+        self.assertEqual(result['warning_list'], [])
+
+    def test_run_get_pack_warnings_multiple_pack(self):
+        action = self.get_action_instance()
+        packs_status = {"py2": "Success.",
+                        "py23": "Success.",
+                        "py22": "Success."}
+
+        result = action.run(packs_status=packs_status)
+        self.assertEqual(len(result['warning_list']), 2)
+        warning0 = result['warning_list'][0]
+        warning1 = result['warning_list'][1]
+        self.assertTrue("DEPRECATION WARNING" in warning0)
+        self.assertTrue("DEPRECATION WARNING" in warning1)
+        self.assertTrue(("Pack py2 only supports Python 2" in warning0 and
+                         "Pack py22 only supports Python 2" in warning1) or 
+                         ("Pack py22 only supports Python 2" in warning0 and
+                         "Pack py2 only supports Python 2" in warning1))

contrib/runners/inquirer_runner/inquirer_runner/runner.yaml

@@ -23,7 +23,7 @@
     roles:
       default: []
       required: false
-      description: A list of roles that are permitted to respond to the action (if nothing provided, all are permitted) - REQUIRES ENTERPRISE FEATURES 
+      description: A list of roles that are permitted to respond to the action (if nothing provided, all are permitted) - REQUIRES RBAC FEATURES 
       type: array
     users:
       default: []

st2api/st2api/controllers/v1/webhooks.py

@@ -19,6 +19,7 @@
 urljoin = urlparse.urljoin
 
 from st2common import log as logging
+from st2common.constants.auth import HEADER_API_KEY_ATTRIBUTE_NAME, HEADER_ATTRIBUTE_NAME
 from st2common.constants.triggers import WEBHOOK_TRIGGER_TYPES
 from st2common.models.api.trace import TraceContext
 from st2common.models.api.trigger import TriggerAPI
@@ -126,6 +127,7 @@ def post(self, hook, webhook_body_api, headers, requester_user):
                                                           permission_type=permission_type)
 
         headers = self._get_headers_as_dict(headers)
+        headers = self._filter_authentication_headers(headers)
 
         # If webhook contains a trace-tag use that else create create a unique trace-tag.
         trace_context = self._create_trace_context(trace_tag=headers.pop(TRACE_TAG_HEADER, None),
@@ -219,6 +221,10 @@ def _get_headers_as_dict(self, headers):
             headers_dict[key] = value
         return headers_dict
 
+    def _filter_authentication_headers(self, headers):
+        auth_headers = [HEADER_API_KEY_ATTRIBUTE_NAME, HEADER_ATTRIBUTE_NAME, 'Cookie']
+        return {key: value for key, value in headers.items() if key not in auth_headers}
+
     def _log_request(self, msg, headers, body, log_method=LOG.debug):
         headers = self._get_headers_as_dict(headers)
         body = str(body)

st2api/st2api/validation.py

@@ -36,16 +36,15 @@ def validate_rbac_is_correctly_configured():
                'You can either enable authentication or disable RBAC.')
         raise ValueError(msg)
 
-    # 2. Verify enterprise backend is set
-    if cfg.CONF.rbac.backend != 'enterprise':
-        msg = ('You have enabled RBAC, but RBAC backend is not set to "enterprise". '
-               'For RBAC to work, you need to install "bwc-enterprise" package, set '
-               '"rbac.backend" config option to "enterprise" and restart st2api service.')
+    # 2. Verify default backend is set
+    if cfg.CONF.rbac.backend != 'default':
+        msg = ('You have enabled RBAC, but RBAC backend is not set to "default". '
+               'For RBAC to work, you need to install "st2-rbac-backend" package, set '
+               '"rbac.backend" config option to "default" and restart st2api service.')
         raise ValueError(msg)
 
-    # 2. Verify enterprise bits are available
-    if 'enterprise' not in available_rbac_backends:
-        msg = ('"enterprise" RBAC backend is not available. Make sure '
-               '"bwc-enterprise" and "st2-rbac-backend" system packages are '
-               'installed.')
+    # 2. Verify default RBAC backend is available
+    if 'default' not in available_rbac_backends:
+        msg = ('"default" RBAC backend is not available. Make sure '
+               '"st2-rbac-backend" system packages are installed.')
         raise ValueError(msg)

st2api/tests/unit/controllers/v1/test_webhooks.py

@@ -320,6 +320,26 @@ def get_webhook_trigger(name, url):
         self.assertTrue(controller._is_valid_hook('with_leading_trailing_slash'))
         self.assertTrue(controller._is_valid_hook('with/mixed/slash'))
 
+    @mock.patch.object(TriggerInstancePublisher, 'publish_trigger', mock.MagicMock(
+        return_value=True))
+    @mock.patch.object(WebhooksController, '_is_valid_hook', mock.MagicMock(
+        return_value=True))
+    @mock.patch.object(HooksHolder, 'get_triggers_for_hook', mock.MagicMock(
+        return_value=[DUMMY_TRIGGER_DICT]))
+    @mock.patch('st2common.transport.reactor.TriggerDispatcher.dispatch')
+    def test_authentication_headers_should_be_removed(self, dispatch_mock):
+        headers = {
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'St2-Api-Key': 'foobar',
+            'X-Auth-Token': 'deadbeaf',
+            'Cookie': 'foo=bar'
+        }
+
+        self.app.post('/v1/webhooks/git', WEBHOOK_1, headers=headers)
+        self.assertNotIn('St2-Api-Key', dispatch_mock.call_args[1]['payload']['headers'])
+        self.assertNotIn('X-Auth-Token', dispatch_mock.call_args[1]['payload']['headers'])
+        self.assertNotIn('Cookie', dispatch_mock.call_args[1]['payload']['headers'])
+
     def __do_post(self, hook, webhook, expect_errors=False, headers=None):
         return self.app.post_json('/v1/webhooks/' + hook,
                                   params=webhook,