Merge pull request #426 from gruebel/update-gha

update GHA files and add Python version test CI

Author: gruebel

.github/dependabot.yml

@@ -11,3 +11,8 @@ updates:
       interval: "daily"
       # Disable all pull requests for Docker dependencies
     open-pull-requests-limit: 0
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+

.github/workflows/bump-version.yml

@@ -7,7 +7,7 @@ jobs:
   bump-version:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
         with:
           ref: master
 

.github/workflows/ci.yml

@@ -2,18 +2,27 @@
 
 name: continuous-integration
 
-on: [push, pull_request, workflow_dispatch]
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+  pull_request:
+
+permissions:
+  contents: read
 
 jobs:
   ci:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Install dependencies
         run: |
@@ -30,3 +39,32 @@ jobs:
       - run: invoke integration.query
       - run: invoke integration.write-policy
       - run: invoke build.uninstall-package
+
+  python-version:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    strategy:
+      fail-fast: true
+      matrix:
+        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
+    steps:
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        with:
+          python-version: ${{ matrix.python }}
+          allow-prereleases: true
+
+      - name: Install dependencies
+        run: |
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+
+      - run: invoke build.install-package
+      - run: invoke integration.clean
+      - run: invoke integration.version
+      - run: invoke integration.initialize
+      - run: invoke unit.pytest
+      - run: invoke integration.query
+      - run: invoke integration.write-policy
+      - run: invoke build.uninstall-package

.github/workflows/publish.yml

@@ -11,12 +11,12 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Install dependencies
         run: |
@@ -38,11 +38,11 @@ jobs:
     needs: test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Install dependencies
         run: |
@@ -57,20 +57,19 @@ jobs:
           pip install setuptools wheel twine
           python -m setup sdist bdist_wheel
       - name: Publish package
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@b7f401de30cb6434a1e19f805ff006643653240e # v1.8.10
         with:
-          user: __token__
           password: ${{ secrets.PYPI_PASSWORD }}
 
   update-brew:
     needs: publish-package
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: 3.7
+          python-version: '3.7'
       - name: publish brew
         run: |
           sleep 5m
@@ -92,7 +91,7 @@ jobs:
     needs: update-brew
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
         with:
           ref: master
 

.github/workflows/python-dependency-updater.yml

@@ -9,16 +9,13 @@ on:
 jobs:
   python-dependency-updater:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ['3.7']
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.7'
 
       - name: Run Pyup.io Dependency updater
         run: |

.github/workflows/release-drafter.yml

@@ -2,13 +2,13 @@ name: Release Drafter
 
 on:
   push:
-    branches: [ main, master ]
+    branches: [ master ]
 
 jobs:
   update_release_draft:
     runs-on: ubuntu-latest
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"
-      - uses: release-drafter/release-drafter@v5
+      - uses: release-drafter/release-drafter@65c5fb495d1e69aa8c08a3317bc44ff8aabe9772 # v5.24.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/update.yml

@@ -11,9 +11,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Setup python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
           python-version: '3.7'
       - name: install dependencies
@@ -31,9 +31,9 @@ jobs:
           cp -rf /tmp/.policy_sentry/data/docs $(pwd)/policy_sentry/shared/data/
       - name: Set outputs
         id: vars
-        run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
       - name: PR if files were updated
-        uses: peter-evans/create-pull-request@v3
+        uses: peter-evans/create-pull-request@153407881ec5c347639a548ade7d8ad1d6740e38 #v5.0.2
         with:
           commit-message: Update database
           title: 'Updates database'
@@ -47,12 +47,12 @@ jobs:
     runs-on: ubuntu-latest
     needs: update-actions
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
 
       - name: Setup Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
         with:
-          python-version: 3.7
+          python-version: '3.7'
 
       - name: Install dependencies
         run: |

docs/contributing/testing.md

@@ -50,7 +50,6 @@ Available tasks:
                                function.
   test.lint                    Linting with `pylint` and `autopep8`
   test.security                Runs `bandit` and `safety check`
-  unit.nose                    Unit testing: Runs unit tests using `nosetests`
   unit.pytest                  Unit testing: Runs unit tests using `pytest`
 
 
@@ -64,8 +63,6 @@ invoke integration.query
 invoke integration.write-policy
 
 invoke test.security
-
-invoke unit.nose
 ```
 
 Local Unit Testing and Integration Testing:
@@ -95,89 +92,37 @@ just run that quick command on your machine.
 Running the Test Suite
 ----------------------
 
-We use [Nose](https://nose.readthedocs.io/en/latest/) for unit testing.
-All tests are placed in the `tests` folder.
+We use [pytest](https://docs.pytest.org/en//) for unit testing.
+All tests are placed in the `test` folder.
 
 -   Just run the following:
 
 ```bash
-nosetests -v
+pytest -v
 
 # This will output the print() statements in your test code
-nosetests -v --nocapture
+pytest -v --show-capture=no
 
 # This will include the debug logging statements in the test output
-nosetests -v --logging-level=DEBUG
+pytest -v --log-level=DEBUG
 ```
 
 -   Alternatively, you can use `invoke`, as mentioned above:
 
 ```bash
-invoke unit.nose
+invoke unit.pytest
 ```
 
 Output:
 
 ```text
-test_overrides_yml_config: Tests the format of the overrides yml file for the RAM service ... ok
-test_passing_overall_iam_action_override: Tests iam:CreateAccessKey ... ok
-test_get_dependent_actions_double (test_actions.ActionsTestCase) ... ok
-test_get_dependent_actions_several (test_actions.ActionsTestCase) ... ok
-test_get_dependent_actions_single (test_actions.ActionsTestCase) ... ok
-test_analyze_by_access_level: Test out calling this as a library ... ok
-test_determine_risky_actions_from_list: Test comparing requested actions to a list of risky actions ... ok
-test_get_actions_from_policy: Verify that the get_actions_from_policy function is grabbing the actions ... ok
-test_get_actions_from_policy_file_with_explicit_actions: Verify that we can get a list of actions from a ... ok
-test_get_actions_from_policy_file_with_wildcards: Verify that we can read the actions from a file, ... ok
-test_remove_actions_not_matching_access_level: Verify remove_actions_not_matching_access_level is working as expected ... ok
-test_get_findings: Ensure that finding.get_findings() combines two risk findings for one policy properly. ... ok
-test_get_findings_by_policy_name: Testing out the 'Findings' object ... ok
-test_add_s3_permissions_management_arn (test_arn_action_group.ArnActionGroupTestCase) ... ok
-test_get_policy_elements (test_arn_action_group.ArnActionGroupTestCase) ... ok
-test_update_actions_for_raw_arn_format (test_arn_action_group.ArnActionGroupTestCase) ... ok
-test_does_arn_match_case_1 (test_arns.ArnsTestCase) ... ok
-test_does_arn_match_case_2 (test_arns.ArnsTestCase) ... ok
-test_does_arn_match_case_4 (test_arns.ArnsTestCase) ... ok
-test_does_arn_match_case_5 (test_arns.ArnsTestCase) ... ok
-test_does_arn_match_case_6 (test_arns.ArnsTestCase) ... ok
-test_does_arn_match_case_bucket (test_arns.ArnsTestCase) ... ok
-test_determine_actions_to_expand: provide expanded list of actions, like ecr:* ... ok
-test_minimize_statement_actions (test_minimize_wildcard_actions.MinimizeWildcardActionsTestCase) ... ok
-test_get_action_data: Tests function that gets details on a specific IAM Action. ... ok
-test_get_actions_at_access_level_that_support_wildcard_arns_only: Test function that gets a list of ... ok
-test_get_actions_for_service: Tests function that gets a list of actions per AWS service. ... ok
-test_get_actions_matching_condition_crud_and_arn: Get a list of IAM Actions matching condition key, ... ok
-test_get_actions_matching_condition_crud_and_wildcard_arn: Get a list of IAM Actions matching condition key ... ok
-test_get_actions_matching_condition_key: Tests a function that gathers all instances in ... ok
-test_get_actions_that_support_wildcard_arns_only: Tests function that shows all ... ok
-test_get_actions_with_access_level: Tests function that gets a list of actions in a ... ok
-test_get_actions_with_arn_type_and_access_level: Tests a function that gets a list of ... ok
-test_get_all_actions_with_access_level: Get all actions with a given access level ... ok
-test_get_arn_type_details: Tests function that grabs details about a specific ARN name ... ok
-test_get_arn_types_for_service: Tests function that grabs arn_type and raw_arn pairs ... ok
-test_get_condition_key_details: Tests function that grabs details about a specific condition key ... ok
-test_get_condition_keys_for_service: Tests function that grabs a list of condition keys per service. ... ok
-test_get_raw_arns_for_service: Tests function that grabs a list of raw ARNs per service ... ok
-test_remove_actions_that_are_not_wildcard_arn_only: Tests function that removes actions from a list that ... ok
-test_actions_template (test_template.TemplateTestCase) ... ok
-test_crud_template (test_template.TemplateTestCase) ... ok
-test_actions_schema: Validates that the user-supplied YAML is working for CRUD mode ... ok
-test_actions_schema: Validates that the user-supplied YAML is working for CRUD mode ... ok
-test_print_policy_with_actions_having_dependencies (test_write_policy.WritePolicyActionsTestCase) ... ok
-test_write_policy (test_write_policy.WritePolicyCrudTestCase) ... ok
-test_write_policy_beijing: Tests ARNs with the partiion `aws-cn` instead of just `aws` ... ok
-test_write_policy_govcloud: Tests ARNs with the partition `aws-us-gov` instead of `aws` ... ok
-test_wildcard_when_not_necessary: Attempts bypass of CRUD mode wildcard-only ... ok
-test_write_actions_policy_with_library_only: Write an actions mode policy without using the command line at all (library only) ... ok
-test_write_crud_policy_with_library_only: Write an actions mode policy without using the command line at all (library only) ... ok
-test_actions_missing_actions: write-policy actions if the actions block is missing ... ok
-test_allow_missing_access_level_categories_in_cfg: write-policy when the YAML file ... ok
-test_allow_empty_access_level_categories_in_cfg: If the content of a list is an empty string, it should sysexit ... ok
-test_actions_missing_arn: write-policy actions command when YAML file block is missing an ARN ... ok
-test_actions_missing_description: write-policy when the YAML file is missing a description ... ok
-test_actions_missing_name: write-policy when the YAML file is missing a name ... ok
-
-Ran 57 tests in 2.694s
-
-OK
+test/analysis/test_analyze.py::AnalysisExpandWildcardActionsTestCase::test_a_determine_actions_to_expand_not_upper_camelcase PASSED  [  0%]
+test/analysis/test_analyze.py::AnalysisExpandWildcardActionsTestCase::test_analyze_by_access_level PASSED                            [  1%]
+test/analysis/test_analyze.py::AnalysisExpandWildcardActionsTestCase::test_analyze_statement_by_access_level PASSED                  [  2%]
+test/analysis/test_analyze.py::AnalysisExpandWildcardActionsTestCase::test_determine_actions_to_expand PASSED                        [  2%]
+test/analysis/test_analyze.py::AnalysisExpandWildcardActionsTestCase::test_gh_162 PASSED                                             [  3%]
+test/analysis/test_expand.py::PolicyExpansionTestCase::test_policy_expansion PASSED                                                  [  4%]
+...
+
+========================================================= 134 passed in 51.04s ============================================================
 ```

policy_sentry/shared/awsdocs.py

@@ -42,7 +42,7 @@ def header_matches(string, table):
 
 def get_links_from_base_actions_resources_conditions_page():
     """Gets the links from the actions, resources, and conditions keys page, and returns their filenames."""
-    html = requests.get(BASE_DOCUMENTATION_URL)
+    html = requests.get(BASE_DOCUMENTATION_URL, timeout=300)
     soup = BeautifulSoup(html.content, "html.parser")
     html_filenames = []
     for i in soup.find("div", {"class": "highlights"}).findAll("a"):
@@ -85,7 +85,7 @@ def update_html_docs_directory(html_docs_destination):
     # html_filenames = [sub.replace(".html", ".partial.html") for sub in html_filenames]
 
     for page in html_filenames:
-        response = requests.get(link_url_prefix + page, allow_redirects=False)
+        response = requests.get(link_url_prefix + page, allow_redirects=False, timeout=300)
         # Replace the CSS stuff. Basically this:
         """
         <link href='href="https://docs.aws.amazon.com/images/favicon.ico"' rel="icon" type="image/ico"/>

requirements-dev.txt

@@ -1,13 +1,12 @@
 # Unit Testing
-pytest==7.1.1
+pytest==7.4.0
 pylint==2.13.5
-nose==1.3.7
-coverage==6.3.2
+coverage==7.2.7
 # Integration tests and tasks
-invoke==1.7.0
+invoke==2.2.0
 # Security testing
 safety==1.10.3
-bandit==1.7.4
+bandit==1.7.5
 # Formatting
 black==22.3.0
 # Other? Maybe this is from the docs? Not sure.

requirements.txt

@@ -2,7 +2,7 @@
 click==8.1.2
 # Web Scraping
 beautifulsoup4==4.11.1
-requests==2.27.1
+requests==2.31.0
 # Config files and schema validation
-PyYAML==6.0
+PyYAML==6.0.1
 schema==0.7.5