modify http headers (#414)

* reworked `vhost` tests with new framework.

* parameterized vhost tests for h2 connection.

* renamed directory and files -  old name does not reflect essence of tests.

Now all tests are inherited from base class - AddHeaderBase.

These directives and other config tests are separated.

* added tests for base logic.

* added tests:
- set large header;
- multiple appending to same header;
- multiple replacing;
- add header from static table;
- add header from dynamic table;

* updated tests:
- test_overwrite_non_singular_header - singular header must not change.
- other - added more checks.

* reworked tests - old version is too hard.
Added tests:
- adding non-exist header for `..._hdr_add`;
- removing one and many headers;
- removing special headers;
- removing non-exist header;
- h2 request tests with dynamic table;
Removed  `test_multiple_replacing_same_header`, `test_overwrite_non_singular_header`, `test_overwrite_singular_header`, `test_overwrite_raw_header` tests because now it is low priority logic.

* added tests for two special headers.

* added checks for all headers.

* moved common class and removed utils.py

* test_logic.py - typo fixed.

`test_add_some_hdrs` is duplicate.Removed.

* added test with many header modifications.

* added tests with long header name and header from static table with cache enabled.

* disabled tests and decreased size of headers because response exceeds SETTINGS_MAX_HEADER_LIST_SIZE.

Author: RomanBelozerov

http2_general/test_h2_headers.py

@@ -129,6 +129,32 @@ def test_transfer_encoding_header_in_request(self):
             "400",
         )
 
+    def test_long_header_name_in_request(self):
+        """Max length for header name - 1024. See fw/http_parser.c HTTP_MAX_HDR_NAME_LEN"""
+        for length, status_code in ((1023, "200"), (1024, "200"), (1025, "400")):
+            with self.subTest(length=length, status_code=status_code):
+                self.start_all_services()
+
+                client = self.get_client("deproxy")
+                client.send_request(self.post_request + [("a" * length, "text")], status_code)
+
+    def test_long_header_name_in_response(self):
+        """Max length for header name - 1024. See fw/http_parser.c HTTP_MAX_HDR_NAME_LEN"""
+        for length, status_code in ((1023, "200"), (1024, "200"), (1025, "502")):
+            with self.subTest(length=length, status_code=status_code):
+                self.start_all_services()
+
+                client = self.get_client("deproxy")
+                server = self.get_server("deproxy")
+                server.set_response(
+                    "HTTP/1.1 200 OK\r\n"
+                    + "Date: test\r\n"
+                    + "Server: debian\r\n"
+                    + f"{'a' * length}: text\r\n"
+                    + "Content-Length: 0\r\n\r\n"
+                )
+                client.send_request(self.post_request, status_code)
+
 
 class DuplicateSingularHeader(H2Base):
     def test_two_header_as_bytes_from_dynamic_table(self):

http_general/test_headers.py

@@ -5,7 +5,7 @@
 from framework import tester
 
 __author__ = "Tempesta Technologies, Inc."
-__copyright__ = "Copyright (C) 2022 Tempesta Technologies, Inc."
+__copyright__ = "Copyright (C) 2023 Tempesta Technologies, Inc."
 __license__ = "GPL2"
 
 
@@ -291,3 +291,68 @@ def test_forwarded_and_empty_host_header(self):
             ),
             expected_status_code="400",
         )
+
+
+class TestHeadersParsing(tester.TempestaTest):
+    backends = [
+        {
+            "id": "deproxy",
+            "type": "deproxy",
+            "port": "8000",
+            "response": "static",
+            "response_content": (
+                "HTTP/1.1 200 OK\r\n"
+                + "Date: test\r\n"
+                + "Server: debian\r\n"
+                + "Content-Length: 0\r\n\r\n"
+            ),
+        }
+    ]
+
+    tempesta = {
+        "config": """
+            listen 80;
+            server ${server_ip}:8000;
+
+            block_action attack reply;
+            block_action error reply;
+        """
+    }
+
+    clients = [
+        {
+            "id": "deproxy",
+            "type": "deproxy",
+            "addr": "${tempesta_ip}",
+            "port": "80",
+        },
+    ]
+
+    def test_long_header_name_in_request(self):
+        """Max length for header name - 1024. See fw/http_parser.c HTTP_MAX_HDR_NAME_LEN"""
+        for length, status_code in ((1023, "200"), (1024, "200"), (1025, "400")):
+            with self.subTest(length=length, status_code=status_code):
+                self.start_all_services()
+
+                client = self.get_client("deproxy")
+                client.send_request(
+                    f"GET / HTTP/1.1\r\nHost: localhost\r\n{'a' * length}: text\r\n\r\n",
+                    status_code,
+                )
+
+    def test_long_header_name_in_response(self):
+        """Max length for header name - 1024. See fw/http_parser.c HTTP_MAX_HDR_NAME_LEN"""
+        for length, status_code in ((1023, "200"), (1024, "200"), (1025, "502")):
+            with self.subTest(length=length, status_code=status_code):
+                self.start_all_services()
+
+                client = self.get_client("deproxy")
+                server = self.get_server("deproxy")
+                server.set_response(
+                    "HTTP/1.1 200 OK\r\n"
+                    + "Date: test\r\n"
+                    + "Server: debian\r\n"
+                    + f"{'a' * length}: text\r\n"
+                    + "Content-Length: 0\r\n\r\n"
+                )
+                client.send_request(f"GET / HTTP/1.1\r\nHost: localhost\r\n\r\n", status_code)

t_modify_http_headers/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["test_common_config", "test_common_config_h2", "test_logic"]
+
+# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

t_modify_http_headers/test_common_config.py

@@ -0,0 +1,230 @@
+"""Functional tests for adding user difined headers."""
+
+__author__ = "Tempesta Technologies, Inc."
+__copyright__ = "Copyright (C) 2017-2023 Tempesta Technologies, Inc."
+__license__ = "GPL2"
+
+from framework import tester
+
+
+class AddHeaderBase(tester.TempestaTest):
+    tempesta = {
+        "config": """
+listen 80;
+listen 443 proto=h2;
+
+server ${server_ip}:8000;
+
+tls_certificate ${tempesta_workdir}/tempesta.crt;
+tls_certificate_key ${tempesta_workdir}/tempesta.key;
+tls_match_any_server_name;
+""",
+    }
+
+    clients = [
+        {
+            "id": "deproxy-1",
+            "type": "deproxy",
+            "addr": "${tempesta_ip}",
+            "port": "80",
+        },
+    ]
+
+    backends = [
+        {
+            "id": "deproxy",
+            "type": "deproxy",
+            "port": "8000",
+            "response": "static",
+            "response_content": (
+                "HTTP/1.1 200 OK\r\n"
+                + "Date: test\r\n"
+                + "Server: debian\r\n"
+                + "Content-Length: 0\r\n\r\n"
+            ),
+        }
+    ]
+
+    request = (
+        f"GET / HTTP/1.1\r\n"
+        + "Host: localhost\r\n"
+        + "Connection: keep-alive\r\n"
+        + "Accept: */*\r\n"
+        + "\r\n"
+    )
+    cache: bool
+    directive: str
+
+    def update_tempesta_config(self, config: str):
+        tempesta = self.get_tempesta()
+        cache = "cache 2;\ncache_fulfill * *;\n" if self.cache else "cache 0;"
+        tempesta.config.defconfig += config + "\n" + cache
+
+    def base_scenario(self, config: str, expected_headers: list):
+        client = self.get_client("deproxy-1")
+        server = self.get_server("deproxy")
+
+        self.update_tempesta_config(config=config)
+
+        self.start_all_services()
+
+        for _ in range(2 if self.cache else 1):
+            client.send_request(self.request, "200")
+
+            for header in expected_headers:
+                if self.directive in ["req_hdr_set", "req_hdr_add"]:
+                    self.assertIn(header[1], list(server.last_request.headers.find_all(header[0])))
+                    self.assertNotIn(
+                        header[1], list(client.last_response.headers.find_all(header[0]))
+                    )
+                else:
+                    self.assertIn(header[1], list(client.last_response.headers.find_all(header[0])))
+                    self.assertNotIn(
+                        header[1], list(server.last_request.headers.find_all(header[0]))
+                    )
+
+        return client, server
+
+
+class TestReqAddHeader(AddHeaderBase):
+    cache = False
+    directive = "req_hdr_add"
+    request = (
+        f"GET / HTTP/1.1\r\n"
+        + "Host: localhost\r\n"
+        + "Connection: keep-alive\r\n"
+        + "Accept: */*\r\n"
+        + "\r\n"
+    )
+
+    def test_add_some_hdrs(self):
+        client, server = self.base_scenario(
+            config=(
+                f'{self.directive} x-my-hdr "some text";\n'
+                f'{self.directive} x-my-hdr-2 "some other text";\n'
+            ),
+            expected_headers=[("x-my-hdr", "some text"), ("x-my-hdr-2", "some other text")],
+        )
+        return client, server
+
+    def test_add_some_hdrs_custom_location(self):
+        client, server = self.base_scenario(
+            config=(
+                'location prefix "/" {\n'
+                f'{self.directive} x-my-hdr "some text";\n'
+                f'{self.directive} x-my-hdr-2 "some other text";\n'
+                "}\n"
+            ),
+            expected_headers=[("x-my-hdr", "some text"), ("x-my-hdr-2", "some other text")],
+        )
+        return client, server
+
+    def test_add_hdrs_derive_config(self):
+        client, server = self.base_scenario(
+            config=(f'{self.directive} x-my-hdr "some text";\n' 'location prefix "/" {}\n'),
+            expected_headers=[("x-my-hdr", "some text")],
+        )
+        return client, server
+
+    def test_add_hdrs_override_config(self):
+        client, server = self.base_scenario(
+            config=(
+                f'{self.directive} x-my-hdr "some text";\n'
+                'location prefix "/" {\n'
+                f'{self.directive} x-my-hdr-2 "some other text";\n'
+                "}\n"
+            ),
+            expected_headers=[("x-my-hdr-2", "some other text")],
+        )
+
+        if self.directive == "req_hdr_add":
+            self.assertNotIn(("x-my-hdr", "some text"), server.last_request.headers.items())
+        else:
+            self.assertNotIn(("x-my-hdr", "some text"), client.last_response.headers.items())
+        return client, server
+
+
+class TestRespAddHeader(TestReqAddHeader):
+    cache = False
+    directive = "resp_hdr_add"
+
+
+class TestCachedRespAddHeader(TestReqAddHeader):
+    cache = True
+    directive = "resp_hdr_add"
+
+
+class TestReqSetHeader(TestReqAddHeader):
+    directive = "req_hdr_set"
+    cache = False
+    request = (
+        f"GET / HTTP/1.1\r\n"
+        + "Host: localhost\r\n"
+        + "Connection: keep-alive\r\n"
+        + "Accept: */*\r\n"
+        + "x-my-hdr: original text\r\n"
+        + "x-my-hdr-2: other original text\r\n"
+        + "\r\n"
+    )
+    backends = [
+        {
+            "id": "deproxy",
+            "type": "deproxy",
+            "port": "8000",
+            "response": "static",
+            "response_content": (
+                "HTTP/1.1 200 OK\r\n"
+                + "Date: test\r\n"
+                + "Server: debian\r\n"
+                + "x-my-hdr: original text\r\n"
+                + "x-my-hdr-2: other original text\r\n"
+                + "Content-Length: 0\r\n\r\n"
+            ),
+        }
+    ]
+
+    def header_not_in(self, header):
+        server = self.get_server("deproxy")
+        self.assertNotIn(header, server.last_request.headers.items())
+
+    def header_in(self, header):
+        server = self.get_server("deproxy")
+        self.assertIn(header, server.last_request.headers.items())
+
+    def test_add_some_hdrs(self):
+        super(TestReqSetHeader, self).test_add_some_hdrs()
+        self.header_not_in(("x-my-hdr", "original text"))
+        self.header_not_in(("x-my-hdr-2", "other original text"))
+
+    def test_add_some_hdrs_custom_location(self):
+        super(TestReqSetHeader, self).test_add_some_hdrs_custom_location()
+        self.header_not_in(("x-my-hdr", "original text"))
+        self.header_not_in(("x-my-hdr-2", "other original text"))
+
+    def test_add_hdrs_derive_config(self):
+        super(TestReqSetHeader, self).test_add_hdrs_derive_config()
+        self.header_not_in(("x-my-hdr", "original text"))
+        self.header_in(("x-my-hdr-2", "other original text"))
+
+    def test_add_hdrs_override_config(self):
+        super(TestReqSetHeader, self).test_add_hdrs_override_config()
+        self.header_in(("x-my-hdr", "original text"))
+        self.header_not_in(("x-my-hdr-2", "other original text"))
+
+
+class TestRespSetHeader(TestReqSetHeader):
+    directive = "resp_hdr_set"
+    cache = False
+
+    def header_in(self, header):
+        client = self.get_client("deproxy-1")
+        self.assertIn(header, client.last_response.headers.items())
+
+    def header_not_in(self, header):
+        client = self.get_client("deproxy-1")
+        self.assertNotIn(header, client.last_response.headers.items())
+
+
+class TestCachedRespSetHeader(TestRespSetHeader):
+    cache = True
+    directive = "resp_hdr_set"