Config/Annotations: Add proxy-busy-buffers-size. (#12433)

Author: bkreitch

docs/user-guide/nginx-configuration/annotations-risk.md

@@ -73,6 +73,7 @@
 | Proxy | proxy-buffer-size | Low | location |
 | Proxy | proxy-buffering | Low | location |
 | Proxy | proxy-buffers-number | Low | location |
+| Proxy | proxy-busy-buffers-size | Low | location |
 | Proxy | proxy-connect-timeout | Low | location |
 | Proxy | proxy-cookie-domain | Medium | location |
 | Proxy | proxy-cookie-path | Medium | location |

docs/user-guide/nginx-configuration/annotations.md

@@ -116,6 +116,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/proxy-buffering](#proxy-buffering)|string|
 |[nginx.ingress.kubernetes.io/proxy-buffers-number](#proxy-buffers-number)|number|
 |[nginx.ingress.kubernetes.io/proxy-buffer-size](#proxy-buffer-size)|string|
+|[nginx.ingress.kubernetes.io/proxy-busy-buffers-size](#proxy-busy-buffers-size)|string|
 |[nginx.ingress.kubernetes.io/proxy-max-temp-file-size](#proxy-max-temp-file-size)|string|
 |[nginx.ingress.kubernetes.io/ssl-ciphers](#ssl-ciphers)|string|
 |[nginx.ingress.kubernetes.io/ssl-prefer-server-ciphers](#ssl-ciphers)|"true" or "false"|
@@ -742,6 +743,18 @@ To configure this setting globally, set `proxy-buffer-size` in [NGINX ConfigMap]
 nginx.ingress.kubernetes.io/proxy-buffer-size: "8k"
 ```
 
+### Proxy busy buffers size
+
+[Limits the total size of buffers that can be busy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size) sending a response to the client while the response is not yet fully read.
+
+By default proxy busy buffers size is set as "8k".
+
+To configure this setting globally, set `proxy-busy-buffers-size` in the [ConfigMap](./configmap.md#proxy-busy-buffers-size). To use custom values in an Ingress rule, define this annotation:
+
+```yaml
+nginx.ingress.kubernetes.io/proxy-busy-buffers-size: "16k"
+```
+
 ### Proxy max temp file size
 
 When [`buffering`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) of responses from the proxied server is enabled, and the whole response does not fit into the buffers set by the [`proxy_buffer_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) and [`proxy_buffers`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) directives, a part of the response can be saved to a temporary file. This directive sets the maximum `size` of the temporary file setting the [`proxy_max_temp_file_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_max_temp_file_size). The size of data written to the temporary file at a time is set by the [`proxy_temp_file_write_size`](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_temp_file_write_size) directive.

docs/user-guide/nginx-configuration/configmap.md

@@ -179,6 +179,7 @@ The following table shows a configuration option's name, type, and the default v
 | [proxy-send-timeout](#proxy-send-timeout)                                       | int          | 60                                                                                                                                                                                                                                                                                                                                                           |                                                                                     |
 | [proxy-buffers-number](#proxy-buffers-number)                                   | int          | 4                                                                                                                                                                                                                                                                                                                                                            |                                                                                     |
 | [proxy-buffer-size](#proxy-buffer-size)                                         | string       | "4k"                                                                                                                                                                                                                                                                                                                                                         |                                                                                     |
+| [proxy-busy-buffers-size](#proxy-busy-buffers-size)                             | string       | "8k"                                                                                                                                                                                                                                                                                                                                                         |                                                                                     |
 | [proxy-cookie-path](#proxy-cookie-path)                                         | string       | "off"                                                                                                                                                                                                                                                                                                                                                        |                                                                                     |
 | [proxy-cookie-domain](#proxy-cookie-domain)                                     | string       | "off"                                                                                                                                                                                                                                                                                                                                                        |                                                                                     |
 | [proxy-next-upstream](#proxy-next-upstream)                                     | string       | "error timeout"                                                                                                                                                                                                                                                                                                                                              |                                                                                     |
@@ -1109,6 +1110,10 @@ Sets the number of the buffer used for [reading the first part of the response](
 
 Sets the size of the buffer used for [reading the first part of the response](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) received from the proxied server. This part usually contains a small response header.
 
+## proxy-busy-buffers-size
+
+[Limits the total size of buffers that can be busy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size) sending a response to the client while the response is not yet fully read.
+
 ## proxy-cookie-path
 
 Sets a text that [should be changed in the path attribute](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path) of the “Set-Cookie” header fields of a proxied server response.

internal/ingress/annotations/proxy/main.go

@@ -31,6 +31,7 @@ const (
 	proxyReadTimeoutAnnotation         = "proxy-read-timeout"
 	proxyBuffersNumberAnnotation       = "proxy-buffers-number"
 	proxyBufferSizeAnnotation          = "proxy-buffer-size"
+	proxyBusyBuffersSizeAnnotation     = "proxy-busy-buffers-size"
 	proxyCookiePathAnnotation          = "proxy-cookie-path"
 	proxyCookieDomainAnnotation        = "proxy-cookie-domain"
 	proxyBodySizeAnnotation            = "proxy-body-size"
@@ -82,6 +83,12 @@ var proxyAnnotations = parser.Annotation{
 			Documentation: `This annotation sets the size of the buffer proxy_buffer_size used for reading the first part of the response received from the proxied server. 
 			By default proxy buffer size is set as "4k".`,
 		},
+		proxyBusyBuffersSizeAnnotation: {
+			Validator:     parser.ValidateRegex(parser.SizeRegex, true),
+			Scope:         parser.AnnotationScopeLocation,
+			Risk:          parser.AnnotationRiskLow,
+			Documentation: `This annotation limits the total size of buffers that can be busy sending a response to the client while the response is not yet fully read. By default proxy busy buffers size is set as "8k".`,
+		},
 		proxyCookiePathAnnotation: {
 			Validator:     parser.ValidateRegex(parser.URLIsValidRegex, true),
 			Scope:         parser.AnnotationScopeLocation,
@@ -167,6 +174,7 @@ type Config struct {
 	ReadTimeout          int    `json:"readTimeout"`
 	BuffersNumber        int    `json:"buffersNumber"`
 	BufferSize           string `json:"bufferSize"`
+	BusyBuffersSize      string `json:"busyBuffersSize"`
 	CookieDomain         string `json:"cookieDomain"`
 	CookiePath           string `json:"cookiePath"`
 	NextUpstream         string `json:"nextUpstream"`
@@ -206,6 +214,9 @@ func (l1 *Config) Equal(l2 *Config) bool {
 	if l1.BufferSize != l2.BufferSize {
 		return false
 	}
+	if l1.BusyBuffersSize != l2.BusyBuffersSize {
+		return false
+	}
 	if l1.CookieDomain != l2.CookieDomain {
 		return false
 	}
@@ -290,6 +301,11 @@ func (a proxy) Parse(ing *networking.Ingress) (interface{}, error) {
 		config.BufferSize = defBackend.ProxyBufferSize
 	}
 
+	config.BusyBuffersSize, err = parser.GetStringAnnotation(proxyBusyBuffersSizeAnnotation, ing, a.annotationConfig.Annotations)
+	if err != nil {
+		config.BusyBuffersSize = defBackend.ProxyBusyBuffersSize
+	}
+
 	config.CookiePath, err = parser.GetStringAnnotation(proxyCookiePathAnnotation, ing, a.annotationConfig.Annotations)
 	if err != nil {
 		config.CookiePath = defBackend.ProxyCookiePath

internal/ingress/annotations/proxy/main_test.go

@@ -88,6 +88,7 @@ func (m mockBackend) GetDefaultBackend() defaults.Backend {
 		ProxyReadTimeout:         20,
 		ProxyBuffersNumber:       4,
 		ProxyBufferSize:          "10k",
+		ProxyBusyBuffersSize:     "15k",
 		ProxyBodySize:            "3k",
 		ProxyNextUpstream:        "error",
 		ProxyNextUpstreamTimeout: 0,
@@ -108,6 +109,7 @@ func TestProxy(t *testing.T) {
 	data[parser.GetAnnotationWithPrefix("proxy-read-timeout")] = "3"
 	data[parser.GetAnnotationWithPrefix("proxy-buffers-number")] = "8"
 	data[parser.GetAnnotationWithPrefix("proxy-buffer-size")] = "1k"
+	data[parser.GetAnnotationWithPrefix("proxy-busy-buffers-size")] = "4k"
 	data[parser.GetAnnotationWithPrefix("proxy-body-size")] = "2k"
 	data[parser.GetAnnotationWithPrefix("proxy-next-upstream")] = off
 	data[parser.GetAnnotationWithPrefix("proxy-next-upstream-timeout")] = "5"
@@ -141,6 +143,9 @@ func TestProxy(t *testing.T) {
 	if p.BufferSize != "1k" {
 		t.Errorf("expected 1k as buffer-size but returned %v", p.BufferSize)
 	}
+	if p.BusyBuffersSize != "4k" {
+		t.Errorf("expected 4k as busy-buffers-size but returned %v", p.BusyBuffersSize)
+	}
 	if p.BodySize != "2k" {
 		t.Errorf("expected 2k as body-size but returned %v", p.BodySize)
 	}
@@ -176,6 +181,7 @@ func TestProxyComplex(t *testing.T) {
 	data[parser.GetAnnotationWithPrefix("proxy-read-timeout")] = "3"
 	data[parser.GetAnnotationWithPrefix("proxy-buffers-number")] = "8"
 	data[parser.GetAnnotationWithPrefix("proxy-buffer-size")] = "1k"
+	data[parser.GetAnnotationWithPrefix("proxy-busy-buffers-size")] = "4k"
 	data[parser.GetAnnotationWithPrefix("proxy-body-size")] = "2k"
 	data[parser.GetAnnotationWithPrefix("proxy-next-upstream")] = "error http_502"
 	data[parser.GetAnnotationWithPrefix("proxy-next-upstream-timeout")] = "5"
@@ -209,6 +215,9 @@ func TestProxyComplex(t *testing.T) {
 	if p.BufferSize != "1k" {
 		t.Errorf("expected 1k as buffer-size but returned %v", p.BufferSize)
 	}
+	if p.BusyBuffersSize != "4k" {
+		t.Errorf("expected 4k as buffer-size but returned %v", p.BusyBuffersSize)
+	}
 	if p.BodySize != "2k" {
 		t.Errorf("expected 2k as body-size but returned %v", p.BodySize)
 	}
@@ -264,6 +273,9 @@ func TestProxyWithNoAnnotation(t *testing.T) {
 	if p.BufferSize != "10k" {
 		t.Errorf("expected 10k as buffer-size but returned %v", p.BufferSize)
 	}
+	if p.BusyBuffersSize != "15k" {
+		t.Errorf("expected 15k as buffer-size but returned %v", p.BusyBuffersSize)
+	}
 	if p.BodySize != "3k" {
 		t.Errorf("expected 3k as body-size but returned %v", p.BodySize)
 	}

internal/ingress/controller/config/config.go

@@ -850,6 +850,7 @@ func NewDefault() Configuration {
 			ProxySendTimeout:            60,
 			ProxyBuffersNumber:          4,
 			ProxyBufferSize:             "4k",
+			ProxyBusyBuffersSize:        "8k",
 			ProxyCookieDomain:           "off",
 			ProxyCookiePath:             "off",
 			ProxyNextUpstream:           "error timeout",

internal/ingress/controller/controller.go

@@ -1255,6 +1255,7 @@ func (n *NGINXController) createServers(data []*ingress.Ingress,
 		ReadTimeout:          bdef.ProxyReadTimeout,
 		BuffersNumber:        bdef.ProxyBuffersNumber,
 		BufferSize:           bdef.ProxyBufferSize,
+		BusyBuffersSize:      bdef.ProxyBusyBuffersSize,
 		CookieDomain:         bdef.ProxyCookieDomain,
 		CookiePath:           bdef.ProxyCookiePath,
 		NextUpstream:         bdef.ProxyNextUpstream,

internal/ingress/defaults/main.go

@@ -69,6 +69,11 @@ type Backend struct {
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size)
 	ProxyBufferSize string `json:"proxy-buffer-size"`
 
+	// Limits the total size of buffers that can be busy sending a response to the client while
+	// the response is not yet fully read.
+	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_busy_buffers_size
+	ProxyBusyBuffersSize string `json:"proxy-busy-buffers-size"`
+
 	// Sets a text that should be changed in the path attribute of the “Set-Cookie” header fields of
 	// a proxied server response.
 	// http://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_cookie_path

rootfs/etc/nginx/template/nginx.tmpl

@@ -1041,6 +1041,7 @@ stream {
             {{ end }}
             proxy_buffer_size                       {{ $location.Proxy.BufferSize }};
             proxy_buffers                           {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }};
+            proxy_busy_buffers_size                 {{ $location.Proxy.BusyBuffersSize }};
             proxy_request_buffering                 {{ $location.Proxy.RequestBuffering }};
 
             proxy_ssl_server_name       on;
@@ -1296,6 +1297,7 @@ stream {
             proxy_buffering                         {{ $location.Proxy.ProxyBuffering }};
             proxy_buffer_size                       {{ $location.Proxy.BufferSize }};
             proxy_buffers                           {{ $location.Proxy.BuffersNumber }} {{ $location.Proxy.BufferSize }};
+            proxy_busy_buffers_size                 {{ $location.Proxy.BusyBuffersSize }};
             {{ if isValidByteSize $location.Proxy.ProxyMaxTempFileSize true }}
             proxy_max_temp_file_size                {{ $location.Proxy.ProxyMaxTempFileSize }};
             {{ end }}

test/e2e/annotations/proxy.go

@@ -160,11 +160,13 @@ var _ = framework.DescribeAnnotation("proxy-*", func() {
 		proxyBuffering := "on"
 		proxyBuffersNumber := "8"
 		proxyBufferSize := "8k"
+		proxyBusyBuffersSize := "16k"
 
 		annotations := make(map[string]string)
 		annotations["nginx.ingress.kubernetes.io/proxy-buffering"] = proxyBuffering
 		annotations["nginx.ingress.kubernetes.io/proxy-buffers-number"] = proxyBuffersNumber
 		annotations["nginx.ingress.kubernetes.io/proxy-buffer-size"] = proxyBufferSize
+		annotations["nginx.ingress.kubernetes.io/proxy-busy-buffers-size"] = proxyBusyBuffersSize
 
 		ing := framework.NewSingleIngress(host, "/", host, f.Namespace, framework.EchoService, 80, annotations)
 		f.EnsureIngress(ing)
@@ -174,6 +176,7 @@ var _ = framework.DescribeAnnotation("proxy-*", func() {
 				return strings.Contains(server, fmt.Sprintf("proxy_buffering %s;", proxyBuffering)) &&
 					strings.Contains(server, fmt.Sprintf("proxy_buffer_size %s;", proxyBufferSize)) &&
 					strings.Contains(server, fmt.Sprintf("proxy_buffers %s %s;", proxyBuffersNumber, proxyBufferSize)) &&
+					strings.Contains(server, fmt.Sprintf("proxy_busy_buffers_size %s;", proxyBusyBuffersSize)) &&
 					strings.Contains(server, fmt.Sprintf("proxy_request_buffering %s;", proxyBuffering))
 			})
 	})