Mekhanik evgenii/1411 7

fw/apm.c

@@ -2230,4 +2230,4 @@ void
 tfw_apm_exit(void)
 {
 	tfw_mod_unregister(&tfw_apm_mod);
-}
+}

fw/hpack.c

@@ -30,6 +30,54 @@
 
 #include "hpack_tbl.h"
 
+static inline int
+tfw_hpack_buffer_sz(unsigned long len)
+{
+	unsigned int order = get_order(sizeof(TfwPoolChunk) + len);
+
+	return (PAGE_SIZE << order) - sizeof(TfwPoolChunk);
+}
+
+/**
+ * There are two cases when we allocate space for header:
+ * - it->rspace is equal to zero. In this case we allocate
+ *   `len` bytes;
+ * - it->rspace is not equal to zero. This means that we
+ *   already have buffer with rspace bytes available.
+ *   If this buffer is rather big to contain the whole
+ *   header or it is greater than sizeof(TfwStr) we use
+ *   it. (Save delta = len - it->rspace bytes. Later when
+ *   `it->rspace` will be exceeded we use it to allocate
+ *   new chunk).
+ *   Otherwise allocate new buffer (if we have some free
+ *   bytes in old buffer, but is is not enough for the
+ *   new header and is less then sizeof(TfwStr)), we allocate
+ *   new buffer to prevent extra header chunking.
+ * - When we restore header from hpack table we use force flag
+ *   for the case when there is no enough space to restore
+ *   header chunk. In this case we allocate new space in pool
+ *   to prevent extra header chunking.
+ */
+static inline int
+tfw_hpack_buffer_get(TfwMsgParseIter *it, unsigned long len, bool force)
+{
+	unsigned int room;
+
+	if (!force && it->rspace
+	    && (len <= it->rspace || it->rspace > sizeof(TfwStr))) {
+		it->to_alloc = len <= it->rspace ?
+			0 : tfw_hpack_buffer_sz(len - it->rspace);
+		return 0;
+	}
+
+	room = TFW_POOL_CHUNK_ROOM(it->pool);
+	len = (room >= len ? room : tfw_hpack_buffer_sz(len));
+	it->rspace = len;
+
+	return (it->pos = tfw_pool_alloc_not_align(it->pool, len)) ?
+		0 : -ENOMEM;
+}
+
 #define HP_HDR_NAME(name)						\
 	(&(TfwStr){							\
 		.chunks = &(TfwStr){					\
@@ -278,23 +326,31 @@ do {								\
 	       last - src);					\
 } while (0)
 
-#define	BUFFER_HDR_INIT(length, it)				\
+#define	BUFFER_HDR_INIT(it)					\
 do {								\
 	(it)->hdr.data = (it)->pos;				\
-	(it)->hdr.len = length;					\
+	(it)->hdr.len = 0;					\
 	(it)->next = 0;						\
 } while (0)
 
 #define	BUFFER_NAME_OPEN(length)				\
 do {								\
 	WARN_ON_ONCE(!TFW_STR_EMPTY(&it->hdr));			\
 	if (state & HPACK_FLAGS_HUFFMAN_NAME) {			\
-		BUFFER_GET(length, it);				\
-		if (!it->pos) {					\
-			r = -ENOMEM;				\
+		/*						\
+		 * Allocate extra 50% bytes. Huffman usually	\
+		 * gives compression benefit about 20 - 60 %.	\
+		 * Since we use extra allocated bytes for the	\
+		 * next header, we can allocate extra 50% bytes	\
+		 * without fear of losing a lot of memory.	\
+		 */						\
+		unsigned long len = length + (length >> 1);	\
+								\
+		r = tfw_hpack_buffer_get(it, len, false);	\
+		if (unlikely(r))				\
 			goto out;				\
-		}						\
-		BUFFER_HDR_INIT(length, it);			\
+								\
+		BUFFER_HDR_INIT(it);				\
 	}							\
 } while (0)
 
@@ -306,19 +362,25 @@ do {								\
 		? it->parsed_hdr->nchunks			\
 		: 1;						\
 	if (state & HPACK_FLAGS_HUFFMAN_VALUE) {		\
-		BUFFER_GET(length, it);				\
-		if (!it->pos) {					\
-			r = -ENOMEM;				\
+		/*						\
+		 * Allocate extra 50% bytes. Huffman usually	\
+		 * gives compression benefit about 20 - 60 %.	\
+		 * Since we use extra allocated bytes for the	\
+		 * next header, we can allocate extra 50% bytes	\
+		 * without fear of losing a lot of memory.	\
+		 */						\
+		unsigned long len = length + (length >> 1);	\
+								\
+		r = tfw_hpack_buffer_get(it, len, false);	\
+		if (unlikely(r))				\
 			goto out;				\
-		}						\
 		if (!TFW_STR_EMPTY(&it->hdr)) {			\
-			r = tfw_hpack_exp_hdr(req->pool, length, \
-					      it); 		\
+			r = tfw_hpack_exp_hdr(req->pool, 0, it); \
 			if (unlikely(r))			\
 				return r;			\
 			it->next = it->hdr.nchunks - 1;		\
 		} else	{					\
-			BUFFER_HDR_INIT(length, it);		\
+			BUFFER_HDR_INIT(it);			\
 		}						\
 	}							\
 } while (0)
@@ -456,41 +518,44 @@ static inline int
 tfw_hpack_huffman_write(char sym, TfwHttpReq *__restrict req)
 {
 	TfwMsgParseIter *it = &req->pit;
-	bool np;
+	unsigned long to_alloc;
 	int r;
 
 	if (it->rspace) {
-		--it->rspace;
-		*it->pos++ = sym;
-		return 0;
-	}
-
-	if (!(it->pos = tfw_pool_alloc_not_align_np(it->pool, 1, &np)))
-		return -ENOMEM;
-
-	*it->pos = sym;
-
-	T_DBG3("%s: it->rspace=%lu, sym=%c, np=%d\n", __func__,
-	       it->rspace, sym, np);
-
-	if (!np) {
 		TfwStr *hdr = &it->hdr;
 		TfwStr *last = TFW_STR_LAST(hdr);
 
 		T_DBG3("%s: add to hdr, hdr->len=%lu, last->len=%lu,"
 		       " last->data=%.*s\n", __func__, hdr->len, last->len,
 		       (int)last->len, last->data);
 
+		--it->rspace;
+		*it->pos++ = sym;
 		++hdr->len;
 		if (!TFW_STR_PLAIN(hdr))
 			++last->len;
 		return 0;
 	}
 
+	to_alloc = it->to_alloc ? it->to_alloc : tfw_hpack_buffer_sz(1);
+	r = tfw_hpack_buffer_get(it, to_alloc, false);
+	if (unlikely(r))
+		return r;
+
+	T_DBG3("%s: it->rspace=%u, sym=%c\n", __func__, it->rspace, sym);
+
+	/*
+	 * If the new page was allocated we should expand
+	 * header.
+	 */
 	r = tfw_hpack_exp_hdr(req->pool, 1, it);
 	if (unlikely(r))
 		return r;
 
+	--it->rspace;
+	*it->pos++ = sym;
+	it->to_alloc = 0;
+
 	return 0;
 }
 
@@ -536,9 +601,8 @@ huffman_decode_tail(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 			 */
 			if (likely(offset == 0)) {
 				if ((i ^ (HT_EOS_HIGH >> 1)) <
-				    (1U << -hp->curr)) {
-					return 0;
-				}
+				    (1U << -hp->curr))
+					return T_OK;
 			}
 			/*
 			 * The first condition here equivalent to the
@@ -562,9 +626,8 @@ huffman_decode_tail(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 		}
 	}
 	if (likely(offset == 0)) {
-		if ((i ^ (HT_EOS_HIGH >> 1)) < (1U << -hp->curr)) {
+		if ((i ^ (HT_EOS_HIGH >> 1)) < (1U << -hp->curr))
 			return T_OK;
-		}
 	}
 	return T_COMPRESSION;
 }
@@ -1179,12 +1242,12 @@ static int
 tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 		       const TfwHPackEntry *__restrict entry)
 {
-	char *data;
 	unsigned int num = entry->name_num;
 	unsigned long sz = entry->name_len;
 	const TfwStr *s, *end, *s_hdr = entry->hdr;
 	TfwMsgParseIter *it = &req->pit;
 	TfwStr *d, *d_hdr = it->parsed_hdr;
+	int r;
 
 	WARN_ON_ONCE(!TFW_STR_EMPTY(d_hdr));
 	if (WARN_ON_ONCE(!num || num > s_hdr->nchunks))
@@ -1207,14 +1270,22 @@ tfw_hpack_hdr_name_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 		goto done;
 	}
 
-	if (!(data = tfw_pool_alloc_not_align(it->pool, sz)))
-		return -ENOMEM;
+	r = tfw_hpack_buffer_get(it, sz, false);
+	if (unlikely(r))
+		return r;
 
 	for (s = s_hdr->chunks, end = s_hdr->chunks + num; s < end; ++s) {
+		if (unlikely(it->rspace < s->len)) {
+			r = tfw_hpack_buffer_get(it, it->to_alloc, true);
+			if (unlikely(r))
+				return r;
+		}
+
 		*d = *s;
-		d->data = data;
-		memcpy_fast(data, s->data, s->len);
-		data += s->len;
+		d->data = it->pos;
+		memcpy_fast(it->pos, s->data, s->len);
+		it->pos += s->len;
+		it->rspace -= s->len;
 		++d;
 	}
 
@@ -1228,12 +1299,11 @@ static int
 tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 		  const TfwHPackEntry *__restrict entry)
 {
-	char *data;
 	unsigned long d_size;
-	TfwMsgParseIter *it = &req->pit;
 	const TfwStr *s, *end, *s_hdr = entry->hdr;
 	TfwHttpParser *parser = &req->stream->parser;
 	TfwStr *d, *d_hdr = &parser->hdr;
+	TfwMsgParseIter *it = &req->pit;
 	int r;
 
 	WARN_ON_ONCE(TFW_STR_PLAIN(s_hdr));
@@ -1262,8 +1332,9 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 	 * (without full copying) only references for statically indexed
 	 * headers (also, see comment above).
 	 */
-	if (!(data = tfw_pool_alloc_not_align(it->pool, s_hdr->len)))
-		return -ENOMEM;
+	r = tfw_hpack_buffer_get(it, s_hdr->len, false);
+	if (unlikely(r))
+		return r;
 
 	d_size = s_hdr->nchunks * sizeof(TfwStr);
 	if (!(d_hdr->chunks = tfw_pool_alloc(req->pool, d_size)))
@@ -1275,10 +1346,17 @@ tfw_hpack_hdr_set(TfwHPack *__restrict hp, TfwHttpReq *__restrict req,
 
 	d = d_hdr->chunks;
 	TFW_STR_FOR_EACH_CHUNK(s, s_hdr, end) {
+		if (unlikely(it->rspace < s->len)) {
+			r = tfw_hpack_buffer_get(it, it->to_alloc, true);
+			if (unlikely(r))
+				return r;
+		}
+
 		*d = *s;
-		d->data = data;
-		memcpy_fast(data, s->data, s->len);
-		data += s->len;
+		d->data = it->pos;
+		memcpy_fast(it->pos, s->data, s->len);
+		it->pos += s->len;
+		it->rspace -= s->len;
 		++d;
 	}
 

fw/hpack.h

@@ -296,17 +296,6 @@ typedef struct {
 	TfwStr			hdr_data;
 } TfwDecodeCacheIter;
 
-#define	BUFFER_GET(len, it)					\
-do {								\
-	BUG_ON(!(len));						\
-	WARN_ON_ONCE((it)->rspace);				\
-	(it)->rspace = len;					\
-	(it)->pos = tfw_pool_alloc_not_align((it)->pool, len);	\
-	T_DBG3("%s: get buffer, len=%lu, it->pos=[%p],"		\
-	       " it->pos=%lu\n", __func__, (unsigned long)len,	\
-	       (it)->pos, (unsigned long)(it)->pos);		\
-} while (0)
-
 void write_int(unsigned long index, unsigned short max, unsigned short mask,
 	       TfwHPackInt *__restrict res_idx);
 int tfw_hpack_init(TfwHPack *__restrict hp, unsigned int htbl_sz);

fw/msg.h

@@ -67,32 +67,35 @@ typedef struct {
  * @pool	- allocation pool for target buffer of decoded headers;
  * @parsed_hdr	- pointer to the message header which is currently processed;
  * @hdrs_len	- accumulated length of message's decoded and parsed headers;
+ * @rspace	- space remained in the allocated chunk;
+ * @pos		- pointer to the currently allocated chunk of decoded headers'
+ *		  buffer;
  * @hdrs_cnt	- count of all headers from message headers block;
  * @__off	- offset for iterator reinitializing before next processing
  *		  stage;
- * @hdr		- descriptor of currently decoded header in target buffer;
- * @pos		- pointer to the currently allocated chunk of decoded headers'
- *		  buffer;
- * @rspace	- space remained in the allocated chunk;
  * @nm_len	- length of the decoded header's name;
+ * @to_alloc	- count of bytes which, should be allocated, after rspace
+ *		  will be exceeded;
  * @nm_num	- chunks number of the decoded header's name;
  * @tag		- tag of currently processed decoded header.
  * @next	- number of chunk which points to the decoded header part
  *		  (name/value) to be parsed next;
+ * @hdr		- descriptor of currently decoded header in target buffer;
  */
 typedef struct {
 	TfwPool		*pool;
 	TfwStr		*parsed_hdr;
-	unsigned long	hdrs_len;
+	unsigned int	hdrs_len;
+	unsigned int	rspace;
+	char		*pos;
 	unsigned int	hdrs_cnt;
 	char		__off[0];
-	TfwStr		hdr;
-	char		*pos;
-	unsigned long	rspace;
-	unsigned long	nm_len;
+	unsigned int	nm_len;
+	unsigned int	to_alloc;
 	unsigned int	nm_num;
 	unsigned int	tag;
 	unsigned int	next;
+	TfwStr		hdr;
 } TfwMsgParseIter;
 
 int tfw_msg_write(TfwMsgIter *it, const TfwStr *data);