net: http: client: Notify application about unprocessed data

The HTTP client would read data from a socket up to the size of the
receiving buffer, however it may not process them all. This is usually
only the case if protocol switching takes place, where the data read may
belong to another protocol. Therefore we need a way to notify the caller
about any potential data that is already present in the buffer and needs
to be processed.

Introduce an new data_len member in struct http_request to provide the
information about the data already available in the receive buffer. If,
after HTTP response processing, the value is non-zero, the data will be
available in the beginning of the user provided receive buffer.

To make this possible however, we need to track how many bytes were
actually processed by the http_parser, therefore the code will no longer
ignore the http_parser_execute() return value. To simplify processing,
it's also been changed how the receive buffer is used. Instead of using
it in a ring-buffer-like way, the offset variable will track how many
bytes are available in the buffer, and in the rare occasions when not
all data from the buffer was processed by the HTTP parser, we'll
memmomve the remaining data to the beginning of the buffer.

Signed-off-by: Robert Lubos <robert.lubos@nordicsemi.no>

Author: rlubos

include/zephyr/net/http/client.h

@@ -260,6 +260,13 @@ struct http_request {
 	/** Length of the user supplied receive buffer */
 	size_t recv_buf_len;
 
+	/** Length of the unprocessed data left inside the user supplied receive
+	 *  buffer. In typical HTTP processing this should be 0, however in case
+	 *  of switching protocols, there may be some data left belonging to the
+	 *  new protocol.
+	 */
+	size_t data_len;
+
 	/** The URL for this request, for example: /index.html */
 	const char *url;
 

subsys/net/lib/http/http_client.c

@@ -480,7 +480,7 @@ static void http_report_progress(struct http_request *req)
 static int http_wait_data(int sock, struct http_request *req, const k_timepoint_t req_end_timepoint)
 {
 	int total_received = 0;
-	size_t offset = 0;
+	size_t offset = 0, processed = 0;
 	int received, ret;
 	struct zsock_pollfd fds[1];
 	int nfds = 1;
@@ -525,35 +525,67 @@ static int http_wait_data(int sock, struct http_request *req, const k_timepoint_
 			} else if (received < 0) {
 				ret = -errno;
 				goto error;
-			} else {
-				req->internal.response.data_len += received;
-
-				(void)http_parser_execute(
-					&req->internal.parser, &req->internal.parser_settings,
-					req->internal.response.recv_buf + offset, received);
 			}
 
 			total_received += received;
 			offset += received;
 
+			processed = http_parser_execute(
+				&req->internal.parser, &req->internal.parser_settings,
+				req->internal.response.recv_buf, offset);
+
+			if (processed > offset) {
+				LOG_ERR("HTTP parser error, too much data consumed");
+				ret = -EBADMSG;
+				goto error;
+			}
+
+			if (req->internal.parser.http_errno != HPE_OK) {
+				LOG_ERR("HTTP parsing error, %d",
+					req->internal.parser.http_errno);
+				ret = -EBADMSG;
+				goto error;
+			}
+
+			req->internal.response.data_len += processed;
+			offset -= processed;
+
 			if (offset >= req->internal.response.recv_buf_len) {
-				offset = 0;
+				/* This means the parser did not consume any data
+				 * and we can't fit any more in the buffer.
+				 */
+				LOG_ERR("HTTP RX buffer full, cannot proceed");
+				ret = -ENOMEM;
+				goto error;
 			}
 
 			if (req->internal.response.message_complete) {
 				http_report_complete(req);
-				break;
-			} else if (offset == 0) {
+			} else {
 				http_report_progress(req);
 
 				/* Re-use the result buffer and start to fill it again */
 				req->internal.response.data_len = 0;
 				req->internal.response.body_frag_start = NULL;
 				req->internal.response.body_frag_len = 0;
 			}
+
+			if (offset > 0) {
+				/* In case there are any unprocessed data left,
+				 * move them to the front of the buffer.
+				 */
+				memmove(req->internal.response.recv_buf,
+					req->internal.response.recv_buf + processed,
+					offset);
+			}
 		}
 
-	} while (true);
+	} while (!req->internal.response.message_complete);
+
+	/* If there's still some data left in the buffer after HTTP processing,
+	 * reflect this in data_len variable.
+	 */
+	req->data_len = offset;
 
 	return total_received;