Improve the processing of rfc3676

rfc3676 formatted messages can have both fixed and flowed lines.
delsp wasn't always being removed
fix folding of lines with quotes

Author: jkbzh

src/hypermail.h

@@ -236,6 +236,7 @@ struct body {
     char attached;		/* part of attachment */
     char demimed;		/* if this is a header, this is set to TRUE if
 				   it has passed the decoderfc2047() function */
+    int format_flowed;          /* TRUE if this a text/plain f=f line */
     int msgnum;
     struct body *next;
 };
@@ -286,8 +287,8 @@ struct emailinfo {
     long datenum;		/* moved here from 'struct header' */
     long flags;			/* added 1999-03-19 for the new thread output functions */
 
-#define PRINT_THREAD 1		/* set if already used in the thread output */
-#define USED_THREAD  2		/* set if already stored in threadlist */
+#define PRINT_THREAD  1		/* set if already used in the thread output */
+#define USED_THREAD   2		/* set if already stored in threadlist */
 
     int initial_next_in_thread;	/* msgnum written as next during normal print*/
 
@@ -330,6 +331,10 @@ struct attach {
 #define BODY_HTMLIZED (1<<1)	/* this is already htmlified */
 #define BODY_HEADER   (1<<2)	/* this is a header line */
 #define BODY_ATTACHED (1<<3)	/* this line was attached */
+#define BODY_FORMAT_FLOWED (1<<4) /* this line is format-flowed */
+#define BODY_DEL_SSQ  (1<<5)    /* remove both space stuffing and
+                                 * quotes where applicable for f=f */
+
 
 struct boundary {
     struct boundary *next;

src/parse.c

@@ -1117,31 +1117,29 @@ static int get_quotelevel (const char *line)
 ** headers), the previous line quotelevel, and a flag saying if the
 ** previous line was marked as a continuing one.
 **
-** The function updates the quotelevel to
-** that of the current parsed line. The function will update the
-** continue_prev_flow_flag to say if the current line should be joined
-** to the previous one, and, if positive, the padding offset that
-** should be applied to the current line when merging it (for skipping
-** quotes or space-stuffing).
+** The function updates the quotelevel to that of the current parsed
+** line. The function will update the continue_prev_flow_flag to say
+** if the current line should be joined to the previous one, and, if
+** positive, the padding offset that should be applied to the current
+** line when merging it (for skipping quotes or space-stuffing).
 **
-** If delsp is true, the function will remove the space in the soft 
+** If delsp is true, the function will remove the space in the soft
 ** line break if the line is flowed.
 **
 ** The function returns true if the current line is flowed.
 **
 */
 static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel, 
-			     bool *continue_prev_flow_flag, int *padding)
+			     bool *continue_prev_flow_flag)
 {
   int new_quotelevel = 0;
-  int space_stuffing = 0;
   int tmp_padding = 0;
   bool sig_sep = FALSE;
   bool flowed = FALSE;
 
   /* rules for evaluation if the flow should stop:
      1. new quote level is different from previous one
-     2. The line ends with a signature "[(quotes)][(ss)]-- \n"
+     2. The line is a signature "[(quotes)][(ss)]-- \n"
      3. The line is a hard break "\n"
      4. The message body has ended
 
@@ -1162,6 +1160,7 @@ static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel,
 
      special case, space-stuffed or f=f? A line that has only this content:
      " \n": this is a space-stuffed newline.
+     @@ test this special case with mutt
   */
 
 
@@ -1176,8 +1175,7 @@ static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel,
   if (rfc3676_ishardlb(line)) {
       /* Hard crlf, reset flags */
       *quotelevel = 0;
-      *padding = 0;
-      /* *continue_prev_flow_flag = FALSE; */
+      *continue_prev_flow_flag = FALSE;
 #if DEBUG_PARSE
       printf("RFC3676: hard CRLF detected. Stopping ff\n");
 #endif
@@ -1207,7 +1205,6 @@ static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel,
   ** skip space stuffing if any 
   */
   if (line[tmp_padding] == ' ') {
-      space_stuffing = 1;
       tmp_padding++;
 #if DEBUG_PARSE
       printf("RFC3676: space-stuffing detected; skipping space\n");
@@ -1221,13 +1218,12 @@ static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel,
       /* Hard crlf, reset flags */
       /* *continue_prev_flow_flag = FALSE; */
       *quotelevel = new_quotelevel;
-      *padding = 0;
 #if DEBUG_PARSE
       printf("RFC3676: hard CRLF detected after quote. Stopping ff\n");
 #endif
       return FALSE;
   }
-    
+
   /*
   ** signature detection
   */
@@ -1280,12 +1276,6 @@ static bool rfc3676_handler (char *line, bool delsp_flag, int *quotelevel,
   */
   *quotelevel = new_quotelevel;
 
-  if (*continue_prev_flow_flag) {
-      *padding = new_quotelevel + space_stuffing;
-  } else {
-    *padding = (new_quotelevel == 0) ? space_stuffing : 0;
-  }
-  
 #if DEBUG_PARSE
   if (*continue_prev_flow_flag)
       printf("RFC3676: Continuing previous flow\n");
@@ -3063,23 +3053,27 @@ msgid);
 			    }
 			}
 			else {
-			  int padding; /* used for skipping padding detected by rfc3676_handler,
-					  which seems smarter than moving all the bytes in data
-					  before injecting it into addbody */
 			  if (!isinheader && (textplain_format == FORMAT_FLOWED)) {
+                              /* remove both space stuffing and quotes
+                               * where applicable for f=f */
+                              bodyflags |= BODY_DEL_SSQ;
                               flowed_line = rfc3676_handler (data, delsp_flag, &quotelevel, 
-                                                             &continue_previous_flow_flag,
-                                                             &padding);
+                                                             &continue_previous_flow_flag);
                               if (continue_previous_flow_flag) {
                                   bodyflags |= BODY_CONTINUE;
                               } else  {
                                   bodyflags &= ~BODY_CONTINUE;
+                                  if (flowed_line) {
+                                      bodyflags |= BODY_FORMAT_FLOWED;
+                                  } else {
+                                      bodyflags &= ~BODY_FORMAT_FLOWED;
+				  }
                               }
                               continue_previous_flow_flag = flowed_line;
 			  } else {
-                              padding = 0;
+                              bodyflags &= ~BODY_DEL_SSQ;
 			  }
-			  bp = addbody(bp, &lp, data + padding,
+			  bp = addbody(bp, &lp, data,
 				       (content == CONTENT_HTML ?
 					BODY_HTMLIZED : 0) | bodyflags);
 			}

src/struct.c

@@ -912,14 +912,38 @@ struct body *addbody(struct body *bp, struct body **lp,	/* points to the last po
 {
     struct body *tempnode;
     struct body *newnode = NULL;
-
+    char *unstuffed_line = line;
+    int free_unstuffed_line = 0;
+
+    /* delete both space stuffing and quotes where applicable for f=f */
+    if (flags & BODY_DEL_SSQ) {
+        if (flags & BODY_CONTINUE) {
+            /* delete all quote levels, we're reusing those of the precedent line */
+            while (*unstuffed_line == '>') {
+                unstuffed_line++;
+            }
+        }
+        /* deleting space-stuffing at beginning of line */
+        if (unstuffed_line[0] == ' ') {
+            unstuffed_line++;
+        } 
+        else if (unstuffed_line[0] == '>') {
+            char *delsp_line = rfc3676_delsp_quotes(unstuffed_line);
+            if (delsp_line) {
+                unstuffed_line = delsp_line;
+                free_unstuffed_line = 1;
+            }
+        }
+    }
+    
     if (!(flags & BODY_CONTINUE)) {
 	newnode = (struct body *)emalloc(sizeof(struct body));
 	memset(newnode, 0, sizeof(struct body));
-	newnode->line = spamify(strsav(line));
+	newnode->line = spamify(strsav(unstuffed_line));
 	newnode->html = (flags & BODY_HTMLIZED) ? 1 : 0;
 	newnode->header = (flags & BODY_HEADER) ? 1 : 0;
 	newnode->attached = (flags & BODY_ATTACHED) ? 1 : 0;
+        newnode->format_flowed = (flags & BODY_FORMAT_FLOWED) ? 1 : 0;
 	newnode->next = NULL;
     }
     if (bp == NULL) {
@@ -935,7 +959,7 @@ struct body *addbody(struct body *bp, struct body **lp,	/* points to the last po
 	    char *newbuf;
 
 	    /* get the new size + 1 for the terminating zero */
-	    newlen = strlen(tempnode->line) + strlen(line) + 1;
+	    newlen = strlen(tempnode->line) + strlen(unstuffed_line) + 1;
 
 	    /* extend the former memory area: */
 	    newbuf = (char *)realloc(tempnode->line, newlen);
@@ -949,7 +973,7 @@ struct body *addbody(struct body *bp, struct body **lp,	/* points to the last po
 		    *lf = 0;
 
 		/* append the new part */
-		strcat(newbuf, line);
+		strcat(newbuf, unstuffed_line);
 
 		/* point out the new buffer instead */
 		tempnode->line = newbuf;
@@ -960,6 +984,11 @@ struct body *addbody(struct body *bp, struct body **lp,	/* points to the last po
 	    *lp = newnode;
 	}
     }
+
+    if (free_unstuffed_line) {
+        free(unstuffed_line);
+    }
+    
     return bp;
 }