Switch to WHATWG’s MIME type parsing algorithm

SimonSapin · SimonSapin · commit 72a19fa69ef8 · 2018-02-02T09:35:21.000+01:00
… and a custom Rust type for MIME type records, because the mime crate does not have a constructor from components: hyperium/mime#78
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,6 @@ authors = ["Simon Sapin <simon.sapin@exyr.org>"]
 
 [dependencies]
 matches = "0.1"
-mime = "0.3"
 
 [dev-dependencies]
 rustc-test = "0.3"
diff --git a/src/lib.rs b/src/lib.rs
@@ -8,19 +8,25 @@
 //! let url = DataUrl::process("data:,Hello%20World!").unwrap();
 //! let (body, fragment) = url.decode_to_vec().unwrap();
 //!
-//! assert_eq!(url.mime_type().type_(), mime::TEXT);
-//! assert_eq!(url.mime_type().subtype(), mime::PLAIN);
-//! assert_eq!(url.mime_type().get_param(mime::CHARSET).unwrap(), "US-ASCII");
+//! assert_eq!(url.mime_type().type_, "text");
+//! assert_eq!(url.mime_type().subtype, "plain");
+//! assert_eq!(url.mime_type().parameters, [("charset".into(), "US-ASCII".into())]);
 //! assert_eq!(body, b"Hello World!");
 //! assert!(fragment.is_none());
 //! ```
 
 #[macro_use] extern crate matches;
-pub extern crate mime;
 
-use forgiving_base64::{InvalidBase64, DecodeError};
+macro_rules! require {
+    ($condition: expr) => {
+        if !$condition {
+            return None
+        }
+    }
+}
 
 pub mod forgiving_base64;
+pub mod mime;
 
 pub struct DataUrl<'a> {
     mime_type: mime::Mime,
@@ -57,20 +63,20 @@ impl<'a> DataUrl<'a> {
     /// Streaming-decode the data URL’s body to `write_body_bytes`,
     /// and return the URL’s fragment identifier if it has one.
     pub fn decode<F, E>(&self, write_body_bytes: F)
-                        -> Result<Option<FragmentIdentifier<'a>>, DecodeError<E>>
+                        -> Result<Option<FragmentIdentifier<'a>>, forgiving_base64::DecodeError<E>>
         where F: FnMut(&[u8]) -> Result<(), E>
     {
         if self.base64 {
             decode_with_base64(self.encoded_body_plus_fragment, write_body_bytes)
         } else {
             decode_without_base64(self.encoded_body_plus_fragment, write_body_bytes)
-                .map_err(DecodeError::WriteError)
+                .map_err(forgiving_base64::DecodeError::WriteError)
         }
     }
 
     /// Return the decoded body, and the URL’s fragment identifier if it has one.
     pub fn decode_to_vec(&self)
-        -> Result<(Vec<u8>, Option<FragmentIdentifier<'a>>), InvalidBase64>
+        -> Result<(Vec<u8>, Option<FragmentIdentifier<'a>>), forgiving_base64::InvalidBase64>
     {
         let mut body = Vec::new();
         let fragment = self.decode(|bytes| Ok(body.extend_from_slice(bytes)))?;
@@ -101,14 +107,6 @@ impl<'a> FragmentIdentifier<'a> {
     }
 }
 
-macro_rules! require {
-    ($condition: expr) => {
-        if !$condition {
-            return None
-        }
-    }
-}
-
 /// Similar to <https://url.spec.whatwg.org/#concept-basic-url-parser>
 /// followed by <https://url.spec.whatwg.org/#concept-url-serializer>
 ///
@@ -196,7 +194,11 @@ fn parse_header(from_colon_to_comma: &str) -> (mime::Mime, bool) {
     // FIXME: does Mime::from_str match the MIME Sniffing Standard’s parsing algorithm?
     // <https://mimesniff.spec.whatwg.org/#parse-a-mime-type>
     let mime_type = string.parse().unwrap_or_else(|_| {
-        "text/plain;charset=US-ASCII".parse().unwrap()
+        mime::Mime {
+            type_: String::from("text"),
+            subtype: String::from("plain"),
+            parameters: vec![(String::from("charset"), String::from("US-ASCII"))],
+        }
     });
 
     (mime_type, base64)
@@ -289,7 +291,7 @@ fn decode_without_base64<F, E>(encoded_body_plus_fragment: &str, mut write_bytes
 /// <https://infra.spec.whatwg.org/#isomorphic-decode> composed with
 /// <https://infra.spec.whatwg.org/#forgiving-base64-decode>.
 fn decode_with_base64<F, E>(encoded_body_plus_fragment: &str, write_bytes: F)
-                            -> Result<Option<FragmentIdentifier>, DecodeError<E>>
+                            -> Result<Option<FragmentIdentifier>, forgiving_base64::DecodeError<E>>
     where F: FnMut(&[u8]) -> Result<(), E>
 {
     let mut decoder = forgiving_base64::Decoder::new(write_bytes);
diff --git a/src/mime.rs b/src/mime.rs
@@ -0,0 +1,170 @@
+use std::fmt::{self, Write};
+use std::str::FromStr;
+
+/// <https://mimesniff.spec.whatwg.org/#mime-type-representation>
+#[derive(Debug, PartialEq, Eq)]
+pub struct Mime {
+    pub type_: String,
+    pub subtype: String,
+    /// (name, value)
+    pub parameters: Vec<(String, String)>
+}
+
+#[derive(Debug)]
+pub struct MimeParsingError(());
+
+/// <https://mimesniff.spec.whatwg.org/#parsing-a-mime-type>
+impl FromStr for Mime {
+    type Err = MimeParsingError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        parse(s).ok_or(MimeParsingError(()))
+    }
+}
+
+fn parse(s: &str) -> Option<Mime> {
+    let trimmed = s.trim_matches(ascii_whitespace);
+
+    let (type_, rest) = split2(trimmed, '/');
+    require!(only_http_token_code_points(type_) && !type_.is_empty());
+
+    let (subtype, rest) = split2(rest?, ';');
+    let subtype = subtype.trim_right_matches(ascii_whitespace);
+    require!(only_http_token_code_points(subtype) && !subtype.is_empty());
+
+    let mut parameters = Vec::new();
+    if let Some(rest) = rest {
+        parse_parameters(rest, &mut parameters)
+    }
+
+    Some(Mime {
+        type_: type_.to_ascii_lowercase(),
+        subtype: subtype.to_ascii_lowercase(),
+        parameters,
+    })
+}
+
+fn split2(s: &str, separator: char) -> (&str, Option<&str>) {
+    let mut iter = s.splitn(2, separator);
+    let first = iter.next().unwrap();
+    (first, iter.next())
+}
+
+fn parse_parameters(s: &str, parameters: &mut Vec<(String, String)>) {
+    let mut semicolon_separated = s.split(';');
+
+    while let Some(piece) = semicolon_separated.next() {
+        let piece = piece.trim_left_matches(ascii_whitespace);
+        let (name, value) = split2(piece, '=');
+        if name.is_empty() || !only_http_token_code_points(name) || contains(&parameters, name) {
+            continue
+        }
+        if let Some(value) = value {
+            let value = if value.starts_with('"') {
+                let max_len = value.len().saturating_sub(2);  // without start or end quotes
+                let mut unescaped_value = String::with_capacity(max_len);
+                let mut chars = value[1..].chars();
+                'until_closing_quote: loop {
+                    while let Some(c) = chars.next() {
+                        match c {
+                            '"' => break 'until_closing_quote,
+                            '\\' => unescaped_value.push(chars.next().unwrap_or('\\')),
+                            _ => unescaped_value.push(c)
+                        }
+                    }
+                    if let Some(piece) = semicolon_separated.next() {
+                        // A semicolon inside a quoted value is not a separator
+                        // for the next parameter, but part of the value.
+                        unescaped_value.push(';');
+                        chars = piece.chars()
+                    } else {
+                        break
+                    }
+                }
+                if !valid_value(&unescaped_value) {
+                    continue
+                }
+                unescaped_value
+            } else {
+                let value = value.trim_right_matches(ascii_whitespace);
+                if !valid_value(value) {
+                    continue
+                }
+                value.to_owned()
+            };
+            parameters.push((name.to_ascii_lowercase(), value))
+        }
+    }
+}
+
+fn contains(parameters: &[(String, String)], name: &str) -> bool {
+    parameters.iter().any(|&(ref n, _)| n == name)
+}
+
+fn valid_value(s: &str) -> bool {
+    s.chars().all(|c| {
+        // <https://mimesniff.spec.whatwg.org/#http-quoted-string-token-code-point>
+        matches!(c, '\t' | ' '...'~' | '\u{80}'...'\u{FF}')
+    }) && !s.is_empty()
+}
+
+/// <https://mimesniff.spec.whatwg.org/#serializing-a-mime-type>
+impl fmt::Display for Mime {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.write_str(&self.type_)?;
+        f.write_str("/")?;
+        f.write_str(&self.subtype)?;
+        for &(ref name, ref value) in &self.parameters {
+            f.write_str(";")?;
+            f.write_str(name)?;
+            f.write_str("=")?;
+            if only_http_token_code_points(value) {
+                f.write_str(value)?
+            } else {
+                f.write_str("\"")?;
+                for c in value.chars() {
+                    if c == '"' || c == '\\' {
+                        f.write_str("\\")?
+                    }
+                    f.write_char(c)?
+                }
+                f.write_str("\"")?
+            }
+        }
+        Ok(())
+    }
+}
+
+fn ascii_whitespace(c: char) -> bool {
+    matches!(c, ' ' | '\t' | '\n' | '\r' | '\x0C')
+}
+
+fn only_http_token_code_points(s: &str) -> bool {
+    s.bytes().all(|byte| IS_HTTP_TOKEN[byte as usize])
+}
+
+macro_rules! byte_map {
+    ($($flag:expr,)*) => ([
+        $($flag != 0,)*
+    ])
+}
+
+// Copied from https://github.com/hyperium/mime/blob/v0.3.5/src/parse.rs#L293
+static IS_HTTP_TOKEN: [bool; 256] = byte_map![
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+    0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+];
diff --git a/tests/wpt.rs b/tests/wpt.rs
@@ -9,9 +9,9 @@ fn run_data_url(input: String, expected_mime: Option<String>, expected_body: Opt
         let url = url.unwrap();
         let (body, _) = url.decode_to_vec().unwrap();
         if expected_mime == "" {
-            assert_eq!(*url.mime_type(), "text/plain;charset=US-ASCII")
+            assert_eq!(url.mime_type().to_string(), "text/plain;charset=US-ASCII")
         } else {
-            assert_eq!(*url.mime_type(), &*expected_mime)
+            assert_eq!(url.mime_type().to_string(), expected_mime)
         }
         if let Some(expected_body) = expected_body {
             assert_eq!(body, expected_body)
@@ -26,19 +26,6 @@ fn collect_data_url<F>(add_test: &mut F)
 {
     let known_failures = [
         "data://test:test/,X",
-        "data:;%62ase64,WA",
-        "data:;base 64,WA",
-        "data:;base64;,WA",
-        "data:;base64;base64,WA",
-        "data:;charset =x,X",
-        "data:;charset,X",
-        "data:;charset=,X",
-        "data:text/plain;,X",
-        "data:text/plain;a=\",\",X",
-        "data:x/x;base64;base64,WA",
-        "data:x/x;base64;base64x,WA",
-        "data:x/x;base64;charset=x,WA",
-        "data:x/x;base64;charset=x;base64,WA",
     ];
 
     #[derive(Deserialize)]
@@ -98,8 +85,8 @@ fn collect_base64<F>(add_test: &mut F)
 fn run_mime(input: String, expected: Option<String>) {
     let result = input.parse::<data_url::mime::Mime>();
     match (result, expected) {
-        (Ok(bytes), Some(expected)) => assert_eq!(bytes, &*expected),
-        (Ok(bytes), None) => panic!("Expected error, got {:?}", bytes),
+        (Ok(mime), Some(expected)) => assert_eq!(mime.to_string(), expected),
+        (Ok(mime), None) => panic!("Expected error, got {:?}", mime),
         (Err(e), Some(expected)) => panic!("Expected {:?}, got error {:?}", expected, e),
         (Err(_), None) => {}
     }
@@ -109,13 +96,7 @@ fn run_mime(input: String, expected: Option<String>) {
 fn collect_mime<F>(add_test: &mut F)
     where F: FnMut(String, bool, rustc_test::TestFn)
 {
-    // Many WPT tests fail with the mime crate’s parser,
-    // since that parser is not written for the same spec.
-    // Only run a few of them for now, since listing all the failures individually is not useful.
-    let only_run_first_n_entries = 5;
-    let known_failures = [
-        "text/html;charset=gbk(",
-    ];
+    let known_failures = [];
 
     #[derive(Deserialize)]
     #[serde(untagged)]
@@ -129,7 +110,7 @@ fn collect_mime<F>(add_test: &mut F)
     let entries = v.into_iter().chain(v2);
 
     let mut last_comment = None;
-    for entry in entries.take(only_run_first_n_entries) {
+    for entry in entries {
         let (input, expected) = match entry {
             Entry::TestCase { input, output } => (input, output),
             Entry::Comment(s) => {
