Auto merge of #361 - servo:percent-encoding-docs, r=SimonSapin,nox

Percent encoding docs

Fixes #298, fixes #355.

This is a rebase of #355 with some additional changes. Original work by @AndyGauge.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/rust-url/361)
<!-- Reviewable:end -->

Author: bors-servo

rust-url-todo

@@ -0,0 +1,14 @@
+* standalone path parsing?
+* Test setters
+  * Test trim C0/space
+  * Test remove tab & newline
+
+
+
+#[test]
+fn test_path_segments() {
+    let mut url = Url::parse("http://example.net").unwrap();
+    url.push_path_segment("foo").unwrap();
+    url.extend_path_segments(&["bar", "b/az"]).unwrap();
+    assert_eq!(url.as_str(), "http://example.net/foo");
+}

src/percent_encoding.rs

@@ -6,6 +6,32 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+//! URLs use special chacters to indicate the parts of the request.  For example, a forward slash
+//! indicates a path.  In order for that charcter to exist outside of a path separator, that
+//! charcter would need to be encoded.
+//!
+//! Percent encoding replaces reserved charcters with the `%` escape charcter followed by hexidecimal
+//! ASCII representaton.  For non-ASCII charcters that are percent encoded, a UTF-8 byte sequence
+//! becomes percent encoded.  A simple example can be seen when the space literal is replaced with
+//! `%20`.
+//!
+//! Percent encoding is further complicated by the fact that different parts of an URL have
+//! different encoding requirements.  In order to support the variety of encoding requirements,
+//! `url::percent_encoding` includes different *encode sets*.
+//! See [URL Standard](https://url.spec.whatwg.org/#percent-encoded-bytes) for details.
+//!
+//! This module provides some `*_ENCODE_SET` constants.
+//! If a different set is required, it can be created with
+//! the [`define_encode_set!`](../macro.define_encode_set!.html) macro.
+//!
+//! # Examples
+//!
+//! ```
+//! use url::percent_encoding::{utf8_percent_encode, DEFAULT_ENCODE_SET};
+//!
+//! assert_eq!(utf8_percent_encode("foo bar?", DEFAULT_ENCODE_SET).to_string(), "foo%20bar%3F");
+//! ```
+
 use encoding;
 use std::ascii::AsciiExt;
 use std::borrow::Cow;
@@ -77,6 +103,9 @@ macro_rules! define_encode_set {
 }
 
 /// This encode set is used for the path of cannot-be-a-base URLs.
+///
+/// All ASCII charcters less than hexidecimal 20 and greater than 7E are encoded.  This includes
+/// special charcters such as line feed, carriage return, NULL, etc.
 #[derive(Copy, Clone, Debug)]
 #[allow(non_camel_case_types)]
 pub struct SIMPLE_ENCODE_SET;
@@ -90,21 +119,39 @@ impl EncodeSet for SIMPLE_ENCODE_SET {
 
 define_encode_set! {
     /// This encode set is used in the URL parser for query strings.
+    ///
+    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+    /// space, double quote ("), hash (#), and inequality qualifiers (<), (>) are encoded.
     pub QUERY_ENCODE_SET = [SIMPLE_ENCODE_SET] | {' ', '"', '#', '<', '>'}
 }
 
 define_encode_set! {
     /// This encode set is used for path components.
+    ///
+    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+    /// question mark (?), and curly brackets ({), (}) are encoded.
     pub DEFAULT_ENCODE_SET = [QUERY_ENCODE_SET] | {'`', '?', '{', '}'}
 }
 
 define_encode_set! {
     /// This encode set is used for on '/'-separated path segment
+    ///
+    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+    /// question mark (?), and curly brackets ({), (}), percent sign (%), forward slash (/) are
+    /// encoded.
     pub PATH_SEGMENT_ENCODE_SET = [DEFAULT_ENCODE_SET] | {'%', '/'}
 }
 
 define_encode_set! {
     /// This encode set is used for username and password.
+    ///
+    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
+    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
+    /// question mark (?), and curly brackets ({), (}), forward slash (/), colon (:), semi-colon (;),
+    /// equality (=), at (@), backslash (\\), square brackets ([), (]), caret (\^), and pipe (|) are
+    /// encoded.
     pub USERINFO_ENCODE_SET = [DEFAULT_ENCODE_SET] | {
         '/', ':', ';', '=', '@', '[', '\\', ']', '^', '|'
     }
@@ -113,6 +160,15 @@ define_encode_set! {
 /// Return the percent-encoding of the given bytes.
 ///
 /// This is unconditional, unlike `percent_encode()` which uses an encode set.
+///
+/// # Examples
+///
+/// ```
+/// use url::percent_encoding::percent_encode_byte;
+///
+/// assert_eq!("foo bar".bytes().map(percent_encode_byte).collect::<String>(),
+///            "%66%6F%6F%20%62%61%72");
+/// ```
 pub fn percent_encode_byte(byte: u8) -> &'static str {
     let index = usize::from(byte) * 3;
     &"\
@@ -146,6 +202,14 @@ pub fn percent_encode_byte(byte: u8) -> &'static str {
 /// that also implements `Display` and `Into<Cow<str>>`.
 /// The latter returns `Cow::Borrowed` when none of the bytes in `input`
 /// are in the given encode set.
+///
+/// # Examples
+///
+/// ```
+/// use url::percent_encoding::{percent_encode, DEFAULT_ENCODE_SET};
+///
+/// assert_eq!(percent_encode(b"foo bar?", DEFAULT_ENCODE_SET).to_string(), "foo%20bar%3F");
+/// ```
 #[inline]
 pub fn percent_encode<E: EncodeSet>(input: &[u8], encode_set: E) -> PercentEncode<E> {
     PercentEncode {
@@ -157,6 +221,14 @@ pub fn percent_encode<E: EncodeSet>(input: &[u8], encode_set: E) -> PercentEncod
 /// Percent-encode the UTF-8 encoding of the given string.
 ///
 /// See `percent_encode()` for how to use the return value.
+///
+/// # Examples
+///
+/// ```
+/// use url::percent_encoding::{utf8_percent_encode, DEFAULT_ENCODE_SET};
+///
+/// assert_eq!(utf8_percent_encode("foo bar?", DEFAULT_ENCODE_SET).to_string(), "foo%20bar%3F");
+/// ```
 #[inline]
 pub fn utf8_percent_encode<E: EncodeSet>(input: &str, encode_set: E) -> PercentEncode<E> {
     percent_encode(input.as_bytes(), encode_set)
@@ -241,6 +313,14 @@ impl<'a, E: EncodeSet> From<PercentEncode<'a, E>> for Cow<'a, str> {
 /// that also implements `Into<Cow<u8>>`
 /// (which returns `Cow::Borrowed` when `input` contains no percent-encoded sequence)
 /// and has `decode_utf8()` and `decode_utf8_lossy()` methods.
+///
+/// # Examples
+///
+/// ```
+/// use url::percent_encoding::percent_decode;
+///
+/// assert_eq!(percent_decode(b"foo%20bar%3F").decode_utf8().unwrap(), "foo bar?");
+/// ```
 #[inline]
 pub fn percent_decode(input: &[u8]) -> PercentDecode {
     PercentDecode {