Make prost a no_std compatible library, and prost-build able to generate no_std code

The alternative is to get collections types from `core` and `alloc`.
In the `no_std` mode in `prost_build`, we force it to always use BTreeMap
since HashMap was not stabilized in `alloc::collections` library.

The functionality is identical and the only incompatibilities in the interface
are that we cannot use `std::error::Error` or `std::io::Error` in `no_std` mode
because these types have not been moved to `alloc` and there is no alternative.

This also uprevs `bytes` dependency to 0.5, and fixes up breakage after IntoBuf api
is removed in `bytes` crate.

Note that for now, we put a patch to `bytes` to pin to a specific commit in master,
since bytes `0.5` is not released yet.

Author: cbeck88

.travis.yml

@@ -15,7 +15,7 @@ os:
   - osx
 
 rust:
-  - 1.32.0
+  - 1.36.0
   - nightly
 
 script:

Cargo.toml

@@ -25,6 +25,7 @@ members = [
   "protobuf",
   "tests",
   "tests-2015",
+  "tests-alloc",
 ]
 exclude = [
   # The fuzz crate can't be compiled or tested without the 'cargo fuzz' command,
@@ -33,11 +34,15 @@ exclude = [
 ]
 
 [features]
-default = ["prost-derive"]
+default = ["prost-derive", "std"]
 no-recursion-limit = []
+std = []
+   # When std is disabled, we attempt to provide no_std support in prost
+   # Config::use_alloc_collections() should be set when using prost_build in your build.rs
+   # so that generated files will not have std:: either, and will use alloc crate instead
 
 [dependencies]
-bytes = "0.4.7"
+bytes = { version = "0.5", default-features = false }
 prost-derive = { version = "0.5.0", path = "prost-derive", optional = true }
 
 [dev-dependencies]
@@ -54,3 +59,6 @@ name = "varint"
 [[bench]]
 name = "benchmark"
 harness = false
+
+[patch.crates-io]
+bytes = { git = "https://github.com/tokio-rs/bytes", rev = "c17e40115f5bb2a2db71ed90dceae6ec643dc024" }

conformance/Cargo.toml

@@ -6,7 +6,7 @@ publish = false
 edition = "2018"
 
 [dependencies]
-bytes = "0.4.7"
+bytes = "0.5"
 env_logger = { version = "0.6", default-features = false }
 log = "0.3"
 prost = { path = ".." }

prost-build/Cargo.toml

@@ -11,7 +11,7 @@ description = "A Protocol Buffers implementation for the Rust Language."
 edition = "2018"
 
 [dependencies]
-bytes = "0.4.7"
+bytes = "0.5"
 heck = "0.3"
 itertools = "0.8"
 log = "0.4"

prost-build/src/code_generator.rs

@@ -17,6 +17,7 @@ use crate::ast::{Comments, Method, Service};
 use crate::extern_paths::ExternPaths;
 use crate::ident::{match_ident, to_snake, to_upper_camel};
 use crate::message_graph::MessageGraph;
+use crate::CollectionsLib;
 use crate::Config;
 
 #[derive(PartialEq)]
@@ -366,12 +367,14 @@ impl<'a> CodeGenerator<'a> {
         self.buf.push_str(&to_snake(field.name()));
         self.buf.push_str(": ");
         if repeated {
-            self.buf.push_str("::std::vec::Vec<");
+            self.buf.push_str(self.config.collections_lib.to_str());
+            self.buf.push_str("::vec::Vec<");
         } else if optional {
-            self.buf.push_str("::std::option::Option<");
+            self.buf.push_str("::core::option::Option<");
         }
         if boxed {
-            self.buf.push_str("::std::boxed::Box<");
+            self.buf.push_str(self.config.collections_lib.to_str());
+            self.buf.push_str("::boxed::Box<");
         }
         self.buf.push_str(&ty);
         if boxed {
@@ -403,7 +406,7 @@ impl<'a> CodeGenerator<'a> {
         self.append_doc();
         self.push_indent();
 
-        let btree_map = self
+        let btree_map = (self.config.collections_lib != CollectionsLib::Std) || self
             .config
             .btree_map
             .iter()
@@ -426,8 +429,9 @@ impl<'a> CodeGenerator<'a> {
         self.append_field_attributes(msg_name, field.name());
         self.push_indent();
         self.buf.push_str(&format!(
-            "pub {}: ::std::collections::{}<{}, {}>,\n",
+            "pub {}: {}::collections::{}<{}, {}>,\n",
             to_snake(field.name()),
+            self.config.collections_lib.to_str(),
             rust_ty,
             key_ty,
             value_ty
@@ -459,7 +463,7 @@ impl<'a> CodeGenerator<'a> {
         self.append_field_attributes(fq_message_name, oneof.name());
         self.push_indent();
         self.buf.push_str(&format!(
-            "pub {}: ::std::option::Option<{}>,\n",
+            "pub {}: ::core::option::Option<{}>,\n",
             to_snake(oneof.name()),
             name
         ));
@@ -713,8 +717,8 @@ impl<'a> CodeGenerator<'a> {
             Type::Int32 | Type::Sfixed32 | Type::Sint32 | Type::Enum => String::from("i32"),
             Type::Int64 | Type::Sfixed64 | Type::Sint64 => String::from("i64"),
             Type::Bool => String::from("bool"),
-            Type::String => String::from("std::string::String"),
-            Type::Bytes => String::from("std::vec::Vec<u8>"),
+            Type::String => format!("{}::string::String", self.config.collections_lib.to_str()),
+            Type::Bytes => format!("{}::vec::Vec<u8>", self.config.collections_lib.to_str()),
             Type::Group | Type::Message => self.resolve_ident(field.type_name()),
         }
     }

prost-build/src/lib.rs

@@ -165,11 +165,32 @@ pub trait ServiceGenerator {
     fn finalize(&mut self, _buf: &mut String) {}
 }
 
+/// Configuration enum for whether to use `std` prefixes or `alloc` prefixes in generated code
+///
+/// This option also forces Btree everywhere, overriding the BtreeMap options,
+/// since HashMap is not in alloc::collections, only std (it requires randomness)
+/// 
+#[derive(PartialEq)]
+pub enum CollectionsLib {
+    Std,
+    Alloc,
+}
+
+impl CollectionsLib {
+    pub fn to_str(&self) -> &'static str {
+        match self {
+            CollectionsLib::Std => { "::std" },
+            CollectionsLib::Alloc => { "::alloc" },
+        }
+    }
+}
+
 /// Configuration options for Protobuf code generation.
 ///
 /// This configuration builder can be used to set non-default code generation options.
 pub struct Config {
     service_generator: Option<Box<dyn ServiceGenerator>>,
+    collections_lib: CollectionsLib,
     btree_map: Vec<String>,
     type_attributes: Vec<(String, String)>,
     field_attributes: Vec<(String, String)>,
@@ -460,6 +481,15 @@ impl Config {
         self
     }
 
+    /// Configure the code generator to use the `::alloc` namespace rather than `::std`, and Btree everywhere
+    /// rather than `std`.
+    ///
+    /// This allows generated code to be used in a `#![no_std]` crate
+    pub fn use_alloc_collections_lib(&mut self) -> &mut Self {
+        self.collections_lib = CollectionsLib::Alloc;
+        self
+    }
+
     /// Configures the output directory where generated Rust files will be written.
     ///
     /// If unset, defaults to the `OUT_DIR` environment variable. `OUT_DIR` is set by Cargo when
@@ -538,7 +568,7 @@ impl Config {
 
         let mut buf = Vec::new();
         fs::File::open(descriptor_set)?.read_to_end(&mut buf)?;
-        let descriptor_set = FileDescriptorSet::decode(&buf)?;
+        let descriptor_set = FileDescriptorSet::decode(&buf[..])?;
 
         let modules = self.generate(descriptor_set.file)?;
         for (module, content) in modules {
@@ -581,6 +611,7 @@ impl default::Default for Config {
     fn default() -> Config {
         Config {
             service_generator: None,
+            collections_lib: CollectionsLib::Std,
             btree_map: Vec::new(),
             type_attributes: Vec::new(),
             field_attributes: Vec::new(),

prost-derive/src/field/group.rs

@@ -120,7 +120,7 @@ impl Field {
 
     pub fn clear(&self, ident: TokenStream) -> TokenStream {
         match self.label {
-            Label::Optional => quote!(#ident = ::std::option::Option::None),
+            Label::Optional => quote!(#ident = ::core::option::Option::None),
             Label::Required => quote!(#ident.clear()),
             Label::Repeated => quote!(#ident.clear()),
         }

prost-derive/src/field/map.rs

@@ -223,10 +223,10 @@ impl Field {
             };
 
             Some(quote! {
-                pub fn #get(&self, key: #key_ref_ty) -> ::std::option::Option<#ty> {
+                pub fn #get(&self, key: #key_ref_ty) -> ::core::option::Option<#ty> {
                     self.#ident.get(#take_ref key).cloned().and_then(#ty::from_i32)
                 }
-                pub fn #insert(&mut self, key: #key_ty, value: #ty) -> ::std::option::Option<#ty> {
+                pub fn #insert(&mut self, key: #key_ty, value: #ty) -> ::core::option::Option<#ty> {
                     self.#ident.insert(key, value as i32).and_then(#ty::from_i32)
                 }
             })
@@ -249,7 +249,7 @@ impl Field {
         let key = self.key_ty.rust_type();
         let value_wrapper = self.value_ty.debug();
         let fmt = quote! {
-            fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+            fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
                 #key_wrapper
                 #value_wrapper
                 let mut builder = f.debug_map();
@@ -263,17 +263,17 @@ impl Field {
             ValueTy::Scalar(ref ty) => {
                 let value = ty.rust_type();
                 quote! {
-                    struct #wrapper_name<'a>(&'a ::std::collections::#type_name<#key, #value>);
-                    impl<'a> ::std::fmt::Debug for #wrapper_name<'a> {
+                    struct #wrapper_name<'a>(&'a ::alloc::collections::#type_name<#key, #value>);
+                    impl<'a> ::core::fmt::Debug for #wrapper_name<'a> {
                         #fmt
                     }
                 }
             }
             ValueTy::Message => quote! {
-                struct #wrapper_name<'a, V: 'a>(&'a ::std::collections::#type_name<#key, V>);
-                impl<'a, V> ::std::fmt::Debug for #wrapper_name<'a, V>
+                struct #wrapper_name<'a, V: 'a>(&'a ::alloc::collections::#type_name<#key, V>);
+                impl<'a, V> ::core::fmt::Debug for #wrapper_name<'a, V>
                 where
-                    V: ::std::fmt::Debug + 'a,
+                    V: ::core::fmt::Debug + 'a,
                 {
                     #fmt
                 }

prost-derive/src/field/message.rs

@@ -123,7 +123,7 @@ impl Field {
 
     pub fn clear(&self, ident: TokenStream) -> TokenStream {
         match self.label {
-            Label::Optional => quote!(#ident = ::std::option::Option::None),
+            Label::Optional => quote!(#ident = ::core::option::Option::None),
             Label::Required => quote!(#ident.clear()),
             Label::Repeated => quote!(#ident.clear()),
         }

prost-derive/src/lib.rs

@@ -191,7 +191,7 @@ fn try_message(input: TokenStream) -> Result<TokenStream, Error> {
                                   wire_type: _prost::encoding::WireType,
                                   buf: &mut B,
                                   ctx: _prost::encoding::DecodeContext,
-                ) -> ::std::result::Result<(), _prost::DecodeError>
+                ) -> ::core::result::Result<(), _prost::DecodeError>
                 where B: _bytes::Buf {
                     #struct_name
                     match tag {
@@ -218,8 +218,8 @@ fn try_message(input: TokenStream) -> Result<TokenStream, Error> {
                 }
             }
 
-            impl ::std::fmt::Debug for #ident {
-                fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+            impl ::core::fmt::Debug for #ident {
+                fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
                     let mut builder = #debug_builder;
                     #(#debugs;)*
                     builder.finish()
@@ -286,7 +286,7 @@ fn try_enumeration(input: TokenStream) -> Result<TokenStream, Error> {
         .iter()
         .map(|&(_, ref value)| quote!(#value => true));
     let from = variants.iter().map(
-        |&(ref variant, ref value)| quote!(#value => ::std::option::Option::Some(#ident::#variant)),
+        |&(ref variant, ref value)| quote!(#value => ::core::option::Option::Some(#ident::#variant)),
     );
 
     let is_valid_doc = format!("Returns `true` if `value` is a variant of `{}`.", ident);
@@ -309,21 +309,21 @@ fn try_enumeration(input: TokenStream) -> Result<TokenStream, Error> {
                 }
 
                 #[doc=#from_i32_doc]
-                pub fn from_i32(value: i32) -> ::std::option::Option<#ident> {
+                pub fn from_i32(value: i32) -> ::core::option::Option<#ident> {
                     match value {
                         #(#from,)*
-                        _ => ::std::option::Option::None,
+                        _ => ::core::option::Option::None,
                     }
                 }
             }
 
-            impl ::std::default::Default for #ident {
+            impl ::core::default::Default for #ident {
                 fn default() -> #ident {
                     #ident::#default
                 }
             }
 
-            impl ::std::convert::From<#ident> for i32 {
+            impl ::core::convert::From<#ident> for i32 {
                 fn from(value: #ident) -> i32 {
                     value as i32
                 }
@@ -411,8 +411,8 @@ fn try_oneof(input: TokenStream) -> Result<TokenStream, Error> {
         let merge = field.merge(quote!(value));
         quote! {
             #tag => {
-                let mut value = ::std::default::Default::default();
-                #merge.map(|_| *field = ::std::option::Option::Some(#ident::#variant_ident(value)))
+                let mut value = ::core::default::Default::default();
+                #merge.map(|_| *field = ::core::option::Option::Some(#ident::#variant_ident(value)))
             }
         }
     });
@@ -445,12 +445,12 @@ fn try_oneof(input: TokenStream) -> Result<TokenStream, Error> {
                     }
                 }
 
-                pub fn merge<B>(field: &mut ::std::option::Option<#ident>,
+                pub fn merge<B>(field: &mut ::core::option::Option<#ident>,
                                 tag: u32,
                                 wire_type: _prost::encoding::WireType,
                                 buf: &mut B,
                                 ctx: _prost::encoding::DecodeContext,
-                ) -> ::std::result::Result<(), _prost::DecodeError>
+                ) -> ::core::result::Result<(), _prost::DecodeError>
                 where B: _bytes::Buf {
                     match tag {
                         #(#merge,)*
@@ -466,8 +466,8 @@ fn try_oneof(input: TokenStream) -> Result<TokenStream, Error> {
                 }
             }
 
-            impl ::std::fmt::Debug for #ident {
-                fn fmt(&self, f: &mut ::std::fmt::Formatter) -> ::std::fmt::Result {
+            impl ::core::fmt::Debug for #ident {
+                fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
                     match *self {
                         #(#debug,)*
                     }