Merge branch 'prost' of https://github.com/justxuewei/ttrpc-rust into prost

Author: jokemanfire

.github/workflows/bvt.yml

@@ -29,6 +29,7 @@ jobs:
           make -C compiler
           make -C ttrpc-codegen
           make -C example build-examples
+          make -C example2 build-examples
         # It's important for windows to fail correctly
         # https://github.com/actions/runner-images/issues/6668 
         shell: bash

Cargo.toml

@@ -12,7 +12,8 @@ description = "A Rust version of ttrpc."
 rust-version = "1.70"
 
 [dependencies]
-protobuf = { version = "3.1.0" }
+prost = { version = "0.11", optional = true }
+protobuf = "3.1.0"
 libc = { version = "0.2.59", features = [ "extra_traits" ] }
 nix = "0.26.2"
 log = "0.4"
@@ -33,11 +34,13 @@ tokio-vsock = { version = "0.4.0", optional = true }
 # lock home to avoid conflict with latest version
 home = "=0.5.9"
 protobuf-codegen = "3.1.0"
+prost-build = { version = "0.11", optional = true }
 
 [features]
 default = ["sync"]
-async = ["async-trait", "tokio", "futures", "tokio-vsock"]
+async = ["dep:async-trait", "dep:tokio", "dep:futures", "dep:tokio-vsock"]
 sync = []
+prost = ["dep:prost", "dep:prost-build"]
 
 [package.metadata.docs.rs]
 all-features = true

Makefile

@@ -1,3 +1,5 @@
+export PROTOC=${HOME}/protoc/bin/protoc
+
 all: debug test
 
 #
@@ -31,7 +33,8 @@ endif
 .PHONY: check
 check:
 	cargo fmt --all -- --check
-	cargo clippy --all-targets --all-features -- -D warnings
+	cargo clippy --all-targets --features sync,async -- -D warnings
+	cargo clippy --all-targets --features sync,async,prost -- -D warnings
 
 .PHONY: check-all
 check-all:

README.md

@@ -138,3 +138,19 @@ cargo update
 cargo install --force --path .
 ```
 3. Build your project.
+
+# ttrpc-rust with the Prost
+
+The new version of the ttrpc-rust is built with the Prost crate, a modern
+protobuf compiler written by Rust. There are certain different behaviors from
+the Rust-protobuf version:
+
+1. The protoc should be installed.
+2. Enabling "prost" feature for the ttrpc-rust.
+3. The Rust files are named based on their package name, rather than the proto
+   filename, e.g. `ttrpc = { version = "1.0", features = ["prost"] }`.
+4. Some variable names are different, e.g. for "cpu", "CPU" is generated by the
+   Rust-protobuf, and "Cpu" is genereated by the Prost.
+
+The "example" is an example with the Rust-protobuf version, and the "example2"
+is an example with the Prost version.

build.rs

@@ -7,6 +7,11 @@ fn main() {
     let path: PathBuf = [out_dir.clone(), "mod.rs".to_string()].iter().collect();
     fs::write(path, "pub mod ttrpc;").unwrap();
 
+    generate_ttrpc(&out_dir);
+}
+
+#[cfg(not(feature = "prost"))]
+fn generate_ttrpc(out_dir: &str) {
     let customize = protobuf_codegen::Customize::default()
         .gen_mod_rs(false)
         .generate_accessors(true);
@@ -20,3 +25,13 @@ fn main() {
         .run()
         .expect("Codegen failed.");
 }
+
+#[cfg(feature = "prost")]
+fn generate_ttrpc(out_dir: &str) {
+    prost_build::Config::new()
+        .out_dir(out_dir)
+        .compile_well_known_types()
+        .protoc_arg("--experimental_allow_proto3_optional")
+        .compile_protos(&["src/ttrpc.proto"], &["src"])
+        .expect("Codegen failed")
+}

codegen/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "ttrpc-codegen"
+version = "1.0.0"
+edition = "2021"
+authors = ["The Ant Group Kata Team <kata@list.alibaba-inc.com>"]
+license = "Apache-2.0"
+keywords = ["codegen", "ttrpc", "protobuf"]
+description = "Rust codegen for ttrpc using prost crate"
+categories = ["network-programming", "development-tools::build-utils"]
+repository = "https://github.com/containerd/ttrpc-rust/tree/master/codegen"
+homepage = "https://github.com/containerd/ttrpc-rust/tree/master/codegen"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+prost = "0.11"
+prost-types = "0.11"
+prost-build = "0.11"
+proc-macro2 = "1.0"
+quote = "1.0"
+anyhow = "^1.0"
+lazy_static = "1.4"
+regex = "1.7"

codegen/Makefile

@@ -0,0 +1,35 @@
+all: debug test
+
+#
+# Build
+#
+
+.PHONY: debug
+debug:
+	cargo build --verbose --all-targets
+
+.PHONY: release
+release:
+	cargo build --release
+
+.PHONY: build
+build: debug
+
+#
+# Tests and linters
+#
+
+.PHONY: test
+test:
+	cargo test --verbose
+
+.PHONY: check
+check:
+	cargo fmt --all -- --check
+	cargo clippy --all-targets --all-features -- -D warnings
+
+.PHONY: deps
+deps:
+	rustup update stable
+	rustup default stable
+	rustup component add rustfmt clippy

codegen/README.md

@@ -0,0 +1,34 @@
+# Ttrpc-rust Codegen
+
+## Getting started
+
+Please ensure that the protoc has been installed on your local environment. Then
+write the following code into "build.rs".
+
+```rust
+let mut protos = vec![
+    "protocols/protos/health.proto",
+    "protocols/protos/agent.proto",
+    "protocols/protos/oci.proto",
+];
+
+let includes = vec!["protocols/protos"];
+
+let codegen = CodegenBuilder::new()
+    .set_out_dir(&"protocols/sync")
+    .set_protos(&protos)
+    .set_includes(&includes)
+    .set_serde(true)
+    .set_async_mode(AsyncMode::None)
+    .set_generate_service(true)
+    .build()
+    .unwrap();
+codegen.generate().unwrap();
+```
+
+Add ttrpc-codegen to "build-dependencies" section in "Cargo.toml".
+
+```toml
+[build-dependencies]
+ttrpc-codegen = "1.0"
+```

codegen/src/codegen.rs

@@ -0,0 +1,189 @@
+use anyhow::{anyhow, Context, Result};
+use prost::Message;
+use prost_build::Config;
+use prost_types::FileDescriptorSet;
+use std::{
+    fs::{self, File},
+    io::{BufReader, Read, Write},
+    path::{Path, PathBuf},
+};
+
+use crate::svcgen::{AsyncMode, TtrpcServiceGenerator};
+
+const FILE_DESCRIPTOR_SET: &str = "fd_set.bin";
+
+pub struct Codegen<'a, P: AsRef<Path>> {
+    out_dir: &'a P,
+    protos: &'a [P],
+    includes: &'a [P],
+    /// Whether to enable serde
+    serde: bool,
+    async_mode: AsyncMode,
+    /// Whether to generate service
+    generate_service: bool,
+}
+
+impl<'a, P> Codegen<'a, P>
+where
+    P: AsRef<Path>,
+{
+    pub fn generate(&self) -> Result<()> {
+        self.compile_protos().context("Compile protos")?;
+        self.write_header().context("Write header")?;
+        self.clean_up().context("Clean up")?;
+
+        Ok(())
+    }
+
+    // TODO: Do not write header to the files that already has the header
+    // TODO: Write header to the files generated by the codegen
+    fn write_header(&self) -> Result<()> {
+        // Read fd_set.bin
+        let f = File::open(PathBuf::from(self.out_dir.as_ref()).join(FILE_DESCRIPTOR_SET))
+            .context("Open fd_set.bin")?;
+        let mut reader = BufReader::new(f);
+        let mut buffer = Vec::new();
+        reader.read_to_end(&mut buffer).context("Read fd_set.bin")?;
+
+        let fd_set = FileDescriptorSet::decode(&buffer as &[u8]).context("Decode fd_set")?;
+
+        for fd in fd_set.file.iter() {
+            let rs_path = PathBuf::from(self.out_dir.as_ref()).join(format!("{}.rs", fd.package()));
+            let mut f = match File::open(&rs_path) {
+                Ok(f) => f,
+                _ => continue,
+            };
+            let header = format!(
+                r#"// This file is generated by ttrpc-codegen {}. Do not edit
+// @generated
+
+"#,
+                env!("CARGO_PKG_VERSION")
+            );
+
+            let mut buf = Vec::<u8>::new();
+            buf.write(header.as_bytes()).context("Write header")?;
+            f.read_to_end(&mut buf)
+                .context(format!("Read from rust file {:?}", rs_path))?;
+            let mut f = File::create(&rs_path).context(format!("Open rust file {:?}", rs_path))?;
+            f.write_all(buf.as_slice())
+                .context(format!("Write to rust file {:?}", rs_path))?;
+        }
+
+        Ok(())
+    }
+
+    fn compile_protos(&self) -> Result<()> {
+        let mut config = Config::new();
+        config.out_dir(self.out_dir.as_ref());
+        config.service_generator(Box::new(TtrpcServiceGenerator::new(self.async_mode)));
+        config.protoc_arg("--experimental_allow_proto3_optional");
+        config.compile_well_known_types();
+        config.file_descriptor_set_path(
+            PathBuf::from(self.out_dir.as_ref()).join(FILE_DESCRIPTOR_SET),
+        );
+        if self.generate_service {
+            config.include_file("_include.rs");
+            if self.serde {
+                config
+                    .message_attribute(".", "#[derive(::serde::Serialize, ::serde::Deserialize)]");
+            }
+        }
+        config
+            .compile_protos(self.protos, self.includes)
+            .context("Compile protos by prost")?;
+        Ok(())
+    }
+
+    fn clean_up(&self) -> Result<()> {
+        fs::remove_file(PathBuf::from(self.out_dir.as_ref()).join(FILE_DESCRIPTOR_SET))
+            .context("Remove fd_set.bin")?;
+        Ok(())
+    }
+}
+
+#[derive(Default)]
+pub struct CodegenBuilder<'a, P: AsRef<Path> + Default> {
+    out_dir: Option<&'a P>,
+    protos: Option<&'a [P]>,
+    includes: Option<&'a [P]>,
+    /// Whether to enable serde
+    serde: Option<bool>,
+    async_mode: Option<AsyncMode>,
+    /// Whether to generate service
+    generate_service: Option<bool>,
+}
+
+impl<'a, P> CodegenBuilder<'a, P>
+where
+    P: AsRef<Path> + Default,
+{
+    pub fn new() -> Self {
+        Default::default()
+    }
+
+    pub fn set_out_dir(mut self, out_dir: &'a P) -> Self {
+        self.out_dir = Some(out_dir);
+        self
+    }
+
+    pub fn set_protos(mut self, protos: &'a [P]) -> Self {
+        self.protos = Some(protos);
+        self
+    }
+
+    pub fn set_includes(mut self, includes: &'a [P]) -> Self {
+        self.includes = Some(includes);
+        self
+    }
+
+    pub fn set_serde(mut self, serde: bool) -> Self {
+        self.serde = Some(serde);
+        self
+    }
+
+    pub fn set_async_mode(mut self, async_mode: AsyncMode) -> Self {
+        self.async_mode = Some(async_mode);
+        self
+    }
+
+    pub fn set_generate_service(mut self, generate_service: bool) -> Self {
+        self.generate_service = Some(generate_service);
+        self
+    }
+
+    pub fn build(&self) -> Result<Codegen<'a, P>> {
+        let out_dir = match self.out_dir {
+            Some(out_dir) => out_dir,
+            None => return Err(anyhow!("The out dir is required.")),
+        };
+
+        let protos = match self.protos {
+            Some(protos) => protos,
+            None => return Err(anyhow!("The protos are required.")),
+        };
+
+        let includes = match self.includes {
+            Some(includes) => includes,
+            None => return Err(anyhow!("The includes are required.")),
+        };
+
+        let serde = self.serde.unwrap_or(false);
+
+        let async_mode = match self.async_mode {
+            Some(mode) => mode,
+            None => AsyncMode::None,
+        };
+
+        let generate_service = self.generate_service.unwrap_or(false);
+
+        Ok(Codegen {
+            out_dir,
+            protos,
+            includes,
+            serde,
+            async_mode,
+            generate_service,
+        })
+    }
+}

codegen/src/lib.rs

@@ -0,0 +1,6 @@
+mod codegen;
+mod svcgen;
+mod util;
+
+pub use codegen::{Codegen, CodegenBuilder};
+pub use svcgen::AsyncMode;