JSON-RPC: add get_http_blob API

Author: link2xt

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+### Changes
+- Add `get_http_blob` JSON-RPC API.
+
 ## [1.112.7] - 2023-04-17
 
 ### Fixes

deltachat-jsonrpc/src/api/mod.rs

@@ -1610,6 +1610,17 @@ impl CommandApi {
         Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
     }
 
+    /// Makes an HTTP GET request and returns base64-encoded contents.
+    ///
+    /// `url` is the HTTP or HTTPS URL.
+    async fn get_http_blob(&self, account_id: u32, url: String) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        let blob = deltachat::net::read_url_blob(&ctx, &url).await?;
+
+        use base64::{engine::general_purpose, Engine as _};
+        Ok(general_purpose::STANDARD_NO_PAD.encode(blob))
+    }
+
     /// Forward messages to another chat.
     ///
     /// All types of messages can be forwarded,

src/configure.rs

@@ -2,7 +2,6 @@
 
 mod auto_mozilla;
 mod auto_outlook;
-mod read_url;
 mod server_params;
 
 use anyhow::{bail, ensure, Context as _, Result};

src/configure/auto_mozilla.rs

@@ -6,10 +6,10 @@ use std::str::FromStr;
 
 use quick_xml::events::{BytesStart, Event};
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 #[derive(Debug)]

src/configure/auto_outlook.rs

@@ -7,9 +7,9 @@ use std::io::BufRead;
 
 use quick_xml::events::Event;
 
-use super::read_url::read_url;
 use super::{Error, ServerParams};
 use crate::context::Context;
+use crate::net::read_url;
 use crate::provider::{Protocol, Socket};
 
 /// Result of parsing a single `Protocol` tag.

src/configure/read_url.rs

@@ -68,7 +68,6 @@ mod decrypt;
 pub mod download;
 mod e2ee;
 pub mod ephemeral;
-mod http;
 mod imap;
 pub mod imex;
 mod scheduler;
@@ -104,7 +103,7 @@ mod dehtml;
 mod authres;
 mod color;
 pub mod html;
-mod net;
+pub mod net;
 pub mod plaintext;
 pub mod summary;
 

src/http.rs

@@ -1,4 +1,4 @@
-///! # Common network utilities.
+//! # Common network utilities.
 use std::net::{IpAddr, SocketAddr};
 use std::pin::Pin;
 use std::str::FromStr;
@@ -12,9 +12,12 @@ use tokio_io_timeout::TimeoutStream;
 use crate::context::Context;
 use crate::tools::time;
 
+pub(crate) mod http;
 pub(crate) mod session;
 pub(crate) mod tls;
 
+pub use http::{read_url, read_url_blob};
+
 async fn connect_tcp_inner(addr: SocketAddr, timeout_val: Duration) -> Result<TcpStream> {
     let tcp_stream = timeout(timeout_val, TcpStream::connect(addr))
         .await

src/lib.rs

@@ -0,0 +1,62 @@
+//! # HTTP module.
+
+use std::time::Duration;
+
+use anyhow::{anyhow, Result};
+
+use crate::context::Context;
+use crate::socks::Socks5Config;
+
+const HTTP_TIMEOUT: Duration = Duration::from_secs(30);
+
+/// Retrieves the text contents of URL using HTTP GET request.
+pub async fn read_url(context: &Context, url: &str) -> Result<String> {
+    Ok(read_url_inner(context, url).await?.text().await?)
+}
+
+/// Retrieves the binary contents of URL using HTTP GET request.
+pub async fn read_url_blob(context: &Context, url: &str) -> Result<Vec<u8>> {
+    Ok(read_url_inner(context, url).await?.bytes().await?.into())
+}
+
+async fn read_url_inner(context: &Context, url: &str) -> Result<reqwest::Response> {
+    let socks5_config = Socks5Config::from_database(&context.sql).await?;
+    let client = get_client(socks5_config)?;
+    let mut url = url.to_string();
+
+    // Follow up to 10 http-redirects
+    for _i in 0..10 {
+        let response = client.get(&url).send().await?;
+        if response.status().is_redirection() {
+            let headers = response.headers();
+            let header = headers
+                .get_all("location")
+                .iter()
+                .last()
+                .ok_or_else(|| anyhow!("Redirection doesn't have a target location"))?
+                .to_str()?;
+            info!(context, "Following redirect to {}", header);
+            url = header.to_string();
+            continue;
+        }
+
+        return Ok(response);
+    }
+
+    Err(anyhow!("Followed 10 redirections"))
+}
+
+pub(crate) fn get_client(socks5_config: Option<Socks5Config>) -> Result<reqwest::Client> {
+    let builder = reqwest::ClientBuilder::new().timeout(HTTP_TIMEOUT);
+    let builder = if let Some(socks5_config) = socks5_config {
+        let proxy = reqwest::Proxy::all(socks5_config.to_url())?;
+        builder.proxy(proxy)
+    } else {
+        // Disable usage of "system" proxy configured via environment variables.
+        // It is enabled by default in `reqwest`, see
+        // <https://docs.rs/reqwest/0.11.14/reqwest/struct.ClientBuilder.html#method.no_proxy>
+        // for documentation.
+        builder.no_proxy()
+    };
+    Ok(builder.build()?)
+}