From 51bccd7e990960142f4dd6cffd4a321775051136 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Mon, 14 Apr 2025 23:10:33 +0200 Subject: [PATCH 01/18] Make self reporting into a setting --- deltachat-jsonrpc/src/api.rs | 3 +- src/config.rs | 7 +++++ src/context.rs | 59 ++++++++++++++++++++++++++++++------ src/context/context_tests.rs | 11 +++---- src/scheduler.rs | 14 +++++++++ 5 files changed, 76 insertions(+), 18 deletions(-) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index a7f21f9bd8..bdf723fd2d 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -375,9 +375,10 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } + /// Deprecated 2025-04. Use the "self_reporting" config instead. async fn draft_self_report(&self, account_id: u32) -> Result { let ctx = self.get_context(account_id).await?; - Ok(ctx.draft_self_report().await?.to_u32()) + Ok(ctx.send_self_report().await?.to_u32()) } /// Sets the given configuration key. diff --git a/src/config.rs b/src/config.rs index 3c143a78b2..8078338394 100644 --- a/src/config.rs +++ b/src/config.rs @@ -428,6 +428,13 @@ pub enum Config { /// used for signatures, encryption to self and included in `Autocrypt` header. KeyId, + /// Send statistics to Delta Chat's developers. + /// Can be exposed to the user as a setting. + SelfReporting, + + /// Last time statistics were sent to Delta Chat's developers + LastSelfReportSent, + /// This key is sent to the self_reporting bot so that the bot can recognize the user /// without storing the email address SelfReportingId, diff --git a/src/context.rs b/src/context.rs index d5cfe1458f..2f88bbf539 100644 --- a/src/context.rs +++ b/src/context.rs @@ -8,32 +8,33 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; -use anyhow::{Context as _, Result, bail, ensure}; +use anyhow::{bail, ensure, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{ChatId, ProtectionStatus, get_chat_cnt}; +use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus}; use crate::chatlist_events; use crate::config::Config; use crate::constants::{ self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, }; -use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified}; +use crate::contact::{import_vcard, mark_contact_id_as_verified, Contact, ContactId}; use crate::debug_logging::DebugLogging; use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_secret_key, self_fingerprint}; +use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _}; +use crate::log::LogExt; use crate::log::{info, warn}; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId}; +use crate::message::{self, Message, MessageState, MsgId, Viewtype}; use crate::param::{Param, Params}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; -use crate::scheduler::{SchedulerState, convert_folder_meaning}; +use crate::scheduler::{convert_folder_meaning, SchedulerState}; use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; @@ -1041,6 +1042,18 @@ impl Context { .await? .to_string(), ); + res.insert( + "self_reporting", + self.get_config_bool(Config::SelfReporting) + .await? + .to_string(), + ); + res.insert( + "last_self_report_sent", + self.get_config_i64(Config::LastSelfReportSent) + .await? + .to_string(), + ); let elapsed = time_elapsed(&self.creation_time); res.insert("uptime", duration_to_str(elapsed)); @@ -1160,7 +1173,8 @@ impl Context { Some(id) => id, None => { let id = create_id(); - self.set_config(Config::SelfReportingId, Some(&id)).await?; + self.set_config_internal(Config::SelfReportingId, Some(&id)) + .await?; id } }; @@ -1174,7 +1188,15 @@ impl Context { /// /// On the other end, a bot will receive the message and make it available /// to Delta Chat's developers. - pub async fn draft_self_report(&self) -> Result { + pub async fn send_self_report(&self) -> Result { + info!(self, "Sending self report."); + // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not + // work out for whatever reason or are interrupted by the OS. + self.set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) + .await + .log_err(self) + .ok(); + const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) .await? @@ -1187,9 +1209,26 @@ impl Context { .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) .await?; - let mut msg = Message::new_text(self.get_self_report().await?); + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + msg.set_file_from_bytes( + self, + "statistics.txt", + self.get_self_report().await?.as_bytes(), + Some("text/plain"), + )?; - chat_id.set_draft(self, Some(&mut msg)).await?; + crate::chat::send_msg(self, chat_id, &mut msg) + .await + .context("Failed to send self_reporting message") + .log_err(self) + .ok(); Ok(chat_id) } diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 8850012841..a43cd403d4 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -602,18 +602,15 @@ async fn test_get_next_msgs() -> Result<()> { async fn test_draft_self_report() -> Result<()> { let alice = TestContext::new_alice().await; - let chat_id = alice.draft_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 1).await; + let chat_id = alice.send_self_report().await?; + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); let chat = Chat::load_from_db(&alice, chat_id).await?; assert!(chat.is_protected()); - let mut draft = chat_id.get_draft(&alice).await?.unwrap(); - assert!(draft.text.starts_with("core_version")); - - // Test that sending into the protected chat works: - let _sent = alice.send_msg(chat_id, &mut draft).await; + let statistics_msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(statistics_msg.get_filename().unwrap(), "statistics.txt"); Ok(()) } diff --git a/src/scheduler.rs b/src/scheduler.rs index d4fc5c3b54..acc041d1c8 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -507,6 +507,20 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; + //#[cfg(target_os = "android")] TODO + if ctx.get_config_bool(Config::SelfReporting).await? { + match ctx.get_config_i64(Config::LastSelfReportSent).await { + Ok(last_selfreport_time) => { + let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_selfreport_time <= time() { + ctx.send_self_report().await?; + } + } + Err(err) => { + warn!(ctx, "Failed to get last self_reporting time: {}", err); + } + } + } match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { From 78417d90f03897ab0bd781398a2d0ffbd49bf64d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 15 Apr 2025 17:23:45 +0200 Subject: [PATCH 02/18] Put the statistics into a json attachment --- src/context.rs | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/context.rs b/src/context.rs index 2f88bbf539..f6b1bb0899 100644 --- a/src/context.rs +++ b/src/context.rs @@ -12,6 +12,7 @@ use anyhow::{bail, ensure, Context as _, Result}; use async_channel::{self as channel, Receiver, Sender}; use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; +use serde::Serialize; use tokio::sync::{Mutex, Notify, RwLock}; use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus}; @@ -1062,7 +1063,17 @@ impl Context { } async fn get_self_report(&self) -> Result { - #[derive(Default)] + #[derive(Serialize)] + struct Statistics { + core_version: String, + num_msgs: u32, + num_chats: u32, + db_size: u64, + key_created: i64, + chat_numbers: ChatNumbers, + self_reporting_id: String, + } + #[derive(Default, Serialize)] struct ChatNumbers { protected: u32, protection_broken: u32, @@ -1072,9 +1083,6 @@ impl Context { unencrypted_mua: u32, } - let mut res = String::new(); - res += &format!("core_version {}\n", get_version_str()); - let num_msgs: u32 = self .sql .query_get_value( @@ -1083,21 +1091,20 @@ impl Context { ) .await? .unwrap_or_default(); - res += &format!("num_msgs {num_msgs}\n"); let num_chats: u32 = self .sql .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) .await? .unwrap_or_default(); - res += &format!("num_chats {num_chats}\n"); let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - res += &format!("db_size_bytes {db_size}\n"); - let secret_key = &load_self_secret_key(self).await?.primary_key; - let key_created = secret_key.public_key().created_at().timestamp(); - res += &format!("key_created {key_created}\n"); + let key_created = load_self_secret_key(self) + .await? + .primary_key + .created_at() + .timestamp(); // how many of the chats active in the last months are: // - protected @@ -1107,7 +1114,7 @@ impl Context { // - unencrypted and the contact uses Delta Chat // - unencrypted and the contact uses a classical MUA let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chats = self + let chat_numbers = self .sql .query_map( "SELECT c.protected, m.param, m.msgrmsg @@ -1162,12 +1169,6 @@ impl Context { }, ) .await?; - res += &format!("chats_protected {}\n", chats.protected); - res += &format!("chats_protection_broken {}\n", chats.protection_broken); - res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc); - res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua); - res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc); - res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua); let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { Some(id) => id, @@ -1178,9 +1179,17 @@ impl Context { id } }; - res += &format!("self_reporting_id {self_reporting_id}"); + let statistics = Statistics { + core_version: get_version_str().to_string(), + num_msgs, + num_chats, + db_size, + key_created, + chat_numbers, + self_reporting_id, + }; - Ok(res) + Ok(serde_json::to_string_pretty(&statistics)?) } /// Drafts a message with statistics about the usage of Delta Chat. From 1473ea07ff151ae0e66838eb20a86b67e0392d82 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 19 Jun 2025 15:39:26 +0200 Subject: [PATCH 03/18] Mute&archive self-reporting bot chat --- src/context.rs | 15 +++++++++++++-- src/receive_imf.rs | 6 +++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/context.rs b/src/context.rs index f6b1bb0899..011976630b 100644 --- a/src/context.rs +++ b/src/context.rs @@ -15,7 +15,7 @@ use ratelimit::Ratelimit; use serde::Serialize; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{get_chat_cnt, ChatId, ProtectionStatus}; +use crate::chat::{self, get_chat_cnt, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use crate::chatlist_events; use crate::config::Config; use crate::constants::{ @@ -1213,7 +1213,18 @@ impl Context { .context("Self reporting bot vCard does not contain a contact")?; mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?; - let chat_id = ChatId::create_for_contact(self, contact_id).await?; + let chat_id = if let Some(res) = ChatId::lookup_by_contact(self, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(self, contact_id).await?; + chat_id + .set_visibility(self, ChatVisibility::Archived) + .await?; + chat::set_muted(self, chat_id, MuteDuration::Forever).await?; + chat_id + }; + chat_id .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) .await?; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 86d1577ace..59d48a6b80 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -1709,7 +1709,11 @@ async fn add_parts( let state = if !mime_parser.incoming { MessageState::OutDelivered - } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent + } else if seen + || is_mdn + || chat_id_blocked == Blocked::Yes + || group_changes.silent + || mime_parser.from.addr == "self_reporting@testrun.org" // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen From a72c5463e7dfeb1b835ed81796a883cef51a0114 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 19 Jun 2025 20:00:36 +0200 Subject: [PATCH 04/18] Move self-reporting logic into new file self_reporting.rs --- src/context.rs | 193 +---------------------- src/context/context_tests.rs | 17 --- src/lib.rs | 1 + src/scheduler.rs | 16 +- src/self_reporting.rs | 289 +++++++++++++++++++++++++++++++++++ 5 files changed, 293 insertions(+), 223 deletions(-) create mode 100644 src/self_reporting.rs diff --git a/src/context.rs b/src/context.rs index 011976630b..69f4db1688 100644 --- a/src/context.rs +++ b/src/context.rs @@ -26,7 +26,7 @@ use crate::debug_logging::DebugLogging; use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _}; +use crate::key::{load_self_public_key, load_self_secret_key, self_fingerprint, DcKey as _}; use crate::log::LogExt; use crate::log::{info, warn}; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; @@ -1062,197 +1062,6 @@ impl Context { Ok(res) } - async fn get_self_report(&self) -> Result { - #[derive(Serialize)] - struct Statistics { - core_version: String, - num_msgs: u32, - num_chats: u32, - db_size: u64, - key_created: i64, - chat_numbers: ChatNumbers, - self_reporting_id: String, - } - #[derive(Default, Serialize)] - struct ChatNumbers { - protected: u32, - protection_broken: u32, - opportunistic_dc: u32, - opportunistic_mua: u32, - unencrypted_dc: u32, - unencrypted_mua: u32, - } - - let num_msgs: u32 = self - .sql - .query_get_value( - "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", - (DC_CHAT_ID_TRASH,), - ) - .await? - .unwrap_or_default(); - - let num_chats: u32 = self - .sql - .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) - .await? - .unwrap_or_default(); - - let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - - let key_created = load_self_secret_key(self) - .await? - .primary_key - .created_at() - .timestamp(); - - // how many of the chats active in the last months are: - // - protected - // - protection-broken - // - opportunistic-encrypted and the contact uses Delta Chat - // - opportunistic-encrypted and the contact uses a classical MUA - // - unencrypted and the contact uses Delta Chat - // - unencrypted and the contact uses a classical MUA - let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chat_numbers = self - .sql - .query_map( - "SELECT c.protected, m.param, m.msgrmsg - FROM chats c - JOIN msgs m - ON c.id=m.chat_id - AND m.id=( - SELECT id - FROM msgs - WHERE chat_id=c.id - AND hidden=0 - AND download_state=? - AND to_id!=? - ORDER BY timestamp DESC, id DESC LIMIT 1) - WHERE c.id>9 - AND (c.blocked=0 OR c.blocked=2) - AND IFNULL(m.timestamp,c.created_timestamp) > ? - GROUP BY c.id", - (DownloadState::Done, ContactId::INFO, three_months_ago), - |row| { - let protected: ProtectionStatus = row.get(0)?; - let message_param: Params = - row.get::<_, String>(1)?.parse().unwrap_or_default(); - let is_dc_message: bool = row.get(2)?; - Ok((protected, message_param, is_dc_message)) - }, - |rows| { - let mut chats = ChatNumbers::default(); - for row in rows { - let (protected, message_param, is_dc_message) = row?; - let encrypted = message_param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or(false); - - if protected == ProtectionStatus::Protected { - chats.protected += 1; - } else if protected == ProtectionStatus::ProtectionBroken { - chats.protection_broken += 1; - } else if encrypted { - if is_dc_message { - chats.opportunistic_dc += 1; - } else { - chats.opportunistic_mua += 1; - } - } else if is_dc_message { - chats.unencrypted_dc += 1; - } else { - chats.unencrypted_mua += 1; - } - } - Ok(chats) - }, - ) - .await?; - - let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { - Some(id) => id, - None => { - let id = create_id(); - self.set_config_internal(Config::SelfReportingId, Some(&id)) - .await?; - id - } - }; - let statistics = Statistics { - core_version: get_version_str().to_string(), - num_msgs, - num_chats, - db_size, - key_created, - chat_numbers, - self_reporting_id, - }; - - Ok(serde_json::to_string_pretty(&statistics)?) - } - - /// Drafts a message with statistics about the usage of Delta Chat. - /// The user can inspect the message if they want, and then hit "Send". - /// - /// On the other end, a bot will receive the message and make it available - /// to Delta Chat's developers. - pub async fn send_self_report(&self) -> Result { - info!(self, "Sending self report."); - // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not - // work out for whatever reason or are interrupted by the OS. - self.set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) - .await - .log_err(self) - .ok(); - - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); - let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(self, contact_id, ContactId::SELF).await?; - - let chat_id = if let Some(res) = ChatId::lookup_by_contact(self, contact_id).await? { - // Already exists, no need to create. - res - } else { - let chat_id = ChatId::get_for_contact(self, contact_id).await?; - chat_id - .set_visibility(self, ChatVisibility::Archived) - .await?; - chat::set_muted(self, chat_id, MuteDuration::Forever).await?; - chat_id - }; - - chat_id - .set_protection(self, ProtectionStatus::Protected, time(), Some(contact_id)) - .await?; - - let mut msg = Message::new(Viewtype::File); - msg.set_text( - "The attachment contains anonymous usage statistics, \ -because you enabled this in the settings. \ -This helps us improve the security of Delta Chat. \ -See TODO[blog post] for more information." - .to_string(), - ); - msg.set_file_from_bytes( - self, - "statistics.txt", - self.get_self_report().await?.as_bytes(), - Some("text/plain"), - )?; - - crate::chat::send_msg(self, chat_id, &mut msg) - .await - .context("Failed to send self_reporting message") - .log_err(self) - .ok(); - - Ok(chat_id) - } - /// Get a list of fresh, unmuted messages in unblocked chats. /// /// The list starts with the most recent message diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index a43cd403d4..254dbcd1f7 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -598,23 +598,6 @@ async fn test_get_next_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = alice.send_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 2).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - - let chat = Chat::load_from_db(&alice, chat_id).await?; - assert!(chat.is_protected()); - - let statistics_msg = get_chat_msg(&alice, chat_id, 1, 2).await; - assert_eq!(statistics_msg.get_filename().unwrap(), "statistics.txt"); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_cache_is_cleared_when_io_is_started() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/lib.rs b/src/lib.rs index 3c6402cbfe..83eeca67b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,6 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; +mod self_reporting; pub mod summary; mod debug_logging; diff --git a/src/scheduler.rs b/src/scheduler.rs index acc041d1c8..21a1a278ab 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -24,6 +24,7 @@ use crate::imap::{FolderMeaning, Imap, session::Session}; use crate::location; use crate::log::{LogExt, error, info, warn}; use crate::message::MsgId; +use crate::self_reporting::maybe_send_self_report; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed}; @@ -507,20 +508,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; - //#[cfg(target_os = "android")] TODO - if ctx.get_config_bool(Config::SelfReporting).await? { - match ctx.get_config_i64(Config::LastSelfReportSent).await { - Ok(last_selfreport_time) => { - let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week - if next_selfreport_time <= time() { - ctx.send_self_report().await?; - } - } - Err(err) => { - warn!(ctx, "Failed to get last self_reporting time: {}", err); - } - } - } + maybe_send_self_report(ctx).await?; match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { diff --git a/src/self_reporting.rs b/src/self_reporting.rs new file mode 100644 index 0000000000..f4b40d9c14 --- /dev/null +++ b/src/self_reporting.rs @@ -0,0 +1,289 @@ +//! TODO doc comment + +use std::collections::{BTreeMap, HashMap}; +use std::ffi::OsString; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::Duration; + +use anyhow::{bail, ensure, Context as _, Result}; +use async_channel::{self as channel, Receiver, Sender}; +use pgp::types::PublicKeyTrait; +use ratelimit::Ratelimit; +use serde::Serialize; +use tokio::sync::{Mutex, Notify, RwLock}; + +use crate::chat::{self, get_chat_cnt, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use crate::chatlist_events; +use crate::config::Config; +use crate::constants::{ + self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, +}; +use crate::contact::{import_vcard, mark_contact_id_as_verified, Contact, ContactId}; +use crate::context::{get_version_str, Context}; +use crate::debug_logging::DebugLogging; +use crate::download::DownloadState; +use crate::events::{Event, EventEmitter, EventType, Events}; +use crate::imap::{FolderMeaning, Imap, ServerMetadata}; +use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _}; +use crate::log::LogExt; +use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; +use crate::message::{self, Message, MessageState, MsgId, Viewtype}; +use crate::param::{Param, Params}; +use crate::peer_channels::Iroh; +use crate::push::PushSubscriber; +use crate::quota::QuotaInfo; +use crate::scheduler::{convert_folder_meaning, SchedulerState}; +use crate::sql::Sql; +use crate::stock_str::StockStrings; +use crate::timesmearing::SmearedTimestamp; +use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; + +#[derive(Serialize)] +struct Statistics { + core_version: String, + num_msgs: u32, + num_chats: u32, + db_size: u64, + key_created: i64, + chat_numbers: ChatNumbers, + self_reporting_id: String, +} +#[derive(Default, Serialize)] +struct ChatNumbers { + protected: u32, + protection_broken: u32, + opportunistic_dc: u32, + opportunistic_mua: u32, + unencrypted_dc: u32, + unencrypted_mua: u32, +} + +pub(crate) async fn maybe_send_self_report(context: &Context) -> Result<()> { + //#[cfg(target_os = "android")] TODO + if context.get_config_bool(Config::SelfReporting).await? { + match context.get_config_i64(Config::LastSelfReportSent).await { + Ok(last_selfreport_time) => { + let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_selfreport_time <= time() { + send_self_report(context).await?; + } + } + Err(err) => { + warn!(context, "Failed to get last self_reporting time: {}", err); + } + } + } + Ok(()) +} + +/// Drafts a message with statistics about the usage of Delta Chat. +/// The user can inspect the message if they want, and then hit "Send". +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. +async fn send_self_report(context: &Context) -> Result { + info!(context, "Sending self report."); + // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not + // work out for whatever reason or are interrupted by the OS. + context + .set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) + .await + .log_err(context) + .ok(); + + const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); + let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) + .await? + .first() + .context("Self reporting bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat_id + .set_visibility(context, ChatVisibility::Archived) + .await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + chat_id + .set_protection( + context, + ProtectionStatus::Protected, + time(), + Some(contact_id), + ) + .await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + msg.set_file_from_bytes( + context, + "statistics.txt", + get_self_report(context).await?.as_bytes(), + Some("text/plain"), + )?; + + crate::chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send self_reporting message") + .log_err(context) + .ok(); + + Ok(chat_id) +} + +async fn get_self_report(context: &Context) -> Result { + let num_msgs: u32 = context + .sql + .query_get_value( + "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", + (DC_CHAT_ID_TRASH,), + ) + .await? + .unwrap_or_default(); + + let num_chats: u32 = context + .sql + .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) + .await? + .unwrap_or_default(); + + let db_size = tokio::fs::metadata(&context.sql.dbfile).await?.len(); + + let key_created = load_self_public_key(context) + .await? + .primary_key + .created_at(); + + // how many of the chats active in the last months are: + // - protected + // - protection-broken + // - opportunistic-encrypted and the contact uses Delta Chat + // - opportunistic-encrypted and the contact uses a classical MUA + // - unencrypted and the contact uses Delta Chat + // - unencrypted and the contact uses a classical MUA + let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); + let chat_numbers = context + .sql + .query_map( + "SELECT c.protected, m.param, m.msgrmsg + FROM chats c + JOIN msgs m + ON c.id=m.chat_id + AND m.id=( + SELECT id + FROM msgs + WHERE chat_id=c.id + AND hidden=0 + AND download_state=? + AND to_id!=? + ORDER BY timestamp DESC, id DESC LIMIT 1) + WHERE c.id>9 + AND (c.blocked=0 OR c.blocked=2) + AND IFNULL(m.timestamp,c.created_timestamp) > ? + GROUP BY c.id", + (DownloadState::Done, ContactId::INFO, three_months_ago), + |row| { + let protected: ProtectionStatus = row.get(0)?; + let message_param: Params = row.get::<_, String>(1)?.parse().unwrap_or_default(); + let is_dc_message: bool = row.get(2)?; + Ok((protected, message_param, is_dc_message)) + }, + |rows| { + let mut chats = ChatNumbers::default(); + for row in rows { + let (protected, message_param, is_dc_message) = row?; + let encrypted = message_param + .get_bool(Param::GuaranteeE2ee) + .unwrap_or(false); + + if protected == ProtectionStatus::Protected { + chats.protected += 1; + } else if protected == ProtectionStatus::ProtectionBroken { + chats.protection_broken += 1; + } else if encrypted { + if is_dc_message { + chats.opportunistic_dc += 1; + } else { + chats.opportunistic_mua += 1; + } + } else if is_dc_message { + chats.unencrypted_dc += 1; + } else { + chats.unencrypted_mua += 1; + } + } + Ok(chats) + }, + ) + .await?; + + let self_reporting_id = match context.get_config(Config::SelfReportingId).await? { + Some(id) => id, + None => { + let id = create_id(); + context + .set_config_internal(Config::SelfReportingId, Some(&id)) + .await?; + id + } + }; + let statistics = Statistics { + core_version: get_version_str().to_string(), + num_msgs, + num_chats, + db_size, + key_created, + chat_numbers, + self_reporting_id, + }; + + Ok(serde_json::to_string_pretty(&statistics)?) +} + +#[cfg(test)] +mod self_reporting_tests { + use anyhow::Context as _; + use strum::IntoEnumIterator; + use tempfile::tempdir; + + use super::*; + use crate::chat::{get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, MuteDuration}; + use crate::chatlist::Chatlist; + use crate::constants::Chattype; + use crate::mimeparser::SystemMessage; + use crate::receive_imf::receive_imf; + use crate::test_utils::{get_chat_msg, TestContext}; + use crate::tools::{create_outgoing_rfc724_mid, SystemTime}; + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_draft_self_report() -> Result<()> { + let alice = TestContext::new_alice().await; + + let chat_id = send_self_report(&alice).await?; + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); + + let chat = Chat::load_from_db(&alice, chat_id).await?; + assert!(chat.is_protected()); + + let statistics_msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(statistics_msg.get_filename().unwrap(), "statistics.txt"); + + Ok(()) + } +} From f0944838acb827eef33c29791eab3b0723ad3090 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 19 Jun 2025 20:10:19 +0200 Subject: [PATCH 05/18] clippy --- src/context.rs | 14 ++++++-------- src/self_reporting.rs | 41 +++++++++-------------------------------- 2 files changed, 15 insertions(+), 40 deletions(-) diff --git a/src/context.rs b/src/context.rs index 69f4db1688..19c42fe17d 100644 --- a/src/context.rs +++ b/src/context.rs @@ -15,23 +15,21 @@ use ratelimit::Ratelimit; use serde::Serialize; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{self, get_chat_cnt, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; +use crate::chat::{get_chat_cnt, ChatId}; use crate::chatlist_events; use crate::config::Config; use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, + self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR, }; -use crate::contact::{import_vcard, mark_contact_id_as_verified, Contact, ContactId}; +use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; -use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_public_key, load_self_secret_key, self_fingerprint, DcKey as _}; +use crate::key::{self_fingerprint, DcKey as _}; use crate::log::LogExt; use crate::log::{info, warn}; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId, Viewtype}; -use crate::param::{Param, Params}; +use crate::message::{self, MessageState, MsgId}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; @@ -39,7 +37,7 @@ use crate::scheduler::{convert_folder_meaning, SchedulerState}; use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{self, duration_to_str, time, time_elapsed}; /// Builder for the [`Context`]. /// diff --git a/src/self_reporting.rs b/src/self_reporting.rs index f4b40d9c14..2e08ead8d0 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -1,45 +1,21 @@ //! TODO doc comment -use std::collections::{BTreeMap, HashMap}; -use std::ffi::OsString; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, OnceLock}; -use std::time::Duration; -use anyhow::{bail, ensure, Context as _, Result}; -use async_channel::{self as channel, Receiver, Sender}; +use anyhow::{Context as _, Result}; use pgp::types::PublicKeyTrait; -use ratelimit::Ratelimit; use serde::Serialize; -use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{self, get_chat_cnt, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; -use crate::chatlist_events; +use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use crate::config::Config; -use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, -}; -use crate::contact::{import_vcard, mark_contact_id_as_verified, Contact, ContactId}; +use crate::constants::DC_CHAT_ID_TRASH; +use crate::contact::{import_vcard, mark_contact_id_as_verified, ContactId}; use crate::context::{get_version_str, Context}; -use crate::debug_logging::DebugLogging; use crate::download::DownloadState; -use crate::events::{Event, EventEmitter, EventType, Events}; -use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_public_key, load_self_secret_key, DcKey as _}; +use crate::key::load_self_public_key; use crate::log::LogExt; -use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId, Viewtype}; +use crate::message::{Message, Viewtype}; use crate::param::{Param, Params}; -use crate::peer_channels::Iroh; -use crate::push::PushSubscriber; -use crate::quota::QuotaInfo; -use crate::scheduler::{convert_folder_meaning, SchedulerState}; -use crate::sql::Sql; -use crate::stock_str::StockStrings; -use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{create_id, time}; #[derive(Serialize)] struct Statistics { @@ -167,7 +143,8 @@ async fn get_self_report(context: &Context) -> Result { let key_created = load_self_public_key(context) .await? .primary_key - .created_at(); + .created_at() + .timestamp(); // how many of the chats active in the last months are: // - protected From 03fc4172c7817e54887862e69be9c5af676f1d06 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 20 Jun 2025 11:17:37 +0200 Subject: [PATCH 06/18] Make everything compile --- deltachat-jsonrpc/src/api.rs | 6 ------ src/context/context_tests.rs | 1 + src/lib.rs | 2 +- src/self_reporting.rs | 3 +-- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index bdf723fd2d..4c5c19ba25 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -375,12 +375,6 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } - /// Deprecated 2025-04. Use the "self_reporting" config instead. - async fn draft_self_report(&self, account_id: u32) -> Result { - let ctx = self.get_context(account_id).await?; - Ok(ctx.send_self_report().await?.to_u32()) - } - /// Sets the given configuration key. async fn set_config(&self, account_id: u32, key: String, value: Option) -> Result<()> { let ctx = self.get_context(account_id).await?; diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 254dbcd1f7..8794afc8fb 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -6,6 +6,7 @@ use super::*; use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted}; use crate::chatlist::Chatlist; use crate::constants::Chattype; +use crate::message::Message; use crate::mimeparser::SystemMessage; use crate::receive_imf::receive_imf; use crate::test_utils::{TestContext, get_chat_msg}; diff --git a/src/lib.rs b/src/lib.rs index 83eeca67b4..64ac262cfd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -98,7 +98,7 @@ pub mod html; pub mod net; pub mod plaintext; mod push; -mod self_reporting; +pub mod self_reporting; pub mod summary; mod debug_logging; diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 2e08ead8d0..be30455380 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -1,6 +1,5 @@ //! TODO doc comment - use anyhow::{Context as _, Result}; use pgp::types::PublicKeyTrait; use serde::Serialize; @@ -37,7 +36,7 @@ struct ChatNumbers { unencrypted_mua: u32, } -pub(crate) async fn maybe_send_self_report(context: &Context) -> Result<()> { +pub async fn maybe_send_self_report(context: &Context) -> Result<()> { //#[cfg(target_os = "android")] TODO if context.get_config_bool(Config::SelfReporting).await? { match context.get_config_i64(Config::LastSelfReportSent).await { From 6e7c2e6e79657b28b5fc2aa000691eb69382e8f8 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 20 Jun 2025 11:22:55 +0200 Subject: [PATCH 07/18] move comment --- src/self_reporting.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index be30455380..c89bde6eee 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -36,6 +36,12 @@ struct ChatNumbers { unencrypted_mua: u32, } +/// Sends a message with statistics about the usage of Delta Chat, +/// if the last time such a message was sent +/// was more than a week ago. +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. pub async fn maybe_send_self_report(context: &Context) -> Result<()> { //#[cfg(target_os = "android")] TODO if context.get_config_bool(Config::SelfReporting).await? { @@ -54,11 +60,6 @@ pub async fn maybe_send_self_report(context: &Context) -> Result<()> { Ok(()) } -/// Drafts a message with statistics about the usage of Delta Chat. -/// The user can inspect the message if they want, and then hit "Send". -/// -/// On the other end, a bot will receive the message and make it available -/// to Delta Chat's developers. async fn send_self_report(context: &Context) -> Result { info!(context, "Sending self report."); // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not From 7123738faae9eff7304840c1725e76d30f41a80e Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 24 Jun 2025 11:34:32 +0200 Subject: [PATCH 08/18] feat: Add contact_infos, add test --- src/context.rs | 15 +- src/context/context_tests.rs | 3 +- src/receive_imf.rs | 3 +- src/self_reporting.rs | 170 ++++++++++++++++----- src/self_reporting/self_reporting_tests.rs | 66 ++++++++ 5 files changed, 210 insertions(+), 47 deletions(-) create mode 100644 src/self_reporting/self_reporting_tests.rs diff --git a/src/context.rs b/src/context.rs index 19c42fe17d..14a0b65585 100644 --- a/src/context.rs +++ b/src/context.rs @@ -8,32 +8,27 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::Duration; -use anyhow::{bail, ensure, Context as _, Result}; +use anyhow::{Context as _, Result, bail, ensure}; use async_channel::{self as channel, Receiver, Sender}; -use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; -use serde::Serialize; use tokio::sync::{Mutex, Notify, RwLock}; -use crate::chat::{get_chat_cnt, ChatId}; +use crate::chat::{ChatId, get_chat_cnt}; use crate::chatlist_events; use crate::config::Config; -use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR, -}; +use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR}; use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{self_fingerprint, DcKey as _}; -use crate::log::LogExt; +use crate::key::self_fingerprint; use crate::log::{info, warn}; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; use crate::message::{self, MessageState, MsgId}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; -use crate::scheduler::{convert_folder_meaning, SchedulerState}; +use crate::scheduler::{SchedulerState, convert_folder_meaning}; use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 8794afc8fb..5d6bad6069 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -7,9 +7,8 @@ use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg use crate::chatlist::Chatlist; use crate::constants::Chattype; use crate::message::Message; -use crate::mimeparser::SystemMessage; use crate::receive_imf::receive_imf; -use crate::test_utils::{TestContext, get_chat_msg}; +use crate::test_utils::TestContext; use crate::tools::{SystemTime, create_outgoing_rfc724_mid}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 59d48a6b80..8443a0a2dd 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -38,6 +38,7 @@ use crate::peer_channels::{add_gossip_peer_from_header, insert_topic_stub}; use crate::reaction::{Reaction, set_msg_reaction}; use crate::rusqlite::OptionalExtension; use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device}; +use crate::self_reporting::SELF_REPORTING_BOT_EMAIL; use crate::simplify; use crate::stock_str; use crate::sync::Sync::*; @@ -1713,7 +1714,7 @@ async fn add_parts( || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent - || mime_parser.from.addr == "self_reporting@testrun.org" + || mime_parser.from.addr == SELF_REPORTING_BOT_EMAIL // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen diff --git a/src/self_reporting.rs b/src/self_reporting.rs index c89bde6eee..a4096d21b0 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -1,13 +1,15 @@ //! TODO doc comment +use std::collections::{BTreeMap, BTreeSet}; + use anyhow::{Context as _, Result}; use pgp::types::PublicKeyTrait; use serde::Serialize; use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use crate::config::Config; -use crate::constants::DC_CHAT_ID_TRASH; -use crate::contact::{import_vcard, mark_contact_id_as_verified, ContactId}; +use crate::constants::{Chattype, DC_CHAT_ID_TRASH}; +use crate::contact::{import_vcard, mark_contact_id_as_verified, ContactId, Origin}; use crate::context::{get_version_str, Context}; use crate::download::DownloadState; use crate::key::load_self_public_key; @@ -16,6 +18,9 @@ use crate::message::{Message, Viewtype}; use crate::param::{Param, Params}; use crate::tools::{create_id, time}; +pub(crate) const SELF_REPORTING_BOT_EMAIL: &str = "self_reporting@testrun.org"; +const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); + #[derive(Serialize)] struct Statistics { core_version: String, @@ -25,6 +30,7 @@ struct Statistics { key_created: i64, chat_numbers: ChatNumbers, self_reporting_id: String, + contact_infos: Vec, } #[derive(Default, Serialize)] struct ChatNumbers { @@ -36,6 +42,132 @@ struct ChatNumbers { unencrypted_mua: u32, } +#[derive(Serialize, PartialEq)] +enum VerifiedStatus { + Direct, + Transitive, + TransitiveViaBot, + Opportunistic, + Unencrypted, +} + +#[derive(Serialize)] +struct ContactInfo { + #[serde(skip_serializing)] + id: ContactId, + + verified: VerifiedStatus, + + #[serde(skip_serializing)] + verifier: ContactId, // TODO unused, could be removed + bot: bool, + direct_chat: bool, + last_seen: u64, + //new: bool, // TODO + #[serde(skip_serializing_if = "Option::is_none")] + transitive_chain: Option, +} + +async fn get_contact_infos(context: &Context) -> Result> { + let mut verified_by_map: BTreeMap = BTreeMap::new(); + let mut bot_ids: BTreeSet = BTreeSet::new(); + + let mut contacts: Vec = context + .sql + .query_map( + "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c + WHERE id>9 AND origin>? AND addr<>?", + (Origin::Hidden, SELF_REPORTING_BOT_EMAIL), + |row| { + let id = row.get(0)?; + let is_encrypted: bool = row.get(1)?; + let verifier: ContactId = row.get(2)?; + let last_seen: u64 = row.get(3)?; + let bot: bool = row.get(4)?; + + let verified = match (is_encrypted, verifier) { + (true, ContactId::SELF) => VerifiedStatus::Direct, + (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, + (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later + (false, _) => VerifiedStatus::Unencrypted, + }; + + if verifier != ContactId::UNDEFINED { + verified_by_map.insert(id, verifier); + } + + if bot { + bot_ids.insert(id); + } + + Ok(ContactInfo { + id, + verified, + verifier, + bot, + direct_chat: false, // will be filled later + last_seen, + transitive_chain: None, // will be filled later + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Fill TransitiveViaBot and transitive_chain + for contact in contacts.iter_mut() { + if contact.verified == VerifiedStatus::Transitive { + let mut transitive_chain: u32 = 0; + let mut has_bot = false; + let mut current_verifier_id = contact.id; + + while current_verifier_id != ContactId::SELF { + current_verifier_id = match verified_by_map.get(¤t_verifier_id) { + Some(id) => *id, + None => { + // The chain ends here, probably because some verification was done + // before we started recording verifiers. + // It's unclear how long the chain really is. + transitive_chain = 0; + break; + } + }; + if bot_ids.contains(¤t_verifier_id) { + has_bot = true; + } + transitive_chain = transitive_chain.saturating_add(1); + } + + if transitive_chain > 0 { + contact.transitive_chain = Some(transitive_chain); + } + + if has_bot { + contact.verified = VerifiedStatus::TransitiveViaBot; + } + } + } + + // Fill direct_chat + for contact in contacts.iter_mut() { + let direct_chat = context + .sql + .exists( + "SELECT COUNT(*) + FROM chats_contacts cc INNER JOIN chats + WHERE cc.contact_id=? AND chats.type=?", + (contact.id, Chattype::Single), + ) + .await?; + contact.direct_chat = direct_chat; + } + + Ok(contacts) +} + /// Sends a message with statistics about the usage of Delta Chat, /// if the last time such a message was sent /// was more than a week ago. @@ -70,7 +202,6 @@ async fn send_self_report(context: &Context) -> Result { .log_err(context) .ok(); - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) .await? .first() @@ -227,40 +358,11 @@ async fn get_self_report(context: &Context) -> Result { key_created, chat_numbers, self_reporting_id, + contact_infos: get_contact_infos(context).await?, }; Ok(serde_json::to_string_pretty(&statistics)?) } #[cfg(test)] -mod self_reporting_tests { - use anyhow::Context as _; - use strum::IntoEnumIterator; - use tempfile::tempdir; - - use super::*; - use crate::chat::{get_chat_contacts, get_chat_msgs, send_msg, set_muted, Chat, MuteDuration}; - use crate::chatlist::Chatlist; - use crate::constants::Chattype; - use crate::mimeparser::SystemMessage; - use crate::receive_imf::receive_imf; - use crate::test_utils::{get_chat_msg, TestContext}; - use crate::tools::{create_outgoing_rfc724_mid, SystemTime}; - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = send_self_report(&alice).await?; - let msg = get_chat_msg(&alice, chat_id, 0, 2).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); - - let chat = Chat::load_from_db(&alice, chat_id).await?; - assert!(chat.is_protected()); - - let statistics_msg = get_chat_msg(&alice, chat_id, 1, 2).await; - assert_eq!(statistics_msg.get_filename().unwrap(), "statistics.txt"); - - Ok(()) - } -} +mod self_reporting_tests; diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs new file mode 100644 index 0000000000..42671aabb4 --- /dev/null +++ b/src/self_reporting/self_reporting_tests.rs @@ -0,0 +1,66 @@ +use super::*; +use crate::chat::Chat; +use crate::mimeparser::SystemMessage; +use crate::test_utils::{get_chat_msg, TestContextManager}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_send_self_report() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + + let chat_id = send_self_report(&alice).await?; + let msg = get_chat_msg(&alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); + + let chat = Chat::load_from_db(&alice, chat_id).await?; + assert!(chat.is_protected()); + + let msg = get_chat_msg(&alice, chat_id, 1, 2).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let report = tokio::fs::read(msg.get_file(&alice).unwrap()).await?; + let report = std::str::from_utf8(&report)?; + println!("\nEmpty account:\n{}\n", report); + assert!(report.contains(r#""contact_infos": []"#)); + + let r: serde_json::Value = serde_json::from_str(&report)?; + assert_eq!( + r.get("contact_infos").unwrap(), + &serde_json::Value::Array(vec![]) + ); + assert_eq!(r.get("core_version").unwrap(), get_version_str()); + + tcm.send_recv_accept(bob, alice, "Hi!").await; + + let report = get_self_report(alice).await?; + + println!("\nWith Bob:\n{report}\n"); + let r2: serde_json::Value = serde_json::from_str(&report)?; + assert_eq!( + r.get("key_created").unwrap(), + r2.get("key_created").unwrap() + ); + assert_eq!( + r.get("self_reporting_id").unwrap(), + r2.get("self_reporting_id").unwrap() + ); + let contact_infos = r2.get("contact_infos").unwrap().as_array().unwrap(); + assert_eq!(contact_infos.len(), 1); + let contact_info = &contact_infos[0]; + assert_eq!( + contact_info.get("bot").unwrap(), + &serde_json::Value::Bool(false) + ); + assert_eq!( + contact_info.get("direct_chat").unwrap(), + &serde_json::Value::Bool(true) + ); + assert!(contact_info.get("transitive_chain").is_none(),); + assert_eq!( + contact_info.get("verified").unwrap(), + &serde_json::Value::String("Opportunistic".to_string()) + ); + + Ok(()) +} From 39afd277480ad9cf4760bc0202aaef5f8816165d Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 24 Jun 2025 16:45:09 +0200 Subject: [PATCH 09/18] clippy --- src/self_reporting.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index a4096d21b0..b67a8776c9 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -57,12 +57,10 @@ struct ContactInfo { id: ContactId, verified: VerifiedStatus, - - #[serde(skip_serializing)] - verifier: ContactId, // TODO unused, could be removed bot: bool, direct_chat: bool, last_seen: u64, + //new: bool, // TODO #[serde(skip_serializing_if = "Option::is_none")] transitive_chain: Option, @@ -103,7 +101,6 @@ async fn get_contact_infos(context: &Context) -> Result> { Ok(ContactInfo { id, verified, - verifier, bot, direct_chat: false, // will be filled later last_seen, From 5ad71e03a024237d66dec2112d949b1a5d2cd613 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 25 Jun 2025 15:52:07 +0200 Subject: [PATCH 10/18] Add message stats. Tests and splitting by bot are missing. --- src/config.rs | 9 ++ src/context.rs | 6 + src/scheduler.rs | 2 +- src/self_reporting.rs | 160 +++++++++++++++------ src/self_reporting/self_reporting_tests.rs | 16 ++- 5 files changed, 143 insertions(+), 50 deletions(-) diff --git a/src/config.rs b/src/config.rs index 8078338394..22577315ec 100644 --- a/src/config.rs +++ b/src/config.rs @@ -439,6 +439,9 @@ pub enum Config { /// without storing the email address SelfReportingId, + /// Timestamp of enabling SelfReporting. + SelfReportingEnabledTimestamp, + /// MsgId of webxdc map integration. WebxdcIntegration, @@ -831,6 +834,12 @@ impl Context { .await?; } } + Config::SelfReporting => { + self.sql.set_raw_config(key.as_ref(), value).await?; + self.sql + .set_raw_config(Config::SelfReportingEnabledTimestamp.as_ref(), value) + .await?; + } _ => { self.sql.set_raw_config(key.as_ref(), value).await?; } diff --git a/src/context.rs b/src/context.rs index 14a0b65585..4a81d591fc 100644 --- a/src/context.rs +++ b/src/context.rs @@ -1048,6 +1048,12 @@ impl Context { .await? .to_string(), ); + res.insert( + "self_reporting_enabled_timestamp", + self.get_config_i64(Config::SelfReportingEnabledTimestamp) + .await? + .to_string(), + ); let elapsed = time_elapsed(&self.creation_time); res.insert("uptime", duration_to_str(elapsed)); diff --git a/src/scheduler.rs b/src/scheduler.rs index 21a1a278ab..ca48e9e6f2 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -508,7 +508,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; - maybe_send_self_report(ctx).await?; + maybe_send_self_report(ctx).await.log_err(ctx).ok(); match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { diff --git a/src/self_reporting.rs b/src/self_reporting.rs index b67a8776c9..8ea155d680 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use anyhow::{Context as _, Result}; +use anyhow::{ensure, Context as _, Result}; use pgp::types::PublicKeyTrait; use serde::Serialize; @@ -30,7 +30,8 @@ struct Statistics { key_created: i64, chat_numbers: ChatNumbers, self_reporting_id: String, - contact_infos: Vec, + contact_stats: Vec, + message_stats: MessageStats, } #[derive(Default, Serialize)] struct ChatNumbers { @@ -52,7 +53,7 @@ enum VerifiedStatus { } #[derive(Serialize)] -struct ContactInfo { +struct ContactStat { #[serde(skip_serializing)] id: ContactId, @@ -61,16 +62,16 @@ struct ContactInfo { direct_chat: bool, last_seen: u64, - //new: bool, // TODO #[serde(skip_serializing_if = "Option::is_none")] transitive_chain: Option, + //new: bool, // TODO } -async fn get_contact_infos(context: &Context) -> Result> { +async fn get_contact_stats(context: &Context) -> Result> { let mut verified_by_map: BTreeMap = BTreeMap::new(); let mut bot_ids: BTreeSet = BTreeSet::new(); - let mut contacts: Vec = context + let mut contacts: Vec = context .sql .query_map( "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c @@ -98,7 +99,7 @@ async fn get_contact_infos(context: &Context) -> Result> { bot_ids.insert(id); } - Ok(ContactInfo { + Ok(ContactStat { id, verified, bot, @@ -165,40 +166,133 @@ async fn get_contact_infos(context: &Context) -> Result> { Ok(contacts) } +#[derive(Serialize)] +struct MessageStats { + to_verified: u32, + unverified_encrypted: u32, + unencrypted: u32, +} + +async fn get_message_stats(context: &Context) -> Result { + let enabled_ts: i64 = context + .get_config_i64(Config::SelfReportingEnabledTimestamp) + .await?; + ensure!(enabled_ts > 0, "Enabled Timestamp missing"); + + let selfreporting_bot_chat_id = get_selfreporting_bot(context).await?; + + let trans_fn = |t: &mut rusqlite::Transaction| { + t.pragma_update(None, "query_only", "0")?; + t.execute( + "CREATE TEMP TABLE temp.verified_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + t.execute( + "INSERT INTO temp.verified_chats + SELECT id FROM chats + WHERE protected=1 AND id>9", + (), + )?; + + let to_verified = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.verified_chats + AND chat_id<>? AND id>9 AND timestamp_sent>?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + let unverified_encrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.verified_chats + AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND chat_id<>? AND id>9 AND timestamp_sent>?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + let unencrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.verified_chats + AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND chat_id<>? AND id>9 AND timestamp_sent>=?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + t.execute("DROP TABLE temp.verified_chats", ())?; + + Ok(MessageStats { + to_verified, + unverified_encrypted, + unencrypted, + }) + }; + + let query_only = true; + let message_stats: MessageStats = context.sql.transaction_ex(query_only, trans_fn).await?; + + Ok(message_stats) +} + /// Sends a message with statistics about the usage of Delta Chat, /// if the last time such a message was sent /// was more than a week ago. /// /// On the other end, a bot will receive the message and make it available /// to Delta Chat's developers. -pub async fn maybe_send_self_report(context: &Context) -> Result<()> { +pub async fn maybe_send_self_report(context: &Context) -> Result> { //#[cfg(target_os = "android")] TODO if context.get_config_bool(Config::SelfReporting).await? { - match context.get_config_i64(Config::LastSelfReportSent).await { - Ok(last_selfreport_time) => { - let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week - if next_selfreport_time <= time() { - send_self_report(context).await?; - } - } - Err(err) => { - warn!(context, "Failed to get last self_reporting time: {}", err); - } + let last_selfreport_time = context.get_config_i64(Config::LastSelfReportSent).await?; + let next_selfreport_time = last_selfreport_time.saturating_add(30); // TODO increase to 1 day or 1 week + if next_selfreport_time <= time() { + return Ok(Some(send_self_report(context).await?)); } } - Ok(()) + Ok(None) } async fn send_self_report(context: &Context) -> Result { info!(context, "Sending self report."); - // Setting `Config::LastHousekeeping` at the beginning avoids endless loops when things do not - // work out for whatever reason or are interrupted by the OS. + // Setting this config at the beginning avoids endless loops when things do not + // work out for whatever reason. context .set_config_internal(Config::LastSelfReportSent, Some(&time().to_string())) .await .log_err(context) .ok(); + let chat_id = get_selfreporting_bot(context).await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text( + "The attachment contains anonymous usage statistics, \ +because you enabled this in the settings. \ +This helps us improve the security of Delta Chat. \ +See TODO[blog post] for more information." + .to_string(), + ); + msg.set_file_from_bytes( + context, + "statistics.txt", + get_self_report(context).await?.as_bytes(), + Some("text/plain"), + )?; + + crate::chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send self_reporting message") + .log_err(context) + .ok(); + + Ok(chat_id) +} + +async fn get_selfreporting_bot(context: &Context) -> Result { let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) .await? .first() @@ -226,27 +320,6 @@ async fn send_self_report(context: &Context) -> Result { ) .await?; - let mut msg = Message::new(Viewtype::File); - msg.set_text( - "The attachment contains anonymous usage statistics, \ -because you enabled this in the settings. \ -This helps us improve the security of Delta Chat. \ -See TODO[blog post] for more information." - .to_string(), - ); - msg.set_file_from_bytes( - context, - "statistics.txt", - get_self_report(context).await?.as_bytes(), - Some("text/plain"), - )?; - - crate::chat::send_msg(context, chat_id, &mut msg) - .await - .context("Failed to send self_reporting message") - .log_err(context) - .ok(); - Ok(chat_id) } @@ -355,7 +428,8 @@ async fn get_self_report(context: &Context) -> Result { key_created, chat_numbers, self_reporting_id, - contact_infos: get_contact_infos(context).await?, + contact_stats: get_contact_stats(context).await?, + message_stats: get_message_stats(context).await?, }; Ok(serde_json::to_string_pretty(&statistics)?) diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs index 42671aabb4..be0b147a4e 100644 --- a/src/self_reporting/self_reporting_tests.rs +++ b/src/self_reporting/self_reporting_tests.rs @@ -9,7 +9,9 @@ async fn test_send_self_report() -> Result<()> { let alice = &tcm.alice().await; let bob = &tcm.bob().await; - let chat_id = send_self_report(&alice).await?; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let chat_id = maybe_send_self_report(&alice).await?.unwrap(); let msg = get_chat_msg(&alice, chat_id, 0, 2).await; assert_eq!(msg.get_info_type(), SystemMessage::ChatProtectionEnabled); @@ -22,15 +24,17 @@ async fn test_send_self_report() -> Result<()> { let report = tokio::fs::read(msg.get_file(&alice).unwrap()).await?; let report = std::str::from_utf8(&report)?; println!("\nEmpty account:\n{}\n", report); - assert!(report.contains(r#""contact_infos": []"#)); + assert!(report.contains(r#""contact_stats": []"#)); let r: serde_json::Value = serde_json::from_str(&report)?; assert_eq!( - r.get("contact_infos").unwrap(), + r.get("contact_stats").unwrap(), &serde_json::Value::Array(vec![]) ); assert_eq!(r.get("core_version").unwrap(), get_version_str()); + assert_eq!(maybe_send_self_report(alice).await?, None); + tcm.send_recv_accept(bob, alice, "Hi!").await; let report = get_self_report(alice).await?; @@ -45,9 +49,9 @@ async fn test_send_self_report() -> Result<()> { r.get("self_reporting_id").unwrap(), r2.get("self_reporting_id").unwrap() ); - let contact_infos = r2.get("contact_infos").unwrap().as_array().unwrap(); - assert_eq!(contact_infos.len(), 1); - let contact_info = &contact_infos[0]; + let contact_stats = r2.get("contact_stats").unwrap().as_array().unwrap(); + assert_eq!(contact_stats.len(), 1); + let contact_info = &contact_stats[0]; assert_eq!( contact_info.get("bot").unwrap(), &serde_json::Value::Bool(false) From 03eff8fcc481079b5909a481c9d5c9ff403c13c0 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 25 Jun 2025 19:16:11 +0200 Subject: [PATCH 11/18] split test into two --- src/self_reporting/self_reporting_tests.rs | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs index be0b147a4e..6b094aaa7b 100644 --- a/src/self_reporting/self_reporting_tests.rs +++ b/src/self_reporting/self_reporting_tests.rs @@ -1,13 +1,11 @@ use super::*; use crate::chat::Chat; use crate::mimeparser::SystemMessage; -use crate::test_utils::{get_chat_msg, TestContextManager}; +use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_self_report() -> Result<()> { - let mut tcm = TestContextManager::new(); - let alice = &tcm.alice().await; - let bob = &tcm.bob().await; + let alice = &TestContext::new_alice().await; alice.set_config_bool(Config::SelfReporting, true).await?; @@ -35,12 +33,25 @@ async fn test_send_self_report() -> Result<()> { assert_eq!(maybe_send_self_report(alice).await?, None); - tcm.send_recv_accept(bob, alice, "Hi!").await; + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_one_contact() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; let report = get_self_report(alice).await?; + let r: serde_json::Value = serde_json::from_str(&report)?; + tcm.send_recv_accept(bob, alice, "Hi!").await; + + let report = get_self_report(alice).await?; println!("\nWith Bob:\n{report}\n"); let r2: serde_json::Value = serde_json::from_str(&report)?; + assert_eq!( r.get("key_created").unwrap(), r2.get("key_created").unwrap() From 67051f4dc0dd3f5d0076640ae9d6b0ecaa8231a7 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 4 Jul 2025 14:47:08 +0200 Subject: [PATCH 12/18] Add SecurejoinSourceStats, not tested yet --- deltachat-jsonrpc/src/api.rs | 4 +-- src/securejoin.rs | 24 ++++++++++++- src/self_reporting.rs | 70 ++++++++++++++++++++++++++++++++++++ src/sql/migrations.rs | 12 +++++++ 4 files changed, 107 insertions(+), 3 deletions(-) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 4c5c19ba25..d3610ae376 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -867,9 +867,9 @@ impl CommandApi { /// **returns**: The chat ID of the joined chat, the UI may redirect to the this chat. /// A returned chat ID does not guarantee that the chat is protected or the belonging contact is verified. /// - async fn secure_join(&self, account_id: u32, qr: String) -> Result { + async fn secure_join(&self, account_id: u32, qr: String, source: Option) -> Result { let ctx = self.get_context(account_id).await?; - let chat_id = securejoin::join_securejoin(&ctx, &qr).await?; + let chat_id = securejoin::join_securejoin_with_source(&ctx, &qr, source).await?; Ok(chat_id.to_u32()) } diff --git a/src/securejoin.rs b/src/securejoin.rs index c779685441..f8e3bea282 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -5,7 +5,6 @@ use deltachat_contact_tools::ContactAddress; use percent_encoding::{NON_ALPHANUMERIC, utf8_percent_encode}; use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ProtectionStatus, get_chat_id_by_grpid}; -use crate::chatlist_events; use crate::config::Config; use crate::constants::{Blocked, Chattype, NON_ALPHANUMERIC_WITHOUT_DOT}; use crate::contact::mark_contact_id_as_verified; @@ -15,6 +14,7 @@ use crate::e2ee::ensure_secret_key_exists; use crate::events::EventType; use crate::headerdef::HeaderDef; use crate::key::{DcKey, Fingerprint, load_self_public_key}; +use crate::log::LogExt as _; use crate::log::{error, info, warn}; use crate::message::{Message, Viewtype}; use crate::mimeparser::{MimeMessage, SystemMessage}; @@ -23,6 +23,7 @@ use crate::qr::check_qr; use crate::securejoin::bob::JoinerProgress; use crate::sync::Sync::*; use crate::token; +use crate::{chatlist_events, self_reporting}; mod bob; mod qrinvite; @@ -151,6 +152,27 @@ pub async fn join_securejoin(context: &Context, qr: &str) -> Result { }) } +/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. +/// +/// This is the start of the process for the joiner. See the module and ffi documentation +/// for more details. +/// +/// The function returns immediately and the handshake will run in background. +pub async fn join_securejoin_with_source( + context: &Context, + qr: &str, + source: Option, +) -> Result { + let res = join_securejoin(context, qr).await?; + + self_reporting::count_securejoin_source(context, source) + .await + .log_err(context) + .ok(); + + Ok(res) +} + async fn securejoin(context: &Context, qr: &str) -> Result { /*======================================================== ==== Bob - the joiner's side ===== diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 8ea155d680..0a4b15dd5f 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use anyhow::{ensure, Context as _, Result}; +use deltachat_derive::FromSql; use pgp::types::PublicKeyTrait; use serde::Serialize; @@ -32,6 +33,7 @@ struct Statistics { self_reporting_id: String, contact_stats: Vec, message_stats: MessageStats, + securejoin_source_stats: SecurejoinSourceStats, } #[derive(Default, Serialize)] struct ChatNumbers { @@ -430,10 +432,78 @@ async fn get_self_report(context: &Context) -> Result { self_reporting_id, contact_stats: get_contact_stats(context).await?, message_stats: get_message_stats(context).await?, + securejoin_source_stats: get_securejoin_source_stats(context).await?, }; Ok(serde_json::to_string_pretty(&statistics)?) } +#[repr(u32)] +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinSource { + Unknown = 0, + ExternalLink = 1, + InternalLink = 2, + Clipboard = 3, + ImageLoaded = 4, + Scan = 5, +} + +#[derive(Serialize)] +struct SecurejoinSourceStats { + unknown: u32, + external_link: u32, + internal_link: u32, + clipboard: u32, + image_loaded: u32, + scan: u32, +} + +pub(crate) async fn count_securejoin_source(context: &Context, source: Option) -> Result<()> { + if context.get_config_bool(Config::SelfReporting).await? { + let source = source + .context("Missing securejoin source") + .log_err(context) + .unwrap_or(0); + + context + .sql + .execute( + "INSERT INTO stats_securejoin_sources VALUES (?, 1) + ON CONFLICT (source) DO UPDATE SET count=count+1;", + (source,), + ) + .await?; + } + Ok(()) +} + +async fn get_securejoin_source_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT source, count FROM stats_securejoin_sources", + (), + |row| { + let source: SecurejoinSource = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((source, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinSourceStats { + unknown: *map.get(&SecurejoinSource::Unknown).unwrap_or(&0), + external_link: *map.get(&SecurejoinSource::ExternalLink).unwrap_or(&0), + internal_link: *map.get(&SecurejoinSource::InternalLink).unwrap_or(&0), + clipboard: *map.get(&SecurejoinSource::Clipboard).unwrap_or(&0), + image_loaded: *map.get(&SecurejoinSource::ImageLoaded).unwrap_or(&0), + scan: *map.get(&SecurejoinSource::Scan).unwrap_or(&0), + }; + + Ok(stats) +} + #[cfg(test)] mod self_reporting_tests; diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 76ea8ee692..507c49a891 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1247,6 +1247,18 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); ); } + inc_and_check(&mut migration_version, 133)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_securejoin_sources( + source INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT", + 133, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? From 8895fd8df7df9273c8fdb993ef5d49e54e0b7635 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Fri, 4 Jul 2025 20:38:54 +0200 Subject: [PATCH 13/18] clippy --- src/self_reporting.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 0a4b15dd5f..553d3159a7 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use anyhow::{ensure, Context as _, Result}; +use anyhow::{Context as _, Result, ensure}; use deltachat_derive::FromSql; use pgp::types::PublicKeyTrait; use serde::Serialize; @@ -10,8 +10,8 @@ use serde::Serialize; use crate::chat::{self, ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use crate::config::Config; use crate::constants::{Chattype, DC_CHAT_ID_TRASH}; -use crate::contact::{import_vcard, mark_contact_id_as_verified, ContactId, Origin}; -use crate::context::{get_version_str, Context}; +use crate::contact::{ContactId, Origin, import_vcard, mark_contact_id_as_verified}; +use crate::context::{Context, get_version_str}; use crate::download::DownloadState; use crate::key::load_self_public_key; use crate::log::LogExt; @@ -118,7 +118,7 @@ async fn get_contact_stats(context: &Context) -> Result> { .await?; // Fill TransitiveViaBot and transitive_chain - for contact in contacts.iter_mut() { + for contact in &mut contacts { if contact.verified == VerifiedStatus::Transitive { let mut transitive_chain: u32 = 0; let mut has_bot = false; @@ -152,7 +152,7 @@ async fn get_contact_stats(context: &Context) -> Result> { } // Fill direct_chat - for contact in contacts.iter_mut() { + for contact in &mut contacts { let direct_chat = context .sql .exists( From e4ce26c53c1a6ac8346e06e73f577b1c915ba40f Mon Sep 17 00:00:00 2001 From: Hocuri Date: Wed, 9 Jul 2025 21:59:38 +0200 Subject: [PATCH 14/18] fix: Don't loop infinitely if there is a loop in the verification chain --- src/self_reporting.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 553d3159a7..754da205ce 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -124,7 +124,7 @@ async fn get_contact_stats(context: &Context) -> Result> { let mut has_bot = false; let mut current_verifier_id = contact.id; - while current_verifier_id != ContactId::SELF { + while current_verifier_id != ContactId::SELF && transitive_chain < 100 { current_verifier_id = match verified_by_map.get(¤t_verifier_id) { Some(id) => *id, None => { From 6ac4a6b724eb5a8d6144b5b5bf3afaa0b35558de Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 10 Jul 2025 15:45:56 +0200 Subject: [PATCH 15/18] Add test for securejoin source stats --- src/securejoin.rs | 14 ++-- src/self_reporting/self_reporting_tests.rs | 74 +++++++++++++++++++++- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/securejoin.rs b/src/securejoin.rs index f8e3bea282..fa8bfc9b9a 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -144,12 +144,7 @@ async fn get_self_fingerprint(context: &Context) -> Result { /// /// The function returns immediately and the handshake will run in background. pub async fn join_securejoin(context: &Context, qr: &str) -> Result { - securejoin(context, qr).await.map_err(|err| { - warn!(context, "Fatal joiner error: {:#}", err); - // The user just scanned this QR code so has context on what failed. - error!(context, "QR process failed"); - err - }) + join_securejoin_with_source(context, qr, None).await } /// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. @@ -163,7 +158,12 @@ pub async fn join_securejoin_with_source( qr: &str, source: Option, ) -> Result { - let res = join_securejoin(context, qr).await?; + let res = securejoin(context, qr).await.map_err(|err| { + warn!(context, "Fatal joiner error: {:#}", err); + // The user just scanned this QR code so has context on what failed. + error!(context, "QR process failed"); + err + })?; self_reporting::count_securejoin_source(context, source) .await diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs index 6b094aaa7b..fbe3a9c802 100644 --- a/src/self_reporting/self_reporting_tests.rs +++ b/src/self_reporting/self_reporting_tests.rs @@ -1,7 +1,9 @@ use super::*; use crate::chat::Chat; use crate::mimeparser::SystemMessage; -use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; +use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_source}; +use crate::test_utils::{TestContext, TestContextManager, get_chat_msg}; +use pretty_assertions::assert_eq; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_send_self_report() -> Result<()> { @@ -79,3 +81,73 @@ async fn test_self_report_one_contact() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_self_report_securejoin_source_stats() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + + let mut expected = SecurejoinSourceStats { + unknown: 0, + external_link: 0, + internal_link: 0, + clipboard: 0, + image_loaded: 0, + scan: 0, + }; + + check_securejoin_report(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32)).await?; + expected.clipboard += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::ExternalLink as u32)).await?; + expected.external_link += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::InternalLink as u32)).await?; + expected.internal_link += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::ImageLoaded as u32)).await?; + expected.image_loaded += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Scan as u32)).await?; + expected.scan += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32)).await?; + expected.clipboard += 1; + check_securejoin_report(alice, &expected).await; + + join_securejoin_with_source(alice, &qr, Some(SecurejoinSource::Clipboard as u32)).await?; + expected.clipboard += 1; + check_securejoin_report(alice, &expected).await; + + Ok(()) +} + +async fn check_securejoin_report(context: &TestContext, expected: &SecurejoinSourceStats) { + let report = get_self_report(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); + let actual = &actual["securejoin_source_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(&expected, actual); +} From 6615c55584a146ee8c57309f51e5ff522bb0a9d0 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 10 Jul 2025 15:50:52 +0200 Subject: [PATCH 16/18] Change sort order of functions in self_reporting.rs --- src/self_reporting.rs | 418 +++++++++++++++++++++--------------------- 1 file changed, 209 insertions(+), 209 deletions(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 754da205ce..d9bf4f6e19 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -69,105 +69,6 @@ struct ContactStat { //new: bool, // TODO } -async fn get_contact_stats(context: &Context) -> Result> { - let mut verified_by_map: BTreeMap = BTreeMap::new(); - let mut bot_ids: BTreeSet = BTreeSet::new(); - - let mut contacts: Vec = context - .sql - .query_map( - "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c - WHERE id>9 AND origin>? AND addr<>?", - (Origin::Hidden, SELF_REPORTING_BOT_EMAIL), - |row| { - let id = row.get(0)?; - let is_encrypted: bool = row.get(1)?; - let verifier: ContactId = row.get(2)?; - let last_seen: u64 = row.get(3)?; - let bot: bool = row.get(4)?; - - let verified = match (is_encrypted, verifier) { - (true, ContactId::SELF) => VerifiedStatus::Direct, - (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, - (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later - (false, _) => VerifiedStatus::Unencrypted, - }; - - if verifier != ContactId::UNDEFINED { - verified_by_map.insert(id, verifier); - } - - if bot { - bot_ids.insert(id); - } - - Ok(ContactStat { - id, - verified, - bot, - direct_chat: false, // will be filled later - last_seen, - transitive_chain: None, // will be filled later - }) - }, - |rows| { - rows.collect::, _>>() - .map_err(Into::into) - }, - ) - .await?; - - // Fill TransitiveViaBot and transitive_chain - for contact in &mut contacts { - if contact.verified == VerifiedStatus::Transitive { - let mut transitive_chain: u32 = 0; - let mut has_bot = false; - let mut current_verifier_id = contact.id; - - while current_verifier_id != ContactId::SELF && transitive_chain < 100 { - current_verifier_id = match verified_by_map.get(¤t_verifier_id) { - Some(id) => *id, - None => { - // The chain ends here, probably because some verification was done - // before we started recording verifiers. - // It's unclear how long the chain really is. - transitive_chain = 0; - break; - } - }; - if bot_ids.contains(¤t_verifier_id) { - has_bot = true; - } - transitive_chain = transitive_chain.saturating_add(1); - } - - if transitive_chain > 0 { - contact.transitive_chain = Some(transitive_chain); - } - - if has_bot { - contact.verified = VerifiedStatus::TransitiveViaBot; - } - } - } - - // Fill direct_chat - for contact in &mut contacts { - let direct_chat = context - .sql - .exists( - "SELECT COUNT(*) - FROM chats_contacts cc INNER JOIN chats - WHERE cc.contact_id=? AND chats.type=?", - (contact.id, Chattype::Single), - ) - .await?; - contact.direct_chat = direct_chat; - } - - Ok(contacts) -} - #[derive(Serialize)] struct MessageStats { to_verified: u32, @@ -175,69 +76,25 @@ struct MessageStats { unencrypted: u32, } -async fn get_message_stats(context: &Context) -> Result { - let enabled_ts: i64 = context - .get_config_i64(Config::SelfReportingEnabledTimestamp) - .await?; - ensure!(enabled_ts > 0, "Enabled Timestamp missing"); - - let selfreporting_bot_chat_id = get_selfreporting_bot(context).await?; - - let trans_fn = |t: &mut rusqlite::Transaction| { - t.pragma_update(None, "query_only", "0")?; - t.execute( - "CREATE TEMP TABLE temp.verified_chats ( - id INTEGER PRIMARY KEY - ) STRICT", - (), - )?; - - t.execute( - "INSERT INTO temp.verified_chats - SELECT id FROM chats - WHERE protected=1 AND id>9", - (), - )?; - - let to_verified = t.query_row( - "SELECT COUNT(*) FROM msgs - WHERE chat_id IN temp.verified_chats - AND chat_id<>? AND id>9 AND timestamp_sent>?", - (selfreporting_bot_chat_id, enabled_ts), - |row| row.get(0), - )?; - - let unverified_encrypted = t.query_row( - "SELECT COUNT(*) FROM msgs - WHERE chat_id not IN temp.verified_chats - AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') - AND chat_id<>? AND id>9 AND timestamp_sent>?", - (selfreporting_bot_chat_id, enabled_ts), - |row| row.get(0), - )?; - - let unencrypted = t.query_row( - "SELECT COUNT(*) FROM msgs - WHERE chat_id not IN temp.verified_chats - AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') - AND chat_id<>? AND id>9 AND timestamp_sent>=?", - (selfreporting_bot_chat_id, enabled_ts), - |row| row.get(0), - )?; - - t.execute("DROP TABLE temp.verified_chats", ())?; - - Ok(MessageStats { - to_verified, - unverified_encrypted, - unencrypted, - }) - }; - - let query_only = true; - let message_stats: MessageStats = context.sql.transaction_ex(query_only, trans_fn).await?; +#[repr(u32)] +#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] +enum SecurejoinSource { + Unknown = 0, + ExternalLink = 1, + InternalLink = 2, + Clipboard = 3, + ImageLoaded = 4, + Scan = 5, +} - Ok(message_stats) +#[derive(Serialize)] +struct SecurejoinSourceStats { + unknown: u32, + external_link: u32, + internal_link: u32, + clipboard: u32, + image_loaded: u32, + scan: u32, } /// Sends a message with statistics about the usage of Delta Chat, @@ -294,37 +151,6 @@ See TODO[blog post] for more information." Ok(chat_id) } -async fn get_selfreporting_bot(context: &Context) -> Result { - let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; - - let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { - // Already exists, no need to create. - res - } else { - let chat_id = ChatId::get_for_contact(context, contact_id).await?; - chat_id - .set_visibility(context, ChatVisibility::Archived) - .await?; - chat::set_muted(context, chat_id, MuteDuration::Forever).await?; - chat_id - }; - - chat_id - .set_protection( - context, - ProtectionStatus::Protected, - time(), - Some(contact_id), - ) - .await?; - - Ok(chat_id) -} - async fn get_self_report(context: &Context) -> Result { let num_msgs: u32 = context .sql @@ -438,25 +264,199 @@ async fn get_self_report(context: &Context) -> Result { Ok(serde_json::to_string_pretty(&statistics)?) } -#[repr(u32)] -#[derive(Debug, Clone, Copy, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord)] -enum SecurejoinSource { - Unknown = 0, - ExternalLink = 1, - InternalLink = 2, - Clipboard = 3, - ImageLoaded = 4, - Scan = 5, +async fn get_selfreporting_bot(context: &Context) -> Result { + let contact_id: ContactId = *import_vcard(context, SELF_REPORTING_BOT_VCARD) + .await? + .first() + .context("Self reporting bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, ContactId::SELF).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat_id + .set_visibility(context, ChatVisibility::Archived) + .await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + chat_id + .set_protection( + context, + ProtectionStatus::Protected, + time(), + Some(contact_id), + ) + .await?; + + Ok(chat_id) } -#[derive(Serialize)] -struct SecurejoinSourceStats { - unknown: u32, - external_link: u32, - internal_link: u32, - clipboard: u32, - image_loaded: u32, - scan: u32, +async fn get_contact_stats(context: &Context) -> Result> { + let mut verified_by_map: BTreeMap = BTreeMap::new(); + let mut bot_ids: BTreeSet = BTreeSet::new(); + + let mut contacts: Vec = context + .sql + .query_map( + "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c + WHERE id>9 AND origin>? AND addr<>?", + (Origin::Hidden, SELF_REPORTING_BOT_EMAIL), + |row| { + let id = row.get(0)?; + let is_encrypted: bool = row.get(1)?; + let verifier: ContactId = row.get(2)?; + let last_seen: u64 = row.get(3)?; + let bot: bool = row.get(4)?; + + let verified = match (is_encrypted, verifier) { + (true, ContactId::SELF) => VerifiedStatus::Direct, + (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, + (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later + (false, _) => VerifiedStatus::Unencrypted, + }; + + if verifier != ContactId::UNDEFINED { + verified_by_map.insert(id, verifier); + } + + if bot { + bot_ids.insert(id); + } + + Ok(ContactStat { + id, + verified, + bot, + direct_chat: false, // will be filled later + last_seen, + transitive_chain: None, // will be filled later + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Fill TransitiveViaBot and transitive_chain + for contact in &mut contacts { + if contact.verified == VerifiedStatus::Transitive { + let mut transitive_chain: u32 = 0; + let mut has_bot = false; + let mut current_verifier_id = contact.id; + + while current_verifier_id != ContactId::SELF && transitive_chain < 100 { + current_verifier_id = match verified_by_map.get(¤t_verifier_id) { + Some(id) => *id, + None => { + // The chain ends here, probably because some verification was done + // before we started recording verifiers. + // It's unclear how long the chain really is. + transitive_chain = 0; + break; + } + }; + if bot_ids.contains(¤t_verifier_id) { + has_bot = true; + } + transitive_chain = transitive_chain.saturating_add(1); + } + + if transitive_chain > 0 { + contact.transitive_chain = Some(transitive_chain); + } + + if has_bot { + contact.verified = VerifiedStatus::TransitiveViaBot; + } + } + } + + // Fill direct_chat + for contact in &mut contacts { + let direct_chat = context + .sql + .exists( + "SELECT COUNT(*) + FROM chats_contacts cc INNER JOIN chats + WHERE cc.contact_id=? AND chats.type=?", + (contact.id, Chattype::Single), + ) + .await?; + contact.direct_chat = direct_chat; + } + + Ok(contacts) +} + +async fn get_message_stats(context: &Context) -> Result { + let enabled_ts: i64 = context + .get_config_i64(Config::SelfReportingEnabledTimestamp) + .await?; + ensure!(enabled_ts > 0, "Enabled Timestamp missing"); + + let selfreporting_bot_chat_id = get_selfreporting_bot(context).await?; + + let trans_fn = |t: &mut rusqlite::Transaction| { + t.pragma_update(None, "query_only", "0")?; + t.execute( + "CREATE TEMP TABLE temp.verified_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + t.execute( + "INSERT INTO temp.verified_chats + SELECT id FROM chats + WHERE protected=1 AND id>9", + (), + )?; + + let to_verified = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.verified_chats + AND chat_id<>? AND id>9 AND timestamp_sent>?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + let unverified_encrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.verified_chats + AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND chat_id<>? AND id>9 AND timestamp_sent>?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + let unencrypted = t.query_row( + "SELECT COUNT(*) FROM msgs + WHERE chat_id not IN temp.verified_chats + AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND chat_id<>? AND id>9 AND timestamp_sent>=?", + (selfreporting_bot_chat_id, enabled_ts), + |row| row.get(0), + )?; + + t.execute("DROP TABLE temp.verified_chats", ())?; + + Ok(MessageStats { + to_verified, + unverified_encrypted, + unencrypted, + }) + }; + + let query_only = true; + let message_stats: MessageStats = context.sql.transaction_ex(query_only, trans_fn).await?; + + Ok(message_stats) } pub(crate) async fn count_securejoin_source(context: &Context, source: Option) -> Result<()> { From 6395dbaaf5d243f0bd72be0c7ef52095e00f0fa2 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 10 Jul 2025 16:52:37 +0200 Subject: [PATCH 17/18] fix: Make it compile & the tests pass --- src/self_reporting.rs | 28 +++++++++++++++------- src/self_reporting/self_reporting_tests.rs | 6 ++--- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/self_reporting.rs b/src/self_reporting.rs index d9bf4f6e19..03803d1b15 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -117,6 +117,9 @@ pub async fn maybe_send_self_report(context: &Context) -> Result> async fn send_self_report(context: &Context) -> Result { info!(context, "Sending self report."); + + let last_selfreport_time = context.get_config_i64(Config::LastSelfReportSent).await?; + // Setting this config at the beginning avoids endless loops when things do not // work out for whatever reason. context @@ -135,10 +138,13 @@ This helps us improve the security of Delta Chat. \ See TODO[blog post] for more information." .to_string(), ); + + let self_report = get_self_report(context, last_selfreport_time).await?; + msg.set_file_from_bytes( context, "statistics.txt", - get_self_report(context).await?.as_bytes(), + self_report.as_bytes(), Some("text/plain"), )?; @@ -151,7 +157,7 @@ See TODO[blog post] for more information." Ok(chat_id) } -async fn get_self_report(context: &Context) -> Result { +async fn get_self_report(context: &Context, last_selfreport_time: i64) -> Result { let num_msgs: u32 = context .sql .query_get_value( @@ -257,7 +263,7 @@ async fn get_self_report(context: &Context) -> Result { chat_numbers, self_reporting_id, contact_stats: get_contact_stats(context).await?, - message_stats: get_message_stats(context).await?, + message_stats: get_message_stats(context, last_selfreport_time).await?, securejoin_source_stats: get_securejoin_source_stats(context).await?, }; @@ -394,11 +400,17 @@ async fn get_contact_stats(context: &Context) -> Result> { Ok(contacts) } -async fn get_message_stats(context: &Context) -> Result { +async fn get_message_stats( + context: &Context, + mut last_selfreport_time: i64, +) -> Result { let enabled_ts: i64 = context .get_config_i64(Config::SelfReportingEnabledTimestamp) .await?; - ensure!(enabled_ts > 0, "Enabled Timestamp missing"); + if last_selfreport_time == 0 { + last_selfreport_time = enabled_ts; + } + ensure!(last_selfreport_time > 0, "Enabled Timestamp missing"); let selfreporting_bot_chat_id = get_selfreporting_bot(context).await?; @@ -422,7 +434,7 @@ async fn get_message_stats(context: &Context) -> Result { "SELECT COUNT(*) FROM msgs WHERE chat_id IN temp.verified_chats AND chat_id<>? AND id>9 AND timestamp_sent>?", - (selfreporting_bot_chat_id, enabled_ts), + (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; @@ -431,7 +443,7 @@ async fn get_message_stats(context: &Context) -> Result { WHERE chat_id not IN temp.verified_chats AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') AND chat_id<>? AND id>9 AND timestamp_sent>?", - (selfreporting_bot_chat_id, enabled_ts), + (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; @@ -440,7 +452,7 @@ async fn get_message_stats(context: &Context) -> Result { WHERE chat_id not IN temp.verified_chats AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') AND chat_id<>? AND id>9 AND timestamp_sent>=?", - (selfreporting_bot_chat_id, enabled_ts), + (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs index fbe3a9c802..827ec6e44a 100644 --- a/src/self_reporting/self_reporting_tests.rs +++ b/src/self_reporting/self_reporting_tests.rs @@ -45,12 +45,12 @@ async fn test_self_report_one_contact() -> Result<()> { let bob = &tcm.bob().await; alice.set_config_bool(Config::SelfReporting, true).await?; - let report = get_self_report(alice).await?; + let report = get_self_report(alice, 0).await?; let r: serde_json::Value = serde_json::from_str(&report)?; tcm.send_recv_accept(bob, alice, "Hi!").await; - let report = get_self_report(alice).await?; + let report = get_self_report(alice, 0).await?; println!("\nWith Bob:\n{report}\n"); let r2: serde_json::Value = serde_json::from_str(&report)?; @@ -142,7 +142,7 @@ async fn test_self_report_securejoin_source_stats() -> Result<()> { } async fn check_securejoin_report(context: &TestContext, expected: &SecurejoinSourceStats) { - let report = get_self_report(context).await.unwrap(); + let report = get_self_report(context, 0).await.unwrap(); let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); let actual = &actual["securejoin_source_stats"]; From 511aaa90651a18a9e80e6ffcba896e380d1ea494 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Thu, 10 Jul 2025 19:08:11 +0200 Subject: [PATCH 18/18] Add test for message stats, and fix bugs --- src/config.rs | 4 +- src/self_reporting.rs | 9 ++-- src/self_reporting/self_reporting_tests.rs | 50 +++++++++++++++++++++- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/src/config.rs b/src/config.rs index 22577315ec..786d420535 100644 --- a/src/config.rs +++ b/src/config.rs @@ -22,7 +22,7 @@ use crate::login_param::ConfiguredLoginParam; use crate::mimefactory::RECOMMENDED_FILE_SIZE; use crate::provider::{Provider, get_provider_by_id}; use crate::sync::{self, Sync::*, SyncData}; -use crate::tools::get_abs_path; +use crate::tools::{get_abs_path, time}; /// The available configuration keys. #[derive( @@ -837,7 +837,7 @@ impl Context { Config::SelfReporting => { self.sql.set_raw_config(key.as_ref(), value).await?; self.sql - .set_raw_config(Config::SelfReportingEnabledTimestamp.as_ref(), value) + .set_raw_config_int64(Config::SelfReportingEnabledTimestamp.as_ref(), time()) .await?; } _ => { diff --git a/src/self_reporting.rs b/src/self_reporting.rs index 03803d1b15..86d3d4573a 100644 --- a/src/self_reporting.rs +++ b/src/self_reporting.rs @@ -433,7 +433,8 @@ async fn get_message_stats( let to_verified = t.query_row( "SELECT COUNT(*) FROM msgs WHERE chat_id IN temp.verified_chats - AND chat_id<>? AND id>9 AND timestamp_sent>?", + AND chat_id<>? AND id>9 AND timestamp>=? AND hidden=0 + AND NOT (param GLOB '*\nS=*' OR param GLOB 'S=*')", (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; @@ -442,7 +443,8 @@ async fn get_message_stats( "SELECT COUNT(*) FROM msgs WHERE chat_id not IN temp.verified_chats AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') - AND chat_id<>? AND id>9 AND timestamp_sent>?", + AND chat_id<>? AND id>9 AND timestamp>=? AND hidden=0 + AND NOT (param GLOB '*\nS=*' OR param GLOB 'S=*')", (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; @@ -451,7 +453,8 @@ async fn get_message_stats( "SELECT COUNT(*) FROM msgs WHERE chat_id not IN temp.verified_chats AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') - AND chat_id<>? AND id>9 AND timestamp_sent>=?", + AND chat_id<>? AND id>9 AND timestamp>=? AND hidden=0 + AND NOT (param GLOB '*\nS=*' OR param GLOB 'S=*')", (selfreporting_bot_chat_id, last_selfreport_time), |row| row.get(0), )?; diff --git a/src/self_reporting/self_reporting_tests.rs b/src/self_reporting/self_reporting_tests.rs index 827ec6e44a..921e466d76 100644 --- a/src/self_reporting/self_reporting_tests.rs +++ b/src/self_reporting/self_reporting_tests.rs @@ -82,6 +82,54 @@ async fn test_self_report_one_contact() -> Result<()> { Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_message_stats() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::SelfReporting, true).await?; + let email_chat = alice.create_email_chat(bob).await; + let encrypted_chat = alice.create_chat(bob).await; + + let mut expected = MessageStats { + to_verified: 0, + unverified_encrypted: 0, + unencrypted: 0, + }; + + check_message_stats_report(alice, &expected).await; + + alice.send_text(email_chat.id, "foo").await; + expected.unencrypted += 1; + check_message_stats_report(alice, &expected).await; + + alice.send_text(encrypted_chat.id, "foo").await; + expected.unverified_encrypted += 1; + check_message_stats_report(alice, &expected).await; + + alice.send_text(encrypted_chat.id, "foo").await; + expected.unverified_encrypted += 1; + check_message_stats_report(alice, &expected).await; + + tcm.execute_securejoin(alice, bob).await; + expected.to_verified = expected.unverified_encrypted; + expected.unverified_encrypted = 0; + check_message_stats_report(alice, &expected).await; + + Ok(()) +} + +async fn check_message_stats_report(context: &TestContext, expected: &MessageStats) { + let report = get_self_report(context, 0).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&report).unwrap(); + let actual = &actual["message_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_self_report_securejoin_source_stats() -> Result<()> { let mut tcm = TestContextManager::new(); @@ -149,5 +197,5 @@ async fn check_securejoin_report(context: &TestContext, expected: &SecurejoinSou let expected = serde_json::to_string_pretty(&expected).unwrap(); let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); - assert_eq!(&expected, actual); + assert_eq!(actual, &expected); }