From 09048a9733cb89c1bb667d5059b3b71470061845 Mon Sep 17 00:00:00 2001 From: Merci Jacob Date: Tue, 18 Feb 2025 17:41:21 +0200 Subject: [PATCH 1/2] enh(backend): concurrently load messages for combined lists using child processes --- composer.json | 5 +- composer.lock | 426 ++++++++++++++++++++++++- modules/imap/functions.php | 116 ++++--- modules/imap/handler_modules.php | 18 +- modules/imap/workers/messages_list.php | 79 +++++ 5 files changed, 575 insertions(+), 69 deletions(-) create mode 100644 modules/imap/workers/messages_list.php diff --git a/composer.json b/composer.json index 4df7e16ee..aa60f686c 100644 --- a/composer.json +++ b/composer.json @@ -57,7 +57,10 @@ "twbs/bootstrap": "^5.3", "twbs/bootstrap-icons": "^1.11", "webklex/composer-info": "^0.0.1", - "zbateson/mail-mime-parser": "^2.4" + "zbateson/mail-mime-parser": "^2.4", + "react/promise": "^3.2.0", + "react/async": "^4.3.0", + "react/child-process": "^0.6.6" }, "require-dev": { "phpunit/phpunit": "^10.5" diff --git a/composer.lock b/composer.lock index d9357206d..7272dec5b 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "6a75aa856405dde3a37887d21821e90a", + "content-hash": "5bffec86ce0fc86baf335beabefe4dba", "packages": [ { "name": "bacon/bacon-qr-code", @@ -244,6 +244,53 @@ }, "time": "2024-07-08T12:26:09+00:00" }, + { + "name": "evenement/evenement", + "version": "v3.0.2", + "source": { + "type": "git", + "url": "https://github.com/igorw/evenement.git", + "reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/igorw/evenement/zipball/0a16b0d71ab13284339abb99d9d2bd813640efbc", + "reference": "0a16b0d71ab13284339abb99d9d2bd813640efbc", + "shasum": "" + }, + "require": { + "php": ">=7.0" + }, + "require-dev": { + "phpunit/phpunit": "^9 || ^6" + }, + "type": "library", + "autoload": { + "psr-4": { + "Evenement\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Igor Wiedler", + "email": "igor@wiedler.ch" + } + ], + "description": "Événement is a very simple event dispatching library for PHP", + "keywords": [ + "event-dispatcher", + "event-emitter" + ], + "support": { + "issues": "https://github.com/igorw/evenement/issues", + "source": "https://github.com/igorw/evenement/tree/v3.0.2" + }, + "time": "2023-08-08T05:53:35+00:00" + }, { "name": "ezyang/htmlpurifier", "version": "v4.18.0", @@ -1554,6 +1601,379 @@ }, "time": "2019-03-08T08:55:37+00:00" }, + { + "name": "react/async", + "version": "v4.3.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/async.git", + "reference": "635d50e30844a484495713e8cb8d9e079c0008a5" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/async/zipball/635d50e30844a484495713e8cb8d9e079c0008a5", + "reference": "635d50e30844a484495713e8cb8d9e079c0008a5", + "shasum": "" + }, + "require": { + "php": ">=8.1", + "react/event-loop": "^1.2", + "react/promise": "^3.2 || ^2.8 || ^1.2.1" + }, + "require-dev": { + "phpstan/phpstan": "1.10.39", + "phpunit/phpunit": "^9.6" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "React\\Async\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "Async utilities and fibers for ReactPHP", + "keywords": [ + "async", + "reactphp" + ], + "support": { + "issues": "https://github.com/reactphp/async/issues", + "source": "https://github.com/reactphp/async/tree/v4.3.0" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2024-06-04T14:40:02+00:00" + }, + { + "name": "react/child-process", + "version": "v0.6.6", + "source": { + "type": "git", + "url": "https://github.com/reactphp/child-process.git", + "reference": "1721e2b93d89b745664353b9cfc8f155ba8a6159" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/child-process/zipball/1721e2b93d89b745664353b9cfc8f155ba8a6159", + "reference": "1721e2b93d89b745664353b9cfc8f155ba8a6159", + "shasum": "" + }, + "require": { + "evenement/evenement": "^3.0 || ^2.0 || ^1.0", + "php": ">=5.3.0", + "react/event-loop": "^1.2", + "react/stream": "^1.4" + }, + "require-dev": { + "phpunit/phpunit": "^9.6 || ^5.7 || ^4.8.36", + "react/socket": "^1.16", + "sebastian/environment": "^5.0 || ^3.0 || ^2.0 || ^1.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "React\\ChildProcess\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "Event-driven library for executing child processes with ReactPHP.", + "keywords": [ + "event-driven", + "process", + "reactphp" + ], + "support": { + "issues": "https://github.com/reactphp/child-process/issues", + "source": "https://github.com/reactphp/child-process/tree/v0.6.6" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2025-01-01T16:37:48+00:00" + }, + { + "name": "react/event-loop", + "version": "v1.5.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/event-loop.git", + "reference": "bbe0bd8c51ffc05ee43f1729087ed3bdf7d53354" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/event-loop/zipball/bbe0bd8c51ffc05ee43f1729087ed3bdf7d53354", + "reference": "bbe0bd8c51ffc05ee43f1729087ed3bdf7d53354", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.6 || ^5.7 || ^4.8.36" + }, + "suggest": { + "ext-pcntl": "For signal handling support when using the StreamSelectLoop" + }, + "type": "library", + "autoload": { + "psr-4": { + "React\\EventLoop\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "ReactPHP's core reactor event loop that libraries can use for evented I/O.", + "keywords": [ + "asynchronous", + "event-loop" + ], + "support": { + "issues": "https://github.com/reactphp/event-loop/issues", + "source": "https://github.com/reactphp/event-loop/tree/v1.5.0" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2023-11-13T13:48:05+00:00" + }, + { + "name": "react/promise", + "version": "v3.2.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/promise.git", + "reference": "8a164643313c71354582dc850b42b33fa12a4b63" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/promise/zipball/8a164643313c71354582dc850b42b33fa12a4b63", + "reference": "8a164643313c71354582dc850b42b33fa12a4b63", + "shasum": "" + }, + "require": { + "php": ">=7.1.0" + }, + "require-dev": { + "phpstan/phpstan": "1.10.39 || 1.4.10", + "phpunit/phpunit": "^9.6 || ^7.5" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions_include.php" + ], + "psr-4": { + "React\\Promise\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "A lightweight implementation of CommonJS Promises/A for PHP", + "keywords": [ + "promise", + "promises" + ], + "support": { + "issues": "https://github.com/reactphp/promise/issues", + "source": "https://github.com/reactphp/promise/tree/v3.2.0" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2024-05-24T10:39:05+00:00" + }, + { + "name": "react/stream", + "version": "v1.4.0", + "source": { + "type": "git", + "url": "https://github.com/reactphp/stream.git", + "reference": "1e5b0acb8fe55143b5b426817155190eb6f5b18d" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/reactphp/stream/zipball/1e5b0acb8fe55143b5b426817155190eb6f5b18d", + "reference": "1e5b0acb8fe55143b5b426817155190eb6f5b18d", + "shasum": "" + }, + "require": { + "evenement/evenement": "^3.0 || ^2.0 || ^1.0", + "php": ">=5.3.8", + "react/event-loop": "^1.2" + }, + "require-dev": { + "clue/stream-filter": "~1.2", + "phpunit/phpunit": "^9.6 || ^5.7 || ^4.8.36" + }, + "type": "library", + "autoload": { + "psr-4": { + "React\\Stream\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Christian Lück", + "email": "christian@clue.engineering", + "homepage": "https://clue.engineering/" + }, + { + "name": "Cees-Jan Kiewiet", + "email": "reactphp@ceesjankiewiet.nl", + "homepage": "https://wyrihaximus.net/" + }, + { + "name": "Jan Sorgalla", + "email": "jsorgalla@gmail.com", + "homepage": "https://sorgalla.com/" + }, + { + "name": "Chris Boden", + "email": "cboden@gmail.com", + "homepage": "https://cboden.dev/" + } + ], + "description": "Event-driven readable and writable streams for non-blocking I/O in ReactPHP", + "keywords": [ + "event-driven", + "io", + "non-blocking", + "pipe", + "reactphp", + "readable", + "stream", + "writable" + ], + "support": { + "issues": "https://github.com/reactphp/stream/issues", + "source": "https://github.com/reactphp/stream/tree/v1.4.0" + }, + "funding": [ + { + "url": "https://opencollective.com/reactphp", + "type": "open_collective" + } + ], + "time": "2024-06-11T12:45:25+00:00" + }, { "name": "symfony/deprecation-contracts", "version": "v3.5.1", @@ -4122,9 +4542,9 @@ "ext-session": "*", "php": ">=8.1" }, - "platform-dev": [], + "platform-dev": {}, "platform-overrides": { "php": "8.1" }, - "plugin-api-version": "2.3.0" + "plugin-api-version": "2.6.0" } diff --git a/modules/imap/functions.php b/modules/imap/functions.php index ae6c0d657..9ef6562cd 100644 --- a/modules/imap/functions.php +++ b/modules/imap/functions.php @@ -1,7 +1,13 @@ 'ALL', 'sort' => 'ARRIVAL', @@ -1625,71 +1631,57 @@ function getCombinedMessagesLists($sources, $cache, $search) { 'defaultOffset' => 0, 'listPage' => 1 ]; - $search = array_merge($defaultSearch, $search); - - $filter = $search['filter']; - $sort = $search['sort']; - $reverse = $search['reverse']; - $searchTerms = $search['terms']; - $limit = $search['limit']; - $offsets = $search['offsets']; - $listPage = $search['listPage']; + $search = array_merge($defaultSearch, $search); + + $promises = array_map(function ($dataSource, $index) use ($context, $search) { + return function () use ($dataSource, $context, $search, $index) { + return new Promise(function ($resolve, $reject) use ($dataSource, $context, $search, $index) { + $process = new Process('php ' . __DIR__ . '/workers/messages_list.php'); + $process->start(Loop::get()); + $process->stdin->write(json_encode([ + 'index' => $index, + 'search' => $search, + 'dataSource' => $dataSource, + 'cache' => serialize($context['cache']), + 'session' => serialize($context['session']), + 'config' => serialize($context['config']) + ])); + $process->stdin->end(); + + $process->stdout->on('data', function ($output) use ($resolve) { + $resolve(json_decode($output, true)); + }); + + $process->on('exit', function ($exitCode) use ($reject) { + $reject(new \Exception('Worker exited with code ' . $exitCode)); + }); + }); + }; + }, $sources, array_keys($sources)); + + $promise = parallel($promises); + + try { + $results = await($promise); + } catch (\Exception $e) { + Hm_Msgs::add($e->getMessage(), 'error'); + return ['lists' => [], 'total' => 0, 'status' => []]; + } $totalMessages = 0; - $offset = $search['defaultOffset']; $messagesLists = []; $status = []; - foreach ($sources as $index => $dataSource) { - - if ($offsets && $listPage > 1) { - if (isset($offsets[$index]) && (int) $offsets[$index] > 0) { - $offset = (int) $offsets[$index] * ($listPage - 1); - } - } - - $mailbox = Hm_IMAP_List::get_connected_mailbox($dataSource['id'], $cache); - if ($mailbox && $mailbox->authed()) { - $connection = $mailbox->get_connection(); - - $folder = $dataSource['folder']; - $mailbox->select_folder(hex2bin($folder)); - $state = $connection->get_mailbox_status(hex2bin($folder)); - $status['imap_'.$dataSource['id'].'_'.$folder] = $state; - - if ($mailbox->is_imap()) { - if ($connection->is_supported( 'SORT' )) { - $sortedUids = $connection->get_message_sort_order($sort, $reverse, $filter); - } else { - $sortedUids = $connection->sort_by_fetch($sort, $reverse, $filter); - } - - $uids = $mailbox->search(hex2bin($folder), $filter, $sortedUids, $searchTerms); - } else { - // EWS - $uids = $connection->search($folder, $sort, $reverse, $filter, 0, $limit, $searchTerms); - } - - $total = count($uids); - $uids = array_slice($uids, $offset, $limit); - - $headers = $mailbox->get_message_list(hex2bin($folder), $uids); - $messages = []; - foreach ($uids as $uid) { - if (isset($headers[$uid])) { - $messages[] = $headers[$uid]; - } - } - - $messagesLists[] = array_map(function($msg) use ($dataSource, $folder) { - $msg['server_id'] = $dataSource['id']; - $msg['server_name'] = $dataSource['name']; - $msg['folder'] = $folder; - return $msg; - }, $messages); - $totalMessages += $total; - } + foreach ($results as $result) { + $totalMessages += $result['total']; + $status['imap_'.$result['dataSource']['id'].'_'.$result['folder']] = $result['status']; + $messagesLists[] = array_map(function($msg) use ($result) { + $msg['server_id'] = $result['dataSource']['id']; + $msg['server_name'] = $result['dataSource']['name']; + $msg['folder'] = $result['folder']; + return $msg; + }, $result['messages']); } - + return ['lists' => $messagesLists, 'total' => $totalMessages, 'status' => $status]; } diff --git a/modules/imap/handler_modules.php b/modules/imap/handler_modules.php index 61f0d1174..ea04ca284 100644 --- a/modules/imap/handler_modules.php +++ b/modules/imap/handler_modules.php @@ -1267,7 +1267,11 @@ public function process() { $offsets = explode(',', $offsets); } - $result = getCombinedMessagesLists($data_sources, $this->cache, [ + $result = getCombinedMessagesLists($data_sources, [ + 'cache' => $this->cache, + 'session' => $this->session, + 'config' => $this->user_config, + ], [ 'terms' => [[search_since_based_on_setting($this->user_config), $date]], 'listPage' => $list_page, 'limit' => $limit, @@ -1345,7 +1349,11 @@ public function process() { } $searchTerms[] = [search_since_based_on_setting($this->user_config), $date]; - $result = getCombinedMessagesLists($data_sources, $this->cache, [ + $result = getCombinedMessagesLists($data_sources, [ + 'cache' => $this->cache, + 'session' => $this->session, + 'config' => $this->user_config, + ], [ 'terms' => $searchTerms, 'listPage' => $list_page, 'limit' => $limit, @@ -2121,7 +2129,11 @@ public function process() { } $searchTerms[] = [search_since_based_on_setting($this->user_config), $date]; - $result = getCombinedMessagesLists($data_sources, $this->cache, [ + $result = getCombinedMessagesLists($data_sources, [ + 'cache' => $this->cache, + 'session' => $this->session, + 'config' => $this->user_config, + ], [ 'listPage' => $list_page, 'limit' => $limit, 'offsets' => $offsets, diff --git a/modules/imap/workers/messages_list.php b/modules/imap/workers/messages_list.php new file mode 100644 index 000000000..c88bb382d --- /dev/null +++ b/modules/imap/workers/messages_list.php @@ -0,0 +1,79 @@ + 1) { + if (isset($offsets[$index]) && (int) $offsets[$index] > 0) { + $offset = (int) $offsets[$index] * ($listPage - 1); + } +} + +Hm_IMAP_List::init($user_config, $session); +$mailbox = Hm_IMAP_List::get_connected_mailbox($dataSource['id'], $cache); +if ($mailbox && $mailbox->authed()) { + $connection = $mailbox->get_connection(); + $folder = $dataSource['folder']; + $mailbox->select_folder(hex2bin($folder)); + $state = $connection->get_mailbox_status(hex2bin($folder)); + + if ($mailbox->is_imap()) { + if ($connection->is_supported('SORT')) { + $sortedUids = $connection->get_message_sort_order($sort, $reverse, $filter); + } else { + $sortedUids = $connection->sort_by_fetch($sort, $reverse, $filter); + } + + $uids = $mailbox->search(hex2bin($folder), $filter, $sortedUids, $searchTerms); + } else { + // EWS + $uids = $connection->search($folder, $sort, $reverse, $filter, 0, $limit, $searchTerms); + } + + $total = count($uids); + $uids = array_slice($uids, $offset, $limit); + + $headers = $mailbox->get_message_list(hex2bin($folder), $uids); + $messages = []; + foreach ($uids as $uid) { + if (isset($headers[$uid])) { + $messages[] = $headers[$uid]; + } + } + + echo json_encode([ + 'uids' => $uids, + 'status' => $state, + 'messages' => $messages, + 'total' => $total, + 'dataSource' => $dataSource, + 'folder' => $folder + ]); +} else { + exit; +} From 90d068fce44e86187e6c5bdf84b5a471d538c87b Mon Sep 17 00:00:00 2001 From: Merci Jacob Date: Tue, 18 Feb 2025 17:48:19 +0200 Subject: [PATCH 2/2] fix(frontend): pagination --- modules/core/js_modules/actions/pagination.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/js_modules/actions/pagination.js b/modules/core/js_modules/actions/pagination.js index ffd779838..26d4c0e9c 100644 --- a/modules/core/js_modules/actions/pagination.js +++ b/modules/core/js_modules/actions/pagination.js @@ -65,7 +65,7 @@ async function changePage(toPage, button, offsets) { try { await messagesStore.load(); Hm_Utils.tbody().attr('id', messagesStore.list); - display_imap_mailbox(messagesStore.rows, null, messagesStore.list, messagesStore); + display_imap_mailbox(messagesStore.rows, messagesStore.list, messagesStore); $(".pagination .current").text(toPage); } catch (error) { Hm_Notices.show("Failed to fetch content", "danger");