TakeV
/
dino
mirror of https://github.com/TakeV-Lambda/dino.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve history sync under load

This commit is contained in:
Marvin W 2023-02-06 21:13:33 +01:00
parent 1d123c7e66
commit d092473fe4
No known key found for this signature in database
GPG Key ID: 072E9235DB996F2A
4 changed files with 102 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
libdino/src/service/history_sync.vala
View File

@ -11,6 +11,8 @@ public class Dino.HistorySync {
private Database db;
public HashMap<Account, HashMap<Jid, int>> current_catchup_id = new HashMap<Account, HashMap<Jid, int>>(Account.hash_func, Account.equals_func);
public WeakMap<Account, XmppStream> sync_streams = new WeakMap<Account, XmppStream>(Account.hash_func, Account.equals_func);
public HashMap<Account, HashMap<Jid, Cancellable>> cancellables = new HashMap<Account, HashMap<Jid, Cancellable>>(Account.hash_func, Account.equals_func);
public HashMap<Account, HashMap<string, DateTime>> mam_times = new HashMap<Account, HashMap<string, DateTime>>();
public HashMap<string, int> hitted_range = new HashMap<string, int>();
@ -27,9 +29,11 @@ public class Dino.HistorySync {
stream_interactor.account_added.connect(on_account_added);
stream_interactor.connection_manager.stream_opened.connect((account, stream) => {
debug("MAM: [%s] Reset catchup_id", account.bare_jid.to_string());
current_catchup_id.unset(account);
stream_interactor.stream_negotiated.connect((account, stream) => {
if (current_catchup_id.has_key(account)) {
debug("MAM: [%s] Reset catchup_id", account.bare_jid.to_string());
current_catchup_id[account].clear();
}
});
}
@ -206,7 +210,7 @@ public class Dino.HistorySync {
PageRequestResult page_result = yield get_mam_page(account, query_params, null, cancellable);
debug("[%s | %s] Latest page result: %s", account.bare_jid.to_string(), mam_server.to_string(), page_result.page_result.to_string());
if (page_result.page_result == PageResult.Error) {
if (page_result.page_result == PageResult.Error || page_result.page_result == PageResult.Cancelled) {
return null;
}
@ -239,7 +243,7 @@ public class Dino.HistorySync {
}
// Either we need to fetch more pages or this is the first db entry ever
debug("[%s | %s] Creating new db range for latest page", mam_server.to_string(), mam_server.to_string());
debug("[%s | %s] Creating new db range for latest page", account.bare_jid.to_string(), mam_server.to_string());
string from_id = page_result.query_result.first;
string to_id = page_result.query_result.last;
@ -328,7 +332,7 @@ public class Dino.HistorySync {
page_result = yield get_mam_page(account, query_params, page_result, cancellable);
debug("Page result %s %b", page_result.page_result.to_string(), page_result.stanzas == null);
if (page_result.page_result == PageResult.Error || page_result.stanzas == null) return page_result;
if (page_result.page_result == PageResult.Error || page_result.page_result == PageResult.Cancelled || page_result.stanzas == null) return page_result;
string earliest_mam_id = page_result.query_result.first;
long earliest_mam_time = (long)mam_times[account][earliest_mam_id].to_unix();
@ -354,7 +358,8 @@ public class Dino.HistorySync {
TargetReached,
NoMoreMessages,
Duplicate,
Error
Error,
Cancelled
}
/**
@ -364,9 +369,9 @@ public class Dino.HistorySync {
XmppStream stream = stream_interactor.get_stream(account);
Xmpp.MessageArchiveManagement.QueryResult query_result = null;
if (prev_page_result == null) {
query_result = yield Xmpp.MessageArchiveManagement.V2.query_archive(stream, query_params);
query_result = yield Xmpp.MessageArchiveManagement.V2.query_archive(stream, query_params, cancellable);
} else {
query_result = yield Xmpp.MessageArchiveManagement.V2.page_through_results(stream, query_params, prev_page_result.query_result);
query_result = yield Xmpp.MessageArchiveManagement.V2.page_through_results(stream, query_params, prev_page_result.query_result, cancellable);
}
return yield process_query_result(account, query_params, query_result, cancellable);
}
@ -394,6 +399,10 @@ public class Dino.HistorySync {
string query_id = query_params.query_id;
string? after_id = query_params.start_id;
if (cancellable != null && cancellable.is_cancelled()) {
return new PageRequestResult(PageResult.Cancelled, query_result, stanzas[query_id]);
}
if (stanzas.has_key(query_id) && !stanzas[query_id].is_empty) {
// Check it we reached our target (from_id)
@ -402,17 +411,21 @@ public class Dino.HistorySync {
if (mam_message_flag != null && mam_message_flag.mam_id != null) {
if (after_id != null && mam_message_flag.mam_id == after_id) {
// Successfully fetched the whole range
var ret = new PageRequestResult(PageResult.TargetReached, query_result, stanzas[query_id]);
send_messages_back_into_pipeline(account, query_id);
return ret;
yield send_messages_back_into_pipeline(account, query_id, cancellable);
if (cancellable != null && cancellable.is_cancelled()) {
return new PageRequestResult(PageResult.Cancelled, query_result, stanzas[query_id]);
}
return new PageRequestResult(PageResult.TargetReached, query_result, stanzas[query_id]);
}
}
}
if (hitted_range.has_key(query_id) && hitted_range[query_id] == -2) {
// Message got filtered out by xmpp-vala, but succesfull range fetch nevertheless
var ret = new PageRequestResult(PageResult.TargetReached, query_result, stanzas[query_id]);
send_messages_back_into_pipeline(account, query_id);
return ret;
// Message got filtered out by xmpp-vala, but succesful range fetch nevertheless
yield send_messages_back_into_pipeline(account, query_id);
if (cancellable != null && cancellable.is_cancelled()) {
return new PageRequestResult(PageResult.Cancelled, query_result, stanzas[query_id]);
}
return new PageRequestResult(PageResult.TargetReached, query_result, stanzas[query_id]);
}
// Check for duplicates. Go through all messages and build a db query.
@ -444,16 +457,19 @@ public class Dino.HistorySync {
}
}
var res = new PageRequestResult(page_result, query_result, stanzas.has_key(query_id) ? stanzas[query_id] : null);
send_messages_back_into_pipeline(account, query_id);
return res;
yield send_messages_back_into_pipeline(account, query_id);
if (cancellable != null && cancellable.is_cancelled()) {
page_result = PageResult.Cancelled;
}
return new PageRequestResult(page_result, query_result, stanzas.has_key(query_id) ? stanzas[query_id] : null);
}
private void send_messages_back_into_pipeline(Account account, string query_id) {
private async void send_messages_back_into_pipeline(Account account, string query_id, Cancellable? cancellable = null) {
if (!stanzas.has_key(query_id)) return;
foreach (Xmpp.MessageStanza message in stanzas[query_id]) {
stream_interactor.get_module(MessageProcessor.IDENTITY).run_pipeline_announce.begin(account, message);
if (cancellable != null && cancellable.is_cancelled()) break;
yield stream_interactor.get_module(MessageProcessor.IDENTITY).run_pipeline_announce(account, message);
}
stanzas.unset(query_id);
}
@ -463,14 +479,16 @@ public class Dino.HistorySync {
mam_times[account] = new HashMap<string, DateTime>();
XmppStream? stream_bak = null;
stream_interactor.module_manager.get_module(account, Xmpp.MessageArchiveManagement.Module.IDENTITY).feature_available.connect( (stream) => {
if (stream == stream_bak) return;
stream_interactor.connection_manager.stream_attached_modules.connect((account, stream) => {
if (!current_catchup_id.has_key(account)) {
current_catchup_id[account] = new HashMap<Jid, int>(Jid.hash_func, Jid.equals_func);
} else {
current_catchup_id[account].clear();
}
});
current_catchup_id[account] = new HashMap<Jid, int>(Jid.hash_func, Jid.equals_func);
stream_bak = stream;
debug("[%s] MAM available", account.bare_jid.to_string());
fetch_everything.begin(account, account.bare_jid);
stream_interactor.module_manager.get_module(account, Xmpp.MessageArchiveManagement.Module.IDENTITY).feature_available.connect((stream) => {
consider_fetch_everything(account, stream);
});
stream_interactor.module_manager.get_module(account, Xmpp.MessageModule.IDENTITY).received_message_unprocessed.connect((stream, message) => {
@ -478,6 +496,24 @@ public class Dino.HistorySync {
});
}
private void consider_fetch_everything(Account account, XmppStream stream) {
if (sync_streams.has(account, stream)) return;
debug("[%s] MAM available", account.bare_jid.to_string());
sync_streams[account] = stream;
if (!cancellables.has_key(account)) {
cancellables[account] = new HashMap<Jid, Cancellable>();
}
if (cancellables[account].has_key(account.bare_jid)) {
cancellables[account][account.bare_jid].cancel();
}
cancellables[account][account.bare_jid] = new Cancellable();
fetch_everything.begin(account, account.bare_jid, cancellables[account][account.bare_jid], new DateTime.from_unix_utc(0), (_, res) => {
fetch_everything.end(res);
cancellables[account].unset(account.bare_jid);
});
}
public static void cleanup_db_ranges(Database db, Account account) {
var ranges = new HashMap<Jid, ArrayList<MamRange>>(Jid.hash_func, Jid.equals_func);
foreach (Row row in db.mam_catchup.select().with(db.mam_catchup.account_id, "=", account.id)) {

36
libdino/src/service/muc_manager.vala
View File

@ -23,6 +23,7 @@ public class MucManager : StreamInteractionModule, Object {
private StreamInteractor stream_interactor;
private HashMap<Account, HashSet<Jid>> mucs_todo = new HashMap<Account, HashSet<Jid>>(Account.hash_func, Account.equals_func);
private HashMap<Account, HashSet<Jid>> mucs_joining = new HashMap<Account, HashSet<Jid>>(Account.hash_func, Account.equals_func);
private HashMap<Account, HashMap<Jid, Cancellable>> mucs_sync_cancellables = new HashMap<Account, HashMap<Jid, Cancellable>>(Account.hash_func, Account.equals_func);
private HashMap<Jid, Xep.Muc.MucEnterError> enter_errors = new HashMap<Jid, Xep.Muc.MucEnterError>(Jid.hash_func, Jid.equals_func);
private ReceivedMessageListener received_message_listener;
private HashMap<Account, BookmarksProvider> bookmarks_provider = new HashMap<Account, BookmarksProvider>(Account.hash_func, Account.equals_func);
@ -56,7 +57,7 @@ public class MucManager : StreamInteractionModule, Object {
}
// already_autojoin: Without this flag we'd be retrieving bookmarks (to check for autojoin) from the sender on every join
public async Muc.JoinResult? join(Account account, Jid jid, string? nick, string? password, bool already_autojoin = false) {
public async Muc.JoinResult? join(Account account, Jid jid, string? nick, string? password, bool already_autojoin = false, Cancellable? cancellable = null) {
XmppStream? stream = stream_interactor.get_stream(account);
if (stream == null) return null;
@ -102,14 +103,22 @@ public class MucManager : StreamInteractionModule, Object {
stream_interactor.get_module(ConversationManager.IDENTITY).start_conversation(joined_conversation);
if (can_do_mam) {
var history_sync = stream_interactor.get_module(MessageProcessor.IDENTITY).history_sync;
if (conversation == null) {
// We never joined the conversation before, just fetch the latest MAM page
yield stream_interactor.get_module(MessageProcessor.IDENTITY).history_sync
.fetch_latest_page(account, jid.bare_jid, null, new DateTime.from_unix_utc(0));
yield history_sync.fetch_latest_page(account, jid.bare_jid, null, new DateTime.from_unix_utc(0), cancellable);
} else {
// Fetch everything up to the last time the user actively joined
stream_interactor.get_module(MessageProcessor.IDENTITY).history_sync
.fetch_everything.begin(account, jid.bare_jid, null, conversation.active_last_changed);
if (!mucs_sync_cancellables.has_key(account)) {
mucs_sync_cancellables[account] = new HashMap<Jid, Cancellable>();
}
if (!mucs_sync_cancellables[account].has_key(jid.bare_jid)) {
mucs_sync_cancellables[account][jid.bare_jid] = new Cancellable();
history_sync.fetch_everything.begin(account, jid.bare_jid, mucs_sync_cancellables[account][jid.bare_jid], conversation.active_last_changed, (_, res) => {
history_sync.fetch_everything.end(res);
mucs_sync_cancellables[account].unset(jid.bare_jid);
});
}
}
}
} else if (res.muc_error != null) {
@ -132,6 +141,14 @@ public class MucManager : StreamInteractionModule, Object {
Conversation? conversation = stream_interactor.get_module(ConversationManager.IDENTITY).get_conversation(jid, account);
if (conversation != null) stream_interactor.get_module(ConversationManager.IDENTITY).close_conversation(conversation);
cancel_sync(account, jid);
}
private void cancel_sync(Account account, Jid jid) {
if (mucs_sync_cancellables.has_key(account) && mucs_sync_cancellables[account].has_key(jid.bare_jid) && !mucs_sync_cancellables[account][jid.bare_jid].is_cancelled()) {
mucs_sync_cancellables[account][jid.bare_jid].cancel();
}
}
public async DataForms.DataForm? get_config_form(Account account, Jid jid) {
@ -403,6 +420,7 @@ public class MucManager : StreamInteractionModule, Object {
private void on_account_added(Account account) {
stream_interactor.module_manager.get_module(account, Xep.Muc.Module.IDENTITY).self_removed_from_room.connect( (stream, jid, code) => {
cancel_sync(account, jid);
left(account, jid);
});
stream_interactor.module_manager.get_module(account, Xep.Muc.Module.IDENTITY).subject_set.connect( (stream, subject, jid) => {
@ -467,6 +485,14 @@ public class MucManager : StreamInteractionModule, Object {
}
private async void on_stream_negotiated(Account account, XmppStream stream) {
if (mucs_sync_cancellables.has_key(account)) {
foreach (Cancellable cancellable in mucs_sync_cancellables[account].values) {
if (!cancellable.is_cancelled()) {
cancellable.cancel();
}
}
}
yield initialize_bookmarks_provider(account);
Set<Conference>? conferences = yield bookmarks_provider[account].get_conferences(stream);

13
libdino/src/util/weak_map.vala
View File

@ -22,12 +22,11 @@ public class WeakMap<K, V> : Gee.AbstractMap<K, V> {
hash_map = new HashMap<K, weak V>();
notify_map = new HashMap<K, WeakNotifyWrapper>();
} else {
hash_map = new HashMap<K, weak V>((v) => { return this.key_hash_func(v); },
(a, b) => { return this.key_equal_func(a, b); },
(a, b) => { return this.value_equal_func(a, b); });
notify_map = new HashMap<K, WeakNotifyWrapper>((v) => { return this.key_hash_func(v); },
(a, b) => { return this.key_equal_func(a, b); },
(a, b) => { return this.value_equal_func(a, b); });
hash_map = new HashMap<K, weak V>((v) => { return this.key_hash_func != null ? this.key_hash_func(v) : 0; },
(a, b) => { return this.key_equal_func != null ? this.key_equal_func(a, b) : a == b; },
(a, b) => { return this.value_equal_func != null ? this.value_equal_func(a, b) : a == b; });
notify_map = new HashMap<K, WeakNotifyWrapper>((v) => { return this.key_hash_func != null ? this.key_hash_func(v) : 0; },
(a, b) => { return this.key_equal_func != null ? this.key_equal_func(a, b) : a == b; });
}
}
@ -49,7 +48,7 @@ public class WeakMap<K, V> : Gee.AbstractMap<K, V> {
}
public override bool has(K key, V value) {
assert_not_reached();
return has_key(key) && (this.value_equal_func != null ? this.value_equal_func(hash_map[key], value) : hash_map[key] == value);
}
public override bool has_key(K key) {

2
xmpp-vala/src/core/xmpp_stream.vala
View File

@ -1,6 +1,6 @@
using Gee;
public abstract class Xmpp.XmppStream {
public abstract class Xmpp.XmppStream : Object {
public signal void received_node(XmppStream stream, StanzaNode node);
public signal void received_root_node(XmppStream stream, StanzaNode node);