From 9182ffb1573c77367ad6b5e4b1f3e4f52b3c3ea4 Mon Sep 17 00:00:00 2001 From: Andrew Branson Date: Mon, 9 Mar 2026 09:43:54 +0100 Subject: Fix Mastodon sync and transfer reliability edge cases --- .../mastodonnotificationssyncadaptor.cpp | 92 +++------------- .../mastodonpostssyncadaptor.cpp | 83 +------------- common/common.pro | 1 + common/mastodontextutils.h | 122 +++++++++++++++++++++ .../mastodonshareservicestatus.cpp | 6 +- .../mastodontransferplugin/mastodonapi.cpp | 47 +++----- .../mastodontransferplugin/mastodonapi.h | 2 +- 7 files changed, 165 insertions(+), 188 deletions(-) create mode 100644 common/mastodontextutils.h diff --git a/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp b/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp index 1fd9a7c..05fd6e9 100644 --- a/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp +++ b/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp @@ -20,11 +20,11 @@ #include "mastodonnotificationssyncadaptor.h" #include "trace.h" +#include "mastodontextutils.h" #include #include #include -#include #include #include #include @@ -55,38 +55,6 @@ namespace { const char *const LastFetchedNotificationIdKey = "LastFetchedNotificationId"; const int NotificationsPageLimit = 80; - QString decodeHtmlEntities(QString text) - { - text.replace(QStringLiteral("""), QStringLiteral("\"")); - text.replace(QStringLiteral("'"), QStringLiteral("'")); - text.replace(QStringLiteral("<"), QStringLiteral("<")); - text.replace(QStringLiteral(">"), QStringLiteral(">")); - text.replace(QStringLiteral("&"), QStringLiteral("&")); - text.replace(QStringLiteral(" "), QStringLiteral(" ")); - - static const QRegularExpression decimalEntity(QStringLiteral("&#(\\d+);")); - QRegularExpressionMatch match; - int index = 0; - while ((index = text.indexOf(decimalEntity, index, &match)) != -1) { - const uint value = match.captured(1).toUInt(); - const QString replacement = value > 0 ? QString(QChar(value)) : QString(); - text.replace(index, match.capturedLength(0), replacement); - index += replacement.size(); - } - - static const QRegularExpression hexEntity(QStringLiteral("&#x([0-9a-fA-F]+);")); - index = 0; - while ((index = text.indexOf(hexEntity, index, &match)) != -1) { - bool ok = false; - const uint value = match.captured(1).toUInt(&ok, 16); - const QString replacement = ok && value > 0 ? QString(QChar(value)) : QString(); - text.replace(index, match.capturedLength(0), replacement); - index += replacement.size(); - } - - return text; - } - QString displayNameForAccount(const QJsonObject &account) { const QString displayName = account.value(QStringLiteral("display_name")).toString().trimmed(); @@ -292,52 +260,12 @@ void MastodonNotificationsSyncAdaptor::finalize(int accountId) QString MastodonNotificationsSyncAdaptor::sanitizeContent(const QString &content) { - QString plain = content; - plain.replace(QRegularExpression(QStringLiteral("<\\s*br\\s*/?\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n")); - plain.replace(QRegularExpression(QStringLiteral("<\\s*/\\s*p\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n")); - plain.remove(QRegularExpression(QStringLiteral("<[^>]+>"), QRegularExpression::CaseInsensitiveOption)); - - return decodeHtmlEntities(plain).trimmed(); + return MastodonTextUtils::sanitizeContent(content); } QDateTime MastodonNotificationsSyncAdaptor::parseTimestamp(const QString ×tampString) { - QDateTime timestamp; - -#if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0) - timestamp = QDateTime::fromString(timestampString, Qt::ISODateWithMs); - if (timestamp.isValid()) { - return timestamp; - } -#endif - - timestamp = QDateTime::fromString(timestampString, Qt::ISODate); - if (timestamp.isValid()) { - return timestamp; - } - - const int timeSeparator = timestampString.indexOf(QLatin1Char('T')); - const int fractionSeparator = timestampString.indexOf(QLatin1Char('.'), timeSeparator + 1); - if (timeSeparator > -1 && fractionSeparator > -1) { - int timezoneSeparator = timestampString.indexOf(QLatin1Char('Z'), fractionSeparator + 1); - if (timezoneSeparator == -1) { - timezoneSeparator = timestampString.indexOf(QLatin1Char('+'), fractionSeparator + 1); - } - if (timezoneSeparator == -1) { - timezoneSeparator = timestampString.indexOf(QLatin1Char('-'), fractionSeparator + 1); - } - - QString stripped = timestampString; - if (timezoneSeparator > -1) { - stripped.remove(fractionSeparator, timezoneSeparator - fractionSeparator); - } else { - stripped.truncate(fractionSeparator); - } - - timestamp = QDateTime::fromString(stripped, Qt::ISODate); - } - - return timestamp; + return MastodonTextUtils::parseTimestamp(timestampString); } int MastodonNotificationsSyncAdaptor::compareNotificationIds(const QString &left, const QString &right) @@ -445,6 +373,11 @@ void MastodonNotificationsSyncAdaptor::finishedUnreadMarkerHandler() if (isError || !ok) { qCWarning(lcSocialPlugin) << "unable to parse notifications marker data from request with account" << accountId << ", got:" << QString::fromUtf8(replyData); + PendingSyncState fallbackState; + fallbackState.accessToken = accessToken; + fallbackState.lastFetchedId = loadLastFetchedId(accountId); + m_pendingSyncStates.insert(accountId, fallbackState); + requestNotifications(accountId, accessToken, fallbackState.lastFetchedId); decrementSemaphore(accountId); return; } @@ -671,12 +604,15 @@ void MastodonNotificationsSyncAdaptor::finishedNotificationsHandler() state.pendingNotifications.insert(notificationId, pendingNotification); } + const QString historyBoundaryId = !state.unreadFloorId.isEmpty() + ? state.unreadFloorId + : state.lastFetchedId; if (notifications.size() >= NotificationsPageLimit && !pageMinNotificationId.isEmpty() - && (state.unreadFloorId.isEmpty() - || compareNotificationIds(pageMinNotificationId, state.unreadFloorId) > 0)) { + && !historyBoundaryId.isEmpty() + && compareNotificationIds(pageMinNotificationId, historyBoundaryId) > 0) { m_pendingSyncStates.insert(accountId, state); - requestNotifications(accountId, state.accessToken, state.unreadFloorId, pageMinNotificationId); + requestNotifications(accountId, state.accessToken, historyBoundaryId, pageMinNotificationId); decrementSemaphore(accountId); return; } diff --git a/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp b/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp index deddb0a..160d6cc 100644 --- a/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp +++ b/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp @@ -20,48 +20,16 @@ #include "mastodonpostssyncadaptor.h" #include "trace.h" +#include "mastodontextutils.h" #include #include #include -#include #include #include #include namespace { - QString decodeHtmlEntities(QString text) - { - text.replace(QStringLiteral("""), QStringLiteral("\"")); - text.replace(QStringLiteral("'"), QStringLiteral("'")); - text.replace(QStringLiteral("<"), QStringLiteral("<")); - text.replace(QStringLiteral(">"), QStringLiteral(">")); - text.replace(QStringLiteral("&"), QStringLiteral("&")); - text.replace(QStringLiteral(" "), QStringLiteral(" ")); - - static const QRegularExpression decimalEntity(QStringLiteral("&#(\\d+);")); - QRegularExpressionMatch match; - int index = 0; - while ((index = text.indexOf(decimalEntity, index, &match)) != -1) { - const uint value = match.captured(1).toUInt(); - const QString replacement = value > 0 ? QString(QChar(value)) : QString(); - text.replace(index, match.capturedLength(0), replacement); - index += replacement.size(); - } - - static const QRegularExpression hexEntity(QStringLiteral("&#x([0-9a-fA-F]+);")); - index = 0; - while ((index = text.indexOf(hexEntity, index, &match)) != -1) { - bool ok = false; - const uint value = match.captured(1).toUInt(&ok, 16); - const QString replacement = ok && value > 0 ? QString(QChar(value)) : QString(); - text.replace(index, match.capturedLength(0), replacement); - index += replacement.size(); - } - - return text; - } - QString displayNameForAccount(const QJsonObject &account) { const QString displayName = account.value(QStringLiteral("display_name")).toString().trimmed(); @@ -124,53 +92,12 @@ void MastodonPostsSyncAdaptor::finalize(int accountId) QString MastodonPostsSyncAdaptor::sanitizeContent(const QString &content) { - QString plain = content; - plain.replace(QRegularExpression(QStringLiteral("<\\s*br\\s*/?\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n")); - plain.replace(QRegularExpression(QStringLiteral("<\\s*/\\s*p\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n")); - plain.remove(QRegularExpression(QStringLiteral("<[^>]+>"), QRegularExpression::CaseInsensitiveOption)); - - return decodeHtmlEntities(plain).trimmed(); + return MastodonTextUtils::sanitizeContent(content); } QDateTime MastodonPostsSyncAdaptor::parseTimestamp(const QString ×tampString) { - QDateTime timestamp; - -#if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0) - timestamp = QDateTime::fromString(timestampString, Qt::ISODateWithMs); - if (timestamp.isValid()) { - return timestamp; - } -#endif - - timestamp = QDateTime::fromString(timestampString, Qt::ISODate); - if (timestamp.isValid()) { - return timestamp; - } - - // Qt 5.6 cannot parse ISO-8601 timestamps with fractional seconds. - const int timeSeparator = timestampString.indexOf(QLatin1Char('T')); - const int fractionSeparator = timestampString.indexOf(QLatin1Char('.'), timeSeparator + 1); - if (timeSeparator > -1 && fractionSeparator > -1) { - int timezoneSeparator = timestampString.indexOf(QLatin1Char('Z'), fractionSeparator + 1); - if (timezoneSeparator == -1) { - timezoneSeparator = timestampString.indexOf(QLatin1Char('+'), fractionSeparator + 1); - } - if (timezoneSeparator == -1) { - timezoneSeparator = timestampString.indexOf(QLatin1Char('-'), fractionSeparator + 1); - } - - QString stripped = timestampString; - if (timezoneSeparator > -1) { - stripped.remove(fractionSeparator, timezoneSeparator - fractionSeparator); - } else { - stripped.truncate(fractionSeparator); - } - - timestamp = QDateTime::fromString(stripped, Qt::ISODate); - } - - return timestamp; + return MastodonTextUtils::parseTimestamp(timestampString); } void MastodonPostsSyncAdaptor::requestPosts(int accountId, const QString &accessToken) @@ -216,14 +143,14 @@ void MastodonPostsSyncAdaptor::finishedPostsHandler() bool ok = false; QJsonArray statuses = parseJsonArrayReplyData(replyData, &ok); if (!isError && ok) { + m_db.removePosts(accountId); + if (!statuses.size()) { qCDebug(lcSocialPlugin) << "no feed posts received for account" << accountId; decrementSemaphore(accountId); return; } - m_db.removePosts(accountId); - const int sinceSpan = m_accountSyncProfile ? m_accountSyncProfile->key(Buteo::KEY_SYNC_SINCE_DAYS_PAST, QStringLiteral("7")).toInt() : 7; diff --git a/common/common.pro b/common/common.pro index 02c78ee..136d882 100644 --- a/common/common.pro +++ b/common/common.pro @@ -11,6 +11,7 @@ TARGET = $$qtLibraryTarget($$TARGET) HEADERS += \ $$PWD/mastodonauthutils.h \ + $$PWD/mastodontextutils.h \ $$PWD/mastodonpostsdatabase.h SOURCES += \ diff --git a/common/mastodontextutils.h b/common/mastodontextutils.h new file mode 100644 index 0000000..bde74c4 --- /dev/null +++ b/common/mastodontextutils.h @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2013-2026 Jolla Ltd. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef MASTODONTEXTUTILS_H +#define MASTODONTEXTUTILS_H + +#include +#include +#include + +namespace MastodonTextUtils { + +inline QString decodeHtmlEntities(QString text) +{ + text.replace(QStringLiteral("""), QStringLiteral("\"")); + text.replace(QStringLiteral("'"), QStringLiteral("'")); + text.replace(QStringLiteral("<"), QStringLiteral("<")); + text.replace(QStringLiteral(">"), QStringLiteral(">")); + text.replace(QStringLiteral("&"), QStringLiteral("&")); + text.replace(QStringLiteral(" "), QStringLiteral(" ")); + + static const QRegularExpression decimalEntity(QStringLiteral("&#(\\d+);")); + QRegularExpressionMatch match; + int index = 0; + while ((index = text.indexOf(decimalEntity, index, &match)) != -1) { + bool ok = false; + const uint value = match.captured(1).toUInt(&ok, 10); + QString replacement; + if (ok && value > 0 && value <= 0x10FFFF) { + replacement = QString::fromUcs4(&value, 1); + } + text.replace(index, match.capturedLength(0), replacement); + index += replacement.size(); + } + + static const QRegularExpression hexEntity(QStringLiteral("&#x([0-9a-fA-F]+);")); + index = 0; + while ((index = text.indexOf(hexEntity, index, &match)) != -1) { + bool ok = false; + const uint value = match.captured(1).toUInt(&ok, 16); + QString replacement; + if (ok && value > 0 && value <= 0x10FFFF) { + replacement = QString::fromUcs4(&value, 1); + } + text.replace(index, match.capturedLength(0), replacement); + index += replacement.size(); + } + + return text; +} + +inline QString sanitizeContent(const QString &content) +{ + QString plain = content; + plain.replace(QRegularExpression(QStringLiteral("<\\s*br\\s*/?\\s*>"), QRegularExpression::CaseInsensitiveOption), + QStringLiteral("\n")); + plain.replace(QRegularExpression(QStringLiteral("<\\s*/\\s*p\\s*>"), QRegularExpression::CaseInsensitiveOption), + QStringLiteral("\n")); + plain.remove(QRegularExpression(QStringLiteral("<[^>]+>"), QRegularExpression::CaseInsensitiveOption)); + + return decodeHtmlEntities(plain).trimmed(); +} + +inline QDateTime parseTimestamp(const QString ×tampString) +{ + QDateTime timestamp; + +#if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0) + timestamp = QDateTime::fromString(timestampString, Qt::ISODateWithMs); + if (timestamp.isValid()) { + return timestamp; + } +#endif + + timestamp = QDateTime::fromString(timestampString, Qt::ISODate); + if (timestamp.isValid()) { + return timestamp; + } + + // Qt 5.6 cannot parse ISO-8601 timestamps with fractional seconds. + const int timeSeparator = timestampString.indexOf(QLatin1Char('T')); + const int fractionSeparator = timestampString.indexOf(QLatin1Char('.'), timeSeparator + 1); + if (timeSeparator > -1 && fractionSeparator > -1) { + int timezoneSeparator = timestampString.indexOf(QLatin1Char('Z'), fractionSeparator + 1); + if (timezoneSeparator == -1) { + timezoneSeparator = timestampString.indexOf(QLatin1Char('+'), fractionSeparator + 1); + } + if (timezoneSeparator == -1) { + timezoneSeparator = timestampString.indexOf(QLatin1Char('-'), fractionSeparator + 1); + } + + QString stripped = timestampString; + if (timezoneSeparator > -1) { + stripped.remove(fractionSeparator, timezoneSeparator - fractionSeparator); + } else { + stripped.truncate(fractionSeparator); + } + + timestamp = QDateTime::fromString(stripped, Qt::ISODate); + } + + return timestamp; +} + +} // namespace MastodonTextUtils + +#endif // MASTODONTEXTUTILS_H diff --git a/transferengine-plugins/mastodonshareservicestatus.cpp b/transferengine-plugins/mastodonshareservicestatus.cpp index f3c96ca..3ac05d5 100644 --- a/transferengine-plugins/mastodonshareservicestatus.cpp +++ b/transferengine-plugins/mastodonshareservicestatus.cpp @@ -186,7 +186,7 @@ void MastodonShareServiceStatus::queryStatus(QueryStatusMode mode) bool signInActive = false; Q_FOREACH (Accounts::AccountId id, m_accountManager->accountList()) { - Accounts::Account *acc = m_accountManager->account(id); + Accounts::Account *acc = Accounts::Account::fromId(m_accountManager, id, this); if (!acc) { qWarning() << Q_FUNC_INFO << "Failed to get account for id:" << id; @@ -206,6 +206,7 @@ void MastodonShareServiceStatus::queryStatus(QueryStatusMode mode) } if (!service.isValid() || !serviceFound) { + acc->deleteLater(); continue; } @@ -214,12 +215,14 @@ void MastodonShareServiceStatus::queryStatus(QueryStatusMode mode) const bool shareServiceEnabled = acc->enabled(); if (!accountEnabled || !shareServiceEnabled) { acc->selectService(Accounts::Service()); + acc->deleteLater(); continue; } if (acc->value(QStringLiteral("CredentialsNeedUpdate")).toBool()) { qWarning() << Q_FUNC_INFO << "Credentials need update for account id:" << id; acc->selectService(Accounts::Service()); + acc->deleteLater(); continue; } @@ -256,6 +259,7 @@ void MastodonShareServiceStatus::queryStatus(QueryStatusMode mode) } acc->selectService(Accounts::Service()); + acc->deleteLater(); } if (!signInActive) { diff --git a/transferengine-plugins/mastodontransferplugin/mastodonapi.cpp b/transferengine-plugins/mastodontransferplugin/mastodonapi.cpp index baf9dd8..24bc6f5 100644 --- a/transferengine-plugins/mastodontransferplugin/mastodonapi.cpp +++ b/transferengine-plugins/mastodontransferplugin/mastodonapi.cpp @@ -3,6 +3,7 @@ */ #include "mastodonapi.h" +#include "mastodonauthutils.h" #include #include @@ -18,6 +19,7 @@ MastodonApi::MastodonApi(QNetworkAccessManager *qnam, QObject *parent) : QObject(parent) + , m_cancelRequested(false) , m_qnam(qnam) { } @@ -26,33 +28,6 @@ MastodonApi::~MastodonApi() { } -QString MastodonApi::normalizeApiHost(const QString &rawHost) -{ - QString host = rawHost.trimmed(); - if (host.isEmpty()) { - host = QStringLiteral("https://mastodon.social"); - } - - if (!host.startsWith(QLatin1String("https://")) - && !host.startsWith(QLatin1String("http://"))) { - host.prepend(QStringLiteral("https://")); - } - - QUrl url(host); - if (!url.isValid() || url.host().isEmpty()) { - return QStringLiteral("https://mastodon.social"); - } - - QString normalized = QString::fromLatin1(url.toEncoded(QUrl::RemovePath - | QUrl::RemoveQuery - | QUrl::RemoveFragment)); - if (normalized.endsWith(QLatin1Char('/'))) { - normalized.chop(1); - } - - return normalized; -} - bool MastodonApi::uploadImage(const QString &filePath, const QString &statusText, const QString &mimeType, @@ -65,7 +40,8 @@ bool MastodonApi::uploadImage(const QString &filePath, return false; } - m_apiHost = normalizeApiHost(apiHost); + m_cancelRequested = false; + m_apiHost = MastodonAuthUtils::normalizeApiHost(apiHost); m_accessToken = accessToken; m_statusText = statusText; @@ -116,7 +92,8 @@ bool MastodonApi::postStatus(const QString &statusText, const QString &apiHost, const QString &accessToken) { - m_apiHost = normalizeApiHost(apiHost); + m_cancelRequested = false; + m_apiHost = MastodonAuthUtils::normalizeApiHost(apiHost); m_accessToken = accessToken; m_statusText = statusText; @@ -172,11 +149,11 @@ void MastodonApi::cancelUpload() return; } + m_cancelRequested = true; const QList replies = m_replies.keys(); Q_FOREACH (QNetworkReply *reply, replies) { reply->abort(); } - m_replies.clear(); } void MastodonApi::replyError(QNetworkReply::NetworkError error) @@ -205,6 +182,14 @@ void MastodonApi::finished() reply->deleteLater(); + if (m_cancelRequested && error == QNetworkReply::OperationCanceledError) { + if (m_replies.isEmpty()) { + m_cancelRequested = false; + emit transferCanceled(); + } + return; + } + if (apiCall == UPLOAD_MEDIA) { if (error != QNetworkReply::NoError || httpCode < 200 || httpCode >= 300) { finishTransfer(error == QNetworkReply::NoError ? QNetworkReply::UnknownNetworkError : error, @@ -241,6 +226,8 @@ void MastodonApi::finished() void MastodonApi::finishTransfer(QNetworkReply::NetworkError error, int httpCode, const QByteArray &data) { + m_cancelRequested = false; + if (httpCode == 401) { emit credentialsExpired(); } diff --git a/transferengine-plugins/mastodontransferplugin/mastodonapi.h b/transferengine-plugins/mastodontransferplugin/mastodonapi.h index 4ac3d80..e24914d 100644 --- a/transferengine-plugins/mastodontransferplugin/mastodonapi.h +++ b/transferengine-plugins/mastodontransferplugin/mastodonapi.h @@ -49,11 +49,11 @@ private Q_SLOTS: void uploadProgress(qint64 received, qint64 total); private: - static QString normalizeApiHost(const QString &rawHost); bool postStatusInternal(const QString &mediaId); void finishTransfer(QNetworkReply::NetworkError error, int httpCode, const QByteArray &data); QMap m_replies; + bool m_cancelRequested; QNetworkAccessManager *m_qnam; QString m_accessToken; QString m_apiHost; -- cgit v1.2.3