Fix Mastodon sync and transfer reliability edge cases

2 files changed, 19 insertions, 156 deletions

diff --git a/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp b/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp
index 1fd9a7c..05fd6e9 100644
--- a/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp
+++ b/buteo-plugins/buteo-sync-plugin-mastodon-notifications/mastodonnotificationssyncadaptor.cpp
@@ -20,11 +20,11 @@
 
 #include "mastodonnotificationssyncadaptor.h"
 #include "trace.h"
+#include "mastodontextutils.h"
 
 #include <QtCore/QJsonArray>
 #include <QtCore/QJsonObject>
 #include <QtCore/QJsonValue>
-#include <QtCore/QRegularExpression>
 #include <QtCore/QUrl>
 #include <QtCore/QUrlQuery>
 #include <QtNetwork/QNetworkRequest>
@@ -55,38 +55,6 @@ namespace {
     const char *const LastFetchedNotificationIdKey = "LastFetchedNotificationId";
     const int NotificationsPageLimit = 80;
 
-    QString decodeHtmlEntities(QString text)
-    {
-        text.replace(QStringLiteral("&quot;"), QStringLiteral("\""));
-        text.replace(QStringLiteral("&apos;"), QStringLiteral("'"));
-        text.replace(QStringLiteral("&lt;"), QStringLiteral("<"));
-        text.replace(QStringLiteral("&gt;"), QStringLiteral(">"));
-        text.replace(QStringLiteral("&amp;"), QStringLiteral("&"));
-        text.replace(QStringLiteral("&nbsp;"), QStringLiteral(" "));
-
-        static const QRegularExpression decimalEntity(QStringLiteral("&#(\\d+);"));
-        QRegularExpressionMatch match;
-        int index = 0;
-        while ((index = text.indexOf(decimalEntity, index, &match)) != -1) {
-            const uint value = match.captured(1).toUInt();
-            const QString replacement = value > 0 ? QString(QChar(value)) : QString();
-            text.replace(index, match.capturedLength(0), replacement);
-            index += replacement.size();
-        }
-
-        static const QRegularExpression hexEntity(QStringLiteral("&#x([0-9a-fA-F]+);"));
-        index = 0;
-        while ((index = text.indexOf(hexEntity, index, &match)) != -1) {
-            bool ok = false;
-            const uint value = match.captured(1).toUInt(&ok, 16);
-            const QString replacement = ok && value > 0 ? QString(QChar(value)) : QString();
-            text.replace(index, match.capturedLength(0), replacement);
-            index += replacement.size();
-        }
-
-        return text;
-    }
-
     QString displayNameForAccount(const QJsonObject &account)
     {
         const QString displayName = account.value(QStringLiteral("display_name")).toString().trimmed();
@@ -292,52 +260,12 @@ void MastodonNotificationsSyncAdaptor::finalize(int accountId)
 
 QString MastodonNotificationsSyncAdaptor::sanitizeContent(const QString &content)
 {
-    QString plain = content;
-    plain.replace(QRegularExpression(QStringLiteral("<\\s*br\\s*/?\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n"));
-    plain.replace(QRegularExpression(QStringLiteral("<\\s*/\\s*p\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n"));
-    plain.remove(QRegularExpression(QStringLiteral("<[^>]+>"), QRegularExpression::CaseInsensitiveOption));
-
-    return decodeHtmlEntities(plain).trimmed();
+    return MastodonTextUtils::sanitizeContent(content);
 }
 
 QDateTime MastodonNotificationsSyncAdaptor::parseTimestamp(const QString &timestampString)
 {
-    QDateTime timestamp;
-
-#if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
-    timestamp = QDateTime::fromString(timestampString, Qt::ISODateWithMs);
-    if (timestamp.isValid()) {
-        return timestamp;
-    }
-#endif
-
-    timestamp = QDateTime::fromString(timestampString, Qt::ISODate);
-    if (timestamp.isValid()) {
-        return timestamp;
-    }
-
-    const int timeSeparator = timestampString.indexOf(QLatin1Char('T'));
-    const int fractionSeparator = timestampString.indexOf(QLatin1Char('.'), timeSeparator + 1);
-    if (timeSeparator > -1 && fractionSeparator > -1) {
-        int timezoneSeparator = timestampString.indexOf(QLatin1Char('Z'), fractionSeparator + 1);
-        if (timezoneSeparator == -1) {
-            timezoneSeparator = timestampString.indexOf(QLatin1Char('+'), fractionSeparator + 1);
-        }
-        if (timezoneSeparator == -1) {
-            timezoneSeparator = timestampString.indexOf(QLatin1Char('-'), fractionSeparator + 1);
-        }
-
-        QString stripped = timestampString;
-        if (timezoneSeparator > -1) {
-            stripped.remove(fractionSeparator, timezoneSeparator - fractionSeparator);
-        } else {
-            stripped.truncate(fractionSeparator);
-        }
-
-        timestamp = QDateTime::fromString(stripped, Qt::ISODate);
-    }
-
-    return timestamp;
+    return MastodonTextUtils::parseTimestamp(timestampString);
 }
 
 int MastodonNotificationsSyncAdaptor::compareNotificationIds(const QString &left, const QString &right)
@@ -445,6 +373,11 @@ void MastodonNotificationsSyncAdaptor::finishedUnreadMarkerHandler()
     if (isError || !ok) {
         qCWarning(lcSocialPlugin) << "unable to parse notifications marker data from request with account"
                                   << accountId << ", got:" << QString::fromUtf8(replyData);
+        PendingSyncState fallbackState;
+        fallbackState.accessToken = accessToken;
+        fallbackState.lastFetchedId = loadLastFetchedId(accountId);
+        m_pendingSyncStates.insert(accountId, fallbackState);
+        requestNotifications(accountId, accessToken, fallbackState.lastFetchedId);
         decrementSemaphore(accountId);
         return;
     }
@@ -671,12 +604,15 @@ void MastodonNotificationsSyncAdaptor::finishedNotificationsHandler()
             state.pendingNotifications.insert(notificationId, pendingNotification);
         }
 
+        const QString historyBoundaryId = !state.unreadFloorId.isEmpty()
+                ? state.unreadFloorId
+                : state.lastFetchedId;
         if (notifications.size() >= NotificationsPageLimit
                 && !pageMinNotificationId.isEmpty()
-                && (state.unreadFloorId.isEmpty()
-                    || compareNotificationIds(pageMinNotificationId, state.unreadFloorId) > 0)) {
+                && !historyBoundaryId.isEmpty()
+                && compareNotificationIds(pageMinNotificationId, historyBoundaryId) > 0) {
             m_pendingSyncStates.insert(accountId, state);
-            requestNotifications(accountId, state.accessToken, state.unreadFloorId, pageMinNotificationId);
+            requestNotifications(accountId, state.accessToken, historyBoundaryId, pageMinNotificationId);
             decrementSemaphore(accountId);
             return;
         }
diff --git a/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp b/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp
index deddb0a..160d6cc 100644
--- a/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp
+++ b/buteo-plugins/buteo-sync-plugin-mastodon-posts/mastodonpostssyncadaptor.cpp
@@ -20,48 +20,16 @@
 
 #include "mastodonpostssyncadaptor.h"
 #include "trace.h"
+#include "mastodontextutils.h"
 
 #include <QtCore/QJsonArray>
 #include <QtCore/QJsonObject>
 #include <QtCore/QJsonValue>
-#include <QtCore/QRegularExpression>
 #include <QtCore/QUrl>
 #include <QtCore/QUrlQuery>
 #include <QtNetwork/QNetworkRequest>
 
 namespace {
-    QString decodeHtmlEntities(QString text)
-    {
-        text.replace(QStringLiteral("&quot;"), QStringLiteral("\""));
-        text.replace(QStringLiteral("&apos;"), QStringLiteral("'"));
-        text.replace(QStringLiteral("&lt;"), QStringLiteral("<"));
-        text.replace(QStringLiteral("&gt;"), QStringLiteral(">"));
-        text.replace(QStringLiteral("&amp;"), QStringLiteral("&"));
-        text.replace(QStringLiteral("&nbsp;"), QStringLiteral(" "));
-
-        static const QRegularExpression decimalEntity(QStringLiteral("&#(\\d+);"));
-        QRegularExpressionMatch match;
-        int index = 0;
-        while ((index = text.indexOf(decimalEntity, index, &match)) != -1) {
-            const uint value = match.captured(1).toUInt();
-            const QString replacement = value > 0 ? QString(QChar(value)) : QString();
-            text.replace(index, match.capturedLength(0), replacement);
-            index += replacement.size();
-        }
-
-        static const QRegularExpression hexEntity(QStringLiteral("&#x([0-9a-fA-F]+);"));
-        index = 0;
-        while ((index = text.indexOf(hexEntity, index, &match)) != -1) {
-            bool ok = false;
-            const uint value = match.captured(1).toUInt(&ok, 16);
-            const QString replacement = ok && value > 0 ? QString(QChar(value)) : QString();
-            text.replace(index, match.capturedLength(0), replacement);
-            index += replacement.size();
-        }
-
-        return text;
-    }
-
     QString displayNameForAccount(const QJsonObject &account)
     {
         const QString displayName = account.value(QStringLiteral("display_name")).toString().trimmed();
@@ -124,53 +92,12 @@ void MastodonPostsSyncAdaptor::finalize(int accountId)
 
 QString MastodonPostsSyncAdaptor::sanitizeContent(const QString &content)
 {
-    QString plain = content;
-    plain.replace(QRegularExpression(QStringLiteral("<\\s*br\\s*/?\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n"));
-    plain.replace(QRegularExpression(QStringLiteral("<\\s*/\\s*p\\s*>"), QRegularExpression::CaseInsensitiveOption), QStringLiteral("\n"));
-    plain.remove(QRegularExpression(QStringLiteral("<[^>]+>"), QRegularExpression::CaseInsensitiveOption));
-
-    return decodeHtmlEntities(plain).trimmed();
+    return MastodonTextUtils::sanitizeContent(content);
 }
 
 QDateTime MastodonPostsSyncAdaptor::parseTimestamp(const QString &timestampString)
 {
-    QDateTime timestamp;
-
-#if QT_VERSION >= QT_VERSION_CHECK(5, 8, 0)
-    timestamp = QDateTime::fromString(timestampString, Qt::ISODateWithMs);
-    if (timestamp.isValid()) {
-        return timestamp;
-    }
-#endif
-
-    timestamp = QDateTime::fromString(timestampString, Qt::ISODate);
-    if (timestamp.isValid()) {
-        return timestamp;
-    }
-
-    // Qt 5.6 cannot parse ISO-8601 timestamps with fractional seconds.
-    const int timeSeparator = timestampString.indexOf(QLatin1Char('T'));
-    const int fractionSeparator = timestampString.indexOf(QLatin1Char('.'), timeSeparator + 1);
-    if (timeSeparator > -1 && fractionSeparator > -1) {
-        int timezoneSeparator = timestampString.indexOf(QLatin1Char('Z'), fractionSeparator + 1);
-        if (timezoneSeparator == -1) {
-            timezoneSeparator = timestampString.indexOf(QLatin1Char('+'), fractionSeparator + 1);
-        }
-        if (timezoneSeparator == -1) {
-            timezoneSeparator = timestampString.indexOf(QLatin1Char('-'), fractionSeparator + 1);
-        }
-
-        QString stripped = timestampString;
-        if (timezoneSeparator > -1) {
-            stripped.remove(fractionSeparator, timezoneSeparator - fractionSeparator);
-        } else {
-            stripped.truncate(fractionSeparator);
-        }
-
-        timestamp = QDateTime::fromString(stripped, Qt::ISODate);
-    }
-
-    return timestamp;
+    return MastodonTextUtils::parseTimestamp(timestampString);
 }
 
 void MastodonPostsSyncAdaptor::requestPosts(int accountId, const QString &accessToken)
@@ -216,14 +143,14 @@ void MastodonPostsSyncAdaptor::finishedPostsHandler()
     bool ok = false;
     QJsonArray statuses = parseJsonArrayReplyData(replyData, &ok);
     if (!isError && ok) {
+        m_db.removePosts(accountId);
+
         if (!statuses.size()) {
             qCDebug(lcSocialPlugin) << "no feed posts received for account" << accountId;
             decrementSemaphore(accountId);
             return;
         }
 
-        m_db.removePosts(accountId);
-
         const int sinceSpan = m_accountSyncProfile
                 ? m_accountSyncProfile->key(Buteo::KEY_SYNC_SINCE_DAYS_PAST, QStringLiteral("7")).toInt()
                 : 7;