fix: use caching for notifications and improve fetching

2026-07-24 15:07:18 -03:00
parent 472de21ea6
commit ba4ccb066f
13 changed files with 250 additions and 81 deletions
@@ -4,5 +4,4 @@ APP_CONSUMER_KEY=app-consumer-key
 APP_CONSUMER_KEY_SECRET=app-consumer-key-secret
 APP_ACCESS_TOKEN=app-access-token
 APP_ACCESS_TOKEN_SECRET=app-access-token-secret
-APP_USER_ID=your-x-user-id
 NOTIFICATION_DB_PATH=/data/notifications.db
@@ -113,7 +113,6 @@ Backend requires all of these at runtime:
 | `APP_ACCESS_TOKEN_SECRET` | X API user access token secret |
 | `REDIS_URL` | Redis URI, e.g. `redis://localhost:6379` or `redis://twitter_redis:6379` |
 | `CORS_ALLOWED_ORIGIN` | Allowed CORS origin, e.g. `http://localhost:5173` |
-| `APP_USER_ID` | Numeric X user id for notification polling |
 | `NOTIFICATION_DB_PATH` | SQLite file path, e.g. `/data/notifications.db` |

 Frontend build-time:
@@ -56,9 +56,9 @@ fun main() {
            accessTokenSecret = requireNotNull(System.getenv("APP_ACCESS_TOKEN_SECRET")) { "APP_ACCESS_TOKEN_SECRET not set" },
        )
        val notificationPoller = NotificationPoller(
-            userId = requireNotNull(System.getenv("APP_USER_ID")) { "APP_USER_ID not set" },
            apiClient = notificationApiClient,
            repository = notificationRepository,
+            userIdProvider = { xApi.resolveUserId() },
        )

        routing {
@@ -69,7 +69,7 @@ fun main() {
            threadRoutes(xApi)
            tweetRoutes(cache, xApi)
            userRoutes(xApi)
-            notificationRoutes(notificationRepository, notificationPoller)
+            notificationRoutes(cache, notificationRepository, notificationPoller)
        }
    }.start(wait = true)
 }
@@ -367,7 +367,7 @@ class XApiClient(
        return buildJsonObject { put("data", JsonArray(enriched)) }.toString()
    }

-    private suspend fun resolveUserId(): String {
+    suspend fun resolveUserId(): String {
        cachedUserId?.let { return it }
        cache.get(USER_ID_CACHE_KEY)?.let {
            cachedUserId = it
@@ -26,7 +26,10 @@ class DbConfig(private val dbPath: String) {
            ?: throw IllegalStateException("schema.sql not found on classpath")
        dataSource.connection.use { conn ->
            conn.createStatement().use { statement ->
-                statement.execute(schema)
+                schema.splitToSequence(";")
+                    .map { it.trim() }
+                    .filter { it.isNotEmpty() }
+                    .forEach { sql -> statement.execute(sql) }
            }
        }
    }
@@ -191,11 +191,23 @@ class NotificationRepository(private val dataSource: DataSource) {
        }
    }

-    fun getTrackedTweets(limit: Int = 50): List<TrackedTweet> {
+    fun getTrackedTweets(
+        limit: Int = 10,
+        minMinutesSinceLastCheck: Int = 15,
+        maxSinceLastCheckHours: Int = 48,
+    ): List<TrackedTweet> {
        val tweets = mutableListOf<TrackedTweet>()
        dataSource.connection.use { conn ->
            conn.prepareStatement(
-                "SELECT tweet_id, text, created_at, last_checked_at FROM tracked_tweets ORDER BY created_at DESC LIMIT ?"
+                """
+                SELECT tweet_id, text, created_at, last_checked_at
+                FROM tracked_tweets
+                WHERE (last_checked_at IS NULL
+                   OR last_checked_at <= datetime('now', '-$minMinutesSinceLastCheck minutes'))
+                  AND created_at >= datetime('now', '-$maxSinceLastCheckHours hours')
+                ORDER BY created_at DESC
+                LIMIT ?
+                """.trimIndent()
            ).use { stmt ->
                stmt.setInt(1, limit)
                stmt.executeQuery().use { rs ->
@@ -293,6 +305,16 @@ class NotificationRepository(private val dataSource: DataSource) {
        }
    }

+    fun getUnreadNotificationCount(): Long {
+        return dataSource.connection.use { conn ->
+            conn.prepareStatement("SELECT COUNT(*) FROM notifications WHERE read_at IS NULL").use { stmt ->
+                stmt.executeQuery().use { rs ->
+                    if (rs.next()) rs.getLong(1) else 0
+                }
+            }
+        }
+    }
+
    fun markRead(id: Long): Boolean {
        return dataSource.connection.use { conn ->
            conn.prepareStatement(
@@ -18,6 +18,8 @@ import kotlinx.serialization.json.jsonObject
 import kotlinx.serialization.json.jsonPrimitive
 import mu.KotlinLogging

+class RateLimitedException(message: String) : Exception(message)
+
@Serializable
 data class XUser(
    val id: String,
@@ -95,6 +97,12 @@ class NotificationApiClient(
            params.forEach { (k, v) -> parameter(k, v) }
            header("Authorization", authHeader)
        }
+        if (response.status.value == 429) {
+            val body = response.bodyAsText()
+            val reset = response.headers["x-rate-limit-reset"]
+            logger.warn { "Rate limited on liking_users for tweet $tweetId; reset=$reset body=$body" }
+            throw RateLimitedException("Rate limited on liking_users: $body")
+        }
        if (!response.status.isSuccess()) {
            throw Exception("X API error fetching liking users: ${response.status} ${response.bodyAsText()}")
        }
@@ -8,21 +8,22 @@ import kotlinx.serialization.json.jsonPrimitive
 import mu.KotlinLogging

 class NotificationPoller(
-    private val userId: String,
    private val apiClient: NotificationApiClient,
    private val repository: NotificationRepository,
+    private val userIdProvider: suspend () -> String,
 ) {
    private val logger = KotlinLogging.logger {}
    private val json = Json { ignoreUnknownKeys = true }

    suspend fun pollAll() {
+        val userId = userIdProvider()
        try {
-            pollFollowers()
+            pollFollowers(userId)
        } catch (e: Exception) {
            logger.error(e) { "Failed to poll followers" }
        }
        try {
-            pollMentions()
+            pollMentions(userId)
        } catch (e: Exception) {
            logger.error(e) { "Failed to poll mentions" }
        }
@@ -33,7 +34,7 @@ class NotificationPoller(
        }
    }

-    suspend fun pollFollowers() {
+    suspend fun pollFollowers(userId: String) {
        val followers = apiClient.fetchAllFollowers(userId)
        repository.upsertUsers(followers)
        val currentIds = followers.map { it.id }.toSet()
@@ -45,7 +46,7 @@ class NotificationPoller(
        logger.info { "Followers polled: ${followers.size} total, ${newFollowers.size} new" }
    }

-    suspend fun pollMentions() {
+    suspend fun pollMentions(userId: String) {
        val sinceId = repository.getSyncCursor("mentions")
        val (mentions, _) = apiClient.getMentions(userId, sinceId)
        mentions.forEach { tweet ->
@@ -62,18 +63,17 @@ class NotificationPoller(
    }

    suspend fun pollLikes() {
-        val trackedTweets = repository.getTrackedTweets()
+        val existingUnreadCount = repository.getUnreadNotificationCount()
+        val trackedTweetsLimit = if (existingUnreadCount == 0L) 50 else 5
+        val trackedTweets = repository.getTrackedTweets(limit = trackedTweetsLimit)
+        if (trackedTweets.isEmpty()) {
+            logger.info { "No tracked tweets eligible for likes polling" }
+            return
+        }
+        logger.info { "Polling likes for ${trackedTweets.size} tracked tweets" }
        trackedTweets.forEach { tweet ->
            try {
-                val likers = apiClient.fetchAllPages { token -> apiClient.getLikingUsers(tweet.tweetId, token) }
-                repository.upsertUsers(likers)
-                val existingLikers = repository.getLikersForTweet(tweet.tweetId)
-                val newLikers = likers.map { it.id }.toSet() - existingLikers
-                newLikers.forEach { likerId ->
-                    repository.upsertTweetLiker(tweet.tweetId, likerId)
-                    repository.recordLikeNotification(likerId, tweet.tweetId)
-                }
-                repository.updateTrackedTweetCheckedAt(tweet.tweetId)
+                pollLikesForTweet(tweet.tweetId)
            } catch (e: Exception) {
                logger.error(e) { "Failed to poll likes for tweet ${tweet.tweetId}" }
            }
@@ -81,6 +81,18 @@ class NotificationPoller(
        repository.setSyncCursor("likes", java.time.Instant.now().toString())
    }

+    private suspend fun pollLikesForTweet(tweetId: String) {
+        val likers = apiClient.fetchAllPages { token -> apiClient.getLikingUsers(tweetId, token) }
+        repository.upsertUsers(likers)
+        val existingLikers = repository.getLikersForTweet(tweetId)
+        val newLikers = likers.map { it.id }.toSet() - existingLikers
+        newLikers.forEach { likerId ->
+            repository.upsertTweetLiker(tweetId, likerId)
+            repository.recordLikeNotification(likerId, tweetId)
+        }
+        repository.updateTrackedTweetCheckedAt(tweetId)
+    }
+
    fun seedTrackedTweetsFromTimeline(timelineJson: String) {
        val root = json.parseToJsonElement(timelineJson).jsonObject
        val tweets = root["data"]?.jsonArray ?: return
@@ -1,48 +1,108 @@
 package com.rmcampos.twitterclient.routes

+import com.rmcampos.twitterclient.cache.CacheService
 import com.rmcampos.twitterclient.db.NotificationRepository
 import com.rmcampos.twitterclient.notifications.NotificationPoller
+import io.ktor.http.ContentType
 import io.ktor.http.HttpStatusCode
 import io.ktor.server.application.call
-import io.ktor.server.response.respond
+import io.ktor.server.response.respondText
 import io.ktor.server.routing.Route
 import io.ktor.server.routing.get
 import io.ktor.server.routing.post
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.coroutines.withContext
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json

-fun Route.notificationRoutes(repository: NotificationRepository, poller: NotificationPoller) {
+private const val NOTIFICATIONS_CACHE_KEY = "notifications:unread"
+private const val NOTIFICATIONS_CACHE_TTL_SECONDS = 300L
+
+@Serializable
+private data class NotificationSubjectResponse(
+    val id: String,
+    val name: String,
+    val username: String,
+    val profile_image_url: String?,
+)
+
+@Serializable
+private data class NotificationResponse(
+    val id: Long,
+    val type: String,
+    val created_at: String,
+    val subject: NotificationSubjectResponse,
+    val ref_tweet_id: String?,
+)
+
+@Serializable
+private data class NotificationsEnvelope(
+    val notifications: List<NotificationResponse>,
+    val unread_count: Int,
+)
+
+fun Route.notificationRoutes(cache: CacheService, repository: NotificationRepository, poller: NotificationPoller) {
    get("/api/notifications") {
-        poller.pollAll()
-        val notifications = repository.getUnreadNotifications()
-        val response = notifications.map {
-            mapOf(
-                "id" to it.id,
-                "type" to it.type,
-                "created_at" to it.createdAt,
-                "subject" to mapOf(
-                    "id" to it.subjectId,
-                    "name" to (it.subjectName ?: ""),
-                    "username" to (it.subjectUsername ?: ""),
-                    "profile_image_url" to it.subjectImageUrl,
-                ),
-                "ref_tweet_id" to it.refTweetId,
+        val cachedJson = cache.get(NOTIFICATIONS_CACHE_KEY)
+        if (cachedJson != null) {
+            launch(Dispatchers.IO) {
+                try {
+                    poller.pollAll()
+                } catch (e: Exception) {
+                    // Background refresh failure is logged by the poller; keep serving cached data.
+                }
+            }
+            call.respondText(cachedJson, ContentType.Application.Json, HttpStatusCode.OK)
+        } else {
+            poller.pollAll()
+            val notifications = repository.getUnreadNotifications()
+            val response = NotificationsEnvelope(
+                notifications = notifications.map {
+                    NotificationResponse(
+                        id = it.id,
+                        type = it.type,
+                        created_at = it.createdAt,
+                        subject = NotificationSubjectResponse(
+                            id = it.subjectId,
+                            name = it.subjectName ?: "",
+                            username = it.subjectUsername ?: "",
+                            profile_image_url = it.subjectImageUrl,
+                        ),
+                        ref_tweet_id = it.refTweetId,
+                    )
+                },
+                unread_count = notifications.size,
            )
+            val json = Json.encodeToString(response)
+            cache.set(NOTIFICATIONS_CACHE_KEY, json, NOTIFICATIONS_CACHE_TTL_SECONDS)
+            call.respondText(json, ContentType.Application.Json, HttpStatusCode.OK)
        }
-        call.respond(HttpStatusCode.OK, mapOf("notifications" to response, "unread_count" to notifications.size))
    }

    post("/api/notifications/{id}/read") {
        val id = call.parameters["id"]?.toLongOrNull()
        if (id == null) {
-            call.respond(HttpStatusCode.BadRequest, mapOf("error" to "invalid notification id"))
+            call.respondText(
+                Json.encodeToString(mapOf("error" to "invalid notification id")),
+                ContentType.Application.Json,
+                HttpStatusCode.BadRequest,
+            )
            return@post
        }
        val updated = withContext(Dispatchers.IO) { repository.markRead(id) }
        if (updated) {
-            call.respond(HttpStatusCode.NoContent)
+            cache.delete(NOTIFICATIONS_CACHE_KEY)
+        }
+        if (updated) {
+            call.respondText("", ContentType.Application.Json, HttpStatusCode.NoContent)
        } else {
-            call.respond(HttpStatusCode.NotFound, mapOf("error" to "notification not found or already read"))
+            call.respondText(
+                Json.encodeToString(mapOf("error" to "notification not found or already read")),
+                ContentType.Application.Json,
+                HttpStatusCode.NotFound,
+            )
        }
    }
 }
@@ -14,7 +14,6 @@ services:
      - APP_ACCESS_TOKEN_SECRET=${APP_ACCESS_TOKEN_SECRET}
      - APP_PASSWORD=${APP_PASSWORD}
      - APP_SECRET=${APP_SECRET}
-      - APP_USER_ID=${APP_USER_ID}
      - REDIS_URL=redis://twitter_redis:6379
      - CORS_ALLOWED_ORIGIN=http://localhost:5173
      - NOTIFICATION_DB_PATH=/data/notifications.db
@@ -1,28 +1,37 @@
 .notifications-tab {
-    padding: 16px;
+    padding: 0;
    text-align: left;
 }

 .notifications-title {
-    margin: 0 0 16px 0;
+    margin: 0;
+    padding: 16px;
    font-size: 20px;
    font-weight: 700;
+    line-height: 24px;
    display: flex;
    align-items: center;
    gap: 8px;
+    border-bottom: 1px solid var(--color-border);
+    position: sticky;
+    top: 0;
+    background: var(--color-bg);
+    z-index: 1;
 }

 .unread-badge {
-    background: #ef4444;
+    background: var(--color-accent);
    color: #fff;
    border-radius: 999px;
    padding: 2px 8px;
    font-size: 12px;
+    font-weight: 700;
    line-height: 1;
 }

 .notifications-placeholder {
    margin: 0;
+    padding: 32px 16px;
    color: var(--color-text-secondary);
    font-size: 15px;
    line-height: 20px;
@@ -30,7 +39,8 @@
 }

 .notifications-error {
-    color: #ef4444;
+    padding: 12px 16px;
+    color: var(--color-error);
    text-align: center;
 }

@@ -40,32 +50,79 @@
    padding: 0;
    display: flex;
    flex-direction: column;
-    gap: 8px;
 }

 .notification-item {
    display: flex;
-    align-items: center;
-    justify-content: space-between;
-    padding: 12px;
-    border: 1px solid var(--color-border, #e5e7eb);
-    border-radius: 8px;
-    background: var(--color-surface, #fff);
+    align-items: flex-start;
+    gap: 12px;
+    padding: 12px 16px;
+    border-bottom: 1px solid var(--color-border);
+    background: var(--color-bg);
+    transition: background-color 0.1s ease;
 }

-.notification-subject {
+.notification-item:hover {
+    background: var(--color-hover-overlay);
+}
+
+.notification-avatar {
+    width: 40px;
+    height: 40px;
+    border-radius: 9999px;
+    object-fit: cover;
+    flex-shrink: 0;
+    background: var(--color-border);
+}
+
+.notification-body {
+    flex: 1;
+    min-width: 0;
+    display: flex;
+    flex-direction: column;
+    gap: 4px;
+}
+
+.notification-text {
+    margin: 0;
+    font-size: 15px;
+    line-height: 20px;
+    color: var(--color-text-primary);
+    word-wrap: break-word;
+}
+
+.notification-text a {
+    color: var(--color-text-primary);
    font-weight: 700;
 }

+.notification-text a:hover {
+    text-decoration: underline;
+}
+
+.notification-meta {
+    font-size: 13px;
+    color: var(--color-text-secondary);
+    line-height: 16px;
+}
+
 .notification-read-btn {
-    padding: 4px 12px;
-    border: 1px solid var(--color-border, #e5e7eb);
-    border-radius: 999px;
+    margin-left: auto;
+    padding: 6px 14px;
+    border: 1px solid var(--color-border);
+    border-radius: 9999px;
    background: transparent;
+    color: var(--color-text-secondary);
    cursor: pointer;
    font-size: 13px;
+    font-weight: 700;
+    line-height: 16px;
+    flex-shrink: 0;
+    transition: background-color 0.1s ease, color 0.1s ease;
 }

 .notification-read-btn:hover {
-    background: var(--color-hover, #f3f4f6);
+    background: rgba(29, 155, 240, 0.1);
+    color: var(--color-accent);
+    border-color: var(--color-accent);
 }
@@ -50,6 +50,24 @@ export function NotificationsTab() {
        }
    };

+    const formatTime = (createdAt: string) => {
+        const date = new Date(createdAt);
+        return date.toLocaleString(undefined, {
+            month: 'short',
+            day: 'numeric',
+            hour: '2-digit',
+            minute: '2-digit',
+        });
+    };
+
+    const renderAvatar = (n: Notification) => {
+        const alt = n.subject.name || n.subject.username || 'subject';
+        if (n.subject.profile_image_url) {
+            return <img src={n.subject.profile_image_url} alt={alt} className="notification-avatar" />;
+        }
+        return <div className="notification-avatar" aria-hidden="true" />;
+    };
+
    return (
        <section className="notifications-tab" aria-label="Notifications">
            <h2 className="notifications-title">
@@ -63,11 +81,19 @@ export function NotificationsTab() {
            <ul className="notifications-list">
                {notifications.map((n) => (
                    <li key={n.id} className="notification-item">
-                        <div className="notification-content">
-                            <span className="notification-subject">
-                                {n.subject.name || `@${n.subject.username}`}
-                            </span>{' '}
-                            {formatMessage(n)}
+                        {renderAvatar(n)}
+                        <div className="notification-body">
+                            <p className="notification-text">
+                                <a
+                                    href={`https://x.com/${n.subject.username}`}
+                                    target="_blank"
+                                    rel="noopener noreferrer"
+                                >
+                                    {n.subject.name || `@${n.subject.username}`}
+                                </a>{' '}
+                                {formatMessage(n)}
+                            </p>
+                            <span className="notification-meta">{formatTime(n.created_at)}</span>
                        </div>
                        <button
                            className="notification-read-btn"
@@ -53,12 +53,6 @@ variable "app_secret" {
  sensitive = true
 }

-variable "app_user_id" {
-  type      = string
-  sensitive = false
-  default   = ""
-}
-
 variable "cors_allowed_origin" {
  type    = string
  default = "https://rmcampos-twc.vercel.app"
@@ -93,7 +87,6 @@ resource "kubernetes_secret_v1" "twitter_client_secrets" {
    app_access_token_secret = var.app_access_token_secret
    app_password            = var.app_password
    app_secret              = var.app_secret
-    app_user_id             = var.app_user_id
  }
 }

@@ -277,15 +270,6 @@ resource "kubernetes_deployment_v1" "twitter_client_backend" {
              }
            }
          }
-          env {
-            name = "APP_USER_ID"
-            value_from {
-              secret_key_ref {
-                name = kubernetes_secret_v1.twitter_client_secrets.metadata[0].name
-                key  = "app_user_id"
-              }
-            }
-          }
          env {
            name  = "NOTIFICATION_DB_PATH"
            value = "/data/notifications.db"