Phase 5: Redis pub/sub for horizontal scaling

Splits the WebSocket layer into three pieces so one app instance and many
behave identically: ConnectionManager stays a purely local socket registry;
RoomBroadcaster publishes chat messages to a per-room Redis channel and
every instance (including the publisher) forwards received messages to its
own local sockets via a single psubscribe("room:*") listener started in
main.py's lifespan; Presence is a Redis-backed refcounted hash per room
tracking who's connected across all instances.

Presence replaces the old process-local connected_user_ids check that
Phase 4's offline-push logic used -- without it, a user connected on a
different instance would look offline and get a redundant push. Fixing
this was scoped in beyond the issue's literal ask (message fan-out only)
since it's a real correctness gap in a phase specifically about running
more than one instance; a known limitation (no heartbeat/TTL, so a hard
crash leaks a presence increment) is documented in the README instead of
solved here.

New tests/test_broadcast.py spins up two independent app instances sharing
one Postgres + Redis to prove delivery and presence both actually cross
the Redis boundary, not just work in-process. Manually verified the same
thing against two real uvicorn processes on different ports.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-14 07:13:06 -06:00
co-authored by Claude Sonnet 5
parent d09bf4a30a
commit 0b995ef75f
11 changed files with 328 additions and 57 deletions
+12 -6
View File
@@ -9,7 +9,7 @@ from app.database import get_db
from app.models import Room, RoomMembership, User
from app.services.message_service import create_message
from app.services.push_service import send_push_to_user
from app.ws.connection_manager import ConnectionManager
from app.ws.presence import Presence
router = APIRouter(tags=["ws"])
@@ -33,7 +33,7 @@ async def _is_room_member(db: AsyncSession, room_id: uuid.UUID, user_id: uuid.UU
async def _notify_offline_members(
db: AsyncSession,
manager: ConnectionManager,
presence: Presence,
room_id: uuid.UUID,
sender: User,
content: str,
@@ -42,7 +42,7 @@ async def _notify_offline_members(
select(RoomMembership.user_id).where(RoomMembership.room_id == room_id)
)
member_ids = {row[0] for row in result.all()}
offline_ids = member_ids - manager.connected_user_ids(room_id)
offline_ids = member_ids - await presence.connected_user_ids(room_id)
if not offline_ids:
return
@@ -70,6 +70,8 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
await websocket.accept()
manager = websocket.app.state.connection_manager
presence: Presence = websocket.app.state.presence
broadcaster = websocket.app.state.broadcaster
joined_rooms: set[uuid.UUID] = set()
try:
@@ -90,7 +92,8 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
{"type": "error", "detail": "Not a member of this room"}
)
continue
manager.join(envelope.room_id, websocket, user.id)
manager.join(envelope.room_id, websocket)
await presence.join(envelope.room_id, user.id)
joined_rooms.add(envelope.room_id)
await websocket.send_json({"type": "joined", "room_id": str(envelope.room_id)})
@@ -99,6 +102,7 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
await websocket.send_json({"type": "error", "detail": "room_id required"})
continue
manager.leave(envelope.room_id, websocket)
await presence.leave(envelope.room_id, user.id)
joined_rooms.discard(envelope.room_id)
elif envelope.type == "message":
@@ -115,7 +119,7 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
)
continue
message = await create_message(db, envelope.room_id, user.id, envelope.content)
await manager.broadcast(
await broadcaster.publish(
envelope.room_id,
{
"type": "message",
@@ -128,7 +132,7 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
},
)
await _notify_offline_members(
db, manager, envelope.room_id, user, envelope.content
db, presence, envelope.room_id, user, envelope.content
)
else:
@@ -140,3 +144,5 @@ async def chat_endpoint(websocket: WebSocket, db: AsyncSession = Depends(get_db)
pass
finally:
manager.leave_all(websocket)
for room_id in joined_rooms:
await presence.leave(room_id, user.id)