Private
Public Access
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
03dd37e69d | ||
|
|
17e7f63e61 | ||
|
|
0bfe9c8076 |
No files matched your search
@@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str:
|
||||
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
||||
|
||||
|
||||
def _content_hash(title: str, content_html: str) -> str:
|
||||
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
||||
def _content_hash(
|
||||
title: str, link: str, author: str, summary: str, content_html: str, published_at
|
||||
) -> str:
|
||||
"""Covers every mutable Article field an entry carries (T6) — a feed
|
||||
editing its link/author/summary/published date, not just title or
|
||||
body, must still be detected as a change worth re-saving."""
|
||||
published_key = published_at.isoformat() if published_at else ""
|
||||
payload = "\n".join([title, link, author, summary, content_html, published_key])
|
||||
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _entry_guid(entry) -> str:
|
||||
@@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
for entry in entries:
|
||||
guid = _entry_guid(entry)
|
||||
title = (getattr(entry, "title", "") or "")[:1000]
|
||||
link = (getattr(entry, "link", "") or "")[:2000]
|
||||
author = (getattr(entry, "author", "") or "")[:500]
|
||||
summary = (getattr(entry, "summary", "") or "")[:5000]
|
||||
content_html = sanitize_html(_entry_content_html(entry))
|
||||
content_hash = _content_hash(title, content_html)
|
||||
published_at = _entry_published_at(entry)
|
||||
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
|
||||
|
||||
article, created = Article.objects.get_or_create(
|
||||
feed=feed,
|
||||
guid=guid,
|
||||
defaults={
|
||||
"link": (getattr(entry, "link", "") or "")[:2000],
|
||||
"link": link,
|
||||
"title": title,
|
||||
"author": (getattr(entry, "author", "") or "")[:500],
|
||||
"author": author,
|
||||
"content_html": content_html,
|
||||
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
||||
"summary": summary,
|
||||
"content_hash": content_hash,
|
||||
"published_at": _entry_published_at(entry),
|
||||
"published_at": published_at,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
||||
new_ids.append(article.pk)
|
||||
elif article.content_hash != content_hash:
|
||||
article.title = title
|
||||
article.link = link
|
||||
article.author = author
|
||||
article.summary = summary
|
||||
article.content_html = content_html
|
||||
article.published_at = published_at
|
||||
article.content_hash = content_hash
|
||||
article.save(update_fields=["title", "content_html", "content_hash"])
|
||||
article.save(
|
||||
update_fields=[
|
||||
"title",
|
||||
"link",
|
||||
"author",
|
||||
"summary",
|
||||
"content_html",
|
||||
"published_at",
|
||||
"content_hash",
|
||||
]
|
||||
)
|
||||
updated_ids.append(article.pk)
|
||||
|
||||
return FetchResult(
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
import requests
|
||||
import responses
|
||||
@@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state():
|
||||
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
|
||||
"""Previously only title/content_html/content_hash were updated on
|
||||
refetch (T6) — a feed correcting its link, author, summary, or
|
||||
publication date without touching the title went unnoticed forever."""
|
||||
feed = Feed.objects.create(feed_url=FEED_URL)
|
||||
|
||||
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
||||
fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
article = Article.objects.get(feed=feed)
|
||||
assert article.link == "https://example.com/1"
|
||||
assert article.author == ""
|
||||
|
||||
edited_xml = GOOD_FEED_XML.replace(
|
||||
"<link>https://example.com/1</link>",
|
||||
"<link>https://example.com/1-moved</link>",
|
||||
).replace(
|
||||
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
|
||||
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
|
||||
)
|
||||
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
assert result.updated_article_ids == [article.pk]
|
||||
article.refresh_from_db()
|
||||
assert article.link == "https://example.com/1-moved"
|
||||
assert article.author == "jane@example.com"
|
||||
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
|
||||
|
||||
|
||||
def test_sanitize_html_caps_stored_output_length():
|
||||
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
||||
cleaned = sanitize_html(huge)
|
||||
|
||||
+8
-3
@@ -54,7 +54,11 @@ GOOD_FEED_XML = """<?xml version="1.0"?>
|
||||
|
||||
@pytest.mark.django_db
|
||||
@responses.activate
|
||||
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
|
||||
def test_add_feed_view_subscribes_and_enqueues_the_initial_fetch(client, two_users):
|
||||
"""The subscription is created synchronously; the first fetch is
|
||||
dispatched via fetch_feed.delay() (T3) rather than run inline — in
|
||||
tests CELERY_TASK_ALWAYS_EAGER still runs it immediately, so the
|
||||
article it ingests is visible by the time the response returns."""
|
||||
alice, _ = two_users
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
|
||||
@@ -62,9 +66,10 @@ def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
|
||||
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
|
||||
|
||||
assert response.status_code == 302
|
||||
assert Subscription.objects.filter(
|
||||
subscription = Subscription.objects.get(
|
||||
user=alice, feed__feed_url="https://example.com/feed.xml"
|
||||
).exists()
|
||||
)
|
||||
assert subscription.feed.articles.filter(guid="guid-1").exists()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
|
||||
+11
-19
@@ -12,8 +12,7 @@ from apps.articles.models import SubscriptionTag, Tag
|
||||
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
||||
from .models import Feed, Subscription
|
||||
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||
from .services.fetcher import fetch_and_ingest
|
||||
from .tasks import import_opml_task
|
||||
from .tasks import fetch_feed, import_opml_task
|
||||
|
||||
OPML_ASYNC_THRESHOLD = 50
|
||||
|
||||
@@ -39,23 +38,16 @@ def add_feed(request):
|
||||
user=request.user, subscription=subscription, tag=tag
|
||||
)
|
||||
|
||||
# Fetch synchronously so the user sees results immediately; the
|
||||
# periodic Celery poll takes over from here for subsequent fetches.
|
||||
result = fetch_and_ingest(feed)
|
||||
feed.save()
|
||||
|
||||
if result.status == Feed.STATUS_ERROR:
|
||||
messages.warning(
|
||||
request,
|
||||
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}",
|
||||
)
|
||||
else:
|
||||
article_count = len(result.new_article_ids)
|
||||
messages.success(
|
||||
request,
|
||||
f"Subscribed to '{subscription.display_title}' — "
|
||||
f"{article_count} article(s) fetched.",
|
||||
)
|
||||
# Enqueue the initial fetch instead of running it inline (T3) — a
|
||||
# slow or large feed would otherwise tie up a Gunicorn worker for
|
||||
# the duration of the request; fetch_feed handles the save/error
|
||||
# bookkeeping the same way the periodic poll's dispatch does.
|
||||
fetch_feed.delay(feed.pk)
|
||||
messages.success(
|
||||
request,
|
||||
f"Subscribed to {feed_url}. Fetching its articles now — "
|
||||
"refresh in a few seconds if you don't see them yet.",
|
||||
)
|
||||
return redirect("reader:stream")
|
||||
|
||||
return render(request, "feeds/add_feed.html", {"form": form})
|
||||
|
||||
@@ -217,3 +217,75 @@ def test_article_rows_carry_their_utc_instant_for_client_side_date_grouping(clie
|
||||
undated_row = re.search(rf'<li id="article-{undated.pk}".*?>', content, flags=re.DOTALL)
|
||||
assert undated_row is not None
|
||||
assert 'data-published-utc=""' in undated_row.group(0)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_reports_nothing_below_the_given_cursor(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
existing = Article.objects.create(feed=feed, guid="existing", title="Existing")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:check_new"), {"filter": "all", "since": existing.pk})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.content.decode().strip() == ""
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_reports_articles_created_after_the_cursor(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
existing = Article.objects.create(feed=feed, guid="existing", title="Existing")
|
||||
Article.objects.create(feed=feed, guid="new-1", title="New one")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:check_new"), {"filter": "all", "since": existing.pk})
|
||||
|
||||
content = response.content.decode()
|
||||
assert "1 new article" in content
|
||||
assert "click to show" in content
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_check_new_articles_respects_the_active_tag_scope(client, alice):
|
||||
from apps.articles.models import SubscriptionTag, Tag
|
||||
|
||||
feed_a = Feed.objects.create(feed_url="https://a.example.com/feed.xml")
|
||||
feed_b = Feed.objects.create(feed_url="https://b.example.com/feed.xml")
|
||||
sub_a = Subscription.objects.create(user=alice, feed=feed_a)
|
||||
Subscription.objects.create(user=alice, feed=feed_b)
|
||||
tag = Tag.objects.create(user=alice, name="work")
|
||||
SubscriptionTag.objects.create(user=alice, subscription=sub_a, tag=tag)
|
||||
|
||||
cursor = Article.objects.create(feed=feed_a, guid="cursor", title="Cursor").pk
|
||||
# New in feed_b, which isn't tagged "work" — shouldn't count for this scope.
|
||||
Article.objects.create(feed=feed_b, guid="untagged-new", title="Untagged new")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(
|
||||
reverse("reader:check_new"), {"filter": "all", "tag": tag.pk, "since": cursor}
|
||||
)
|
||||
|
||||
assert response.content.decode().strip() == ""
|
||||
|
||||
# New in feed_a (tagged "work") — should count.
|
||||
Article.objects.create(feed=feed_a, guid="tagged-new", title="Tagged new")
|
||||
response = client.get(
|
||||
reverse("reader:check_new"), {"filter": "all", "tag": tag.pk, "since": cursor}
|
||||
)
|
||||
assert "1 new article" in response.content.decode()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_stream_exposes_the_latest_article_id_and_scope_for_the_banner(client, alice):
|
||||
feed = Feed.objects.create(feed_url="https://example.com/feed.xml")
|
||||
Subscription.objects.create(user=alice, feed=feed)
|
||||
article = Article.objects.create(feed=feed, guid="g1", title="Hello")
|
||||
|
||||
client.login(username="alice", password="pw-alice-123")
|
||||
response = client.get(reverse("reader:stream"), {"filter": "all"})
|
||||
content = response.content.decode()
|
||||
|
||||
assert f"since={article.pk}" in content
|
||||
assert "filter=all" in content
|
||||
@@ -6,6 +6,7 @@ app_name = "reader"
|
||||
|
||||
urlpatterns = [
|
||||
path("", views.stream, name="stream"),
|
||||
path("check-new/", views.check_new_articles, name="check_new"),
|
||||
path("mark-all-read/", views.mark_all_read, name="mark_all_read"),
|
||||
path("articles/<int:article_id>/", views.article_detail, name="article_detail"),
|
||||
path(
|
||||
|
||||
+42
-4
@@ -1,7 +1,8 @@
|
||||
from itertools import groupby
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.db.models import Q
|
||||
from django.db.models import Max, Q
|
||||
from django.http import Http404
|
||||
from django.shortcuts import get_object_or_404, redirect, render
|
||||
from django.urls import reverse
|
||||
@@ -117,8 +118,7 @@ def _group_by_date(articles):
|
||||
]
|
||||
|
||||
|
||||
@login_required
|
||||
def stream(request):
|
||||
def _resolve_active_filter(request):
|
||||
requested_filter = request.GET.get("filter")
|
||||
if requested_filter in VALID_FILTERS:
|
||||
active_filter = requested_filter
|
||||
@@ -128,7 +128,14 @@ def stream(request):
|
||||
# selected, rather than always resetting to "unread".
|
||||
active_filter = request.session.get("stream_filter", "unread")
|
||||
request.session["stream_filter"] = active_filter
|
||||
return active_filter
|
||||
|
||||
|
||||
def _filtered_queryset(request, active_filter):
|
||||
"""The stream's fully-scoped queryset (filter tab + tag/feed scope),
|
||||
unpaginated — shared by stream() and check_new_articles() so the "any
|
||||
new articles?" check always matches exactly what the stream itself
|
||||
would show."""
|
||||
queryset = _base_queryset(request.user)
|
||||
if active_filter == "unread":
|
||||
queryset = queryset.exclude(user_states__user=request.user, user_states__is_read=True)
|
||||
@@ -146,8 +153,13 @@ def stream(request):
|
||||
article_tags__tag__name=SAVED_TAG_NAME,
|
||||
article_tags__tag__is_system=True,
|
||||
)
|
||||
return _apply_scope(request, queryset)
|
||||
|
||||
queryset, active_tag, active_feed = _apply_scope(request, queryset)
|
||||
|
||||
@login_required
|
||||
def stream(request):
|
||||
active_filter = _resolve_active_filter(request)
|
||||
queryset, active_tag, active_feed = _filtered_queryset(request, active_filter)
|
||||
|
||||
try:
|
||||
page = max(int(request.GET.get("page", 1)), 1)
|
||||
@@ -158,6 +170,12 @@ def stream(request):
|
||||
total_count = queryset.count()
|
||||
articles = with_user_state(queryset[start:end], request.user)
|
||||
|
||||
scope_params = {"filter": active_filter}
|
||||
if active_tag:
|
||||
scope_params["tag"] = active_tag.pk
|
||||
if active_feed:
|
||||
scope_params["feed"] = active_feed.pk
|
||||
|
||||
context = {
|
||||
"date_groups": _group_by_date(articles),
|
||||
"active_filter": active_filter,
|
||||
@@ -168,10 +186,30 @@ def stream(request):
|
||||
"page": page,
|
||||
"has_next_page": end < total_count,
|
||||
"has_previous_page": page > 1,
|
||||
# Lets the "new articles" banner ask, on this exact same scope,
|
||||
# whether anything with a higher id has shown up since this page
|
||||
# was rendered — id (not published_at) because it's monotonic
|
||||
# regardless of how a feed backdates an entry's published date.
|
||||
"latest_article_id": queryset.aggregate(Max("id"))["id__max"] or 0,
|
||||
"new_check_qs": urlencode(scope_params),
|
||||
}
|
||||
return render(request, "reader/stream.html", context)
|
||||
|
||||
|
||||
@login_required
|
||||
def check_new_articles(request):
|
||||
active_filter = _resolve_active_filter(request)
|
||||
queryset, _active_tag, _active_feed = _filtered_queryset(request, active_filter)
|
||||
|
||||
try:
|
||||
since_id = int(request.GET.get("since", 0))
|
||||
except ValueError:
|
||||
since_id = 0
|
||||
|
||||
new_count = queryset.filter(id__gt=since_id).count()
|
||||
return render(request, "reader/_new_articles_banner.html", {"new_count": new_count})
|
||||
|
||||
|
||||
@login_required
|
||||
@require_POST
|
||||
def mark_all_read(request):
|
||||
|
||||
@@ -113,6 +113,19 @@ form p {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
.new-articles-banner {
|
||||
display: block;
|
||||
width: 100%;
|
||||
margin: 0 0 0.75rem;
|
||||
padding: 0.5rem 0.75rem;
|
||||
border: none;
|
||||
border-radius: 0.375rem;
|
||||
background: var(--accent);
|
||||
color: #fff;
|
||||
font-weight: 600;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
opacity: 0.6;
|
||||
padding: 2rem 0;
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{% if new_count %}
|
||||
<button type="button" class="new-articles-banner" onclick="location.reload()">
|
||||
{{ new_count }} new article{{ new_count|pluralize }} — click to show
|
||||
</button>
|
||||
{% endif %}
|
||||
@@ -23,5 +23,11 @@
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div id="new-articles-banner"
|
||||
hx-get="{% url 'reader:check_new' %}?since={{ latest_article_id }}&{{ new_check_qs }}"
|
||||
hx-trigger="every 30s"
|
||||
hx-target="this"
|
||||
hx-swap="innerHTML"></div>
|
||||
|
||||
{% include "reader/_article_list.html" %}
|
||||
{% endblock %}
|
||||
Reference in new issue
Block a user