Private
Public Access
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
17e7f63e61 | ||
|
|
0bfe9c8076 |
No files matched your search
@@ -97,8 +97,15 @@ def sanitize_html(raw_html: str) -> str:
|
|||||||
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
return cleaned[:MAX_CONTENT_HTML_CHARS]
|
||||||
|
|
||||||
|
|
||||||
def _content_hash(title: str, content_html: str) -> str:
|
def _content_hash(
|
||||||
return hashlib.sha256(f"{title}\n{content_html}".encode("utf-8")).hexdigest()
|
title: str, link: str, author: str, summary: str, content_html: str, published_at
|
||||||
|
) -> str:
|
||||||
|
"""Covers every mutable Article field an entry carries (T6) — a feed
|
||||||
|
editing its link/author/summary/published date, not just title or
|
||||||
|
body, must still be detected as a change worth re-saving."""
|
||||||
|
published_key = published_at.isoformat() if published_at else ""
|
||||||
|
payload = "\n".join([title, link, author, summary, content_html, published_key])
|
||||||
|
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def _entry_guid(entry) -> str:
|
def _entry_guid(entry) -> str:
|
||||||
@@ -213,20 +220,24 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
for entry in entries:
|
for entry in entries:
|
||||||
guid = _entry_guid(entry)
|
guid = _entry_guid(entry)
|
||||||
title = (getattr(entry, "title", "") or "")[:1000]
|
title = (getattr(entry, "title", "") or "")[:1000]
|
||||||
|
link = (getattr(entry, "link", "") or "")[:2000]
|
||||||
|
author = (getattr(entry, "author", "") or "")[:500]
|
||||||
|
summary = (getattr(entry, "summary", "") or "")[:5000]
|
||||||
content_html = sanitize_html(_entry_content_html(entry))
|
content_html = sanitize_html(_entry_content_html(entry))
|
||||||
content_hash = _content_hash(title, content_html)
|
published_at = _entry_published_at(entry)
|
||||||
|
content_hash = _content_hash(title, link, author, summary, content_html, published_at)
|
||||||
|
|
||||||
article, created = Article.objects.get_or_create(
|
article, created = Article.objects.get_or_create(
|
||||||
feed=feed,
|
feed=feed,
|
||||||
guid=guid,
|
guid=guid,
|
||||||
defaults={
|
defaults={
|
||||||
"link": (getattr(entry, "link", "") or "")[:2000],
|
"link": link,
|
||||||
"title": title,
|
"title": title,
|
||||||
"author": (getattr(entry, "author", "") or "")[:500],
|
"author": author,
|
||||||
"content_html": content_html,
|
"content_html": content_html,
|
||||||
"summary": (getattr(entry, "summary", "") or "")[:5000],
|
"summary": summary,
|
||||||
"content_hash": content_hash,
|
"content_hash": content_hash,
|
||||||
"published_at": _entry_published_at(entry),
|
"published_at": published_at,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -234,9 +245,23 @@ def fetch_and_ingest(feed: Feed) -> FetchResult:
|
|||||||
new_ids.append(article.pk)
|
new_ids.append(article.pk)
|
||||||
elif article.content_hash != content_hash:
|
elif article.content_hash != content_hash:
|
||||||
article.title = title
|
article.title = title
|
||||||
|
article.link = link
|
||||||
|
article.author = author
|
||||||
|
article.summary = summary
|
||||||
article.content_html = content_html
|
article.content_html = content_html
|
||||||
|
article.published_at = published_at
|
||||||
article.content_hash = content_hash
|
article.content_hash = content_hash
|
||||||
article.save(update_fields=["title", "content_html", "content_hash"])
|
article.save(
|
||||||
|
update_fields=[
|
||||||
|
"title",
|
||||||
|
"link",
|
||||||
|
"author",
|
||||||
|
"summary",
|
||||||
|
"content_html",
|
||||||
|
"published_at",
|
||||||
|
"content_hash",
|
||||||
|
]
|
||||||
|
)
|
||||||
updated_ids.append(article.pk)
|
updated_ids.append(article.pk)
|
||||||
|
|
||||||
return FetchResult(
|
return FetchResult(
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
import datetime as dt
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import requests
|
import requests
|
||||||
import responses
|
import responses
|
||||||
@@ -204,6 +206,40 @@ def test_refetching_updated_content_does_not_reset_user_state():
|
|||||||
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
assert ArticleTag.objects.filter(user=user, article=article, tag=starred_tag).exists()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.django_db
|
||||||
|
@responses.activate
|
||||||
|
def test_refetching_picks_up_link_author_summary_and_published_at_changes():
|
||||||
|
"""Previously only title/content_html/content_hash were updated on
|
||||||
|
refetch (T6) — a feed correcting its link, author, summary, or
|
||||||
|
publication date without touching the title went unnoticed forever."""
|
||||||
|
feed = Feed.objects.create(feed_url=FEED_URL)
|
||||||
|
|
||||||
|
responses.add(responses.GET, FEED_URL, body=GOOD_FEED_XML, status=200)
|
||||||
|
fetch_and_ingest(feed)
|
||||||
|
feed.save()
|
||||||
|
|
||||||
|
article = Article.objects.get(feed=feed)
|
||||||
|
assert article.link == "https://example.com/1"
|
||||||
|
assert article.author == ""
|
||||||
|
|
||||||
|
edited_xml = GOOD_FEED_XML.replace(
|
||||||
|
"<link>https://example.com/1</link>",
|
||||||
|
"<link>https://example.com/1-moved</link>",
|
||||||
|
).replace(
|
||||||
|
"<pubDate>Mon, 01 Jan 2024 00:00:00 GMT</pubDate>",
|
||||||
|
"<pubDate>Tue, 02 Jan 2024 00:00:00 GMT</pubDate>\n <author>jane@example.com</author>",
|
||||||
|
)
|
||||||
|
responses.add(responses.GET, FEED_URL, body=edited_xml, status=200)
|
||||||
|
result = fetch_and_ingest(feed)
|
||||||
|
feed.save()
|
||||||
|
|
||||||
|
assert result.updated_article_ids == [article.pk]
|
||||||
|
article.refresh_from_db()
|
||||||
|
assert article.link == "https://example.com/1-moved"
|
||||||
|
assert article.author == "jane@example.com"
|
||||||
|
assert article.published_at == dt.datetime(2024, 1, 2, tzinfo=dt.timezone.utc)
|
||||||
|
|
||||||
|
|
||||||
def test_sanitize_html_caps_stored_output_length():
|
def test_sanitize_html_caps_stored_output_length():
|
||||||
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
huge = "<p>" + ("a" * (MAX_CONTENT_HTML_CHARS + 10_000)) + "</p>"
|
||||||
cleaned = sanitize_html(huge)
|
cleaned = sanitize_html(huge)
|
||||||
|
|||||||
+8
-3
@@ -54,7 +54,11 @@ GOOD_FEED_XML = """<?xml version="1.0"?>
|
|||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@responses.activate
|
@responses.activate
|
||||||
def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
|
def test_add_feed_view_subscribes_and_enqueues_the_initial_fetch(client, two_users):
|
||||||
|
"""The subscription is created synchronously; the first fetch is
|
||||||
|
dispatched via fetch_feed.delay() (T3) rather than run inline — in
|
||||||
|
tests CELERY_TASK_ALWAYS_EAGER still runs it immediately, so the
|
||||||
|
article it ingests is visible by the time the response returns."""
|
||||||
alice, _ = two_users
|
alice, _ = two_users
|
||||||
client.login(username="alice", password="pw-alice-123")
|
client.login(username="alice", password="pw-alice-123")
|
||||||
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
|
responses.add(responses.GET, "https://example.com/feed.xml", body=GOOD_FEED_XML, status=200)
|
||||||
@@ -62,9 +66,10 @@ def test_add_feed_view_subscribes_and_fetches_immediately(client, two_users):
|
|||||||
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
|
response = client.post(reverse("feeds:add_feed"), {"feed_url": "https://example.com/feed.xml"})
|
||||||
|
|
||||||
assert response.status_code == 302
|
assert response.status_code == 302
|
||||||
assert Subscription.objects.filter(
|
subscription = Subscription.objects.get(
|
||||||
user=alice, feed__feed_url="https://example.com/feed.xml"
|
user=alice, feed__feed_url="https://example.com/feed.xml"
|
||||||
).exists()
|
)
|
||||||
|
assert subscription.feed.articles.filter(guid="guid-1").exists()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
|||||||
+8
-16
@@ -12,8 +12,7 @@ from apps.articles.models import SubscriptionTag, Tag
|
|||||||
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
from .forms import AddFeedForm, EditSubscriptionForm, OPMLImportForm
|
||||||
from .models import Feed, Subscription
|
from .models import Feed, Subscription
|
||||||
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
from .opml import export_opml, extract_opml_bytes, import_opml, parse_opml
|
||||||
from .services.fetcher import fetch_and_ingest
|
from .tasks import fetch_feed, import_opml_task
|
||||||
from .tasks import import_opml_task
|
|
||||||
|
|
||||||
OPML_ASYNC_THRESHOLD = 50
|
OPML_ASYNC_THRESHOLD = 50
|
||||||
|
|
||||||
@@ -39,22 +38,15 @@ def add_feed(request):
|
|||||||
user=request.user, subscription=subscription, tag=tag
|
user=request.user, subscription=subscription, tag=tag
|
||||||
)
|
)
|
||||||
|
|
||||||
# Fetch synchronously so the user sees results immediately; the
|
# Enqueue the initial fetch instead of running it inline (T3) — a
|
||||||
# periodic Celery poll takes over from here for subsequent fetches.
|
# slow or large feed would otherwise tie up a Gunicorn worker for
|
||||||
result = fetch_and_ingest(feed)
|
# the duration of the request; fetch_feed handles the save/error
|
||||||
feed.save()
|
# bookkeeping the same way the periodic poll's dispatch does.
|
||||||
|
fetch_feed.delay(feed.pk)
|
||||||
if result.status == Feed.STATUS_ERROR:
|
|
||||||
messages.warning(
|
|
||||||
request,
|
|
||||||
f"Subscribed to {feed_url}, but the initial fetch failed: {result.error_message}",
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
article_count = len(result.new_article_ids)
|
|
||||||
messages.success(
|
messages.success(
|
||||||
request,
|
request,
|
||||||
f"Subscribed to '{subscription.display_title}' — "
|
f"Subscribed to {feed_url}. Fetching its articles now — "
|
||||||
f"{article_count} article(s) fetched.",
|
"refresh in a few seconds if you don't see them yet.",
|
||||||
)
|
)
|
||||||
return redirect("reader:stream")
|
return redirect("reader:stream")
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user