From 3482a35e549370fcff6b4685af83ba74a55fe99d Mon Sep 17 00:00:00 2001 From: "J. King" Date: Sat, 30 Sep 2017 11:43:43 -0400 Subject: [PATCH] Implement feed discovery; fixes #110 --- lib/Database.php | 8 ++++---- lib/Feed.php | 13 +++++++------ tests/Feed/TestFeed.php | 9 +++++++++ tests/docroot/Feed/Discovery/Feed.php | 12 ++++++++++++ tests/docroot/Feed/Discovery/Invalid.php | 8 ++++++++ tests/docroot/Feed/Discovery/Valid.php | 9 +++++++++ tests/lib/Database/SeriesSubscription.php | 4 ++-- 7 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 tests/docroot/Feed/Discovery/Feed.php create mode 100644 tests/docroot/Feed/Discovery/Invalid.php create mode 100644 tests/docroot/Feed/Discovery/Valid.php diff --git a/lib/Database.php b/lib/Database.php index 757188d8..e952404d 100644 --- a/lib/Database.php +++ b/lib/Database.php @@ -406,7 +406,7 @@ class Database { } } - public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = ""): int { + public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = "", bool $discover = true): int { if (!Arsse::$user->authorize($user, __FUNCTION__)) { throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]); } @@ -417,7 +417,7 @@ class Database { $feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId(); try { // perform an initial update on the newly added feed - $this->feedUpdate($feedID, true); + $this->feedUpdate($feedID, true, $discover); } catch (\Throwable $e) { // if the update fails, delete the feed we just added $this->db->prepare('DELETE from arsse_feeds where id is ?', 'int')->run($feedID); @@ -548,7 +548,7 @@ class Database { return array_column($feeds, 'id'); } - public function feedUpdate($feedID, bool $throwError = false): bool { + public function feedUpdate($feedID, bool $throwError = false, bool $discover = false): bool { $tr = $this->db->begin(); // check to make sure the feed exists if (!ValueInfo::id($feedID)) { @@ -564,7 +564,7 @@ class Database { // here. When an exception is thrown it should update the database with the // error instead of failing; if other exceptions are thrown, we should simply roll back try { - $feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape); + $feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape, $discover); if (!$feed->modified) { // if the feed hasn't changed, just compute the next fetch time and record it $this->db->prepare("UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ? WHERE id is ?", 'datetime', 'int')->run($feed->nextFetch, $feedID); diff --git a/lib/Feed.php b/lib/Feed.php index a4dc4764..ee6114cb 100644 --- a/lib/Feed.php +++ b/lib/Feed.php @@ -21,7 +21,7 @@ class Feed { public $newItems = []; public $changedItems = []; - public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false) { + public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false, bool $discover = false) { // set the configuration $userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf('Arsse/%s (%s %s; %s; https://code.jkingweb.ca/jking/arsse) PicoFeed (https://github.com/fguillot/picoFeed)', VERSION, // Arsse version @@ -36,7 +36,7 @@ class Feed { $this->config->setClientUserAgent($userAgent); $this->config->setGrabberUserAgent($userAgent); // fetch the feed - $this->download($url, $lastModified, $etag, $username, $password); + $this->download($url, $lastModified, $etag, $username, $password, $discover); // format the HTTP Last-Modified date returned $lastMod = $this->resource->getLastModified(); if (strlen($lastMod)) { @@ -65,10 +65,11 @@ class Feed { $this->nextFetch = $this->computeNextFetch(); } - protected function download(string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = ''): bool { + protected function download(string $url, string $lastModified, string $etag, string $username, string $password, bool $discover): bool { + $action = $discover ? "discover" : "download"; try { $this->reader = new Reader($this->config); - $this->resource = $this->reader->download($url, $lastModified, $etag, $username, $password); + $this->resource = $this->reader->$action($url, $lastModified, $etag, $username, $password); } catch (PicoFeedException $e) { throw new Feed\Exception($url, $e); } @@ -361,13 +362,13 @@ class Feed { protected function computeLastModified() { if (!$this->modified) { - return $this->lastModified; + return $this->lastModified; // @codeCoverageIgnore } $dates = $this->gatherDates(); if (sizeof($dates)) { return Date::normalize($dates[0]); } else { - return null; + return null; // @codeCoverageIgnore } } diff --git a/tests/Feed/TestFeed.php b/tests/Feed/TestFeed.php index 74c2426e..8957ba80 100644 --- a/tests/Feed/TestFeed.php +++ b/tests/Feed/TestFeed.php @@ -133,6 +133,15 @@ class TestFeed extends Test\AbstractTest { $this->assertSame($categories, $f->data->items[5]->categories); } + public function testDiscoverAFeedSuccessfully() { + $this->assertInstanceOf(Feed::class, new Feed(null, $this->base."Discovery/Valid", "", "", "", "", false, true)); + } + + public function testDiscoverAFeedUnsuccessfully() { + $this->assertException("subscriptionNotFound", "Feed"); + new Feed(null, $this->base."Discovery/Invalid", "", "", "", "", false, true); + } + public function testParseEntityExpansionAttack() { $this->assertException("xmlEntity", "Feed"); new Feed(null, $this->base."Parsing/XEEAttack"); diff --git a/tests/docroot/Feed/Discovery/Feed.php b/tests/docroot/Feed/Discovery/Feed.php new file mode 100644 index 00000000..a13398ac --- /dev/null +++ b/tests/docroot/Feed/Discovery/Feed.php @@ -0,0 +1,12 @@ + "application/rss+xml", + 'content' => << + + Test feed + http://example.com/ + Example newsfeed title + + +MESSAGE_BODY +]; diff --git a/tests/docroot/Feed/Discovery/Invalid.php b/tests/docroot/Feed/Discovery/Invalid.php new file mode 100644 index 00000000..59f7e6ec --- /dev/null +++ b/tests/docroot/Feed/Discovery/Invalid.php @@ -0,0 +1,8 @@ + "text/html", + 'content' => << +Example article + +MESSAGE_BODY +]; \ No newline at end of file diff --git a/tests/docroot/Feed/Discovery/Valid.php b/tests/docroot/Feed/Discovery/Valid.php new file mode 100644 index 00000000..293ea191 --- /dev/null +++ b/tests/docroot/Feed/Discovery/Valid.php @@ -0,0 +1,9 @@ + "text/html", + 'content' => << +Example article + + +MESSAGE_BODY +]; \ No newline at end of file diff --git a/tests/lib/Database/SeriesSubscription.php b/tests/lib/Database/SeriesSubscription.php index b68ecb47..fc528c37 100644 --- a/tests/lib/Database/SeriesSubscription.php +++ b/tests/lib/Database/SeriesSubscription.php @@ -135,7 +135,7 @@ trait SeriesSubscription { Phake::when(Arsse::$db)->feedUpdate->thenReturn(true); $this->assertSame($subID, Arsse::$db->subscriptionAdd($this->user, $url)); Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); - Phake::verify(Arsse::$db)->feedUpdate($feedID, true); + Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true); $state = $this->primeExpectations($this->data, [ 'arsse_feeds' => ['id','url','username','password'], 'arsse_subscriptions' => ['id','owner','feed'], @@ -153,7 +153,7 @@ trait SeriesSubscription { Arsse::$db->subscriptionAdd($this->user, $url); } catch (FeedException $e) { Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); - Phake::verify(Arsse::$db)->feedUpdate($feedID, true); + Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true); $state = $this->primeExpectations($this->data, [ 'arsse_feeds' => ['id','url','username','password'], 'arsse_subscriptions' => ['id','owner','feed'],