1
1
Fork 0
mirror of https://code.mensbeam.com/MensBeam/Arsse.git synced 2024-12-23 08:04:54 +00:00

Implement feed discovery; fixes #110

This commit is contained in:
J. King 2017-09-30 11:43:43 -04:00
parent d1e4c6eed3
commit 3482a35e54
7 changed files with 51 additions and 12 deletions

View file

@ -406,7 +406,7 @@ class Database {
} }
} }
public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = ""): int { public function subscriptionAdd(string $user, string $url, string $fetchUser = "", string $fetchPassword = "", bool $discover = true): int {
if (!Arsse::$user->authorize($user, __FUNCTION__)) { if (!Arsse::$user->authorize($user, __FUNCTION__)) {
throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]); throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
} }
@ -417,7 +417,7 @@ class Database {
$feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId(); $feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId();
try { try {
// perform an initial update on the newly added feed // perform an initial update on the newly added feed
$this->feedUpdate($feedID, true); $this->feedUpdate($feedID, true, $discover);
} catch (\Throwable $e) { } catch (\Throwable $e) {
// if the update fails, delete the feed we just added // if the update fails, delete the feed we just added
$this->db->prepare('DELETE from arsse_feeds where id is ?', 'int')->run($feedID); $this->db->prepare('DELETE from arsse_feeds where id is ?', 'int')->run($feedID);
@ -548,7 +548,7 @@ class Database {
return array_column($feeds, 'id'); return array_column($feeds, 'id');
} }
public function feedUpdate($feedID, bool $throwError = false): bool { public function feedUpdate($feedID, bool $throwError = false, bool $discover = false): bool {
$tr = $this->db->begin(); $tr = $this->db->begin();
// check to make sure the feed exists // check to make sure the feed exists
if (!ValueInfo::id($feedID)) { if (!ValueInfo::id($feedID)) {
@ -564,7 +564,7 @@ class Database {
// here. When an exception is thrown it should update the database with the // here. When an exception is thrown it should update the database with the
// error instead of failing; if other exceptions are thrown, we should simply roll back // error instead of failing; if other exceptions are thrown, we should simply roll back
try { try {
$feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape); $feed = new Feed((int) $feedID, $f['url'], (string) Date::transform($f['modified'], "http", "sql"), $f['etag'], $f['username'], $f['password'], $scrape, $discover);
if (!$feed->modified) { if (!$feed->modified) {
// if the feed hasn't changed, just compute the next fetch time and record it // if the feed hasn't changed, just compute the next fetch time and record it
$this->db->prepare("UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ? WHERE id is ?", 'datetime', 'int')->run($feed->nextFetch, $feedID); $this->db->prepare("UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ? WHERE id is ?", 'datetime', 'int')->run($feed->nextFetch, $feedID);

View file

@ -21,7 +21,7 @@ class Feed {
public $newItems = []; public $newItems = [];
public $changedItems = []; public $changedItems = [];
public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false) { public function __construct(int $feedID = null, string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = '', bool $scrape = false, bool $discover = false) {
// set the configuration // set the configuration
$userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf('Arsse/%s (%s %s; %s; https://code.jkingweb.ca/jking/arsse) PicoFeed (https://github.com/fguillot/picoFeed)', $userAgent = Arsse::$conf->fetchUserAgentString ?? sprintf('Arsse/%s (%s %s; %s; https://code.jkingweb.ca/jking/arsse) PicoFeed (https://github.com/fguillot/picoFeed)',
VERSION, // Arsse version VERSION, // Arsse version
@ -36,7 +36,7 @@ class Feed {
$this->config->setClientUserAgent($userAgent); $this->config->setClientUserAgent($userAgent);
$this->config->setGrabberUserAgent($userAgent); $this->config->setGrabberUserAgent($userAgent);
// fetch the feed // fetch the feed
$this->download($url, $lastModified, $etag, $username, $password); $this->download($url, $lastModified, $etag, $username, $password, $discover);
// format the HTTP Last-Modified date returned // format the HTTP Last-Modified date returned
$lastMod = $this->resource->getLastModified(); $lastMod = $this->resource->getLastModified();
if (strlen($lastMod)) { if (strlen($lastMod)) {
@ -65,10 +65,11 @@ class Feed {
$this->nextFetch = $this->computeNextFetch(); $this->nextFetch = $this->computeNextFetch();
} }
protected function download(string $url, string $lastModified = '', string $etag = '', string $username = '', string $password = ''): bool { protected function download(string $url, string $lastModified, string $etag, string $username, string $password, bool $discover): bool {
$action = $discover ? "discover" : "download";
try { try {
$this->reader = new Reader($this->config); $this->reader = new Reader($this->config);
$this->resource = $this->reader->download($url, $lastModified, $etag, $username, $password); $this->resource = $this->reader->$action($url, $lastModified, $etag, $username, $password);
} catch (PicoFeedException $e) { } catch (PicoFeedException $e) {
throw new Feed\Exception($url, $e); throw new Feed\Exception($url, $e);
} }
@ -361,13 +362,13 @@ class Feed {
protected function computeLastModified() { protected function computeLastModified() {
if (!$this->modified) { if (!$this->modified) {
return $this->lastModified; return $this->lastModified; // @codeCoverageIgnore
} }
$dates = $this->gatherDates(); $dates = $this->gatherDates();
if (sizeof($dates)) { if (sizeof($dates)) {
return Date::normalize($dates[0]); return Date::normalize($dates[0]);
} else { } else {
return null; return null; // @codeCoverageIgnore
} }
} }

View file

@ -133,6 +133,15 @@ class TestFeed extends Test\AbstractTest {
$this->assertSame($categories, $f->data->items[5]->categories); $this->assertSame($categories, $f->data->items[5]->categories);
} }
public function testDiscoverAFeedSuccessfully() {
$this->assertInstanceOf(Feed::class, new Feed(null, $this->base."Discovery/Valid", "", "", "", "", false, true));
}
public function testDiscoverAFeedUnsuccessfully() {
$this->assertException("subscriptionNotFound", "Feed");
new Feed(null, $this->base."Discovery/Invalid", "", "", "", "", false, true);
}
public function testParseEntityExpansionAttack() { public function testParseEntityExpansionAttack() {
$this->assertException("xmlEntity", "Feed"); $this->assertException("xmlEntity", "Feed");
new Feed(null, $this->base."Parsing/XEEAttack"); new Feed(null, $this->base."Parsing/XEEAttack");

View file

@ -0,0 +1,12 @@
<?php return [
'mime' => "application/rss+xml",
'content' => <<<MESSAGE_BODY
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom" xmlns:dc="http://purl.org/dc/elements/1.1/">
<channel>
<title>Test feed</title>
<link>http://example.com/</link>
<description>Example newsfeed title</description>
</channel>
</rss>
MESSAGE_BODY
];

View file

@ -0,0 +1,8 @@
<?php return [
'mime' => "text/html",
'content' => <<<MESSAGE_BODY
<html>
<title>Example article</title>
</html>
MESSAGE_BODY
];

View file

@ -0,0 +1,9 @@
<?php return [
'mime' => "text/html",
'content' => <<<MESSAGE_BODY
<html>
<title>Example article</title>
<link rel="alternate" type="application/rss+xml" href="http://localhost:8000/Feed/Discovery/Feed">
</html>
MESSAGE_BODY
];

View file

@ -135,7 +135,7 @@ trait SeriesSubscription {
Phake::when(Arsse::$db)->feedUpdate->thenReturn(true); Phake::when(Arsse::$db)->feedUpdate->thenReturn(true);
$this->assertSame($subID, Arsse::$db->subscriptionAdd($this->user, $url)); $this->assertSame($subID, Arsse::$db->subscriptionAdd($this->user, $url));
Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd");
Phake::verify(Arsse::$db)->feedUpdate($feedID, true); Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true);
$state = $this->primeExpectations($this->data, [ $state = $this->primeExpectations($this->data, [
'arsse_feeds' => ['id','url','username','password'], 'arsse_feeds' => ['id','url','username','password'],
'arsse_subscriptions' => ['id','owner','feed'], 'arsse_subscriptions' => ['id','owner','feed'],
@ -153,7 +153,7 @@ trait SeriesSubscription {
Arsse::$db->subscriptionAdd($this->user, $url); Arsse::$db->subscriptionAdd($this->user, $url);
} catch (FeedException $e) { } catch (FeedException $e) {
Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd"); Phake::verify(Arsse::$user)->authorize($this->user, "subscriptionAdd");
Phake::verify(Arsse::$db)->feedUpdate($feedID, true); Phake::verify(Arsse::$db)->feedUpdate($feedID, true, true);
$state = $this->primeExpectations($this->data, [ $state = $this->primeExpectations($this->data, [
'arsse_feeds' => ['id','url','username','password'], 'arsse_feeds' => ['id','url','username','password'],
'arsse_subscriptions' => ['id','owner','feed'], 'arsse_subscriptions' => ['id','owner','feed'],