1
1
Fork 0
mirror of https://code.mensbeam.com/MensBeam/Arsse.git synced 2024-12-22 21:22:40 +00:00

Fix feed fetching

There are almost certainly other bugs; proper tests forthcoming
This commit is contained in:
J. King 2017-05-04 14:42:40 -04:00
parent 9dccdb32a7
commit 9c7f4710aa
4 changed files with 30 additions and 10 deletions

View file

@ -398,7 +398,7 @@ class Database {
if(!$this->userExists($user)) throw new User\Exception("doesNotExist", ["user" => $user, "action" => __FUNCTION__]); if(!$this->userExists($user)) throw new User\Exception("doesNotExist", ["user" => $user, "action" => __FUNCTION__]);
// check to see if the feed exists // check to see if the feed exists
$feedID = $this->db->prepare("SELECT id from arsse_feeds where url is ? and username is ? and password is ?", "str", "str", "str")->run($url, $fetchUser, $fetchPassword)->getValue(); $feedID = $this->db->prepare("SELECT id from arsse_feeds where url is ? and username is ? and password is ?", "str", "str", "str")->run($url, $fetchUser, $fetchPassword)->getValue();
if(is_nill($feedID)) { if(is_null($feedID)) {
// if the feed doesn't exist add it to the database; we do this unconditionally so as to lock SQLite databases for as little time as possible // if the feed doesn't exist add it to the database; we do this unconditionally so as to lock SQLite databases for as little time as possible
$feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId(); $feedID = $this->db->prepare('INSERT INTO arsse_feeds(url,username,password) values(?,?,?)', 'str', 'str', 'str')->run($url, $fetchUser, $fetchPassword)->lastId();
try { try {
@ -419,6 +419,18 @@ class Database {
return (bool) $this->db->prepare("DELETE from arsse_subscriptions where owner is ? and id is ?", "str", "int")->run($user, $id)->changes(); return (bool) $this->db->prepare("DELETE from arsse_subscriptions where owner is ? and id is ?", "str", "int")->run($user, $id)->changes();
} }
public function subscriptionList(string $user, int $folder = null): Db\Result {
if(!Data::$user->authorize($user, __FUNCTION__)) throw new User\ExceptionAuthz("notAuthorized", ["action" => __FUNCTION__, "user" => $user]);
if(!$this->userExists($user)) throw new User\Exception("doesNotExist", ["user" => $user, "action" => __FUNCTION__]);
// check to make sure the folder exists, if one is specified
if(!is_null($folder)) {
if(!$this->db->prepare("SELECT count(*) from arsse_folders where owner is ? and id is ?", "str", "int")->run($user, $folder)->getValue()) {
throw new Db\ExceptionInput("idMissing", ["action" => __FUNCTION__, "field" => "folder", 'id' => $folder]);
}
}
return $this->db->prepare("SELECT arsse_subscriptions.id, arsse_feeds.url, arsse_feeds.title from arsse_subscriptions join arsse_feeds on feed = arsse_feeds.id where arsse_subscriptions.owner is ?", "str")->run($user);
}
public function feedUpdate(int $feedID, bool $throwError = false): bool { public function feedUpdate(int $feedID, bool $throwError = false): bool {
$this->db->begin(); $this->db->begin();
try { try {
@ -452,7 +464,7 @@ class Database {
//prepare the necessary statements to perform the update //prepare the necessary statements to perform the update
if(sizeof($feed->newItems) || sizeof($feed->changedItems)) { if(sizeof($feed->newItems) || sizeof($feed->changedItems)) {
$qInsertCategory = $this->db->prepare('INSERT INTO arsse_categories(article,name) values(?,?)', 'int', 'str'); $qInsertCategory = $this->db->prepare('INSERT INTO arsse_categories(article,name) values(?,?)', 'int', 'str');
$qInsertEdition = $this->db->prepare('INSERT INTO arse_editions(article) values(?)', 'int'); $qInsertEdition = $this->db->prepare('INSERT INTO arsse_editions(article) values(?)', 'int');
} }
if(sizeof($feed->newItems)) { if(sizeof($feed->newItems)) {
$qInsertArticle = $this->db->prepare( $qInsertArticle = $this->db->prepare(
@ -547,7 +559,7 @@ class Database {
// perform the query // perform the query
return $articles = $this->db->prepare( return $articles = $this->db->prepare(
'SELECT id, DATEFORMAT("unix", edited) AS edited_date, guid, url_title_hash, url_content_hash, title_content_hash FROM arsse_articles '. 'SELECT id, DATEFORMAT("unix", edited) AS edited_date, guid, url_title_hash, url_content_hash, title_content_hash FROM arsse_articles '.
'WHERE feed is ? and (guid in($cId) or url_title_hash in($cHashUT) or url_content_hash in($cHashUC) or title_content_hash in($cHashTC)', 'WHERE feed is ? and (guid in($cId) or url_title_hash in($cHashUT) or url_content_hash in($cHashUC) or title_content_hash in($cHashTC))',
'int', $tId, $tHashUT, $tHashUC, $tHashTC 'int', $tId, $tHashUT, $tHashUC, $tHashTC
)->run($feedID, $ids, $hashesUT, $hashesUC, $hashesTC); )->run($feedID, $ids, $hashesUT, $hashesUC, $hashesTC);
} }

View file

@ -52,7 +52,7 @@ class Result implements \JKingWeb\Arsse\Db\Result {
} }
public function __destruct() { public function __destruct() {
$this->set->finalize(); try{$this->set->finalize();} catch(\Throwable $e) {}
unset($this->set); unset($this->set);
} }

View file

@ -24,7 +24,7 @@ class Feed {
// format the HTTP Last-Modified date returned // format the HTTP Last-Modified date returned
$lastMod = $this->resource->getLastModified(); $lastMod = $this->resource->getLastModified();
if(strlen($lastMod)) { if(strlen($lastMod)) {
$this->$lastModified = \DateTime::createFromFormat("!D, d M Y H:i:s e", $lastMod); $this->lastModified = \DateTime::createFromFormat("!D, d M Y H:i:s e", $lastMod);
} }
$this->modified = $this->resource->isModified(); $this->modified = $this->resource->isModified();
//parse the feed, if it has been modified //parse the feed, if it has been modified
@ -162,6 +162,7 @@ class Feed {
$new = $tentative = $edited = []; $new = $tentative = $edited = [];
// iterate through the articles and for each determine whether it is existing, edited, or entirely new // iterate through the articles and for each determine whether it is existing, edited, or entirely new
foreach($items as $index => $i) { foreach($items as $index => $i) {
$found = false;
foreach($articles as $a) { foreach($articles as $a) {
if( if(
// the item matches if the GUID matches... // the item matches if the GUID matches...
@ -174,28 +175,29 @@ class Feed {
if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) { if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) {
// if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited // if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited
// we store the item index and database record ID as a key/value pair // we store the item index and database record ID as a key/value pair
$found = true;
$edited[$index] = $a['id']; $edited[$index] = $a['id'];
break; break;
} else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) { } else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) {
// if any of the hashes do not match, then the article has been edited // if any of the hashes do not match, then the article has been edited
$found = true;
$edited[$index] = $a['id']; $edited[$index] = $a['id'];
break; break;
} else { } else {
// otherwise the item is unchanged and we can ignore it // otherwise the item is unchanged and we can ignore it
$found = true;
break; break;
} }
} else {
// if we don't have a match, add the item to the tentatively new list
$tentative[] = $index;
} }
} }
if(!$found) $tentative[] = $index;
} }
if(sizeof($tentative)) { if(sizeof($tentative)) {
// if we need to, perform a second pass on the database looking specifically for IDs and hashes of the new items // if we need to, perform a second pass on the database looking specifically for IDs and hashes of the new items
$ids = $hashesUT = $hashesUC = $hashesTC = []; $ids = $hashesUT = $hashesUC = $hashesTC = [];
foreach($tentative as $index) { foreach($tentative as $index) {
$i = $items[$index]; $i = $items[$index];
if($i->id) $ids[] = $id->id; if($i->id) $ids[] = $i->id;
$hashesUT[] = $i->urlTitleHash; $hashesUT[] = $i->urlTitleHash;
$hashesUC[] = $i->urlContentHash; $hashesUC[] = $i->urlContentHash;
$hashesTC[] = $i->titleContentHash; $hashesTC[] = $i->titleContentHash;
@ -203,6 +205,7 @@ class Feed {
$articles = Data::$db->articleMatchIds($feedID, $ids, $hashesUT, $hashesUC, $hashesTC); $articles = Data::$db->articleMatchIds($feedID, $ids, $hashesUT, $hashesUC, $hashesTC);
foreach($tentative as $index) { foreach($tentative as $index) {
$i = $items[$index]; $i = $items[$index];
$found = false;
foreach($articles as $a) { foreach($articles as $a) {
if( if(
// the item matches if the GUID matches... // the item matches if the GUID matches...
@ -215,14 +218,17 @@ class Feed {
if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) { if($i->updatedDate && $i->updatedDate->getTimestamp() !== $match['edited_date']) {
// if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited // if the item has an edit timestamp and it doesn't match that of the article in the database, the the article has been edited
// we store the item index and database record ID as a key/value pair // we store the item index and database record ID as a key/value pair
$found = true;
$edited[$index] = $a['id']; $edited[$index] = $a['id'];
break; break;
} else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) { } else if($i->urlTitleHash !== $a['url_title_hash'] || $i->urlContentHash !== $a['url_content_hash'] || $i->titleContentHash !== $a['title_content_hash']) {
// if any of the hashes do not match, then the article has been edited // if any of the hashes do not match, then the article has been edited
$found = true;
$edited[$index] = $a['id']; $edited[$index] = $a['id'];
break; break;
} else { } else {
// otherwise the item is unchanged and we can ignore it // otherwise the item is unchanged and we can ignore it
$found = true;
break; break;
} }
} else { } else {
@ -230,6 +236,7 @@ class Feed {
$new[] = $index; $new[] = $index;
} }
} }
if(!$found) $new[] = $index;
} }
} }
// FIXME: fetch full content when appropriate // FIXME: fetch full content when appropriate

View file

@ -20,4 +20,5 @@ Data::$user->authorizationEnabled(false);
Data::$user->rightsSet($user, User\Driver::RIGHTS_GLOBAL_ADMIN); Data::$user->rightsSet($user, User\Driver::RIGHTS_GLOBAL_ADMIN);
Data::$user->authorizationEnabled(true); Data::$user->authorizationEnabled(true);
Data::$db->folderAdd($user, ['name' => 'ook']); Data::$db->folderAdd($user, ['name' => 'ook']);
Data::$db->subscriptionAdd($user, "http://www.pcgamer.com/rss/"); Data::$db->subscriptionAdd($user, "https://jkingweb.ca/test.atom");
var_export(Data::$db->subscriptionList($user)->getAll());