From 0514321a76adafa6e4ba9f85a38db40335e4ada5 Mon Sep 17 00:00:00 2001 From: "J. King" Date: Sat, 3 Jun 2017 14:08:33 -0400 Subject: [PATCH] Save enclosures and categories properly in feedUpdate --- lib/Database.php | 23 ++++--- tests/lib/Database/SeriesFeed.php | 103 +++++++++++++++++++----------- 2 files changed, 82 insertions(+), 44 deletions(-) diff --git a/lib/Database.php b/lib/Database.php index 6b777d75..11254fee 100644 --- a/lib/Database.php +++ b/lib/Database.php @@ -495,7 +495,7 @@ class Database { } catch (Feed\Exception $e) { // update the database with the resultant error and the next fetch time, incrementing the error count $this->db->prepare( - 'UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ?, err_count = err_count + 1, err_msg = ? WHERE id is ?', + "UPDATE arsse_feeds SET updated = CURRENT_TIMESTAMP, next_fetch = ?, err_count = err_count + 1, err_msg = ? WHERE id is ?", 'datetime', 'str', 'int' )->run(Feed::nextFetchOnError($f['err_count']), $e->getMessage(),$feedID); $tr->commit(); @@ -504,8 +504,9 @@ class Database { } //prepare the necessary statements to perform the update if(sizeof($feed->newItems) || sizeof($feed->changedItems)) { - $qInsertCategory = $this->db->prepare('INSERT INTO arsse_categories(article,name) values(?,?)', 'int', 'str'); - $qInsertEdition = $this->db->prepare('INSERT INTO arsse_editions(article) values(?)', 'int'); + $qInsertEnclosure = $this->db->prepare("INSERT INTO arsse_enclosures(article,url,type) values(?,?,?)", 'int', 'str', 'str'); + $qInsertCategory = $this->db->prepare("INSERT INTO arsse_categories(article,name) values(?,?)", 'int', 'str'); + $qInsertEdition = $this->db->prepare("INSERT INTO arsse_editions(article) values(?)", 'int'); } if(sizeof($feed->newItems)) { $qInsertArticle = $this->db->prepare( @@ -514,6 +515,7 @@ class Database { ); } if(sizeof($feed->changedItems)) { + $qDeleteEnclosures = $this->db->prepare('DELETE FROM arsse_enclosures WHERE article is ?', 'int'); $qDeleteCategories = $this->db->prepare('DELETE FROM arsse_categories WHERE article is ?', 'int'); $qClearReadMarks = $this->db->prepare('UPDATE arsse_marks SET read = 0, modified = CURRENT_TIMESTAMP WHERE article is ? and read is 1', 'int'); $qUpdateArticle = $this->db->prepare( @@ -536,8 +538,10 @@ class Database { $article->titleContentHash, $feedID )->lastId(); - // FIXME: Need to insert enclosures - foreach($article->getTag('category') as $c) { + if($article->enclosureUrl) { + $qInsertEnclosure->run($articleID,$article->enclosureUrl,$article->enclosureType); + } + foreach($article->categories as $c) { $qInsertCategory->run($articleID, $c); } $qInsertEdition->run($articleID); @@ -556,9 +560,12 @@ class Database { $article->titleContentHash, $articleID ); - // FIXME: Need to refresh enclosures + $qDeleteEnclosures->run($articleID); $qDeleteCategories->run($articleID); - foreach($article->getTag('category') as $c) { + if($article->enclosureUrl) { + $qInsertEnclosure->run($articleID,$article->enclosureUrl,$article->enclosureType); + } + foreach($article->categories as $c) { $qInsertCategory->run($articleID, $c); } $qInsertEdition->run($articleID); @@ -584,7 +591,7 @@ class Database { public function feedMatchLatest(int $feedID, int $count): Db\Result { return $this->db->prepare( - 'SELECT id, DATEFORMAT("unix", edited) AS edited_date, guid, url_title_hash, url_content_hash, title_content_hash FROM arsse_articles WHERE feed is ? ORDER BY edited desc limit ?', + 'SELECT id, DATEFORMAT("unix", edited) AS edited_date, guid, url_title_hash, url_content_hash, title_content_hash FROM arsse_articles WHERE feed is ? ORDER BY modified desc, id desc limit ?', 'int', 'int' )->run($feedID, $count); } diff --git a/tests/lib/Database/SeriesFeed.php b/tests/lib/Database/SeriesFeed.php index 6d505ff7..9db29633 100644 --- a/tests/lib/Database/SeriesFeed.php +++ b/tests/lib/Database/SeriesFeed.php @@ -10,9 +10,9 @@ use Phake; trait SeriesFeed { function setUpSeries() { - $ts = gmdate("Y-m-d H:i:s",strtotime("now - 1 minute")); - $ts2 = gmdate("Y-m-d H:i:s",strtotime("now + 1 minute")); - $ts3 = gmdate("Y-m-d H:i:s",strtotime("now")); + $past = gmdate("Y-m-d H:i:s",strtotime("now - 1 minute")); + $future = gmdate("Y-m-d H:i:s",strtotime("now + 1 minute")); + $now = gmdate("Y-m-d H:i:s",strtotime("now")); $data = [ 'arsse_feeds' => [ 'columns' => [ @@ -25,10 +25,11 @@ trait SeriesFeed { 'next_fetch' => "datetime", ], 'rows' => [ - [1,"http://localhost:8000/Feed/Matching/3","Ook",0,"",$ts,$ts], - [2,"http://localhost:8000/Feed/Matching/1","Eek",5,"There was an error last time",$ts,$ts2], - [3,"http://localhost:8000/Feed/Fetching/Error?code=404","Ack",0,"",$ts,$ts3], - [4,"http://localhost:8000/Feed/NextFetch/NotModified?t=".time(),"Ooook",0,"",$ts,$ts], + [1,"http://localhost:8000/Feed/Matching/3","Ook",0,"",$past,$past], + [2,"http://localhost:8000/Feed/Matching/1","Eek",5,"There was an error last time",$past,$future], + [3,"http://localhost:8000/Feed/Fetching/Error?code=404","Ack",0,"",$past,$now], + [4,"http://localhost:8000/Feed/NextFetch/NotModified?t=".time(),"Ooook",0,"",$past,$past], + [5,"http://localhost:8000/Feed/Parsing/Valid","Ooook",0,"",$past,$future], ] ], 'arsse_articles' => [ @@ -48,11 +49,11 @@ trait SeriesFeed { 'modified' => "datetime", ], 'rows' => [ - [1,1,'http://example.com/1','Article title 1','','2000-01-01 00:00:00','2000-01-01 00:00:00','

Article content 1

','e433653cef2e572eee4215fa299a4a5af9137b2cefd6283c85bd69a32915beda','f5cb8bfc1c7396dc9816af212a3e2ac5221585c2a00bf7ccb6aabd95dcfcd6a6','fb0bc8f8cb08913dc5a497db700e327f1d34e4987402687d494a5891f24714d4','18fdd4fa93d693128c43b004399e5c9cea6c261ddfa002518d3669f55d8c2207',$ts], - [2,1,'http://example.com/2','Article title 2','','2000-01-02 00:00:00','2000-01-02 00:00:00','

Article content 2

','5be8a5a46ecd52ed132191c8d27fb1af6b3d4edc00234c5d9f8f0e10562ed3b7','0e86d2de822a174fe3c44a466953e63ca1f1a58a19cbf475fce0855d4e3d5153','13075894189c47ffcfafd1dfe7fbb539f7c74a69d35a399b3abf8518952714f9','2abd0a8cba83b8214a66c8f0293ba63e467d720540e29ff8ddcdab069d4f1c9e',$ts], - [3,1,'http://example.com/3','Article title 3','','2000-01-03 00:00:00','2000-01-03 00:00:00','

Article content 3

','31a6594500a48b59fcc8a075ce82b946c9c3c782460d088bd7b8ef3ede97ad92','f74b06b240bd08abf4d3fdfc20dba6a6f6eb8b4f1a00e9a617efd63a87180a4b','b278380e984cefe63f0e412b88ffc9cb0befdfa06fdc00bace1da99a8daff406','ad622b31e739cd3a3f3c788991082cf4d2f7a8773773008e75f0572e58cd373b',$ts], - [4,1,'http://example.com/4','Article title 4','','2000-01-04 00:00:00','2000-01-04 00:00:00','

Article content 4

','804e517d623390e71497982c77cf6823180342ebcd2e7d5e32da1e55b09dd180','f3615c7f16336d3ea242d35cf3fc17dbc4ee3afb78376bf49da2dd7a5a25dec8','f11c2b4046f207579aeb9c69a8c20ca5461cef49756ccfa5ba5e2344266da3b3','ab2da63276acce431250b18d3d49b988b226a99c7faadf275c90b751aee05be9',$ts], - [5,1,'http://example.com/5','Article title 5','','2000-01-05 00:00:00','2000-01-05 00:00:00','

Article content 5

','db3e736c2c492f5def5c5da33ddcbea1824040e9ced2142069276b0a6e291a41','d40da96e39eea6c55948ccbe9b3d275b5f931298288dbe953990c5f496097022','834240f84501b5341d375414718204ec421561f3825d34c22bf9182203e42900','43b970ac6ec5f8a9647b2c7e4eed8b1d7f62e154a95eed748b0294c1256764ba',$ts], + [1,1,'http://example.com/1','Article title 1','','2000-01-01 00:00:00','2000-01-01 00:00:00','

Article content 1

','e433653cef2e572eee4215fa299a4a5af9137b2cefd6283c85bd69a32915beda','f5cb8bfc1c7396dc9816af212a3e2ac5221585c2a00bf7ccb6aabd95dcfcd6a6','fb0bc8f8cb08913dc5a497db700e327f1d34e4987402687d494a5891f24714d4','18fdd4fa93d693128c43b004399e5c9cea6c261ddfa002518d3669f55d8c2207',$past], + [2,1,'http://example.com/2','Article title 2','','2000-01-02 00:00:00','2000-01-02 00:00:00','

Article content 2

','5be8a5a46ecd52ed132191c8d27fb1af6b3d4edc00234c5d9f8f0e10562ed3b7','0e86d2de822a174fe3c44a466953e63ca1f1a58a19cbf475fce0855d4e3d5153','13075894189c47ffcfafd1dfe7fbb539f7c74a69d35a399b3abf8518952714f9','2abd0a8cba83b8214a66c8f0293ba63e467d720540e29ff8ddcdab069d4f1c9e',$past], + [3,1,'http://example.com/3','Article title 3','','2000-01-03 00:00:00','2000-01-03 00:00:00','

Article content 3

','31a6594500a48b59fcc8a075ce82b946c9c3c782460d088bd7b8ef3ede97ad92','f74b06b240bd08abf4d3fdfc20dba6a6f6eb8b4f1a00e9a617efd63a87180a4b','b278380e984cefe63f0e412b88ffc9cb0befdfa06fdc00bace1da99a8daff406','ad622b31e739cd3a3f3c788991082cf4d2f7a8773773008e75f0572e58cd373b',$past], + [4,1,'http://example.com/4','Article title 4','','2000-01-04 00:00:00','2000-01-04 00:00:00','

Article content 4

','804e517d623390e71497982c77cf6823180342ebcd2e7d5e32da1e55b09dd180','f3615c7f16336d3ea242d35cf3fc17dbc4ee3afb78376bf49da2dd7a5a25dec8','f11c2b4046f207579aeb9c69a8c20ca5461cef49756ccfa5ba5e2344266da3b3','ab2da63276acce431250b18d3d49b988b226a99c7faadf275c90b751aee05be9',$past], + [5,1,'http://example.com/5','Article title 5','','2000-01-05 00:00:00','2000-01-05 00:00:00','

Article content 5

','db3e736c2c492f5def5c5da33ddcbea1824040e9ced2142069276b0a6e291a41','d40da96e39eea6c55948ccbe9b3d275b5f931298288dbe953990c5f496097022','834240f84501b5341d375414718204ec421561f3825d34c22bf9182203e42900','43b970ac6ec5f8a9647b2c7e4eed8b1d7f62e154a95eed748b0294c1256764ba',$past], ] ], 'arsse_editions' => [ @@ -62,11 +63,11 @@ trait SeriesFeed { 'modified' => "datetime", ], 'rows' => [ - [1,1,$ts], - [2,2,$ts], - [3,3,$ts], - [4,4,$ts], - [5,5,$ts], + [1,1,$past], + [2,2,$past], + [3,3,$past], + [4,4,$past], + [5,5,$past], ] ], 'arsse_marks' => [ @@ -79,14 +80,29 @@ trait SeriesFeed { 'modified' => "datetime", ], 'rows' => [ - [1,1,"jane.doe@example.com",1,0,$ts], - [2,2,"jane.doe@example.com",1,0,$ts], - [3,3,"jane.doe@example.com",1,1,$ts], - [4,4,"jane.doe@example.com",1,0,$ts], - [5,5,"jane.doe@example.com",1,1,$ts], - [9, 1,"john.doe@example.com",1,0,$ts], - [10,3,"john.doe@example.com",1,0,$ts], - [11,4,"john.doe@example.com",0,1,$ts], + [1,1,"jane.doe@example.com",1,0,$past], + [2,2,"jane.doe@example.com",1,0,$past], + [3,3,"jane.doe@example.com",1,1,$past], + [4,4,"jane.doe@example.com",1,0,$past], + [5,5,"jane.doe@example.com",1,1,$past], + [9, 1,"john.doe@example.com",1,0,$past], + [10,3,"john.doe@example.com",1,0,$past], + [11,4,"john.doe@example.com",0,1,$past], + ] + ], + 'arsse_enclosures' => [ + 'columns' => [ + 'url' => "str", + 'type' => "str", + ], + 'rows' => [ + ] + ], + 'arsse_categories' => [ + 'columns' => [ + 'name' => "str", + ], + 'rows' => [ ] ], ]; @@ -99,23 +115,23 @@ trait SeriesFeed { function testUpdateAFeed() { // update a valid feed with both new and changed items Data::$db->feedUpdate(1); - $ts = gmdate("Y-m-d H:i:s"); + $now = gmdate("Y-m-d H:i:s"); $state = $this->primeExpectations($this->data, [ 'arsse_articles' => ["id", "feed","url","title","author","published","edited","content","guid","url_title_hash","url_content_hash","title_content_hash","modified"], 'arsse_editions' => ["id","article","modified"], 'arsse_marks' => ["id","article","read","starred","modified"], ]); - $state['arsse_articles']['rows'][2] = [3,1,'http://example.com/3','Article title 3 (updated)','','2000-01-03 00:00:00','2000-01-03 00:00:00','

Article content 3

','31a6594500a48b59fcc8a075ce82b946c9c3c782460d088bd7b8ef3ede97ad92','6cc99be662ef3486fef35a890123f18d74c29a32d714802d743c5b4ef713315a','b278380e984cefe63f0e412b88ffc9cb0befdfa06fdc00bace1da99a8daff406','d5faccc13bf8267850a1e8e61f95950a0f34167df2c8c58011c0aaa6367026ac',$ts]; - $state['arsse_articles']['rows'][3] = [4,1,'http://example.com/4','Article title 4','','2000-01-04 00:00:00','2000-01-04 00:00:01','

Article content 4

','804e517d623390e71497982c77cf6823180342ebcd2e7d5e32da1e55b09dd180','f3615c7f16336d3ea242d35cf3fc17dbc4ee3afb78376bf49da2dd7a5a25dec8','f11c2b4046f207579aeb9c69a8c20ca5461cef49756ccfa5ba5e2344266da3b3','ab2da63276acce431250b18d3d49b988b226a99c7faadf275c90b751aee05be9',$ts]; - $state['arsse_articles']['rows'][5] = [6,1,'http://example.com/6','Article title 6','','2000-01-06 00:00:00','2000-01-06 00:00:00','

Article content 6

','b3461ab8e8759eeb1d65a818c65051ec00c1dfbbb32a3c8f6999434e3e3b76ab','91d051a8e6749d014506848acd45e959af50bf876427c4f0e3a1ec0f04777b51','211d78b1a040d40d17e747a363cc283f58767b2e502630d8de9b8f1d5e941d18','5ed68ccb64243b8c1931241d2c9276274c3b1d87f223634aa7a1ab0141292ca7',$ts]; + $state['arsse_articles']['rows'][2] = [3,1,'http://example.com/3','Article title 3 (updated)','','2000-01-03 00:00:00','2000-01-03 00:00:00','

Article content 3

','31a6594500a48b59fcc8a075ce82b946c9c3c782460d088bd7b8ef3ede97ad92','6cc99be662ef3486fef35a890123f18d74c29a32d714802d743c5b4ef713315a','b278380e984cefe63f0e412b88ffc9cb0befdfa06fdc00bace1da99a8daff406','d5faccc13bf8267850a1e8e61f95950a0f34167df2c8c58011c0aaa6367026ac',$now]; + $state['arsse_articles']['rows'][3] = [4,1,'http://example.com/4','Article title 4','','2000-01-04 00:00:00','2000-01-04 00:00:01','

Article content 4

','804e517d623390e71497982c77cf6823180342ebcd2e7d5e32da1e55b09dd180','f3615c7f16336d3ea242d35cf3fc17dbc4ee3afb78376bf49da2dd7a5a25dec8','f11c2b4046f207579aeb9c69a8c20ca5461cef49756ccfa5ba5e2344266da3b3','ab2da63276acce431250b18d3d49b988b226a99c7faadf275c90b751aee05be9',$now]; + $state['arsse_articles']['rows'][5] = [6,1,'http://example.com/6','Article title 6','','2000-01-06 00:00:00','2000-01-06 00:00:00','

Article content 6

','b3461ab8e8759eeb1d65a818c65051ec00c1dfbbb32a3c8f6999434e3e3b76ab','91d051a8e6749d014506848acd45e959af50bf876427c4f0e3a1ec0f04777b51','211d78b1a040d40d17e747a363cc283f58767b2e502630d8de9b8f1d5e941d18','5ed68ccb64243b8c1931241d2c9276274c3b1d87f223634aa7a1ab0141292ca7',$now]; $state['arsse_editions']['rows'] = array_merge($state['arsse_editions']['rows'], [ - [6,6,$ts], - [7,3,$ts], - [8,4,$ts], + [6,6,$now], + [7,3,$now], + [8,4,$now], ]); - $state['arsse_marks']['rows'][2] = [3,3,0,1,$ts]; - $state['arsse_marks']['rows'][3] = [4,4,0,0,$ts]; - $state['arsse_marks']['rows'][6] = [10,3,0,0,$ts]; + $state['arsse_marks']['rows'][2] = [3,3,0,1,$now]; + $state['arsse_marks']['rows'][3] = [4,4,0,0,$now]; + $state['arsse_marks']['rows'][6] = [10,3,0,0,$now]; $this->compareExpectations($state); // update a valid feed which previously had an error Data::$db->feedUpdate(2); @@ -132,7 +148,6 @@ trait SeriesFeed { Data::$db->feedUpdate(3); $state['arsse_feeds']['rows'][2] = [3,3,'Feed URL "http://localhost:8000/Feed/Fetching/Error?code=404" is invalid']; $this->compareExpectations($state); - // FIXME: Need to test enclosures } function testUpdateAFeedThrowingExceptions() { @@ -140,6 +155,22 @@ trait SeriesFeed { Data::$db->feedUpdate(3, true); } + function testUpdateAFeedWithEnclosuresAndCategories() { + Data::$db->feedUpdate(5); + $state = $this->primeExpectations($this->data, [ + 'arsse_enclosures' => ["url","type"], + 'arsse_categories' => ["name"], + ]); + $state['arsse_enclosures']['rows'][] = ['http://example.com/text','text/plain']; + $state['arsse_categories']['rows'] = array_merge($state['arsse_categories']['rows'], [ + ["Aniki!"], + ["Beams"], + ["Bodybuilders"], + ["Men"], + ]); + $this->compareExpectations($state); + } + function testListStaleFeeds() { $this->assertSame([1,3,4], Data::$db->feedListStale()); Data::$db->feedUpdate(3);