From 9178d937c4009959dc0b32cc10b349b605efc186 Mon Sep 17 00:00:00 2001 From: sgiehl Date: Fri, 12 Jul 2024 16:53:12 +0200 Subject: [PATCH] Use database for performing check for intersecting periods being processed --- core/CronArchive/QueueConsumer.php | 49 +- core/DataAccess/Model.php | 10 + .../CronArchive/QueueConsumerTest.php | 763 ++++++++++-------- 3 files changed, 475 insertions(+), 347 deletions(-) diff --git a/core/CronArchive/QueueConsumer.php b/core/CronArchive/QueueConsumer.php index a49f82852639..1601622fb0a5 100644 --- a/core/CronArchive/QueueConsumer.php +++ b/core/CronArchive/QueueConsumer.php @@ -291,7 +291,7 @@ public function getNextArchivesToProcess() } $reason = $this->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($invalidatedArchive); - if ($reason) { + if ($reason !== null) { $this->logger->debug("Skipping invalidated archive, $reason: $invalidationDesc"); $invalidationsToExcludeInBatch[$invalidatedArchive['idinvalidation']] = true; $this->addInvalidationToExclude($invalidatedArchive); @@ -430,50 +430,50 @@ public function canSkipArchiveBecauseNoPoint(array $invalidatedArchive) return $loader->canSkipThisArchive(); // if no point in archiving, skip } - public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess) + public function shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(array $archiveToProcess): ?string { - $inProgressArchives = $this->cliMultiRequestParser->getInProgressArchivingCommands(); + $inProgressArchives = $this->model->getInvalidationsInProgress( + (int) $archiveToProcess['idsite'] + ); + + $periods = array_flip(Piwik::$idPeriods); foreach ($inProgressArchives as $archiveBeingProcessed) { - if ( - empty($archiveBeingProcessed['period']) - || empty($archiveBeingProcessed['date']) - ) { - continue; - } + $this->findSegmentForArchive($archiveBeingProcessed); - if ( - empty($archiveBeingProcessed['idSite']) - || $archiveBeingProcessed['idSite'] != $archiveToProcess['idsite'] - ) { + if ($archiveBeingProcessed['idsite'] != $archiveToProcess['idsite']) { continue; // different site } // we don't care about lower periods being concurrent if they are for different segments (that are not "all visits") if ( - !empty($archiveBeingProcessed['segment']) - && !empty($archiveToProcess['segment']) + !empty($archiveBeingProcessed['segment']) && !empty($archiveToProcess['segment']) && $archiveBeingProcessed['segment'] != $archiveToProcess['segment'] - && urldecode($archiveBeingProcessed['segment']) != $archiveToProcess['segment'] ) { continue; } - $archiveBeingProcessed['periodObj'] = PeriodFactory::build($archiveBeingProcessed['period'], $archiveBeingProcessed['date']); + $archiveBeingProcessed['periodObj'] = PeriodFactory::build($periods[$archiveBeingProcessed['period']], $archiveBeingProcessed['date1']); + + if (!$this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) { + continue; + } - if ($this->isArchiveOfLowerPeriod($archiveToProcess, $archiveBeingProcessed)) { - return "lower or same period in progress in another local climulti process (period = {$archiveBeingProcessed['period']}, date = {$archiveBeingProcessed['date']})"; + if (empty($archiveToProcess['segment']) && !empty($archiveBeingProcessed['segment'])) { + return "segment archive in progress for same site with lower or same period ({$archiveBeingProcessed['segment']}, period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})"; } - if ($this->isArchiveNonSegmentAndInProgressArchiveSegment($archiveToProcess, $archiveBeingProcessed)) { - return "segment archive in progress for same site/period ({$archiveBeingProcessed['segment']})"; + if (!empty($archiveToProcess['segment']) && empty($archiveBeingProcessed['segment'])) { + return "all visits archive in progress for same site with lower or same period (period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})"; } + + return "lower or same period in progress (period = {$periods[$archiveBeingProcessed['period']]}, date = {$archiveBeingProcessed['date1']})"; } - return false; + return null; } - private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed) + private function isArchiveOfLowerPeriod(array $archiveToProcess, $archiveBeingProcessed): bool { /** @var Period $archiveToProcessPeriodObj */ $archiveToProcessPeriodObj = $archiveToProcess['periodObj']; @@ -494,8 +494,7 @@ private function isArchiveNonSegmentAndInProgressArchiveSegment(array $archiveTo { // archive is for different site/period if ( - empty($archiveBeingProcessed['idSite']) - || $archiveToProcess['idsite'] != $archiveBeingProcessed['idSite'] + $archiveToProcess['idsite'] != $archiveBeingProcessed['idsite'] || $archiveToProcess['periodObj']->getId() != $archiveBeingProcessed['periodObj']->getId() || $archiveToProcess['periodObj']->getDateStart()->toString() != $archiveBeingProcessed['periodObj']->getDateStart()->toString() ) { diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 8f48a75df1f9..3aab4bb76f84 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -787,6 +787,16 @@ public function isSimilarArchiveInProgress($invalidation) return !empty($result); } + public function getInvalidationsInProgress(int $idSite): array + { + $table = Common::prefixTable('archive_invalidations'); + + $bind = [$idSite, ArchiveInvalidator::INVALIDATION_STATUS_IN_PROGRESS]; + + $sql = "SELECT idsite, period, date1, date2, name FROM `$table` WHERE idsite = ? AND `status` = ? AND ts_started IS NOT NULL"; + return Db::fetchAll($sql, $bind); + } + /** * Gets the next invalidated archive that should be archived in a table. * diff --git a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php index 579c983e14d7..7193b4a8b869 100644 --- a/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php +++ b/tests/PHPUnit/Integration/CronArchive/QueueConsumerTest.php @@ -33,33 +33,31 @@ use Piwik\Log\LoggerInterface; use Piwik\Log\NullLogger; -class MockCronArchive extends CronArchive -{ - public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain($idSiteToInvalidate) - { - // empty (skip in these tests so we don't invalidate today) - } -} - class QueueConsumerTest extends IntegrationTestCase { public function testConsumerIgnoresPeriodsThatHaveBeenDisabledInApi() { Fixture::createWebsite('2015-02-03'); + Fixture::createWebsite('2015-02-03'); + Fixture::createWebsite('2015-02-03'); + Fixture::createWebsite('2015-02-03'); // force archiving so we don't skip those without visits Piwik::addAction('Archiving.getIdSitesToArchiveWhenNoVisits', function (&$idSites) { $idSites[] = 1; + $idSites[] = 2; + $idSites[] = 3; + $idSites[] = 4; }); - $cronArchive = new MockCronArchive(); + $cronArchive = $this->getMockCronArchive(); $cronArchive->init(); $archiveFilter = $this->makeTestArchiveFilter(); $queueConsumer = new QueueConsumer( StaticContainer::get(LoggerInterface::class), - new FixedSiteIds([1]), + new FixedSiteIds([1, 2, 3, 4]), 3, 24, new Model(), @@ -71,13 +69,11 @@ public function testConsumerIgnoresPeriodsThatHaveBeenDisabledInApi() $invalidations = [ ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], - ['idarchive' => 2, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], - ['idarchive' => 3, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], - ['idarchive' => 4, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-01-01', 'date2' => '2018-12-31', 'period' => 4, 'report' => null], + ['idarchive' => 2, 'name' => 'done', 'idsite' => 2, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], + ['idarchive' => 3, 'name' => 'done', 'idsite' => 3, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], + ['idarchive' => 4, 'name' => 'done', 'idsite' => 4, 'date1' => '2018-01-01', 'date2' => '2018-12-31', 'period' => 4, 'report' => null], ]; - shuffle($invalidations); - $this->insertInvalidations($invalidations); Config::getInstance()->General['enabled_periods_API'] = 'day,week,range'; @@ -116,11 +112,12 @@ public function testConsumerIgnoresPeriodsThatHaveBeenDisabledInApi() 'segment' => '', ], ], + [], [ [ 'idarchive' => '2', 'name' => 'done', - 'idsite' => '1', + 'idsite' => '2', 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => '2', @@ -132,6 +129,8 @@ public function testConsumerIgnoresPeriodsThatHaveBeenDisabledInApi() ], ], [], + [], + [], ]; $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations, "Invalidations inserted:\n" . var_export($invalidations, true)); @@ -156,7 +155,7 @@ public function testInvalidateConsumeOrder() $idSites[] = 2; }); - $cronArchive = new MockCronArchive(); + $cronArchive = $this->getMockCronArchive(); $cronArchive->init(); $archiveFilter = $this->makeTestArchiveFilter(); @@ -181,18 +180,18 @@ public function testInvalidateConsumeOrder() ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-07', 'date2' => '2018-03-07', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-08', 'date2' => '2018-03-08', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], - ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], - ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], // intersecting period + ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], // intersecting period - ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => 'testReport'], - ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => 'testReport'], - ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => 'testReport'], + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => 'testReport'], // intersecting period + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => 'testReport'], // intersecting period + ['idarchive' => 1, 'name' => 'done.Actions', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => 'testReport'], // intersecting period // some or all subperiods before site was created ['idarchive' => 1, 'name' => 'done', 'idsite' => 2, 'date1' => '2020-04-04', 'date2' => '2020-04-04', 'period' => 1, 'report' => 'testReport'], - ['idarchive' => 1, 'name' => 'done', 'idsite' => 2, 'date1' => '2020-03-30', 'date2' => '2020-04-05', 'period' => 2, 'report' => 'testReport'], + ['idarchive' => 1, 'name' => 'done', 'idsite' => 2, 'date1' => '2020-03-30', 'date2' => '2020-04-05', 'period' => 2, 'report' => 'testReport'], // intersecting period - // segments + // segments are skipped due to insersecting period with all visits ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-04', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-07', 'date2' => '2018-03-07', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-08', 'date2' => '2018-03-08', 'period' => 1, 'report' => null], @@ -200,17 +199,20 @@ public function testInvalidateConsumeOrder() ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], ['idarchive' => 1, 'name' => 'done' . $segmentHash, 'idsite' => 2, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], + + // removed as segment not configured to auto archive ['idarchive' => 1, 'name' => 'done' . $segmentHash2, 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => null], - // invalid plugin + // removed as invalid plugin ['idarchive' => 1, 'name' => 'done.MyPlugin', 'idsite' => 1, 'date1' => '2018-03-04', 'date2' => '2018-03-11', 'period' => 2, 'report' => 'testReport'], - // duplicates + // removed as duplicates ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-06', 'date2' => '2018-03-06', 'period' => 1, 'report' => null], + // dupliactes not removed as skipped due to intersecting period ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], ['idarchive' => 1, 'name' => 'done', 'idsite' => 1, 'date1' => '2018-03-01', 'date2' => '2018-03-31', 'period' => 3, 'report' => null], @@ -244,8 +246,8 @@ public function testInvalidateConsumeOrder() } $expectedInvalidationsFound = [ - array ( - array ( + [ + [ 'idarchive' => '1', 'idsite' => '1', 'date1' => '2018-03-08', @@ -257,8 +259,8 @@ public function testInvalidateConsumeOrder() 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - array ( + ], + [ 'idarchive' => '1', 'idsite' => '1', 'date1' => '2018-03-07', @@ -270,8 +272,8 @@ public function testInvalidateConsumeOrder() 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - array ( + ], + [ 'idarchive' => '1', 'idsite' => '1', 'date1' => '2018-03-06', @@ -283,64 +285,10 @@ public function testInvalidateConsumeOrder() 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-08', - 'date2' => '2018-03-08', - 'period' => '1', - 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE;dimension1==val', - 'ts_started' => null, - 'status' => '0', - ), - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-07', - 'date2' => '2018-03-07', - 'period' => '1', - 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE;dimension1==val', - 'ts_started' => null, - 'status' => '0', - ), - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-06', - 'date2' => '2018-03-06', - 'period' => '1', - 'name' => 'done.Actions', - 'report' => 'testReport', - 'plugin' => 'Actions', - 'segment' => '', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-06', - 'date2' => '2018-03-06', - 'period' => '1', - 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE;dimension1==val', - 'ts_started' => null, - 'status' => '0', - ), - array ( + ], + ], + [ + [ 'idarchive' => '1', 'idsite' => '1', 'date1' => '2018-03-04', @@ -352,139 +300,150 @@ public function testInvalidateConsumeOrder() 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-04', - 'period' => '1', - 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE;dimension1==val', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( - array ( + ], + ], + [], + [], // end of idsite=1 + [ + [ 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-11', + 'idsite' => '2', + 'date1' => '2020-03-30', + 'date2' => '2020-04-05', 'period' => '2', 'name' => 'done', - 'report' => null, - 'plugin' => null, - 'segment' => '', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-11', - 'period' => '2', - 'name' => 'done.Actions', 'report' => 'testReport', - 'plugin' => 'Actions', + 'plugin' => null, 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( + ], + ], + [ // end of idsite=2 + ], + [ // end of idsite=3 + ], + ]; + + $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations); + + // automated check for no duplicates + $invalidationDescs = []; + foreach ($iteratedInvalidations as $group) { + foreach ($group as $invalidation) { + unset($invalidation['idarchive']); + $invalidationDescs[] = implode('.', $invalidation); + } + } + $uniqueInvalidationDescs = array_unique($invalidationDescs); + + $this->assertEquals($uniqueInvalidationDescs, $invalidationDescs, "Found duplicate archives being processed."); + + // check that segment hash 2 is no longer in the invalidations table + $count = Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archive_invalidations') . ' WHERE name = ?', [ + 'done' . $segmentHash2, + ]); + $this->assertEquals(0, $count); + + // simulate a second run after the first round being finished + Db::query('DELETE FROM ' . Common::prefixTable('archive_invalidations') . ' WHERE status = 1'); + + $queueConsumer = new QueueConsumer( + StaticContainer::get(LoggerInterface::class), + new FixedSiteIds([1,2,3]), + 3, + 24, + new Model(), + new SegmentArchiving(), + $cronArchive, + new RequestParser(true), + $archiveFilter + ); + + $iteratedInvalidations = []; + while (true) { + $next = $queueConsumer->getNextArchivesToProcess(); + if ($next === null) { + break; + } + + foreach ($next as &$item) { + $this->simulateJobStart($item['idinvalidation']); + + unset($item['periodObj']); + unset($item['idinvalidation']); + unset($item['ts_invalidated']); + } + + $iteratedInvalidations[] = $next; + } + + $expectedInvalidationsFound = [ + [ + [ 'idarchive' => '1', 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-11', - 'period' => '2', + 'date1' => '2018-03-08', + 'date2' => '2018-03-08', + 'period' => '1', 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', 'report' => null, 'plugin' => null, 'segment' => 'browserCode==IE;dimension1==val', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( + ], + [ 'idarchive' => '1', 'idsite' => '1', - 'date1' => '2018-03-01', - 'date2' => '2018-03-31', - 'period' => '3', - 'name' => 'done', + 'date1' => '2018-03-07', + 'date2' => '2018-03-07', + 'period' => '1', + 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', 'report' => null, 'plugin' => null, - 'segment' => '', + 'segment' => 'browserCode==IE;dimension1==val', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( + ], + [ 'idarchive' => '1', 'idsite' => '1', - 'date1' => '2018-03-01', - 'date2' => '2018-03-31', - 'period' => '3', + 'date1' => '2018-03-06', + 'date2' => '2018-03-06', + 'period' => '1', 'name' => 'done.Actions', 'report' => 'testReport', 'plugin' => 'Actions', 'segment' => '', 'ts_started' => null, 'status' => '0', - ), - ), - array ( - array ( + ], + ], + [ + [ 'idarchive' => '1', 'idsite' => '1', - 'date1' => '2018-03-01', - 'date2' => '2018-03-31', - 'period' => '3', + 'date1' => '2018-03-04', + 'date2' => '2018-03-04', + 'period' => '1', 'name' => 'donec3afbf588c35606b9cd9ecd1ac781428', 'report' => null, 'plugin' => null, 'segment' => 'browserCode==IE;dimension1==val', 'ts_started' => null, 'status' => '0', - ), - ), - array ( // end of idsite=1 - ), - array ( - array ( - 'idarchive' => '1', - 'idsite' => '2', - 'date1' => '2020-03-30', - 'date2' => '2020-04-05', - 'period' => '2', - 'name' => 'done', - 'report' => 'testReport', - 'plugin' => null, - 'segment' => '', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( // end of idsite=2 - ), - array ( // end of idsite=3 - ), + ], + ], + [], + [], // end of idsite=1 + [], // end of idsite=2 + [], // end of idsite=3 ]; - $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations, "Invalidations inserted:\n" . var_export($invalidations, true)); + $this->assertEquals($expectedInvalidationsFound, $iteratedInvalidations); - // automated ccheck for no duplicates + // automated check for no duplicates $invalidationDescs = []; foreach ($iteratedInvalidations as $group) { foreach ($group as $invalidation) { @@ -495,16 +454,8 @@ public function testInvalidateConsumeOrder() $uniqueInvalidationDescs = array_unique($invalidationDescs); $this->assertEquals($uniqueInvalidationDescs, $invalidationDescs, "Found duplicate archives being processed."); - - // check that segment hash 2 is no longer in the invalidations table - $count = Db::fetchOne('SELECT COUNT(*) FROM ' . Common::prefixTable('archive_invalidations') . ' WHERE name = ?', [ - 'done' . $segmentHash2, - ]); - $this->assertEquals(0, $count); } - - public function testPluginInvalidationDeletedIfUsableArchiveExists() { Fixture::createWebsite('2015-02-03'); @@ -514,7 +465,7 @@ public function testPluginInvalidationDeletedIfUsableArchiveExists() $idSites[] = 1; }); - $cronArchive = new MockCronArchive(); + $cronArchive = $this->getMockCronArchive(); $cronArchive->init(); $archiveFilter = $this->makeTestArchiveFilter(); @@ -595,7 +546,7 @@ public function testSkipSegmentsToday() $idSites[] = 1; }); - $cronArchive = new MockCronArchive(); + $cronArchive = $this->getMockCronArchive(); $cronArchive->init(); $archiveFilter = $this->makeTestArchiveFilter(null, null, null, false, true); @@ -646,52 +597,36 @@ public function testSkipSegmentsToday() } $expectedInvalidationsFound = [ - array ( - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-04', - 'date2' => '2018-03-04', - 'period' => '1', - 'name' => 'done', - 'report' => null, - 'plugin' => null, - 'segment' => '', - 'ts_started' => null, - 'status' => '0', - ), - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-03', - 'date2' => '2018-03-03', - 'period' => '1', - 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE', - 'ts_started' => null, - 'status' => '0', - ), - ), - array ( - 0 => - array ( - 'idarchive' => '1', - 'idsite' => '1', - 'date1' => '2018-03-01', - 'date2' => '2018-03-31', - 'period' => '3', - 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', - 'report' => null, - 'plugin' => null, - 'segment' => 'browserCode==IE', - 'ts_started' => null, - 'status' => '0', - ), - ), - array (// end of idsite=1 - ), + [ + [ + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-04', + 'date2' => '2018-03-04', + 'period' => '1', + 'name' => 'done', + 'report' => null, + 'plugin' => null, + 'segment' => '', + 'ts_started' => null, + 'status' => '0', + ], + [ + 'idarchive' => '1', + 'idsite' => '1', + 'date1' => '2018-03-03', + 'date2' => '2018-03-03', + 'period' => '1', + 'name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', + 'report' => null, + 'plugin' => null, + 'segment' => 'browserCode==IE', + 'ts_started' => null, + 'status' => '0', + ], + ], + [], + [] ]; try { @@ -727,7 +662,7 @@ public function testMaxWebsitesToProcess() $idSites[] = 3; }); - $cronArchive = new MockCronArchive(); + $cronArchive = $this->getMockCronArchive(); $cronArchive->init(); $archiveFilter = $this->makeTestArchiveFilter(); @@ -829,17 +764,19 @@ private function insertInvalidations(array $invalidations) $table = Common::prefixTable('archive_invalidations'); foreach ($invalidations as $inv) { $bind = [ - $inv['idarchive'], + $inv['idarchive'] ?? null, $inv['name'], $inv['idsite'], $inv['date1'], $inv['date2'], $inv['period'], isset($inv['ts_invalidated']) ? $inv['ts_invalidated'] : $now, - $inv['report'], + $inv['report'] ?? null, + $inv['status'] ?? 0, + $inv['ts_started'] ?? null, ]; - Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0)", $bind); + Db::query("INSERT INTO `$table` (idarchive, name, idsite, date1, date2, period, ts_invalidated, report, status, ts_started) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", $bind); } } @@ -1163,112 +1100,294 @@ public function getTestDataForHasIntersectingPeriod() /** * @dataProvider getTestDataForShouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress */ - public function testShouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($cliMultiProcesses, $archiveToProcess, $expected) + public function testShouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($existingInvalidations, $archiveToProcess, $expected) { - $cliRequestProcessor = $this->getMockRequestParser($cliMultiProcesses); + Fixture::createWebsite('2020-01-01 00:00:01'); + Fixture::createWebsite('2020-01-01 00:00:01'); + + Rules::setBrowserTriggerArchiving(false); + API::getInstance()->add('testegment', 'browserCode==IE', 0, true); + API::getInstance()->add('testegment2', 'browserCode==FF', 0, true); + Rules::setBrowserTriggerArchiving(true); + + $this->insertInvalidations($existingInvalidations); + $cliRequestProcessor = $this->getMockRequestParser([]); /** @var QueueConsumer $queueConsumer */ $queueConsumer = $this->getQueueConsumerWithMocks($cliRequestProcessor); $periods = array_flip(Piwik::$idPeriods); - $archiveToProcess['periodObj'] = Factory::build($periods[$archiveToProcess['period']], $archiveToProcess['date']); + $archiveToProcess['periodObj'] = Factory::build($periods[$archiveToProcess['period']], $archiveToProcess['date1']); $actual = $queueConsumer->shouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress($archiveToProcess); - $this->assertEquals($expected, $actual); + $this->assertSame($expected, $actual); } - public function getTestDataForShouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress() + public function getTestDataForShouldSkipArchiveBecauseLowerPeriodOrSegmentIsInProgress(): iterable { - return [ - // test idSite different - [ - [ - ['idSite' => 5, 'date' => '2020-03-04', 'period' => 'day'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1], - false, + yield 'different period and different idSite should not be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 3, 'date1' => '2022-03-04', 'date2' => '2022-03-04', 'period' => 1], + 'expected' => null + ]; - // test no period/date - [ - [ - ['idSite' => 3], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1], - false, + yield 'same period, but different idSite should not be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 3, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => null + ]; - // test same segment - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => 'pageUrl=@%2C'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1, 'segment' => 'pageUrl=@%2C'], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + yield 'same day period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => 'pageUrl=@%252C'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1, 'segment' => 'pageUrl=@%2C'], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + yield 'week period should be detected as intersecting when day is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; - // test different segment - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => 'pageUrl=@%2C'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1, 'segment' => 'pageUrl=@%2Ca'], - false, + yield 'month period should be detected as intersecting when day is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => 'pageUrl=@%252C'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1, 'segment' => 'pageUrl%3D%40%252C'], - false, + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + yield 'year period should be detected as intersecting when day is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; - // test lower periods together - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + yield 'same week period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day'], - ], - ['idsite' => 3, 'date' => '2020-03-01', 'period' => 3], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2], + 'expected' => 'lower or same period in progress (period = week, date = 2020-03-02)' + ]; + + yield 'day period should not be detected as intersecting when week is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], - [ - [ - ['idSite' => 3, 'date' => '2020-03-01', 'period' => 'month'], - ], - ['idsite' => 3, 'date' => '2020-03-01', 'period' => 1], - false, + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => null + ]; + + yield 'month period should be detected as intersecting when week is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3], + 'expected' => 'lower or same period in progress (period = week, date = 2020-03-02)' + ]; - // test segment w/ non-segment - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => ''], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 2, 'segment' => 'pageUrl=@%2C'], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + yield 'year period should be detected as intersecting when week is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], - [ - [ - ['idSite' => 3, 'date' => '2020-03-04', 'period' => 'day', 'segment' => 'pageUrl=@%2C'], - ], - ['idsite' => 3, 'date' => '2020-03-04', 'period' => 1, 'segment' => ''], - 'lower or same period in progress in another local climulti process (period = day, date = 2020-03-04)', + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4], + 'expected' => 'lower or same period in progress (period = week, date = 2020-03-02)' + ]; + + yield 'same month period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3], + 'expected' => 'lower or same period in progress (period = month, date = 2020-03-01)' + ]; + + yield 'day period should not be detected as intersecting when month is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => null + ]; + + yield 'week period should not be detected as intersecting when month is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2], + 'expected' => null + ]; + + yield 'year period should be detected as intersecting when month is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4], + 'expected' => 'lower or same period in progress (period = month, date = 2020-03-01)' + ]; + + yield 'same year period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4], + 'expected' => 'lower or same period in progress (period = year, date = 2020-01-01)' + ]; + + yield 'day period should not be detected as intersecting when year is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => null + ]; + + yield 'week period should not be detected as intersecting when year is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2], + 'expected' => null ]; + + yield 'month period should be detected as intersecting when year is processed' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 5, 'date1' => '2020-01-01', 'date2' => '2020-12-31', 'period' => 4], + 'expected' => 'lower or same period in progress (period = month, date = 2020-03-01)' + ]; + + yield 'plugin and normal invalidation for same day period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done.Actions', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + // @todo is this needed? + yield 'different plugin invalidations for same day period should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done.VisitsSummary', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done.Actions', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + yield 'week period should be detected as intersecting when day is processed for a segment' => [ + 'existingInvalidations' => [ + ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'segment' => 'browserCode==IE'], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + yield 'plugin archive should be detected as intersecting when lower period is processed for a segment' => [ + 'existingInvalidations' => [ + ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 5, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.Actions', 'idsite' => 5, 'date1' => '2020-03-02', 'date2' => '2020-03-08', 'period' => 2, 'segment' => 'browserCode==IE'], + 'expected' => 'lower or same period in progress (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving during "all visits" archiving should be detected as intersecting with same period' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'segment' => 'browserCode==IE'], + 'expected' => 'all visits archive in progress for same site with lower or same period (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving for plugin during "all visits" archiving should be detected as intersecting with same period' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.VisitsSummary', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'segment' => 'browserCode==IE'], + 'expected' => 'all visits archive in progress for same site with lower or same period (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving during "all visits" plugin archiving should be detected as intersecting with same period' => [ + 'existingInvalidations' => [ + ['name' => 'done.VisitsSummary', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'segment' => 'browserCode==IE'], + 'expected' => 'all visits archive in progress for same site with lower or same period (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving for plugin during "all visits" plugin archiving should be detected as intersecting with same period' => [ + 'existingInvalidations' => [ + ['name' => 'done.VisitsSummary', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc.Actions', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'segment' => 'browserCode==IE'], + 'expected' => 'all visits archive in progress for same site with lower or same period (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving during "all visits" archiving should be detected as intersecting with lower period' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3, 'segment' => 'browserCode==IE'], + 'expected' => 'all visits archive in progress for same site with lower or same period (period = day, date = 2020-03-04)' + ]; + + yield 'segment archiving during "all visits" archiving not should be detected as intersecting with different periods' => [ + 'existingInvalidations' => [ + ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-04-01', 'date2' => '2020-04-30', 'period' => 3, 'segment' => 'browserCode==IE'], + 'expected' => null + ]; + + yield '"all visits" archiving, while running a segment should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => 'segment archive in progress for same site with lower or same period (browserCode==IE, period = day, date = 2020-03-04)' + ]; + + yield '"all visits" plugin archiving, while running a segment should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done.VisitsSummary', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1], + 'expected' => 'segment archive in progress for same site with lower or same period (browserCode==IE, period = day, date = 2020-03-04)' + ]; + + yield '"all visits" archiving with bigger period, while running a segment should be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done', 'idsite' => 1, 'date1' => '2020-03-01', 'date2' => '2020-03-31', 'period' => 3], + 'expected' => 'segment archive in progress for same site with lower or same period (browserCode==IE, period = day, date = 2020-03-04)' + ]; + + yield 'same period, but different segments archiving should not be detected as intersecting' => [ + 'existingInvalidations' => [ + ['name' => 'done3736b708e4d20cfc10610e816a1b2341', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'status' => 1, 'ts_started' => date('Y-m-d H:i:s')], + ], + 'archiveToProcess' => ['name' => 'done5f4f9bafeda3443c3c2d4b2ef4dffadc', 'idsite' => 1, 'date1' => '2020-03-04', 'date2' => '2020-03-04', 'period' => 1, 'segment' => 'browserCode==IE'], + 'expected' => null + ]; + } + + private function getMockCronArchive() + { + return $this->getMockBuilder(CronArchive::class) + ->onlyMethods(['invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain']) + ->getMock(); } private function getMockRequestParser($cliMultiProcesses)