Use rollup queries in custom dimension reports

core/DataAccess/LogAggregator.php

@@ -339,12 +339,13 @@ private function createTemporaryTable($unprefixedSegmentTableName, $segmentSelec
      * @param             $orderBy
      * @param int         $limit
      * @param int         $offset
+     * @param bool        $withRollup
      *
      * @return array|mixed|string
      * @throws \Piwik\Exception\DI\DependencyException
      * @throws \Piwik\Exception\DI\NotFoundException
      */
-    public function generateQuery($select, $from, $where, $groupBy, $orderBy, $limit = 0, $offset = 0)
+    public function generateQuery($select, $from, $where, $groupBy, $orderBy, $limit = 0, $offset = 0, bool $withRollup = false)
     {
         $segment = $this->segment;
         $bind = $this->getGeneralQueryBindParams();
@@ -393,7 +394,7 @@ public function generateQuery($select, $from, $where, $groupBy, $orderBy, $limit
             }
         }
 
-        $query = $segment->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy, $limit, $offset);
+        $query = $segment->getSelectQuery($select, $from, $where, $bind, $orderBy, $groupBy, $limit, $offset, $forceGroupBy = false, $withRollup);
 
         if (is_array($query) && array_key_exists('sql', $query)) {
             $query['sql'] = DbHelper::addOriginHintToQuery($query['sql'], $this->queryOriginHint, $this->dateStart, $this->dateEnd, $this->sites, $this->segment);

core/DataAccess/LogQueryBuilder.php

@@ -57,7 +57,8 @@ public function getSelectQueryString(
         $bind,
         $groupBy,
         $orderBy,
-        $limitAndOffset
+        $limitAndOffset,
+        bool $withRollup = false
     ) {
         if (!is_array($from)) {
             $from = array($from);
@@ -96,7 +97,7 @@ public function getSelectQueryString(
         } elseif ($joinWithSubSelect) {
             $sql = $this->buildWrappedSelectQuery($select, $from, $where, $groupBy, $orderBy, $limitAndOffset, $tables);
         } else {
-            $sql = $this->buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limitAndOffset);
+            $sql = $this->buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limitAndOffset, $withRollup);
         }
         return array(
             'sql' => $sql,
@@ -238,7 +239,7 @@ private function buildWrappedSelectQuery($select, $from, $where, $groupBy, $orde
      * @param string|int $limitAndOffset limit by clause eg '5' for Limit 5 Offset 0 or '10, 5' for Limit 5 Offset 10
      * @return string
      */
-    private function buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limitAndOffset)
+    private function buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $limitAndOffset, bool $withRollup = false)
     {
         $sql = "
 			SELECT
@@ -256,9 +257,21 @@ private function buildSelectQuery($select, $from, $where, $groupBy, $orderBy, $l
             $sql .= "
 			GROUP BY
 				$groupBy";
+
+            if ($withRollup) {
+                $sql .= "
+				WITH ROLLUP";
+            }
         }
 
         if ($orderBy) {
+            if ($withRollup) {
+                $sql = "
+			SELECT * FROM (
+				$sql
+			) rollupQuery";
+            }
+
             $sql .= "
 			ORDER BY
 				$orderBy";

core/RankingQuery.php

@@ -284,18 +284,28 @@ private function splitPartitions(&$data)
      *                            itself.
      * @return string             The entire ranking query SQL.
      */
-    public function generateRankingQuery($innerQuery)
+    public function generateRankingQuery($innerQuery, bool $withRollup = false)
     {
         // +1 to include "Others"
         $limit = $this->limit + 1;
-        $counterExpression = $this->getCounterExpression($limit);
+        $counterExpression = $this->getCounterExpression($limit, $withRollup);
 
         // generate select clauses for label columns
         $labelColumnsString = '`' . implode('`, `', array_keys($this->labelColumns)) . '`';
-        $labelColumnsOthersSwitch = array();
+        $labelColumnsOthersSwitch = [];
+        $withRollupColumns = [];
+
         foreach ($this->labelColumns as $column => $true) {
+            $rollupWhen = '';
+
+            if ($withRollup) {
+                $rollupWhen = "WHEN counterRollup > 0 THEN `$column`";
+                $withRollupColumns[] = $column;
+            }
+
             $labelColumnsOthersSwitch[] = "
 				CASE
+					$rollupWhen
 					WHEN counter = $limit THEN '" . $this->othersLabelValue . "'
 					ELSE `$column`
 				END AS `$column`
@@ -325,6 +335,26 @@ public function generateRankingQuery($innerQuery)
             $initCounter = '( SELECT @counter:=0 ) initCounter,';
         }
 
+        $counterRollupExpression = '';
+
+        if ([] !== $withRollupColumns) {
+            $initCounter .= ' ( SELECT @counterRollup:=0 ) initCounterRollup,';
+            $counterRollupWhen = '';
+
+            foreach ($withRollupColumns as $withRollupColumn) {
+                $counterRollupWhen .= "
+					WHEN `$withRollupColumn` IS NULL THEN @counterRollup := @counterRollup + 1";
+            }
+
+            $counterRollupExpression = "
+				,
+				CASE
+					$counterRollupWhen
+					ELSE 0
+				END AS counterRollup
+            ";
+        }
+
         if (false === strpos(' LIMIT ', $innerQuery) && !Schema::getInstance()->supportsSortingInSubquery()) {
             // Setting a limit for the inner query forces the optimizer to use a temporary table, which uses the sorting
             $innerQuery .= ' LIMIT 18446744073709551615';
@@ -336,6 +366,7 @@ public function generateRankingQuery($innerQuery)
 			SELECT
 				$labelColumnsString,
 				$counterExpression AS counter
+				$counterRollupExpression
 				$additionalColumnsString
 			FROM
 				$initCounter
@@ -344,9 +375,15 @@ public function generateRankingQuery($innerQuery)
 
         // group by the counter - this groups "Others" because the counter stops at $limit
         $groupBy = 'counter';
+
+        if ('' !== $counterRollupExpression) {
+            $groupBy .= ', counterRollup';
+        }
+
         if ($this->partitionColumn !== false) {
             $groupBy .= ', `' . $this->partitionColumn . '`';
         }
+
         $groupOthers = "
 			SELECT
 				$labelColumnsOthersSwitch
@@ -363,7 +400,7 @@ public function generateRankingQuery($innerQuery)
         return $groupOthers;
     }
 
-    private function getCounterExpression($limit)
+    private function getCounterExpression($limit, bool $withRollup = false)
     {
         $whens = array();
 
@@ -375,6 +412,12 @@ private function getCounterExpression($limit)
             $whens[] = "WHEN {$this->columnToMarkExcludedRows} != 0 THEN -1 * {$this->columnToMarkExcludedRows}";
         }
 
+        if ($withRollup) {
+            foreach ($this->labelColumns as $rollupColumn) {
+                $whens[] = "WHEN `$rollupColumn` IS NULL THEN -1";
+            }
+        }
+
         if ($this->partitionColumn !== false) {
             // partition: one counter per possible value
             foreach ($this->partitionColumnValues as $value) {

core/Segment.php

@@ -584,7 +584,7 @@ public static function getSegmentHash($definition)
      *              Use only when you're not aggregating or it will sample the data.
      * @return array The entire select query.
      */
-    public function getSelectQuery($select, $from, $where = false, $bind = array(), $orderBy = false, $groupBy = false, $limit = 0, $offset = 0, $forceGroupBy = false)
+    public function getSelectQuery($select, $from, $where = false, $bind = array(), $orderBy = false, $groupBy = false, $limit = 0, $offset = 0, $forceGroupBy = false, bool $withRollup = false)
     {
         $segmentExpression = $this->segmentExpression;
 
@@ -605,7 +605,8 @@ public function getSelectQuery($select, $from, $where = false, $bind = array(),
                 $bind,
                 $groupBy,
                 $orderBy,
-                $limitAndOffset
+                $limitAndOffset,
+                $withRollup
             );
         } catch (Exception $e) {
             if ($forceGroupBy && $groupBy) {

plugins/CustomDimensions/RecordBuilders/CustomDimension.php

@@ -180,31 +180,62 @@ protected function aggregateFromActions(DataTable $report, LogAggregator $logAgg
         $metricIds[] = Metrics::INDEX_BOUNCE_COUNT;
         $metricIds[] = Metrics::INDEX_PAGE_EXIT_NB_VISITS;
 
+        $actionRows = [];
+
         while ($row = $resultSet->fetch()) {
             if (!isset($row[Metrics::INDEX_NB_VISITS])) {
                 return;
             }
 
             $label = $row[$valueField];
-            $label = $this->cleanCustomDimensionValue($label);
+            $url = $row['url'];
+
+            if (null === $label) {
+                continue;
+            }
+
+            if (null !== $url) {
+                $actionRows[] = $row;
+                continue;
+            }
 
             $columns = [];
+
             foreach ($metricIds as $id) {
                 $columns[$id] = (float) ($row[$id] ?? 0);
             }
 
-            $tableRow = $report->sumRowWithLabel($label, $columns);
+            $label = $this->cleanCustomDimensionValue($label);
+
+            $report->sumRowWithLabel($label, $columns);
+        }
+
+        foreach ($actionRows as $row) {
+            if (!isset($row[Metrics::INDEX_NB_VISITS])) {
+                return;
+            }
 
+            $label = $row[$valueField];
             $url = $row['url'];
-            if (empty($url)) {
+
+            if (null === $label || null === $url) {
                 continue;
             }
 
+            $columns = [];
+
+            foreach ($metricIds as $id) {
+                $columns[$id] = (float) ($row[$id] ?? 0);
+            }
+
+            $label = $this->cleanCustomDimensionValue($label);
+            $tableRow = $report->getRowFromLabel($label);
+
             // make sure we always work with normalized URL no matter how the individual action stores it
             $normalized = Tracker\PageUrl::normalizeUrl($url);
             $url = $normalized['url'];
 
-            if (empty($url)) {
+            if (empty($url) || empty($tableRow)) {
                 continue;
             }
 
@@ -215,7 +246,7 @@ protected function aggregateFromActions(DataTable $report, LogAggregator $logAgg
     public function queryCustomDimensionActions(array $metricsConfig, LogAggregator $logAggregator, $valueField, $additionalWhere = '')
     {
         $select = "log_link_visit_action.$valueField,
-                  log_action.name as url,
+                  COALESCE(log_action.name, '') as url,
                   sum(log_link_visit_action.time_spent) as `" . Metrics::INDEX_PAGE_SUM_TIME_SPENT . "`,
                   sum(case log_visit.visit_total_actions when 1 then 1 when 0 then 1 else 0 end) as `" . Metrics::INDEX_BOUNCE_COUNT . "`,
                   sum(IF(log_visit.last_idlink_va = log_link_visit_action.idlink_va, 1, 0)) as `" . Metrics::INDEX_PAGE_EXIT_NB_VISITS . "`";
@@ -245,7 +276,16 @@ public function queryCustomDimensionActions(array $metricsConfig, LogAggregator
         $orderBy = "`" . Metrics::INDEX_PAGE_NB_HITS . "` DESC";
 
         // get query with segmentation
-        $query     = $logAggregator->generateQuery($select, $from, $where, $groupBy, $orderBy);
+        $query     = $logAggregator->generateQuery(
+            $select,
+            $from,
+            $where,
+            $groupBy,
+            $orderBy,
+            $limit = 0,
+            $offset = 0,
+            $withRollup = true
+        );
 
         if ($this->rankingQueryLimit > 0) {
             $rankingQuery = new RankingQuery($this->rankingQueryLimit);
@@ -267,7 +307,7 @@ public function queryCustomDimensionActions(array $metricsConfig, LogAggregator
                 $rankingQuery->addColumn($column, $config['aggregation']);
             }
 
-            $query['sql'] = $rankingQuery->generateRankingQuery($query['sql']);
+            $query['sql'] = $rankingQuery->generateRankingQuery($query['sql'], $withRollup = true);
         }
 
         $db        = $logAggregator->getDb();
@@ -289,7 +329,7 @@ private function getRankingQueryLimit(): int
 
     protected function cleanCustomDimensionValue(string $value): string
     {
-        if (isset($value) && strlen($value)) {
+        if ('' !== $value) {
             return $value;
         }
 

plugins/CustomDimensions/tests/Fixtures/TrackVisitsWithCustomDimensionsFixture.php

@@ -127,6 +127,10 @@ protected function trackFirstVisit()
         $t->setUrl('http://example.com/sub_en/page?param=en_US');
         self::checkResponse($t->doTrackPageView('Third page view'));
 
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.4)->getDatetime());
+        $t->setUrl('http://example.com/sub_en/page?param=en_US');
+        self::checkResponse($t->doTrackPageView('Fourth page view'));
+
         $t->setForceVisitDateTime(Date::factory($this->dateTime)->addDay(0.4)->getDatetime());
         $t->setUrl('http://example.com/sub_en/page?param=en_US');
         self::checkResponse($t->doTrackPageView('Fourth page view'));