export/import/count pageleave_visitors

Author: RobertJoonas

lib/plausible/exports.ex

@@ -445,7 +445,8 @@ defmodule Plausible.Exports do
                 "if(isNull(sum(?)), NULL, toUInt64(sum(?)))",
                 p.max_scroll_depth,
                 p.max_scroll_depth
-              )
+              ),
+            pageleave_visitors: count(p.user_id)
           },
           group_by: [:date, :page]
         )
@@ -465,7 +466,8 @@ defmodule Plausible.Exports do
           ),
           visitors(e),
           selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :pageviews),
-          selected_as(fragment("any(?)", s.scroll_depth), :scroll_depth)
+          selected_as(fragment("any(?)", s.scroll_depth), :scroll_depth),
+          selected_as(fragment("any(?)", s.pageleave_visitors), :pageleave_visitors)
         ]
       )
       |> group_by([e], [selected_as(:date), selected_as(:page)])

lib/plausible/imported/csv_importer.ex

@@ -161,7 +161,7 @@ defmodule Plausible.Imported.CSVImporter do
     "imported_operating_systems" =>
       "date Date, operating_system String, operating_system_version String, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
     "imported_pages" =>
-      "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, scroll_depth Nullable(UInt64)",
+      "date Date, hostname String, page String, visits UInt64, visitors UInt64, pageviews UInt64, scroll_depth Nullable(UInt64), pageleave_visitors UInt64",
     "imported_sources" =>
       "date Date, source String, referrer String, utm_source String, utm_medium String, utm_campaign String, utm_content String, utm_term String, pageviews UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32",
     "imported_visitors" =>

lib/plausible/imported/page.ex

@@ -16,5 +16,6 @@ defmodule Plausible.Imported.Page do
     field :exits, Ch, type: "UInt64"
     field :time_on_page, Ch, type: "UInt64"
     field :scroll_depth, Ch, type: "Nullable(UInt64)"
+    field :pageleave_visitors, Ch, type: "UInt64"
   end
 end

lib/plausible/stats/imported/sql/expression.ex

@@ -118,7 +118,7 @@ defmodule Plausible.Stats.Imported.SQL.Expression do
   defp select_metric(:scroll_depth, "imported_pages") do
     wrap_alias([i], %{
       scroll_depth_sum: sum(i.scroll_depth),
-      total_visitors: fragment("sumIf(?, isNotNull(?))", i.visitors, i.scroll_depth)
+      total_visitors: sum(i.pageleave_visitors)
     })
   end
 

test/plausible/imported/csv_importer_test.exs

@@ -1190,14 +1190,19 @@ defmodule Plausible.Imported.CSVImporterTest do
           select: %{
             date: i.date,
             page: i.page,
-            scroll_depth: i.scroll_depth
+            scroll_depth: i.scroll_depth,
+            pageleave_visitors: i.pageleave_visitors
           }
         )
         |> Plausible.IngestRepo.all()
 
-      assert %{date: ~D[2020-01-01], page: "/another", scroll_depth: 50} in imported_data
-      assert %{date: ~D[2020-01-01], page: "/blog", scroll_depth: 180} in imported_data
-      assert %{date: ~D[2020-01-02], page: "/blog", scroll_depth: nil} in imported_data
+      assert %{date: ~D[2020-01-01], page: "/", scroll_depth: 20, pageleave_visitors: 1} in imported_data
+
+      assert %{date: ~D[2020-01-01], page: "/another", scroll_depth: 50, pageleave_visitors: 2} in imported_data
+
+      assert %{date: ~D[2020-01-01], page: "/blog", scroll_depth: 180, pageleave_visitors: 3} in imported_data
+
+      assert %{date: ~D[2020-01-02], page: "/blog", scroll_depth: nil, pageleave_visitors: 0} in imported_data
 
       # assert via stats queries that scroll_depth from imported
       # data matches the scroll_depth from native data

test/plausible_web/controllers/api/stats_controller/main_graph_test.exs

@@ -648,9 +648,21 @@ defmodule PlausibleWeb.Api.StatsController.MainGraphTest do
         # 2020-01-02 - both imported and native data
         build(:pageview, user_id: 56, timestamp: ~N[2020-01-02 00:00:00]),
         build(:pageleave, user_id: 56, timestamp: ~N[2020-01-02 00:01:00], scroll_depth: 20),
-        build(:imported_pages, date: ~D[2020-01-02], page: "/", visitors: 1, scroll_depth: 40),
+        build(:imported_pages,
+          date: ~D[2020-01-02],
+          page: "/",
+          visitors: 1,
+          scroll_depth: 40,
+          pageleave_visitors: 1
+        ),
         # 2020-01-03 - only imported data
-        build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 1, scroll_depth: 90),
+        build(:imported_pages,
+          date: ~D[2020-01-03],
+          page: "/",
+          visitors: 1,
+          scroll_depth: 90,
+          pageleave_visitors: 1
+        ),
         build(:imported_pages, date: ~D[2020-01-03], page: "/", visitors: 100, scroll_depth: nil)
       ])
 

test/plausible_web/controllers/api/stats_controller/pages_test.exs

@@ -666,7 +666,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           pageviews: 3,
           time_on_page: 90,
           page: "/blog",
-          scroll_depth: 120
+          scroll_depth: 120,
+          pageleave_visitors: 3
         )
       ])
 
@@ -717,19 +718,21 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
         ),
         build(:imported_pages,
           date: ~D[2020-01-01],
-          visitors: 3,
-          pageviews: 3,
+          visitors: 4,
+          pageviews: 4,
           time_on_page: 180,
           page: "/native-and-imported",
-          scroll_depth: 120
+          scroll_depth: 120,
+          pageleave_visitors: 3
         ),
         build(:imported_pages,
           date: ~D[2020-01-01],
-          visitors: 10,
-          pageviews: 10,
+          visitors: 20,
+          pageviews: 30,
           time_on_page: 300,
           page: "/imported-only",
-          scroll_depth: 100
+          scroll_depth: 100,
+          pageleave_visitors: 10
         )
       ])
 
@@ -742,10 +745,10 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
       assert json_response(conn, 200)["results"] == [
                %{
                  "name" => "/native-and-imported",
-                 "visitors" => 4,
-                 "pageviews" => 4,
+                 "visitors" => 5,
+                 "pageviews" => 5,
                  "bounce_rate" => 0,
-                 "time_on_page" => 60,
+                 "time_on_page" => 48,
                  "scroll_depth" => 50
                },
                %{
@@ -758,10 +761,10 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
                },
                %{
                  "name" => "/imported-only",
-                 "visitors" => 10,
-                 "pageviews" => 10,
+                 "visitors" => 20,
+                 "pageviews" => 30,
                  "bounce_rate" => 0,
-                 "time_on_page" => 30.0,
+                 "time_on_page" => 10.0,
                  "scroll_depth" => 10
                }
              ]
@@ -778,7 +781,8 @@ defmodule PlausibleWeb.Api.StatsController.PagesTest do
           visitors: 10,
           pageviews: 10,
           page: "/blog",
-          scroll_depth: 100
+          scroll_depth: 100,
+          pageleave_visitors: 10
         ),
         build(:imported_pages,
           date: ~D[2020-01-01],

test/plausible_web/controllers/api/stats_controller/top_stats_test.exs

@@ -1000,7 +1000,13 @@ defmodule PlausibleWeb.Api.StatsController.TopStatsTest do
         build(:pageleave, user_id: 123, timestamp: ~N[2021-01-01 00:00:20], scroll_depth: 60),
         build(:pageview, user_id: 456, timestamp: ~N[2021-01-01 00:00:00]),
         build(:pageleave, user_id: 456, timestamp: ~N[2021-01-01 00:00:10], scroll_depth: 80),
-        build(:imported_pages, page: "/", date: ~D[2021-01-01], visitors: 8, scroll_depth: 410),
+        build(:imported_pages,
+          page: "/",
+          date: ~D[2021-01-01],
+          visitors: 8,
+          scroll_depth: 410,
+          pageleave_visitors: 8
+        ),
         build(:imported_pages, page: "/", date: ~D[2021-01-02], visitors: 100, scroll_depth: nil)
       ])