Skip to content

Commit 23ca853

Browse files
muetymuety
committed
fix: finish implementing additional real-valued time column for heartbeats and durations tables on sqlite (resolve #882)
1 parent 8515bdf commit 23ca853

File tree

11 files changed

+1076
-699
lines changed

11 files changed

+1076
-699
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,7 @@ You can specify configuration options either via a config file (default: `config
241241

242242
Wakapi uses [GORM](https://gorm.io) as an ORM. As a consequence, a set of different relational databases is supported.
243243

244-
* [SQLite](https://sqlite.org/) (_default, easy setup_)
244+
* [SQLite](https://sqlite.org/) (>= 3.31) (_default, easy setup_)
245245
* [MySQL](https://hub.docker.com/_/mysql) (_recommended, because most extensively tested_)
246246
* [MariaDB](https://hub.docker.com/_/mariadb) (_open-source MySQL alternative_)
247247
* [Postgres](https://hub.docker.com/_/postgres) (_open-source as well_)

‎coverage/coverage.out‎

Lines changed: 735 additions & 633 deletions
Large diffs are not rendered by default.

‎migrations/20260111_sqlite_real_valued_time_column_durations.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
package migrations
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/muety/wakapi/config"
8+
"github.com/muety/wakapi/models"
9+
"gorm.io/gorm"
10+
)
11+
12+
// https://github.com/muety/wakapi/issues/882
13+
14+
func init() {
15+
const name = "20260111-sqlite_real_valued_time_column_durations"
16+
17+
f := migrationFunc{
18+
name: name,
19+
background: false,
20+
f: func(db *gorm.DB, cfg *config.Config) error {
21+
if hasRun(name, db) {
22+
return nil
23+
}
24+
25+
if !cfg.Db.IsSQLite() {
26+
return nil
27+
}
28+
29+
if err := db.Transaction(func(tx *gorm.DB) error {
30+
// drop the indexes to allow them be recreated by next auto-migrate
31+
indexes, err := tx.Migrator().GetIndexes(&models.Duration{})
32+
if err != nil {
33+
return err
34+
}
35+
for _, index := range indexes {
36+
if err := tx.Migrator().DropIndex(&models.Duration{}, index.Name()); err != nil {
37+
return err
38+
}
39+
}
40+
41+
// create identical copy of the durations table
42+
var createDdl string
43+
if err := tx.Raw("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'durations'").Scan(&createDdl).Error; err != nil {
44+
return err
45+
}
46+
createDdl = strings.ToLower(createDdl)
47+
createDdl = strings.Replace(createDdl, "create table \"durations\"", "create table \"durations_new\"", 1)
48+
createDdl = strings.Replace(createDdl, "create table durations", "create table \"durations_new\"", 1)
49+
createDdl = strings.Replace(createDdl, "create table `durations`", "create table \"durations_new\"", 1)
50+
if idx := strings.LastIndex(createDdl, ")"); idx != -1 { // inject new column definition
51+
createDdl = createDdl[:idx] + ", time_real real as (julianday(time)) stored" + createDdl[idx:]
52+
} else {
53+
return fmt.Errorf("could not modify ddl for durations table")
54+
}
55+
if err := tx.Exec(createDdl).Error; err != nil {
56+
return err
57+
}
58+
59+
// copy data dynamically
60+
var columns []string
61+
type colInfo struct{ Name string }
62+
var info []colInfo
63+
if err := tx.Raw("pragma table_info(durations)").Scan(&info).Error; err != nil {
64+
return err
65+
}
66+
for _, c := range info {
67+
columns = append(columns, c.Name)
68+
}
69+
quotedCols := "\"" + strings.Join(columns, "\", \"") + "\""
70+
if err := tx.Exec(fmt.Sprintf("insert into durations_new (%s) select %s from durations", quotedCols, quotedCols)).Error; err != nil {
71+
return err
72+
}
73+
74+
// swap tables
75+
if err := tx.Exec("drop table durations").Error; err != nil {
76+
return err
77+
}
78+
if err := tx.Exec("alter table durations_new rename to durations").Error; err != nil {
79+
return err
80+
}
81+
82+
// add new indexes for real-valued time column
83+
if err := tx.Exec("create index idx_time_real_duration_user on durations(user_id, time_real)").Error; err != nil {
84+
return err
85+
}
86+
87+
// auto-migrate to recreate all other indexes and constraints
88+
if err := tx.Migrator().AutoMigrate(&models.Duration{}); err != nil {
89+
return err
90+
}
91+
92+
return nil
93+
}); err != nil {
94+
return err
95+
}
96+
97+
setHasRun(name, db)
98+
return nil
99+
},
100+
}
101+
102+
registerPostMigration(f)
103+
}

‎migrations/20260111_sqlite_real_valued_time_column_heartbeats.go‎

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package migrations
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
7+
"github.com/muety/wakapi/config"
8+
"github.com/muety/wakapi/models"
9+
"gorm.io/gorm"
10+
)
11+
12+
// https://github.com/muety/wakapi/issues/882
13+
14+
func init() {
15+
const name = "20260111-sqlite_real_valued_time_column_heartbeats"
16+
17+
f := migrationFunc{
18+
name: name,
19+
background: false,
20+
f: func(db *gorm.DB, cfg *config.Config) error {
21+
if hasRun(name, db) {
22+
return nil
23+
}
24+
25+
if !cfg.Db.IsSQLite() {
26+
return nil
27+
}
28+
29+
if err := db.Transaction(func(tx *gorm.DB) error {
30+
// drop view to recreate it later, then referencing the new column
31+
var viewExists int
32+
if err := tx.Raw("select count(*) from sqlite_master where type = 'view' and name = 'user_heartbeats_range';").Scan(&viewExists).
33+
Error; err != nil {
34+
return err
35+
}
36+
if viewExists == 1 {
37+
if err := tx.Migrator().DropView("user_heartbeats_range"); err != nil {
38+
return err
39+
}
40+
}
41+
42+
// drop the indexes to allow them be recreated by next auto-migrate
43+
indexes, err := tx.Migrator().GetIndexes(&models.Heartbeat{})
44+
if err != nil {
45+
return err
46+
}
47+
for _, index := range indexes {
48+
if err := tx.Migrator().DropIndex(&models.Heartbeat{}, index.Name()); err != nil {
49+
return err
50+
}
51+
}
52+
53+
// create identical copy of the heartbeats table
54+
var createDdl string
55+
if err := tx.Raw("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'heartbeats'").Scan(&createDdl).Error; err != nil {
56+
return err
57+
}
58+
createDdl = strings.ToLower(createDdl)
59+
createDdl = strings.Replace(createDdl, "create table \"heartbeats\"", "create table \"heartbeats_new\"", 1)
60+
createDdl = strings.Replace(createDdl, "create table heartbeats", "create table \"heartbeats_new\"", 1)
61+
createDdl = strings.Replace(createDdl, "create table `heartbeats`", "create table \"heartbeats_new\"", 1)
62+
if idx := strings.Index(createDdl, "constraint"); idx != -1 { // inject new column definition
63+
createDdl = createDdl[:idx] + " time_real real as (julianday(time)) stored, " + createDdl[idx:]
64+
} else {
65+
return fmt.Errorf("could not modify ddl for heartbeats table")
66+
}
67+
if err := tx.Exec(createDdl).Error; err != nil {
68+
return err
69+
}
70+
71+
// copy data dynamically
72+
var columns []string
73+
type colInfo struct{ Name string }
74+
var info []colInfo
75+
if err := tx.Raw("pragma table_info(heartbeats)").Scan(&info).Error; err != nil {
76+
return err
77+
}
78+
for _, c := range info {
79+
columns = append(columns, c.Name)
80+
}
81+
quotedCols := "\"" + strings.Join(columns, "\", \"") + "\""
82+
if err := tx.Exec(fmt.Sprintf("insert into heartbeats_new (%s) select %s from heartbeats", quotedCols, quotedCols)).Error; err != nil {
83+
return err
84+
}
85+
86+
// swap tables
87+
if err := tx.Exec("drop table heartbeats").Error; err != nil {
88+
return err
89+
}
90+
if err := tx.Exec("alter table heartbeats_new rename to heartbeats").Error; err != nil {
91+
return err
92+
}
93+
94+
// recreate view, involving new column now
95+
const viewDdl = "select u.id as user_id, concat(datetime(min(h.time_real)), '+00:00') as first, concat(datetime(max(h.time_real)), '+00:00') as last " +
96+
"from users u left join heartbeats h on u.id = h.user_id " +
97+
"group by u.id"
98+
if err := tx.Migrator().CreateView("user_heartbeats_range", gorm.ViewOption{
99+
Query: db.Raw(viewDdl),
100+
Replace: !cfg.Db.IsSQLite(),
101+
}); err != nil {
102+
return err
103+
}
104+
105+
// add new indexes for real-valued time column
106+
if err := tx.Exec("create index idx_time_real on heartbeats(time_real)").Error; err != nil {
107+
return err
108+
}
109+
if err := tx.Exec("create index idx_time_real_user on heartbeats(user_id, time_real)").Error; err != nil {
110+
return err
111+
}
112+
113+
// auto-migrate to recreate all other indexes and constraints
114+
if err := tx.Migrator().AutoMigrate(&models.Heartbeat{}); err != nil {
115+
return err
116+
}
117+
118+
return nil
119+
}); err != nil {
120+
return err
121+
}
122+
123+
setHasRun(name, db)
124+
return nil
125+
},
126+
}
127+
128+
registerPostMigration(f)
129+
}

‎models/duration.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,23 @@ package models
22

33
import (
44
"fmt"
5-
"github.com/cespare/xxhash/v2"
6-
"github.com/gohugoio/hashstructure"
7-
"github.com/muety/wakapi/models/lib"
85
"log/slog"
96
"strings"
107
"time"
118
"unicode"
9+
10+
"github.com/cespare/xxhash/v2"
11+
"github.com/gohugoio/hashstructure"
12+
"github.com/muety/wakapi/models/lib"
1213
)
1314

1415
// TODO: support multiple durations per time per user for different heartbeat timeouts
1516
// see discussion at https://github.com/muety/wakapi/issues/675
1617
type Duration struct {
17-
ID int64 `json:"-" gorm:"primaryKey; autoIncrement"` // https://github.com/muety/wakapi/issues/777
18-
UserID string `json:"user_id" gorm:"not null; index:idx_time_duration_user"`
18+
ID int64 `json:"-" gorm:"primaryKey; autoIncrement"` // https://github.com/muety/wakapi/issues/777
19+
UserID string `json:"user_id" gorm:"not null; index:idx_time_duration_user"`
20+
// note: on sqlite, table will have an additional column `time_real`, introduced "manually" by migration 20260111
21+
// see https://github.com/muety/wakapi/issues/882 for details
1922
Time CustomTime `json:"time" hash:"ignore" gorm:"not null; index:idx_time_duration_user"` // time of first heartbeat of this duration
2023
Duration time.Duration `json:"duration" hash:"ignore" gorm:"not null"`
2124
Project string `json:"project"`

‎models/heartbeat.go‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,30 +2,33 @@ package models
22

33
import (
44
"fmt"
5-
"github.com/cespare/xxhash/v2"
65
"strings"
76
"time"
87

8+
"github.com/cespare/xxhash/v2"
9+
910
"log/slog"
1011

1112
"github.com/gohugoio/hashstructure"
1213
)
1314

1415
type Heartbeat struct {
15-
ID uint64 `json:"-" gorm:"primary_key" hash:"ignore"`
16-
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" hash:"ignore"`
17-
UserID string `json:"-" gorm:"not null; index:idx_time_user; index:idx_user_project"` // idx_user_project is for quickly fetching a user's project list (settings page)
18-
Entity string `json:"entity" gorm:"not null"`
19-
Type string `json:"type" gorm:"size:255"`
20-
Category string `json:"category" gorm:"size:255"`
21-
Project string `json:"project" gorm:"index:idx_project; index:idx_user_project"`
22-
Branch string `json:"branch" gorm:"index:idx_branch"`
23-
Language string `json:"language" gorm:"index:idx_language"`
24-
IsWrite bool `json:"is_write"`
25-
Editor string `json:"editor" gorm:"index:idx_editor" hash:"ignore"` // ignored because editor might be parsed differently by wakatime
26-
OperatingSystem string `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
27-
Machine string `json:"machine" gorm:"index:idx_machine" hash:"ignore"` // ignored because wakatime api doesn't return machines currently
28-
UserAgent string `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
16+
ID uint64 `json:"-" gorm:"primary_key" hash:"ignore"`
17+
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" hash:"ignore"`
18+
UserID string `json:"-" gorm:"not null; index:idx_time_user; index:idx_user_project"` // idx_user_project is for quickly fetching a user's project list (settings page)
19+
Entity string `json:"entity" gorm:"not null"`
20+
Type string `json:"type" gorm:"size:255"`
21+
Category string `json:"category" gorm:"size:255"`
22+
Project string `json:"project" gorm:"index:idx_project; index:idx_user_project"`
23+
Branch string `json:"branch" gorm:"index:idx_branch"`
24+
Language string `json:"language" gorm:"index:idx_language"`
25+
IsWrite bool `json:"is_write"`
26+
Editor string `json:"editor" gorm:"index:idx_editor" hash:"ignore"` // ignored because editor might be parsed differently by wakatime
27+
OperatingSystem string `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
28+
Machine string `json:"machine" gorm:"index:idx_machine" hash:"ignore"` // ignored because wakatime api doesn't return machines currently
29+
UserAgent string `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
30+
// note: on sqlite, table will have an additional column `time_real`, introduced "manually" by migration 20260111
31+
// see https://github.com/muety/wakapi/issues/882 for details
2932
Time CustomTime `json:"time" gorm:"timeScale:3; index:idx_time; index:idx_time_user; not null" swaggertype:"primitive,number"`
3033
Hash string `json:"-" gorm:"type:varchar(17); uniqueIndex"`
3134
Origin string `json:"-" hash:"ignore" gorm:"type:varchar(255)"`

0 commit comments

Comments
 (0)