Skip to content

Commit 2dd91d5

Browse files
niheavenniheaven
authored
feat(sqlite): Use SQLite for caching apps to speed up local search (#5851)
1 parent 105e416 commit 2dd91d5

File tree

10 files changed

+494
-64
lines changed

10 files changed

+494
-64
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ test/installer/tmp/*
66
test/tmp/*
77
*~
88
TestResults.xml
9+
supporting/sqlite/*

‎CHANGELOG.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
## [Unreleased](https://github.com/ScoopInstaller/Scoop/compare/master...develop)
2+
3+
### Features
4+
5+
- **scoop-search:** Use SQLite for caching apps to speed up local search ([#5851](https://github.com/ScoopInstaller/Scoop/issues/5851))
6+
17
## [v0.4.0](https://github.com/ScoopInstaller/Scoop/compare/v0.3.1...v0.4.0) - 2024-04-18
28

39
### Features

‎lib/core.ps1

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,13 @@ function Complete-ConfigChange {
216216
}
217217
}
218218
}
219+
220+
if ($Name -eq 'use_sqlite_cache' -and $Value -eq $true) {
221+
. "$PSScriptRoot\..\lib\database.ps1"
222+
. "$PSScriptRoot\..\lib\manifest.ps1"
223+
info 'Initializing SQLite cache in progress... This may take a while, please wait.'
224+
Set-ScoopDB
225+
}
219226
}
220227

221228
function setup_proxy() {

‎lib/database.ps1

Lines changed: 359 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
# Description: Functions for interacting with the Scoop database cache
2+
3+
<#
4+
.SYNOPSIS
5+
Get SQLite .NET driver
6+
.DESCRIPTION
7+
Download and extract the SQLite .NET driver from NuGet.
8+
.PARAMETER Version
9+
System.String
10+
The version of the SQLite .NET driver to download.
11+
.INPUTS
12+
None
13+
.OUTPUTS
14+
System.Boolean
15+
True if the SQLite .NET driver was successfully downloaded and extracted, otherwise false.
16+
#>
17+
function Get-SQLite {
18+
param (
19+
[string]$Version = '1.0.118'
20+
)
21+
# Install SQLite
22+
try {
23+
Write-Host "Downloading SQLite $Version..." -ForegroundColor DarkYellow
24+
$sqlitePkgPath = "$env:TEMP\sqlite.nupkg"
25+
$sqliteTempPath = "$env:TEMP\sqlite"
26+
$sqlitePath = "$PSScriptRoot\..\supporting\sqlite"
27+
Invoke-WebRequest -Uri "https://api.nuget.org/v3-flatcontainer/stub.system.data.sqlite.core.netframework/$version/stub.system.data.sqlite.core.netframework.$version.nupkg" -OutFile $sqlitePkgPath
28+
Write-Host "Extracting SQLite $Version..." -ForegroundColor DarkYellow -NoNewline
29+
Expand-Archive -Path $sqlitePkgPath -DestinationPath $sqliteTempPath -Force
30+
New-Item -Path $sqlitePath -ItemType Directory -Force | Out-Null
31+
Move-Item -Path "$sqliteTempPath\build\net45\*" -Destination $sqlitePath -Exclude '*.targets' -Force
32+
Move-Item -Path "$sqliteTempPath\lib\net45\System.Data.SQLite.dll" -Destination $sqlitePath -Force
33+
Remove-Item -Path $sqlitePkgPath, $sqliteTempPath -Recurse -Force
34+
Write-Host ' Done' -ForegroundColor DarkYellow
35+
return $true
36+
} catch {
37+
return $false
38+
}
39+
}
40+
41+
<#
42+
.SYNOPSIS
43+
Close a SQLite database.
44+
.DESCRIPTION
45+
Close a SQLite database connection.
46+
.PARAMETER InputObject
47+
System.Data.SQLite.SQLiteConnection
48+
The SQLite database connection to close.
49+
.INPUTS
50+
System.Data.SQLite.SQLiteConnection
51+
.OUTPUTS
52+
None
53+
#>
54+
function Close-ScoopDB {
55+
[CmdletBinding()]
56+
param (
57+
[Parameter(Mandatory, ValueFromPipeline)]
58+
[System.Data.SQLite.SQLiteConnection]
59+
$InputObject
60+
)
61+
process {
62+
$InputObject.Dispose()
63+
}
64+
}
65+
66+
<#
67+
.SYNOPSIS
68+
Create a new SQLite database.
69+
.DESCRIPTION
70+
Create a new SQLite database connection and create the necessary tables.
71+
.PARAMETER PassThru
72+
System.Management.Automation.SwitchParameter
73+
Return the SQLite database connection.
74+
.INPUTS
75+
None
76+
.OUTPUTS
77+
None
78+
Default
79+
80+
System.Data.SQLite.SQLiteConnection
81+
The SQLite database connection if **PassThru** is used.
82+
#>
83+
function New-ScoopDB ([switch]$PassThru) {
84+
# Load System.Data.SQLite
85+
if (!('System.Data.SQLite.SQLiteConnection' -as [Type])) {
86+
try {
87+
if (!(Test-Path -Path "$PSScriptRoot\..\supporting\sqlite\System.Data.SQLite.dll")) {
88+
Get-SQLite | Out-Null
89+
}
90+
Add-Type -Path "$PSScriptRoot\..\supporting\sqlite\System.Data.SQLite.dll"
91+
} catch {
92+
throw "Scoop's Database cache requires the ADO.NET driver:`n`thttp://system.data.sqlite.org/index.html/doc/trunk/www/downloads.wiki"
93+
}
94+
}
95+
$dbPath = Join-Path $scoopdir 'scoop.db'
96+
$db = New-Object -TypeName System.Data.SQLite.SQLiteConnection
97+
$db.ConnectionString = "Data Source=$dbPath"
98+
$db.ParseViaFramework = $true # Allow UNC path
99+
$db.Open()
100+
$tableCommand = $db.CreateCommand()
101+
$tableCommand.CommandText = "CREATE TABLE IF NOT EXISTS 'app' (
102+
name TEXT NOT NULL COLLATE NOCASE,
103+
description TEXT NOT NULL,
104+
version TEXT NOT NULL,
105+
bucket VARCHAR NOT NULL,
106+
manifest JSON NOT NULL,
107+
binary TEXT,
108+
shortcut TEXT,
109+
dependency TEXT,
110+
suggest TEXT,
111+
PRIMARY KEY (name, version, bucket)
112+
)"
113+
$tableCommand.ExecuteNonQuery() | Out-Null
114+
$tableCommand.Dispose()
115+
if ($PassThru) {
116+
return $db
117+
} else {
118+
$db.Dispose()
119+
}
120+
}
121+
122+
<#
123+
.SYNOPSIS
124+
Set Scoop database item(s).
125+
.DESCRIPTION
126+
Insert or replace Scoop database item(s) into the database.
127+
.PARAMETER InputObject
128+
System.Object[]
129+
The database item(s) to insert or replace.
130+
.INPUTS
131+
System.Object[]
132+
.OUTPUTS
133+
None
134+
#>
135+
function Set-ScoopDBItem {
136+
[CmdletBinding()]
137+
param (
138+
[Parameter(Mandatory, Position = 0, ValueFromPipeline)]
139+
[psobject[]]
140+
$InputObject
141+
)
142+
143+
begin {
144+
$db = New-ScoopDB -PassThru
145+
$dbTrans = $db.BeginTransaction()
146+
# TODO Support [hashtable]$InputObject
147+
$colName = @($InputObject | Get-Member -MemberType NoteProperty | Select-Object -ExpandProperty Name)
148+
$dbQuery = "INSERT OR REPLACE INTO app ($($colName -join ', ')) VALUES ($('@' + ($colName -join ', @')))"
149+
$dbCommand = $db.CreateCommand()
150+
$dbCommand.CommandText = $dbQuery
151+
}
152+
process {
153+
foreach ($item in $InputObject) {
154+
$item.PSObject.Properties | ForEach-Object {
155+
$dbCommand.Parameters.AddWithValue("@$($_.Name)", $_.Value) | Out-Null
156+
}
157+
$dbCommand.ExecuteNonQuery() | Out-Null
158+
}
159+
}
160+
end {
161+
try {
162+
$dbTrans.Commit()
163+
} catch {
164+
$dbTrans.Rollback()
165+
throw $_
166+
} finally {
167+
$db.Dispose()
168+
}
169+
}
170+
}
171+
172+
<#
173+
.SYNOPSIS
174+
Set Scoop app database item(s).
175+
.DESCRIPTION
176+
Insert or replace Scoop app(s) into the database.
177+
.PARAMETER Path
178+
System.String
179+
The path to the bucket.
180+
.PARAMETER CommitHash
181+
System.String
182+
The commit hash to compare with the HEAD.
183+
.INPUTS
184+
None
185+
.OUTPUTS
186+
None
187+
#>
188+
function Set-ScoopDB {
189+
[CmdletBinding()]
190+
param (
191+
[Parameter(Position = 0, ValueFromPipeline)]
192+
[string[]]
193+
$Path
194+
)
195+
196+
begin {
197+
$list = [System.Collections.Generic.List[PSCustomObject]]::new()
198+
$arch = Get-DefaultArchitecture
199+
}
200+
process {
201+
if ($Path.Count -eq 0) {
202+
$bucketPath = Get-LocalBucket | ForEach-Object { Join-Path $bucketsdir $_ }
203+
$Path = (Get-ChildItem $bucketPath -Filter '*.json' -Recurse).FullName
204+
}
205+
$Path | ForEach-Object {
206+
$manifestRaw = [System.IO.File]::ReadAllText($_)
207+
$manifest = $manifestRaw | ConvertFrom-Json -ErrorAction Continue
208+
if ($null -ne $manifest.version) {
209+
$list.Add([pscustomobject]@{
210+
name = $($_ -replace '.*[\\/]([^\\/]+)\.json$', '$1')
211+
description = if ($manifest.description) { $manifest.description } else { '' }
212+
version = $manifest.version
213+
bucket = $($_ -replace '.*buckets[\\/]([^\\/]+)(?:[\\/].*)', '$1')
214+
manifest = $manifestRaw
215+
binary = $(
216+
$result = @()
217+
@(arch_specific 'bin' $manifest $arch) | ForEach-Object {
218+
if ($_ -is [System.Array]) {
219+
$result += "$($_[1]).$($_[0].Split('.')[-1])"
220+
} else {
221+
$result += $_
222+
}
223+
}
224+
$result -replace '.*?([^\\/]+)?(\.(exe|bat|cmd|ps1|jar|py))$', '$1' -join ' | '
225+
)
226+
shortcut = $(
227+
$result = @()
228+
@(arch_specific 'shortcuts' $manifest $arch) | ForEach-Object {
229+
$result += $_[1]
230+
}
231+
$result -replace '.*?([^\\/]+$)', '$1' -join ' | '
232+
)
233+
dependency = $manifest.depends -join ' | '
234+
suggest = $(
235+
$suggest_output = @()
236+
$manifest.suggest.PSObject.Properties | ForEach-Object {
237+
$suggest_output += $_.Value -join ' | '
238+
}
239+
$suggest_output -join ' | '
240+
)
241+
})
242+
}
243+
}
244+
}
245+
end {
246+
if ($list.Count -ne 0) {
247+
Set-ScoopDBItem $list
248+
}
249+
}
250+
}
251+
252+
<#
253+
.SYNOPSIS
254+
Select Scoop database item(s).
255+
.DESCRIPTION
256+
Select Scoop database item(s) from the database.
257+
The pattern is matched against the name, binaries, and shortcuts columns for apps.
258+
.PARAMETER Pattern
259+
System.String
260+
The pattern to search for. If is an empty string, all items will be returned.
261+
.INPUTS
262+
System.String
263+
.OUTPUTS
264+
System.Data.DataTable
265+
The selected database item(s).
266+
#>
267+
function Select-ScoopDBItem {
268+
[CmdletBinding()]
269+
param (
270+
[Parameter(Mandatory, Position = 0, ValueFromPipeline)]
271+
[AllowEmptyString()]
272+
[string]
273+
$Pattern,
274+
[Parameter(Mandatory, Position = 1)]
275+
[string[]]
276+
$From
277+
)
278+
279+
begin {
280+
$db = New-ScoopDB -PassThru
281+
$dbAdapter = New-Object -TypeName System.Data.SQLite.SQLiteDataAdapter
282+
$result = New-Object System.Data.DataTable
283+
$dbQuery = "SELECT * FROM app WHERE $(($From -join ' LIKE @Pattern OR ') + ' LIKE @Pattern')"
284+
$dbQuery = "SELECT * FROM ($($dbQuery + ' ORDER BY version DESC')) GROUP BY name, bucket"
285+
$dbCommand = $db.CreateCommand()
286+
$dbCommand.CommandText = $dbQuery
287+
$dbCommand.CommandType = [System.Data.CommandType]::Text
288+
}
289+
process {
290+
$dbCommand.Parameters.AddWithValue('@Pattern', $(if ($Pattern -eq '') { '%' } else { '%' + $Pattern + '%' })) | Out-Null
291+
$dbAdapter.SelectCommand = $dbCommand
292+
[void]$dbAdapter.Fill($result)
293+
}
294+
end {
295+
$db.Dispose()
296+
return $result
297+
}
298+
}
299+
300+
<#
301+
.SYNOPSIS
302+
Get Scoop database item.
303+
.DESCRIPTION
304+
Get Scoop database item from the database.
305+
.PARAMETER Name
306+
System.String
307+
The name of the item to get.
308+
.PARAMETER Bucket
309+
System.String
310+
The bucket of the item to get.
311+
.PARAMETER Version
312+
System.String
313+
The version of the item to get. If not provided, the latest version will be returned.
314+
.INPUTS
315+
System.String
316+
.OUTPUTS
317+
System.Data.DataTable
318+
The selected database item.
319+
#>
320+
function Get-ScoopDBItem {
321+
[CmdletBinding()]
322+
param (
323+
[Parameter(Mandatory, Position = 0, ValueFromPipeline)]
324+
[string]
325+
$Name,
326+
[Parameter(Mandatory, Position = 1)]
327+
[string]
328+
$Bucket,
329+
[Parameter(Position = 2)]
330+
[string]
331+
$Version
332+
)
333+
334+
begin {
335+
$db = New-ScoopDB -PassThru
336+
$dbAdapter = New-Object -TypeName System.Data.SQLite.SQLiteDataAdapter
337+
$result = New-Object System.Data.DataTable
338+
$dbQuery = 'SELECT * FROM app WHERE name = @Name AND bucket = @Bucket'
339+
if ($Version) {
340+
$dbQuery += ' AND version = @Version'
341+
} else {
342+
$dbQuery += ' ORDER BY version DESC LIMIT 1'
343+
}
344+
$dbCommand = $db.CreateCommand()
345+
$dbCommand.CommandText = $dbQuery
346+
$dbCommand.CommandType = [System.Data.CommandType]::Text
347+
}
348+
process {
349+
$dbCommand.Parameters.AddWithValue('@Name', $Name) | Out-Null
350+
$dbCommand.Parameters.AddWithValue('@Bucket', $Bucket) | Out-Null
351+
$dbCommand.Parameters.AddWithValue('@Version', $Version) | Out-Null
352+
$dbAdapter.SelectCommand = $dbCommand
353+
[void]$dbAdapter.Fill($result)
354+
}
355+
end {
356+
$db.Dispose()
357+
return $result
358+
}
359+
}

0 commit comments

Comments
 (0)