Skip to content

Commit 54e3a08

Browse files
hdonnayhdonnay
committed
libvuln: port to jsonblob's iterator interface
This also adds a test that can be used to profile the import process. Signed-off-by: Hank Donnay <hdonnay@redhat.com> Change-Id: I387d9289aad128724e57b9cebd213a3c6a6a6964
1 parent b66ea48 commit 54e3a08

2 files changed

Lines changed: 199 additions & 30 deletions

File tree

‎libvuln/offlineimport_test.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package libvuln
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"io"
8+
"log/slog"
9+
"os"
10+
"testing"
11+
12+
"github.com/google/uuid"
13+
"github.com/jackc/pgx/v5/pgxpool"
14+
"github.com/klauspost/compress/zstd"
15+
16+
"github.com/quay/claircore/datastore/postgres"
17+
"github.com/quay/claircore/libvuln/driver"
18+
"github.com/quay/claircore/libvuln/jsonblob"
19+
"github.com/quay/claircore/test"
20+
"github.com/quay/claircore/test/integration"
21+
pgtest "github.com/quay/claircore/test/postgres"
22+
)
23+
24+
var importFile *string
25+
26+
func init() {
27+
flag.Func(`load-file`, "run the integration test reading from `FILE` (must be zstd compressed)", func(v string) error {
28+
importFile = &v
29+
return nil
30+
})
31+
}
32+
33+
func TestMain(m *testing.M) {
34+
var c int
35+
defer func() { os.Exit(c) }()
36+
defer integration.DBSetup()()
37+
c = m.Run()
38+
}
39+
40+
// TestLiveOfflineImport is meant to be used for profiling and testing the
41+
// system on a "real" export as produced by `clairctl`.
42+
func TestOfflineImport(t *testing.T) {
43+
if importFile == nil {
44+
t.Skip(`needed flag "-load-file" not provided`)
45+
}
46+
integration.NeedDB(t)
47+
48+
t.Run("Old", testOneOfflineImport(oldOfflineImport))
49+
t.Run("New", testOneOfflineImport(OfflineImport))
50+
}
51+
52+
func testOneOfflineImport(inner func(context.Context, *pgxpool.Pool, io.Reader) error) func(*testing.T) {
53+
return func(t *testing.T) {
54+
ctx := test.Logging(t)
55+
56+
f, err := os.Open(*importFile)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
defer f.Close()
61+
zr, err := zstd.NewReader(f)
62+
if err != nil {
63+
t.Fatal(err)
64+
}
65+
defer zr.Close()
66+
67+
pool := pgtest.TestMatcherDB(ctx, t)
68+
if err := inner(ctx, pool, zr); err != nil {
69+
t.Error(err)
70+
}
71+
}
72+
}
73+
74+
// OldOfflineImport is a copy of the previous implementation of [OfflineImport],
75+
// kept here for the above test.
76+
func oldOfflineImport(ctx context.Context, pool *pgxpool.Pool, in io.Reader) error {
77+
s := postgres.NewMatcherStore(pool)
78+
l, err := jsonblob.NewLoader(ctx, in)
79+
if err != nil {
80+
return err
81+
}
82+
83+
ops, err := s.GetUpdateOperations(ctx, driver.VulnerabilityKind)
84+
if err != nil {
85+
return err
86+
}
87+
88+
Update:
89+
for l.Next() {
90+
e := l.Entry()
91+
log := slog.With("updater", e.Updater)
92+
for _, op := range ops[e.Updater] {
93+
// This only helps if updaters don't keep something that
94+
// changes in the fingerprint.
95+
if op.Fingerprint == e.Fingerprint {
96+
log.InfoContext(ctx, "fingerprint match, skipping")
97+
continue Update
98+
}
99+
}
100+
var ref uuid.UUID
101+
if e.Enrichment != nil {
102+
if ref, err = s.UpdateEnrichments(ctx, e.Updater, e.Fingerprint, e.Enrichment); err != nil {
103+
return fmt.Errorf("updating enrichements: %w", err)
104+
}
105+
}
106+
if e.Vuln != nil {
107+
if ref, err = s.UpdateVulnerabilities(ctx, e.Updater, e.Fingerprint, e.Vuln); err != nil {
108+
return fmt.Errorf("updating vulnerabilities: %w", err)
109+
}
110+
}
111+
log.InfoContext(ctx, "update imported",
112+
"ref", ref,
113+
"vuln_count", len(e.Vuln),
114+
"enrichment_count", len(e.Enrichment))
115+
}
116+
if err := l.Err(); err != nil {
117+
return err
118+
}
119+
return nil
120+
}

‎libvuln/updates.go‎

Lines changed: 79 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -2,67 +2,116 @@ package libvuln
22

33
import (
44
"context"
5-
"fmt"
5+
"errors"
66
"io"
77
"log/slog"
8+
"slices"
89

910
"github.com/google/uuid"
1011
"github.com/jackc/pgx/v5/pgxpool"
1112

13+
"github.com/quay/claircore"
1214
"github.com/quay/claircore/datastore/postgres"
1315
"github.com/quay/claircore/libvuln/driver"
1416
"github.com/quay/claircore/libvuln/jsonblob"
1517
)
1618

19+
// BUG(hank) The OfflineImport function is a wart, needed to work around
20+
// some package namespacing issues. It should get refactored if claircore
21+
// gets merged into clair.
22+
1723
// OfflineImport takes the format written into the io.Writer provided to
1824
// NewOfflineUpdater and imports the contents into the provided pgxpool.Pool.
1925
//
2026
// The format provided on "in" should be the same output from [jsonblob.Store], with
2127
// any compression undone.
2228
func OfflineImport(ctx context.Context, pool *pgxpool.Pool, in io.Reader) error {
23-
// BUG(hank) The OfflineImport function is a wart, needed to work around
24-
// some package namespacing issues. It should get refactored if claircore
25-
// gets merged into clair.
2629
s := postgres.NewMatcherStore(pool)
27-
l, err := jsonblob.Load(ctx, in)
30+
ld, err := jsonblob.NewLoader(ctx, in)
2831
if err != nil {
2932
return err
3033
}
31-
32-
ops, err := s.GetUpdateOperations(ctx, driver.VulnerabilityKind)
34+
vulns, err := s.GetUpdateOperations(ctx, driver.VulnerabilityKind)
35+
if err != nil {
36+
return err
37+
}
38+
enrichers, err := s.GetUpdateOperations(ctx, driver.EnrichmentKind)
3339
if err != nil {
3440
return err
3541
}
3642

37-
Update:
38-
for l.Next() {
39-
e := l.Entry()
40-
log := slog.With("updater", e.Updater)
41-
for _, op := range ops[e.Updater] {
42-
// This only helps if updaters don't keep something that
43-
// changes in the fingerprint.
44-
if op.Fingerprint == e.Fingerprint {
45-
log.InfoContext(ctx, "fingerprint match, skipping")
46-
continue Update
47-
}
43+
for e, seq := range ld.All() {
44+
l := slog.With("updater", e.Updater)
45+
var check []driver.UpdateOperation
46+
switch e.Kind {
47+
case driver.VulnerabilityKind:
48+
check = vulns[e.Updater]
49+
case driver.EnrichmentKind:
50+
check = enrichers[e.Updater]
4851
}
52+
seen := slices.ContainsFunc(check, func(op driver.UpdateOperation) bool {
53+
return op.Fingerprint == e.Fingerprint
54+
})
55+
if seen {
56+
l.InfoContext(ctx, "fingerprint match, skipping")
57+
continue
58+
}
59+
60+
l.InfoContext(ctx, "new update")
4961
var ref uuid.UUID
50-
if e.Enrichment != nil {
51-
if ref, err = s.UpdateEnrichments(ctx, e.Updater, e.Fingerprint, e.Enrichment); err != nil {
52-
return fmt.Errorf("updating enrichements: %w", err)
62+
var vulnCt, enrichCt int
63+
switch e.Kind {
64+
case driver.VulnerabilityKind:
65+
wrap := func(yield func(*claircore.Vulnerability, error) bool) {
66+
for v := range seq {
67+
if v == nil {
68+
if !yield(nil, ld.Err()) {
69+
return
70+
}
71+
continue
72+
}
73+
vulnCt++
74+
if !yield(v, nil) {
75+
return
76+
}
77+
jsonblob.ReturnVulnerability(v)
78+
}
5379
}
54-
}
55-
if e.Vuln != nil {
56-
if ref, err = s.UpdateVulnerabilities(ctx, e.Updater, e.Fingerprint, e.Vuln); err != nil {
57-
return fmt.Errorf("updating vulnerabilities: %w", err)
80+
ref, err = s.UpdateVulnerabilitiesIter(ctx, e.Updater, e.Fingerprint, wrap)
81+
case driver.EnrichmentKind:
82+
wrap := func(yield func(*driver.EnrichmentRecord, error) bool) {
83+
for _, e := range seq {
84+
if e == nil {
85+
if !yield(nil, ld.Err()) {
86+
return
87+
}
88+
continue
89+
}
90+
enrichCt++
91+
if !yield(e, nil) {
92+
return
93+
}
94+
jsonblob.ReturnEnrichment(e)
95+
}
5896
}
97+
ref, err = s.UpdateEnrichmentsIter(ctx, e.Updater, e.Fingerprint, wrap)
98+
default:
99+
panic("unreachable")
100+
}
101+
if err == nil {
102+
l.InfoContext(
103+
ctx, "update imported",
104+
"ref", ref,
105+
"vuln_count", vulnCt,
106+
"enrichment_count", enrichCt,
107+
)
108+
} else {
109+
l.InfoContext(ctx, "update failed", "reason", err)
110+
break
59111
}
60-
log.InfoContext(ctx, "update imported",
61-
"ref", ref,
62-
"vuln_count", len(e.Vuln),
63-
"enrichment_count", len(e.Enrichment))
64112
}
65-
if err := l.Err(); err != nil {
113+
114+
if err := errors.Join(err, ld.Err()); err != nil {
66115
return err
67116
}
68117
return nil

0 commit comments

Comments
 (0)