Skip to content

Commit 571e0d3

Browse files
hdonnayhdonnay
committed
libvuln: port to jsonblob's iterator interface
This also adds a test that can be used to profile the import process. Signed-off-by: Hank Donnay <hdonnay@redhat.com> Change-Id: I387d9289aad128724e57b9cebd213a3c6a6a6964
1 parent 6d9a28d commit 571e0d3

2 files changed

Lines changed: 170 additions & 33 deletions

File tree

‎libvuln/offlineimport_test.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package libvuln
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"io"
8+
"log/slog"
9+
"os"
10+
"testing"
11+
12+
"github.com/google/uuid"
13+
"github.com/jackc/pgx/v5/pgxpool"
14+
"github.com/klauspost/compress/zstd"
15+
16+
"github.com/quay/claircore/datastore/postgres"
17+
"github.com/quay/claircore/libvuln/driver"
18+
"github.com/quay/claircore/libvuln/jsonblob"
19+
"github.com/quay/claircore/test"
20+
"github.com/quay/claircore/test/integration"
21+
pgtest "github.com/quay/claircore/test/postgres"
22+
)
23+
24+
var importFile *string
25+
26+
func init() {
27+
flag.Func(`load-file`, "run the integration test reading from `FILE` (must be zstd compressed)", func(v string) error {
28+
importFile = &v
29+
return nil
30+
})
31+
}
32+
33+
func TestMain(m *testing.M) {
34+
var c int
35+
defer func() { os.Exit(c) }()
36+
defer integration.DBSetup()()
37+
c = m.Run()
38+
}
39+
40+
// TestLiveOfflineImport is meant to be used for profiling and testing the
41+
// system on a "real" export as produced by `clairctl`.
42+
func TestOfflineImport(t *testing.T) {
43+
if importFile == nil {
44+
t.Skip(`needed flag "-load-export" not provided`)
45+
}
46+
integration.NeedDB(t)
47+
48+
t.Run("Old", testOneOfflineImport(oldOfflineImport))
49+
t.Run("New", testOneOfflineImport(OfflineImport))
50+
}
51+
52+
func testOneOfflineImport(inner func(context.Context, *pgxpool.Pool, io.Reader) error) func(*testing.T) {
53+
return func(t *testing.T) {
54+
ctx := test.Logging(t)
55+
56+
f, err := os.Open(*importFile)
57+
if err != nil {
58+
t.Fatal(err)
59+
}
60+
defer f.Close()
61+
zr, err := zstd.NewReader(f)
62+
if err != nil {
63+
t.Fatal(err)
64+
}
65+
defer zr.Close()
66+
67+
pool := pgtest.TestMatcherDB(ctx, t)
68+
if err := inner(ctx, pool, zr); err != nil {
69+
t.Error(err)
70+
}
71+
}
72+
}
73+
74+
// OldOfflineImport is a copy of the previous implementation of [OfflineImport],
75+
// kept here for the above test.
76+
func oldOfflineImport(ctx context.Context, pool *pgxpool.Pool, in io.Reader) error {
77+
s := postgres.NewMatcherStore(pool)
78+
l, err := jsonblob.NewLoader(ctx, in)
79+
if err != nil {
80+
return err
81+
}
82+
83+
ops, err := s.GetUpdateOperations(ctx, driver.VulnerabilityKind)
84+
if err != nil {
85+
return err
86+
}
87+
88+
Update:
89+
for l.Next() {
90+
e := l.Entry()
91+
log := slog.With("updater", e.Updater)
92+
for _, op := range ops[e.Updater] {
93+
// This only helps if updaters don't keep something that
94+
// changes in the fingerprint.
95+
if op.Fingerprint == e.Fingerprint {
96+
log.InfoContext(ctx, "fingerprint match, skipping")
97+
continue Update
98+
}
99+
}
100+
var ref uuid.UUID
101+
if e.Enrichment != nil {
102+
if ref, err = s.UpdateEnrichments(ctx, e.Updater, e.Fingerprint, e.Enrichment); err != nil {
103+
return fmt.Errorf("updating enrichements: %w", err)
104+
}
105+
}
106+
if e.Vuln != nil {
107+
if ref, err = s.UpdateVulnerabilities(ctx, e.Updater, e.Fingerprint, e.Vuln); err != nil {
108+
return fmt.Errorf("updating vulnerabilities: %w", err)
109+
}
110+
}
111+
log.InfoContext(ctx, "update imported",
112+
"ref", ref,
113+
"vuln_count", len(e.Vuln),
114+
"enrichment_count", len(e.Enrichment))
115+
}
116+
if err := l.Err(); err != nil {
117+
return err
118+
}
119+
return nil
120+
}

‎libvuln/updates.go‎

Lines changed: 50 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -2,67 +2,84 @@ package libvuln
22

33
import (
44
"context"
5-
"fmt"
5+
"errors"
66
"io"
77
"log/slog"
88

99
"github.com/google/uuid"
1010
"github.com/jackc/pgx/v5/pgxpool"
1111

12+
"github.com/quay/claircore"
1213
"github.com/quay/claircore/datastore/postgres"
1314
"github.com/quay/claircore/libvuln/driver"
1415
"github.com/quay/claircore/libvuln/jsonblob"
1516
)
1617

18+
// BUG(hank) The OfflineImport function is a wart, needed to work around
19+
// some package namespacing issues. It should get refactored if claircore
20+
// gets merged into clair.
21+
1722
// OfflineImport takes the format written into the io.Writer provided to
1823
// NewOfflineUpdater and imports the contents into the provided pgxpool.Pool.
1924
//
2025
// The format provided on "in" should be the same output from [jsonblob.Store], with
2126
// any compression undone.
2227
func OfflineImport(ctx context.Context, pool *pgxpool.Pool, in io.Reader) error {
23-
// BUG(hank) The OfflineImport function is a wart, needed to work around
24-
// some package namespacing issues. It should get refactored if claircore
25-
// gets merged into clair.
2628
s := postgres.NewMatcherStore(pool)
27-
l, err := jsonblob.Load(ctx, in)
29+
l, err := jsonblob.NewLoader(ctx, in)
2830
if err != nil {
2931
return err
3032
}
3133

32-
ops, err := s.GetUpdateOperations(ctx, driver.VulnerabilityKind)
33-
if err != nil {
34-
return err
35-
}
34+
// Don't bother checking the fingerprint, just gogogo.
35+
36+
for e, seq := range l.All() {
37+
l := slog.With("updater", e.Updater)
38+
l.InfoContext(ctx, "new update")
3639

37-
Update:
38-
for l.Next() {
39-
e := l.Entry()
40-
log := slog.With("updater", e.Updater)
41-
for _, op := range ops[e.Updater] {
42-
// This only helps if updaters don't keep something that
43-
// changes in the fingerprint.
44-
if op.Fingerprint == e.Fingerprint {
45-
log.InfoContext(ctx, "fingerprint match, skipping")
46-
continue Update
47-
}
48-
}
4940
var ref uuid.UUID
50-
if e.Enrichment != nil {
51-
if ref, err = s.UpdateEnrichments(ctx, e.Updater, e.Fingerprint, e.Enrichment); err != nil {
52-
return fmt.Errorf("updating enrichements: %w", err)
41+
var err error
42+
var vulnCt, enrichCt int
43+
switch e.Kind {
44+
case driver.VulnerabilityKind:
45+
wrap := func(yield func(*claircore.Vulnerability, error) bool) {
46+
for v := range seq {
47+
vulnCt++
48+
if !yield(v, nil) {
49+
return
50+
}
51+
jsonblob.ReturnVulnerability(v)
52+
}
5353
}
54-
}
55-
if e.Vuln != nil {
56-
if ref, err = s.UpdateVulnerabilities(ctx, e.Updater, e.Fingerprint, e.Vuln); err != nil {
57-
return fmt.Errorf("updating vulnerabilities: %w", err)
54+
ref, err = s.UpdateVulnerabilitiesIter(ctx, e.Updater, e.Fingerprint, wrap)
55+
case driver.EnrichmentKind:
56+
wrap := func(yield func(*driver.EnrichmentRecord, error) bool) {
57+
for _, e := range seq {
58+
enrichCt++
59+
if !yield(e, nil) {
60+
return
61+
}
62+
jsonblob.ReturnEnrichment(e)
63+
}
5864
}
65+
ref, err = s.UpdateEnrichmentsIter(ctx, e.Updater, e.Fingerprint, wrap)
66+
default:
67+
panic("unreachable")
68+
}
69+
if err == nil {
70+
l.InfoContext(
71+
ctx, "update imported",
72+
"ref", ref,
73+
"vuln_count", len(e.Vuln),
74+
"enrichment_count", len(e.Enrichment),
75+
)
76+
} else {
77+
l.InfoContext(ctx, "update failed", "reason", err)
78+
break
5979
}
60-
log.InfoContext(ctx, "update imported",
61-
"ref", ref,
62-
"vuln_count", len(e.Vuln),
63-
"enrichment_count", len(e.Enrichment))
6480
}
65-
if err := l.Err(); err != nil {
81+
82+
if err := errors.Join(err, l.Err()); err != nil {
6683
return err
6784
}
6885
return nil

0 commit comments

Comments
 (0)