Skip to content

Commit f22bf6f

Browse files
FirstyearFirstyear
committed
Improve memory use by shrinking arrays during operations
1 parent 55342a1 commit f22bf6f

File tree

3 files changed

+52
-16
lines changed

3 files changed

+52
-16
lines changed

‎Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[package]
22
name = "idlset"
3-
version = "0.2.4"
3+
version = "0.2.5"
44
authors = ["William Brown <[email protected]>"]
55
edition = "2021"
66

‎src/v1.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ impl IDLBitRange {
201201
let r = self.list.binary_search(&candidate);
202202
match r {
203203
Ok(idx) => {
204-
let mut existing = self.list.get_mut(idx).unwrap();
204+
let existing = self.list.get_mut(idx).unwrap();
205205
existing.mask |= candidate.mask;
206206
}
207207
Err(idx) => {
@@ -227,7 +227,7 @@ impl IDLBitRange {
227227
//
228228
// To do this, we not the candidate, so all other bits remain,
229229
// then we perform and &= so that the existing bits survive.
230-
let mut existing = self.list.get_mut(idx).unwrap();
230+
let existing = self.list.get_mut(idx).unwrap();
231231

232232
existing.mask &= !candidate.mask;
233233

@@ -252,7 +252,7 @@ impl IDLBitRange {
252252

253253
// Get the highest IDLRange out:
254254
if let Some(last) = self.list.last_mut() {
255-
if (*last).range == range {
255+
if last.range == range {
256256
// Insert the bit.
257257
(*last).push_id(bvalue);
258258
return;

‎src/v2.rs

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,13 @@ enum IDLState {
8585
}
8686

8787
impl IDLState {
88+
fn shrink_to_fit(&mut self) {
89+
match self {
90+
IDLState::Sparse(svec) => svec.shrink_to_fit(),
91+
IDLState::Compressed(vec) => vec.shrink_to_fit(),
92+
}
93+
}
94+
8895
fn sparse_bitand_fast_path(smol: &[u64], lrg: &[u64]) -> Self {
8996
let mut nlist = SmallVec::with_capacity(smol.len());
9097
// We cache the idx inbetween to narrow the bst sizes.
@@ -99,6 +106,8 @@ impl IDLState {
99106
idx_min = idx;
100107
}
101108
});
109+
110+
nlist.shrink_to_fit();
102111
IDLState::Sparse(nlist)
103112
}
104113

@@ -120,6 +129,7 @@ impl IDLState {
120129
}
121130
});
122131

132+
nlist.shrink_to_fit();
123133
IDLState::Sparse(nlist)
124134
}
125135
}
@@ -201,7 +211,7 @@ impl Default for IDLBitRange {
201211
/// Construct a new, empty set.
202212
fn default() -> Self {
203213
IDLBitRange {
204-
state: IDLState::Sparse(SmallVec::new()),
214+
state: IDLState::Sparse(SmallVec::with_capacity(0)),
205215
}
206216
}
207217
}
@@ -334,8 +344,8 @@ impl IDLBitRange {
334344
let range: u64 = id - bvalue;
335345

336346
if let Some(last) = list.last_mut() {
337-
debug_assert!(id >= (*last).range);
338-
if (*last).range == range {
347+
debug_assert!(id >= last.range);
348+
if last.range == range {
339349
// Insert the bit.
340350
(*last).push_id(bvalue);
341351
return;
@@ -366,7 +376,7 @@ impl IDLBitRange {
366376
let r = list.binary_search(&candidate);
367377
match r {
368378
Ok(idx) => {
369-
let mut existing = list.get_mut(idx).unwrap();
379+
let existing = list.get_mut(idx).unwrap();
370380
existing.mask |= candidate.mask;
371381
}
372382
Err(idx) => {
@@ -402,7 +412,7 @@ impl IDLBitRange {
402412
//
403413
// To do this, we not the candidate, so all other bits remain,
404414
// then we perform and &= so that the existing bits survive.
405-
let mut existing = list.get_mut(idx).unwrap();
415+
let existing = list.get_mut(idx).unwrap();
406416

407417
existing.mask &= !candidate.mask;
408418

@@ -422,7 +432,7 @@ impl IDLBitRange {
422432
if self.is_compressed() {
423433
return;
424434
}
425-
let mut prev_state = IDLState::Compressed(Vec::new());
435+
let mut prev_state = IDLState::Compressed(Vec::with_capacity(0));
426436
std::mem::swap(&mut prev_state, &mut self.state);
427437
match prev_state {
428438
IDLState::Sparse(list) => list.into_iter().for_each(|i| unsafe {
@@ -441,14 +451,15 @@ impl IDLBitRange {
441451
None
442452
} else {
443453
let mut maybe = IDLBitRange {
444-
state: IDLState::Compressed(Vec::new()),
454+
state: IDLState::Compressed(Vec::with_capacity(0)),
445455
};
446456
list.iter().for_each(|id| unsafe { maybe.push_id(*id) });
447457

448458
if maybe.len_ranges() > 0
449459
&& (maybe.len() / maybe.len_ranges()) >= AVG_RANGE_COMP_REQ
450460
{
451-
let IDLBitRange { state } = maybe;
461+
let IDLBitRange { mut state } = maybe;
462+
state.shrink_to_fit();
452463
Some(state)
453464
} else {
454465
None
@@ -507,6 +518,9 @@ impl IDLBitRange {
507518
}
508519
}
509520
}
521+
522+
nlist.shrink_to_fit();
523+
510524
IDLState::Sparse(nlist)
511525
};
512526

@@ -540,12 +554,14 @@ impl IDLBitRange {
540554
}
541555
});
542556

557+
nlist.shrink_to_fit();
558+
543559
IDLBitRange {
544560
state: IDLState::Sparse(nlist),
545561
}
546562
}
547563
(IDLState::Compressed(list1), IDLState::Compressed(list2)) => {
548-
let mut nlist = Vec::new();
564+
let mut nlist = Vec::with_capacity(0);
549565
let mut liter = list1.iter();
550566
let mut riter = list2.iter();
551567

@@ -577,6 +593,7 @@ impl IDLBitRange {
577593
if nlist.is_empty() {
578594
IDLBitRange::new()
579595
} else {
596+
nlist.shrink_to_fit();
580597
IDLBitRange {
581598
state: IDLState::Compressed(nlist),
582599
}
@@ -636,6 +653,8 @@ impl IDLBitRange {
636653
rnext = riter.next();
637654
}
638655

656+
nlist.shrink_to_fit();
657+
639658
IDLState::Sparse(nlist)
640659
};
641660

@@ -672,7 +691,7 @@ impl IDLBitRange {
672691
Ok(idx) => {
673692
debug_assert!(Ok(idx + idx_min) == list.binary_search(&candidate));
674693
let idx = idx + idx_min;
675-
let mut existing = list.get_mut(idx).unwrap();
694+
let existing = list.get_mut(idx).unwrap();
676695
existing.mask |= candidate.mask;
677696
debug_assert!(idx >= idx_min);
678697
idx_min = idx;
@@ -689,6 +708,8 @@ impl IDLBitRange {
689708
};
690709
});
691710

711+
list.shrink_to_fit();
712+
692713
IDLBitRange {
693714
state: IDLState::Compressed(list),
694715
}
@@ -743,6 +764,9 @@ impl IDLBitRange {
743764
nlist.push(newrange);
744765
rnextrange = riter.next();
745766
}
767+
768+
nlist.shrink_to_fit();
769+
746770
IDLBitRange {
747771
state: IDLState::Compressed(nlist),
748772
}
@@ -788,6 +812,8 @@ impl IDLBitRange {
788812
lnext = liter.next();
789813
}
790814

815+
nlist.shrink_to_fit();
816+
791817
IDLBitRange {
792818
state: IDLState::Sparse(nlist),
793819
}
@@ -832,6 +858,8 @@ impl IDLBitRange {
832858
}
833859
});
834860

861+
nlist.shrink_to_fit();
862+
835863
IDLBitRange {
836864
state: IDLState::Sparse(nlist),
837865
}
@@ -850,7 +878,7 @@ impl IDLBitRange {
850878
if let Ok(idx) = partition.binary_search(&candidate) {
851879
debug_assert!(Ok(idx + idx_min) == nlist.binary_search(&candidate));
852880
let idx = idx + idx_min;
853-
let mut existing = nlist.get_mut(idx).unwrap();
881+
let existing = nlist.get_mut(idx).unwrap();
854882
existing.mask &= !candidate.mask;
855883
if existing.mask == 0 {
856884
nlist.remove(idx);
@@ -859,6 +887,9 @@ impl IDLBitRange {
859887
idx_min = idx;
860888
}
861889
});
890+
891+
nlist.shrink_to_fit();
892+
862893
IDLBitRange {
863894
state: IDLState::Compressed(nlist),
864895
}
@@ -908,6 +939,8 @@ impl IDLBitRange {
908939
lnextrange = liter.next();
909940
}
910941

942+
nlist.shrink_to_fit();
943+
911944
IDLBitRange {
912945
state: IDLState::Compressed(nlist),
913946
}
@@ -943,7 +976,10 @@ impl FromIterator<u64> for IDLBitRange {
943976
}
944977
});
945978

946-
new_sparse.maybe_compress();
979+
if !new_sparse.maybe_compress() {
980+
// If the compression didn't occur, trim the vec anyway.
981+
new_sparse.state.shrink_to_fit();
982+
}
947983
new_sparse
948984
}
949985
}

0 commit comments

Comments
 (0)