@@ -1164,7 +1164,7 @@ void rans_set_cpu(int opts) {
1164
1164
unsigned char * rans_compress_to_4x16 (unsigned char * in , unsigned int in_size ,
1165
1165
unsigned char * out ,unsigned int * out_size ,
1166
1166
int order ) {
1167
- if (in_size > INT_MAX ) {
1167
+ if (in_size > INT_MAX || ( out && * out_size == 0 ) ) {
1168
1168
* out_size = 0 ;
1169
1169
return NULL ;
1170
1170
}
@@ -1177,8 +1177,10 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1177
1177
* out_size = rans_compress_bound_4x16 (in_size , order );
1178
1178
if (* out_size == 0 )
1179
1179
return NULL ;
1180
- if (!(out_free = out = malloc (* out_size )))
1180
+ if (!(out_free = out = malloc (* out_size ))) {
1181
+ * out_size = 0 ;
1181
1182
return NULL ;
1183
+ }
1182
1184
}
1183
1185
1184
1186
unsigned char * out_end = out + * out_size ;
@@ -1199,11 +1201,15 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1199
1201
int N = (order >>8 ) & 0xff ;
1200
1202
if (N == 0 ) N = 4 ; // default for compatibility with old tests
1201
1203
1204
+ if (N > in_size )
1205
+ N = in_size ;
1206
+
1202
1207
unsigned char * transposed = malloc (in_size );
1203
1208
unsigned int part_len [256 ];
1204
1209
unsigned int idx [256 ];
1205
1210
if (!transposed ) {
1206
1211
free (out_free );
1212
+ * out_size = 0 ;
1207
1213
return NULL ;
1208
1214
}
1209
1215
int i , j , x ;
@@ -1241,6 +1247,13 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1241
1247
c_meta_len = 1 ;
1242
1248
* out = order & ~RANS_ORDER_NOSZ ;
1243
1249
c_meta_len += var_put_u32 (out + c_meta_len , out_end , in_size );
1250
+ if (c_meta_len >= * out_size ) {
1251
+ free (out_free );
1252
+ free (transposed );
1253
+ * out_size = 0 ;
1254
+ return NULL ;
1255
+ }
1256
+
1244
1257
out [c_meta_len ++ ] = N ;
1245
1258
1246
1259
unsigned char * out_best = NULL ;
@@ -1249,26 +1262,33 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1249
1262
out2_start = out2 = out + 7 + 5 * N ; // shares a buffer with c_meta
1250
1263
for (i = 0 ; i < N ; i ++ ) {
1251
1264
// Brute force try all methods.
1252
- int j , m [] = {1 ,64 ,128 ,0 }, best_j = 0 , best_sz = in_size + 10 ;
1265
+ uint8_t * r ;
1266
+ int j , m [] = {1 ,64 ,128 ,0 }, best_j = 0 , best_sz = INT_MAX ;
1253
1267
for (j = 0 ; j < sizeof (m )/sizeof (* m ); j ++ ) {
1254
1268
if ((order & m [j ]) != m [j ])
1255
1269
continue ;
1256
1270
1257
1271
// order-1 *only*; bit check above cannot elide order-0
1258
1272
if ((order & RANS_ORDER_STRIPE_NO0 ) && (m [j ]& 1 ) == 0 )
1259
1273
continue ;
1274
+
1275
+ if (out2 - out > * out_size )
1276
+ continue ; // an error, but caught in best_sz check later
1277
+
1260
1278
olen2 = * out_size - (out2 - out );
1261
- rans_compress_to_4x16 (transposed + idx [i ], part_len [i ],
1262
- out2 , & olen2 ,
1263
- m [j ] | RANS_ORDER_NOSZ
1264
- | (order & RANS_ORDER_X32 ));
1265
- if (best_sz > olen2 ) {
1279
+ r = rans_compress_to_4x16 (transposed + idx [i ], part_len [i ],
1280
+ out2 , & olen2 ,
1281
+ m [j ] | RANS_ORDER_NOSZ
1282
+ | (order & RANS_ORDER_X32 ));
1283
+ if (r && olen2 && best_sz > olen2 ) {
1266
1284
best_sz = olen2 ;
1267
1285
best_j = j ;
1268
1286
if (j < sizeof (m )/sizeof (* m ) && olen2 > out_best_len ) {
1269
1287
unsigned char * tmp = realloc (out_best , olen2 );
1270
1288
if (!tmp ) {
1271
1289
free (out_free );
1290
+ free (transposed );
1291
+ * out_size = 0 ;
1272
1292
return NULL ;
1273
1293
}
1274
1294
out_best = tmp ;
@@ -1279,6 +1299,15 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1279
1299
memcpy (out_best , out2 , olen2 );
1280
1300
}
1281
1301
}
1302
+
1303
+ if (best_sz == INT_MAX ) {
1304
+ free (out_best );
1305
+ free (out_free );
1306
+ free (transposed );
1307
+ * out_size = 0 ;
1308
+ return NULL ;
1309
+ }
1310
+
1282
1311
if (best_j < sizeof (m )/sizeof (* m )) {
1283
1312
// Copy the best compression to output buffer if not current
1284
1313
memcpy (out2 , out_best , best_sz );
@@ -1301,6 +1330,11 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1301
1330
out [0 ] = RANS_ORDER_CAT ;
1302
1331
c_meta_len = 1 ;
1303
1332
c_meta_len += var_put_u32 (& out [1 ], out_end , in_size );
1333
+
1334
+ if (c_meta_len + in_size > * out_size ) {
1335
+ * out_size = 0 ;
1336
+ return NULL ;
1337
+ }
1304
1338
if (in_size )
1305
1339
memcpy (out + c_meta_len , in , in_size );
1306
1340
* out_size = c_meta_len + in_size ;
@@ -1329,6 +1363,11 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1329
1363
// PACK 2, 4 or 8 symbols into one byte.
1330
1364
int pmeta_len ;
1331
1365
uint64_t packed_len ;
1366
+ if (c_meta_len + 256 > * out_size ) {
1367
+ free (out_free );
1368
+ * out_size = 0 ;
1369
+ return NULL ;
1370
+ }
1332
1371
packed = hts_pack (in , in_size , out + c_meta_len , & pmeta_len , & packed_len );
1333
1372
if (!packed ) {
1334
1373
out [0 ] &= ~RANS_ORDER_PACK ;
@@ -1357,6 +1396,7 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1357
1396
c_rmeta_len = in_size + 257 ;
1358
1397
if (!(meta = malloc (c_rmeta_len ))) {
1359
1398
free (out_free );
1399
+ * out_size = 0 ;
1360
1400
return NULL ;
1361
1401
}
1362
1402
@@ -1380,8 +1420,23 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1380
1420
// Compress lengths with O0 and literals with O0/O1 ("order" param)
1381
1421
int sz = var_put_u32 (out + c_meta_len , out_end , rmeta_len * 2 ), sz2 ;
1382
1422
sz += var_put_u32 (out + c_meta_len + sz , out_end , rle_len );
1423
+ if ((c_meta_len + sz + 5 ) > * out_size ) {
1424
+ free (out_free );
1425
+ free (rle );
1426
+ free (meta );
1427
+ free (packed );
1428
+ * out_size = 0 ;
1429
+ return NULL ;
1430
+ }
1383
1431
c_rmeta_len = * out_size - (c_meta_len + sz + 5 );
1384
- rans_enc_func (do_simd , 0 )(meta , rmeta_len , out + c_meta_len + sz + 5 , & c_rmeta_len );
1432
+ if (!rans_enc_func (do_simd , 0 )(meta , rmeta_len , out + c_meta_len + sz + 5 , & c_rmeta_len )) {
1433
+ free (out_free );
1434
+ free (rle );
1435
+ free (meta );
1436
+ free (packed );
1437
+ * out_size = 0 ;
1438
+ return NULL ;
1439
+ }
1385
1440
if (c_rmeta_len < rmeta_len ) {
1386
1441
sz2 = var_put_u32 (out + c_meta_len + sz , out_end , c_rmeta_len );
1387
1442
memmove (out + c_meta_len + sz + sz2 , out + c_meta_len + sz + 5 , c_rmeta_len );
@@ -1404,17 +1459,36 @@ unsigned char *rans_compress_to_4x16(unsigned char *in, unsigned int in_size,
1404
1459
out [0 ] &= ~RANS_ORDER_RLE ;
1405
1460
}
1406
1461
1462
+ if (c_meta_len > * out_size ) {
1463
+ free (rle );
1464
+ free (packed );
1465
+ * out_size = 0 ;
1466
+ return NULL ;
1467
+ }
1468
+
1407
1469
* out_size -= c_meta_len ;
1408
1470
if (order && in_size < 8 ) {
1409
1471
out [0 ] &= ~1 ;
1410
1472
order &= ~1 ;
1411
1473
}
1412
1474
1413
- rans_enc_func (do_simd , order )(in , in_size , out + c_meta_len , out_size );
1475
+ if (!rans_enc_func (do_simd , order )(in , in_size , out + c_meta_len , out_size )) {
1476
+ free (rle );
1477
+ free (packed );
1478
+ * out_size = 0 ;
1479
+ return NULL ;
1480
+ }
1414
1481
1415
1482
if (* out_size >= in_size ) {
1416
1483
out [0 ] &= ~3 ;
1417
1484
out [0 ] |= RANS_ORDER_CAT | no_size ;
1485
+
1486
+ if (out + c_meta_len + in_size > out_end ) {
1487
+ free (rle );
1488
+ free (packed );
1489
+ * out_size = 0 ;
1490
+ return NULL ;
1491
+ }
1418
1492
if (in_size )
1419
1493
memcpy (out + c_meta_len , in , in_size );
1420
1494
* out_size = in_size ;
0 commit comments