ESE parsing improvements (#140)

.changes/unreleased/Changed-20240520-003633.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: Major improvements to the ESE parser
+time: 2024-05-20T00:36:33.3387505-04:00

common/src/windows.rs

@@ -101,18 +101,17 @@ pub struct BitsInfo {
  pub job_id: String,
  pub file_id: String,
  pub owner_sid: String,
- pub username: String,
  pub created: i64,
  pub modified: i64,
  pub completed: i64,
  pub expiration: i64,
  pub files_total: u32,
  pub bytes_downloaded: u64,
- pub bytes_tranferred: u64,
+ pub bytes_transferred: u64,
  pub job_name: String,
  pub job_description: String,
  pub job_command: String,
- pub job_arguements: String,
+ pub job_arguments: String,
  pub error_count: u32,
  pub job_type: JobType,
  pub job_state: JobState,
@@ -143,7 +142,7 @@ pub struct FileInfo {
  pub volume: String,
  pub url: String,
  pub download_bytes_size: u64,
- pub trasfer_bytes_size: u64,
+ pub transfer_bytes_size: u64,
  pub files_transferred: u32,
 }
 
@@ -159,7 +158,7 @@ pub struct JobInfo {
  pub job_name: String,
  pub job_description: String,
  pub job_command: String,
- pub job_arguements: String,
+ pub job_arguments: String,
  pub error_count: u32,
  pub transient_error_count: u32,
  pub job_type: JobType,
@@ -339,7 +338,7 @@ pub struct TableDump {
  pub column_data: String,
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub enum ColumnType {
  Nil,
  Bit,

core/src/artifacts/os/windows/bits/background.rs

@@ -5,67 +5,46 @@ use super::{
  jobs::{get_jobs, get_legacy_jobs},
 };
 use crate::{
- artifacts::os::windows::{accounts::parser::get_users, ese::parser::grab_ese_tables},
+ artifacts::os::windows::ese::{
+ helper::{get_all_pages, get_catalog_info, get_page_data},
+ tables::table_info,
+ },
  filesystem::{files::is_file, ntfs::raw_files::raw_read_file},
 };
-use common::windows::{BitsInfo, WindowsBits};
+use common::windows::{BitsInfo, TableDump, WindowsBits};
 use log::error;
 
 /**
  * Parse modern version (Win10+) of BITS which is an ESE database by dumping the `Jobs` and `Files` tables and parsing their contents 
  */
 pub(crate) fn parse_ese_bits(bits_path: &str, carve: bool) -> Result<WindowsBits, BitsError> {
- let tables = vec![String::from("Jobs"), String::from("Files")];
  // Dump the Jobs and Files tables from the BITS database
- let ese_results = grab_ese_tables(bits_path, &tables);
- let bits_tables = match ese_results {
- Ok(results) => results,
- Err(err) => {
- error!("[bits] Failed to parse ESE file: {err:?}");
- return Err(BitsError::ParseEse);
- }
- };
-
- let jobs = if let Some(values) = bits_tables.get("Jobs") {
- values
- } else {
- return Err(BitsError::MissingJobs);
- };
-
- let jobs_info = get_jobs(jobs)?;
+ let files = get_bits_ese(bits_path, "Files")?;
+ let jobs_info = get_bits_ese(bits_path, "Jobs")?;
 
- let files = if let Some(values) = bits_tables.get("Files") {
- values
- } else {
- return Err(BitsError::MissingFiles);
- };
+ let jobs = get_jobs(&jobs_info)?;
 
- let files_info = get_files(files)?;
+ let files_info = get_files(&files)?;
  let mut bits_info: Vec<BitsInfo> = Vec::new();
- let users = get_users().unwrap_or_default();
 
- for job in &jobs_info {
+ for job in &jobs {
  for file in &files_info {
  if job.file_id == file.file_id {
  let bit_info = BitsInfo {
  job_id: job.job_id.clone(),
  file_id: job.file_id.clone(),
  owner_sid: job.owner_sid.clone(),
- username: users
- .get(&job.owner_sid.clone())
- .unwrap_or(&String::new())
- .to_string(),
  created: job.created,
  modified: job.modified,
  completed: job.completed,
  expiration: job.expiration,
  files_total: file.files_transferred,
  bytes_downloaded: file.download_bytes_size,
- bytes_tranferred: file.trasfer_bytes_size,
+ bytes_transferred: file.transfer_bytes_size,
  job_name: job.job_name.clone(),
  job_description: job.job_description.clone(),
  job_command: job.job_command.clone(),
- job_arguements: job.job_arguements.clone(),
+ job_arguments: job.job_arguments.clone(),
  error_count: job.error_count,
  job_type: job.job_type.clone(),
  job_state: job.job_state.clone(),
@@ -114,6 +93,39 @@ pub(crate) fn parse_ese_bits(bits_path: &str, carve: bool) -> Result<WindowsBits
  Ok(windows_bits)
 }
 
+/// Extract BITs info from ESE database
+pub(crate) fn get_bits_ese(path: &str, table: &str) -> Result<Vec<Vec<TableDump>>, BitsError> {
+ let catalog_result = get_catalog_info(path);
+ let catalog = match catalog_result {
+ Ok(result) => result,
+ Err(err) => {
+ error!("[bits] Failed to parse {path} catalog: {err:?}");
+ return Err(BitsError::ParseEse);
+ }
+ };
+
+ let mut info = table_info(&catalog, table);
+ let pages_result = get_all_pages(path, &(info.table_page as u32));
+ let pages = match pages_result {
+ Ok(result) => result,
+ Err(err) => {
+ error!("[bits] Failed to get {table} pages at {path}: {err:?}");
+ return Err(BitsError::ParseEse);
+ }
+ };
+
+ let rows_results = get_page_data(path, &pages, &mut info, table);
+ let table_rows = match rows_results {
+ Ok(result) => result,
+ Err(err) => {
+ error!("[bits] Failed to parse {table} table at {path}: {err:?}");
+ return Err(BitsError::ParseEse);
+ }
+ };
+
+ Ok(table_rows.get(table).unwrap_or(&Vec::new()).clone())
+}
+
 /**
  * Parse older version (pre-Win10) of BITS which is a custom binary format
  */
@@ -196,7 +208,9 @@ fn parse_carve(data: &[u8], is_legacy: bool) -> WinBits {
 mod tests {
  use super::parse_ese_bits;
  use crate::{
- artifacts::os::windows::bits::background::{legacy_bits, parse_carve, parse_legacy_bits},
+ artifacts::os::windows::bits::background::{
+ get_bits_ese, legacy_bits, parse_carve, parse_legacy_bits,
+ },
  filesystem::files::read_file,
  };
  use std::path::PathBuf;
@@ -209,6 +223,14 @@ mod tests {
  assert_eq!(results.bits.len(), 1);
  }
 
+ #[test]
+ fn test_get_bits_ese() {
+ let mut test_location = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+ test_location.push("tests\\test_data\\windows\\ese\\win10\\qmgr.db");
+ let results = get_bits_ese(test_location.to_str().unwrap(), "Files").unwrap();
+ assert_eq!(results.len(), 1);
+ }
+
  #[test]
  fn test_parse_legacy_bits() {
  let results = parse_legacy_bits(&'C', false).unwrap();

core/src/artifacts/os/windows/bits/carve.rs

@@ -1,9 +1,4 @@
-use std::collections::HashMap;
-
-use crate::{
- artifacts::os::windows::accounts::parser::get_users,
- utils::nom_helper::{nom_unsigned_four_bytes, nom_unsigned_sixteen_bytes, Endian},
-};
+use crate::utils::nom_helper::{nom_unsigned_four_bytes, nom_unsigned_sixteen_bytes, Endian};
 use common::windows::{BitsInfo, FileInfo, JobFlags, JobInfo, JobPriority, JobState, JobType};
 use nom::bytes::complete::take_until;
 
@@ -58,7 +53,7 @@ pub(crate) fn carve_bits(data: &[u8], is_legacy: bool) -> nom::IResult<&[u8], Wi
  // Start by scanning for known job delimiters
  for job in job_delimiters {
  while !job_data.is_empty() {
- let scan_results = scan_delimter(job_data, &job);
+ let scan_results = scan_delimiter(job_data, &job);
  // If no hits move on to next delimiter
  let hit_data = match scan_results {
  Ok((input, _)) => input,
@@ -85,7 +80,7 @@ pub(crate) fn carve_bits(data: &[u8], is_legacy: bool) -> nom::IResult<&[u8], Wi
  job_name: String::new(),
  job_description: String::new(),
  job_command: String::new(),
- job_arguements: String::new(),
+ job_arguments: String::new(),
  error_count: 0,
  job_type: JobType::Unknown,
  job_state: JobState::Unknown,
@@ -107,8 +102,7 @@ pub(crate) fn carve_bits(data: &[u8], is_legacy: bool) -> nom::IResult<&[u8], Wi
 
  job_data = remaining_input;
  let carved = true;
- let users = get_users().unwrap_or_default();
- bits.push(combine_file_and_job(&job, &file, carved, &users));
+ bits.push(combine_file_and_job(&job, &file, carved));
  continue;
  }
  let remaining_input_result = job_details(input, &mut job, is_legacy);
@@ -133,13 +127,13 @@ pub(crate) fn carve_bits(data: &[u8], is_legacy: bool) -> nom::IResult<&[u8], Wi
  let (hit_data, file) = match scan_results {
  Ok(results) => results,
  Err(_err) => {
- // Before we break because of parsing error, check one more time for the file delimter
+ // Before we break because of parsing error, check one more time for the file delimiter
  // If we find another delimiter, keep trying to parse the data
  let (check_data, _) = nom_unsigned_sixteen_bytes(file_data, Endian::Le)?;
  let file_delimiter = [
  228, 207, 158, 81, 70, 217, 151, 67, 183, 62, 38, 133, 19, 5, 26, 178,
  ];
- let scan_results = scan_delimter(check_data, &file_delimiter);
+ let scan_results = scan_delimiter(check_data, &file_delimiter);
  match scan_results {
  Ok((input, _)) => file_data = input,
  Err(_err) => break,
@@ -156,31 +150,22 @@ pub(crate) fn carve_bits(data: &[u8], is_legacy: bool) -> nom::IResult<&[u8], Wi
  Ok((data, (bits, jobs, files)))
 }
 
-/// The legacy BITS format has both job and file info in same structure, we combine them both here into one strcuture
-pub(crate) fn combine_file_and_job(
- job: &JobInfo,
- file: &FileInfo,
- carved: bool,
- users: &HashMap<String, String>,
-) -> BitsInfo {
+/// The legacy BITS format has both job and file info in same structure, we combine them both here into one structure
+pub(crate) fn combine_file_and_job(job: &JobInfo, file: &FileInfo, carved: bool) -> BitsInfo {
  BitsInfo {
  job_id: job.job_id.clone(),
  file_id: job.file_id.clone(),
  owner_sid: job.owner_sid.clone(),
- username: users
- .get(&job.owner_sid.clone())
- .unwrap_or(&String::new())
- .to_string(),
  created: job.created,
  modified: job.modified,
  completed: job.completed,
  files_total: file.files_transferred,
  bytes_downloaded: file.download_bytes_size,
- bytes_tranferred: file.trasfer_bytes_size,
+ bytes_transferred: file.transfer_bytes_size,
  job_name: job.job_name.clone(),
  job_description: job.job_description.clone(),
  job_command: job.job_command.clone(),
- job_arguements: job.job_arguements.clone(),
+ job_arguments: job.job_arguments.clone(),
  error_count: job.error_count,
  job_type: job.job_type.clone(),
  job_state: job.job_state.clone(),
@@ -203,17 +188,17 @@ pub(crate) fn combine_file_and_job(
  }
 }
 
-pub(crate) fn scan_delimter<'a>(data: &'a [u8], delimter: &[u8]) -> nom::IResult<&'a [u8], ()> {
- let (input, _) = take_until(delimter)(data)?;
+pub(crate) fn scan_delimiter<'a>(data: &'a [u8], delimiter: &[u8]) -> nom::IResult<&'a [u8], ()> {
+ let (input, _) = take_until(delimiter)(data)?;
  Ok((input, ()))
 }
 
 #[cfg(test)]
 mod tests {
- use super::{carve_bits, combine_file_and_job, scan_delimter};
+ use super::{carve_bits, combine_file_and_job, scan_delimiter};
  use crate::filesystem::files::read_file;
  use common::windows::{FileInfo, JobFlags, JobInfo, JobPriority, JobState, JobType};
- use std::{collections::HashMap, path::PathBuf};
+ use std::path::PathBuf;
 
  #[test]
  fn test_carve_bits() {
@@ -266,7 +251,7 @@ mod tests {
  job_name: String::new(),
  job_description: String::new(),
  job_command: String::new(),
- job_arguements: String::new(),
+ job_arguments: String::new(),
  error_count: 0,
  job_type: JobType::Unknown,
  job_state: JobState::Unknown,
@@ -290,23 +275,23 @@ mod tests {
  volume: String::new(),
  url: String::new(),
  download_bytes_size: 0,
- trasfer_bytes_size: 0,
+ transfer_bytes_size: 0,
  files_transferred: 0,
  };
 
- let bit_info = combine_file_and_job(&job, &file, true, &HashMap::new());
+ let bit_info = combine_file_and_job(&job, &file, true);
  assert_eq!(bit_info.carved, true);
  }
 
  #[test]
- fn test_scan_delimter() {
+ fn test_scan_delimiter() {
  let file_delimiter = [
  228, 207, 158, 81, 70, 217, 151, 67, 183, 62, 38, 133, 19, 5, 26, 178,
  ];
  let mut test_location = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
  test_location.push("tests/test_data/windows/ese/win10/qmgr.db");
  let data = read_file(test_location.to_str().unwrap()).unwrap();
- let (scan_results, _) = scan_delimter(&data, &file_delimiter).unwrap();
+ let (scan_results, _) = scan_delimiter(&data, &file_delimiter).unwrap();
 
  assert_eq!(scan_results.len(), 769801);
  }

core/src/artifacts/os/windows/bits/error.rs

@@ -6,8 +6,6 @@ pub(crate) enum BitsError {
  Systemdrive,
  ParseEse,
  ParseLegacyBits,
- MissingJobs,
- MissingFiles,
 }
 
 impl std::error::Error for BitsError {}
@@ -19,8 +17,6 @@ impl fmt::Display for BitsError {
  BitsError::Systemdrive => write!(f, "Failed to get systemdrive"),
  BitsError::ParseEse => write!(f, "Failed to parse ESE db"),
  BitsError::ParseLegacyBits => write!(f, "Failed to parse legacy BITS format"),
- BitsError::MissingJobs => write!(f, "No Jobs table in ESE db"),
- BitsError::MissingFiles => write!(f, "No Files table in ESE db"),
  }
  }
 }