fuse: partial file reads

Author: Maxqia

recuperabit/fs/core_types.py

@@ -50,6 +50,7 @@ def __init__(self, index, name, size, is_directory=False,
         self.children = set()
         self.children_names = set()     # Avoid name clashes breaking restore
         self.offset = None  # Offset from beginning of disk
+        self.isopen = False # if the file is currently open
 
     def set_parent(self, parent):
         """Set a pointer to the parent directory."""
@@ -107,6 +108,13 @@ def get_content(self, partition):
         if self.is_directory or self.is_ghost:
             return None
         raise NotImplementedError
+    
+    def open(self, partition):
+        raise NotImplementedError
+    def close(self, partition):
+        pass
+    def read(self, partition, offset, size):
+        raise NotImplementedError
 
     # pylint: disable=R0201
     def ignore(self):

recuperabit/fs/ntfs.py

@@ -320,72 +320,63 @@ def _padded_bytes(image, offset, size):
                 '{}'.format(offset, size))
             dump += bytearray('\x00' * (size - len(dump)))
         return dump
-
-    def content_iterator(self, partition, image, datas):
-        """Return an iterator for the contents of this file."""
+    
+    def parse_data(self, partition, datas):
         vcn = 0
         spc = partition.sec_per_clus
+        
+        output = [] # (vcn start, vcn end, sector offset (-1 if zeros))
         for attr in datas:
             diff = attr['start_VCN'] - vcn
             if diff > 0:
                 logging.warning(
                     u'Missing part for {}, filling {} clusters '
                     'with zeros'.format(self, diff)
                 )
-            while diff > 0:
-                amount = min(max_sectors//spc, diff)
-                vcn += amount
-                diff -= amount
-                yield '\x00' * sector_size * spc * amount
-
+                output.append((vcn, attr['start_VCN'], -1))
+                
+            vcn = attr['start_VCN']
             clusters_pos = 0
             size = attr['real_size']
 
             if 'runlist' not in attr:
-                logging.error(
-                    u'Cannot restore {}, missing runlist'.format(self)
-                )
+                raise ValueError(u'Cannot restore {}, missing runlist'.format(self))
                 break
 
             for entry in attr['runlist']:
                 length = min(entry['length'] * spc * sector_size, size)
                 size -= length
                 # Sparse runlist
                 if entry['offset'] is None:
-                    while length > 0:
-                        amount = min(max_sectors*sector_size, length)
-                        length -= amount
-                        yield '\x00' * amount
-                    continue
-                # Normal runlists
-                clusters_pos += entry['offset']
-                real_pos = clusters_pos * spc + partition.offset
-                # Avoid to fill memory with huge blocks
-                offset = 0
-                while length > 0:
-                    amount = min(max_sectors*sector_size, length)
-                    position = real_pos*sector_size + offset
-                    partial = self._padded_bytes(image, position, amount)
-                    length -= amount
-                    offset += amount
-                    yield str(partial)
+                    output.append((vcn, vcn+entry['length'], -1))
+                else:
+                    # Normal runlists
+                    clusters_pos += entry['offset']
+                    real_pos = clusters_pos * spc + partition.offset
+                    output.append((vcn, vcn+entry['length'], real_pos))
+                vcn += entry['length']
+            if vcn != attr['end_VCN'] + 1:
+                logging.error("VCN miscalcuation! {} {}".format(vcn, attr['end_VCN'] + 1))
             vcn = attr['end_VCN'] + 1
-
-    def get_content(self, partition):
-        """Extract the content of the file.
+        return output
+    
+    def open(self, partition):
+        """Opens the file and get the sector locations of the file.
 
         This method works by extracting the $DATA attribute."""
+        if self.isopen:
+            logging.warning(u'Tried to open already open file {}!'.format(self))
+            return # already open!
+            
         if self.is_ghost:
-            logging.error(u'Cannot restore ghost file {}'.format(self))
-            return None
+            raise ValueError(u'Cannot open ghost file {}'.format(self))
 
         image = DiskScanner.get_image(partition.scanner)
         dump = sectors(image, File.get_offset(self), FILE_size)
         parsed = parse_file_record(dump)
 
         if not parsed['valid'] or 'attributes' not in parsed:
-            logging.error(u'Invalid MFT entry for {}'.format(self))
-            return None
+            raise ValueError(u'Invalid MFT entry for {}'.format(self))
         attrs = parsed['attributes']
         if ('$ATTRIBUTE_LIST' in attrs and
                 partition.sec_per_clus is not None):
@@ -395,32 +386,30 @@ def get_content(self, partition):
         datas = [d for d in attrs['$DATA'] if d['name'] == self.ads]
         if not len(datas):
             if not self.is_directory:
-                logging.error(u'Cannot restore $DATA attribute(s) '
+                raise ValueError(u'Cannot restore $DATA attribute(s) '
                               'for {}'.format(self))
-            return None
 
         # TODO implemented compressed attributes
         for d in datas:
             if d['flags'] & 0x01:
-                logging.error(u'Cannot restore compressed $DATA attribute(s) '
+                raise ValueError(u'Cannot restore compressed $DATA attribute(s) '
                               'for {}'.format(self))
-                return None
             elif d['flags'] & 0x4000:
                 logging.warning(u'Found encrypted $DATA attribute(s) '
                                 'for {}'.format(self))
-
+        self.isopen = True
         # Handle resident file content
         if len(datas) == 1 and not datas[0]['non_resident']:
             single = datas[0]
             start = single['dump_offset'] + single['content_off']
             end = start + single['content_size']
-            content = dump[start:end]
-            return str(content)
+            self.resident = True
+            self.content = dump[start:end]
+            return
         else:
             if partition.sec_per_clus is None:
-                logging.error(u'Cannot restore non-resident $DATA '
+                raise ValueError(u'Cannot restore non-resident $DATA '
                               'attribute(s) for {}'.format(self))
-                return None
             non_resident = sorted(
                 (d for d in attrs['$DATA'] if d['non_resident']),
                 key=lambda x: x['start_VCN']
@@ -430,7 +419,77 @@ def get_content(self, partition):
                     u'Found leftover resident $DATA attributes for '
                     '{}'.format(self)
                 )
-            return self.content_iterator(partition, image, non_resident)
+            self.resident = False
+            self.content = self.parse_data(partition, non_resident)
+            return
+    
+    def content_iterator(self, partition, image, datas):
+        """Return an iterator for the contents of this file."""
+        
+        spc = partition.sec_per_clus
+        bpc = sector_size*spc # bytes per cluster
+        
+        curlen = 0
+        for attr in self.content:
+            (attr_start, attr_end, sectoroff) = attr
+            curoff = (attr_end - attr_start) * bpc
+            length = min(self.size - curlen, curoff)
+            
+            if length <= 0:
+                break
+            if sectoroff == -1:
+                yield '\x00' * length;
+            else:
+                yield self._padded_bytes(image, sectoroff*sector_size, length)
+                
+    def get_content(self, partition):
+        """Extract the entire content of the file."""
+        self.open(partition)
+        assert self.isopen
+        if self.resident:
+            return bytes(self.content) # typecast from bytearray -> bytes
+        else:
+            image = DiskScanner.get_image(partition.scanner)
+            return self.content_iterator(partition, image, self.content)
+        
+    # TODO it can technically read off the end of the file a bit....
+    def read(self, partition, roffset, rsize):
+        if not self.isopen:
+            raise RuntimeError("tried to read file that wasn't open!")
+        if self.resident:
+            trim = self.content[roffset:roffset+rsize]
+            return bytes(trim) # typecast from bytearray -> bytes
+        
+        image = DiskScanner.get_image(partition.scanner)
+        spc = partition.sec_per_clus
+        bpc = sector_size*spc # bytes per cluster
+        
+        start_vcn = roffset // bpc
+        offset_startvcn = roffset % bpc
+        end_vcn = (roffset+rsize) // bpc
+        
+        value = bytearray()
+        for attr in self.content:
+            (attr_start, attr_end, sectoroff) = attr
+            vcn_off = 0
+            if start_vcn > attr_end:
+                continue
+            elif start_vcn >= attr_start:
+                vcn_off = start_vcn - attr_start
+                
+            
+            length = attr_end - (attr_start + vcn_off)
+            offset = sectoroff + (spc*vcn_off)
+            if sectoroff == -1:
+                value.extend('\x00' * bpc * length)
+            else:
+                value.extend(self._padded_bytes(image, offset*sector_size, length*bpc))
+                
+            if end_vcn < attr_end:
+                break
+        
+        trim = value[offset_startvcn:offset_startvcn+rsize]
+        return bytes(trim) # typecast from bytearray -> bytes
 
     def ignore(self):
         """Determine which files should be ignored."""

recuperabit/ifuse.py

@@ -8,6 +8,7 @@
 import time
 from datetime import datetime
 from fs.core_types import File
+import traceback
 
 # was originally named fuse.py until i realized it conflicted with fusepy
 
@@ -137,46 +138,31 @@ def open(self, path, flags):
         part = self.get_part_from_path(path)
 
         try:
-            content = file.get_content(part)
-        except NotImplementedError:
-            logging.error(u'Restore of #%s is not supported', file.index)
+            file.open(part)
+        except Exception, e:
+            track = traceback.format_exc()
+            logging.error(e)
+            logging.error(track)
             raise FuseOSError(EIO)
 
-        
-        if file.is_directory and content is not None:
-            logging.warning(u'Directory %s has data content!', file.file_path)
-
-        binarray = bytearray()
-        if content is not None:
-            logging.info(u'Restoring #%s %s', file.index, path)
-            if hasattr(content, '__iter__'):
-                for piece in content:
-                    binarray.extend(piece)
-            else:
-                binarray.extend(content)
-        """else:
-            if not is_directory:
-                # Empty file
-                pass
-            else:
-                raise FuseOSError(EIO)"""
-
-        binout = bytes(binarray)
-        #print(type(binout))
-        
         self.fd += 1
-        self.files[self.fd] = (file, binout)
+        self.files[self.fd] = file
         return self.fd
 
     def release(self, path, fh):
         self.files[fh] = None
         return 0
 
     def read(self, path, size, offset, fh):
-        content = self.files[fh][1]
-        if content is None:
+        file = self.get_file_from_path(path)
+        part = self.get_part_from_path(path)
+        try:
+            return file.read(part, offset, size)
+        except Exception, e:
+            track = traceback.format_exc()
+            logging.error(e)
+            logging.error(track)
             raise FuseOSError(EIO)
-        return content[offset:offset+size]
 
 class PartView(AbstractView):
     def __init__(self, part, root):