myraid : extract the math part in a separate file, make it re-usable

2 files changed, 156 insertions, 124 deletions

diff --git a/myraid.py b/myraid.py
index fc240b1..5847e6e 100644
--- a/myraid.py
+++ b/myraid.py
@@ -18,57 +18,14 @@
 # You should have received a copy of the GNU General Public License
 # along with RaidGuessFS. If not, see <http://www.gnu.org/licenses/>
 
-import logging, numpy
-import mydisks
+import logging
+from myraidmaths import MyRaidMaths
 
 class MyRaid():
     """Auxiliary class, managing RAID layer"""
     RAID_TYPES = [ '0', '1', '5', '5+0' ]
     RAID5_LAYOUTS = [ 'la', 'ra', 'ls', 'rs' ]
 
-    @staticmethod
-    def xor_blocks(fd_list, offset, size):
-        """Compute bitwise XOR against a bunch of disks slice"""
-        logging.info("Enter xor_blocks(fd_list(%i),0x%011x,%d)"%(len(fd_list), offset, size))
-        
-        if size % 8 != 0:
-            raise ValueError('xor_blocks : size must be multiple of 8')
-        dt = numpy.dtype('<Q8')
-
-        fd_list[0].seek(offset)
-        str_b1=fd_list[0].read(size)
-        numpy_b1 = numpy.fromstring(str_b1, dtype=dt)
-        all_zero = (numpy.count_nonzero(numpy_b1) == 0 )
-        any_zero = all_zero
-
-        for fd in fd_list[1:]:
-            fd.seek(offset)
-            str_b2=fd.read(size)
-            numpy_b2 = numpy.fromstring(str_b2, dtype=dt)
-            b2_zero = (numpy.count_nonzero(numpy_b2) == 0 )
-            if all_zero == True:
-                all_zero = b2_zero
-            if any_zero == False:
-                any_zero = b2_zero
-
-            numpy.bitwise_xor(numpy_b1,numpy_b2,numpy_b1)
-
-        if all_zero == True:
-            result = 'z'
-        elif numpy.count_nonzero(numpy_b1) == 0:
-            if any_zero:
-                result = 'g'
-            else:
-                result = 'G'
-        else:
-            result = 'b'
-
-        logging.info("Exit. xor_blocks(fd_list,%d,%d)"%(offset, size))
-        #import binascii
-        #logging.debug(binascii.hexlify(numpy_b1))
-        return (result,numpy_b1)
-
-
     def __init__(self, *args, **kwargs):
         self.d = None
         self.raid_start = 0
@@ -161,7 +118,7 @@ class MyRaid():
 
     def read_disk_xor(self,raid_type,offset,size):
         """Returns raw bitwise XOR against a bunch of disks slice"""
-        return MyRaid.xor_blocks(self.raid_disks,offset,size)[1].tostring()
+        return MyRaidMaths.xor_blocks(self.raid_disks,offset,size)[1]
 
     def read_disk_parity(self,raid_type,offset,size):
         """Returns textual information about parity status of each sector"""
@@ -182,7 +139,7 @@ class MyRaid():
         #TODO : improove for nested levels
         if raid_type in ['1','5', '5+0']:
             result = ''.join(
-                    [ '0x%011x %c\n'%( addr, MyRaid.xor_blocks(self.raid_disks, addr, self.raid_sector_size)[0])
+                    [ '0x%011x %c\n'%( addr, MyRaidMaths.xor_blocks(self.raid_disks, addr, self.raid_sector_size)[0])
                             for addr in xrange(start, end, self.raid_sector_size)
                     ])
         else:
@@ -194,87 +151,22 @@ class MyRaid():
 
     def read_raid_result(self,raid_type,offset,size):
         """Returns actual RAID data"""
-        if raid_type == '0':
-            segment_no = offset / self.raid_chunk_size
-            segment_off = offset % self.raid_chunk_size
-            stripe_no = segment_no / self.raid_disk_count
-            subraid_no = -1
-            par_disk = -1
-            data_disk = segment_no % self.raid_disk_count
-            off_disk = self.raid_start + stripe_no * self.raid_chunk_size + segment_off
-            size2 = min(size, (segment_no+1) * self.raid_chunk_size - offset)
-
-        elif raid_type == '1':
-            segment_no = -1
-            segment_off = -1
-            stripe_no = -1
-            subraid_no = -1
-            par_disk = 1
-            data_disk = 0
-            off_disk = self.raid_start + offset
-            size2 = size
-
-        elif raid_type == '5':
-            segment_no = offset / self.raid_chunk_size
-            segment_off = offset % self.raid_chunk_size
-            stripe_no = segment_no / (self.raid_disk_count-1)
-            subraid_no = -1
-
-            if self.raid_layout in ['ls','la']:
-                par_disk = (self.raid_disk_count-1) - (stripe_no % self.raid_disk_count)
-            else: # self.raid_layout in ['rs','ra']:
-                par_disk = stripe_no % self.raid_disk_count
-
-            if self.raid_layout in ['ls','rs']:
-                data_disk = (par_disk+1 + (segment_no % (self.raid_disk_count-1)) ) % self.raid_disk_count
-            else: # self.raid_layout in ['la','ra']:
-                data_disk = segment_no % (self.raid_disk_count-1)
-                if data_disk >= par_disk:
-                    data_disk = data_disk + 1
-
-            off_disk = self.raid_start + stripe_no * self.raid_chunk_size + segment_off
-            # Note : could make error-free shorter reads than asked but convince the reader to be chunck aligned, which is great for perf
-            size2 = min(size, (segment_no+1) * self.raid_chunk_size - offset)
-
-        elif raid_type == '5+0':
-            subraid_disk_count = self.raid_disk_count / self.nested_subraid
-            segment_no = offset / self.raid_chunk_size
-            segment_off = offset % self.raid_chunk_size
-            stripe_no = segment_no / (self.raid_disk_count - self.nested_subraid)                # segment_no / 12
-            subraid_no = (segment_no / (subraid_disk_count-1) ) % self.nested_subraid            # (segment_no/6) mod 2
-
-            if self.raid_layout in ['ls','la']:
-                subraid_par_disk = (subraid_disk_count-1) - (stripe_no % subraid_disk_count)
-            else: # self.raid_layout in ['rs','ra']:
-                subraid_par_disk = stripe_no % subraid_disk_count
-
-            if self.raid_layout in ['ls','rs']:
-                subraid_data_disk = (subraid_par_disk+1 + (segment_no % (subraid_disk_count-1)) ) % subraid_disk_count
-            else: # self.raid_layout in ['la','ra']:
-                subraid_data_disk = segment_no % (subraid_disk_count-1)
-                if subraid_data_disk >= subraid_par_disk:
-                    subraid_data_disk = subraid_data_disk + 1
-
-            par_disk  = subraid_no * subraid_disk_count + subraid_par_disk
-            data_disk = subraid_no * subraid_disk_count + subraid_data_disk
-
-            off_disk = self.raid_start + stripe_no * self.raid_chunk_size + segment_off
-            # Note : could make error-free shorter reads than asked but convince the reader to be chunck aligned, which is great for perf
-            size2 = min(size, (segment_no+1) * self.raid_chunk_size - offset)
+        res = MyRaidMaths.apply_raid_layout(offset, size, raid_type, self.raid_layout, self.raid_chunk_size, self.raid_disk_count, self.raid_start, self.nested_subraid);
+        
+        (segment_no, segment_off, stripe_no, subraid_no, par_disk, data_disk, off_disk, aligned_read_size) = res
 
-        else:
-            raise Exception('Unimplemented read_raid_result() for raid_type == %s', raid_type)
 
-        logging.debug("raid.read_result(%s): offset=%d,segment_no=%d,segment_off=%d,stripe_no=%d,subraid_no=%d,par_disk=%d(disk%02d),data_disk=%d(disk%02d),off_disk=%d,size2=%d,segment_off+size2=%d" 
-        % (raid_type,offset,segment_no,segment_off,stripe_no,subraid_no,par_disk,self.raid_disk_order[par_disk],data_disk,self.raid_disk_order[data_disk],off_disk,size2,segment_off+size2) )
+        logging.debug("raid.read_result(%s): offset=%d,segment_no=%d,segment_off=%d,stripe_no=%d,subraid_no=%d,par_disk=%d(disk%02d),data_disk=%d(disk%02d),off_disk=%d,aligned_read_size=%d,segment_off+aligned_read_size=%d" 
+        % (raid_type,offset,segment_no,segment_off,stripe_no,subraid_no,par_disk,self.raid_disk_order[par_disk],data_disk,self.raid_disk_order[data_disk],off_disk,aligned_read_size,segment_off+aligned_read_size) )
 
         data_fd = self.raid_disks[data_disk]
 
-        if self.d.is_readable(self.raid_disk_order[data_disk],off_disk,size2):
+        if self.d.is_readable(self.raid_disk_order[data_disk],off_disk,aligned_read_size):
             # No damaged sectors until the end of the chunck, so just read the data disk
             data_fd.seek(off_disk)
-            data = data_fd.read(size2)
+            data = data_fd.read(aligned_read_size)
         else:
+            # TODO : this is only for RAID5
             logging.warn('Try to recovering damaged chunck (raid_offset: 0x%011x, data_disk: %i, disk_offset: 0x%011x'
                     % (offset, self.raid_disk_order[data_disk], off_disk) )
             # Damaged sectors, check / recover every sector
@@ -284,7 +176,7 @@ class MyRaid():
             other_fds.remove(data_fd)
 
             data_arr = []
-            for s in xrange(off_disk, off_disk+size2, self.raid_sector_size):
+            for s in xrange(off_disk, off_disk+aligned_read_size, self.raid_sector_size):
                 if self.d.is_readable(self.raid_disk_order[data_disk],s,self.raid_sector_size):
                     # Current sector is readable from data disk, read it
                     logging.debug('-> 0x%011x : readable'%s)
@@ -297,15 +189,16 @@ class MyRaid():
                     ])
                     if recoverable:
                         logging.info('-> 0x%011x : recoverable'%s)
-                        data_arr.append( MyRaid.xor_blocks(other_fds, s,self.raid_sector_size)[1].tostring() )
+                        data_arr.append( MyRaidMaths.xor_blocks(other_fds, s,self.raid_sector_size)[1] )
                     else:
                         logging.warn('-> 0x%011x : unrecoverable'%s)
                         data_arr.append( '\0' * self.raid_sector_size)
             data = ''.join(data_arr)
 
         # Prevent short reads, seems mandatory for losetup'ing raid_result but kills performance
-        if size2 < size:
-            return ''.join( (data, self.read_raid_result(raid_type,offset+size2,size-size2) ) )
+        #TODO : make it activable per config
+        if aligned_read_size < size:
+            return ''.join( (data, self.read_raid_result(raid_type,offset+aligned_read_size,size-aligned_read_size) ) )
 
         return data
 
diff --git a/myraidmaths.py b/myraidmaths.py
new file mode 100644
index 0000000..c0d3b8b
--- /dev/null
+++ b/myraidmaths.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+
+# RaidGuessFS, a FUSE pseudo-filesystem to guess RAID parameters of a damaged device
+# Copyright (C) 2015 Ludovic Pouzenc <ludovic@pouzenc.fr>
+#
+# This file is part of RaidGuessFS.
+#
+# RaidGuessFS is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# RaidGuessFS is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with RaidGuessFS. If not, see <http://www.gnu.org/licenses/>
+
+import numpy
+
+class MyRaidMaths():
+    """Auxiliary class, managing RAID layer, mathematics algorithms"""
+
+    @staticmethod
+    def xor_blocks(fd_list, offset, size):
+        """Compute bitwise XOR against a bunch of disks slice"""
+        logging.info("Enter xor_blocks(fd_list(%i),0x%011x,%d)"%(len(fd_list), offset, size))
+        
+        if size % 8 != 0:
+            raise ValueError('xor_blocks : size must be multiple of 8')
+        dt = numpy.dtype('<Q8')
+
+        fd_list[0].seek(offset)
+        str_b1=fd_list[0].read(size)
+        numpy_b1 = numpy.fromstring(str_b1, dtype=dt)
+        all_zero = (numpy.count_nonzero(numpy_b1) == 0 )
+        any_zero = all_zero
+
+        for fd in fd_list[1:]:
+            fd.seek(offset)
+            str_b2=fd.read(size)
+            numpy_b2 = numpy.fromstring(str_b2, dtype=dt)
+            b2_zero = (numpy.count_nonzero(numpy_b2) == 0 )
+            if all_zero == True:
+                all_zero = b2_zero
+            if any_zero == False:
+                any_zero = b2_zero
+
+            numpy.bitwise_xor(numpy_b1,numpy_b2,numpy_b1)
+
+        if all_zero == True:
+            result = 'z'
+        elif numpy.count_nonzero(numpy_b1) == 0:
+            if any_zero:
+                result = 'g'
+            else:
+                result = 'G'
+        else:
+            result = 'b'
+
+        #import sys, binascii
+        #print sys.stderr.write(binascii.hexlify(numpy_b1))
+        return (result,numpy_b1.tostring())
+
+    @staticmethod
+    def apply_raid_layout(wanted_raid_offset, wanted_read_size, raid_type, raid_layout, raid_chunk_size, raid_disk_count, raid_start, nested_subraid):
+        """Returns disk numbers, offset and so on to known where read data for a given RAID offset/size and layout"""
+
+        if raid_type == '0':
+            segment_no = wanted_raid_offset / raid_chunk_size
+            segment_off = wanted_raid_offset % raid_chunk_size
+            stripe_no = segment_no / raid_disk_count
+            subraid_no = -1
+            par_disk = -1
+            data_disk = segment_no % raid_disk_count
+            off_disk = raid_start + stripe_no * raid_chunk_size + segment_off
+            aligned_read_size = min(wanted_read_size, (segment_no+1) * raid_chunk_size - wanted_raid_offset)
+
+        elif raid_type == '1':
+            segment_no = -1
+            segment_off = -1
+            stripe_no = -1
+            subraid_no = -1
+            par_disk = 1
+            data_disk = 0
+            off_disk = raid_start + wanted_raid_offset
+            aligned_read_size = wanted_read_size
+
+        elif raid_type == '5':
+            segment_no = wanted_raid_offset / raid_chunk_size
+            segment_off = wanted_raid_offset % raid_chunk_size
+            stripe_no = segment_no / (raid_disk_count-1)
+            subraid_no = -1
+
+            if raid_layout in ['ls','la']:
+                par_disk = (raid_disk_count-1) - (stripe_no % raid_disk_count)
+            else: # raid_layout in ['rs','ra']:
+                par_disk = stripe_no % raid_disk_count
+
+            if raid_layout in ['ls','rs']:
+                data_disk = (par_disk+1 + (segment_no % (raid_disk_count-1)) ) % raid_disk_count
+            else: # raid_layout in ['la','ra']:
+                data_disk = segment_no % (raid_disk_count-1)
+                if data_disk >= par_disk:
+                    data_disk = data_disk + 1
+
+            off_disk = raid_start + stripe_no * raid_chunk_size + segment_off
+            # Note : could make error-free shorter reads than asked but convince the reader to be chunck aligned, which is great for perf
+            aligned_read_size = min(wanted_read_size, (segment_no+1) * raid_chunk_size - wanted_raid_offset)
+
+        elif raid_type == '5+0':
+            subraid_disk_count = raid_disk_count / nested_subraid
+            segment_no = wanted_raid_offset / raid_chunk_size
+            segment_off = wanted_raid_offset % raid_chunk_size
+            stripe_no = segment_no / (raid_disk_count - nested_subraid)
+            subraid_no = (segment_no / (subraid_disk_count-1) ) % nested_subraid
+
+            if raid_layout in ['ls','la']:
+                subraid_par_disk = (subraid_disk_count-1) - (stripe_no % subraid_disk_count)
+            else: # raid_layout in ['rs','ra']:
+                subraid_par_disk = stripe_no % subraid_disk_count
+
+            if raid_layout in ['ls','rs']:
+                subraid_data_disk = (subraid_par_disk+1 + (segment_no % (subraid_disk_count-1)) ) % subraid_disk_count
+            else: # raid_layout in ['la','ra']:
+                subraid_data_disk = segment_no % (subraid_disk_count-1)
+                if subraid_data_disk >= subraid_par_disk:
+                    subraid_data_disk = subraid_data_disk + 1
+
+            par_disk  = subraid_no * subraid_disk_count + subraid_par_disk
+            data_disk = subraid_no * subraid_disk_count + subraid_data_disk
+
+            off_disk = raid_start + stripe_no * raid_chunk_size + segment_off
+            # Note : could make error-free shorter reads than asked but convince the reader to be chunck aligned, which is great for perf
+            aligned_read_size = min(wanted_read_size, (segment_no+1) * raid_chunk_size - wanted_raid_offset)
+
+        return (segment_no, segment_off, stripe_no, subraid_no, par_disk, data_disk, off_disk, aligned_read_size)