"""Module containing a memory memory manager which provides a sliding window on a number of memory mapped files""" import os import sys from mmap import mmap, ACCESS_READ from mmap import ALLOCATIONGRANULARITY __all__ = ["align_to_mmap", "is_64_bit", "MapWindow", "MapRegion", "MapRegionList", "ALLOCATIONGRANULARITY"] #{ Utilities def align_to_mmap(num, round_up): """ Align the given integer number to the closest page offset, which usually is 4096 bytes. :param round_up: if True, the next higher multiple of page size is used, otherwise the lower page_size will be used (i.e. if True, 1 becomes 4096, otherwise it becomes 0) :return: num rounded to closest page""" res = (num // ALLOCATIONGRANULARITY) * ALLOCATIONGRANULARITY if round_up and (res != num): res += ALLOCATIONGRANULARITY # END handle size return res def is_64_bit(): """:return: True if the system is 64 bit. Otherwise it can be assumed to be 32 bit""" return sys.maxsize > (1 << 32) - 1 #}END utilities #{ Utility Classes class MapWindow: """Utility type which is used to snap windows towards each other, and to adjust their size""" __slots__ = ( 'ofs', # offset into the file in bytes 'size' # size of the window in bytes ) def __init__(self, offset, size): self.ofs = offset self.size = size def __repr__(self): return "MapWindow(%i, %i)" % (self.ofs, self.size) @classmethod def from_region(cls, region): """:return: new window from a region""" return cls(region._b, region.size()) def ofs_end(self): return self.ofs + self.size def align(self): """Assures the previous window area is contained in the new one""" nofs = align_to_mmap(self.ofs, 0) self.size += self.ofs - nofs # keep size constant self.ofs = nofs self.size = align_to_mmap(self.size, 1) def extend_left_to(self, window, max_size): """Adjust the offset to start where the given window on our left ends if possible, but don't make yourself larger than max_size. The resize will assure that the new window still contains the old window area""" rofs = self.ofs - window.ofs_end() nsize = rofs + self.size rofs -= nsize - min(nsize, max_size) self.ofs = self.ofs - rofs self.size += rofs def extend_right_to(self, window, max_size): """Adjust the size to make our window end where the right window begins, but don't get larger than max_size""" self.size = min(self.size + (window.ofs - self.ofs_end()), max_size) class MapRegion: """Defines a mapped region of memory, aligned to pagesizes **Note:** deallocates used region automatically on destruction""" __slots__ = [ '_b', # beginning of mapping '_mf', # mapped memory chunk (as returned by mmap) '_uc', # total amount of usages '_size', # cached size of our memory map '__weakref__' ] #{ Configuration #} END configuration def __init__(self, path_or_fd, ofs, size, flags=0): """Initialize a region, allocate the memory map :param path_or_fd: path to the file to map, or the opened file descriptor :param ofs: **aligned** offset into the file to be mapped :param size: if size is larger then the file on disk, the whole file will be allocated the the size automatically adjusted :param flags: additional flags to be given when opening the file. :raise Exception: if no memory can be allocated""" self._b = ofs self._size = 0 self._uc = 0 if isinstance(path_or_fd, int): fd = path_or_fd else: fd = os.open(path_or_fd, os.O_RDONLY | getattr(os, 'O_BINARY', 0) | flags) # END handle fd try: kwargs = dict(access=ACCESS_READ, offset=ofs) corrected_size = size sizeofs = ofs # have to correct size, otherwise (instead of the c version) it will # bark that the size is too large ... many extra file accesses because # if this ... argh ! actual_size = min(os.fstat(fd).st_size - sizeofs, corrected_size) self._mf = mmap(fd, actual_size, **kwargs) # END handle memory mode self._size = len(self._mf) finally: if isinstance(path_or_fd, str): os.close(fd) # END only close it if we opened it # END close file handle # We assume the first one to use us keeps us around self.increment_client_count() def __repr__(self): return "MapRegion<%i, %i>" % (self._b, self.size()) #{ Interface def buffer(self): """:return: a buffer containing the memory""" return self._mf def map(self): """:return: a memory map containing the memory""" return self._mf def ofs_begin(self): """:return: absolute byte offset to the first byte of the mapping""" return self._b def size(self): """:return: total size of the mapped region in bytes""" return self._size def ofs_end(self): """:return: Absolute offset to one byte beyond the mapping into the file""" return self._b + self._size def includes_ofs(self, ofs): """:return: True if the given offset can be read in our mapped region""" return self._b <= ofs < self._b + self._size def client_count(self): """:return: number of clients currently using this region""" return self._uc def increment_client_count(self, ofs = 1): """Adjust the usage count by the given positive or negative offset. If usage count equals 0, we will auto-release our resources :return: True if we released resources, False otherwise. In the latter case, we can still be used""" self._uc += ofs assert self._uc > -1, "Increments must match decrements, usage counter negative: %i" % self._uc if self.client_count() == 0: self.release() return True else: return False # end handle release def release(self): """Release all resources this instance might hold. Must only be called if there usage_count() is zero""" self._mf.close() #} END interface class MapRegionList(list): """List of MapRegion instances associating a path with a list of regions.""" __slots__ = ( '_path_or_fd', # path or file descriptor which is mapped by all our regions '_file_size' # total size of the file we map ) def __new__(cls, path): return super().__new__(cls) def __init__(self, path_or_fd): self._path_or_fd = path_or_fd self._file_size = None def path_or_fd(self): """:return: path or file descriptor we are attached to""" return self._path_or_fd def file_size(self): """:return: size of file we manager""" if self._file_size is None: if isinstance(self._path_or_fd, str): self._file_size = os.stat(self._path_or_fd).st_size else: self._file_size = os.fstat(self._path_or_fd).st_size # END handle path type # END update file size return self._file_size #} END utility classes