PageRenderTime 112ms CodeModel.GetById 48ms app.highlight 56ms RepoModel.GetById 2ms app.codeStats 0ms

/pypy/rlib/rmmap.py

http://github.com/pypy/pypy
Python | 863 lines | 803 code | 37 blank | 23 comment | 23 complexity | 6cd13b71d23568658afccb7af8e980c0 MD5 | raw file
  1
  2from pypy.rpython.tool import rffi_platform
  3from pypy.rpython.lltypesystem import rffi, lltype, llmemory
  4from pypy.rlib import rposix
  5from pypy.translator.tool.cbuild import ExternalCompilationInfo
  6from pypy.rlib.nonconst import NonConstant
  7
  8import sys
  9import os
 10import platform
 11import stat
 12
 13_POSIX = os.name == "posix"
 14_MS_WINDOWS = os.name == "nt"
 15_LINUX = "linux" in sys.platform
 16_64BIT = "64bit" in platform.architecture()[0]
 17
 18class RValueError(Exception):
 19    def __init__(self, message):
 20        self.message = message
 21
 22class RTypeError(Exception):
 23    def __init__(self, message):
 24        self.message = message
 25
 26includes = ["sys/types.h"]
 27if _POSIX:
 28    includes += ['unistd.h', 'sys/mman.h']
 29elif _MS_WINDOWS:
 30    includes += ['winsock2.h','windows.h']
 31
 32class CConfig:
 33    _compilation_info_ = ExternalCompilationInfo(
 34        includes=includes,
 35        #pre_include_bits=['#ifndef _GNU_SOURCE\n' +
 36        #                  '#define _GNU_SOURCE\n' +
 37        #                  '#endif']
 38        # ^^^ _GNU_SOURCE is always defined by the ExternalCompilationInfo now
 39    )
 40    size_t = rffi_platform.SimpleType("size_t", rffi.LONG)
 41    off_t = rffi_platform.SimpleType("off_t", rffi.LONG)
 42
 43constants = {}
 44if _POSIX:
 45    # constants, look in sys/mman.h and platform docs for the meaning
 46    # some constants are linux only so they will be correctly exposed outside
 47    # depending on the OS
 48    constant_names = ['MAP_SHARED', 'MAP_PRIVATE',
 49                      'PROT_READ', 'PROT_WRITE',
 50                      'MS_SYNC']
 51    opt_constant_names = ['MAP_ANON', 'MAP_ANONYMOUS', 'MAP_NORESERVE',
 52                          'PROT_EXEC',
 53                          'MAP_DENYWRITE', 'MAP_EXECUTABLE']
 54    for name in constant_names:
 55        setattr(CConfig, name, rffi_platform.ConstantInteger(name))
 56    for name in opt_constant_names:
 57        setattr(CConfig, name, rffi_platform.DefinedConstantInteger(name))
 58
 59    CConfig.MREMAP_MAYMOVE = (
 60        rffi_platform.DefinedConstantInteger("MREMAP_MAYMOVE"))
 61    CConfig.has_mremap = rffi_platform.Has('mremap(NULL, 0, 0, 0)')
 62    # a dirty hack, this is probably a macro
 63
 64elif _MS_WINDOWS:
 65    constant_names = ['PAGE_READONLY', 'PAGE_READWRITE', 'PAGE_WRITECOPY',
 66                      'FILE_MAP_READ', 'FILE_MAP_WRITE', 'FILE_MAP_COPY',
 67                      'DUPLICATE_SAME_ACCESS', 'MEM_COMMIT', 'MEM_RESERVE',
 68                      'MEM_RELEASE', 'PAGE_EXECUTE_READWRITE', 'PAGE_NOACCESS']
 69    for name in constant_names:
 70        setattr(CConfig, name, rffi_platform.ConstantInteger(name))
 71
 72    from pypy.rlib import rwin32
 73
 74    from pypy.rlib.rwin32 import HANDLE, LPHANDLE
 75    from pypy.rlib.rwin32 import NULL_HANDLE, INVALID_HANDLE_VALUE
 76    from pypy.rlib.rwin32 import DWORD, WORD, DWORD_PTR, LPDWORD
 77    from pypy.rlib.rwin32 import BOOL, LPVOID, LPCSTR, SIZE_T
 78    from pypy.rlib.rwin32 import INT, LONG, PLONG
 79
 80# export the constants inside and outside. see __init__.py
 81cConfig = rffi_platform.configure(CConfig)
 82constants.update(cConfig)
 83
 84if _POSIX:
 85    # MAP_ANONYMOUS is not always present but it's always available at CPython level
 86    if constants["MAP_ANONYMOUS"] is None:
 87        constants["MAP_ANONYMOUS"] = constants["MAP_ANON"]
 88    assert constants["MAP_ANONYMOUS"] is not None
 89    constants["MAP_ANON"] = constants["MAP_ANONYMOUS"]
 90
 91locals().update(constants)
 92
 93_ACCESS_DEFAULT, ACCESS_READ, ACCESS_WRITE, ACCESS_COPY = range(4)
 94
 95if rffi.sizeof(off_t) > rffi.sizeof(lltype.Signed):
 96    HAVE_LARGEFILE_SUPPORT = True
 97else:
 98    HAVE_LARGEFILE_SUPPORT = False
 99
100def external(name, args, result, **kwargs):
101    unsafe = rffi.llexternal(name, args, result,
102                             compilation_info=CConfig._compilation_info_,
103                             **kwargs)
104    safe = rffi.llexternal(name, args, result,
105                           compilation_info=CConfig._compilation_info_,
106                           sandboxsafe=True, threadsafe=False,
107                           **kwargs)
108    return unsafe, safe
109
110def winexternal(name, args, result, **kwargs):
111    return rffi.llexternal(name, args, result,
112                           compilation_info=CConfig._compilation_info_,
113                           calling_conv='win',
114                           **kwargs)
115
116PTR = rffi.CCHARP
117
118c_memmove, _ = external('memmove', [PTR, PTR, size_t], lltype.Void)
119
120if _POSIX:
121    has_mremap = cConfig['has_mremap']
122    c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT,
123                               rffi.INT, off_t], PTR, macro=True)
124    # 'mmap' on linux32 is a macro that calls 'mmap64'
125    _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT)
126    c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT)
127    if has_mremap:
128        c_mremap, _ = external('mremap',
129                               [PTR, size_t, size_t, rffi.ULONG], PTR)
130
131    # this one is always safe
132    _, _get_page_size = external('getpagesize', [], rffi.INT)
133    _get_allocation_granularity = _get_page_size
134
135elif _MS_WINDOWS:
136
137    class ComplexCConfig:
138        _compilation_info_ = CConfig._compilation_info_
139
140        SYSINFO_STRUCT = rffi.CStruct(
141            'SYSINFO_STRUCT',
142                ("wProcessorArchitecture", WORD),
143                ("wReserved", WORD),
144            )
145
146        SYSINFO_UNION = rffi.CStruct(
147            'union SYSINFO_UNION',
148                ("dwOemId", DWORD),
149                ("_struct_", SYSINFO_STRUCT),
150            )
151        # sorry, I can't find a way to insert the above
152        # because the union field has no name
153        SYSTEM_INFO = rffi_platform.Struct(
154            'SYSTEM_INFO', [
155                ## ("_union_", SYSINFO_UNION),
156                ## instead, we put the smaller fields, here
157                ("wProcessorArchitecture", WORD),
158                ("wReserved", WORD),
159                ## should be a union. dwOemId is obsolete, anyway
160                ("dwPageSize", DWORD),
161                ("lpMinimumApplicationAddress", LPVOID),
162                ("lpMaximumApplicationAddress", LPVOID),
163                ("dwActiveProcessorMask", DWORD_PTR),
164                ("dwNumberOfProcessors", DWORD),
165                ("dwProcessorType", DWORD),
166                ("dwAllocationGranularity", DWORD),
167                ("wProcessorLevel", WORD),
168                ("wProcessorRevision", WORD),
169            ])
170
171    config = rffi_platform.configure(ComplexCConfig)
172    SYSTEM_INFO = config['SYSTEM_INFO']
173    SYSTEM_INFO_P = lltype.Ptr(SYSTEM_INFO)
174
175    GetSystemInfo = winexternal('GetSystemInfo', [SYSTEM_INFO_P], lltype.Void)
176    GetFileSize = winexternal('GetFileSize', [HANDLE, LPDWORD], DWORD)
177    GetCurrentProcess = winexternal('GetCurrentProcess', [], HANDLE)
178    DuplicateHandle = winexternal('DuplicateHandle', [HANDLE, HANDLE, HANDLE, LPHANDLE, DWORD, BOOL, DWORD], BOOL)
179    CreateFileMapping = winexternal('CreateFileMappingA', [HANDLE, rwin32.LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCSTR], HANDLE)
180    MapViewOfFile = winexternal('MapViewOfFile', [HANDLE, DWORD, DWORD, DWORD, SIZE_T], LPCSTR)##!!LPVOID)
181    UnmapViewOfFile = winexternal('UnmapViewOfFile', [LPCSTR], BOOL,
182                                  threadsafe=False)
183    FlushViewOfFile = winexternal('FlushViewOfFile', [LPCSTR, SIZE_T], BOOL)
184    SetFilePointer = winexternal('SetFilePointer', [HANDLE, LONG, PLONG, DWORD], DWORD)
185    SetEndOfFile = winexternal('SetEndOfFile', [HANDLE], BOOL)
186    VirtualAlloc = winexternal('VirtualAlloc',
187                               [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD],
188                               rffi.VOIDP)
189    # VirtualProtect is used in llarena and should not release the GIL
190    _VirtualProtect = winexternal('VirtualProtect',
191                                  [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD],
192                                  BOOL,
193                                  _nowrapper=True)
194    def VirtualProtect(addr, size, mode, oldmode_ptr):
195        return _VirtualProtect(addr,
196                               rffi.cast(rffi.SIZE_T, size),
197                               rffi.cast(DWORD, mode),
198                               oldmode_ptr)
199    VirtualProtect._annspecialcase_ = 'specialize:ll'
200    VirtualFree = winexternal('VirtualFree',
201                              [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL)
202
203
204    def _get_page_size():
205        try:
206            si = rffi.make(SYSTEM_INFO)
207            GetSystemInfo(si)
208            return int(si.c_dwPageSize)
209        finally:
210            lltype.free(si, flavor="raw")
211
212    def _get_allocation_granularity():
213        try:
214            si = rffi.make(SYSTEM_INFO)
215            GetSystemInfo(si)
216            return int(si.c_dwAllocationGranularity)
217        finally:
218            lltype.free(si, flavor="raw")
219
220    def _get_file_size(handle):
221        # XXX use native Windows types like WORD
222        high_ref = lltype.malloc(LPDWORD.TO, 1, flavor='raw')
223        try:
224            low = GetFileSize(handle, high_ref)
225            low = rffi.cast(lltype.Signed, low)
226            # XXX should be propagate the real type, allowing
227            # for 2*sys.maxint?
228            high = high_ref[0]
229            high = rffi.cast(lltype.Signed, high)
230            # low might just happen to have the value INVALID_FILE_SIZE
231            # so we need to check the last error also
232            INVALID_FILE_SIZE = -1
233            if low == INVALID_FILE_SIZE:
234                err = rwin32.GetLastError()
235                if err:
236                    raise WindowsError(err, "mmap")
237            return low, high
238        finally:
239            lltype.free(high_ref, flavor='raw')
240
241    INVALID_HANDLE = INVALID_HANDLE_VALUE
242
243PAGESIZE = _get_page_size()
244ALLOCATIONGRANULARITY = _get_allocation_granularity()
245NULL = lltype.nullptr(PTR.TO)
246NODATA = lltype.nullptr(PTR.TO)
247
248class MMap(object):
249    def __init__(self, access, offset):
250        self.size = 0
251        self.pos = 0
252        self.access = access
253        self.offset = offset
254
255        if _MS_WINDOWS:
256            self.map_handle = NULL_HANDLE
257            self.file_handle = NULL_HANDLE
258            self.tagname = ""
259        elif _POSIX:
260            self.fd = -1
261            self.closed = False
262
263    def check_valid(self):
264        if _MS_WINDOWS:
265            to_close = self.map_handle == INVALID_HANDLE
266        elif _POSIX:
267            to_close = self.closed
268
269        if to_close:
270            raise RValueError("map closed or invalid")
271
272    def check_writeable(self):
273        if not (self.access != ACCESS_READ):
274            raise RTypeError("mmap can't modify a readonly memory map.")
275
276    def check_resizeable(self):
277        if not (self.access == ACCESS_WRITE or self.access == _ACCESS_DEFAULT):
278            raise RTypeError("mmap can't resize a readonly or copy-on-write memory map.")
279
280    def setdata(self, data, size):
281        """Set the internal data and map size from a PTR."""
282        assert size >= 0
283        self.data = data
284        self.size = size
285
286    def close(self):
287        if _MS_WINDOWS:
288            if self.size > 0:
289                self.unmapview()
290                self.setdata(NODATA, 0)
291            if self.map_handle != INVALID_HANDLE:
292                rwin32.CloseHandle(self.map_handle)
293                self.map_handle = INVALID_HANDLE
294            if self.file_handle != INVALID_HANDLE:
295                rwin32.CloseHandle(self.file_handle)
296                self.file_handle = INVALID_HANDLE
297        elif _POSIX:
298            self.closed = True
299            if self.fd != -1:
300                # XXX this is buggy - raising in an RPython del is not a good
301                #     idea, we should swallow the exception or ignore the
302                #     underlaying close error code
303                os.close(self.fd)
304                self.fd = -1
305            if self.size > 0:
306                c_munmap_safe(self.getptr(0), self.size)
307                self.setdata(NODATA, 0)
308
309    def __del__(self):
310        self.close()
311
312    def unmapview(self):
313        UnmapViewOfFile(self.getptr(0))
314
315    def read_byte(self):
316        self.check_valid()
317
318        if self.pos < self.size:
319            value = self.data[self.pos]
320            self.pos += 1
321            return value
322        else:
323            raise RValueError("read byte out of range")
324
325    def readline(self):
326        self.check_valid()
327
328        data = self.data
329        for pos in xrange(self.pos, self.size):
330            if data[pos] == '\n':
331                eol = pos + 1 # we're interested in the position after new line
332                break
333        else: # no '\n' found
334            eol = self.size
335
336        res = "".join([data[i] for i in range(self.pos, eol)])
337        self.pos += len(res)
338        return res
339
340    def read(self, num=-1):
341        self.check_valid()
342
343        if num < 0:
344            # read all
345            eol = self.size
346        else:
347            eol = self.pos + num
348            # silently adjust out of range requests
349            if eol > self.size:
350                eol = self.size
351
352        res = [self.data[i] for i in range(self.pos, eol)]
353        res = "".join(res)
354        self.pos += len(res)
355        return res
356
357    def find(self, tofind, start, end, reverse=False):
358        self.check_valid()
359
360        # XXX naive! how can we reuse the rstr algorithm?
361        if start < 0:
362            start += self.size
363            if start < 0:
364                start = 0
365        if end < 0:
366            end += self.size
367            if end < 0:
368                end = 0
369        elif end > self.size:
370            end = self.size
371        #
372        upto = end - len(tofind)
373        if not reverse:
374            step = 1
375            p = start
376            if p > upto:
377                return -1      # failure (empty range to search)
378        else:
379            step = -1
380            p = upto
381            upto = start
382            if p < upto:
383                return -1      # failure (empty range to search)
384        #
385        data = self.data
386        while True:
387            assert p >= 0
388            for q in range(len(tofind)):
389                if data[p+q] != tofind[q]:
390                    break     # position 'p' is not a match
391            else:
392                # full match
393                return p
394            #
395            if p == upto:
396                return -1   # failure
397            p += step
398
399    def seek(self, pos, whence=0):
400        self.check_valid()
401
402        dist = pos
403        how = whence
404
405        if how == 0: # relative to start
406            where = dist
407        elif how == 1: # relative to current position
408            where = self.pos + dist
409        elif how == 2: # relative to the end
410            where = self.size + dist
411        else:
412            raise RValueError("unknown seek type")
413
414        if not (0 <= where <= self.size):
415            raise RValueError("seek out of range")
416
417        self.pos = where
418
419    def tell(self):
420        self.check_valid()
421        return self.pos
422
423    def file_size(self):
424        self.check_valid()
425
426        size = self.size
427        if _MS_WINDOWS:
428            if self.file_handle != INVALID_HANDLE:
429                low, high = _get_file_size(self.file_handle)
430                if not high and low <= sys.maxint:
431                    return low
432                # not so sure if the signed/unsigned strictness is a good idea:
433                high = rffi.cast(lltype.Unsigned, high)
434                low = rffi.cast(lltype.Unsigned, low)
435                size = (high << 32) + low
436                size = rffi.cast(lltype.Signed, size)
437        elif _POSIX:
438            st = os.fstat(self.fd)
439            size = st[stat.ST_SIZE]
440        return size
441
442    def write(self, data):
443        self.check_valid()
444        self.check_writeable()
445
446        data_len = len(data)
447        if self.pos + data_len > self.size:
448            raise RValueError("data out of range")
449
450        internaldata = self.data
451        start = self.pos
452        for i in range(data_len):
453            internaldata[start+i] = data[i]
454        self.pos = start + data_len
455
456    def write_byte(self, byte):
457        self.check_valid()
458
459        if len(byte) != 1:
460            raise RTypeError("write_byte() argument must be char")
461
462        self.check_writeable()
463        if self.pos >= self.size:
464            raise RValueError("write byte out of range")
465
466        self.data[self.pos] = byte[0]
467        self.pos += 1
468
469    def getptr(self, offset):
470        return rffi.ptradd(self.data, offset)
471
472    def flush(self, offset=0, size=0):
473        self.check_valid()
474
475        if size == 0:
476            size = self.size
477        if offset < 0 or size < 0 or offset + size > self.size:
478            raise RValueError("flush values out of range")
479        else:
480            start = self.getptr(offset)
481            if _MS_WINDOWS:
482                res = FlushViewOfFile(start, size)
483                # XXX res == 0 means that an error occurred, but in CPython
484                # this is not checked
485                return res
486            elif _POSIX:
487##                XXX why is this code here?  There is no equivalent in CPython
488##                if _LINUX:
489##                    # alignment of the address
490##                    value = cast(self.data, c_void_p).value
491##                    aligned_value = value & ~(PAGESIZE - 1)
492##                    # the size should be increased too. otherwise the final
493##                    # part is not "msynced"
494##                    new_size = size + value & (PAGESIZE - 1)
495                res = c_msync(start, size, MS_SYNC)
496                if res == -1:
497                    errno = rposix.get_errno()
498                    raise OSError(errno, os.strerror(errno))
499
500        return 0
501
502    def move(self, dest, src, count):
503        self.check_valid()
504
505        self.check_writeable()
506
507        # check boundings
508        if (src < 0 or dest < 0 or count < 0 or
509            src + count > self.size or dest + count > self.size):
510            raise RValueError("source or destination out of range")
511
512        datasrc = self.getptr(src)
513        datadest = self.getptr(dest)
514        c_memmove(datadest, datasrc, count)
515
516    def resize(self, newsize):
517        self.check_valid()
518
519        self.check_resizeable()
520
521        if _POSIX:
522            if not has_mremap:
523                raise RValueError("mmap: resizing not available--no mremap()")
524
525            # resize the underlying file first
526            os.ftruncate(self.fd, self.offset + newsize)
527
528            # now resize the mmap
529            newdata = c_mremap(self.getptr(0), self.size, newsize,
530                               MREMAP_MAYMOVE or 0)
531            self.setdata(newdata, newsize)
532        elif _MS_WINDOWS:
533            # disconnect the mapping
534            self.unmapview()
535            rwin32.CloseHandle(self.map_handle)
536
537            # move to the desired EOF position
538            if _64BIT:
539                newsize_high = (self.offset + newsize) >> 32
540                newsize_low = (self.offset + newsize) & 0xFFFFFFFF
541                offset_high = self.offset >> 32
542                offset_low = self.offset & 0xFFFFFFFF
543            else:
544                newsize_high = 0
545                newsize_low = self.offset + newsize
546                offset_high = 0
547                offset_low = self.offset
548
549            FILE_BEGIN = 0
550            high_ref = lltype.malloc(PLONG.TO, 1, flavor='raw')
551            try:
552                high_ref[0] = rffi.cast(LONG, newsize_high)
553                SetFilePointer(self.file_handle, newsize_low, high_ref,
554                               FILE_BEGIN)
555            finally:
556                lltype.free(high_ref, flavor='raw')
557            # resize the file
558            SetEndOfFile(self.file_handle)
559            # create another mapping object and remap the file view
560            res = CreateFileMapping(self.file_handle, NULL, PAGE_READWRITE,
561                                 newsize_high, newsize_low, self.tagname)
562            self.map_handle = res
563
564            dwErrCode = 0
565            if self.map_handle:
566                data = MapViewOfFile(self.map_handle, FILE_MAP_WRITE,
567                                     offset_high, offset_low, newsize)
568                if data:
569                    # XXX we should have a real LPVOID which must always be casted
570                    charp = rffi.cast(LPCSTR, data)
571                    self.setdata(charp, newsize)
572                    return
573            winerror = rwin32.lastWindowsError()
574            if self.map_handle:
575                rwin32.CloseHandle(self.map_handle)
576            self.map_handle = INVALID_HANDLE
577            raise winerror
578
579    def len(self):
580        self.check_valid()
581
582        return self.size
583
584    def getitem(self, index):
585        self.check_valid()
586        # simplified version, for rpython
587        if index < 0:
588            index += self.size
589        return self.data[index]
590
591    def setitem(self, index, value):
592        self.check_valid()
593        self.check_writeable()
594
595        if len(value) != 1:
596            raise RValueError("mmap assignment must be "
597                             "single-character string")
598        if index < 0:
599            index += self.size
600        self.data[index] = value[0]
601
602def _check_map_size(size):
603    if size < 0:
604        raise RTypeError("memory mapped size must be positive")
605
606if _POSIX:
607    def mmap(fileno, length, flags=MAP_SHARED,
608        prot=PROT_WRITE | PROT_READ, access=_ACCESS_DEFAULT, offset=0):
609
610        fd = fileno
611
612        # check access is not there when flags and prot are there
613        if access != _ACCESS_DEFAULT and ((flags != MAP_SHARED) or\
614                                          (prot != (PROT_WRITE | PROT_READ))):
615            raise RValueError("mmap can't specify both access and flags, prot.")
616
617        # check size boundaries
618        _check_map_size(length)
619        map_size = length
620        if offset < 0:
621            raise RValueError("negative offset")
622
623        if access == ACCESS_READ:
624            flags = MAP_SHARED
625            prot = PROT_READ
626        elif access == ACCESS_WRITE:
627            flags = MAP_SHARED
628            prot = PROT_READ | PROT_WRITE
629        elif access == ACCESS_COPY:
630            flags = MAP_PRIVATE
631            prot = PROT_READ | PROT_WRITE
632        elif access == _ACCESS_DEFAULT:
633            # map prot to access type
634            if prot & PROT_READ and prot & PROT_WRITE:
635                pass  # _ACCESS_DEFAULT
636            elif prot & PROT_WRITE:
637                access = ACCESS_WRITE
638            else:
639                access = ACCESS_READ
640        else:
641            raise RValueError("mmap invalid access parameter.")
642
643        # check file size
644        try:
645            st = os.fstat(fd)
646        except OSError:
647            pass     # ignore errors and trust map_size
648        else:
649            mode = st[stat.ST_MODE]
650            size = st[stat.ST_SIZE]
651            if stat.S_ISREG(mode):
652                if map_size == 0:
653                    if offset > size:
654                        raise RValueError(
655                            "mmap offset is greater than file size")
656                    map_size = int(size - offset)
657                    if map_size != size - offset:
658                        raise RValueError("mmap length is too large")
659                elif offset + map_size > size:
660                    raise RValueError("mmap length is greater than file size")
661
662        m = MMap(access, offset)
663        if fd == -1:
664            # Assume the caller wants to map anonymous memory.
665            # This is the same behaviour as Windows.  mmap.mmap(-1, size)
666            # on both Windows and Unix map anonymous memory.
667            m.fd = -1
668
669            flags |= MAP_ANONYMOUS
670
671        else:
672            m.fd = os.dup(fd)
673
674        # XXX if we use hintp below in alloc, the NonConstant
675        #     is necessary since we want a general version of c_mmap
676        #     to be annotated with a non-constant pointer.
677        res = c_mmap(NonConstant(NULL), map_size, prot, flags, fd, offset)
678        if res == rffi.cast(PTR, -1):
679            errno = rposix.get_errno()
680            raise OSError(errno, os.strerror(errno))
681
682        m.setdata(res, map_size)
683        return m
684
685    # XXX is this really necessary?
686    class Hint:
687        pos = -0x4fff0000   # for reproducible results
688    hint = Hint()
689
690    def alloc(map_size):
691        """Allocate memory.  This is intended to be used by the JIT,
692        so the memory has the executable bit set and gets allocated
693        internally in case of a sandboxed process.
694        """
695        flags = MAP_PRIVATE | MAP_ANONYMOUS
696        prot = PROT_EXEC | PROT_READ | PROT_WRITE
697        hintp = rffi.cast(PTR, hint.pos)
698        res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
699        if res == rffi.cast(PTR, -1):
700            # some systems (some versions of OS/X?) complain if they
701            # are passed a non-zero address.  Try again.
702            hintp = rffi.cast(PTR, 0)
703            res = c_mmap_safe(hintp, map_size, prot, flags, -1, 0)
704            if res == rffi.cast(PTR, -1):
705                raise MemoryError
706        else:
707            hint.pos += map_size
708        return res
709    alloc._annenforceargs_ = (int,)
710
711    free = c_munmap_safe
712
713elif _MS_WINDOWS:
714    def mmap(fileno, length, tagname="", access=_ACCESS_DEFAULT, offset=0):
715        # XXX flags is or-ed into access by now.
716        flags = 0
717        # check size boundaries
718        _check_map_size(length)
719        map_size = length
720        if offset < 0:
721            raise RValueError("negative offset")
722
723        flProtect = 0
724        dwDesiredAccess = 0
725        fh = NULL_HANDLE
726
727        if access == ACCESS_READ:
728            flProtect = PAGE_READONLY
729            dwDesiredAccess = FILE_MAP_READ
730        elif access == _ACCESS_DEFAULT or access == ACCESS_WRITE:
731            flProtect = PAGE_READWRITE
732            dwDesiredAccess = FILE_MAP_WRITE
733        elif access == ACCESS_COPY:
734            flProtect = PAGE_WRITECOPY
735            dwDesiredAccess = FILE_MAP_COPY
736        else:
737            raise RValueError("mmap invalid access parameter.")
738
739        # assume -1 and 0 both mean invalid file descriptor
740        # to 'anonymously' map memory.
741        if fileno != -1 and fileno != 0:
742            fh = rwin32.get_osfhandle(fileno)
743            # Win9x appears to need us seeked to zero
744            # SEEK_SET = 0
745            # libc._lseek(fileno, 0, SEEK_SET)
746
747            # check file size
748            try:
749                low, high = _get_file_size(fh)
750            except OSError:
751                pass     # ignore non-seeking files and errors and trust map_size
752            else:
753                if not high and low <= sys.maxint:
754                   size = low
755                else:   
756                    # not so sure if the signed/unsigned strictness is a good idea:
757                    high = rffi.cast(lltype.Unsigned, high)
758                    low = rffi.cast(lltype.Unsigned, low)
759                    size = (high << 32) + low
760                    size = rffi.cast(lltype.Signed, size)
761                if map_size == 0:
762                    if offset > size:
763                        raise RValueError(
764                            "mmap offset is greater than file size")
765                    map_size = int(size - offset)
766                    if map_size != size - offset:
767                        raise RValueError("mmap length is too large")
768                elif offset + map_size > size:
769                    raise RValueError("mmap length is greater than file size")
770
771        m = MMap(access, offset)
772        m.file_handle = INVALID_HANDLE
773        m.map_handle = INVALID_HANDLE
774        if fh:
775            # it is necessary to duplicate the handle, so the
776            # Python code can close it on us
777            handle_ref = lltype.malloc(LPHANDLE.TO, 1, flavor='raw')
778            handle_ref[0] = m.file_handle
779            try:
780                res = DuplicateHandle(GetCurrentProcess(), # source process handle
781                                      fh, # handle to be duplicated
782                                      GetCurrentProcess(), # target process handle
783                                      handle_ref, # result
784                                      0, # access - ignored due to options value
785                                      False, # inherited by child procs?
786                                      DUPLICATE_SAME_ACCESS) # options
787                if not res:
788                    raise rwin32.lastWindowsError()
789                m.file_handle = handle_ref[0]
790            finally:
791                lltype.free(handle_ref, flavor='raw')
792
793            if not map_size:
794                low, high = _get_file_size(fh)
795                if _64BIT:
796                    map_size = (low << 32) + 1
797                else:
798                    if high:
799                        # file is too large to map completely
800                        map_size = -1
801                    else:
802                        map_size = low
803
804        if tagname:
805            m.tagname = tagname
806
807        # DWORD is a 4-byte int. If int > 4-byte it must be divided
808        if _64BIT:
809            size_hi = (map_size + offset) >> 32
810            size_lo = (map_size + offset) & 0xFFFFFFFF
811            offset_hi = offset >> 32
812            offset_lo = offset & 0xFFFFFFFF
813        else:
814            size_hi = 0
815            size_lo = map_size + offset
816            offset_hi = 0
817            offset_lo = offset
818
819        flProtect |= flags
820        m.map_handle = CreateFileMapping(m.file_handle, NULL, flProtect,
821                                         size_hi, size_lo, m.tagname)
822
823        if m.map_handle:
824            data = MapViewOfFile(m.map_handle, dwDesiredAccess,
825                                 offset_hi, offset_lo, length)
826            if data:
827                # XXX we should have a real LPVOID which must always be casted
828                charp = rffi.cast(LPCSTR, data)
829                m.setdata(charp, map_size)
830                return m
831        winerror = rwin32.lastWindowsError()
832        if m.map_handle:
833            rwin32.CloseHandle(m.map_handle)
834        m.map_handle = INVALID_HANDLE
835        raise winerror
836
837    class Hint:
838        pos = -0x4fff0000   # for reproducible results
839    hint = Hint()
840    # XXX this has no effect on windows
841
842    def alloc(map_size):
843        """Allocate memory.  This is intended to be used by the JIT,
844        so the memory has the executable bit set.
845        XXX implement me: it should get allocated internally in
846        case of a sandboxed process
847        """
848        null = lltype.nullptr(rffi.VOIDP.TO)
849        res = VirtualAlloc(null, map_size, MEM_COMMIT|MEM_RESERVE,
850                           PAGE_EXECUTE_READWRITE)
851        if not res:
852            raise MemoryError
853        arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw')
854        VirtualProtect(res, map_size, PAGE_EXECUTE_READWRITE, arg)
855        lltype.free(arg, flavor='raw')
856        # ignore errors, just try
857        return res
858    alloc._annenforceargs_ = (int,)
859
860    def free(ptr, map_size):
861        VirtualFree(ptr, 0, MEM_RELEASE)
862
863# register_external here?