/js/src/nanojit/CodeAlloc.cpp

  1/* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
  2/* vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5) */
  3/* ***** BEGIN LICENSE BLOCK *****
  4 * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  5 *
  6 * The contents of this file are subject to the Mozilla Public License Version
  7 * 1.1 (the "License"); you may not use this file except in compliance with
  8 * the License. You may obtain a copy of the License at
  9 * http://www.mozilla.org/MPL/
 10 *
 11 * Software distributed under the License is distributed on an "AS IS" basis,
 12 * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
 13 * for the specific language governing rights and limitations under the
 14 * License.
 15 *
 16 * The Original Code is [Open Source Virtual Machine].
 17 *
 18 * The Initial Developer of the Original Code is
 19 * Adobe System Incorporated.
 20 * Portions created by the Initial Developer are Copyright (C) 2004-2007
 21 * the Initial Developer. All Rights Reserved.
 22 *
 23 * Contributor(s):
 24 *   Adobe AS3 Team
 25 *
 26 * Alternatively, the contents of this file may be used under the terms of
 27 * either the GNU General Public License Version 2 or later (the "GPL"), or
 28 * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 29 * in which case the provisions of the GPL or the LGPL are applicable instead
 30 * of those above. If you wish to allow use of your version of this file only
 31 * under the terms of either the GPL or the LGPL, and not to allow others to
 32 * use your version of this file under the terms of the MPL, indicate your
 33 * decision by deleting the provisions above and replace them with the notice
 34 * and other provisions required by the GPL or the LGPL. If you do not delete
 35 * the provisions above, a recipient may use your version of this file under
 36 * the terms of any one of the MPL, the GPL or the LGPL.
 37 *
 38 * ***** END LICENSE BLOCK ***** */
 39
 40#include "nanojit.h"
 41
 42//#define DOPROF
 43#include "../vprof/vprof.h"
 44
 45#ifdef FEATURE_NANOJIT
 46
 47namespace nanojit
 48{
 49    static const bool verbose = false;
 50#ifdef VMCFG_VTUNE
 51    // vtune jit profiling api can't handle non-contiguous methods,
 52    // so make the allocation size huge to avoid non-contiguous methods
 53    static const int pagesPerAlloc = 128; // 1MB
 54#elif defined(NANOJIT_ARM)
 55    // ARM requires single-page allocations, due to the constant pool that
 56    // lives on each page that must be reachable by a 4kb pcrel load.
 57    static const int pagesPerAlloc = 1;
 58#else
 59    static const int pagesPerAlloc = 16;
 60#endif
 61
 62    CodeAlloc::CodeAlloc()
 63        : heapblocks(0)
 64        , availblocks(0)
 65        , totalAllocated(0)
 66        , bytesPerPage(VMPI_getVMPageSize())
 67        , bytesPerAlloc(pagesPerAlloc * bytesPerPage)
 68    {
 69    }
 70
 71    CodeAlloc::~CodeAlloc() {
 72        reset();
 73    }
 74
 75    void CodeAlloc::reset() {
 76        // give all memory back to gcheap.  Assumption is that all
 77        // code is done being used by now.
 78        for (CodeList* hb = heapblocks; hb != 0; ) {
 79            _nvprof("free page",1);
 80            CodeList* next = hb->next;
 81            CodeList* fb = firstBlock(hb);
 82            markBlockWrite(fb);
 83            freeCodeChunk(fb, bytesPerAlloc);
 84            totalAllocated -= bytesPerAlloc;
 85            hb = next;
 86        }
 87        NanoAssert(!totalAllocated);
 88        heapblocks = availblocks = 0;
 89    }
 90
 91    CodeList* CodeAlloc::firstBlock(CodeList* term) {
 92        // use uintptr_t, rather than char*, to avoid "increases required alignment" warning
 93        uintptr_t end = (uintptr_t)alignUp(term, bytesPerPage);
 94        return (CodeList*) (end - (uintptr_t)bytesPerAlloc);
 95    }
 96
 97    static int round(size_t x) {
 98        return (int)((x + 512) >> 10);
 99    }
100
101    void CodeAlloc::getStats(size_t& total, size_t& frag_size, size_t& free_size) {
102        total = 0;
103        frag_size = 0;
104        free_size = 0;
105        int free_count = 0;
106        for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) {
107            total += bytesPerAlloc;
108            for (CodeList* b = hb->lower; b != 0; b = b->lower) {
109                if (b->isFree) {
110                    free_count++;
111                    free_size += b->blockSize();
112                    if (b->size() < minAllocSize)
113                        frag_size += b->blockSize();
114                }
115            }
116        }
117    }
118
119    void CodeAlloc::logStats() {
120        size_t total, frag_size, free_size;
121        getStats(total, frag_size, free_size);
122        avmplus::AvmLog("code-heap: %dk free %dk fragmented %d\n",
123            round(total), round(free_size), frag_size);
124    }
125
126    inline void CodeAlloc::markBlockWrite(CodeList* b) {
127        NanoAssert(b->terminator != NULL);
128        CodeList* term = b->terminator;
129        if (term->isExec) {
130            markCodeChunkWrite(firstBlock(term), bytesPerAlloc);
131            term->isExec = false;
132        }
133    }
134
135   void CodeAlloc::alloc(NIns* &start, NIns* &end, size_t byteLimit) {
136        if (!availblocks) {
137            // no free mem, get more
138            addMem();
139        }
140
141        // grab a block
142        NanoAssert(!byteLimit || byteLimit > blkSpaceFor(2));  // if a limit is imposed it must be bigger than 2x minimum block size (see below)
143        markBlockWrite(availblocks);
144        CodeList* b = removeBlock(availblocks);
145
146        // limit imposed (byteLimit > 0) and the block is too big?  then break it apart
147        if (byteLimit > 0 && b->size() > byteLimit) {
148
149            size_t consume;   // # bytes to extract from the free block
150
151            // enough space to carve out a perfectly sized blk?  (leaving at least a full free blk)
152            if (b->size() >= byteLimit + headerSpaceFor(1) + blkSpaceFor(1)) {
153                // yes, then take exactly what we need
154                consume = byteLimit + headerSpaceFor(1);
155            } else {
156                // no, then we should only take the min amount
157                consume = blkSpaceFor(1);
158
159                // ... and since b->size() > byteLimit && byteLimit > blkSpaceFor(2)
160                NanoAssert( b->size() > blkSpaceFor(2) );
161                NanoAssert( b->size() - consume > blkSpaceFor(1) );  // thus, we know that at least 1 blk left.
162            }
163
164            // break block into 2 pieces, returning the lower portion to the free list
165            CodeList* higher = b->higher;
166            b->end = (NIns*) ( (uintptr_t)b->end - consume );
167            CodeList* b1 = b->higher;
168            higher->lower = b1;
169            b1->higher = higher;
170            b1->lower = b;
171            b1->terminator = b->terminator;
172            NanoAssert(b->size() > minAllocSize);
173            addBlock(availblocks, b);  // put back the rest of the block
174            b = b1;
175        }
176        NanoAssert(b->size() >= minAllocSize);
177        b->next = 0; // not technically needed (except for debug builds), but good hygiene.
178        b->isFree = false;
179        start = b->start();
180        end = b->end;
181        if (verbose)
182            avmplus::AvmLog("CodeAlloc(%p).alloc %p-%p %d\n", this, start, end, int(end-start));
183        debug_only(sanity_check();)
184    }
185
186    void CodeAlloc::free(NIns* start, NIns *end) {
187        NanoAssert(heapblocks);
188        CodeList *blk = getBlock(start, end);
189        if (verbose)
190            avmplus::AvmLog("free %p-%p %d\n", start, end, (int)blk->size());
191
192        NanoAssert(!blk->isFree);
193
194        // coalesce adjacent blocks.
195        bool already_on_avail_list;
196
197        if (blk->lower && blk->lower->isFree) {
198            // combine blk into blk->lower (destroy blk)
199            CodeList* lower = blk->lower;
200            CodeList* higher = blk->higher;
201            already_on_avail_list = lower->size() >= minAllocSize;
202            lower->higher = higher;
203            higher->lower = lower;
204            blk = lower;
205        }
206        else
207            already_on_avail_list = false;
208
209        // the last block in each heapblock is a terminator block,
210        // which is never free, therefore blk->higher != null
211        if (blk->higher->isFree) {
212            CodeList *higher = blk->higher->higher;
213            CodeList *coalescedBlock = blk->higher;
214
215            if ( coalescedBlock->size() >= minAllocSize ) {
216                // Unlink coalescedBlock from the available block chain.
217                if ( availblocks == coalescedBlock ) {
218                    removeBlock(availblocks);
219                }
220                else {
221                    CodeList* free_block = availblocks;
222                    while (free_block->next != coalescedBlock) {
223                        NanoAssert(free_block->size() >= minAllocSize);
224                        NanoAssert(free_block->isFree);
225                        NanoAssert(free_block->next);
226                        free_block = free_block->next;
227                    }
228                    NanoAssert(free_block->next == coalescedBlock);
229                    free_block->next = coalescedBlock->next;
230                }
231            }
232
233            // combine blk->higher into blk (destroy coalescedBlock)
234            blk->higher = higher;
235            higher->lower = blk;
236        }
237        blk->isFree = true;
238        NanoAssert(!blk->lower || !blk->lower->isFree);
239        NanoAssert(blk->higher && !blk->higher->isFree);
240        //memset(blk->start(), 0xCC, blk->size()); // INT 3 instruction
241        if ( !already_on_avail_list && blk->size() >= minAllocSize )
242            addBlock(availblocks, blk);
243
244        NanoAssert(heapblocks);
245        debug_only(sanity_check();)
246    }
247
248    void CodeAlloc::freeAll(CodeList* &code) {
249        while (code) {
250            CodeList *b = removeBlock(code);
251            free(b->start(), b->end);
252        }
253    }
254
255    void CodeAlloc::flushICache(CodeList* &blocks) {
256        for (CodeList *b = blocks; b != 0; b = b->next)
257            flushICache(b->start(), b->size());
258    }
259
260#if defined(AVMPLUS_UNIX) && defined(NANOJIT_ARM)
261#if defined(__APPLE__)
262#include <libkern/OSCacheControl.h>
263#else
264#include <asm/unistd.h>
265extern "C" void __clear_cache(char *BEG, char *END);
266#endif
267#endif
268
269#if defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS)
270#include <asm/cachectl.h>
271extern  "C" int cacheflush(char *addr, int nbytes, int cache);
272#endif
273
274#ifdef AVMPLUS_SPARC
275// Note: the linux #define provided by the compiler.
276#ifdef linux  // bugzilla 502369
277void sync_instruction_memory(caddr_t v, u_int len)
278{
279    caddr_t end = v + len;
280    caddr_t p = v;
281    while (p < end) {
282        asm("flush %0" : : "r" (p));
283        p += 32;
284    }
285}
286#else
287extern  "C" void sync_instruction_memory(caddr_t v, u_int len);
288#endif
289#endif
290
291#if defined NANOJIT_IA32 || defined NANOJIT_X64
292    // intel chips have dcache/icache interlock
293    void CodeAlloc::flushICache(void *start, size_t len) {
294        // Tell Valgrind that new code has been generated, and it must flush
295        // any translations it has for the memory range generated into.
296        (void)start;
297        (void)len;
298        VALGRIND_DISCARD_TRANSLATIONS(start, len);
299    }
300
301#elif defined NANOJIT_ARM && defined DARWIN
302    void CodeAlloc::flushICache(void *, size_t) {
303        VMPI_debugBreak();
304    }
305
306#elif defined AVMPLUS_MAC && defined NANOJIT_PPC
307
308#  ifdef NANOJIT_64BIT
309    extern "C" void sys_icache_invalidate(const void*, size_t len);
310    extern "C" void sys_dcache_flush(const void*, size_t len);
311
312    // mac 64bit requires 10.5 so use that api
313    void CodeAlloc::flushICache(void *start, size_t len) {
314        sys_dcache_flush(start, len);
315        sys_icache_invalidate(start, len);
316    }
317#  else
318    // mac ppc 32 could be 10.0 or later
319    // uses MakeDataExecutable() from Carbon api, OSUtils.h
320    // see http://developer.apple.com/documentation/Carbon/Reference/Memory_Manag_nt_Utilities/Reference/reference.html#//apple_ref/c/func/MakeDataExecutable
321    void CodeAlloc::flushICache(void *start, size_t len) {
322        MakeDataExecutable(start, len);
323    }
324#  endif
325
326#elif defined NANOJIT_ARM && defined VMCFG_SYMBIAN
327    void CodeAlloc::flushICache(void *ptr, size_t len) {
328        uint32_t start = (uint32_t)ptr;
329        uint32_t rangeEnd = start + len;
330        User::IMB_Range((TAny*)start, (TAny*)rangeEnd);
331    }
332
333#elif defined AVMPLUS_SPARC
334    // fixme: sync_instruction_memory is a solaris api, test for solaris not sparc
335    void CodeAlloc::flushICache(void *start, size_t len) {
336            sync_instruction_memory((char*)start, len);
337    }
338
339#elif defined NANOJIT_SH4
340#include <asm/cachectl.h> /* CACHEFLUSH_*, */
341#include <sys/syscall.h>  /* __NR_cacheflush, */
342    void CodeAlloc::flushICache(void *start, size_t len) {
343        syscall(__NR_cacheflush, start, len, CACHEFLUSH_D_WB | CACHEFLUSH_I);
344    }
345
346#elif defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS)
347    void CodeAlloc::flushICache(void *start, size_t len) {
348        // FIXME Use synci on MIPS32R2
349        cacheflush((char *)start, len, BCACHE);
350    }
351
352#elif defined AVMPLUS_UNIX
353    #ifdef ANDROID
354    void CodeAlloc::flushICache(void *start, size_t len) {
355        cacheflush((int)start, (int)start + len, 0);
356    }
357    #elif defined(AVMPLUS_ARM) && defined(__APPLE__)
358    void CodeAlloc::flushICache(void *start, size_t len) {
359        sys_dcache_flush(start, len);
360    }
361    #else
362    // fixme: __clear_cache is a libgcc feature, test for libgcc or gcc
363    void CodeAlloc::flushICache(void *start, size_t len) {
364        __clear_cache((char*)start, (char*)start + len);
365    }
366    #endif
367#endif // AVMPLUS_MAC && NANOJIT_PPC
368
369    void CodeAlloc::addBlock(CodeList* &blocks, CodeList* b) {
370        NanoAssert(b->terminator != NULL);  // should not be mucking with terminator blocks
371        b->next = blocks;
372        blocks = b;
373    }
374
375    void CodeAlloc::addMem() {
376        void *mem = allocCodeChunk(bytesPerAlloc); // allocations never fail
377        totalAllocated += bytesPerAlloc;
378        NanoAssert(mem != NULL); // see allocCodeChunk contract in CodeAlloc.h
379        _nvprof("alloc page", uintptr_t(mem)>>12);
380
381        CodeList* b = (CodeList*)mem;
382        b->lower = 0;
383        b->next = 0;
384        b->end = (NIns*) (uintptr_t(mem) + bytesPerAlloc - sizeofMinBlock);
385        b->isFree = true;
386
387        // create a tiny terminator block, add to fragmented list, this way
388        // all other blocks have a valid block at b->higher
389        CodeList* terminator = b->higher;
390        b->terminator = terminator;
391        terminator->lower = b;
392        terminator->end = 0; // this is how we identify the terminator
393        terminator->isFree = false;
394        terminator->isExec = false;
395        terminator->terminator = 0;
396        debug_only(sanity_check();)
397
398        // add terminator to heapblocks list so we can track whole blocks
399        terminator->next = heapblocks;
400        heapblocks = terminator;
401
402        addBlock(availblocks, b); // add to free list
403    }
404
405    CodeList* CodeAlloc::getBlock(NIns* start, NIns* end) {
406        CodeList* b = (CodeList*) (uintptr_t(start) - offsetof(CodeList, code));
407        NanoAssert(b->end == end && b->next == 0); (void) end;
408        return b;
409    }
410
411    CodeList* CodeAlloc::removeBlock(CodeList* &blocks) {
412        CodeList* b = blocks;
413        NanoAssert(b != NULL);
414        NanoAssert(b->terminator != NULL);  // should not be mucking with terminator blocks
415        blocks = b->next;
416        b->next = 0;
417        return b;
418    }
419
420    void CodeAlloc::add(CodeList* &blocks, NIns* start, NIns* end) {
421        addBlock(blocks, getBlock(start, end));
422    }
423
424    /**
425     * split a block by freeing the hole in the middle defined by [holeStart,holeEnd),
426     * and adding the used prefix and suffix parts to the blocks CodeList.
427     */
428    void CodeAlloc::addRemainder(CodeList* &blocks, NIns* start, NIns* end, NIns* holeStart, NIns* holeEnd) {
429        NanoAssert(start < end && start <= holeStart && holeStart <= holeEnd && holeEnd <= end);
430        // shrink the hole by aligning holeStart forward and holeEnd backward
431        holeStart = (NIns*) ((uintptr_t(holeStart) + sizeof(NIns*)-1) & ~(sizeof(NIns*)-1));
432        holeEnd = (NIns*) (uintptr_t(holeEnd) & ~(sizeof(NIns*)-1));
433        // hole needs to be big enough for 2 headers + 1 block of free space (subtraction not used in check to avoid wraparound)
434        size_t minHole = headerSpaceFor(2) + blkSpaceFor(1);
435        if (uintptr_t(holeEnd) < minHole + uintptr_t(holeStart) ) {
436            // the hole is too small to make a new free block and a new used block. just keep
437            // the whole original block and don't free anything.
438            add(blocks, start, end);
439        } else if (holeStart == start && holeEnd == end) {
440            // totally empty block.  free whole start-end range
441            this->free(start, end);
442        } else if (holeStart == start) {
443            // hole is lower-aligned with start, so just need one new block
444            // b1 b2
445            CodeList* b1 = getBlock(start, end);
446            CodeList* b2 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code));
447            b2->terminator = b1->terminator;
448            b2->isFree = false;
449            b2->next = 0;
450            b2->higher = b1->higher;
451            b2->lower = b1;
452            b2->higher->lower = b2;
453            b1->higher = b2;
454            debug_only(sanity_check();)
455            this->free(b1->start(), b1->end);
456            addBlock(blocks, b2);
457        } else if (holeEnd == end) {
458            // hole is right-aligned with end, just need one new block
459            // todo
460            NanoAssert(false);
461        } else {
462            // there's enough space left to split into three blocks (two new ones)
463            CodeList* b1 = getBlock(start, end);
464            CodeList* b2 = (CodeList*) (void*) holeStart;
465            CodeList* b3 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code));
466            b1->higher = b2;
467            b2->lower = b1;
468            b2->higher = b3;
469            b2->isFree = false; // redundant, since we're about to free, but good hygiene
470            b2->terminator = b1->terminator;
471            b3->lower = b2;
472            b3->end = end;
473            b3->isFree = false;
474            b3->higher->lower = b3;
475            b3->terminator = b1->terminator;
476            b2->next = 0;
477            b3->next = 0;
478            debug_only(sanity_check();)
479            this->free(b2->start(), b2->end);
480            addBlock(blocks, b3);
481            addBlock(blocks, b1);
482        }
483    }
484
485#ifdef PERFM
486    // This method is used only for profiling purposes.
487    // See CodegenLIR::emitMD() in Tamarin for an example.
488
489    size_t CodeAlloc::size(const CodeList* blocks) {
490        size_t size = 0;
491        for (const CodeList* b = blocks; b != 0; b = b->next)
492            size += int((uintptr_t)b->end - (uintptr_t)b);
493        return size;
494    }
495#endif
496
497    size_t CodeAlloc::size() {
498        return totalAllocated;
499    }
500
501    // check that all block neighbors are correct
502    #ifdef _DEBUG
503    void CodeAlloc::sanity_check() {
504        for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) {
505            NanoAssert(hb->higher == 0);
506            for (CodeList* b = hb->lower; b != 0; b = b->lower) {
507                NanoAssert(b->higher->lower == b);
508            }
509            bool b = checkChunkMark(firstBlock(hb), bytesPerAlloc, hb->isExec);
510            NanoAssertMsg(b, "Chunk access mode differs from that expected");
511        }
512        for (CodeList* avail = this->availblocks; avail; avail = avail->next) {
513            NanoAssert(avail->isFree && avail->size() >= minAllocSize);
514        }
515
516        #if CROSS_CHECK_FREE_LIST
517        for(CodeList* term = heapblocks; term; term = term->next) {
518            for(CodeList* hb = term->lower; hb; hb = hb->lower) {
519                if (hb->isFree && hb->size() >= minAllocSize) {
520                    bool found_on_avail = false;
521                    for (CodeList* avail = this->availblocks; !found_on_avail && avail; avail = avail->next) {
522                        found_on_avail = avail == hb;
523                    }
524
525                    NanoAssert(found_on_avail);
526                }
527            }
528        }
529        for (CodeList* avail = this->availblocks; avail; avail = avail->next) {
530            bool found_in_heapblocks = false;
531            for(CodeList* term = heapblocks; !found_in_heapblocks && term; term = term->next) {
532                for(CodeList* hb = term->lower; !found_in_heapblocks && hb; hb = hb->lower) {
533                    found_in_heapblocks = hb == avail;
534                }
535            }
536            NanoAssert(found_in_heapblocks);
537        }
538        #endif /* CROSS_CHECK_FREE_LIST */
539    }
540    #endif
541
542    // Loop through a list of blocks marking the chunks executable.  If we encounter
543    // multiple blocks in the same chunk, only the first block will cause the
544    // chunk to become executable, the other calls will no-op (isExec flag checked)
545    void CodeAlloc::markExec(CodeList* &blocks) {
546        for (CodeList *b = blocks; b != 0; b = b->next) {
547            markChunkExec(b->terminator);
548        }
549    }
550
551    // Variant of markExec(CodeList*) that walks all heapblocks (i.e. chunks) marking
552    // each one executable.   On systems where bytesPerAlloc is low (i.e. have lots
553    // of elements in the list) this can be expensive.
554    void CodeAlloc::markAllExec() {
555        for (CodeList* hb = heapblocks; hb != NULL; hb = hb->next) {
556            markChunkExec(hb);
557        }
558    }
559
560    // make an entire chunk executable
561    void CodeAlloc::markChunkExec(CodeList* term) {
562        NanoAssert(term->terminator == NULL);
563        if (!term->isExec) {
564            term->isExec = true;
565            markCodeChunkExec(firstBlock(term), bytesPerAlloc);
566        }
567        debug_only(sanity_check();)
568    }
569}
570#endif // FEATURE_NANOJIT