/js/src/nanojit/CodeAlloc.cpp

http://github.com/zpao/v8monkey · C++ · 570 lines · 410 code · 61 blank · 99 comment · 96 complexity · 31258570cd81584b36ec5185b81d9dcd MD5 · raw file

  1. /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 4 -*- */
  2. /* vi: set ts=4 sw=4 expandtab: (add to ~/.vimrc: set modeline modelines=5) */
  3. /* ***** BEGIN LICENSE BLOCK *****
  4. * Version: MPL 1.1/GPL 2.0/LGPL 2.1
  5. *
  6. * The contents of this file are subject to the Mozilla Public License Version
  7. * 1.1 (the "License"); you may not use this file except in compliance with
  8. * the License. You may obtain a copy of the License at
  9. * http://www.mozilla.org/MPL/
  10. *
  11. * Software distributed under the License is distributed on an "AS IS" basis,
  12. * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13. * for the specific language governing rights and limitations under the
  14. * License.
  15. *
  16. * The Original Code is [Open Source Virtual Machine].
  17. *
  18. * The Initial Developer of the Original Code is
  19. * Adobe System Incorporated.
  20. * Portions created by the Initial Developer are Copyright (C) 2004-2007
  21. * the Initial Developer. All Rights Reserved.
  22. *
  23. * Contributor(s):
  24. * Adobe AS3 Team
  25. *
  26. * Alternatively, the contents of this file may be used under the terms of
  27. * either the GNU General Public License Version 2 or later (the "GPL"), or
  28. * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29. * in which case the provisions of the GPL or the LGPL are applicable instead
  30. * of those above. If you wish to allow use of your version of this file only
  31. * under the terms of either the GPL or the LGPL, and not to allow others to
  32. * use your version of this file under the terms of the MPL, indicate your
  33. * decision by deleting the provisions above and replace them with the notice
  34. * and other provisions required by the GPL or the LGPL. If you do not delete
  35. * the provisions above, a recipient may use your version of this file under
  36. * the terms of any one of the MPL, the GPL or the LGPL.
  37. *
  38. * ***** END LICENSE BLOCK ***** */
  39. #include "nanojit.h"
  40. //#define DOPROF
  41. #include "../vprof/vprof.h"
  42. #ifdef FEATURE_NANOJIT
  43. namespace nanojit
  44. {
  45. static const bool verbose = false;
  46. #ifdef VMCFG_VTUNE
  47. // vtune jit profiling api can't handle non-contiguous methods,
  48. // so make the allocation size huge to avoid non-contiguous methods
  49. static const int pagesPerAlloc = 128; // 1MB
  50. #elif defined(NANOJIT_ARM)
  51. // ARM requires single-page allocations, due to the constant pool that
  52. // lives on each page that must be reachable by a 4kb pcrel load.
  53. static const int pagesPerAlloc = 1;
  54. #else
  55. static const int pagesPerAlloc = 16;
  56. #endif
  57. CodeAlloc::CodeAlloc()
  58. : heapblocks(0)
  59. , availblocks(0)
  60. , totalAllocated(0)
  61. , bytesPerPage(VMPI_getVMPageSize())
  62. , bytesPerAlloc(pagesPerAlloc * bytesPerPage)
  63. {
  64. }
  65. CodeAlloc::~CodeAlloc() {
  66. reset();
  67. }
  68. void CodeAlloc::reset() {
  69. // give all memory back to gcheap. Assumption is that all
  70. // code is done being used by now.
  71. for (CodeList* hb = heapblocks; hb != 0; ) {
  72. _nvprof("free page",1);
  73. CodeList* next = hb->next;
  74. CodeList* fb = firstBlock(hb);
  75. markBlockWrite(fb);
  76. freeCodeChunk(fb, bytesPerAlloc);
  77. totalAllocated -= bytesPerAlloc;
  78. hb = next;
  79. }
  80. NanoAssert(!totalAllocated);
  81. heapblocks = availblocks = 0;
  82. }
  83. CodeList* CodeAlloc::firstBlock(CodeList* term) {
  84. // use uintptr_t, rather than char*, to avoid "increases required alignment" warning
  85. uintptr_t end = (uintptr_t)alignUp(term, bytesPerPage);
  86. return (CodeList*) (end - (uintptr_t)bytesPerAlloc);
  87. }
  88. static int round(size_t x) {
  89. return (int)((x + 512) >> 10);
  90. }
  91. void CodeAlloc::getStats(size_t& total, size_t& frag_size, size_t& free_size) {
  92. total = 0;
  93. frag_size = 0;
  94. free_size = 0;
  95. int free_count = 0;
  96. for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) {
  97. total += bytesPerAlloc;
  98. for (CodeList* b = hb->lower; b != 0; b = b->lower) {
  99. if (b->isFree) {
  100. free_count++;
  101. free_size += b->blockSize();
  102. if (b->size() < minAllocSize)
  103. frag_size += b->blockSize();
  104. }
  105. }
  106. }
  107. }
  108. void CodeAlloc::logStats() {
  109. size_t total, frag_size, free_size;
  110. getStats(total, frag_size, free_size);
  111. avmplus::AvmLog("code-heap: %dk free %dk fragmented %d\n",
  112. round(total), round(free_size), frag_size);
  113. }
  114. inline void CodeAlloc::markBlockWrite(CodeList* b) {
  115. NanoAssert(b->terminator != NULL);
  116. CodeList* term = b->terminator;
  117. if (term->isExec) {
  118. markCodeChunkWrite(firstBlock(term), bytesPerAlloc);
  119. term->isExec = false;
  120. }
  121. }
  122. void CodeAlloc::alloc(NIns* &start, NIns* &end, size_t byteLimit) {
  123. if (!availblocks) {
  124. // no free mem, get more
  125. addMem();
  126. }
  127. // grab a block
  128. NanoAssert(!byteLimit || byteLimit > blkSpaceFor(2)); // if a limit is imposed it must be bigger than 2x minimum block size (see below)
  129. markBlockWrite(availblocks);
  130. CodeList* b = removeBlock(availblocks);
  131. // limit imposed (byteLimit > 0) and the block is too big? then break it apart
  132. if (byteLimit > 0 && b->size() > byteLimit) {
  133. size_t consume; // # bytes to extract from the free block
  134. // enough space to carve out a perfectly sized blk? (leaving at least a full free blk)
  135. if (b->size() >= byteLimit + headerSpaceFor(1) + blkSpaceFor(1)) {
  136. // yes, then take exactly what we need
  137. consume = byteLimit + headerSpaceFor(1);
  138. } else {
  139. // no, then we should only take the min amount
  140. consume = blkSpaceFor(1);
  141. // ... and since b->size() > byteLimit && byteLimit > blkSpaceFor(2)
  142. NanoAssert( b->size() > blkSpaceFor(2) );
  143. NanoAssert( b->size() - consume > blkSpaceFor(1) ); // thus, we know that at least 1 blk left.
  144. }
  145. // break block into 2 pieces, returning the lower portion to the free list
  146. CodeList* higher = b->higher;
  147. b->end = (NIns*) ( (uintptr_t)b->end - consume );
  148. CodeList* b1 = b->higher;
  149. higher->lower = b1;
  150. b1->higher = higher;
  151. b1->lower = b;
  152. b1->terminator = b->terminator;
  153. NanoAssert(b->size() > minAllocSize);
  154. addBlock(availblocks, b); // put back the rest of the block
  155. b = b1;
  156. }
  157. NanoAssert(b->size() >= minAllocSize);
  158. b->next = 0; // not technically needed (except for debug builds), but good hygiene.
  159. b->isFree = false;
  160. start = b->start();
  161. end = b->end;
  162. if (verbose)
  163. avmplus::AvmLog("CodeAlloc(%p).alloc %p-%p %d\n", this, start, end, int(end-start));
  164. debug_only(sanity_check();)
  165. }
  166. void CodeAlloc::free(NIns* start, NIns *end) {
  167. NanoAssert(heapblocks);
  168. CodeList *blk = getBlock(start, end);
  169. if (verbose)
  170. avmplus::AvmLog("free %p-%p %d\n", start, end, (int)blk->size());
  171. NanoAssert(!blk->isFree);
  172. // coalesce adjacent blocks.
  173. bool already_on_avail_list;
  174. if (blk->lower && blk->lower->isFree) {
  175. // combine blk into blk->lower (destroy blk)
  176. CodeList* lower = blk->lower;
  177. CodeList* higher = blk->higher;
  178. already_on_avail_list = lower->size() >= minAllocSize;
  179. lower->higher = higher;
  180. higher->lower = lower;
  181. blk = lower;
  182. }
  183. else
  184. already_on_avail_list = false;
  185. // the last block in each heapblock is a terminator block,
  186. // which is never free, therefore blk->higher != null
  187. if (blk->higher->isFree) {
  188. CodeList *higher = blk->higher->higher;
  189. CodeList *coalescedBlock = blk->higher;
  190. if ( coalescedBlock->size() >= minAllocSize ) {
  191. // Unlink coalescedBlock from the available block chain.
  192. if ( availblocks == coalescedBlock ) {
  193. removeBlock(availblocks);
  194. }
  195. else {
  196. CodeList* free_block = availblocks;
  197. while (free_block->next != coalescedBlock) {
  198. NanoAssert(free_block->size() >= minAllocSize);
  199. NanoAssert(free_block->isFree);
  200. NanoAssert(free_block->next);
  201. free_block = free_block->next;
  202. }
  203. NanoAssert(free_block->next == coalescedBlock);
  204. free_block->next = coalescedBlock->next;
  205. }
  206. }
  207. // combine blk->higher into blk (destroy coalescedBlock)
  208. blk->higher = higher;
  209. higher->lower = blk;
  210. }
  211. blk->isFree = true;
  212. NanoAssert(!blk->lower || !blk->lower->isFree);
  213. NanoAssert(blk->higher && !blk->higher->isFree);
  214. //memset(blk->start(), 0xCC, blk->size()); // INT 3 instruction
  215. if ( !already_on_avail_list && blk->size() >= minAllocSize )
  216. addBlock(availblocks, blk);
  217. NanoAssert(heapblocks);
  218. debug_only(sanity_check();)
  219. }
  220. void CodeAlloc::freeAll(CodeList* &code) {
  221. while (code) {
  222. CodeList *b = removeBlock(code);
  223. free(b->start(), b->end);
  224. }
  225. }
  226. void CodeAlloc::flushICache(CodeList* &blocks) {
  227. for (CodeList *b = blocks; b != 0; b = b->next)
  228. flushICache(b->start(), b->size());
  229. }
  230. #if defined(AVMPLUS_UNIX) && defined(NANOJIT_ARM)
  231. #if defined(__APPLE__)
  232. #include <libkern/OSCacheControl.h>
  233. #else
  234. #include <asm/unistd.h>
  235. extern "C" void __clear_cache(char *BEG, char *END);
  236. #endif
  237. #endif
  238. #if defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS)
  239. #include <asm/cachectl.h>
  240. extern "C" int cacheflush(char *addr, int nbytes, int cache);
  241. #endif
  242. #ifdef AVMPLUS_SPARC
  243. // Note: the linux #define provided by the compiler.
  244. #ifdef linux // bugzilla 502369
  245. void sync_instruction_memory(caddr_t v, u_int len)
  246. {
  247. caddr_t end = v + len;
  248. caddr_t p = v;
  249. while (p < end) {
  250. asm("flush %0" : : "r" (p));
  251. p += 32;
  252. }
  253. }
  254. #else
  255. extern "C" void sync_instruction_memory(caddr_t v, u_int len);
  256. #endif
  257. #endif
  258. #if defined NANOJIT_IA32 || defined NANOJIT_X64
  259. // intel chips have dcache/icache interlock
  260. void CodeAlloc::flushICache(void *start, size_t len) {
  261. // Tell Valgrind that new code has been generated, and it must flush
  262. // any translations it has for the memory range generated into.
  263. (void)start;
  264. (void)len;
  265. VALGRIND_DISCARD_TRANSLATIONS(start, len);
  266. }
  267. #elif defined NANOJIT_ARM && defined DARWIN
  268. void CodeAlloc::flushICache(void *, size_t) {
  269. VMPI_debugBreak();
  270. }
  271. #elif defined AVMPLUS_MAC && defined NANOJIT_PPC
  272. # ifdef NANOJIT_64BIT
  273. extern "C" void sys_icache_invalidate(const void*, size_t len);
  274. extern "C" void sys_dcache_flush(const void*, size_t len);
  275. // mac 64bit requires 10.5 so use that api
  276. void CodeAlloc::flushICache(void *start, size_t len) {
  277. sys_dcache_flush(start, len);
  278. sys_icache_invalidate(start, len);
  279. }
  280. # else
  281. // mac ppc 32 could be 10.0 or later
  282. // uses MakeDataExecutable() from Carbon api, OSUtils.h
  283. // see http://developer.apple.com/documentation/Carbon/Reference/Memory_Manag_nt_Utilities/Reference/reference.html#//apple_ref/c/func/MakeDataExecutable
  284. void CodeAlloc::flushICache(void *start, size_t len) {
  285. MakeDataExecutable(start, len);
  286. }
  287. # endif
  288. #elif defined NANOJIT_ARM && defined VMCFG_SYMBIAN
  289. void CodeAlloc::flushICache(void *ptr, size_t len) {
  290. uint32_t start = (uint32_t)ptr;
  291. uint32_t rangeEnd = start + len;
  292. User::IMB_Range((TAny*)start, (TAny*)rangeEnd);
  293. }
  294. #elif defined AVMPLUS_SPARC
  295. // fixme: sync_instruction_memory is a solaris api, test for solaris not sparc
  296. void CodeAlloc::flushICache(void *start, size_t len) {
  297. sync_instruction_memory((char*)start, len);
  298. }
  299. #elif defined NANOJIT_SH4
  300. #include <asm/cachectl.h> /* CACHEFLUSH_*, */
  301. #include <sys/syscall.h> /* __NR_cacheflush, */
  302. void CodeAlloc::flushICache(void *start, size_t len) {
  303. syscall(__NR_cacheflush, start, len, CACHEFLUSH_D_WB | CACHEFLUSH_I);
  304. }
  305. #elif defined(AVMPLUS_UNIX) && defined(NANOJIT_MIPS)
  306. void CodeAlloc::flushICache(void *start, size_t len) {
  307. // FIXME Use synci on MIPS32R2
  308. cacheflush((char *)start, len, BCACHE);
  309. }
  310. #elif defined AVMPLUS_UNIX
  311. #ifdef ANDROID
  312. void CodeAlloc::flushICache(void *start, size_t len) {
  313. cacheflush((int)start, (int)start + len, 0);
  314. }
  315. #elif defined(AVMPLUS_ARM) && defined(__APPLE__)
  316. void CodeAlloc::flushICache(void *start, size_t len) {
  317. sys_dcache_flush(start, len);
  318. }
  319. #else
  320. // fixme: __clear_cache is a libgcc feature, test for libgcc or gcc
  321. void CodeAlloc::flushICache(void *start, size_t len) {
  322. __clear_cache((char*)start, (char*)start + len);
  323. }
  324. #endif
  325. #endif // AVMPLUS_MAC && NANOJIT_PPC
  326. void CodeAlloc::addBlock(CodeList* &blocks, CodeList* b) {
  327. NanoAssert(b->terminator != NULL); // should not be mucking with terminator blocks
  328. b->next = blocks;
  329. blocks = b;
  330. }
  331. void CodeAlloc::addMem() {
  332. void *mem = allocCodeChunk(bytesPerAlloc); // allocations never fail
  333. totalAllocated += bytesPerAlloc;
  334. NanoAssert(mem != NULL); // see allocCodeChunk contract in CodeAlloc.h
  335. _nvprof("alloc page", uintptr_t(mem)>>12);
  336. CodeList* b = (CodeList*)mem;
  337. b->lower = 0;
  338. b->next = 0;
  339. b->end = (NIns*) (uintptr_t(mem) + bytesPerAlloc - sizeofMinBlock);
  340. b->isFree = true;
  341. // create a tiny terminator block, add to fragmented list, this way
  342. // all other blocks have a valid block at b->higher
  343. CodeList* terminator = b->higher;
  344. b->terminator = terminator;
  345. terminator->lower = b;
  346. terminator->end = 0; // this is how we identify the terminator
  347. terminator->isFree = false;
  348. terminator->isExec = false;
  349. terminator->terminator = 0;
  350. debug_only(sanity_check();)
  351. // add terminator to heapblocks list so we can track whole blocks
  352. terminator->next = heapblocks;
  353. heapblocks = terminator;
  354. addBlock(availblocks, b); // add to free list
  355. }
  356. CodeList* CodeAlloc::getBlock(NIns* start, NIns* end) {
  357. CodeList* b = (CodeList*) (uintptr_t(start) - offsetof(CodeList, code));
  358. NanoAssert(b->end == end && b->next == 0); (void) end;
  359. return b;
  360. }
  361. CodeList* CodeAlloc::removeBlock(CodeList* &blocks) {
  362. CodeList* b = blocks;
  363. NanoAssert(b != NULL);
  364. NanoAssert(b->terminator != NULL); // should not be mucking with terminator blocks
  365. blocks = b->next;
  366. b->next = 0;
  367. return b;
  368. }
  369. void CodeAlloc::add(CodeList* &blocks, NIns* start, NIns* end) {
  370. addBlock(blocks, getBlock(start, end));
  371. }
  372. /**
  373. * split a block by freeing the hole in the middle defined by [holeStart,holeEnd),
  374. * and adding the used prefix and suffix parts to the blocks CodeList.
  375. */
  376. void CodeAlloc::addRemainder(CodeList* &blocks, NIns* start, NIns* end, NIns* holeStart, NIns* holeEnd) {
  377. NanoAssert(start < end && start <= holeStart && holeStart <= holeEnd && holeEnd <= end);
  378. // shrink the hole by aligning holeStart forward and holeEnd backward
  379. holeStart = (NIns*) ((uintptr_t(holeStart) + sizeof(NIns*)-1) & ~(sizeof(NIns*)-1));
  380. holeEnd = (NIns*) (uintptr_t(holeEnd) & ~(sizeof(NIns*)-1));
  381. // hole needs to be big enough for 2 headers + 1 block of free space (subtraction not used in check to avoid wraparound)
  382. size_t minHole = headerSpaceFor(2) + blkSpaceFor(1);
  383. if (uintptr_t(holeEnd) < minHole + uintptr_t(holeStart) ) {
  384. // the hole is too small to make a new free block and a new used block. just keep
  385. // the whole original block and don't free anything.
  386. add(blocks, start, end);
  387. } else if (holeStart == start && holeEnd == end) {
  388. // totally empty block. free whole start-end range
  389. this->free(start, end);
  390. } else if (holeStart == start) {
  391. // hole is lower-aligned with start, so just need one new block
  392. // b1 b2
  393. CodeList* b1 = getBlock(start, end);
  394. CodeList* b2 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code));
  395. b2->terminator = b1->terminator;
  396. b2->isFree = false;
  397. b2->next = 0;
  398. b2->higher = b1->higher;
  399. b2->lower = b1;
  400. b2->higher->lower = b2;
  401. b1->higher = b2;
  402. debug_only(sanity_check();)
  403. this->free(b1->start(), b1->end);
  404. addBlock(blocks, b2);
  405. } else if (holeEnd == end) {
  406. // hole is right-aligned with end, just need one new block
  407. // todo
  408. NanoAssert(false);
  409. } else {
  410. // there's enough space left to split into three blocks (two new ones)
  411. CodeList* b1 = getBlock(start, end);
  412. CodeList* b2 = (CodeList*) (void*) holeStart;
  413. CodeList* b3 = (CodeList*) (uintptr_t(holeEnd) - offsetof(CodeList, code));
  414. b1->higher = b2;
  415. b2->lower = b1;
  416. b2->higher = b3;
  417. b2->isFree = false; // redundant, since we're about to free, but good hygiene
  418. b2->terminator = b1->terminator;
  419. b3->lower = b2;
  420. b3->end = end;
  421. b3->isFree = false;
  422. b3->higher->lower = b3;
  423. b3->terminator = b1->terminator;
  424. b2->next = 0;
  425. b3->next = 0;
  426. debug_only(sanity_check();)
  427. this->free(b2->start(), b2->end);
  428. addBlock(blocks, b3);
  429. addBlock(blocks, b1);
  430. }
  431. }
  432. #ifdef PERFM
  433. // This method is used only for profiling purposes.
  434. // See CodegenLIR::emitMD() in Tamarin for an example.
  435. size_t CodeAlloc::size(const CodeList* blocks) {
  436. size_t size = 0;
  437. for (const CodeList* b = blocks; b != 0; b = b->next)
  438. size += int((uintptr_t)b->end - (uintptr_t)b);
  439. return size;
  440. }
  441. #endif
  442. size_t CodeAlloc::size() {
  443. return totalAllocated;
  444. }
  445. // check that all block neighbors are correct
  446. #ifdef _DEBUG
  447. void CodeAlloc::sanity_check() {
  448. for (CodeList* hb = heapblocks; hb != 0; hb = hb->next) {
  449. NanoAssert(hb->higher == 0);
  450. for (CodeList* b = hb->lower; b != 0; b = b->lower) {
  451. NanoAssert(b->higher->lower == b);
  452. }
  453. bool b = checkChunkMark(firstBlock(hb), bytesPerAlloc, hb->isExec);
  454. NanoAssertMsg(b, "Chunk access mode differs from that expected");
  455. }
  456. for (CodeList* avail = this->availblocks; avail; avail = avail->next) {
  457. NanoAssert(avail->isFree && avail->size() >= minAllocSize);
  458. }
  459. #if CROSS_CHECK_FREE_LIST
  460. for(CodeList* term = heapblocks; term; term = term->next) {
  461. for(CodeList* hb = term->lower; hb; hb = hb->lower) {
  462. if (hb->isFree && hb->size() >= minAllocSize) {
  463. bool found_on_avail = false;
  464. for (CodeList* avail = this->availblocks; !found_on_avail && avail; avail = avail->next) {
  465. found_on_avail = avail == hb;
  466. }
  467. NanoAssert(found_on_avail);
  468. }
  469. }
  470. }
  471. for (CodeList* avail = this->availblocks; avail; avail = avail->next) {
  472. bool found_in_heapblocks = false;
  473. for(CodeList* term = heapblocks; !found_in_heapblocks && term; term = term->next) {
  474. for(CodeList* hb = term->lower; !found_in_heapblocks && hb; hb = hb->lower) {
  475. found_in_heapblocks = hb == avail;
  476. }
  477. }
  478. NanoAssert(found_in_heapblocks);
  479. }
  480. #endif /* CROSS_CHECK_FREE_LIST */
  481. }
  482. #endif
  483. // Loop through a list of blocks marking the chunks executable. If we encounter
  484. // multiple blocks in the same chunk, only the first block will cause the
  485. // chunk to become executable, the other calls will no-op (isExec flag checked)
  486. void CodeAlloc::markExec(CodeList* &blocks) {
  487. for (CodeList *b = blocks; b != 0; b = b->next) {
  488. markChunkExec(b->terminator);
  489. }
  490. }
  491. // Variant of markExec(CodeList*) that walks all heapblocks (i.e. chunks) marking
  492. // each one executable. On systems where bytesPerAlloc is low (i.e. have lots
  493. // of elements in the list) this can be expensive.
  494. void CodeAlloc::markAllExec() {
  495. for (CodeList* hb = heapblocks; hb != NULL; hb = hb->next) {
  496. markChunkExec(hb);
  497. }
  498. }
  499. // make an entire chunk executable
  500. void CodeAlloc::markChunkExec(CodeList* term) {
  501. NanoAssert(term->terminator == NULL);
  502. if (!term->isExec) {
  503. term->isExec = true;
  504. markCodeChunkExec(firstBlock(term), bytesPerAlloc);
  505. }
  506. debug_only(sanity_check();)
  507. }
  508. }
  509. #endif // FEATURE_NANOJIT