PageRenderTime 62ms CodeModel.GetById 25ms RepoModel.GetById 0ms app.codeStats 1ms

/arch/ia64/kernel/ivt.S

https://bitbucket.org/cresqo/cm7-p500-kernel
Assembly | 1689 lines | 1561 code | 45 blank | 83 comment | 34 complexity | 4b871dde0387bf86e5bbac9492f5de0c MD5 | raw file
Possible License(s): LGPL-2.0, AGPL-1.0, GPL-2.0
  1. /*
  2. * arch/ia64/kernel/ivt.S
  3. *
  4. * Copyright (C) 1998-2001, 2003, 2005 Hewlett-Packard Co
  5. * Stephane Eranian <eranian@hpl.hp.com>
  6. * David Mosberger <davidm@hpl.hp.com>
  7. * Copyright (C) 2000, 2002-2003 Intel Co
  8. * Asit Mallick <asit.k.mallick@intel.com>
  9. * Suresh Siddha <suresh.b.siddha@intel.com>
  10. * Kenneth Chen <kenneth.w.chen@intel.com>
  11. * Fenghua Yu <fenghua.yu@intel.com>
  12. *
  13. * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> TLB handling for SMP
  14. * 00/12/20 David Mosberger-Tang <davidm@hpl.hp.com> DTLB/ITLB handler now uses virtual PT.
  15. *
  16. * Copyright (C) 2005 Hewlett-Packard Co
  17. * Dan Magenheimer <dan.magenheimer@hp.com>
  18. * Xen paravirtualization
  19. * Copyright (c) 2008 Isaku Yamahata <yamahata at valinux co jp>
  20. * VA Linux Systems Japan K.K.
  21. * pv_ops.
  22. * Yaozu (Eddie) Dong <eddie.dong@intel.com>
  23. */
  24. /*
  25. * This file defines the interruption vector table used by the CPU.
  26. * It does not include one entry per possible cause of interruption.
  27. *
  28. * The first 20 entries of the table contain 64 bundles each while the
  29. * remaining 48 entries contain only 16 bundles each.
  30. *
  31. * The 64 bundles are used to allow inlining the whole handler for critical
  32. * interruptions like TLB misses.
  33. *
  34. * For each entry, the comment is as follows:
  35. *
  36. * // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
  37. * entry offset ----/ / / / /
  38. * entry number ---------/ / / /
  39. * size of the entry -------------/ / /
  40. * vector name -------------------------------------/ /
  41. * interruptions triggering this vector ----------------------/
  42. *
  43. * The table is 32KB in size and must be aligned on 32KB boundary.
  44. * (The CPU ignores the 15 lower bits of the address)
  45. *
  46. * Table is based upon EAS2.6 (Oct 1999)
  47. */
  48. #include <asm/asmmacro.h>
  49. #include <asm/break.h>
  50. #include <asm/kregs.h>
  51. #include <asm/asm-offsets.h>
  52. #include <asm/pgtable.h>
  53. #include <asm/processor.h>
  54. #include <asm/ptrace.h>
  55. #include <asm/system.h>
  56. #include <asm/thread_info.h>
  57. #include <asm/unistd.h>
  58. #include <asm/errno.h>
  59. #if 1
  60. # define PSR_DEFAULT_BITS psr.ac
  61. #else
  62. # define PSR_DEFAULT_BITS 0
  63. #endif
  64. #if 0
  65. /*
  66. * This lets you track the last eight faults that occurred on the CPU. Make sure ar.k2 isn't
  67. * needed for something else before enabling this...
  68. */
  69. # define DBG_FAULT(i) mov r16=ar.k2;; shl r16=r16,8;; add r16=(i),r16;;mov ar.k2=r16
  70. #else
  71. # define DBG_FAULT(i)
  72. #endif
  73. #include "minstate.h"
  74. #define FAULT(n) \
  75. mov r31=pr; \
  76. mov r19=n;; /* prepare to save predicates */ \
  77. br.sptk.many dispatch_to_fault_handler
  78. .section .text..ivt,"ax"
  79. .align 32768 // align on 32KB boundary
  80. .global ia64_ivt
  81. ia64_ivt:
  82. /////////////////////////////////////////////////////////////////////////////////////////
  83. // 0x0000 Entry 0 (size 64 bundles) VHPT Translation (8,20,47)
  84. ENTRY(vhpt_miss)
  85. DBG_FAULT(0)
  86. /*
  87. * The VHPT vector is invoked when the TLB entry for the virtual page table
  88. * is missing. This happens only as a result of a previous
  89. * (the "original") TLB miss, which may either be caused by an instruction
  90. * fetch or a data access (or non-access).
  91. *
  92. * What we do here is normal TLB miss handing for the _original_ miss,
  93. * followed by inserting the TLB entry for the virtual page table page
  94. * that the VHPT walker was attempting to access. The latter gets
  95. * inserted as long as page table entry above pte level have valid
  96. * mappings for the faulting address. The TLB entry for the original
  97. * miss gets inserted only if the pte entry indicates that the page is
  98. * present.
  99. *
  100. * do_page_fault gets invoked in the following cases:
  101. * - the faulting virtual address uses unimplemented address bits
  102. * - the faulting virtual address has no valid page table mapping
  103. */
  104. MOV_FROM_IFA(r16) // get address that caused the TLB miss
  105. #ifdef CONFIG_HUGETLB_PAGE
  106. movl r18=PAGE_SHIFT
  107. MOV_FROM_ITIR(r25)
  108. #endif
  109. ;;
  110. RSM_PSR_DT // use physical addressing for data
  111. mov r31=pr // save the predicate registers
  112. mov r19=IA64_KR(PT_BASE) // get page table base address
  113. shl r21=r16,3 // shift bit 60 into sign bit
  114. shr.u r17=r16,61 // get the region number into r17
  115. ;;
  116. shr.u r22=r21,3
  117. #ifdef CONFIG_HUGETLB_PAGE
  118. extr.u r26=r25,2,6
  119. ;;
  120. cmp.ne p8,p0=r18,r26
  121. sub r27=r26,r18
  122. ;;
  123. (p8) dep r25=r18,r25,2,6
  124. (p8) shr r22=r22,r27
  125. #endif
  126. ;;
  127. cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
  128. shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
  129. ;;
  130. (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
  131. srlz.d
  132. LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
  133. .pred.rel "mutex", p6, p7
  134. (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
  135. (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
  136. ;;
  137. (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
  138. (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
  139. cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
  140. #ifdef CONFIG_PGTABLE_4
  141. shr.u r28=r22,PUD_SHIFT // shift pud index into position
  142. #else
  143. shr.u r18=r22,PMD_SHIFT // shift pmd index into position
  144. #endif
  145. ;;
  146. ld8 r17=[r17] // get *pgd (may be 0)
  147. ;;
  148. (p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
  149. #ifdef CONFIG_PGTABLE_4
  150. dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
  151. ;;
  152. shr.u r18=r22,PMD_SHIFT // shift pmd index into position
  153. (p7) ld8 r29=[r28] // get *pud (may be 0)
  154. ;;
  155. (p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
  156. dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
  157. #else
  158. dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
  159. #endif
  160. ;;
  161. (p7) ld8 r20=[r17] // get *pmd (may be 0)
  162. shr.u r19=r22,PAGE_SHIFT // shift pte index into position
  163. ;;
  164. (p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
  165. dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
  166. ;;
  167. (p7) ld8 r18=[r21] // read *pte
  168. MOV_FROM_ISR(r19) // cr.isr bit 32 tells us if this is an insn miss
  169. ;;
  170. (p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
  171. MOV_FROM_IHA(r22) // get the VHPT address that caused the TLB miss
  172. ;; // avoid RAW on p7
  173. (p7) tbit.nz.unc p10,p11=r19,32 // is it an instruction TLB miss?
  174. dep r23=0,r20,0,PAGE_SHIFT // clear low bits to get page address
  175. ;;
  176. ITC_I_AND_D(p10, p11, r18, r24) // insert the instruction TLB entry and
  177. // insert the data TLB entry
  178. (p6) br.cond.spnt.many page_fault // handle bad address/page not present (page fault)
  179. MOV_TO_IFA(r22, r24)
  180. #ifdef CONFIG_HUGETLB_PAGE
  181. MOV_TO_ITIR(p8, r25, r24) // change to default page-size for VHPT
  182. #endif
  183. /*
  184. * Now compute and insert the TLB entry for the virtual page table. We never
  185. * execute in a page table page so there is no need to set the exception deferral
  186. * bit.
  187. */
  188. adds r24=__DIRTY_BITS_NO_ED|_PAGE_PL_0|_PAGE_AR_RW,r23
  189. ;;
  190. ITC_D(p7, r24, r25)
  191. ;;
  192. #ifdef CONFIG_SMP
  193. /*
  194. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  195. * cannot possibly affect the following loads:
  196. */
  197. dv_serialize_data
  198. /*
  199. * Re-check pagetable entry. If they changed, we may have received a ptc.g
  200. * between reading the pagetable and the "itc". If so, flush the entry we
  201. * inserted and retry. At this point, we have:
  202. *
  203. * r28 = equivalent of pud_offset(pgd, ifa)
  204. * r17 = equivalent of pmd_offset(pud, ifa)
  205. * r21 = equivalent of pte_offset(pmd, ifa)
  206. *
  207. * r29 = *pud
  208. * r20 = *pmd
  209. * r18 = *pte
  210. */
  211. ld8 r25=[r21] // read *pte again
  212. ld8 r26=[r17] // read *pmd again
  213. #ifdef CONFIG_PGTABLE_4
  214. ld8 r19=[r28] // read *pud again
  215. #endif
  216. cmp.ne p6,p7=r0,r0
  217. ;;
  218. cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
  219. #ifdef CONFIG_PGTABLE_4
  220. cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
  221. #endif
  222. mov r27=PAGE_SHIFT<<2
  223. ;;
  224. (p6) ptc.l r22,r27 // purge PTE page translation
  225. (p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
  226. ;;
  227. (p6) ptc.l r16,r27 // purge translation
  228. #endif
  229. mov pr=r31,-1 // restore predicate registers
  230. RFI
  231. END(vhpt_miss)
  232. .org ia64_ivt+0x400
  233. /////////////////////////////////////////////////////////////////////////////////////////
  234. // 0x0400 Entry 1 (size 64 bundles) ITLB (21)
  235. ENTRY(itlb_miss)
  236. DBG_FAULT(1)
  237. /*
  238. * The ITLB handler accesses the PTE via the virtually mapped linear
  239. * page table. If a nested TLB miss occurs, we switch into physical
  240. * mode, walk the page table, and then re-execute the PTE read and
  241. * go on normally after that.
  242. */
  243. MOV_FROM_IFA(r16) // get virtual address
  244. mov r29=b0 // save b0
  245. mov r31=pr // save predicates
  246. .itlb_fault:
  247. MOV_FROM_IHA(r17) // get virtual address of PTE
  248. movl r30=1f // load nested fault continuation point
  249. ;;
  250. 1: ld8 r18=[r17] // read *pte
  251. ;;
  252. mov b0=r29
  253. tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
  254. (p6) br.cond.spnt page_fault
  255. ;;
  256. ITC_I(p0, r18, r19)
  257. ;;
  258. #ifdef CONFIG_SMP
  259. /*
  260. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  261. * cannot possibly affect the following loads:
  262. */
  263. dv_serialize_data
  264. ld8 r19=[r17] // read *pte again and see if same
  265. mov r20=PAGE_SHIFT<<2 // setup page size for purge
  266. ;;
  267. cmp.ne p7,p0=r18,r19
  268. ;;
  269. (p7) ptc.l r16,r20
  270. #endif
  271. mov pr=r31,-1
  272. RFI
  273. END(itlb_miss)
  274. .org ia64_ivt+0x0800
  275. /////////////////////////////////////////////////////////////////////////////////////////
  276. // 0x0800 Entry 2 (size 64 bundles) DTLB (9,48)
  277. ENTRY(dtlb_miss)
  278. DBG_FAULT(2)
  279. /*
  280. * The DTLB handler accesses the PTE via the virtually mapped linear
  281. * page table. If a nested TLB miss occurs, we switch into physical
  282. * mode, walk the page table, and then re-execute the PTE read and
  283. * go on normally after that.
  284. */
  285. MOV_FROM_IFA(r16) // get virtual address
  286. mov r29=b0 // save b0
  287. mov r31=pr // save predicates
  288. dtlb_fault:
  289. MOV_FROM_IHA(r17) // get virtual address of PTE
  290. movl r30=1f // load nested fault continuation point
  291. ;;
  292. 1: ld8 r18=[r17] // read *pte
  293. ;;
  294. mov b0=r29
  295. tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
  296. (p6) br.cond.spnt page_fault
  297. ;;
  298. ITC_D(p0, r18, r19)
  299. ;;
  300. #ifdef CONFIG_SMP
  301. /*
  302. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  303. * cannot possibly affect the following loads:
  304. */
  305. dv_serialize_data
  306. ld8 r19=[r17] // read *pte again and see if same
  307. mov r20=PAGE_SHIFT<<2 // setup page size for purge
  308. ;;
  309. cmp.ne p7,p0=r18,r19
  310. ;;
  311. (p7) ptc.l r16,r20
  312. #endif
  313. mov pr=r31,-1
  314. RFI
  315. END(dtlb_miss)
  316. .org ia64_ivt+0x0c00
  317. /////////////////////////////////////////////////////////////////////////////////////////
  318. // 0x0c00 Entry 3 (size 64 bundles) Alt ITLB (19)
  319. ENTRY(alt_itlb_miss)
  320. DBG_FAULT(3)
  321. MOV_FROM_IFA(r16) // get address that caused the TLB miss
  322. movl r17=PAGE_KERNEL
  323. MOV_FROM_IPSR(p0, r21)
  324. movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
  325. mov r31=pr
  326. ;;
  327. #ifdef CONFIG_DISABLE_VHPT
  328. shr.u r22=r16,61 // get the region number into r21
  329. ;;
  330. cmp.gt p8,p0=6,r22 // user mode
  331. ;;
  332. THASH(p8, r17, r16, r23)
  333. ;;
  334. MOV_TO_IHA(p8, r17, r23)
  335. (p8) mov r29=b0 // save b0
  336. (p8) br.cond.dptk .itlb_fault
  337. #endif
  338. extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
  339. and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
  340. shr.u r18=r16,57 // move address bit 61 to bit 4
  341. ;;
  342. andcm r18=0x10,r18 // bit 4=~address-bit(61)
  343. cmp.ne p8,p0=r0,r23 // psr.cpl != 0?
  344. or r19=r17,r19 // insert PTE control bits into r19
  345. ;;
  346. or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
  347. (p8) br.cond.spnt page_fault
  348. ;;
  349. ITC_I(p0, r19, r18) // insert the TLB entry
  350. mov pr=r31,-1
  351. RFI
  352. END(alt_itlb_miss)
  353. .org ia64_ivt+0x1000
  354. /////////////////////////////////////////////////////////////////////////////////////////
  355. // 0x1000 Entry 4 (size 64 bundles) Alt DTLB (7,46)
  356. ENTRY(alt_dtlb_miss)
  357. DBG_FAULT(4)
  358. MOV_FROM_IFA(r16) // get address that caused the TLB miss
  359. movl r17=PAGE_KERNEL
  360. MOV_FROM_ISR(r20)
  361. movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
  362. MOV_FROM_IPSR(p0, r21)
  363. mov r31=pr
  364. mov r24=PERCPU_ADDR
  365. ;;
  366. #ifdef CONFIG_DISABLE_VHPT
  367. shr.u r22=r16,61 // get the region number into r21
  368. ;;
  369. cmp.gt p8,p0=6,r22 // access to region 0-5
  370. ;;
  371. THASH(p8, r17, r16, r25)
  372. ;;
  373. MOV_TO_IHA(p8, r17, r25)
  374. (p8) mov r29=b0 // save b0
  375. (p8) br.cond.dptk dtlb_fault
  376. #endif
  377. cmp.ge p10,p11=r16,r24 // access to per_cpu_data?
  378. tbit.z p12,p0=r16,61 // access to region 6?
  379. mov r25=PERCPU_PAGE_SHIFT << 2
  380. mov r26=PERCPU_PAGE_SIZE
  381. nop.m 0
  382. nop.b 0
  383. ;;
  384. (p10) mov r19=IA64_KR(PER_CPU_DATA)
  385. (p11) and r19=r19,r16 // clear non-ppn fields
  386. extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
  387. and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
  388. tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
  389. tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
  390. ;;
  391. (p10) sub r19=r19,r26
  392. MOV_TO_ITIR(p10, r25, r24)
  393. cmp.ne p8,p0=r0,r23
  394. (p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
  395. (p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr
  396. (p8) br.cond.spnt page_fault
  397. dep r21=-1,r21,IA64_PSR_ED_BIT,1
  398. ;;
  399. or r19=r19,r17 // insert PTE control bits into r19
  400. MOV_TO_IPSR(p6, r21, r24)
  401. ;;
  402. ITC_D(p7, r19, r18) // insert the TLB entry
  403. mov pr=r31,-1
  404. RFI
  405. END(alt_dtlb_miss)
  406. .org ia64_ivt+0x1400
  407. /////////////////////////////////////////////////////////////////////////////////////////
  408. // 0x1400 Entry 5 (size 64 bundles) Data nested TLB (6,45)
  409. ENTRY(nested_dtlb_miss)
  410. /*
  411. * In the absence of kernel bugs, we get here when the virtually mapped linear
  412. * page table is accessed non-speculatively (e.g., in the Dirty-bit, Instruction
  413. * Access-bit, or Data Access-bit faults). If the DTLB entry for the virtual page
  414. * table is missing, a nested TLB miss fault is triggered and control is
  415. * transferred to this point. When this happens, we lookup the pte for the
  416. * faulting address by walking the page table in physical mode and return to the
  417. * continuation point passed in register r30 (or call page_fault if the address is
  418. * not mapped).
  419. *
  420. * Input: r16: faulting address
  421. * r29: saved b0
  422. * r30: continuation address
  423. * r31: saved pr
  424. *
  425. * Output: r17: physical address of PTE of faulting address
  426. * r29: saved b0
  427. * r30: continuation address
  428. * r31: saved pr
  429. *
  430. * Clobbered: b0, r18, r19, r21, r22, psr.dt (cleared)
  431. */
  432. RSM_PSR_DT // switch to using physical data addressing
  433. mov r19=IA64_KR(PT_BASE) // get the page table base address
  434. shl r21=r16,3 // shift bit 60 into sign bit
  435. MOV_FROM_ITIR(r18)
  436. ;;
  437. shr.u r17=r16,61 // get the region number into r17
  438. extr.u r18=r18,2,6 // get the faulting page size
  439. ;;
  440. cmp.eq p6,p7=5,r17 // is faulting address in region 5?
  441. add r22=-PAGE_SHIFT,r18 // adjustment for hugetlb address
  442. add r18=PGDIR_SHIFT-PAGE_SHIFT,r18
  443. ;;
  444. shr.u r22=r16,r22
  445. shr.u r18=r16,r18
  446. (p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
  447. srlz.d
  448. LOAD_PHYSICAL(p6, r19, swapper_pg_dir) // region 5 is rooted at swapper_pg_dir
  449. .pred.rel "mutex", p6, p7
  450. (p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
  451. (p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
  452. ;;
  453. (p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
  454. (p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
  455. cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
  456. #ifdef CONFIG_PGTABLE_4
  457. shr.u r18=r22,PUD_SHIFT // shift pud index into position
  458. #else
  459. shr.u r18=r22,PMD_SHIFT // shift pmd index into position
  460. #endif
  461. ;;
  462. ld8 r17=[r17] // get *pgd (may be 0)
  463. ;;
  464. (p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
  465. dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
  466. ;;
  467. #ifdef CONFIG_PGTABLE_4
  468. (p7) ld8 r17=[r17] // get *pud (may be 0)
  469. shr.u r18=r22,PMD_SHIFT // shift pmd index into position
  470. ;;
  471. (p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
  472. dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
  473. ;;
  474. #endif
  475. (p7) ld8 r17=[r17] // get *pmd (may be 0)
  476. shr.u r19=r22,PAGE_SHIFT // shift pte index into position
  477. ;;
  478. (p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
  479. dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
  480. (p6) br.cond.spnt page_fault
  481. mov b0=r30
  482. br.sptk.many b0 // return to continuation point
  483. END(nested_dtlb_miss)
  484. .org ia64_ivt+0x1800
  485. /////////////////////////////////////////////////////////////////////////////////////////
  486. // 0x1800 Entry 6 (size 64 bundles) Instruction Key Miss (24)
  487. ENTRY(ikey_miss)
  488. DBG_FAULT(6)
  489. FAULT(6)
  490. END(ikey_miss)
  491. .org ia64_ivt+0x1c00
  492. /////////////////////////////////////////////////////////////////////////////////////////
  493. // 0x1c00 Entry 7 (size 64 bundles) Data Key Miss (12,51)
  494. ENTRY(dkey_miss)
  495. DBG_FAULT(7)
  496. FAULT(7)
  497. END(dkey_miss)
  498. .org ia64_ivt+0x2000
  499. /////////////////////////////////////////////////////////////////////////////////////////
  500. // 0x2000 Entry 8 (size 64 bundles) Dirty-bit (54)
  501. ENTRY(dirty_bit)
  502. DBG_FAULT(8)
  503. /*
  504. * What we do here is to simply turn on the dirty bit in the PTE. We need to
  505. * update both the page-table and the TLB entry. To efficiently access the PTE,
  506. * we address it through the virtual page table. Most likely, the TLB entry for
  507. * the relevant virtual page table page is still present in the TLB so we can
  508. * normally do this without additional TLB misses. In case the necessary virtual
  509. * page table TLB entry isn't present, we take a nested TLB miss hit where we look
  510. * up the physical address of the L3 PTE and then continue at label 1 below.
  511. */
  512. MOV_FROM_IFA(r16) // get the address that caused the fault
  513. movl r30=1f // load continuation point in case of nested fault
  514. ;;
  515. THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
  516. mov r29=b0 // save b0 in case of nested fault
  517. mov r31=pr // save pr
  518. #ifdef CONFIG_SMP
  519. mov r28=ar.ccv // save ar.ccv
  520. ;;
  521. 1: ld8 r18=[r17]
  522. ;; // avoid RAW on r18
  523. mov ar.ccv=r18 // set compare value for cmpxchg
  524. or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
  525. tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
  526. ;;
  527. (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
  528. mov r24=PAGE_SHIFT<<2
  529. ;;
  530. (p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
  531. ;;
  532. ITC_D(p6, r25, r18) // install updated PTE
  533. ;;
  534. /*
  535. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  536. * cannot possibly affect the following loads:
  537. */
  538. dv_serialize_data
  539. ld8 r18=[r17] // read PTE again
  540. ;;
  541. cmp.eq p6,p7=r18,r25 // is it same as the newly installed
  542. ;;
  543. (p7) ptc.l r16,r24
  544. mov b0=r29 // restore b0
  545. mov ar.ccv=r28
  546. #else
  547. ;;
  548. 1: ld8 r18=[r17]
  549. ;; // avoid RAW on r18
  550. or r18=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
  551. mov b0=r29 // restore b0
  552. ;;
  553. st8 [r17]=r18 // store back updated PTE
  554. ITC_D(p0, r18, r16) // install updated PTE
  555. #endif
  556. mov pr=r31,-1 // restore pr
  557. RFI
  558. END(dirty_bit)
  559. .org ia64_ivt+0x2400
  560. /////////////////////////////////////////////////////////////////////////////////////////
  561. // 0x2400 Entry 9 (size 64 bundles) Instruction Access-bit (27)
  562. ENTRY(iaccess_bit)
  563. DBG_FAULT(9)
  564. // Like Entry 8, except for instruction access
  565. MOV_FROM_IFA(r16) // get the address that caused the fault
  566. movl r30=1f // load continuation point in case of nested fault
  567. mov r31=pr // save predicates
  568. #ifdef CONFIG_ITANIUM
  569. /*
  570. * Erratum 10 (IFA may contain incorrect address) has "NoFix" status.
  571. */
  572. MOV_FROM_IPSR(p0, r17)
  573. ;;
  574. MOV_FROM_IIP(r18)
  575. tbit.z p6,p0=r17,IA64_PSR_IS_BIT // IA64 instruction set?
  576. ;;
  577. (p6) mov r16=r18 // if so, use cr.iip instead of cr.ifa
  578. #endif /* CONFIG_ITANIUM */
  579. ;;
  580. THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
  581. mov r29=b0 // save b0 in case of nested fault)
  582. #ifdef CONFIG_SMP
  583. mov r28=ar.ccv // save ar.ccv
  584. ;;
  585. 1: ld8 r18=[r17]
  586. ;;
  587. mov ar.ccv=r18 // set compare value for cmpxchg
  588. or r25=_PAGE_A,r18 // set the accessed bit
  589. tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
  590. ;;
  591. (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
  592. mov r24=PAGE_SHIFT<<2
  593. ;;
  594. (p6) cmp.eq p6,p7=r26,r18 // Only if page present
  595. ;;
  596. ITC_I(p6, r25, r26) // install updated PTE
  597. ;;
  598. /*
  599. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  600. * cannot possibly affect the following loads:
  601. */
  602. dv_serialize_data
  603. ld8 r18=[r17] // read PTE again
  604. ;;
  605. cmp.eq p6,p7=r18,r25 // is it same as the newly installed
  606. ;;
  607. (p7) ptc.l r16,r24
  608. mov b0=r29 // restore b0
  609. mov ar.ccv=r28
  610. #else /* !CONFIG_SMP */
  611. ;;
  612. 1: ld8 r18=[r17]
  613. ;;
  614. or r18=_PAGE_A,r18 // set the accessed bit
  615. mov b0=r29 // restore b0
  616. ;;
  617. st8 [r17]=r18 // store back updated PTE
  618. ITC_I(p0, r18, r16) // install updated PTE
  619. #endif /* !CONFIG_SMP */
  620. mov pr=r31,-1
  621. RFI
  622. END(iaccess_bit)
  623. .org ia64_ivt+0x2800
  624. /////////////////////////////////////////////////////////////////////////////////////////
  625. // 0x2800 Entry 10 (size 64 bundles) Data Access-bit (15,55)
  626. ENTRY(daccess_bit)
  627. DBG_FAULT(10)
  628. // Like Entry 8, except for data access
  629. MOV_FROM_IFA(r16) // get the address that caused the fault
  630. movl r30=1f // load continuation point in case of nested fault
  631. ;;
  632. THASH(p0, r17, r16, r18) // compute virtual address of L3 PTE
  633. mov r31=pr
  634. mov r29=b0 // save b0 in case of nested fault)
  635. #ifdef CONFIG_SMP
  636. mov r28=ar.ccv // save ar.ccv
  637. ;;
  638. 1: ld8 r18=[r17]
  639. ;; // avoid RAW on r18
  640. mov ar.ccv=r18 // set compare value for cmpxchg
  641. or r25=_PAGE_A,r18 // set the dirty bit
  642. tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
  643. ;;
  644. (p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
  645. mov r24=PAGE_SHIFT<<2
  646. ;;
  647. (p6) cmp.eq p6,p7=r26,r18 // Only if page is present
  648. ;;
  649. ITC_D(p6, r25, r26) // install updated PTE
  650. /*
  651. * Tell the assemblers dependency-violation checker that the above "itc" instructions
  652. * cannot possibly affect the following loads:
  653. */
  654. dv_serialize_data
  655. ;;
  656. ld8 r18=[r17] // read PTE again
  657. ;;
  658. cmp.eq p6,p7=r18,r25 // is it same as the newly installed
  659. ;;
  660. (p7) ptc.l r16,r24
  661. mov ar.ccv=r28
  662. #else
  663. ;;
  664. 1: ld8 r18=[r17]
  665. ;; // avoid RAW on r18
  666. or r18=_PAGE_A,r18 // set the accessed bit
  667. ;;
  668. st8 [r17]=r18 // store back updated PTE
  669. ITC_D(p0, r18, r16) // install updated PTE
  670. #endif
  671. mov b0=r29 // restore b0
  672. mov pr=r31,-1
  673. RFI
  674. END(daccess_bit)
  675. .org ia64_ivt+0x2c00
  676. /////////////////////////////////////////////////////////////////////////////////////////
  677. // 0x2c00 Entry 11 (size 64 bundles) Break instruction (33)
  678. ENTRY(break_fault)
  679. /*
  680. * The streamlined system call entry/exit paths only save/restore the initial part
  681. * of pt_regs. This implies that the callers of system-calls must adhere to the
  682. * normal procedure calling conventions.
  683. *
  684. * Registers to be saved & restored:
  685. * CR registers: cr.ipsr, cr.iip, cr.ifs
  686. * AR registers: ar.unat, ar.pfs, ar.rsc, ar.rnat, ar.bspstore, ar.fpsr
  687. * others: pr, b0, b6, loadrs, r1, r11, r12, r13, r15
  688. * Registers to be restored only:
  689. * r8-r11: output value from the system call.
  690. *
  691. * During system call exit, scratch registers (including r15) are modified/cleared
  692. * to prevent leaking bits from kernel to user level.
  693. */
  694. DBG_FAULT(11)
  695. mov.m r16=IA64_KR(CURRENT) // M2 r16 <- current task (12 cyc)
  696. MOV_FROM_IPSR(p0, r29) // M2 (12 cyc)
  697. mov r31=pr // I0 (2 cyc)
  698. MOV_FROM_IIM(r17) // M2 (2 cyc)
  699. mov.m r27=ar.rsc // M2 (12 cyc)
  700. mov r18=__IA64_BREAK_SYSCALL // A
  701. mov.m ar.rsc=0 // M2
  702. mov.m r21=ar.fpsr // M2 (12 cyc)
  703. mov r19=b6 // I0 (2 cyc)
  704. ;;
  705. mov.m r23=ar.bspstore // M2 (12 cyc)
  706. mov.m r24=ar.rnat // M2 (5 cyc)
  707. mov.i r26=ar.pfs // I0 (2 cyc)
  708. invala // M0|1
  709. nop.m 0 // M
  710. mov r20=r1 // A save r1
  711. nop.m 0
  712. movl r30=sys_call_table // X
  713. MOV_FROM_IIP(r28) // M2 (2 cyc)
  714. cmp.eq p0,p7=r18,r17 // I0 is this a system call?
  715. (p7) br.cond.spnt non_syscall // B no ->
  716. //
  717. // From this point on, we are definitely on the syscall-path
  718. // and we can use (non-banked) scratch registers.
  719. //
  720. ///////////////////////////////////////////////////////////////////////
  721. mov r1=r16 // A move task-pointer to "addl"-addressable reg
  722. mov r2=r16 // A setup r2 for ia64_syscall_setup
  723. add r9=TI_FLAGS+IA64_TASK_SIZE,r16 // A r9 = &current_thread_info()->flags
  724. adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16
  725. adds r15=-1024,r15 // A subtract 1024 from syscall number
  726. mov r3=NR_syscalls - 1
  727. ;;
  728. ld1.bias r17=[r16] // M0|1 r17 = current->thread.on_ustack flag
  729. ld4 r9=[r9] // M0|1 r9 = current_thread_info()->flags
  730. extr.u r8=r29,41,2 // I0 extract ei field from cr.ipsr
  731. shladd r30=r15,3,r30 // A r30 = sys_call_table + 8*(syscall-1024)
  732. addl r22=IA64_RBS_OFFSET,r1 // A compute base of RBS
  733. cmp.leu p6,p7=r15,r3 // A syscall number in range?
  734. ;;
  735. lfetch.fault.excl.nt1 [r22] // M0|1 prefetch RBS
  736. (p6) ld8 r30=[r30] // M0|1 load address of syscall entry point
  737. tnat.nz.or p7,p0=r15 // I0 is syscall nr a NaT?
  738. mov.m ar.bspstore=r22 // M2 switch to kernel RBS
  739. cmp.eq p8,p9=2,r8 // A isr.ei==2?
  740. ;;
  741. (p8) mov r8=0 // A clear ei to 0
  742. (p7) movl r30=sys_ni_syscall // X
  743. (p8) adds r28=16,r28 // A switch cr.iip to next bundle
  744. (p9) adds r8=1,r8 // A increment ei to next slot
  745. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  746. ;;
  747. mov b6=r30 // I0 setup syscall handler branch reg early
  748. #else
  749. nop.i 0
  750. ;;
  751. #endif
  752. mov.m r25=ar.unat // M2 (5 cyc)
  753. dep r29=r8,r29,41,2 // I0 insert new ei into cr.ipsr
  754. adds r15=1024,r15 // A restore original syscall number
  755. //
  756. // If any of the above loads miss in L1D, we'll stall here until
  757. // the data arrives.
  758. //
  759. ///////////////////////////////////////////////////////////////////////
  760. st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
  761. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  762. MOV_FROM_ITC(p0, p14, r30, r18) // M get cycle for accounting
  763. #else
  764. mov b6=r30 // I0 setup syscall handler branch reg early
  765. #endif
  766. cmp.eq pKStk,pUStk=r0,r17 // A were we on kernel stacks already?
  767. and r9=_TIF_SYSCALL_TRACEAUDIT,r9 // A mask trace or audit
  768. mov r18=ar.bsp // M2 (12 cyc)
  769. (pKStk) br.cond.spnt .break_fixup // B we're already in kernel-mode -- fix up RBS
  770. ;;
  771. .back_from_break_fixup:
  772. (pUStk) addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r1 // A compute base of memory stack
  773. cmp.eq p14,p0=r9,r0 // A are syscalls being traced/audited?
  774. br.call.sptk.many b7=ia64_syscall_setup // B
  775. 1:
  776. #ifdef CONFIG_VIRT_CPU_ACCOUNTING
  777. // mov.m r30=ar.itc is called in advance, and r13 is current
  778. add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13 // A
  779. add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13 // A
  780. (pKStk) br.cond.spnt .skip_accounting // B unlikely skip
  781. ;;
  782. ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // M get last stamp
  783. ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // M time at leave
  784. ;;
  785. ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // M cumulated stime
  786. ld8 r21=[r17] // M cumulated utime
  787. sub r22=r19,r18 // A stime before leave
  788. ;;
  789. st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // M update stamp
  790. sub r18=r30,r19 // A elapsed time in user
  791. ;;
  792. add r20=r20,r22 // A sum stime
  793. add r21=r21,r18 // A sum utime
  794. ;;
  795. st8 [r16]=r20 // M update stime
  796. st8 [r17]=r21 // M update utime
  797. ;;
  798. .skip_accounting:
  799. #endif
  800. mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
  801. nop 0
  802. BSW_1(r2, r14) // B (6 cyc) regs are saved, switch to bank 1
  803. ;;
  804. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r16) // M2 now it's safe to re-enable intr.-collection
  805. // M0 ensure interruption collection is on
  806. movl r3=ia64_ret_from_syscall // X
  807. ;;
  808. mov rp=r3 // I0 set the real return addr
  809. (p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
  810. SSM_PSR_I(p15, p15, r16) // M2 restore psr.i
  811. (p14) br.call.sptk.many b6=b6 // B invoke syscall-handker (ignore return addr)
  812. br.cond.spnt.many ia64_trace_syscall // B do syscall-tracing thingamagic
  813. // NOT REACHED
  814. ///////////////////////////////////////////////////////////////////////
  815. // On entry, we optimistically assumed that we're coming from user-space.
  816. // For the rare cases where a system-call is done from within the kernel,
  817. // we fix things up at this point:
  818. .break_fixup:
  819. add r1=-IA64_PT_REGS_SIZE,sp // A allocate space for pt_regs structure
  820. mov ar.rnat=r24 // M2 restore kernel's AR.RNAT
  821. ;;
  822. mov ar.bspstore=r23 // M2 restore kernel's AR.BSPSTORE
  823. br.cond.sptk .back_from_break_fixup
  824. END(break_fault)
  825. .org ia64_ivt+0x3000
  826. /////////////////////////////////////////////////////////////////////////////////////////
  827. // 0x3000 Entry 12 (size 64 bundles) External Interrupt (4)
  828. ENTRY(interrupt)
  829. /* interrupt handler has become too big to fit this area. */
  830. br.sptk.many __interrupt
  831. END(interrupt)
  832. .org ia64_ivt+0x3400
  833. /////////////////////////////////////////////////////////////////////////////////////////
  834. // 0x3400 Entry 13 (size 64 bundles) Reserved
  835. DBG_FAULT(13)
  836. FAULT(13)
  837. .org ia64_ivt+0x3800
  838. /////////////////////////////////////////////////////////////////////////////////////////
  839. // 0x3800 Entry 14 (size 64 bundles) Reserved
  840. DBG_FAULT(14)
  841. FAULT(14)
  842. /*
  843. * There is no particular reason for this code to be here, other than that
  844. * there happens to be space here that would go unused otherwise. If this
  845. * fault ever gets "unreserved", simply moved the following code to a more
  846. * suitable spot...
  847. *
  848. * ia64_syscall_setup() is a separate subroutine so that it can
  849. * allocate stacked registers so it can safely demine any
  850. * potential NaT values from the input registers.
  851. *
  852. * On entry:
  853. * - executing on bank 0 or bank 1 register set (doesn't matter)
  854. * - r1: stack pointer
  855. * - r2: current task pointer
  856. * - r3: preserved
  857. * - r11: original contents (saved ar.pfs to be saved)
  858. * - r12: original contents (sp to be saved)
  859. * - r13: original contents (tp to be saved)
  860. * - r15: original contents (syscall # to be saved)
  861. * - r18: saved bsp (after switching to kernel stack)
  862. * - r19: saved b6
  863. * - r20: saved r1 (gp)
  864. * - r21: saved ar.fpsr
  865. * - r22: kernel's register backing store base (krbs_base)
  866. * - r23: saved ar.bspstore
  867. * - r24: saved ar.rnat
  868. * - r25: saved ar.unat
  869. * - r26: saved ar.pfs
  870. * - r27: saved ar.rsc
  871. * - r28: saved cr.iip
  872. * - r29: saved cr.ipsr
  873. * - r30: ar.itc for accounting (don't touch)
  874. * - r31: saved pr
  875. * - b0: original contents (to be saved)
  876. * On exit:
  877. * - p10: TRUE if syscall is invoked with more than 8 out
  878. * registers or r15's Nat is true
  879. * - r1: kernel's gp
  880. * - r3: preserved (same as on entry)
  881. * - r8: -EINVAL if p10 is true
  882. * - r12: points to kernel stack
  883. * - r13: points to current task
  884. * - r14: preserved (same as on entry)
  885. * - p13: preserved
  886. * - p15: TRUE if interrupts need to be re-enabled
  887. * - ar.fpsr: set to kernel settings
  888. * - b6: preserved (same as on entry)
  889. */
  890. #ifdef __IA64_ASM_PARAVIRTUALIZED_NATIVE
  891. GLOBAL_ENTRY(ia64_syscall_setup)
  892. #if PT(B6) != 0
  893. # error This code assumes that b6 is the first field in pt_regs.
  894. #endif
  895. st8 [r1]=r19 // save b6
  896. add r16=PT(CR_IPSR),r1 // initialize first base pointer
  897. add r17=PT(R11),r1 // initialize second base pointer
  898. ;;
  899. alloc r19=ar.pfs,8,0,0,0 // ensure in0-in7 are writable
  900. st8 [r16]=r29,PT(AR_PFS)-PT(CR_IPSR) // save cr.ipsr
  901. tnat.nz p8,p0=in0
  902. st8.spill [r17]=r11,PT(CR_IIP)-PT(R11) // save r11
  903. tnat.nz p9,p0=in1
  904. (pKStk) mov r18=r0 // make sure r18 isn't NaT
  905. ;;
  906. st8 [r16]=r26,PT(CR_IFS)-PT(AR_PFS) // save ar.pfs
  907. st8 [r17]=r28,PT(AR_UNAT)-PT(CR_IIP) // save cr.iip
  908. mov r28=b0 // save b0 (2 cyc)
  909. ;;
  910. st8 [r17]=r25,PT(AR_RSC)-PT(AR_UNAT) // save ar.unat
  911. dep r19=0,r19,38,26 // clear all bits but 0..37 [I0]
  912. (p8) mov in0=-1
  913. ;;
  914. st8 [r16]=r19,PT(AR_RNAT)-PT(CR_IFS) // store ar.pfs.pfm in cr.ifs
  915. extr.u r11=r19,7,7 // I0 // get sol of ar.pfs
  916. and r8=0x7f,r19 // A // get sof of ar.pfs
  917. st8 [r17]=r27,PT(AR_BSPSTORE)-PT(AR_RSC)// save ar.rsc
  918. tbit.nz p15,p0=r29,IA64_PSR_I_BIT // I0
  919. (p9) mov in1=-1
  920. ;;
  921. (pUStk) sub r18=r18,r22 // r18=RSE.ndirty*8
  922. tnat.nz p10,p0=in2
  923. add r11=8,r11
  924. ;;
  925. (pKStk) adds r16=PT(PR)-PT(AR_RNAT),r16 // skip over ar_rnat field
  926. (pKStk) adds r17=PT(B0)-PT(AR_BSPSTORE),r17 // skip over ar_bspstore field
  927. tnat.nz p11,p0=in3
  928. ;;
  929. (p10) mov in2=-1
  930. tnat.nz p12,p0=in4 // [I0]
  931. (p11) mov in3=-1
  932. ;;
  933. (pUStk) st8 [r16]=r24,PT(PR)-PT(AR_RNAT) // save ar.rnat
  934. (pUStk) st8 [r17]=r23,PT(B0)-PT(AR_BSPSTORE) // save ar.bspstore
  935. shl r18=r18,16 // compute ar.rsc to be used for "loadrs"
  936. ;;
  937. st8 [r16]=r31,PT(LOADRS)-PT(PR) // save predicates
  938. st8 [r17]=r28,PT(R1)-PT(B0) // save b0
  939. tnat.nz p13,p0=in5 // [I0]
  940. ;;
  941. st8 [r16]=r18,PT(R12)-PT(LOADRS) // save ar.rsc value for "loadrs"
  942. st8.spill [r17]=r20,PT(R13)-PT(R1) // save original r1
  943. (p12) mov in4=-1
  944. ;;
  945. .mem.offset 0,0; st8.spill [r16]=r12,PT(AR_FPSR)-PT(R12) // save r12
  946. .mem.offset 8,0; st8.spill [r17]=r13,PT(R15)-PT(R13) // save r13
  947. (p13) mov in5=-1
  948. ;;
  949. st8 [r16]=r21,PT(R8)-PT(AR_FPSR) // save ar.fpsr
  950. tnat.nz p13,p0=in6
  951. cmp.lt p10,p9=r11,r8 // frame size can't be more than local+8
  952. ;;
  953. mov r8=1
  954. (p9) tnat.nz p10,p0=r15
  955. adds r12=-16,r1 // switch to kernel memory stack (with 16 bytes of scratch)
  956. st8.spill [r17]=r15 // save r15
  957. tnat.nz p8,p0=in7
  958. nop.i 0
  959. mov r13=r2 // establish `current'
  960. movl r1=__gp // establish kernel global pointer
  961. ;;
  962. st8 [r16]=r8 // ensure pt_regs.r8 != 0 (see handle_syscall_error)
  963. (p13) mov in6=-1
  964. (p8) mov in7=-1
  965. cmp.eq pSys,pNonSys=r0,r0 // set pSys=1, pNonSys=0
  966. movl r17=FPSR_DEFAULT
  967. ;;
  968. mov.m ar.fpsr=r17 // set ar.fpsr to kernel default value
  969. (p10) mov r8=-EINVAL
  970. br.ret.sptk.many b7
  971. END(ia64_syscall_setup)
  972. #endif /* __IA64_ASM_PARAVIRTUALIZED_NATIVE */
  973. .org ia64_ivt+0x3c00
  974. /////////////////////////////////////////////////////////////////////////////////////////
  975. // 0x3c00 Entry 15 (size 64 bundles) Reserved
  976. DBG_FAULT(15)
  977. FAULT(15)
  978. .org ia64_ivt+0x4000
  979. /////////////////////////////////////////////////////////////////////////////////////////
  980. // 0x4000 Entry 16 (size 64 bundles) Reserved
  981. DBG_FAULT(16)
  982. FAULT(16)
  983. #if defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(__IA64_ASM_PARAVIRTUALIZED_NATIVE)
  984. /*
  985. * There is no particular reason for this code to be here, other than
  986. * that there happens to be space here that would go unused otherwise.
  987. * If this fault ever gets "unreserved", simply moved the following
  988. * code to a more suitable spot...
  989. *
  990. * account_sys_enter is called from SAVE_MIN* macros if accounting is
  991. * enabled and if the macro is entered from user mode.
  992. */
  993. GLOBAL_ENTRY(account_sys_enter)
  994. // mov.m r20=ar.itc is called in advance, and r13 is current
  995. add r16=TI_AC_STAMP+IA64_TASK_SIZE,r13
  996. add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r13
  997. ;;
  998. ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
  999. ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at left from kernel
  1000. ;;
  1001. ld8 r23=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
  1002. ld8 r21=[r17] // cumulated utime
  1003. sub r22=r19,r18 // stime before leave kernel
  1004. ;;
  1005. st8 [r16]=r20,TI_AC_STIME-TI_AC_STAMP // update stamp
  1006. sub r18=r20,r19 // elapsed time in user mode
  1007. ;;
  1008. add r23=r23,r22 // sum stime
  1009. add r21=r21,r18 // sum utime
  1010. ;;
  1011. st8 [r16]=r23 // update stime
  1012. st8 [r17]=r21 // update utime
  1013. ;;
  1014. br.ret.sptk.many rp
  1015. END(account_sys_enter)
  1016. #endif
  1017. .org ia64_ivt+0x4400
  1018. /////////////////////////////////////////////////////////////////////////////////////////
  1019. // 0x4400 Entry 17 (size 64 bundles) Reserved
  1020. DBG_FAULT(17)
  1021. FAULT(17)
  1022. .org ia64_ivt+0x4800
  1023. /////////////////////////////////////////////////////////////////////////////////////////
  1024. // 0x4800 Entry 18 (size 64 bundles) Reserved
  1025. DBG_FAULT(18)
  1026. FAULT(18)
  1027. .org ia64_ivt+0x4c00
  1028. /////////////////////////////////////////////////////////////////////////////////////////
  1029. // 0x4c00 Entry 19 (size 64 bundles) Reserved
  1030. DBG_FAULT(19)
  1031. FAULT(19)
  1032. //
  1033. // --- End of long entries, Beginning of short entries
  1034. //
  1035. .org ia64_ivt+0x5000
  1036. /////////////////////////////////////////////////////////////////////////////////////////
  1037. // 0x5000 Entry 20 (size 16 bundles) Page Not Present (10,22,49)
  1038. ENTRY(page_not_present)
  1039. DBG_FAULT(20)
  1040. MOV_FROM_IFA(r16)
  1041. RSM_PSR_DT
  1042. /*
  1043. * The Linux page fault handler doesn't expect non-present pages to be in
  1044. * the TLB. Flush the existing entry now, so we meet that expectation.
  1045. */
  1046. mov r17=PAGE_SHIFT<<2
  1047. ;;
  1048. ptc.l r16,r17
  1049. ;;
  1050. mov r31=pr
  1051. srlz.d
  1052. br.sptk.many page_fault
  1053. END(page_not_present)
  1054. .org ia64_ivt+0x5100
  1055. /////////////////////////////////////////////////////////////////////////////////////////
  1056. // 0x5100 Entry 21 (size 16 bundles) Key Permission (13,25,52)
  1057. ENTRY(key_permission)
  1058. DBG_FAULT(21)
  1059. MOV_FROM_IFA(r16)
  1060. RSM_PSR_DT
  1061. mov r31=pr
  1062. ;;
  1063. srlz.d
  1064. br.sptk.many page_fault
  1065. END(key_permission)
  1066. .org ia64_ivt+0x5200
  1067. /////////////////////////////////////////////////////////////////////////////////////////
  1068. // 0x5200 Entry 22 (size 16 bundles) Instruction Access Rights (26)
  1069. ENTRY(iaccess_rights)
  1070. DBG_FAULT(22)
  1071. MOV_FROM_IFA(r16)
  1072. RSM_PSR_DT
  1073. mov r31=pr
  1074. ;;
  1075. srlz.d
  1076. br.sptk.many page_fault
  1077. END(iaccess_rights)
  1078. .org ia64_ivt+0x5300
  1079. /////////////////////////////////////////////////////////////////////////////////////////
  1080. // 0x5300 Entry 23 (size 16 bundles) Data Access Rights (14,53)
  1081. ENTRY(daccess_rights)
  1082. DBG_FAULT(23)
  1083. MOV_FROM_IFA(r16)
  1084. RSM_PSR_DT
  1085. mov r31=pr
  1086. ;;
  1087. srlz.d
  1088. br.sptk.many page_fault
  1089. END(daccess_rights)
  1090. .org ia64_ivt+0x5400
  1091. /////////////////////////////////////////////////////////////////////////////////////////
  1092. // 0x5400 Entry 24 (size 16 bundles) General Exception (5,32,34,36,38,39)
  1093. ENTRY(general_exception)
  1094. DBG_FAULT(24)
  1095. MOV_FROM_ISR(r16)
  1096. mov r31=pr
  1097. ;;
  1098. cmp4.eq p6,p0=0,r16
  1099. (p6) br.sptk.many dispatch_illegal_op_fault
  1100. ;;
  1101. mov r19=24 // fault number
  1102. br.sptk.many dispatch_to_fault_handler
  1103. END(general_exception)
  1104. .org ia64_ivt+0x5500
  1105. /////////////////////////////////////////////////////////////////////////////////////////
  1106. // 0x5500 Entry 25 (size 16 bundles) Disabled FP-Register (35)
  1107. ENTRY(disabled_fp_reg)
  1108. DBG_FAULT(25)
  1109. rsm psr.dfh // ensure we can access fph
  1110. ;;
  1111. srlz.d
  1112. mov r31=pr
  1113. mov r19=25
  1114. br.sptk.many dispatch_to_fault_handler
  1115. END(disabled_fp_reg)
  1116. .org ia64_ivt+0x5600
  1117. /////////////////////////////////////////////////////////////////////////////////////////
  1118. // 0x5600 Entry 26 (size 16 bundles) Nat Consumption (11,23,37,50)
  1119. ENTRY(nat_consumption)
  1120. DBG_FAULT(26)
  1121. MOV_FROM_IPSR(p0, r16)
  1122. MOV_FROM_ISR(r17)
  1123. mov r31=pr // save PR
  1124. ;;
  1125. and r18=0xf,r17 // r18 = cr.ipsr.code{3:0}
  1126. tbit.z p6,p0=r17,IA64_ISR_NA_BIT
  1127. ;;
  1128. cmp.ne.or p6,p0=IA64_ISR_CODE_LFETCH,r18
  1129. dep r16=-1,r16,IA64_PSR_ED_BIT,1
  1130. (p6) br.cond.spnt 1f // branch if (cr.ispr.na == 0 || cr.ipsr.code{3:0} != LFETCH)
  1131. ;;
  1132. MOV_TO_IPSR(p0, r16, r18)
  1133. mov pr=r31,-1
  1134. ;;
  1135. RFI
  1136. 1: mov pr=r31,-1
  1137. ;;
  1138. FAULT(26)
  1139. END(nat_consumption)
  1140. .org ia64_ivt+0x5700
  1141. /////////////////////////////////////////////////////////////////////////////////////////
  1142. // 0x5700 Entry 27 (size 16 bundles) Speculation (40)
  1143. ENTRY(speculation_vector)
  1144. DBG_FAULT(27)
  1145. /*
  1146. * A [f]chk.[as] instruction needs to take the branch to the recovery code but
  1147. * this part of the architecture is not implemented in hardware on some CPUs, such
  1148. * as Itanium. Thus, in general we need to emulate the behavior. IIM contains
  1149. * the relative target (not yet sign extended). So after sign extending it we
  1150. * simply add it to IIP. We also need to reset the EI field of the IPSR to zero,
  1151. * i.e., the slot to restart into.
  1152. *
  1153. * cr.imm contains zero_ext(imm21)
  1154. */
  1155. MOV_FROM_IIM(r18)
  1156. ;;
  1157. MOV_FROM_IIP(r17)
  1158. shl r18=r18,43 // put sign bit in position (43=64-21)
  1159. ;;
  1160. MOV_FROM_IPSR(p0, r16)
  1161. shr r18=r18,39 // sign extend (39=43-4)
  1162. ;;
  1163. add r17=r17,r18 // now add the offset
  1164. ;;
  1165. MOV_TO_IIP(r17, r19)
  1166. dep r16=0,r16,41,2 // clear EI
  1167. ;;
  1168. MOV_TO_IPSR(p0, r16, r19)
  1169. ;;
  1170. RFI
  1171. END(speculation_vector)
  1172. .org ia64_ivt+0x5800
  1173. /////////////////////////////////////////////////////////////////////////////////////////
  1174. // 0x5800 Entry 28 (size 16 bundles) Reserved
  1175. DBG_FAULT(28)
  1176. FAULT(28)
  1177. .org ia64_ivt+0x5900
  1178. /////////////////////////////////////////////////////////////////////////////////////////
  1179. // 0x5900 Entry 29 (size 16 bundles) Debug (16,28,56)
  1180. ENTRY(debug_vector)
  1181. DBG_FAULT(29)
  1182. FAULT(29)
  1183. END(debug_vector)
  1184. .org ia64_ivt+0x5a00
  1185. /////////////////////////////////////////////////////////////////////////////////////////
  1186. // 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
  1187. ENTRY(unaligned_access)
  1188. DBG_FAULT(30)
  1189. mov r31=pr // prepare to save predicates
  1190. ;;
  1191. br.sptk.many dispatch_unaligned_handler
  1192. END(unaligned_access)
  1193. .org ia64_ivt+0x5b00
  1194. /////////////////////////////////////////////////////////////////////////////////////////
  1195. // 0x5b00 Entry 31 (size 16 bundles) Unsupported Data Reference (57)
  1196. ENTRY(unsupported_data_reference)
  1197. DBG_FAULT(31)
  1198. FAULT(31)
  1199. END(unsupported_data_reference)
  1200. .org ia64_ivt+0x5c00
  1201. /////////////////////////////////////////////////////////////////////////////////////////
  1202. // 0x5c00 Entry 32 (size 16 bundles) Floating-Point Fault (64)
  1203. ENTRY(floating_point_fault)
  1204. DBG_FAULT(32)
  1205. FAULT(32)
  1206. END(floating_point_fault)
  1207. .org ia64_ivt+0x5d00
  1208. /////////////////////////////////////////////////////////////////////////////////////////
  1209. // 0x5d00 Entry 33 (size 16 bundles) Floating Point Trap (66)
  1210. ENTRY(floating_point_trap)
  1211. DBG_FAULT(33)
  1212. FAULT(33)
  1213. END(floating_point_trap)
  1214. .org ia64_ivt+0x5e00
  1215. /////////////////////////////////////////////////////////////////////////////////////////
  1216. // 0x5e00 Entry 34 (size 16 bundles) Lower Privilege Transfer Trap (66)
  1217. ENTRY(lower_privilege_trap)
  1218. DBG_FAULT(34)
  1219. FAULT(34)
  1220. END(lower_privilege_trap)
  1221. .org ia64_ivt+0x5f00
  1222. /////////////////////////////////////////////////////////////////////////////////////////
  1223. // 0x5f00 Entry 35 (size 16 bundles) Taken Branch Trap (68)
  1224. ENTRY(taken_branch_trap)
  1225. DBG_FAULT(35)
  1226. FAULT(35)
  1227. END(taken_branch_trap)
  1228. .org ia64_ivt+0x6000
  1229. /////////////////////////////////////////////////////////////////////////////////////////
  1230. // 0x6000 Entry 36 (size 16 bundles) Single Step Trap (69)
  1231. ENTRY(single_step_trap)
  1232. DBG_FAULT(36)
  1233. FAULT(36)
  1234. END(single_step_trap)
  1235. .org ia64_ivt+0x6100
  1236. /////////////////////////////////////////////////////////////////////////////////////////
  1237. // 0x6100 Entry 37 (size 16 bundles) Reserved
  1238. DBG_FAULT(37)
  1239. FAULT(37)
  1240. .org ia64_ivt+0x6200
  1241. /////////////////////////////////////////////////////////////////////////////////////////
  1242. // 0x6200 Entry 38 (size 16 bundles) Reserved
  1243. DBG_FAULT(38)
  1244. FAULT(38)
  1245. .org ia64_ivt+0x6300
  1246. /////////////////////////////////////////////////////////////////////////////////////////
  1247. // 0x6300 Entry 39 (size 16 bundles) Reserved
  1248. DBG_FAULT(39)
  1249. FAULT(39)
  1250. .org ia64_ivt+0x6400
  1251. /////////////////////////////////////////////////////////////////////////////////////////
  1252. // 0x6400 Entry 40 (size 16 bundles) Reserved
  1253. DBG_FAULT(40)
  1254. FAULT(40)
  1255. .org ia64_ivt+0x6500
  1256. /////////////////////////////////////////////////////////////////////////////////////////
  1257. // 0x6500 Entry 41 (size 16 bundles) Reserved
  1258. DBG_FAULT(41)
  1259. FAULT(41)
  1260. .org ia64_ivt+0x6600
  1261. /////////////////////////////////////////////////////////////////////////////////////////
  1262. // 0x6600 Entry 42 (size 16 bundles) Reserved
  1263. DBG_FAULT(42)
  1264. FAULT(42)
  1265. .org ia64_ivt+0x6700
  1266. /////////////////////////////////////////////////////////////////////////////////////////
  1267. // 0x6700 Entry 43 (size 16 bundles) Reserved
  1268. DBG_FAULT(43)
  1269. FAULT(43)
  1270. .org ia64_ivt+0x6800
  1271. /////////////////////////////////////////////////////////////////////////////////////////
  1272. // 0x6800 Entry 44 (size 16 bundles) Reserved
  1273. DBG_FAULT(44)
  1274. FAULT(44)
  1275. .org ia64_ivt+0x6900
  1276. /////////////////////////////////////////////////////////////////////////////////////////
  1277. // 0x6900 Entry 45 (size 16 bundles) IA-32 Exeception (17,18,29,41,42,43,44,58,60,61,62,72,73,75,76,77)
  1278. ENTRY(ia32_exception)
  1279. DBG_FAULT(45)
  1280. FAULT(45)
  1281. END(ia32_exception)
  1282. .org ia64_ivt+0x6a00
  1283. /////////////////////////////////////////////////////////////////////////////////////////
  1284. // 0x6a00 Entry 46 (size 16 bundles) IA-32 Intercept (30,31,59,70,71)
  1285. ENTRY(ia32_intercept)
  1286. DBG_FAULT(46)
  1287. FAULT(46)
  1288. END(ia32_intercept)
  1289. .org ia64_ivt+0x6b00
  1290. /////////////////////////////////////////////////////////////////////////////////////////
  1291. // 0x6b00 Entry 47 (size 16 bundles) IA-32 Interrupt (74)
  1292. ENTRY(ia32_interrupt)
  1293. DBG_FAULT(47)
  1294. FAULT(47)
  1295. END(ia32_interrupt)
  1296. .org ia64_ivt+0x6c00
  1297. /////////////////////////////////////////////////////////////////////////////////////////
  1298. // 0x6c00 Entry 48 (size 16 bundles) Reserved
  1299. DBG_FAULT(48)
  1300. FAULT(48)
  1301. .org ia64_ivt+0x6d00
  1302. /////////////////////////////////////////////////////////////////////////////////////////
  1303. // 0x6d00 Entry 49 (size 16 bundles) Reserved
  1304. DBG_FAULT(49)
  1305. FAULT(49)
  1306. .org ia64_ivt+0x6e00
  1307. /////////////////////////////////////////////////////////////////////////////////////////
  1308. // 0x6e00 Entry 50 (size 16 bundles) Reserved
  1309. DBG_FAULT(50)
  1310. FAULT(50)
  1311. .org ia64_ivt+0x6f00
  1312. /////////////////////////////////////////////////////////////////////////////////////////
  1313. // 0x6f00 Entry 51 (size 16 bundles) Reserved
  1314. DBG_FAULT(51)
  1315. FAULT(51)
  1316. .org ia64_ivt+0x7000
  1317. /////////////////////////////////////////////////////////////////////////////////////////
  1318. // 0x7000 Entry 52 (size 16 bundles) Reserved
  1319. DBG_FAULT(52)
  1320. FAULT(52)
  1321. .org ia64_ivt+0x7100
  1322. /////////////////////////////////////////////////////////////////////////////////////////
  1323. // 0x7100 Entry 53 (size 16 bundles) Reserved
  1324. DBG_FAULT(53)
  1325. FAULT(53)
  1326. .org ia64_ivt+0x7200
  1327. /////////////////////////////////////////////////////////////////////////////////////////
  1328. // 0x7200 Entry 54 (size 16 bundles) Reserved
  1329. DBG_FAULT(54)
  1330. FAULT(54)
  1331. .org ia64_ivt+0x7300
  1332. /////////////////////////////////////////////////////////////////////////////////////////
  1333. // 0x7300 Entry 55 (size 16 bundles) Reserved
  1334. DBG_FAULT(55)
  1335. FAULT(55)
  1336. .org ia64_ivt+0x7400
  1337. /////////////////////////////////////////////////////////////////////////////////////////
  1338. // 0x7400 Entry 56 (size 16 bundles) Reserved
  1339. DBG_FAULT(56)
  1340. FAULT(56)
  1341. .org ia64_ivt+0x7500
  1342. /////////////////////////////////////////////////////////////////////////////////////////
  1343. // 0x7500 Entry 57 (size 16 bundles) Reserved
  1344. DBG_FAULT(57)
  1345. FAULT(57)
  1346. .org ia64_ivt+0x7600
  1347. /////////////////////////////////////////////////////////////////////////////////////////
  1348. // 0x7600 Entry 58 (size 16 bundles) Reserved
  1349. DBG_FAULT(58)
  1350. FAULT(58)
  1351. .org ia64_ivt+0x7700
  1352. /////////////////////////////////////////////////////////////////////////////////////////
  1353. // 0x7700 Entry 59 (size 16 bundles) Reserved
  1354. DBG_FAULT(59)
  1355. FAULT(59)
  1356. .org ia64_ivt+0x7800
  1357. /////////////////////////////////////////////////////////////////////////////////////////
  1358. // 0x7800 Entry 60 (size 16 bundles) Reserved
  1359. DBG_FAULT(60)
  1360. FAULT(60)
  1361. .org ia64_ivt+0x7900
  1362. /////////////////////////////////////////////////////////////////////////////////////////
  1363. // 0x7900 Entry 61 (size 16 bundles) Reserved
  1364. DBG_FAULT(61)
  1365. FAULT(61)
  1366. .org ia64_ivt+0x7a00
  1367. /////////////////////////////////////////////////////////////////////////////////////////
  1368. // 0x7a00 Entry 62 (size 16 bundles) Reserved
  1369. DBG_FAULT(62)
  1370. FAULT(62)
  1371. .org ia64_ivt+0x7b00
  1372. /////////////////////////////////////////////////////////////////////////////////////////
  1373. // 0x7b00 Entry 63 (size 16 bundles) Reserved
  1374. DBG_FAULT(63)
  1375. FAULT(63)
  1376. .org ia64_ivt+0x7c00
  1377. /////////////////////////////////////////////////////////////////////////////////////////
  1378. // 0x7c00 Entry 64 (size 16 bundles) Reserved
  1379. DBG_FAULT(64)
  1380. FAULT(64)
  1381. .org ia64_ivt+0x7d00
  1382. /////////////////////////////////////////////////////////////////////////////////////////
  1383. // 0x7d00 Entry 65 (size 16 bundles) Reserved
  1384. DBG_FAULT(65)
  1385. FAULT(65)
  1386. .org ia64_ivt+0x7e00
  1387. /////////////////////////////////////////////////////////////////////////////////////////
  1388. // 0x7e00 Entry 66 (size 16 bundles) Reserved
  1389. DBG_FAULT(66)
  1390. FAULT(66)
  1391. .org ia64_ivt+0x7f00
  1392. /////////////////////////////////////////////////////////////////////////////////////////
  1393. // 0x7f00 Entry 67 (size 16 bundles) Reserved
  1394. DBG_FAULT(67)
  1395. FAULT(67)
  1396. //-----------------------------------------------------------------------------------
  1397. // call do_page_fault (predicates are in r31, psr.dt may be off, r16 is faulting address)
  1398. ENTRY(page_fault)
  1399. SSM_PSR_DT_AND_SRLZ_I
  1400. ;;
  1401. SAVE_MIN_WITH_COVER
  1402. alloc r15=ar.pfs,0,0,3,0
  1403. MOV_FROM_IFA(out0)
  1404. MOV_FROM_ISR(out1)
  1405. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r14, r3)
  1406. adds r3=8,r2 // set up second base pointer
  1407. SSM_PSR_I(p15, p15, r14) // restore psr.i
  1408. movl r14=ia64_leave_kernel
  1409. ;;
  1410. SAVE_REST
  1411. mov rp=r14
  1412. ;;
  1413. adds out2=16,r12 // out2 = pointer to pt_regs
  1414. br.call.sptk.many b6=ia64_do_page_fault // ignore return address
  1415. END(page_fault)
  1416. ENTRY(non_syscall)
  1417. mov ar.rsc=r27 // restore ar.rsc before SAVE_MIN_WITH_COVER
  1418. ;;
  1419. SAVE_MIN_WITH_COVER
  1420. // There is no particular reason for this code to be here, other than that
  1421. // there happens to be space here that would go unused otherwise. If this
  1422. // fault ever gets "unreserved", simply moved the following code to a more
  1423. // suitable spot...
  1424. alloc r14=ar.pfs,0,0,2,0
  1425. MOV_FROM_IIM(out0)
  1426. add out1=16,sp
  1427. adds r3=8,r2 // set up second base pointer for SAVE_REST
  1428. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r15, r24)
  1429. // guarantee that interruption collection is on
  1430. SSM_PSR_I(p15, p15, r15) // restore psr.i
  1431. movl r15=ia64_leave_kernel
  1432. ;;
  1433. SAVE_REST
  1434. mov rp=r15
  1435. ;;
  1436. br.call.sptk.many b6=ia64_bad_break // avoid WAW on CFM and ignore return addr
  1437. END(non_syscall)
  1438. ENTRY(__interrupt)
  1439. DBG_FAULT(12)
  1440. mov r31=pr // prepare to save predicates
  1441. ;;
  1442. SAVE_MIN_WITH_COVER // uses r31; defines r2 and r3
  1443. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r14)
  1444. // ensure everybody knows psr.ic is back on
  1445. adds r3=8,r2 // set up second base pointer for SAVE_REST
  1446. ;;
  1447. SAVE_REST
  1448. ;;
  1449. MCA_RECOVER_RANGE(interrupt)
  1450. alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group
  1451. MOV_FROM_IVR(out0, r8) // pass cr.ivr as first arg
  1452. add out1=16,sp // pass pointer to pt_regs as second arg
  1453. ;;
  1454. srlz.d // make sure we see the effect of cr.ivr
  1455. movl r14=ia64_leave_kernel
  1456. ;;
  1457. mov rp=r14
  1458. br.call.sptk.many b6=ia64_handle_irq
  1459. END(__interrupt)
  1460. /*
  1461. * There is no particular reason for this code to be here, other than that
  1462. * there happens to be space here that would go unused otherwise. If this
  1463. * fault ever gets "unreserved", simply moved the following code to a more
  1464. * suitable spot...
  1465. */
  1466. ENTRY(dispatch_unaligned_handler)
  1467. SAVE_MIN_WITH_COVER
  1468. ;;
  1469. alloc r14=ar.pfs,0,0,2,0 // now it's safe (must be first in insn group!)
  1470. MOV_FROM_IFA(out0)
  1471. adds out1=16,sp
  1472. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24)
  1473. // guarantee that interruption collection is on
  1474. SSM_PSR_I(p15, p15, r3) // restore psr.i
  1475. adds r3=8,r2 // set up second base pointer
  1476. ;;
  1477. SAVE_REST
  1478. movl r14=ia64_leave_kernel
  1479. ;;
  1480. mov rp=r14
  1481. br.sptk.many ia64_prepare_handle_unaligned
  1482. END(dispatch_unaligned_handler)
  1483. /*
  1484. * There is no particular reason for this code to be here, other than that
  1485. * there happens to be space here that would go unused otherwise. If this
  1486. * fault ever gets "unreserved", simply moved the following code to a more
  1487. * suitable spot...
  1488. */
  1489. ENTRY(dispatch_to_fault_handler)
  1490. /*
  1491. * Input:
  1492. * psr.ic: off
  1493. * r19: fault vector number (e.g., 24 for General Exception)
  1494. * r31: contains saved predicates (pr)
  1495. */
  1496. SAVE_MIN_WITH_COVER_R19
  1497. alloc r14=ar.pfs,0,0,5,0
  1498. MOV_FROM_ISR(out1)
  1499. MOV_FROM_IFA(out2)
  1500. MOV_FROM_IIM(out3)
  1501. MOV_FROM_ITIR(out4)
  1502. ;;
  1503. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, out0)
  1504. // guarantee that interruption collection is on
  1505. mov out0=r15
  1506. ;;
  1507. SSM_PSR_I(p15, p15, r3) // restore psr.i
  1508. adds r3=8,r2 // set up second base pointer for SAVE_REST
  1509. ;;
  1510. SAVE_REST
  1511. movl r14=ia64_leave_kernel
  1512. ;;
  1513. mov rp=r14
  1514. br.call.sptk.many b6=ia64_fault
  1515. END(dispatch_to_fault_handler)
  1516. /*
  1517. * Squatting in this space ...
  1518. *
  1519. * This special case dispatcher for illegal operation faults allows preserved
  1520. * registers to be modified through a callback function (asm only) that is handed
  1521. * back from the fault handler in r8. Up to three arguments can be passed to the
  1522. * callback function by returning an aggregate with the callback as its first
  1523. * element, followed by the arguments.
  1524. */
  1525. ENTRY(dispatch_illegal_op_fault)
  1526. .prologue
  1527. .body
  1528. SAVE_MIN_WITH_COVER
  1529. SSM_PSR_IC_AND_DEFAULT_BITS_AND_SRLZ_I(r3, r24)
  1530. // guarantee that interruption collection is on
  1531. ;;
  1532. SSM_PSR_I(p15, p15, r3) // restore psr.i
  1533. adds r3=8,r2 // set up second base pointer for SAVE_REST
  1534. ;;
  1535. alloc r14=ar.pfs,0,0,1,0 // must be first in insn group
  1536. mov out0=ar.ec
  1537. ;;
  1538. SAVE_REST
  1539. PT_REGS_UNWIND_INFO(0)
  1540. ;;
  1541. br.call.sptk.many rp=ia64_illegal_op_fault
  1542. .ret0: ;;
  1543. alloc r14=ar.pfs,0,0,3,0 // must be first in insn group
  1544. mov out0=r9
  1545. mov out1=r10
  1546. mov out2=r11
  1547. movl r15=ia64_leave_kernel
  1548. ;;
  1549. mov rp=r15
  1550. mov b6=r8
  1551. ;;
  1552. cmp.ne p6,p0=0,r8
  1553. (p6) br.call.dpnt.many b6=b6 // call returns to ia64_leave_kernel
  1554. br.sptk.many ia64_leave_kernel
  1555. END(dispatch_illegal_op_fault)