/media/libjpeg/simd/jiss2fst-64.asm

http://github.com/zpao/v8monkey · Assembly · 492 lines · 332 code · 90 blank · 70 comment · 2 complexity · f85348298dfa0341865dff2d9a0cc489 MD5 · raw file

  1. ;
  2. ; jiss2fst-64.asm - fast integer IDCT (64-bit SSE2)
  3. ;
  4. ; Copyright 2009 Pierre Ossman <ossman@cendio.se> for Cendio AB
  5. ; Copyright 2009 D. R. Commander
  6. ;
  7. ; Based on
  8. ; x86 SIMD extension for IJG JPEG library
  9. ; Copyright (C) 1999-2006, MIYASAKA Masaru.
  10. ; For conditions of distribution and use, see copyright notice in jsimdext.inc
  11. ;
  12. ; This file should be assembled with NASM (Netwide Assembler),
  13. ; can *not* be assembled with Microsoft's MASM or any compatible
  14. ; assembler (including Borland's Turbo Assembler).
  15. ; NASM is available from http://nasm.sourceforge.net/ or
  16. ; http://sourceforge.net/projecpt/showfiles.php?group_id=6208
  17. ;
  18. ; This file contains a fast, not so accurate integer implementation of
  19. ; the inverse DCT (Discrete Cosine Transform). The following code is
  20. ; based directly on the IJG's original jidctfst.c; see the jidctfst.c
  21. ; for more details.
  22. ;
  23. ; [TAB8]
  24. %include "jsimdext.inc"
  25. %include "jdct.inc"
  26. ; --------------------------------------------------------------------------
  27. %define CONST_BITS 8 ; 14 is also OK.
  28. %define PASS1_BITS 2
  29. %if IFAST_SCALE_BITS != PASS1_BITS
  30. %error "'IFAST_SCALE_BITS' must be equal to 'PASS1_BITS'."
  31. %endif
  32. %if CONST_BITS == 8
  33. F_1_082 equ 277 ; FIX(1.082392200)
  34. F_1_414 equ 362 ; FIX(1.414213562)
  35. F_1_847 equ 473 ; FIX(1.847759065)
  36. F_2_613 equ 669 ; FIX(2.613125930)
  37. F_1_613 equ (F_2_613 - 256) ; FIX(2.613125930) - FIX(1)
  38. %else
  39. ; NASM cannot do compile-time arithmetic on floating-point constants.
  40. %define DESCALE(x,n) (((x)+(1<<((n)-1)))>>(n))
  41. F_1_082 equ DESCALE(1162209775,30-CONST_BITS) ; FIX(1.082392200)
  42. F_1_414 equ DESCALE(1518500249,30-CONST_BITS) ; FIX(1.414213562)
  43. F_1_847 equ DESCALE(1984016188,30-CONST_BITS) ; FIX(1.847759065)
  44. F_2_613 equ DESCALE(2805822602,30-CONST_BITS) ; FIX(2.613125930)
  45. F_1_613 equ (F_2_613 - (1 << CONST_BITS)) ; FIX(2.613125930) - FIX(1)
  46. %endif
  47. ; --------------------------------------------------------------------------
  48. SECTION SEG_CONST
  49. ; PRE_MULTIPLY_SCALE_BITS <= 2 (to avoid overflow)
  50. ; CONST_BITS + CONST_SHIFT + PRE_MULTIPLY_SCALE_BITS == 16 (for pmulhw)
  51. %define PRE_MULTIPLY_SCALE_BITS 2
  52. %define CONST_SHIFT (16 - PRE_MULTIPLY_SCALE_BITS - CONST_BITS)
  53. alignz 16
  54. global EXTN(jconst_idct_ifast_sse2)
  55. EXTN(jconst_idct_ifast_sse2):
  56. PW_F1414 times 8 dw F_1_414 << CONST_SHIFT
  57. PW_F1847 times 8 dw F_1_847 << CONST_SHIFT
  58. PW_MF1613 times 8 dw -F_1_613 << CONST_SHIFT
  59. PW_F1082 times 8 dw F_1_082 << CONST_SHIFT
  60. PB_CENTERJSAMP times 16 db CENTERJSAMPLE
  61. alignz 16
  62. ; --------------------------------------------------------------------------
  63. SECTION SEG_TEXT
  64. BITS 64
  65. ;
  66. ; Perform dequantization and inverse DCT on one block of coefficients.
  67. ;
  68. ; GLOBAL(void)
  69. ; jsimd_idct_ifast_sse2 (void * dct_table, JCOEFPTR coef_block,
  70. ; JSAMPARRAY output_buf, JDIMENSION output_col)
  71. ;
  72. ; r10 = jpeg_component_info * compptr
  73. ; r11 = JCOEFPTR coef_block
  74. ; r12 = JSAMPARRAY output_buf
  75. ; r13 = JDIMENSION output_col
  76. %define original_rbp rbp+0
  77. %define wk(i) rbp-(WK_NUM-(i))*SIZEOF_XMMWORD ; xmmword wk[WK_NUM]
  78. %define WK_NUM 2
  79. align 16
  80. global EXTN(jsimd_idct_ifast_sse2)
  81. EXTN(jsimd_idct_ifast_sse2):
  82. push rbp
  83. mov rax,rsp ; rax = original rbp
  84. sub rsp, byte 4
  85. and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits
  86. mov [rsp],rax
  87. mov rbp,rsp ; rbp = aligned rbp
  88. lea rsp, [wk(0)]
  89. collect_args
  90. ; ---- Pass 1: process columns from input.
  91. mov rdx, r10 ; quantptr
  92. mov rsi, r11 ; inptr
  93. %ifndef NO_ZERO_COLUMN_TEST_IFAST_SSE2
  94. mov eax, DWORD [DWBLOCK(1,0,rsi,SIZEOF_JCOEF)]
  95. or eax, DWORD [DWBLOCK(2,0,rsi,SIZEOF_JCOEF)]
  96. jnz near .columnDCT
  97. movdqa xmm0, XMMWORD [XMMBLOCK(1,0,rsi,SIZEOF_JCOEF)]
  98. movdqa xmm1, XMMWORD [XMMBLOCK(2,0,rsi,SIZEOF_JCOEF)]
  99. por xmm0, XMMWORD [XMMBLOCK(3,0,rsi,SIZEOF_JCOEF)]
  100. por xmm1, XMMWORD [XMMBLOCK(4,0,rsi,SIZEOF_JCOEF)]
  101. por xmm0, XMMWORD [XMMBLOCK(5,0,rsi,SIZEOF_JCOEF)]
  102. por xmm1, XMMWORD [XMMBLOCK(6,0,rsi,SIZEOF_JCOEF)]
  103. por xmm0, XMMWORD [XMMBLOCK(7,0,rsi,SIZEOF_JCOEF)]
  104. por xmm1,xmm0
  105. packsswb xmm1,xmm1
  106. packsswb xmm1,xmm1
  107. movd eax,xmm1
  108. test rax,rax
  109. jnz short .columnDCT
  110. ; -- AC terms all zero
  111. movdqa xmm0, XMMWORD [XMMBLOCK(0,0,rsi,SIZEOF_JCOEF)]
  112. pmullw xmm0, XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_ISLOW_MULT_TYPE)]
  113. movdqa xmm7,xmm0 ; xmm0=in0=(00 01 02 03 04 05 06 07)
  114. punpcklwd xmm0,xmm0 ; xmm0=(00 00 01 01 02 02 03 03)
  115. punpckhwd xmm7,xmm7 ; xmm7=(04 04 05 05 06 06 07 07)
  116. pshufd xmm6,xmm0,0x00 ; xmm6=col0=(00 00 00 00 00 00 00 00)
  117. pshufd xmm2,xmm0,0x55 ; xmm2=col1=(01 01 01 01 01 01 01 01)
  118. pshufd xmm5,xmm0,0xAA ; xmm5=col2=(02 02 02 02 02 02 02 02)
  119. pshufd xmm0,xmm0,0xFF ; xmm0=col3=(03 03 03 03 03 03 03 03)
  120. pshufd xmm1,xmm7,0x00 ; xmm1=col4=(04 04 04 04 04 04 04 04)
  121. pshufd xmm4,xmm7,0x55 ; xmm4=col5=(05 05 05 05 05 05 05 05)
  122. pshufd xmm3,xmm7,0xAA ; xmm3=col6=(06 06 06 06 06 06 06 06)
  123. pshufd xmm7,xmm7,0xFF ; xmm7=col7=(07 07 07 07 07 07 07 07)
  124. movdqa XMMWORD [wk(0)], xmm2 ; wk(0)=col1
  125. movdqa XMMWORD [wk(1)], xmm0 ; wk(1)=col3
  126. jmp near .column_end
  127. %endif
  128. .columnDCT:
  129. ; -- Even part
  130. movdqa xmm0, XMMWORD [XMMBLOCK(0,0,rsi,SIZEOF_JCOEF)]
  131. movdqa xmm1, XMMWORD [XMMBLOCK(2,0,rsi,SIZEOF_JCOEF)]
  132. pmullw xmm0, XMMWORD [XMMBLOCK(0,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  133. pmullw xmm1, XMMWORD [XMMBLOCK(2,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  134. movdqa xmm2, XMMWORD [XMMBLOCK(4,0,rsi,SIZEOF_JCOEF)]
  135. movdqa xmm3, XMMWORD [XMMBLOCK(6,0,rsi,SIZEOF_JCOEF)]
  136. pmullw xmm2, XMMWORD [XMMBLOCK(4,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  137. pmullw xmm3, XMMWORD [XMMBLOCK(6,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  138. movdqa xmm4,xmm0
  139. movdqa xmm5,xmm1
  140. psubw xmm0,xmm2 ; xmm0=tmp11
  141. psubw xmm1,xmm3
  142. paddw xmm4,xmm2 ; xmm4=tmp10
  143. paddw xmm5,xmm3 ; xmm5=tmp13
  144. psllw xmm1,PRE_MULTIPLY_SCALE_BITS
  145. pmulhw xmm1,[rel PW_F1414]
  146. psubw xmm1,xmm5 ; xmm1=tmp12
  147. movdqa xmm6,xmm4
  148. movdqa xmm7,xmm0
  149. psubw xmm4,xmm5 ; xmm4=tmp3
  150. psubw xmm0,xmm1 ; xmm0=tmp2
  151. paddw xmm6,xmm5 ; xmm6=tmp0
  152. paddw xmm7,xmm1 ; xmm7=tmp1
  153. movdqa XMMWORD [wk(1)], xmm4 ; wk(1)=tmp3
  154. movdqa XMMWORD [wk(0)], xmm0 ; wk(0)=tmp2
  155. ; -- Odd part
  156. movdqa xmm2, XMMWORD [XMMBLOCK(1,0,rsi,SIZEOF_JCOEF)]
  157. movdqa xmm3, XMMWORD [XMMBLOCK(3,0,rsi,SIZEOF_JCOEF)]
  158. pmullw xmm2, XMMWORD [XMMBLOCK(1,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  159. pmullw xmm3, XMMWORD [XMMBLOCK(3,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  160. movdqa xmm5, XMMWORD [XMMBLOCK(5,0,rsi,SIZEOF_JCOEF)]
  161. movdqa xmm1, XMMWORD [XMMBLOCK(7,0,rsi,SIZEOF_JCOEF)]
  162. pmullw xmm5, XMMWORD [XMMBLOCK(5,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  163. pmullw xmm1, XMMWORD [XMMBLOCK(7,0,rdx,SIZEOF_IFAST_MULT_TYPE)]
  164. movdqa xmm4,xmm2
  165. movdqa xmm0,xmm5
  166. psubw xmm2,xmm1 ; xmm2=z12
  167. psubw xmm5,xmm3 ; xmm5=z10
  168. paddw xmm4,xmm1 ; xmm4=z11
  169. paddw xmm0,xmm3 ; xmm0=z13
  170. movdqa xmm1,xmm5 ; xmm1=z10(unscaled)
  171. psllw xmm2,PRE_MULTIPLY_SCALE_BITS
  172. psllw xmm5,PRE_MULTIPLY_SCALE_BITS
  173. movdqa xmm3,xmm4
  174. psubw xmm4,xmm0
  175. paddw xmm3,xmm0 ; xmm3=tmp7
  176. psllw xmm4,PRE_MULTIPLY_SCALE_BITS
  177. pmulhw xmm4,[rel PW_F1414] ; xmm4=tmp11
  178. ; To avoid overflow...
  179. ;
  180. ; (Original)
  181. ; tmp12 = -2.613125930 * z10 + z5;
  182. ;
  183. ; (This implementation)
  184. ; tmp12 = (-1.613125930 - 1) * z10 + z5;
  185. ; = -1.613125930 * z10 - z10 + z5;
  186. movdqa xmm0,xmm5
  187. paddw xmm5,xmm2
  188. pmulhw xmm5,[rel PW_F1847] ; xmm5=z5
  189. pmulhw xmm0,[rel PW_MF1613]
  190. pmulhw xmm2,[rel PW_F1082]
  191. psubw xmm0,xmm1
  192. psubw xmm2,xmm5 ; xmm2=tmp10
  193. paddw xmm0,xmm5 ; xmm0=tmp12
  194. ; -- Final output stage
  195. psubw xmm0,xmm3 ; xmm0=tmp6
  196. movdqa xmm1,xmm6
  197. movdqa xmm5,xmm7
  198. paddw xmm6,xmm3 ; xmm6=data0=(00 01 02 03 04 05 06 07)
  199. paddw xmm7,xmm0 ; xmm7=data1=(10 11 12 13 14 15 16 17)
  200. psubw xmm1,xmm3 ; xmm1=data7=(70 71 72 73 74 75 76 77)
  201. psubw xmm5,xmm0 ; xmm5=data6=(60 61 62 63 64 65 66 67)
  202. psubw xmm4,xmm0 ; xmm4=tmp5
  203. movdqa xmm3,xmm6 ; transpose coefficients(phase 1)
  204. punpcklwd xmm6,xmm7 ; xmm6=(00 10 01 11 02 12 03 13)
  205. punpckhwd xmm3,xmm7 ; xmm3=(04 14 05 15 06 16 07 17)
  206. movdqa xmm0,xmm5 ; transpose coefficients(phase 1)
  207. punpcklwd xmm5,xmm1 ; xmm5=(60 70 61 71 62 72 63 73)
  208. punpckhwd xmm0,xmm1 ; xmm0=(64 74 65 75 66 76 67 77)
  209. movdqa xmm7, XMMWORD [wk(0)] ; xmm7=tmp2
  210. movdqa xmm1, XMMWORD [wk(1)] ; xmm1=tmp3
  211. movdqa XMMWORD [wk(0)], xmm5 ; wk(0)=(60 70 61 71 62 72 63 73)
  212. movdqa XMMWORD [wk(1)], xmm0 ; wk(1)=(64 74 65 75 66 76 67 77)
  213. paddw xmm2,xmm4 ; xmm2=tmp4
  214. movdqa xmm5,xmm7
  215. movdqa xmm0,xmm1
  216. paddw xmm7,xmm4 ; xmm7=data2=(20 21 22 23 24 25 26 27)
  217. paddw xmm1,xmm2 ; xmm1=data4=(40 41 42 43 44 45 46 47)
  218. psubw xmm5,xmm4 ; xmm5=data5=(50 51 52 53 54 55 56 57)
  219. psubw xmm0,xmm2 ; xmm0=data3=(30 31 32 33 34 35 36 37)
  220. movdqa xmm4,xmm7 ; transpose coefficients(phase 1)
  221. punpcklwd xmm7,xmm0 ; xmm7=(20 30 21 31 22 32 23 33)
  222. punpckhwd xmm4,xmm0 ; xmm4=(24 34 25 35 26 36 27 37)
  223. movdqa xmm2,xmm1 ; transpose coefficients(phase 1)
  224. punpcklwd xmm1,xmm5 ; xmm1=(40 50 41 51 42 52 43 53)
  225. punpckhwd xmm2,xmm5 ; xmm2=(44 54 45 55 46 56 47 57)
  226. movdqa xmm0,xmm3 ; transpose coefficients(phase 2)
  227. punpckldq xmm3,xmm4 ; xmm3=(04 14 24 34 05 15 25 35)
  228. punpckhdq xmm0,xmm4 ; xmm0=(06 16 26 36 07 17 27 37)
  229. movdqa xmm5,xmm6 ; transpose coefficients(phase 2)
  230. punpckldq xmm6,xmm7 ; xmm6=(00 10 20 30 01 11 21 31)
  231. punpckhdq xmm5,xmm7 ; xmm5=(02 12 22 32 03 13 23 33)
  232. movdqa xmm4, XMMWORD [wk(0)] ; xmm4=(60 70 61 71 62 72 63 73)
  233. movdqa xmm7, XMMWORD [wk(1)] ; xmm7=(64 74 65 75 66 76 67 77)
  234. movdqa XMMWORD [wk(0)], xmm3 ; wk(0)=(04 14 24 34 05 15 25 35)
  235. movdqa XMMWORD [wk(1)], xmm0 ; wk(1)=(06 16 26 36 07 17 27 37)
  236. movdqa xmm3,xmm1 ; transpose coefficients(phase 2)
  237. punpckldq xmm1,xmm4 ; xmm1=(40 50 60 70 41 51 61 71)
  238. punpckhdq xmm3,xmm4 ; xmm3=(42 52 62 72 43 53 63 73)
  239. movdqa xmm0,xmm2 ; transpose coefficients(phase 2)
  240. punpckldq xmm2,xmm7 ; xmm2=(44 54 64 74 45 55 65 75)
  241. punpckhdq xmm0,xmm7 ; xmm0=(46 56 66 76 47 57 67 77)
  242. movdqa xmm4,xmm6 ; transpose coefficients(phase 3)
  243. punpcklqdq xmm6,xmm1 ; xmm6=col0=(00 10 20 30 40 50 60 70)
  244. punpckhqdq xmm4,xmm1 ; xmm4=col1=(01 11 21 31 41 51 61 71)
  245. movdqa xmm7,xmm5 ; transpose coefficients(phase 3)
  246. punpcklqdq xmm5,xmm3 ; xmm5=col2=(02 12 22 32 42 52 62 72)
  247. punpckhqdq xmm7,xmm3 ; xmm7=col3=(03 13 23 33 43 53 63 73)
  248. movdqa xmm1, XMMWORD [wk(0)] ; xmm1=(04 14 24 34 05 15 25 35)
  249. movdqa xmm3, XMMWORD [wk(1)] ; xmm3=(06 16 26 36 07 17 27 37)
  250. movdqa XMMWORD [wk(0)], xmm4 ; wk(0)=col1
  251. movdqa XMMWORD [wk(1)], xmm7 ; wk(1)=col3
  252. movdqa xmm4,xmm1 ; transpose coefficients(phase 3)
  253. punpcklqdq xmm1,xmm2 ; xmm1=col4=(04 14 24 34 44 54 64 74)
  254. punpckhqdq xmm4,xmm2 ; xmm4=col5=(05 15 25 35 45 55 65 75)
  255. movdqa xmm7,xmm3 ; transpose coefficients(phase 3)
  256. punpcklqdq xmm3,xmm0 ; xmm3=col6=(06 16 26 36 46 56 66 76)
  257. punpckhqdq xmm7,xmm0 ; xmm7=col7=(07 17 27 37 47 57 67 77)
  258. .column_end:
  259. ; -- Prefetch the next coefficient block
  260. prefetchnta [rsi + DCTSIZE2*SIZEOF_JCOEF + 0*32]
  261. prefetchnta [rsi + DCTSIZE2*SIZEOF_JCOEF + 1*32]
  262. prefetchnta [rsi + DCTSIZE2*SIZEOF_JCOEF + 2*32]
  263. prefetchnta [rsi + DCTSIZE2*SIZEOF_JCOEF + 3*32]
  264. ; ---- Pass 2: process rows from work array, store into output array.
  265. mov rax, [original_rbp]
  266. mov rdi, r12 ; (JSAMPROW *)
  267. mov rax, r13
  268. ; -- Even part
  269. ; xmm6=col0, xmm5=col2, xmm1=col4, xmm3=col6
  270. movdqa xmm2,xmm6
  271. movdqa xmm0,xmm5
  272. psubw xmm6,xmm1 ; xmm6=tmp11
  273. psubw xmm5,xmm3
  274. paddw xmm2,xmm1 ; xmm2=tmp10
  275. paddw xmm0,xmm3 ; xmm0=tmp13
  276. psllw xmm5,PRE_MULTIPLY_SCALE_BITS
  277. pmulhw xmm5,[rel PW_F1414]
  278. psubw xmm5,xmm0 ; xmm5=tmp12
  279. movdqa xmm1,xmm2
  280. movdqa xmm3,xmm6
  281. psubw xmm2,xmm0 ; xmm2=tmp3
  282. psubw xmm6,xmm5 ; xmm6=tmp2
  283. paddw xmm1,xmm0 ; xmm1=tmp0
  284. paddw xmm3,xmm5 ; xmm3=tmp1
  285. movdqa xmm0, XMMWORD [wk(0)] ; xmm0=col1
  286. movdqa xmm5, XMMWORD [wk(1)] ; xmm5=col3
  287. movdqa XMMWORD [wk(0)], xmm2 ; wk(0)=tmp3
  288. movdqa XMMWORD [wk(1)], xmm6 ; wk(1)=tmp2
  289. ; -- Odd part
  290. ; xmm0=col1, xmm5=col3, xmm4=col5, xmm7=col7
  291. movdqa xmm2,xmm0
  292. movdqa xmm6,xmm4
  293. psubw xmm0,xmm7 ; xmm0=z12
  294. psubw xmm4,xmm5 ; xmm4=z10
  295. paddw xmm2,xmm7 ; xmm2=z11
  296. paddw xmm6,xmm5 ; xmm6=z13
  297. movdqa xmm7,xmm4 ; xmm7=z10(unscaled)
  298. psllw xmm0,PRE_MULTIPLY_SCALE_BITS
  299. psllw xmm4,PRE_MULTIPLY_SCALE_BITS
  300. movdqa xmm5,xmm2
  301. psubw xmm2,xmm6
  302. paddw xmm5,xmm6 ; xmm5=tmp7
  303. psllw xmm2,PRE_MULTIPLY_SCALE_BITS
  304. pmulhw xmm2,[rel PW_F1414] ; xmm2=tmp11
  305. ; To avoid overflow...
  306. ;
  307. ; (Original)
  308. ; tmp12 = -2.613125930 * z10 + z5;
  309. ;
  310. ; (This implementation)
  311. ; tmp12 = (-1.613125930 - 1) * z10 + z5;
  312. ; = -1.613125930 * z10 - z10 + z5;
  313. movdqa xmm6,xmm4
  314. paddw xmm4,xmm0
  315. pmulhw xmm4,[rel PW_F1847] ; xmm4=z5
  316. pmulhw xmm6,[rel PW_MF1613]
  317. pmulhw xmm0,[rel PW_F1082]
  318. psubw xmm6,xmm7
  319. psubw xmm0,xmm4 ; xmm0=tmp10
  320. paddw xmm6,xmm4 ; xmm6=tmp12
  321. ; -- Final output stage
  322. psubw xmm6,xmm5 ; xmm6=tmp6
  323. movdqa xmm7,xmm1
  324. movdqa xmm4,xmm3
  325. paddw xmm1,xmm5 ; xmm1=data0=(00 10 20 30 40 50 60 70)
  326. paddw xmm3,xmm6 ; xmm3=data1=(01 11 21 31 41 51 61 71)
  327. psraw xmm1,(PASS1_BITS+3) ; descale
  328. psraw xmm3,(PASS1_BITS+3) ; descale
  329. psubw xmm7,xmm5 ; xmm7=data7=(07 17 27 37 47 57 67 77)
  330. psubw xmm4,xmm6 ; xmm4=data6=(06 16 26 36 46 56 66 76)
  331. psraw xmm7,(PASS1_BITS+3) ; descale
  332. psraw xmm4,(PASS1_BITS+3) ; descale
  333. psubw xmm2,xmm6 ; xmm2=tmp5
  334. packsswb xmm1,xmm4 ; xmm1=(00 10 20 30 40 50 60 70 06 16 26 36 46 56 66 76)
  335. packsswb xmm3,xmm7 ; xmm3=(01 11 21 31 41 51 61 71 07 17 27 37 47 57 67 77)
  336. movdqa xmm5, XMMWORD [wk(1)] ; xmm5=tmp2
  337. movdqa xmm6, XMMWORD [wk(0)] ; xmm6=tmp3
  338. paddw xmm0,xmm2 ; xmm0=tmp4
  339. movdqa xmm4,xmm5
  340. movdqa xmm7,xmm6
  341. paddw xmm5,xmm2 ; xmm5=data2=(02 12 22 32 42 52 62 72)
  342. paddw xmm6,xmm0 ; xmm6=data4=(04 14 24 34 44 54 64 74)
  343. psraw xmm5,(PASS1_BITS+3) ; descale
  344. psraw xmm6,(PASS1_BITS+3) ; descale
  345. psubw xmm4,xmm2 ; xmm4=data5=(05 15 25 35 45 55 65 75)
  346. psubw xmm7,xmm0 ; xmm7=data3=(03 13 23 33 43 53 63 73)
  347. psraw xmm4,(PASS1_BITS+3) ; descale
  348. psraw xmm7,(PASS1_BITS+3) ; descale
  349. movdqa xmm2,[rel PB_CENTERJSAMP] ; xmm2=[rel PB_CENTERJSAMP]
  350. packsswb xmm5,xmm6 ; xmm5=(02 12 22 32 42 52 62 72 04 14 24 34 44 54 64 74)
  351. packsswb xmm7,xmm4 ; xmm7=(03 13 23 33 43 53 63 73 05 15 25 35 45 55 65 75)
  352. paddb xmm1,xmm2
  353. paddb xmm3,xmm2
  354. paddb xmm5,xmm2
  355. paddb xmm7,xmm2
  356. movdqa xmm0,xmm1 ; transpose coefficients(phase 1)
  357. punpcklbw xmm1,xmm3 ; xmm1=(00 01 10 11 20 21 30 31 40 41 50 51 60 61 70 71)
  358. punpckhbw xmm0,xmm3 ; xmm0=(06 07 16 17 26 27 36 37 46 47 56 57 66 67 76 77)
  359. movdqa xmm6,xmm5 ; transpose coefficients(phase 1)
  360. punpcklbw xmm5,xmm7 ; xmm5=(02 03 12 13 22 23 32 33 42 43 52 53 62 63 72 73)
  361. punpckhbw xmm6,xmm7 ; xmm6=(04 05 14 15 24 25 34 35 44 45 54 55 64 65 74 75)
  362. movdqa xmm4,xmm1 ; transpose coefficients(phase 2)
  363. punpcklwd xmm1,xmm5 ; xmm1=(00 01 02 03 10 11 12 13 20 21 22 23 30 31 32 33)
  364. punpckhwd xmm4,xmm5 ; xmm4=(40 41 42 43 50 51 52 53 60 61 62 63 70 71 72 73)
  365. movdqa xmm2,xmm6 ; transpose coefficients(phase 2)
  366. punpcklwd xmm6,xmm0 ; xmm6=(04 05 06 07 14 15 16 17 24 25 26 27 34 35 36 37)
  367. punpckhwd xmm2,xmm0 ; xmm2=(44 45 46 47 54 55 56 57 64 65 66 67 74 75 76 77)
  368. movdqa xmm3,xmm1 ; transpose coefficients(phase 3)
  369. punpckldq xmm1,xmm6 ; xmm1=(00 01 02 03 04 05 06 07 10 11 12 13 14 15 16 17)
  370. punpckhdq xmm3,xmm6 ; xmm3=(20 21 22 23 24 25 26 27 30 31 32 33 34 35 36 37)
  371. movdqa xmm7,xmm4 ; transpose coefficients(phase 3)
  372. punpckldq xmm4,xmm2 ; xmm4=(40 41 42 43 44 45 46 47 50 51 52 53 54 55 56 57)
  373. punpckhdq xmm7,xmm2 ; xmm7=(60 61 62 63 64 65 66 67 70 71 72 73 74 75 76 77)
  374. pshufd xmm5,xmm1,0x4E ; xmm5=(10 11 12 13 14 15 16 17 00 01 02 03 04 05 06 07)
  375. pshufd xmm0,xmm3,0x4E ; xmm0=(30 31 32 33 34 35 36 37 20 21 22 23 24 25 26 27)
  376. pshufd xmm6,xmm4,0x4E ; xmm6=(50 51 52 53 54 55 56 57 40 41 42 43 44 45 46 47)
  377. pshufd xmm2,xmm7,0x4E ; xmm2=(70 71 72 73 74 75 76 77 60 61 62 63 64 65 66 67)
  378. mov rdx, JSAMPROW [rdi+0*SIZEOF_JSAMPROW]
  379. mov rsi, JSAMPROW [rdi+2*SIZEOF_JSAMPROW]
  380. movq XMM_MMWORD [rdx+rax*SIZEOF_JSAMPLE], xmm1
  381. movq XMM_MMWORD [rsi+rax*SIZEOF_JSAMPLE], xmm3
  382. mov rdx, JSAMPROW [rdi+4*SIZEOF_JSAMPROW]
  383. mov rsi, JSAMPROW [rdi+6*SIZEOF_JSAMPROW]
  384. movq XMM_MMWORD [rdx+rax*SIZEOF_JSAMPLE], xmm4
  385. movq XMM_MMWORD [rsi+rax*SIZEOF_JSAMPLE], xmm7
  386. mov rdx, JSAMPROW [rdi+1*SIZEOF_JSAMPROW]
  387. mov rsi, JSAMPROW [rdi+3*SIZEOF_JSAMPROW]
  388. movq XMM_MMWORD [rdx+rax*SIZEOF_JSAMPLE], xmm5
  389. movq XMM_MMWORD [rsi+rax*SIZEOF_JSAMPLE], xmm0
  390. mov rdx, JSAMPROW [rdi+5*SIZEOF_JSAMPROW]
  391. mov rsi, JSAMPROW [rdi+7*SIZEOF_JSAMPROW]
  392. movq XMM_MMWORD [rdx+rax*SIZEOF_JSAMPLE], xmm6
  393. movq XMM_MMWORD [rsi+rax*SIZEOF_JSAMPLE], xmm2
  394. uncollect_args
  395. mov rsp,rbp ; rsp <- aligned rbp
  396. pop rsp ; rsp <- original rbp
  397. pop rbp
  398. ret
  399. ret
  400. ; For some reason, the OS X linker does not honor the request to align the
  401. ; segment unless we do this.
  402. align 16