PageRenderTime 42ms CodeModel.GetById 0ms RepoModel.GetById 0ms app.codeStats 1ms

/lib/ffmpeg/libswscale/x86/rgb2rgb_template.c

https://bitbucket.org/bgiorgini/xbmc
C | 2595 lines | 2386 code | 131 blank | 78 comment | 78 complexity | 69133a8698389e86b7d9121a15e4a22d MD5 | raw file
Possible License(s): GPL-3.0, CC-BY-SA-3.0, BSD-3-Clause, GPL-2.0, LGPL-3.0, 0BSD, LGPL-2.0, AGPL-1.0, LGPL-2.1 
    

  1. /*
    2. 

  2.  * software RGB to RGB converter
    3. 

  3.  * pluralize by software PAL8 to RGB converter
    4. 

  4.  *              software YUV to YUV converter
    5. 

  5.  *              software YUV to RGB converter
    6. 

  6.  * Written by Nick Kurshev.
    7. 

  7.  * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
    8. 

  8.  * lot of big-endian byte order fixes by Alex Beregszaszi
    9. 

  9.  *
    10. 

  10.  * This file is part of FFmpeg.
    11. 

  11.  *
    12. 

  12.  * FFmpeg is free software; you can redistribute it and/or
    13. 

  13.  * modify it under the terms of the GNU Lesser General Public
    14. 

  14.  * License as published by the Free Software Foundation; either
    15. 

  15.  * version 2.1 of the License, or (at your option) any later version.
    16. 

  16.  *
    17. 

  17.  * FFmpeg is distributed in the hope that it will be useful,
    18. 

  18.  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    19. 

  19.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    20. 

  20.  * Lesser General Public License for more details.
    21. 

  21.  *
    22. 

  22.  * You should have received a copy of the GNU Lesser General Public
    23. 

  23.  * License along with FFmpeg; if not, write to the Free Software
    24. 

  24.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
    25. 

  25.  */
    26. 

  26. 

  27. #include <stddef.h>
    28. 

  28. 

  29. #undef PREFETCH
    30. 

  30. #undef MOVNTQ
    31. 

  31. #undef EMMS
    32. 

  32. #undef SFENCE
    33. 

  33. #undef PAVGB
    34. 

  34. 

  35. #if COMPILE_TEMPLATE_AMD3DNOW
    36. 

  36. #define PREFETCH  "prefetch"
    37. 

  37. #define PAVGB     "pavgusb"
    38. 

  38. #elif COMPILE_TEMPLATE_MMX2
    39. 

  39. #define PREFETCH "prefetchnta"
    40. 

  40. #define PAVGB     "pavgb"
    41. 

  41. #else
    42. 

  42. #define PREFETCH  " # nop"
    43. 

  43. #endif
    44. 

  44. 

  45. #if COMPILE_TEMPLATE_AMD3DNOW
    46. 

  46. /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
    47. 

  47. #define EMMS     "femms"
    48. 

  48. #else
    49. 

  49. #define EMMS     "emms"
    50. 

  50. #endif
    51. 

  51. 

  52. #if COMPILE_TEMPLATE_MMX2
    53. 

  53. #define MOVNTQ "movntq"
    54. 

  54. #define SFENCE "sfence"
    55. 

  55. #else
    56. 

  56. #define MOVNTQ "movq"
    57. 

  57. #define SFENCE " # nop"
    58. 

  58. #endif
    59. 

  59. 

  60. #if !COMPILE_TEMPLATE_SSE2
    61. 

  61. 

  62. #if !COMPILE_TEMPLATE_AMD3DNOW
    63. 

  63. 

  64. static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, int src_size)
    65. 

  65. {
    66. 

  66.     uint8_t *dest = dst;
    67. 

  67.     const uint8_t *s = src;
    68. 

  68.     const uint8_t *end;
    69. 

  69.     const uint8_t *mm_end;
    70. 

  70.     end = s + src_size;
    71. 

  71.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    72. 

  72.     mm_end = end - 23;
    73. 

  73.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask32a):"memory");
    74. 

  74.     while (s < mm_end) {
    75. 

  75.         __asm__ volatile(
    76. 

  76.             PREFETCH"    32%1           \n\t"
    77. 

  77.             "movd          %1, %%mm0    \n\t"
    78. 

  78.             "punpckldq    3%1, %%mm0    \n\t"
    79. 

  79.             "movd         6%1, %%mm1    \n\t"
    80. 

  80.             "punpckldq    9%1, %%mm1    \n\t"
    81. 

  81.             "movd        12%1, %%mm2    \n\t"
    82. 

  82.             "punpckldq   15%1, %%mm2    \n\t"
    83. 

  83.             "movd        18%1, %%mm3    \n\t"
    84. 

  84.             "punpckldq   21%1, %%mm3    \n\t"
    85. 

  85.             "por        %%mm7, %%mm0    \n\t"
    86. 

  86.             "por        %%mm7, %%mm1    \n\t"
    87. 

  87.             "por        %%mm7, %%mm2    \n\t"
    88. 

  88.             "por        %%mm7, %%mm3    \n\t"
    89. 

  89.             MOVNTQ"     %%mm0,   %0     \n\t"
    90. 

  90.             MOVNTQ"     %%mm1,  8%0     \n\t"
    91. 

  91.             MOVNTQ"     %%mm2, 16%0     \n\t"
    92. 

  92.             MOVNTQ"     %%mm3, 24%0"
    93. 

  93.             :"=m"(*dest)
    94. 

  94.             :"m"(*s)
    95. 

  95.             :"memory");
    96. 

  96.         dest += 32;
    97. 

  97.         s += 24;
    98. 

  98.     }
    99. 

  99.     __asm__ volatile(SFENCE:::"memory");
    100. 

  100.     __asm__ volatile(EMMS:::"memory");
    101. 

  101.     while (s < end) {
    102. 

  102.         *dest++ = *s++;
    103. 

  103.         *dest++ = *s++;
    104. 

  104.         *dest++ = *s++;
    105. 

  105.         *dest++ = 255;
    106. 

  106.     }
    107. 

  107. }
    108. 

  108. 

  109. #define STORE_BGR24_MMX \
    110. 

  110.             "psrlq         $8, %%mm2    \n\t" \
    111. 

  111.             "psrlq         $8, %%mm3    \n\t" \
    112. 

  112.             "psrlq         $8, %%mm6    \n\t" \
    113. 

  113.             "psrlq         $8, %%mm7    \n\t" \
    114. 

  114.             "pand "MANGLE(mask24l)", %%mm0\n\t" \
    115. 

  115.             "pand "MANGLE(mask24l)", %%mm1\n\t" \
    116. 

  116.             "pand "MANGLE(mask24l)", %%mm4\n\t" \
    117. 

  117.             "pand "MANGLE(mask24l)", %%mm5\n\t" \
    118. 

  118.             "pand "MANGLE(mask24h)", %%mm2\n\t" \
    119. 

  119.             "pand "MANGLE(mask24h)", %%mm3\n\t" \
    120. 

  120.             "pand "MANGLE(mask24h)", %%mm6\n\t" \
    121. 

  121.             "pand "MANGLE(mask24h)", %%mm7\n\t" \
    122. 

  122.             "por        %%mm2, %%mm0    \n\t" \
    123. 

  123.             "por        %%mm3, %%mm1    \n\t" \
    124. 

  124.             "por        %%mm6, %%mm4    \n\t" \
    125. 

  125.             "por        %%mm7, %%mm5    \n\t" \
    126. 

  126.  \
    127. 

  127.             "movq       %%mm1, %%mm2    \n\t" \
    128. 

  128.             "movq       %%mm4, %%mm3    \n\t" \
    129. 

  129.             "psllq        $48, %%mm2    \n\t" \
    130. 

  130.             "psllq        $32, %%mm3    \n\t" \
    131. 

  131.             "pand "MANGLE(mask24hh)", %%mm2\n\t" \
    132. 

  132.             "pand "MANGLE(mask24hhh)", %%mm3\n\t" \
    133. 

  133.             "por        %%mm2, %%mm0    \n\t" \
    134. 

  134.             "psrlq        $16, %%mm1    \n\t" \
    135. 

  135.             "psrlq        $32, %%mm4    \n\t" \
    136. 

  136.             "psllq        $16, %%mm5    \n\t" \
    137. 

  137.             "por        %%mm3, %%mm1    \n\t" \
    138. 

  138.             "pand  "MANGLE(mask24hhhh)", %%mm5\n\t" \
    139. 

  139.             "por        %%mm5, %%mm4    \n\t" \
    140. 

  140.  \
    141. 

  141.             MOVNTQ"     %%mm0,   %0     \n\t" \
    142. 

  142.             MOVNTQ"     %%mm1,  8%0     \n\t" \
    143. 

  143.             MOVNTQ"     %%mm4, 16%0"
    144. 

  144. 

  145. 

  146. static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    147. 

  147. {
    148. 

  148.     uint8_t *dest = dst;
    149. 

  149.     const uint8_t *s = src;
    150. 

  150.     const uint8_t *end;
    151. 

  151.     const uint8_t *mm_end;
    152. 

  152.     end = s + src_size;
    153. 

  153.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    154. 

  154.     mm_end = end - 31;
    155. 

  155.     while (s < mm_end) {
    156. 

  156.         __asm__ volatile(
    157. 

  157.             PREFETCH"    32%1           \n\t"
    158. 

  158.             "movq          %1, %%mm0    \n\t"
    159. 

  159.             "movq         8%1, %%mm1    \n\t"
    160. 

  160.             "movq        16%1, %%mm4    \n\t"
    161. 

  161.             "movq        24%1, %%mm5    \n\t"
    162. 

  162.             "movq       %%mm0, %%mm2    \n\t"
    163. 

  163.             "movq       %%mm1, %%mm3    \n\t"
    164. 

  164.             "movq       %%mm4, %%mm6    \n\t"
    165. 

  165.             "movq       %%mm5, %%mm7    \n\t"
    166. 

  166.             STORE_BGR24_MMX
    167. 

  167.             :"=m"(*dest)
    168. 

  168.             :"m"(*s)
    169. 

  169.             :"memory");
    170. 

  170.         dest += 24;
    171. 

  171.         s += 32;
    172. 

  172.     }
    173. 

  173.     __asm__ volatile(SFENCE:::"memory");
    174. 

  174.     __asm__ volatile(EMMS:::"memory");
    175. 

  175.     while (s < end) {
    176. 

  176.         *dest++ = *s++;
    177. 

  177.         *dest++ = *s++;
    178. 

  178.         *dest++ = *s++;
    179. 

  179.         s++;
    180. 

  180.     }
    181. 

  181. }
    182. 

  182. 

  183. /*
    184. 

  184.  original by Strepto/Astral
    185. 

  185.  ported to gcc & bugfixed: A'rpi
    186. 

  186.  MMX2, 3DNOW optimization by Nick Kurshev
    187. 

  187.  32-bit C version, and and&add trick by Michael Niedermayer
    188. 

  188. */
    189. 

  189. static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, int src_size)
    190. 

  190. {
    191. 

  191.     register const uint8_t* s=src;
    192. 

  192.     register uint8_t* d=dst;
    193. 

  193.     register const uint8_t *end;
    194. 

  194.     const uint8_t *mm_end;
    195. 

  195.     end = s + src_size;
    196. 

  196.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));
    197. 

  197.     __asm__ volatile("movq        %0, %%mm4"::"m"(mask15s));
    198. 

  198.     mm_end = end - 15;
    199. 

  199.     while (s<mm_end) {
    200. 

  200.         __asm__ volatile(
    201. 

  201.             PREFETCH"  32%1         \n\t"
    202. 

  202.             "movq        %1, %%mm0  \n\t"
    203. 

  203.             "movq       8%1, %%mm2  \n\t"
    204. 

  204.             "movq     %%mm0, %%mm1  \n\t"
    205. 

  205.             "movq     %%mm2, %%mm3  \n\t"
    206. 

  206.             "pand     %%mm4, %%mm0  \n\t"
    207. 

  207.             "pand     %%mm4, %%mm2  \n\t"
    208. 

  208.             "paddw    %%mm1, %%mm0  \n\t"
    209. 

  209.             "paddw    %%mm3, %%mm2  \n\t"
    210. 

  210.             MOVNTQ"   %%mm0,  %0    \n\t"
    211. 

  211.             MOVNTQ"   %%mm2, 8%0"
    212. 

  212.             :"=m"(*d)
    213. 

  213.             :"m"(*s)
    214. 

  214.         );
    215. 

  215.         d+=16;
    216. 

  216.         s+=16;
    217. 

  217.     }
    218. 

  218.     __asm__ volatile(SFENCE:::"memory");
    219. 

  219.     __asm__ volatile(EMMS:::"memory");
    220. 

  220.     mm_end = end - 3;
    221. 

  221.     while (s < mm_end) {
    222. 

  222.         register unsigned x= *((const uint32_t *)s);
    223. 

  223.         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
    224. 

  224.         d+=4;
    225. 

  225.         s+=4;
    226. 

  226.     }
    227. 

  227.     if (s < end) {
    228. 

  228.         register unsigned short x= *((const uint16_t *)s);
    229. 

  229.         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
    230. 

  230.     }
    231. 

  231. }
    232. 

  232. 

  233. static inline void RENAME(rgb16to15)(const uint8_t *src, uint8_t *dst, int src_size)
    234. 

  234. {
    235. 

  235.     register const uint8_t* s=src;
    236. 

  236.     register uint8_t* d=dst;
    237. 

  237.     register const uint8_t *end;
    238. 

  238.     const uint8_t *mm_end;
    239. 

  239.     end = s + src_size;
    240. 

  240.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));
    241. 

  241.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask15rg));
    242. 

  242.     __asm__ volatile("movq        %0, %%mm6"::"m"(mask15b));
    243. 

  243.     mm_end = end - 15;
    244. 

  244.     while (s<mm_end) {
    245. 

  245.         __asm__ volatile(
    246. 

  246.             PREFETCH"  32%1         \n\t"
    247. 

  247.             "movq        %1, %%mm0  \n\t"
    248. 

  248.             "movq       8%1, %%mm2  \n\t"
    249. 

  249.             "movq     %%mm0, %%mm1  \n\t"
    250. 

  250.             "movq     %%mm2, %%mm3  \n\t"
    251. 

  251.             "psrlq       $1, %%mm0  \n\t"
    252. 

  252.             "psrlq       $1, %%mm2  \n\t"
    253. 

  253.             "pand     %%mm7, %%mm0  \n\t"
    254. 

  254.             "pand     %%mm7, %%mm2  \n\t"
    255. 

  255.             "pand     %%mm6, %%mm1  \n\t"
    256. 

  256.             "pand     %%mm6, %%mm3  \n\t"
    257. 

  257.             "por      %%mm1, %%mm0  \n\t"
    258. 

  258.             "por      %%mm3, %%mm2  \n\t"
    259. 

  259.             MOVNTQ"   %%mm0,  %0    \n\t"
    260. 

  260.             MOVNTQ"   %%mm2, 8%0"
    261. 

  261.             :"=m"(*d)
    262. 

  262.             :"m"(*s)
    263. 

  263.         );
    264. 

  264.         d+=16;
    265. 

  265.         s+=16;
    266. 

  266.     }
    267. 

  267.     __asm__ volatile(SFENCE:::"memory");
    268. 

  268.     __asm__ volatile(EMMS:::"memory");
    269. 

  269.     mm_end = end - 3;
    270. 

  270.     while (s < mm_end) {
    271. 

  271.         register uint32_t x= *((const uint32_t*)s);
    272. 

  272.         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
    273. 

  273.         s+=4;
    274. 

  274.         d+=4;
    275. 

  275.     }
    276. 

  276.     if (s < end) {
    277. 

  277.         register uint16_t x= *((const uint16_t*)s);
    278. 

  278.         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
    279. 

  279.     }
    280. 

  280. }
    281. 

  281. 

  282. static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, int src_size)
    283. 

  283. {
    284. 

  284.     const uint8_t *s = src;
    285. 

  285.     const uint8_t *end;
    286. 

  286.     const uint8_t *mm_end;
    287. 

  287.     uint16_t *d = (uint16_t *)dst;
    288. 

  288.     end = s + src_size;
    289. 

  289.     mm_end = end - 15;
    290. 

  290. #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster)
    291. 

  291.     __asm__ volatile(
    292. 

  292.         "movq           %3, %%mm5   \n\t"
    293. 

  293.         "movq           %4, %%mm6   \n\t"
    294. 

  294.         "movq           %5, %%mm7   \n\t"
    295. 

  295.         "jmp 2f                     \n\t"
    296. 

  296.         ".p2align        4          \n\t"
    297. 

  297.         "1:                         \n\t"
    298. 

  298.         PREFETCH"   32(%1)          \n\t"
    299. 

  299.         "movd         (%1), %%mm0   \n\t"
    300. 

  300.         "movd        4(%1), %%mm3   \n\t"
    301. 

  301.         "punpckldq   8(%1), %%mm0   \n\t"
    302. 

  302.         "punpckldq  12(%1), %%mm3   \n\t"
    303. 

  303.         "movq        %%mm0, %%mm1   \n\t"
    304. 

  304.         "movq        %%mm3, %%mm4   \n\t"
    305. 

  305.         "pand        %%mm6, %%mm0   \n\t"
    306. 

  306.         "pand        %%mm6, %%mm3   \n\t"
    307. 

  307.         "pmaddwd     %%mm7, %%mm0   \n\t"
    308. 

  308.         "pmaddwd     %%mm7, %%mm3   \n\t"
    309. 

  309.         "pand        %%mm5, %%mm1   \n\t"
    310. 

  310.         "pand        %%mm5, %%mm4   \n\t"
    311. 

  311.         "por         %%mm1, %%mm0   \n\t"
    312. 

  312.         "por         %%mm4, %%mm3   \n\t"
    313. 

  313.         "psrld          $5, %%mm0   \n\t"
    314. 

  314.         "pslld         $11, %%mm3   \n\t"
    315. 

  315.         "por         %%mm3, %%mm0   \n\t"
    316. 

  316.         MOVNTQ"      %%mm0, (%0)    \n\t"
    317. 

  317.         "add           $16,  %1     \n\t"
    318. 

  318.         "add            $8,  %0     \n\t"
    319. 

  319.         "2:                         \n\t"
    320. 

  320.         "cmp            %2,  %1     \n\t"
    321. 

  321.         " jb            1b          \n\t"
    322. 

  322.         : "+r" (d), "+r"(s)
    323. 

  323.         : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
    324. 

  324.     );
    325. 

  325. #else
    326. 

  326.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    327. 

  327.     __asm__ volatile(
    328. 

  328.         "movq    %0, %%mm7    \n\t"
    329. 

  329.         "movq    %1, %%mm6    \n\t"
    330. 

  330.         ::"m"(red_16mask),"m"(green_16mask));
    331. 

  331.     while (s < mm_end) {
    332. 

  332.         __asm__ volatile(
    333. 

  333.             PREFETCH"    32%1           \n\t"
    334. 

  334.             "movd          %1, %%mm0    \n\t"
    335. 

  335.             "movd         4%1, %%mm3    \n\t"
    336. 

  336.             "punpckldq    8%1, %%mm0    \n\t"
    337. 

  337.             "punpckldq   12%1, %%mm3    \n\t"
    338. 

  338.             "movq       %%mm0, %%mm1    \n\t"
    339. 

  339.             "movq       %%mm0, %%mm2    \n\t"
    340. 

  340.             "movq       %%mm3, %%mm4    \n\t"
    341. 

  341.             "movq       %%mm3, %%mm5    \n\t"
    342. 

  342.             "psrlq         $3, %%mm0    \n\t"
    343. 

  343.             "psrlq         $3, %%mm3    \n\t"
    344. 

  344.             "pand          %2, %%mm0    \n\t"
    345. 

  345.             "pand          %2, %%mm3    \n\t"
    346. 

  346.             "psrlq         $5, %%mm1    \n\t"
    347. 

  347.             "psrlq         $5, %%mm4    \n\t"
    348. 

  348.             "pand       %%mm6, %%mm1    \n\t"
    349. 

  349.             "pand       %%mm6, %%mm4    \n\t"
    350. 

  350.             "psrlq         $8, %%mm2    \n\t"
    351. 

  351.             "psrlq         $8, %%mm5    \n\t"
    352. 

  352.             "pand       %%mm7, %%mm2    \n\t"
    353. 

  353.             "pand       %%mm7, %%mm5    \n\t"
    354. 

  354.             "por        %%mm1, %%mm0    \n\t"
    355. 

  355.             "por        %%mm4, %%mm3    \n\t"
    356. 

  356.             "por        %%mm2, %%mm0    \n\t"
    357. 

  357.             "por        %%mm5, %%mm3    \n\t"
    358. 

  358.             "psllq        $16, %%mm3    \n\t"
    359. 

  359.             "por        %%mm3, %%mm0    \n\t"
    360. 

  360.             MOVNTQ"     %%mm0, %0       \n\t"
    361. 

  361.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    362. 

  362.         d += 4;
    363. 

  363.         s += 16;
    364. 

  364.     }
    365. 

  365. #endif
    366. 

  366.     __asm__ volatile(SFENCE:::"memory");
    367. 

  367.     __asm__ volatile(EMMS:::"memory");
    368. 

  368.     while (s < end) {
    369. 

  369.         register int rgb = *(const uint32_t*)s; s += 4;
    370. 

  370.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
    371. 

  371.     }
    372. 

  372. }
    373. 

  373. 

  374. static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
    375. 

  375. {
    376. 

  376.     const uint8_t *s = src;
    377. 

  377.     const uint8_t *end;
    378. 

  378.     const uint8_t *mm_end;
    379. 

  379.     uint16_t *d = (uint16_t *)dst;
    380. 

  380.     end = s + src_size;
    381. 

  381.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    382. 

  382.     __asm__ volatile(
    383. 

  383.         "movq          %0, %%mm7    \n\t"
    384. 

  384.         "movq          %1, %%mm6    \n\t"
    385. 

  385.         ::"m"(red_16mask),"m"(green_16mask));
    386. 

  386.     mm_end = end - 15;
    387. 

  387.     while (s < mm_end) {
    388. 

  388.         __asm__ volatile(
    389. 

  389.             PREFETCH"    32%1           \n\t"
    390. 

  390.             "movd          %1, %%mm0    \n\t"
    391. 

  391.             "movd         4%1, %%mm3    \n\t"
    392. 

  392.             "punpckldq    8%1, %%mm0    \n\t"
    393. 

  393.             "punpckldq   12%1, %%mm3    \n\t"
    394. 

  394.             "movq       %%mm0, %%mm1    \n\t"
    395. 

  395.             "movq       %%mm0, %%mm2    \n\t"
    396. 

  396.             "movq       %%mm3, %%mm4    \n\t"
    397. 

  397.             "movq       %%mm3, %%mm5    \n\t"
    398. 

  398.             "psllq         $8, %%mm0    \n\t"
    399. 

  399.             "psllq         $8, %%mm3    \n\t"
    400. 

  400.             "pand       %%mm7, %%mm0    \n\t"
    401. 

  401.             "pand       %%mm7, %%mm3    \n\t"
    402. 

  402.             "psrlq         $5, %%mm1    \n\t"
    403. 

  403.             "psrlq         $5, %%mm4    \n\t"
    404. 

  404.             "pand       %%mm6, %%mm1    \n\t"
    405. 

  405.             "pand       %%mm6, %%mm4    \n\t"
    406. 

  406.             "psrlq        $19, %%mm2    \n\t"
    407. 

  407.             "psrlq        $19, %%mm5    \n\t"
    408. 

  408.             "pand          %2, %%mm2    \n\t"
    409. 

  409.             "pand          %2, %%mm5    \n\t"
    410. 

  410.             "por        %%mm1, %%mm0    \n\t"
    411. 

  411.             "por        %%mm4, %%mm3    \n\t"
    412. 

  412.             "por        %%mm2, %%mm0    \n\t"
    413. 

  413.             "por        %%mm5, %%mm3    \n\t"
    414. 

  414.             "psllq        $16, %%mm3    \n\t"
    415. 

  415.             "por        %%mm3, %%mm0    \n\t"
    416. 

  416.             MOVNTQ"     %%mm0, %0       \n\t"
    417. 

  417.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    418. 

  418.         d += 4;
    419. 

  419.         s += 16;
    420. 

  420.     }
    421. 

  421.     __asm__ volatile(SFENCE:::"memory");
    422. 

  422.     __asm__ volatile(EMMS:::"memory");
    423. 

  423.     while (s < end) {
    424. 

  424.         register int rgb = *(const uint32_t*)s; s += 4;
    425. 

  425.         *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
    426. 

  426.     }
    427. 

  427. }
    428. 

  428. 

  429. static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, int src_size)
    430. 

  430. {
    431. 

  431.     const uint8_t *s = src;
    432. 

  432.     const uint8_t *end;
    433. 

  433.     const uint8_t *mm_end;
    434. 

  434.     uint16_t *d = (uint16_t *)dst;
    435. 

  435.     end = s + src_size;
    436. 

  436.     mm_end = end - 15;
    437. 

  437. #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster)
    438. 

  438.     __asm__ volatile(
    439. 

  439.         "movq           %3, %%mm5   \n\t"
    440. 

  440.         "movq           %4, %%mm6   \n\t"
    441. 

  441.         "movq           %5, %%mm7   \n\t"
    442. 

  442.         "jmp            2f          \n\t"
    443. 

  443.         ".p2align        4          \n\t"
    444. 

  444.         "1:                         \n\t"
    445. 

  445.         PREFETCH"   32(%1)          \n\t"
    446. 

  446.         "movd         (%1), %%mm0   \n\t"
    447. 

  447.         "movd        4(%1), %%mm3   \n\t"
    448. 

  448.         "punpckldq   8(%1), %%mm0   \n\t"
    449. 

  449.         "punpckldq  12(%1), %%mm3   \n\t"
    450. 

  450.         "movq        %%mm0, %%mm1   \n\t"
    451. 

  451.         "movq        %%mm3, %%mm4   \n\t"
    452. 

  452.         "pand        %%mm6, %%mm0   \n\t"
    453. 

  453.         "pand        %%mm6, %%mm3   \n\t"
    454. 

  454.         "pmaddwd     %%mm7, %%mm0   \n\t"
    455. 

  455.         "pmaddwd     %%mm7, %%mm3   \n\t"
    456. 

  456.         "pand        %%mm5, %%mm1   \n\t"
    457. 

  457.         "pand        %%mm5, %%mm4   \n\t"
    458. 

  458.         "por         %%mm1, %%mm0   \n\t"
    459. 

  459.         "por         %%mm4, %%mm3   \n\t"
    460. 

  460.         "psrld          $6, %%mm0   \n\t"
    461. 

  461.         "pslld         $10, %%mm3   \n\t"
    462. 

  462.         "por         %%mm3, %%mm0   \n\t"
    463. 

  463.         MOVNTQ"      %%mm0, (%0)    \n\t"
    464. 

  464.         "add           $16,  %1     \n\t"
    465. 

  465.         "add            $8,  %0     \n\t"
    466. 

  466.         "2:                         \n\t"
    467. 

  467.         "cmp            %2,  %1     \n\t"
    468. 

  468.         " jb            1b          \n\t"
    469. 

  469.         : "+r" (d), "+r"(s)
    470. 

  470.         : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
    471. 

  471.     );
    472. 

  472. #else
    473. 

  473.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    474. 

  474.     __asm__ volatile(
    475. 

  475.         "movq          %0, %%mm7    \n\t"
    476. 

  476.         "movq          %1, %%mm6    \n\t"
    477. 

  477.         ::"m"(red_15mask),"m"(green_15mask));
    478. 

  478.     while (s < mm_end) {
    479. 

  479.         __asm__ volatile(
    480. 

  480.             PREFETCH"    32%1           \n\t"
    481. 

  481.             "movd          %1, %%mm0    \n\t"
    482. 

  482.             "movd         4%1, %%mm3    \n\t"
    483. 

  483.             "punpckldq    8%1, %%mm0    \n\t"
    484. 

  484.             "punpckldq   12%1, %%mm3    \n\t"
    485. 

  485.             "movq       %%mm0, %%mm1    \n\t"
    486. 

  486.             "movq       %%mm0, %%mm2    \n\t"
    487. 

  487.             "movq       %%mm3, %%mm4    \n\t"
    488. 

  488.             "movq       %%mm3, %%mm5    \n\t"
    489. 

  489.             "psrlq         $3, %%mm0    \n\t"
    490. 

  490.             "psrlq         $3, %%mm3    \n\t"
    491. 

  491.             "pand          %2, %%mm0    \n\t"
    492. 

  492.             "pand          %2, %%mm3    \n\t"
    493. 

  493.             "psrlq         $6, %%mm1    \n\t"
    494. 

  494.             "psrlq         $6, %%mm4    \n\t"
    495. 

  495.             "pand       %%mm6, %%mm1    \n\t"
    496. 

  496.             "pand       %%mm6, %%mm4    \n\t"
    497. 

  497.             "psrlq         $9, %%mm2    \n\t"
    498. 

  498.             "psrlq         $9, %%mm5    \n\t"
    499. 

  499.             "pand       %%mm7, %%mm2    \n\t"
    500. 

  500.             "pand       %%mm7, %%mm5    \n\t"
    501. 

  501.             "por        %%mm1, %%mm0    \n\t"
    502. 

  502.             "por        %%mm4, %%mm3    \n\t"
    503. 

  503.             "por        %%mm2, %%mm0    \n\t"
    504. 

  504.             "por        %%mm5, %%mm3    \n\t"
    505. 

  505.             "psllq        $16, %%mm3    \n\t"
    506. 

  506.             "por        %%mm3, %%mm0    \n\t"
    507. 

  507.             MOVNTQ"     %%mm0, %0       \n\t"
    508. 

  508.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    509. 

  509.         d += 4;
    510. 

  510.         s += 16;
    511. 

  511.     }
    512. 

  512. #endif
    513. 

  513.     __asm__ volatile(SFENCE:::"memory");
    514. 

  514.     __asm__ volatile(EMMS:::"memory");
    515. 

  515.     while (s < end) {
    516. 

  516.         register int rgb = *(const uint32_t*)s; s += 4;
    517. 

  517.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
    518. 

  518.     }
    519. 

  519. }
    520. 

  520. 

  521. static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
    522. 

  522. {
    523. 

  523.     const uint8_t *s = src;
    524. 

  524.     const uint8_t *end;
    525. 

  525.     const uint8_t *mm_end;
    526. 

  526.     uint16_t *d = (uint16_t *)dst;
    527. 

  527.     end = s + src_size;
    528. 

  528.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    529. 

  529.     __asm__ volatile(
    530. 

  530.         "movq          %0, %%mm7    \n\t"
    531. 

  531.         "movq          %1, %%mm6    \n\t"
    532. 

  532.         ::"m"(red_15mask),"m"(green_15mask));
    533. 

  533.     mm_end = end - 15;
    534. 

  534.     while (s < mm_end) {
    535. 

  535.         __asm__ volatile(
    536. 

  536.             PREFETCH"    32%1           \n\t"
    537. 

  537.             "movd          %1, %%mm0    \n\t"
    538. 

  538.             "movd         4%1, %%mm3    \n\t"
    539. 

  539.             "punpckldq    8%1, %%mm0    \n\t"
    540. 

  540.             "punpckldq   12%1, %%mm3    \n\t"
    541. 

  541.             "movq       %%mm0, %%mm1    \n\t"
    542. 

  542.             "movq       %%mm0, %%mm2    \n\t"
    543. 

  543.             "movq       %%mm3, %%mm4    \n\t"
    544. 

  544.             "movq       %%mm3, %%mm5    \n\t"
    545. 

  545.             "psllq         $7, %%mm0    \n\t"
    546. 

  546.             "psllq         $7, %%mm3    \n\t"
    547. 

  547.             "pand       %%mm7, %%mm0    \n\t"
    548. 

  548.             "pand       %%mm7, %%mm3    \n\t"
    549. 

  549.             "psrlq         $6, %%mm1    \n\t"
    550. 

  550.             "psrlq         $6, %%mm4    \n\t"
    551. 

  551.             "pand       %%mm6, %%mm1    \n\t"
    552. 

  552.             "pand       %%mm6, %%mm4    \n\t"
    553. 

  553.             "psrlq        $19, %%mm2    \n\t"
    554. 

  554.             "psrlq        $19, %%mm5    \n\t"
    555. 

  555.             "pand          %2, %%mm2    \n\t"
    556. 

  556.             "pand          %2, %%mm5    \n\t"
    557. 

  557.             "por        %%mm1, %%mm0    \n\t"
    558. 

  558.             "por        %%mm4, %%mm3    \n\t"
    559. 

  559.             "por        %%mm2, %%mm0    \n\t"
    560. 

  560.             "por        %%mm5, %%mm3    \n\t"
    561. 

  561.             "psllq        $16, %%mm3    \n\t"
    562. 

  562.             "por        %%mm3, %%mm0    \n\t"
    563. 

  563.             MOVNTQ"     %%mm0, %0       \n\t"
    564. 

  564.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    565. 

  565.         d += 4;
    566. 

  566.         s += 16;
    567. 

  567.     }
    568. 

  568.     __asm__ volatile(SFENCE:::"memory");
    569. 

  569.     __asm__ volatile(EMMS:::"memory");
    570. 

  570.     while (s < end) {
    571. 

  571.         register int rgb = *(const uint32_t*)s; s += 4;
    572. 

  572.         *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
    573. 

  573.     }
    574. 

  574. }
    575. 

  575. 

  576. static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
    577. 

  577. {
    578. 

  578.     const uint8_t *s = src;
    579. 

  579.     const uint8_t *end;
    580. 

  580.     const uint8_t *mm_end;
    581. 

  581.     uint16_t *d = (uint16_t *)dst;
    582. 

  582.     end = s + src_size;
    583. 

  583.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    584. 

  584.     __asm__ volatile(
    585. 

  585.         "movq         %0, %%mm7     \n\t"
    586. 

  586.         "movq         %1, %%mm6     \n\t"
    587. 

  587.         ::"m"(red_16mask),"m"(green_16mask));
    588. 

  588.     mm_end = end - 11;
    589. 

  589.     while (s < mm_end) {
    590. 

  590.         __asm__ volatile(
    591. 

  591.             PREFETCH"    32%1           \n\t"
    592. 

  592.             "movd          %1, %%mm0    \n\t"
    593. 

  593.             "movd         3%1, %%mm3    \n\t"
    594. 

  594.             "punpckldq    6%1, %%mm0    \n\t"
    595. 

  595.             "punpckldq    9%1, %%mm3    \n\t"
    596. 

  596.             "movq       %%mm0, %%mm1    \n\t"
    597. 

  597.             "movq       %%mm0, %%mm2    \n\t"
    598. 

  598.             "movq       %%mm3, %%mm4    \n\t"
    599. 

  599.             "movq       %%mm3, %%mm5    \n\t"
    600. 

  600.             "psrlq         $3, %%mm0    \n\t"
    601. 

  601.             "psrlq         $3, %%mm3    \n\t"
    602. 

  602.             "pand          %2, %%mm0    \n\t"
    603. 

  603.             "pand          %2, %%mm3    \n\t"
    604. 

  604.             "psrlq         $5, %%mm1    \n\t"
    605. 

  605.             "psrlq         $5, %%mm4    \n\t"
    606. 

  606.             "pand       %%mm6, %%mm1    \n\t"
    607. 

  607.             "pand       %%mm6, %%mm4    \n\t"
    608. 

  608.             "psrlq         $8, %%mm2    \n\t"
    609. 

  609.             "psrlq         $8, %%mm5    \n\t"
    610. 

  610.             "pand       %%mm7, %%mm2    \n\t"
    611. 

  611.             "pand       %%mm7, %%mm5    \n\t"
    612. 

  612.             "por        %%mm1, %%mm0    \n\t"
    613. 

  613.             "por        %%mm4, %%mm3    \n\t"
    614. 

  614.             "por        %%mm2, %%mm0    \n\t"
    615. 

  615.             "por        %%mm5, %%mm3    \n\t"
    616. 

  616.             "psllq        $16, %%mm3    \n\t"
    617. 

  617.             "por        %%mm3, %%mm0    \n\t"
    618. 

  618.             MOVNTQ"     %%mm0, %0       \n\t"
    619. 

  619.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    620. 

  620.         d += 4;
    621. 

  621.         s += 12;
    622. 

  622.     }
    623. 

  623.     __asm__ volatile(SFENCE:::"memory");
    624. 

  624.     __asm__ volatile(EMMS:::"memory");
    625. 

  625.     while (s < end) {
    626. 

  626.         const int b = *s++;
    627. 

  627.         const int g = *s++;
    628. 

  628.         const int r = *s++;
    629. 

  629.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
    630. 

  630.     }
    631. 

  631. }
    632. 

  632. 

  633. static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, int src_size)
    634. 

  634. {
    635. 

  635.     const uint8_t *s = src;
    636. 

  636.     const uint8_t *end;
    637. 

  637.     const uint8_t *mm_end;
    638. 

  638.     uint16_t *d = (uint16_t *)dst;
    639. 

  639.     end = s + src_size;
    640. 

  640.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    641. 

  641.     __asm__ volatile(
    642. 

  642.         "movq         %0, %%mm7     \n\t"
    643. 

  643.         "movq         %1, %%mm6     \n\t"
    644. 

  644.         ::"m"(red_16mask),"m"(green_16mask));
    645. 

  645.     mm_end = end - 15;
    646. 

  646.     while (s < mm_end) {
    647. 

  647.         __asm__ volatile(
    648. 

  648.             PREFETCH"    32%1           \n\t"
    649. 

  649.             "movd          %1, %%mm0    \n\t"
    650. 

  650.             "movd         3%1, %%mm3    \n\t"
    651. 

  651.             "punpckldq    6%1, %%mm0    \n\t"
    652. 

  652.             "punpckldq    9%1, %%mm3    \n\t"
    653. 

  653.             "movq       %%mm0, %%mm1    \n\t"
    654. 

  654.             "movq       %%mm0, %%mm2    \n\t"
    655. 

  655.             "movq       %%mm3, %%mm4    \n\t"
    656. 

  656.             "movq       %%mm3, %%mm5    \n\t"
    657. 

  657.             "psllq         $8, %%mm0    \n\t"
    658. 

  658.             "psllq         $8, %%mm3    \n\t"
    659. 

  659.             "pand       %%mm7, %%mm0    \n\t"
    660. 

  660.             "pand       %%mm7, %%mm3    \n\t"
    661. 

  661.             "psrlq         $5, %%mm1    \n\t"
    662. 

  662.             "psrlq         $5, %%mm4    \n\t"
    663. 

  663.             "pand       %%mm6, %%mm1    \n\t"
    664. 

  664.             "pand       %%mm6, %%mm4    \n\t"
    665. 

  665.             "psrlq        $19, %%mm2    \n\t"
    666. 

  666.             "psrlq        $19, %%mm5    \n\t"
    667. 

  667.             "pand          %2, %%mm2    \n\t"
    668. 

  668.             "pand          %2, %%mm5    \n\t"
    669. 

  669.             "por        %%mm1, %%mm0    \n\t"
    670. 

  670.             "por        %%mm4, %%mm3    \n\t"
    671. 

  671.             "por        %%mm2, %%mm0    \n\t"
    672. 

  672.             "por        %%mm5, %%mm3    \n\t"
    673. 

  673.             "psllq        $16, %%mm3    \n\t"
    674. 

  674.             "por        %%mm3, %%mm0    \n\t"
    675. 

  675.             MOVNTQ"     %%mm0, %0       \n\t"
    676. 

  676.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    677. 

  677.         d += 4;
    678. 

  678.         s += 12;
    679. 

  679.     }
    680. 

  680.     __asm__ volatile(SFENCE:::"memory");
    681. 

  681.     __asm__ volatile(EMMS:::"memory");
    682. 

  682.     while (s < end) {
    683. 

  683.         const int r = *s++;
    684. 

  684.         const int g = *s++;
    685. 

  685.         const int b = *s++;
    686. 

  686.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
    687. 

  687.     }
    688. 

  688. }
    689. 

  689. 

  690. static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
    691. 

  691. {
    692. 

  692.     const uint8_t *s = src;
    693. 

  693.     const uint8_t *end;
    694. 

  694.     const uint8_t *mm_end;
    695. 

  695.     uint16_t *d = (uint16_t *)dst;
    696. 

  696.     end = s + src_size;
    697. 

  697.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    698. 

  698.     __asm__ volatile(
    699. 

  699.         "movq          %0, %%mm7    \n\t"
    700. 

  700.         "movq          %1, %%mm6    \n\t"
    701. 

  701.         ::"m"(red_15mask),"m"(green_15mask));
    702. 

  702.     mm_end = end - 11;
    703. 

  703.     while (s < mm_end) {
    704. 

  704.         __asm__ volatile(
    705. 

  705.             PREFETCH"    32%1           \n\t"
    706. 

  706.             "movd          %1, %%mm0    \n\t"
    707. 

  707.             "movd         3%1, %%mm3    \n\t"
    708. 

  708.             "punpckldq    6%1, %%mm0    \n\t"
    709. 

  709.             "punpckldq    9%1, %%mm3    \n\t"
    710. 

  710.             "movq       %%mm0, %%mm1    \n\t"
    711. 

  711.             "movq       %%mm0, %%mm2    \n\t"
    712. 

  712.             "movq       %%mm3, %%mm4    \n\t"
    713. 

  713.             "movq       %%mm3, %%mm5    \n\t"
    714. 

  714.             "psrlq         $3, %%mm0    \n\t"
    715. 

  715.             "psrlq         $3, %%mm3    \n\t"
    716. 

  716.             "pand          %2, %%mm0    \n\t"
    717. 

  717.             "pand          %2, %%mm3    \n\t"
    718. 

  718.             "psrlq         $6, %%mm1    \n\t"
    719. 

  719.             "psrlq         $6, %%mm4    \n\t"
    720. 

  720.             "pand       %%mm6, %%mm1    \n\t"
    721. 

  721.             "pand       %%mm6, %%mm4    \n\t"
    722. 

  722.             "psrlq         $9, %%mm2    \n\t"
    723. 

  723.             "psrlq         $9, %%mm5    \n\t"
    724. 

  724.             "pand       %%mm7, %%mm2    \n\t"
    725. 

  725.             "pand       %%mm7, %%mm5    \n\t"
    726. 

  726.             "por        %%mm1, %%mm0    \n\t"
    727. 

  727.             "por        %%mm4, %%mm3    \n\t"
    728. 

  728.             "por        %%mm2, %%mm0    \n\t"
    729. 

  729.             "por        %%mm5, %%mm3    \n\t"
    730. 

  730.             "psllq        $16, %%mm3    \n\t"
    731. 

  731.             "por        %%mm3, %%mm0    \n\t"
    732. 

  732.             MOVNTQ"     %%mm0, %0       \n\t"
    733. 

  733.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    734. 

  734.         d += 4;
    735. 

  735.         s += 12;
    736. 

  736.     }
    737. 

  737.     __asm__ volatile(SFENCE:::"memory");
    738. 

  738.     __asm__ volatile(EMMS:::"memory");
    739. 

  739.     while (s < end) {
    740. 

  740.         const int b = *s++;
    741. 

  741.         const int g = *s++;
    742. 

  742.         const int r = *s++;
    743. 

  743.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
    744. 

  744.     }
    745. 

  745. }
    746. 

  746. 

  747. static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, int src_size)
    748. 

  748. {
    749. 

  749.     const uint8_t *s = src;
    750. 

  750.     const uint8_t *end;
    751. 

  751.     const uint8_t *mm_end;
    752. 

  752.     uint16_t *d = (uint16_t *)dst;
    753. 

  753.     end = s + src_size;
    754. 

  754.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    755. 

  755.     __asm__ volatile(
    756. 

  756.         "movq         %0, %%mm7     \n\t"
    757. 

  757.         "movq         %1, %%mm6     \n\t"
    758. 

  758.         ::"m"(red_15mask),"m"(green_15mask));
    759. 

  759.     mm_end = end - 15;
    760. 

  760.     while (s < mm_end) {
    761. 

  761.         __asm__ volatile(
    762. 

  762.             PREFETCH"   32%1            \n\t"
    763. 

  763.             "movd         %1, %%mm0     \n\t"
    764. 

  764.             "movd        3%1, %%mm3     \n\t"
    765. 

  765.             "punpckldq   6%1, %%mm0     \n\t"
    766. 

  766.             "punpckldq   9%1, %%mm3     \n\t"
    767. 

  767.             "movq      %%mm0, %%mm1     \n\t"
    768. 

  768.             "movq      %%mm0, %%mm2     \n\t"
    769. 

  769.             "movq      %%mm3, %%mm4     \n\t"
    770. 

  770.             "movq      %%mm3, %%mm5     \n\t"
    771. 

  771.             "psllq        $7, %%mm0     \n\t"
    772. 

  772.             "psllq        $7, %%mm3     \n\t"
    773. 

  773.             "pand      %%mm7, %%mm0     \n\t"
    774. 

  774.             "pand      %%mm7, %%mm3     \n\t"
    775. 

  775.             "psrlq        $6, %%mm1     \n\t"
    776. 

  776.             "psrlq        $6, %%mm4     \n\t"
    777. 

  777.             "pand      %%mm6, %%mm1     \n\t"
    778. 

  778.             "pand      %%mm6, %%mm4     \n\t"
    779. 

  779.             "psrlq       $19, %%mm2     \n\t"
    780. 

  780.             "psrlq       $19, %%mm5     \n\t"
    781. 

  781.             "pand         %2, %%mm2     \n\t"
    782. 

  782.             "pand         %2, %%mm5     \n\t"
    783. 

  783.             "por       %%mm1, %%mm0     \n\t"
    784. 

  784.             "por       %%mm4, %%mm3     \n\t"
    785. 

  785.             "por       %%mm2, %%mm0     \n\t"
    786. 

  786.             "por       %%mm5, %%mm3     \n\t"
    787. 

  787.             "psllq       $16, %%mm3     \n\t"
    788. 

  788.             "por       %%mm3, %%mm0     \n\t"
    789. 

  789.             MOVNTQ"    %%mm0, %0        \n\t"
    790. 

  790.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    791. 

  791.         d += 4;
    792. 

  792.         s += 12;
    793. 

  793.     }
    794. 

  794.     __asm__ volatile(SFENCE:::"memory");
    795. 

  795.     __asm__ volatile(EMMS:::"memory");
    796. 

  796.     while (s < end) {
    797. 

  797.         const int r = *s++;
    798. 

  798.         const int g = *s++;
    799. 

  799.         const int b = *s++;
    800. 

  800.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
    801. 

  801.     }
    802. 

  802. }
    803. 

  803. 

  804. static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    805. 

  805. {
    806. 

  806.     const uint16_t *end;
    807. 

  807.     const uint16_t *mm_end;
    808. 

  808.     uint8_t *d = dst;
    809. 

  809.     const uint16_t *s = (const uint16_t*)src;
    810. 

  810.     end = s + src_size/2;
    811. 

  811.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    812. 

  812.     mm_end = end - 7;
    813. 

  813.     while (s < mm_end) {
    814. 

  814.         __asm__ volatile(
    815. 

  815.             PREFETCH"    32%1           \n\t"
    816. 

  816.             "movq          %1, %%mm0    \n\t"
    817. 

  817.             "movq          %1, %%mm1    \n\t"
    818. 

  818.             "movq          %1, %%mm2    \n\t"
    819. 

  819.             "pand          %2, %%mm0    \n\t"
    820. 

  820.             "pand          %3, %%mm1    \n\t"
    821. 

  821.             "pand          %4, %%mm2    \n\t"
    822. 

  822.             "psllq         $5, %%mm0    \n\t"
    823. 

  823.             "pmulhw        %6, %%mm0    \n\t"
    824. 

  824.             "pmulhw        %6, %%mm1    \n\t"
    825. 

  825.             "pmulhw        %7, %%mm2    \n\t"
    826. 

  826.             "movq       %%mm0, %%mm3    \n\t"
    827. 

  827.             "movq       %%mm1, %%mm4    \n\t"
    828. 

  828.             "movq       %%mm2, %%mm5    \n\t"
    829. 

  829.             "punpcklwd     %5, %%mm0    \n\t"
    830. 

  830.             "punpcklwd     %5, %%mm1    \n\t"
    831. 

  831.             "punpcklwd     %5, %%mm2    \n\t"
    832. 

  832.             "punpckhwd     %5, %%mm3    \n\t"
    833. 

  833.             "punpckhwd     %5, %%mm4    \n\t"
    834. 

  834.             "punpckhwd     %5, %%mm5    \n\t"
    835. 

  835.             "psllq         $8, %%mm1    \n\t"
    836. 

  836.             "psllq        $16, %%mm2    \n\t"
    837. 

  837.             "por        %%mm1, %%mm0    \n\t"
    838. 

  838.             "por        %%mm2, %%mm0    \n\t"
    839. 

  839.             "psllq         $8, %%mm4    \n\t"
    840. 

  840.             "psllq        $16, %%mm5    \n\t"
    841. 

  841.             "por        %%mm4, %%mm3    \n\t"
    842. 

  842.             "por        %%mm5, %%mm3    \n\t"
    843. 

  843. 

  844.             "movq       %%mm0, %%mm6    \n\t"
    845. 

  845.             "movq       %%mm3, %%mm7    \n\t"
    846. 

  846. 

  847.             "movq         8%1, %%mm0    \n\t"
    848. 

  848.             "movq         8%1, %%mm1    \n\t"
    849. 

  849.             "movq         8%1, %%mm2    \n\t"
    850. 

  850.             "pand          %2, %%mm0    \n\t"
    851. 

  851.             "pand          %3, %%mm1    \n\t"
    852. 

  852.             "pand          %4, %%mm2    \n\t"
    853. 

  853.             "psllq         $5, %%mm0    \n\t"
    854. 

  854.             "pmulhw        %6, %%mm0    \n\t"
    855. 

  855.             "pmulhw        %6, %%mm1    \n\t"
    856. 

  856.             "pmulhw        %7, %%mm2    \n\t"
    857. 

  857.             "movq       %%mm0, %%mm3    \n\t"
    858. 

  858.             "movq       %%mm1, %%mm4    \n\t"
    859. 

  859.             "movq       %%mm2, %%mm5    \n\t"
    860. 

  860.             "punpcklwd     %5, %%mm0    \n\t"
    861. 

  861.             "punpcklwd     %5, %%mm1    \n\t"
    862. 

  862.             "punpcklwd     %5, %%mm2    \n\t"
    863. 

  863.             "punpckhwd     %5, %%mm3    \n\t"
    864. 

  864.             "punpckhwd     %5, %%mm4    \n\t"
    865. 

  865.             "punpckhwd     %5, %%mm5    \n\t"
    866. 

  866.             "psllq         $8, %%mm1    \n\t"
    867. 

  867.             "psllq        $16, %%mm2    \n\t"
    868. 

  868.             "por        %%mm1, %%mm0    \n\t"
    869. 

  869.             "por        %%mm2, %%mm0    \n\t"
    870. 

  870.             "psllq         $8, %%mm4    \n\t"
    871. 

  871.             "psllq        $16, %%mm5    \n\t"
    872. 

  872.             "por        %%mm4, %%mm3    \n\t"
    873. 

  873.             "por        %%mm5, %%mm3    \n\t"
    874. 

  874. 

  875.             :"=m"(*d)
    876. 

  876.             :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r),"m"(mmx_null),"m"(mul15_mid),"m"(mul15_hi)
    877. 

  877.             :"memory");
    878. 

  878.         /* borrowed 32 to 24 */
    879. 

  879.         __asm__ volatile(
    880. 

  880.             "movq       %%mm0, %%mm4    \n\t"
    881. 

  881.             "movq       %%mm3, %%mm5    \n\t"
    882. 

  882.             "movq       %%mm6, %%mm0    \n\t"
    883. 

  883.             "movq       %%mm7, %%mm1    \n\t"
    884. 

  884. 

  885.             "movq       %%mm4, %%mm6    \n\t"
    886. 

  886.             "movq       %%mm5, %%mm7    \n\t"
    887. 

  887.             "movq       %%mm0, %%mm2    \n\t"
    888. 

  888.             "movq       %%mm1, %%mm3    \n\t"
    889. 

  889. 

  890.             STORE_BGR24_MMX
    891. 

  891. 

  892.             :"=m"(*d)
    893. 

  893.             :"m"(*s)
    894. 

  894.             :"memory");
    895. 

  895.         d += 24;
    896. 

  896.         s += 8;
    897. 

  897.     }
    898. 

  898.     __asm__ volatile(SFENCE:::"memory");
    899. 

  899.     __asm__ volatile(EMMS:::"memory");
    900. 

  900.     while (s < end) {
    901. 

  901.         register uint16_t bgr;
    902. 

  902.         bgr = *s++;
    903. 

  903.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    904. 

  904.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
    905. 

  905.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
    906. 

  906.     }
    907. 

  907. }
    908. 

  908. 

  909. static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    910. 

  910. {
    911. 

  911.     const uint16_t *end;
    912. 

  912.     const uint16_t *mm_end;
    913. 

  913.     uint8_t *d = (uint8_t *)dst;
    914. 

  914.     const uint16_t *s = (const uint16_t *)src;
    915. 

  915.     end = s + src_size/2;
    916. 

  916.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    917. 

  917.     mm_end = end - 7;
    918. 

  918.     while (s < mm_end) {
    919. 

  919.         __asm__ volatile(
    920. 

  920.             PREFETCH"    32%1           \n\t"
    921. 

  921.             "movq          %1, %%mm0    \n\t"
    922. 

  922.             "movq          %1, %%mm1    \n\t"
    923. 

  923.             "movq          %1, %%mm2    \n\t"
    924. 

  924.             "pand          %2, %%mm0    \n\t"
    925. 

  925.             "pand          %3, %%mm1    \n\t"
    926. 

  926.             "pand          %4, %%mm2    \n\t"
    927. 

  927.             "psllq         $5, %%mm0    \n\t"
    928. 

  928.             "psrlq         $1, %%mm2    \n\t"
    929. 

  929.             "pmulhw        %6, %%mm0    \n\t"
    930. 

  930.             "pmulhw        %8, %%mm1    \n\t"
    931. 

  931.             "pmulhw        %7, %%mm2    \n\t"
    932. 

  932.             "movq       %%mm0, %%mm3    \n\t"
    933. 

  933.             "movq       %%mm1, %%mm4    \n\t"
    934. 

  934.             "movq       %%mm2, %%mm5    \n\t"
    935. 

  935.             "punpcklwd     %5, %%mm0    \n\t"
    936. 

  936.             "punpcklwd     %5, %%mm1    \n\t"
    937. 

  937.             "punpcklwd     %5, %%mm2    \n\t"
    938. 

  938.             "punpckhwd     %5, %%mm3    \n\t"
    939. 

  939.             "punpckhwd     %5, %%mm4    \n\t"
    940. 

  940.             "punpckhwd     %5, %%mm5    \n\t"
    941. 

  941.             "psllq         $8, %%mm1    \n\t"
    942. 

  942.             "psllq        $16, %%mm2    \n\t"
    943. 

  943.             "por        %%mm1, %%mm0    \n\t"
    944. 

  944.             "por        %%mm2, %%mm0    \n\t"
    945. 

  945.             "psllq         $8, %%mm4    \n\t"
    946. 

  946.             "psllq        $16, %%mm5    \n\t"
    947. 

  947.             "por        %%mm4, %%mm3    \n\t"
    948. 

  948.             "por        %%mm5, %%mm3    \n\t"
    949. 

  949. 

  950.             "movq       %%mm0, %%mm6    \n\t"
    951. 

  951.             "movq       %%mm3, %%mm7    \n\t"
    952. 

  952. 

  953.             "movq         8%1, %%mm0    \n\t"
    954. 

  954.             "movq         8%1, %%mm1    \n\t"
    955. 

  955.             "movq         8%1, %%mm2    \n\t"
    956. 

  956.             "pand          %2, %%mm0    \n\t"
    957. 

  957.             "pand          %3, %%mm1    \n\t"
    958. 

  958.             "pand          %4, %%mm2    \n\t"
    959. 

  959.             "psllq         $5, %%mm0    \n\t"
    960. 

  960.             "psrlq         $1, %%mm2    \n\t"
    961. 

  961.             "pmulhw        %6, %%mm0    \n\t"
    962. 

  962.             "pmulhw        %8, %%mm1    \n\t"
    963. 

  963.             "pmulhw        %7, %%mm2    \n\t"
    964. 

  964.             "movq       %%mm0, %%mm3    \n\t"
    965. 

  965.             "movq       %%mm1, %%mm4    \n\t"
    966. 

  966.             "movq       %%mm2, %%mm5    \n\t"
    967. 

  967.             "punpcklwd     %5, %%mm0    \n\t"
    968. 

  968.             "punpcklwd     %5, %%mm1    \n\t"
    969. 

  969.             "punpcklwd     %5, %%mm2    \n\t"
    970. 

  970.             "punpckhwd     %5, %%mm3    \n\t"
    971. 

  971.             "punpckhwd     %5, %%mm4    \n\t"
    972. 

  972.             "punpckhwd     %5, %%mm5    \n\t"
    973. 

  973.             "psllq         $8, %%mm1    \n\t"
    974. 

  974.             "psllq        $16, %%mm2    \n\t"
    975. 

  975.             "por        %%mm1, %%mm0    \n\t"
    976. 

  976.             "por        %%mm2, %%mm0    \n\t"
    977. 

  977.             "psllq         $8, %%mm4    \n\t"
    978. 

  978.             "psllq        $16, %%mm5    \n\t"
    979. 

  979.             "por        %%mm4, %%mm3    \n\t"
    980. 

  980.             "por        %%mm5, %%mm3    \n\t"
    981. 

  981.             :"=m"(*d)
    982. 

  982.             :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null),"m"(mul15_mid),"m"(mul15_hi),"m"(mul16_mid)
    983. 

  983.             :"memory");
    984. 

  984.         /* borrowed 32 to 24 */
    985. 

  985.         __asm__ volatile(
    986. 

  986.             "movq       %%mm0, %%mm4    \n\t"
    987. 

  987.             "movq       %%mm3, %%mm5    \n\t"
    988. 

  988.             "movq       %%mm6, %%mm0    \n\t"
    989. 

  989.             "movq       %%mm7, %%mm1    \n\t"
    990. 

  990. 

  991.             "movq       %%mm4, %%mm6    \n\t"
    992. 

  992.             "movq       %%mm5, %%mm7    \n\t"
    993. 

  993.             "movq       %%mm0, %%mm2    \n\t"
    994. 

  994.             "movq       %%mm1, %%mm3    \n\t"
    995. 

  995. 

  996.             STORE_BGR24_MMX
    997. 

  997. 

  998.             :"=m"(*d)
    999. 

  999.             :"m"(*s)
    1000. 

  1000.             :"memory");
    1001. 

  1001.         d += 24;
    1002. 

  1002.         s += 8;
    1003. 

  1003.     }
    1004. 

  1004.     __asm__ volatile(SFENCE:::"memory");
    1005. 

  1005.     __asm__ volatile(EMMS:::"memory");
    1006. 

  1006.     while (s < end) {
    1007. 

  1007.         register uint16_t bgr;
    1008. 

  1008.         bgr = *s++;
    1009. 

  1009.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1010. 

  1010.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
    1011. 

  1011.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
    1012. 

  1012.     }
    1013. 

  1013. }
    1014. 

  1014. 

  1015. /*
    1016. 

  1016.  * mm0 = 00 B3 00 B2 00 B1 00 B0
    1017. 

  1017.  * mm1 = 00 G3 00 G2 00 G1 00 G0
    1018. 

  1018.  * mm2 = 00 R3 00 R2 00 R1 00 R0
    1019. 

  1019.  * mm6 = FF FF FF FF FF FF FF FF
    1020. 

  1020.  * mm7 = 00 00 00 00 00 00 00 00
    1021. 

  1021.  */
    1022. 

  1022. #define PACK_RGB32 \
    1023. 

  1023.     "packuswb   %%mm7, %%mm0    \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \
    1024. 

  1024.     "packuswb   %%mm7, %%mm1    \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \
    1025. 

  1025.     "packuswb   %%mm7, %%mm2    \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \
    1026. 

  1026.     "punpcklbw  %%mm1, %%mm0    \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \
    1027. 

  1027.     "punpcklbw  %%mm6, %%mm2    \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \
    1028. 

  1028.     "movq       %%mm0, %%mm3    \n\t"                               \
    1029. 

  1029.     "punpcklwd  %%mm2, %%mm0    \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \
    1030. 

  1030.     "punpckhwd  %%mm2, %%mm3    \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \
    1031. 

  1031.     MOVNTQ"     %%mm0,  %0      \n\t"                               \
    1032. 

  1032.     MOVNTQ"     %%mm3, 8%0      \n\t"                               \
    1033. 

  1033. 

  1034. static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, int src_size)
    1035. 

  1035. {
    1036. 

  1036.     const uint16_t *end;
    1037. 

  1037.     const uint16_t *mm_end;
    1038. 

  1038.     uint8_t *d = dst;
    1039. 

  1039.     const uint16_t *s = (const uint16_t *)src;
    1040. 

  1040.     end = s + src_size/2;
    1041. 

  1041.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    1042. 

  1042.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");
    1043. 

  1043.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");
    1044. 

  1044.     mm_end = end - 3;
    1045. 

  1045.     while (s < mm_end) {
    1046. 

  1046.         __asm__ volatile(
    1047. 

  1047.             PREFETCH"    32%1           \n\t"
    1048. 

  1048.             "movq          %1, %%mm0    \n\t"
    1049. 

  1049.             "movq          %1, %%mm1    \n\t"
    1050. 

  1050.             "movq          %1, %%mm2    \n\t"
    1051. 

  1051.             "pand          %2, %%mm0    \n\t"
    1052. 

  1052.             "pand          %3, %%mm1    \n\t"
    1053. 

  1053.             "pand          %4, %%mm2    \n\t"
    1054. 

  1054.             "psllq         $5, %%mm0    \n\t"
    1055. 

  1055.             "pmulhw        %5, %%mm0    \n\t"
    1056. 

  1056.             "pmulhw        %5, %%mm1    \n\t"
    1057. 

  1057.             "pmulhw        %6, %%mm2    \n\t"
    1058. 

  1058.             PACK_RGB32
    1059. 

  1059.             :"=m"(*d)
    1060. 

  1060.             :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r),"m"(mul15_mid),"m"(mul15_hi)
    1061. 

  1061.             :"memory");
    1062. 

  1062.         d += 16;
    1063. 

  1063.         s += 4;
    1064. 

  1064.     }
    1065. 

  1065.     __asm__ volatile(SFENCE:::"memory");
    1066. 

  1066.     __asm__ volatile(EMMS:::"memory");
    1067. 

  1067.     while (s < end) {
    1068. 

  1068.         register uint16_t bgr;
    1069. 

  1069.         bgr = *s++;
    1070. 

  1070.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1071. 

  1071.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
    1072. 

  1072.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
    1073. 

  1073.         *d++ = 255;
    1074. 

  1074.     }
    1075. 

  1075. }
    1076. 

  1076. 

  1077. static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, int src_size)
    1078. 

  1078. {
    1079. 

  1079.     const uint16_t *end;
    1080. 

  1080.     const uint16_t *mm_end;
    1081. 

  1081.     uint8_t *d = dst;
    1082. 

  1082.     const uint16_t *s = (const uint16_t*)src;
    1083. 

  1083.     end = s + src_size/2;
    1084. 

  1084.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    1085. 

  1085.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");
    1086. 

  1086.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");
    1087. 

  1087.     mm_end = end - 3;
    1088. 

  1088.     while (s < mm_end) {
    1089. 

  1089.         __asm__ volatile(
    1090. 

  1090.             PREFETCH"    32%1           \n\t"
    1091. 

  1091.             "movq          %1, %%mm0    \n\t"
    1092. 

  1092.             "movq          %1, %%mm1    \n\t"
    1093. 

  1093.             "movq          %1, %%mm2    \n\t"
    1094. 

  1094.             "pand          %2, %%mm0    \n\t"
    1095. 

  1095.             "pand          %3, %%mm1    \n\t"
    1096. 

  1096.             "pand          %4, %%mm2    \n\t"
    1097. 

  1097.             "psllq         $5, %%mm0    \n\t"
    1098. 

  1098.             "psrlq         $1, %%mm2    \n\t"
    1099. 

  1099.             "pmulhw        %5, %%mm0    \n\t"
    1100. 

  1100.             "pmulhw        %7, %%mm1    \n\t"
    1101. 

  1101.             "pmulhw        %6, %%mm2    \n\t"
    1102. 

  1102.             PACK_RGB32
    1103. 

  1103.             :"=m"(*d)
    1104. 

  1104.             :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mul15_mid),"m"(mul15_hi),"m"(mul16_mid)
    1105. 

  1105.             :"memory");
    1106. 

  1106.         d += 16;
    1107. 

  1107.         s += 4;
    1108. 

  1108.     }
    1109. 

  1109.     __asm__ volatile(SFENCE:::"memory");
    1110. 

  1110.     __asm__ volatile(EMMS:::"memory");
    1111. 

  1111.     while (s < end) {
    1112. 

  1112.         register uint16_t bgr;
    1113. 

  1113.         bgr = *s++;
    1114. 

  1114.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1115. 

  1115.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
    1116. 

  1116.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
    1117. 

  1117.         *d++ = 255;
    1118. 

  1118.     }
    1119. 

  1119. }
    1120. 

  1120. 

  1121. static inline void RENAME(shuffle_bytes_2103)(const uint8_t *src, uint8_t *dst, int src_size)
    1122. 

  1122. {
    1123. 

  1123.     x86_reg idx = 15 - src_size;
    1124. 

  1124.     const uint8_t *s = src-idx;
    1125. 

  1125.     uint8_t *d = dst-idx;
    1126. 

  1126.     __asm__ volatile(
    1127. 

  1127.         "test          %0, %0           \n\t"
    1128. 

  1128.         "jns           2f               \n\t"
    1129. 

  1129.         PREFETCH"       (%1, %0)        \n\t"
    1130. 

  1130.         "movq          %3, %%mm7        \n\t"
    1131. 

  1131.         "pxor          %4, %%mm7        \n\t"
    1132. 

  1132.         "movq       %%mm7, %%mm6        \n\t"
    1133. 

  1133.         "pxor          %5, %%mm7        \n\t"
    1134. 

  1134.         ".p2align       4               \n\t"
    1135. 

  1135.         "1:                             \n\t"
    1136. 

  1136.         PREFETCH"     32(%1, %0)        \n\t"
    1137. 

  1137.         "movq           (%1, %0), %%mm0 \n\t"
    1138. 

  1138.         "movq          8(%1, %0), %%mm1 \n\t"
    1139. 

  1139. # if COMPILE_TEMPLATE_MMX2
    1140. 

  1140.         "pshufw      $177, %%mm0, %%mm3 \n\t"
    1141. 

  1141.         "pshufw      $177, %%mm1, %%mm5 \n\t"
    1142. 

  1142.         "pand       %%mm7, %%mm0        \n\t"
    1143. 

  1143.         "pand       %%mm6, %%mm3        \n\t"
    1144. 

  1144.         "pand       %%mm7, %%mm1        \n\t"
    1145. 

  1145.         "pand       %%mm6, %%mm5        \n\t"
    1146. 

  1146.         "por        %%mm3, %%mm0        \n\t"
    1147. 

  1147.         "por        %%mm5, %%mm1        \n\t"
    1148. 

  1148. # else
    1149. 

  1149.         "movq       %%mm0, %%mm2        \n\t"
    1150. 

  1150.         "movq       %%mm1, %%mm4        \n\t"
    1151. 

  1151.         "pand       %%mm7, %%mm0        \n\t"
    1152. 

  1152.         "pand       %%mm6, %%mm2        \n\t"
    1153. 

  1153.         "pand       %%mm7, %%mm1        \n\t"
    1154. 

  1154.         "pand       %%mm6, %%mm4        \n\t"
    1155. 

  1155.         "movq       %%mm2, %%mm3        \n\t"
    1156. 

  1156.         "movq       %%mm4, %%mm5        \n\t"
    1157. 

  1157.         "pslld        $16, %%mm2        \n\t"
    1158. 

  1158.         "psrld        $16, %%mm3        \n\t"
    1159. 

  1159.         "pslld        $16, %%mm4        \n\t"
    1160. 

  1160.         "psrld        $16, %%mm5        \n\t"
    1161. 

  1161.         "por        %%mm2, %%mm0        \n\t"
    1162. 

  1162.         "por        %%mm4, %%mm1        \n\t"
    1163. 

  1163.         "por        %%mm3, %%mm0        \n\t"
    1164. 

  1164.         "por        %%mm5, %%mm1        \n\t"
    1165. 

  1165. # endif
    1166. 

  1166.         MOVNTQ"     %%mm0,  (%2, %0)    \n\t"
    1167. 

  1167.         MOVNTQ"     %%mm1, 8(%2, %0)    \n\t"
    1168. 

  1168.         "add          $16, %0           \n\t"
    1169. 

  1169.         "js            1b               \n\t"
    1170. 

  1170.         SFENCE"                         \n\t"
    1171. 

  1171.         EMMS"                           \n\t"
    1172. 

  1172.         "2:                             \n\t"
    1173. 

  1173.         : "+&r"(idx)
    1174. 

  1174.         : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
    1175. 

  1175.         : "memory");
    1176. 

  1176.     for (; idx<15; idx+=4) {
    1177. 

  1177.         register int v = *(const uint32_t *)&s[idx], g = v & 0xff00ff00;
    1178. 

  1178.         v &= 0xff00ff;
    1179. 

  1179.         *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
    1180. 

  1180.     }
    1181. 

  1181. }
    1182. 

  1182. 

  1183. static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    1184. 

  1184. {
    1185. 

  1185.     unsigned i;
    1186. 

  1186.     x86_reg mmx_size= 23 - src_size;
    1187. 

  1187.     __asm__ volatile (
    1188. 

  1188.         "test             %%"REG_a", %%"REG_a"          \n\t"
    1189. 

  1189.         "jns                     2f                     \n\t"
    1190. 

  1190.         "movq     "MANGLE(mask24r)", %%mm5              \n\t"
    1191. 

  1191.         "movq     "MANGLE(mask24g)", %%mm6              \n\t"
    1192. 

  1192.         "movq     "MANGLE(mask24b)", %%mm7              \n\t"
    1193. 

  1193.         ".p2align                 4                     \n\t"
    1194. 

  1194.         "1:                                             \n\t"
    1195. 

  1195.         PREFETCH" 32(%1, %%"REG_a")                     \n\t"
    1196. 

  1196.         "movq       (%1, %%"REG_a"), %%mm0              \n\t" // BGR BGR BG
    1197. 

  1197.         "movq       (%1, %%"REG_a"), %%mm1              \n\t" // BGR BGR BG
    1198. 

  1198.         "movq      2(%1, %%"REG_a"), %%mm2              \n\t" // R BGR BGR B
    1199. 

  1199.         "psllq                  $16, %%mm0              \n\t" // 00 BGR BGR
    1200. 

  1200.         "pand                 %%mm5, %%mm0              \n\t"
    1201. 

  1201.         "pand                 %%mm6, %%mm1              \n\t"
    1202. 

  1202.         "pand                 %%mm7, %%mm2              \n\t"
    1203. 

  1203.         "por                  %%mm0, %%mm1              \n\t"
    1204. 

  1204.         "por                  %%mm2, %%mm1              \n\t"
    1205. 

  1205.         "movq      6(%1, %%"REG_a"), %%mm0              \n\t" // BGR BGR BG
    1206. 

  1206.         MOVNTQ"               %%mm1,   (%2, %%"REG_a")  \n\t" // RGB RGB RG
    1207. 

  1207.         "movq      8(%1, %%"REG_a"), %%mm1              \n\t" // R BGR BGR B
    1208. 

  1208.         "movq     10(%1, %%"REG_a"), %%mm2              \n\t" // GR BGR BGR
    1209. 

  1209.         "pand                 %%mm7, %%mm0              \n\t"
    1210. 

  1210.         "pand                 %%mm5, %%mm1              \n\t"
    1211. 

  1211.         "pand                 %%mm6, %%mm2              \n\t"
    1212. 

  1212.         "por                  %%mm0, %%mm1              \n\t"
    1213. 

  1213.         "por                  %%mm2, %%mm1              \n\t"
    1214. 

  1214.         "movq     14(%1, %%"REG_a"), %%mm0              \n\t" // R BGR BGR B
    1215. 

  1215.         MOVNTQ"               %%mm1,  8(%2, %%"REG_a")  \n\t" // B RGB RGB R
    1216. 

  1216.         "movq     16(%1, %%"REG_a"), %%mm1              \n\t" // GR BGR BGR
    1217. 

  1217.         "movq     18(%1, %%"REG_a"), %%mm2              \n\t" // BGR BGR BG
    1218. 

  1218.         "pand                 %%mm6, %%mm0              \n\t"
    1219. 

  1219.         "pand                 %%mm7, %%mm1              \n\t"
    1220. 

  1220.         "pand                 %%mm5, %%mm2              \n\t"
    1221. 

  1221.         "por                  %%mm0, %%mm1              \n\t"
    1222. 

  1222.         "por                  %%mm2, %%mm1              \n\t"
    1223. 

  1223.         MOVNTQ"               %%mm1, 16(%2, %%"REG_a")  \n\t"
    1224. 

  1224.         "add                    $24, %%"REG_a"          \n\t"
    1225. 

  1225.         " js                     1b                     \n\t"
    1226. 

  1226.         "2:                                             \n\t"
    1227. 

  1227.         : "+a" (mmx_size)
    1228. 

  1228.         : "r" (src-mmx_size), "r"(dst-mmx_size)
    1229. 

  1229.     );
    1230. 

  1230. 

  1231.     __asm__ volatile(SFENCE:::"memory");
    1232. 

  1232.     __asm__ volatile(EMMS:::"memory");
    1233. 

  1233. 

  1234.     if (mmx_size==23) return; //finished, was multiple of 8
    1235. 

  1235. 

  1236.     src+= src_size;
    1237. 

  1237.     dst+= src_size;
    1238. 

  1238.     src_size= 23-mmx_size;
    1239. 

  1239.     src-= src_size;
    1240. 

  1240.     dst-= src_size;
    1241. 

  1241.     for (i=0; i<src_size; i+=3) {
    1242. 

  1242.         register uint8_t x;
    1243. 

  1243.         x          = src[i + 2];
    1244. 

  1244.         dst[i + 1] = src[i + 1];
    1245. 

  1245.         dst[i + 2] = src[i + 0];
    1246. 

  1246.         dst[i + 0] = x;
    1247. 

  1247.     }
    1248. 

  1248. }
    1249. 

  1249. 

  1250. static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1251. 

  1251.                                            int width, int height,
    1252. 

  1252.                                            int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
    1253. 

  1253. {
    1254. 

  1254.     int y;
    1255. 

  1255.     const x86_reg chromWidth= width>>1;
    1256. 

  1256.     for (y=0; y<height; y++) {
    1257. 

  1257.         //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
    1258. 

  1258.         __asm__ volatile(
    1259. 

  1259.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1260. 

  1260.             ".p2align                    4              \n\t"
    1261. 

  1261.             "1:                                         \n\t"
    1262. 

  1262.             PREFETCH"    32(%1, %%"REG_a", 2)           \n\t"
    1263. 

  1263.             PREFETCH"    32(%2, %%"REG_a")              \n\t"
    1264. 

  1264.             PREFETCH"    32(%3, %%"REG_a")              \n\t"
    1265. 

  1265.             "movq          (%2, %%"REG_a"), %%mm0       \n\t" // U(0)
    1266. 

  1266.             "movq                    %%mm0, %%mm2       \n\t" // U(0)
    1267. 

  1267.             "movq          (%3, %%"REG_a"), %%mm1       \n\t" // V(0)
    1268. 

  1268.             "punpcklbw               %%mm1, %%mm0       \n\t" // UVUV UVUV(0)
    1269. 

  1269.             "punpckhbw               %%mm1, %%mm2       \n\t" // UVUV UVUV(8)
    1270. 

  1270. 

  1271.             "movq        (%1, %%"REG_a",2), %%mm3       \n\t" // Y(0)
    1272. 

  1272.             "movq       8(%1, %%"REG_a",2), %%mm5       \n\t" // Y(8)
    1273. 

  1273.             "movq                    %%mm3, %%mm4       \n\t" // Y(0)
    1274. 

  1274.             "movq                    %%mm5, %%mm6       \n\t" // Y(8)
    1275. 

  1275.             "punpcklbw               %%mm0, %%mm3       \n\t" // YUYV YUYV(0)
    1276. 

  1276.             "punpckhbw               %%mm0, %%mm4       \n\t" // YUYV YUYV(4)
    1277. 

  1277.             "punpcklbw               %%mm2, %%mm5       \n\t" // YUYV YUYV(8)
    1278. 

  1278.             "punpckhbw               %%mm2, %%mm6       \n\t" // YUYV YUYV(12)
    1279. 

  1279. 

  1280.             MOVNTQ"                  %%mm3,   (%0, %%"REG_a", 4)    \n\t"
    1281. 

  1281.             MOVNTQ"                  %%mm4,  8(%0, %%"REG_a", 4)    \n\t"
    1282. 

  1282.             MOVNTQ"                  %%mm5, 16(%0, %%"REG_a", 4)    \n\t"
    1283. 

  1283.             MOVNTQ"                  %%mm6, 24(%0, %%"REG_a", 4)    \n\t"
    1284. 

  1284. 

  1285.             "add                        $8, %%"REG_a"   \n\t"
    1286. 

  1286.             "cmp                        %4, %%"REG_a"   \n\t"
    1287. 

  1287.             " jb                        1b              \n\t"
    1288. 

  1288.             ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
    1289. 

  1289.             : "%"REG_a
    1290. 

  1290.         );
    1291. 

  1291.         if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
    1292. 

  1292.             usrc += chromStride;
    1293. 

  1293.             vsrc += chromStride;
    1294. 

  1294.         }
    1295. 

  1295.         ysrc += lumStride;
    1296. 

  1296.         dst  += dstStride;
    1297. 

  1297.     }
    1298. 

  1298.     __asm__(EMMS"       \n\t"
    1299. 

  1299.             SFENCE"     \n\t"
    1300. 

  1300.             :::"memory");
    1301. 

  1301. }
    1302. 

  1302. 

  1303. /**
    1304. 

  1304.  * Height should be a multiple of 2 and width should be a multiple of 16.
    1305. 

  1305.  * (If this is a problem for anyone then tell me, and I will fix it.)
    1306. 

  1306.  */
    1307. 

  1307. static inline void RENAME(yv12toyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1308. 

  1308.                                       int width, int height,
    1309. 

  1309.                                       int lumStride, int chromStride, int dstStride)
    1310. 

  1310. {
    1311. 

  1311.     //FIXME interpolate chroma
    1312. 

  1312.     RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
    1313. 

  1313. }
    1314. 

  1314. 

  1315. static inline void RENAME(yuvPlanartouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1316. 

  1316.                                            int width, int height,
    1317. 

  1317.                                            int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
    1318. 

  1318. {
    1319. 

  1319.     int y;
    1320. 

  1320.     const x86_reg chromWidth= width>>1;
    1321. 

  1321.     for (y=0; y<height; y++) {
    1322. 

  1322.         //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
    1323. 

  1323.         __asm__ volatile(
    1324. 

  1324.             "xor                %%"REG_a", %%"REG_a"    \n\t"
    1325. 

  1325.             ".p2align                   4               \n\t"
    1326. 

  1326.             "1:                                         \n\t"
    1327. 

  1327.             PREFETCH"   32(%1, %%"REG_a", 2)            \n\t"
    1328. 

  1328.             PREFETCH"   32(%2, %%"REG_a")               \n\t"
    1329. 

  1329.             PREFETCH"   32(%3, %%"REG_a")               \n\t"
    1330. 

  1330.             "movq         (%2, %%"REG_a"), %%mm0        \n\t" // U(0)
    1331. 

  1331.             "movq                   %%mm0, %%mm2        \n\t" // U(0)
    1332. 

  1332.             "movq         (%3, %%"REG_a"), %%mm1        \n\t" // V(0)
    1333. 

  1333.             "punpcklbw              %%mm1, %%mm0        \n\t" // UVUV UVUV(0)
    1334. 

  1334.             "punpckhbw              %%mm1, %%mm2        \n\t" // UVUV UVUV(8)
    1335. 

  1335. 

  1336.             "movq       (%1, %%"REG_a",2), %%mm3        \n\t" // Y(0)
    1337. 

  1337.             "movq      8(%1, %%"REG_a",2), %%mm5        \n\t" // Y(8)
    1338. 

  1338.             "movq                   %%mm0, %%mm4        \n\t" // Y(0)
    1339. 

  1339.             "movq                   %%mm2, %%mm6        \n\t" // Y(8)
    1340. 

  1340.             "punpcklbw              %%mm3, %%mm0        \n\t" // YUYV YUYV(0)
    1341. 

  1341.             "punpckhbw              %%mm3, %%mm4        \n\t" // YUYV YUYV(4)
    1342. 

  1342.             "punpcklbw              %%mm5, %%mm2        \n\t" // YUYV YUYV(8)
    1343. 

  1343.             "punpckhbw              %%mm5, %%mm6        \n\t" // YUYV YUYV(12)
    1344. 

  1344. 

  1345.             MOVNTQ"                 %%mm0,   (%0, %%"REG_a", 4)     \n\t"
    1346. 

  1346.             MOVNTQ"                 %%mm4,  8(%0, %%"REG_a", 4)     \n\t"
    1347. 

  1347.             MOVNTQ"                 %%mm2, 16(%0, %%"REG_a", 4)     \n\t"
    1348. 

  1348.             MOVNTQ"                 %%mm6, 24(%0, %%"REG_a", 4)     \n\t"
    1349. 

  1349. 

  1350.             "add                       $8, %%"REG_a"    \n\t"
    1351. 

  1351.             "cmp                       %4, %%"REG_a"    \n\t"
    1352. 

  1352.             " jb                       1b               \n\t"
    1353. 

  1353.             ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
    1354. 

  1354.             : "%"REG_a
    1355. 

  1355.         );
    1356. 

  1356.         if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
    1357. 

  1357.             usrc += chromStride;
    1358. 

  1358.             vsrc += chromStride;
    1359. 

  1359.         }
    1360. 

  1360.         ysrc += lumStride;
    1361. 

  1361.         dst += dstStride;
    1362. 

  1362.     }
    1363. 

  1363.     __asm__(EMMS"       \n\t"
    1364. 

  1364.             SFENCE"     \n\t"
    1365. 

  1365.             :::"memory");
    1366. 

  1366. }
    1367. 

  1367. 

  1368. /**
    1369. 

  1369.  * Height should be a multiple of 2 and width should be a multiple of 16
    1370. 

  1370.  * (If this is a problem for anyone then tell me, and I will fix it.)
    1371. 

  1371.  */
    1372. 

  1372. static inline void RENAME(yv12touyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1373. 

  1373.                                       int width, int height,
    1374. 

  1374.                                       int lumStride, int chromStride, int dstStride)
    1375. 

  1375. {
    1376. 

  1376.     //FIXME interpolate chroma
    1377. 

  1377.     RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 2);
    1378. 

  1378. }
    1379. 

  1379. 

  1380. /**
    1381. 

  1381.  * Width should be a multiple of 16.
    1382. 

  1382.  */
    1383. 

  1383. static inline void RENAME(yuv422ptouyvy)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1384. 

  1384.                                          int width, int height,
    1385. 

  1385.                                          int lumStride, int chromStride, int dstStride)
    1386. 

  1386. {
    1387. 

  1387.     RENAME(yuvPlanartouyvy)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
    1388. 

  1388. }
    1389. 

  1389. 

  1390. /**
    1391. 

  1391.  * Width should be a multiple of 16.
    1392. 

  1392.  */
    1393. 

  1393. static inline void RENAME(yuv422ptoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1394. 

  1394.                                          int width, int height,
    1395. 

  1395.                                          int lumStride, int chromStride, int dstStride)
    1396. 

  1396. {
    1397. 

  1397.     RENAME(yuvPlanartoyuy2)(ysrc, usrc, vsrc, dst, width, height, lumStride, chromStride, dstStride, 1);
    1398. 

  1398. }
    1399. 

  1399. 

  1400. /**
    1401. 

  1401.  * Height should be a multiple of 2 and width should be a multiple of 16.
    1402. 

  1402.  * (If this is a problem for anyone then tell me, and I will fix it.)
    1403. 

  1403.  */
    1404. 

  1404. static inline void RENAME(yuy2toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
    1405. 

  1405.                                       int width, int height,
    1406. 

  1406.                                       int lumStride, int chromStride, int srcStride)
    1407. 

  1407. {
    1408. 

  1408.     int y;
    1409. 

  1409.     const x86_reg chromWidth= width>>1;
    1410. 

  1410.     for (y=0; y<height; y+=2) {
    1411. 

  1411.         __asm__ volatile(
    1412. 

  1412.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1413. 

  1413.             "pcmpeqw                 %%mm7, %%mm7       \n\t"
    1414. 

  1414.             "psrlw                      $8, %%mm7       \n\t" // FF,00,FF,00...
    1415. 

  1415.             ".p2align                    4              \n\t"
    1416. 

  1416.             "1:                \n\t"
    1417. 

  1417.             PREFETCH" 64(%0, %%"REG_a", 4)              \n\t"
    1418. 

  1418.             "movq       (%0, %%"REG_a", 4), %%mm0       \n\t" // YUYV YUYV(0)
    1419. 

  1419.             "movq      8(%0, %%"REG_a", 4), %%mm1       \n\t" // YUYV YUYV(4)
    1420. 

  1420.             "movq                    %%mm0, %%mm2       \n\t" // YUYV YUYV(0)
    1421. 

  1421.             "movq                    %%mm1, %%mm3       \n\t" // YUYV YUYV(4)
    1422. 

  1422.             "psrlw                      $8, %%mm0       \n\t" // U0V0 U0V0(0)
    1423. 

  1423.             "psrlw                      $8, %%mm1       \n\t" // U0V0 U0V0(4)
    1424. 

  1424.             "pand                    %%mm7, %%mm2       \n\t" // Y0Y0 Y0Y0(0)
    1425. 

  1425.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(4)
    1426. 

  1426.             "packuswb                %%mm1, %%mm0       \n\t" // UVUV UVUV(0)
    1427. 

  1427.             "packuswb                %%mm3, %%mm2       \n\t" // YYYY YYYY(0)
    1428. 

  1428. 

  1429.             MOVNTQ"                  %%mm2, (%1, %%"REG_a", 2)  \n\t"
    1430. 

  1430. 

  1431.             "movq     16(%0, %%"REG_a", 4), %%mm1       \n\t" // YUYV YUYV(8)
    1432. 

  1432.             "movq     24(%0, %%"REG_a", 4), %%mm2       \n\t" // YUYV YUYV(12)
    1433. 

  1433.             "movq                    %%mm1, %%mm3       \n\t" // YUYV YUYV(8)
    1434. 

  1434.             "movq                    %%mm2, %%mm4       \n\t" // YUYV YUYV(12)
    1435. 

  1435.             "psrlw                      $8, %%mm1       \n\t" // U0V0 U0V0(8)
    1436. 

  1436.             "psrlw                      $8, %%mm2       \n\t" // U0V0 U0V0(12)
    1437. 

  1437.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(8)
    1438. 

  1438.             "pand                    %%mm7, %%mm4       \n\t" // Y0Y0 Y0Y0(12)
    1439. 

  1439.             "packuswb                %%mm2, %%mm1       \n\t" // UVUV UVUV(8)
    1440. 

  1440.             "packuswb                %%mm4, %%mm3       \n\t" // YYYY YYYY(8)
    1441. 

  1441. 

  1442.             MOVNTQ"                  %%mm3, 8(%1, %%"REG_a", 2) \n\t"
    1443. 

  1443. 

  1444.             "movq                    %%mm0, %%mm2       \n\t" // UVUV UVUV(0)
    1445. 

  1445.             "movq                    %%mm1, %%mm3       \n\t" // UVUV UVUV(8)
    1446. 

  1446.             "psrlw                      $8, %%mm0       \n\t" // V0V0 V0V0(0)
    1447. 

  1447.             "psrlw                      $8, %%mm1       \n\t" // V0V0 V0V0(8)
    1448. 

  1448.             "pand                    %%mm7, %%mm2       \n\t" // U0U0 U0U0(0)
    1449. 

  1449.             "pand                    %%mm7, %%mm3       \n\t" // U0U0 U0U0(8)
    1450. 

  1450.             "packuswb                %%mm1, %%mm0       \n\t" // VVVV VVVV(0)
    1451. 

  1451.             "packuswb                %%mm3, %%mm2       \n\t" // UUUU UUUU(0)
    1452. 

  1452. 

  1453.             MOVNTQ"                  %%mm0, (%3, %%"REG_a")     \n\t"
    1454. 

  1454.             MOVNTQ"                  %%mm2, (%2, %%"REG_a")     \n\t"
    1455. 

  1455. 

  1456.             "add                        $8, %%"REG_a"   \n\t"
    1457. 

  1457.             "cmp                        %4, %%"REG_a"   \n\t"
    1458. 

  1458.             " jb                        1b              \n\t"
    1459. 

  1459.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
    1460. 

  1460.             : "memory", "%"REG_a
    1461. 

  1461.         );
    1462. 

  1462. 

  1463.         ydst += lumStride;
    1464. 

  1464.         src  += srcStride;
    1465. 

  1465. 

  1466.         __asm__ volatile(
    1467. 

  1467.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1468. 

  1468.             ".p2align                    4              \n\t"
    1469. 

  1469.             "1:                                         \n\t"
    1470. 

  1470.             PREFETCH" 64(%0, %%"REG_a", 4)              \n\t"
    1471. 

  1471.             "movq       (%0, %%"REG_a", 4), %%mm0       \n\t" // YUYV YUYV(0)
    1472. 

  1472.             "movq      8(%0, %%"REG_a", 4), %%mm1       \n\t" // YUYV YUYV(4)
    1473. 

  1473.             "movq     16(%0, %%"REG_a", 4), %%mm2       \n\t" // YUYV YUYV(8)
    1474. 

  1474.             "movq     24(%0, %%"REG_a", 4), %%mm3       \n\t" // YUYV YUYV(12)
    1475. 

  1475.             "pand                    %%mm7, %%mm0       \n\t" // Y0Y0 Y0Y0(0)
    1476. 

  1476.             "pand                    %%mm7, %%mm1       \n\t" // Y0Y0 Y0Y0(4)
    1477. 

  1477.             "pand                    %%mm7, %%mm2       \n\t" // Y0Y0 Y0Y0(8)
    1478. 

  1478.             "pand                    %%mm7, %%mm3       \n\t" // Y0Y0 Y0Y0(12)
    1479. 

  1479.             "packuswb                %%mm1, %%mm0       \n\t" // YYYY YYYY(0)
    1480. 

  1480.             "packuswb                %%mm3, %%mm2       \n\t" // YYYY YYYY(8)
    1481. 

  1481. 

  1482.             MOVNTQ"                  %%mm0,  (%1, %%"REG_a", 2) \n\t"
    1483. 

  1483.             MOVNTQ"                  %%mm2, 8(%1, %%"REG_a", 2) \n\t"
    1484. 

  1484. 

  1485.             "add                        $8, %%"REG_a"   \n\t"
    1486. 

  1486.             "cmp                        %4, %%"REG_a"   \n\t"
    1487. 

  1487.             " jb                        1b              \n\t"
    1488. 

  1488. 

  1489.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
    1490. 

  1490.             : "memory", "%"REG_a
    1491. 

  1491.         );
    1492. 

  1492.         udst += chromStride;
    1493. 

  1493.         vdst += chromStride;
    1494. 

  1494.         ydst += lumStride;
    1495. 

  1495.         src  += srcStride;
    1496. 

  1496.     }
    1497. 

  1497.     __asm__ volatile(EMMS"       \n\t"
    1498. 

  1498.                      SFENCE"     \n\t"
    1499. 

  1499.                      :::"memory");
    1500. 

  1500. }
    1501. 

  1501. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    1502. 

  1502. 

  1503. #if COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW
    1504. 

  1504. static inline void RENAME(planar2x)(const uint8_t *src, uint8_t *dst, int srcWidth, int srcHeight, int srcStride, int dstStride)
    1505. 

  1505. {
    1506. 

  1506.     int x,y;
    1507. 

  1507. 

  1508.     dst[0]= src[0];
    1509. 

  1509. 

  1510.     // first line
    1511. 

  1511.     for (x=0; x<srcWidth-1; x++) {
    1512. 

  1512.         dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
    1513. 

  1513.         dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
    1514. 

  1514.     }
    1515. 

  1515.     dst[2*srcWidth-1]= src[srcWidth-1];
    1516. 

  1516. 

  1517.     dst+= dstStride;
    1518. 

  1518. 

  1519.     for (y=1; y<srcHeight; y++) {
    1520. 

  1520.         const x86_reg mmxSize= srcWidth&~15;
    1521. 

  1521.         __asm__ volatile(
    1522. 

  1522.             "mov           %4, %%"REG_a"            \n\t"
    1523. 

  1523.             "movq        "MANGLE(mmx_ff)", %%mm0    \n\t"
    1524. 

  1524.             "movq         (%0, %%"REG_a"), %%mm4    \n\t"
    1525. 

  1525.             "movq                   %%mm4, %%mm2    \n\t"
    1526. 

  1526.             "psllq                     $8, %%mm4    \n\t"
    1527. 

  1527.             "pand                   %%mm0, %%mm2    \n\t"
    1528. 

  1528.             "por                    %%mm2, %%mm4    \n\t"
    1529. 

  1529.             "movq         (%1, %%"REG_a"), %%mm5    \n\t"
    1530. 

  1530.             "movq                   %%mm5, %%mm3    \n\t"
    1531. 

  1531.             "psllq                     $8, %%mm5    \n\t"
    1532. 

  1532.             "pand                   %%mm0, %%mm3    \n\t"
    1533. 

  1533.             "por                    %%mm3, %%mm5    \n\t"
    1534. 

  1534.             "1:                                     \n\t"
    1535. 

  1535.             "movq         (%0, %%"REG_a"), %%mm0    \n\t"
    1536. 

  1536.             "movq         (%1, %%"REG_a"), %%mm1    \n\t"
    1537. 

  1537.             "movq        1(%0, %%"REG_a"), %%mm2    \n\t"
    1538. 

  1538.             "movq        1(%1, %%"REG_a"), %%mm3    \n\t"
    1539. 

  1539.             PAVGB"                  %%mm0, %%mm5    \n\t"
    1540. 

  1540.             PAVGB"                  %%mm0, %%mm3    \n\t"
    1541. 

  1541.             PAVGB"                  %%mm0, %%mm5    \n\t"
    1542. 

  1542.             PAVGB"                  %%mm0, %%mm3    \n\t"
    1543. 

  1543.             PAVGB"                  %%mm1, %%mm4    \n\t"
    1544. 

  1544.             PAVGB"                  %%mm1, %%mm2    \n\t"
    1545. 

  1545.             PAVGB"                  %%mm1, %%mm4    \n\t"
    1546. 

  1546.             PAVGB"                  %%mm1, %%mm2    \n\t"
    1547. 

  1547.             "movq                   %%mm5, %%mm7    \n\t"
    1548. 

  1548.             "movq                   %%mm4, %%mm6    \n\t"
    1549. 

  1549.             "punpcklbw              %%mm3, %%mm5    \n\t"
    1550. 

  1550.             "punpckhbw              %%mm3, %%mm7    \n\t"
    1551. 

  1551.             "punpcklbw              %%mm2, %%mm4    \n\t"
    1552. 

  1552.             "punpckhbw              %%mm2, %%mm6    \n\t"
    1553. 

  1553.             MOVNTQ"                 %%mm5,  (%2, %%"REG_a", 2)  \n\t"
    1554. 

  1554.             MOVNTQ"                 %%mm7, 8(%2, %%"REG_a", 2)  \n\t"
    1555. 

  1555.             MOVNTQ"                 %%mm4,  (%3, %%"REG_a", 2)  \n\t"
    1556. 

  1556.             MOVNTQ"                 %%mm6, 8(%3, %%"REG_a", 2)  \n\t"
    1557. 

  1557.             "add                       $8, %%"REG_a"            \n\t"
    1558. 

  1558.             "movq       -1(%0, %%"REG_a"), %%mm4    \n\t"
    1559. 

  1559.             "movq       -1(%1, %%"REG_a"), %%mm5    \n\t"
    1560. 

  1560.             " js                       1b                       \n\t"
    1561. 

  1561.             :: "r" (src + mmxSize  ), "r" (src + srcStride + mmxSize  ),
    1562. 

  1562.                "r" (dst + mmxSize*2), "r" (dst + dstStride + mmxSize*2),
    1563. 

  1563.                "g" (-mmxSize)
    1564. 

  1564.             : "%"REG_a
    1565. 

  1565.         );
    1566. 

  1566. 

  1567.         for (x=mmxSize-1; x<srcWidth-1; x++) {
    1568. 

  1568.             dst[2*x          +1]= (3*src[x+0] +   src[x+srcStride+1])>>2;
    1569. 

  1569.             dst[2*x+dstStride+2]= (  src[x+0] + 3*src[x+srcStride+1])>>2;
    1570. 

  1570.             dst[2*x+dstStride+1]= (  src[x+1] + 3*src[x+srcStride  ])>>2;
    1571. 

  1571.             dst[2*x          +2]= (3*src[x+1] +   src[x+srcStride  ])>>2;
    1572. 

  1572.         }
    1573. 

  1573.         dst[srcWidth*2 -1            ]= (3*src[srcWidth-1] +   src[srcWidth-1 + srcStride])>>2;
    1574. 

  1574.         dst[srcWidth*2 -1 + dstStride]= (  src[srcWidth-1] + 3*src[srcWidth-1 + srcStride])>>2;
    1575. 

  1575. 

  1576.         dst+=dstStride*2;
    1577. 

  1577.         src+=srcStride;
    1578. 

  1578.     }
    1579. 

  1579. 

  1580.     // last line
    1581. 

  1581.     dst[0]= src[0];
    1582. 

  1582. 

  1583.     for (x=0; x<srcWidth-1; x++) {
    1584. 

  1584.         dst[2*x+1]= (3*src[x] +   src[x+1])>>2;
    1585. 

  1585.         dst[2*x+2]= (  src[x] + 3*src[x+1])>>2;
    1586. 

  1586.     }
    1587. 

  1587.     dst[2*srcWidth-1]= src[srcWidth-1];
    1588. 

  1588. 

  1589.     __asm__ volatile(EMMS"       \n\t"
    1590. 

  1590.                      SFENCE"     \n\t"
    1591. 

  1591.                      :::"memory");
    1592. 

  1592. }
    1593. 

  1593. #endif /* COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW */
    1594. 

  1594. 

  1595. #if !COMPILE_TEMPLATE_AMD3DNOW
    1596. 

  1596. /**
    1597. 

  1597.  * Height should be a multiple of 2 and width should be a multiple of 16.
    1598. 

  1598.  * (If this is a problem for anyone then tell me, and I will fix it.)
    1599. 

  1599.  * Chrominance data is only taken from every second line, others are ignored.
    1600. 

  1600.  * FIXME: Write HQ version.
    1601. 

  1601.  */
    1602. 

  1602. static inline void RENAME(uyvytoyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
    1603. 

  1603.                                       int width, int height,
    1604. 

  1604.                                       int lumStride, int chromStride, int srcStride)
    1605. 

  1605. {
    1606. 

  1606.     int y;
    1607. 

  1607.     const x86_reg chromWidth= width>>1;
    1608. 

  1608.     for (y=0; y<height; y+=2) {
    1609. 

  1609.         __asm__ volatile(
    1610. 

  1610.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1611. 

  1611.             "pcmpeqw             %%mm7, %%mm7   \n\t"
    1612. 

  1612.             "psrlw                  $8, %%mm7   \n\t" // FF,00,FF,00...
    1613. 

  1613.             ".p2align                4          \n\t"
    1614. 

  1614.             "1:                                 \n\t"
    1615. 

  1615.             PREFETCH" 64(%0, %%"REG_a", 4)          \n\t"
    1616. 

  1616.             "movq       (%0, %%"REG_a", 4), %%mm0   \n\t" // UYVY UYVY(0)
    1617. 

  1617.             "movq      8(%0, %%"REG_a", 4), %%mm1   \n\t" // UYVY UYVY(4)
    1618. 

  1618.             "movq                %%mm0, %%mm2   \n\t" // UYVY UYVY(0)
    1619. 

  1619.             "movq                %%mm1, %%mm3   \n\t" // UYVY UYVY(4)
    1620. 

  1620.             "pand                %%mm7, %%mm0   \n\t" // U0V0 U0V0(0)
    1621. 

  1621.             "pand                %%mm7, %%mm1   \n\t" // U0V0 U0V0(4)
    1622. 

  1622.             "psrlw                  $8, %%mm2   \n\t" // Y0Y0 Y0Y0(0)
    1623. 

  1623.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(4)
    1624. 

  1624.             "packuswb            %%mm1, %%mm0   \n\t" // UVUV UVUV(0)
    1625. 

  1625.             "packuswb            %%mm3, %%mm2   \n\t" // YYYY YYYY(0)
    1626. 

  1626. 

  1627.             MOVNTQ"              %%mm2,  (%1, %%"REG_a", 2) \n\t"
    1628. 

  1628. 

  1629.             "movq     16(%0, %%"REG_a", 4), %%mm1   \n\t" // UYVY UYVY(8)
    1630. 

  1630.             "movq     24(%0, %%"REG_a", 4), %%mm2   \n\t" // UYVY UYVY(12)
    1631. 

  1631.             "movq                %%mm1, %%mm3   \n\t" // UYVY UYVY(8)
    1632. 

  1632.             "movq                %%mm2, %%mm4   \n\t" // UYVY UYVY(12)
    1633. 

  1633.             "pand                %%mm7, %%mm1   \n\t" // U0V0 U0V0(8)
    1634. 

  1634.             "pand                %%mm7, %%mm2   \n\t" // U0V0 U0V0(12)
    1635. 

  1635.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(8)
    1636. 

  1636.             "psrlw                  $8, %%mm4   \n\t" // Y0Y0 Y0Y0(12)
    1637. 

  1637.             "packuswb            %%mm2, %%mm1   \n\t" // UVUV UVUV(8)
    1638. 

  1638.             "packuswb            %%mm4, %%mm3   \n\t" // YYYY YYYY(8)
    1639. 

  1639. 

  1640.             MOVNTQ"              %%mm3, 8(%1, %%"REG_a", 2) \n\t"
    1641. 

  1641. 

  1642.             "movq                %%mm0, %%mm2   \n\t" // UVUV UVUV(0)
    1643. 

  1643.             "movq                %%mm1, %%mm3   \n\t" // UVUV UVUV(8)
    1644. 

  1644.             "psrlw                  $8, %%mm0   \n\t" // V0V0 V0V0(0)
    1645. 

  1645.             "psrlw                  $8, %%mm1   \n\t" // V0V0 V0V0(8)
    1646. 

  1646.             "pand                %%mm7, %%mm2   \n\t" // U0U0 U0U0(0)
    1647. 

  1647.             "pand                %%mm7, %%mm3   \n\t" // U0U0 U0U0(8)
    1648. 

  1648.             "packuswb            %%mm1, %%mm0   \n\t" // VVVV VVVV(0)
    1649. 

  1649.             "packuswb            %%mm3, %%mm2   \n\t" // UUUU UUUU(0)
    1650. 

  1650. 

  1651.             MOVNTQ"              %%mm0, (%3, %%"REG_a") \n\t"
    1652. 

  1652.             MOVNTQ"              %%mm2, (%2, %%"REG_a") \n\t"
    1653. 

  1653. 

  1654.             "add                    $8, %%"REG_a"   \n\t"
    1655. 

  1655.             "cmp                    %4, %%"REG_a"   \n\t"
    1656. 

  1656.             " jb                    1b          \n\t"
    1657. 

  1657.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
    1658. 

  1658.             : "memory", "%"REG_a
    1659. 

  1659.         );
    1660. 

  1660. 

  1661.         ydst += lumStride;
    1662. 

  1662.         src  += srcStride;
    1663. 

  1663. 

  1664.         __asm__ volatile(
    1665. 

  1665.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1666. 

  1666.             ".p2align                    4              \n\t"
    1667. 

  1667.             "1:                                 \n\t"
    1668. 

  1668.             PREFETCH" 64(%0, %%"REG_a", 4)          \n\t"
    1669. 

  1669.             "movq       (%0, %%"REG_a", 4), %%mm0   \n\t" // YUYV YUYV(0)
    1670. 

  1670.             "movq      8(%0, %%"REG_a", 4), %%mm1   \n\t" // YUYV YUYV(4)
    1671. 

  1671.             "movq     16(%0, %%"REG_a", 4), %%mm2   \n\t" // YUYV YUYV(8)
    1672. 

  1672.             "movq     24(%0, %%"REG_a", 4), %%mm3   \n\t" // YUYV YUYV(12)
    1673. 

  1673.             "psrlw                  $8, %%mm0   \n\t" // Y0Y0 Y0Y0(0)
    1674. 

  1674.             "psrlw                  $8, %%mm1   \n\t" // Y0Y0 Y0Y0(4)
    1675. 

  1675.             "psrlw                  $8, %%mm2   \n\t" // Y0Y0 Y0Y0(8)
    1676. 

  1676.             "psrlw                  $8, %%mm3   \n\t" // Y0Y0 Y0Y0(12)
    1677. 

  1677.             "packuswb            %%mm1, %%mm0   \n\t" // YYYY YYYY(0)
    1678. 

  1678.             "packuswb            %%mm3, %%mm2   \n\t" // YYYY YYYY(8)
    1679. 

  1679. 

  1680.             MOVNTQ"              %%mm0,  (%1, %%"REG_a", 2) \n\t"
    1681. 

  1681.             MOVNTQ"              %%mm2, 8(%1, %%"REG_a", 2) \n\t"
    1682. 

  1682. 

  1683.             "add                    $8, %%"REG_a"   \n\t"
    1684. 

  1684.             "cmp                    %4, %%"REG_a"   \n\t"
    1685. 

  1685.             " jb                    1b          \n\t"
    1686. 

  1686. 

  1687.             ::"r"(src), "r"(ydst), "r"(udst), "r"(vdst), "g" (chromWidth)
    1688. 

  1688.             : "memory", "%"REG_a
    1689. 

  1689.         );
    1690. 

  1690.         udst += chromStride;
    1691. 

  1691.         vdst += chromStride;
    1692. 

  1692.         ydst += lumStride;
    1693. 

  1693.         src  += srcStride;
    1694. 

  1694.     }
    1695. 

  1695.     __asm__ volatile(EMMS"       \n\t"
    1696. 

  1696.                      SFENCE"     \n\t"
    1697. 

  1697.                      :::"memory");
    1698. 

  1698. }
    1699. 

  1699. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    1700. 

  1700. 

  1701. /**
    1702. 

  1702.  * Height should be a multiple of 2 and width should be a multiple of 2.
    1703. 

  1703.  * (If this is a problem for anyone then tell me, and I will fix it.)
    1704. 

  1704.  * Chrominance data is only taken from every second line,
    1705. 

  1705.  * others are ignored in the C version.
    1706. 

  1706.  * FIXME: Write HQ version.
    1707. 

  1707.  */
    1708. 

  1708. static inline void RENAME(rgb24toyv12)(const uint8_t *src, uint8_t *ydst, uint8_t *udst, uint8_t *vdst,
    1709. 

  1709.                                        int width, int height,
    1710. 

  1710.                                        int lumStride, int chromStride, int srcStride)
    1711. 

  1711. {
    1712. 

  1712.     int y;
    1713. 

  1713.     const x86_reg chromWidth= width>>1;
    1714. 

  1714.     for (y=0; y<height-2; y+=2) {
    1715. 

  1715.         int i;
    1716. 

  1716.         for (i=0; i<2; i++) {
    1717. 

  1717.             __asm__ volatile(
    1718. 

  1718.                 "mov                        %2, %%"REG_a"   \n\t"
    1719. 

  1719.                 "movq  "MANGLE(ff_bgr2YCoeff)", %%mm6       \n\t"
    1720. 

  1720.                 "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"
    1721. 

  1721.                 "pxor                    %%mm7, %%mm7       \n\t"
    1722. 

  1722.                 "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"   \n\t"
    1723. 

  1723.                 ".p2align                    4              \n\t"
    1724. 

  1724.                 "1:                                         \n\t"
    1725. 

  1725.                 PREFETCH"    64(%0, %%"REG_d")              \n\t"
    1726. 

  1726.                 "movd          (%0, %%"REG_d"), %%mm0       \n\t"
    1727. 

  1727.                 "movd         3(%0, %%"REG_d"), %%mm1       \n\t"
    1728. 

  1728.                 "punpcklbw               %%mm7, %%mm0       \n\t"
    1729. 

  1729.                 "punpcklbw               %%mm7, %%mm1       \n\t"
    1730. 

  1730.                 "movd         6(%0, %%"REG_d"), %%mm2       \n\t"
    1731. 

  1731.                 "movd         9(%0, %%"REG_d"), %%mm3       \n\t"
    1732. 

  1732.                 "punpcklbw               %%mm7, %%mm2       \n\t"
    1733. 

  1733.                 "punpcklbw               %%mm7, %%mm3       \n\t"
    1734. 

  1734.                 "pmaddwd                 %%mm6, %%mm0       \n\t"
    1735. 

  1735.                 "pmaddwd                 %%mm6, %%mm1       \n\t"
    1736. 

  1736.                 "pmaddwd                 %%mm6, %%mm2       \n\t"
    1737. 

  1737.                 "pmaddwd                 %%mm6, %%mm3       \n\t"
    1738. 

  1738. #ifndef FAST_BGR2YV12
    1739. 

  1739.                 "psrad                      $8, %%mm0       \n\t"
    1740. 

  1740.                 "psrad                      $8, %%mm1       \n\t"
    1741. 

  1741.                 "psrad                      $8, %%mm2       \n\t"
    1742. 

  1742.                 "psrad                      $8, %%mm3       \n\t"
    1743. 

  1743. #endif
    1744. 

  1744.                 "packssdw                %%mm1, %%mm0       \n\t"
    1745. 

  1745.                 "packssdw                %%mm3, %%mm2       \n\t"
    1746. 

  1746.                 "pmaddwd                 %%mm5, %%mm0       \n\t"
    1747. 

  1747.                 "pmaddwd                 %%mm5, %%mm2       \n\t"
    1748. 

  1748.                 "packssdw                %%mm2, %%mm0       \n\t"
    1749. 

  1749.                 "psraw                      $7, %%mm0       \n\t"
    1750. 

  1750. 

  1751.                 "movd        12(%0, %%"REG_d"), %%mm4       \n\t"
    1752. 

  1752.                 "movd        15(%0, %%"REG_d"), %%mm1       \n\t"
    1753. 

  1753.                 "punpcklbw               %%mm7, %%mm4       \n\t"
    1754. 

  1754.                 "punpcklbw               %%mm7, %%mm1       \n\t"
    1755. 

  1755.                 "movd        18(%0, %%"REG_d"), %%mm2       \n\t"
    1756. 

  1756.                 "movd        21(%0, %%"REG_d"), %%mm3       \n\t"
    1757. 

  1757.                 "punpcklbw               %%mm7, %%mm2       \n\t"
    1758. 

  1758.                 "punpcklbw               %%mm7, %%mm3       \n\t"
    1759. 

  1759.                 "pmaddwd                 %%mm6, %%mm4       \n\t"
    1760. 

  1760.                 "pmaddwd                 %%mm6, %%mm1       \n\t"
    1761. 

  1761.                 "pmaddwd                 %%mm6, %%mm2       \n\t"
    1762. 

  1762.                 "pmaddwd                 %%mm6, %%mm3       \n\t"
    1763. 

  1763. #ifndef FAST_BGR2YV12
    1764. 

  1764.                 "psrad                      $8, %%mm4       \n\t"
    1765. 

  1765.                 "psrad                      $8, %%mm1       \n\t"
    1766. 

  1766.                 "psrad                      $8, %%mm2       \n\t"
    1767. 

  1767.                 "psrad                      $8, %%mm3       \n\t"
    1768. 

  1768. #endif
    1769. 

  1769.                 "packssdw                %%mm1, %%mm4       \n\t"
    1770. 

  1770.                 "packssdw                %%mm3, %%mm2       \n\t"
    1771. 

  1771.                 "pmaddwd                 %%mm5, %%mm4       \n\t"
    1772. 

  1772.                 "pmaddwd                 %%mm5, %%mm2       \n\t"
    1773. 

  1773.                 "add                       $24, %%"REG_d"   \n\t"
    1774. 

  1774.                 "packssdw                %%mm2, %%mm4       \n\t"
    1775. 

  1775.                 "psraw                      $7, %%mm4       \n\t"
    1776. 

  1776. 

  1777.                 "packuswb                %%mm4, %%mm0       \n\t"
    1778. 

  1778.                 "paddusb "MANGLE(ff_bgr2YOffset)", %%mm0    \n\t"
    1779. 

  1779. 

  1780.                 MOVNTQ"                  %%mm0, (%1, %%"REG_a") \n\t"
    1781. 

  1781.                 "add                        $8,      %%"REG_a"  \n\t"
    1782. 

  1782.                 " js                        1b                  \n\t"
    1783. 

  1783.                 : : "r" (src+width*3), "r" (ydst+width), "g" ((x86_reg)-width)
    1784. 

  1784.                 : "%"REG_a, "%"REG_d
    1785. 

  1785.             );
    1786. 

  1786.             ydst += lumStride;
    1787. 

  1787.             src  += srcStride;
    1788. 

  1788.         }
    1789. 

  1789.         src -= srcStride*2;
    1790. 

  1790.         __asm__ volatile(
    1791. 

  1791.             "mov                        %4, %%"REG_a"   \n\t"
    1792. 

  1792.             "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"
    1793. 

  1793.             "movq  "MANGLE(ff_bgr2UCoeff)", %%mm6       \n\t"
    1794. 

  1794.             "pxor                    %%mm7, %%mm7       \n\t"
    1795. 

  1795.             "lea (%%"REG_a", %%"REG_a", 2), %%"REG_d"   \n\t"
    1796. 

  1796.             "add                 %%"REG_d", %%"REG_d"   \n\t"
    1797. 

  1797.             ".p2align                    4              \n\t"
    1798. 

  1798.             "1:                                         \n\t"
    1799. 

  1799.             PREFETCH"    64(%0, %%"REG_d")              \n\t"
    1800. 

  1800.             PREFETCH"    64(%1, %%"REG_d")              \n\t"
    1801. 

  1801. #if COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW
    1802. 

  1802.             "movq          (%0, %%"REG_d"), %%mm0       \n\t"
    1803. 

  1803.             "movq          (%1, %%"REG_d"), %%mm1       \n\t"
    1804. 

  1804.             "movq         6(%0, %%"REG_d"), %%mm2       \n\t"
    1805. 

  1805.             "movq         6(%1, %%"REG_d"), %%mm3       \n\t"
    1806. 

  1806.             PAVGB"                   %%mm1, %%mm0       \n\t"
    1807. 

  1807.             PAVGB"                   %%mm3, %%mm2       \n\t"
    1808. 

  1808.             "movq                    %%mm0, %%mm1       \n\t"
    1809. 

  1809.             "movq                    %%mm2, %%mm3       \n\t"
    1810. 

  1810.             "psrlq                     $24, %%mm0       \n\t"
    1811. 

  1811.             "psrlq                     $24, %%mm2       \n\t"
    1812. 

  1812.             PAVGB"                   %%mm1, %%mm0       \n\t"
    1813. 

  1813.             PAVGB"                   %%mm3, %%mm2       \n\t"
    1814. 

  1814.             "punpcklbw               %%mm7, %%mm0       \n\t"
    1815. 

  1815.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1816. 

  1816. #else
    1817. 

  1817.             "movd          (%0, %%"REG_d"), %%mm0       \n\t"
    1818. 

  1818.             "movd          (%1, %%"REG_d"), %%mm1       \n\t"
    1819. 

  1819.             "movd         3(%0, %%"REG_d"), %%mm2       \n\t"
    1820. 

  1820.             "movd         3(%1, %%"REG_d"), %%mm3       \n\t"
    1821. 

  1821.             "punpcklbw               %%mm7, %%mm0       \n\t"
    1822. 

  1822.             "punpcklbw               %%mm7, %%mm1       \n\t"
    1823. 

  1823.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1824. 

  1824.             "punpcklbw               %%mm7, %%mm3       \n\t"
    1825. 

  1825.             "paddw                   %%mm1, %%mm0       \n\t"
    1826. 

  1826.             "paddw                   %%mm3, %%mm2       \n\t"
    1827. 

  1827.             "paddw                   %%mm2, %%mm0       \n\t"
    1828. 

  1828.             "movd         6(%0, %%"REG_d"), %%mm4       \n\t"
    1829. 

  1829.             "movd         6(%1, %%"REG_d"), %%mm1       \n\t"
    1830. 

  1830.             "movd         9(%0, %%"REG_d"), %%mm2       \n\t"
    1831. 

  1831.             "movd         9(%1, %%"REG_d"), %%mm3       \n\t"
    1832. 

  1832.             "punpcklbw               %%mm7, %%mm4       \n\t"
    1833. 

  1833.             "punpcklbw               %%mm7, %%mm1       \n\t"
    1834. 

  1834.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1835. 

  1835.             "punpcklbw               %%mm7, %%mm3       \n\t"
    1836. 

  1836.             "paddw                   %%mm1, %%mm4       \n\t"
    1837. 

  1837.             "paddw                   %%mm3, %%mm2       \n\t"
    1838. 

  1838.             "paddw                   %%mm4, %%mm2       \n\t"
    1839. 

  1839.             "psrlw                      $2, %%mm0       \n\t"
    1840. 

  1840.             "psrlw                      $2, %%mm2       \n\t"
    1841. 

  1841. #endif
    1842. 

  1842.             "movq  "MANGLE(ff_bgr2VCoeff)", %%mm1       \n\t"
    1843. 

  1843.             "movq  "MANGLE(ff_bgr2VCoeff)", %%mm3       \n\t"
    1844. 

  1844. 

  1845.             "pmaddwd                 %%mm0, %%mm1       \n\t"
    1846. 

  1846.             "pmaddwd                 %%mm2, %%mm3       \n\t"
    1847. 

  1847.             "pmaddwd                 %%mm6, %%mm0       \n\t"
    1848. 

  1848.             "pmaddwd                 %%mm6, %%mm2       \n\t"
    1849. 

  1849. #ifndef FAST_BGR2YV12
    1850. 

  1850.             "psrad                      $8, %%mm0       \n\t"
    1851. 

  1851.             "psrad                      $8, %%mm1       \n\t"
    1852. 

  1852.             "psrad                      $8, %%mm2       \n\t"
    1853. 

  1853.             "psrad                      $8, %%mm3       \n\t"
    1854. 

  1854. #endif
    1855. 

  1855.             "packssdw                %%mm2, %%mm0       \n\t"
    1856. 

  1856.             "packssdw                %%mm3, %%mm1       \n\t"
    1857. 

  1857.             "pmaddwd                 %%mm5, %%mm0       \n\t"
    1858. 

  1858.             "pmaddwd                 %%mm5, %%mm1       \n\t"
    1859. 

  1859.             "packssdw                %%mm1, %%mm0       \n\t" // V1 V0 U1 U0
    1860. 

  1860.             "psraw                      $7, %%mm0       \n\t"
    1861. 

  1861. 

  1862. #if COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW
    1863. 

  1863.             "movq        12(%0, %%"REG_d"), %%mm4       \n\t"
    1864. 

  1864.             "movq        12(%1, %%"REG_d"), %%mm1       \n\t"
    1865. 

  1865.             "movq        18(%0, %%"REG_d"), %%mm2       \n\t"
    1866. 

  1866.             "movq        18(%1, %%"REG_d"), %%mm3       \n\t"
    1867. 

  1867.             PAVGB"                   %%mm1, %%mm4       \n\t"
    1868. 

  1868.             PAVGB"                   %%mm3, %%mm2       \n\t"
    1869. 

  1869.             "movq                    %%mm4, %%mm1       \n\t"
    1870. 

  1870.             "movq                    %%mm2, %%mm3       \n\t"
    1871. 

  1871.             "psrlq                     $24, %%mm4       \n\t"
    1872. 

  1872.             "psrlq                     $24, %%mm2       \n\t"
    1873. 

  1873.             PAVGB"                   %%mm1, %%mm4       \n\t"
    1874. 

  1874.             PAVGB"                   %%mm3, %%mm2       \n\t"
    1875. 

  1875.             "punpcklbw               %%mm7, %%mm4       \n\t"
    1876. 

  1876.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1877. 

  1877. #else
    1878. 

  1878.             "movd        12(%0, %%"REG_d"), %%mm4       \n\t"
    1879. 

  1879.             "movd        12(%1, %%"REG_d"), %%mm1       \n\t"
    1880. 

  1880.             "movd        15(%0, %%"REG_d"), %%mm2       \n\t"
    1881. 

  1881.             "movd        15(%1, %%"REG_d"), %%mm3       \n\t"
    1882. 

  1882.             "punpcklbw               %%mm7, %%mm4       \n\t"
    1883. 

  1883.             "punpcklbw               %%mm7, %%mm1       \n\t"
    1884. 

  1884.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1885. 

  1885.             "punpcklbw               %%mm7, %%mm3       \n\t"
    1886. 

  1886.             "paddw                   %%mm1, %%mm4       \n\t"
    1887. 

  1887.             "paddw                   %%mm3, %%mm2       \n\t"
    1888. 

  1888.             "paddw                   %%mm2, %%mm4       \n\t"
    1889. 

  1889.             "movd        18(%0, %%"REG_d"), %%mm5       \n\t"
    1890. 

  1890.             "movd        18(%1, %%"REG_d"), %%mm1       \n\t"
    1891. 

  1891.             "movd        21(%0, %%"REG_d"), %%mm2       \n\t"
    1892. 

  1892.             "movd        21(%1, %%"REG_d"), %%mm3       \n\t"
    1893. 

  1893.             "punpcklbw               %%mm7, %%mm5       \n\t"
    1894. 

  1894.             "punpcklbw               %%mm7, %%mm1       \n\t"
    1895. 

  1895.             "punpcklbw               %%mm7, %%mm2       \n\t"
    1896. 

  1896.             "punpcklbw               %%mm7, %%mm3       \n\t"
    1897. 

  1897.             "paddw                   %%mm1, %%mm5       \n\t"
    1898. 

  1898.             "paddw                   %%mm3, %%mm2       \n\t"
    1899. 

  1899.             "paddw                   %%mm5, %%mm2       \n\t"
    1900. 

  1900.             "movq       "MANGLE(ff_w1111)", %%mm5       \n\t"
    1901. 

  1901.             "psrlw                      $2, %%mm4       \n\t"
    1902. 

  1902.             "psrlw                      $2, %%mm2       \n\t"
    1903. 

  1903. #endif
    1904. 

  1904.             "movq  "MANGLE(ff_bgr2VCoeff)", %%mm1       \n\t"
    1905. 

  1905.             "movq  "MANGLE(ff_bgr2VCoeff)", %%mm3       \n\t"
    1906. 

  1906. 

  1907.             "pmaddwd                 %%mm4, %%mm1       \n\t"
    1908. 

  1908.             "pmaddwd                 %%mm2, %%mm3       \n\t"
    1909. 

  1909.             "pmaddwd                 %%mm6, %%mm4       \n\t"
    1910. 

  1910.             "pmaddwd                 %%mm6, %%mm2       \n\t"
    1911. 

  1911. #ifndef FAST_BGR2YV12
    1912. 

  1912.             "psrad                      $8, %%mm4       \n\t"
    1913. 

  1913.             "psrad                      $8, %%mm1       \n\t"
    1914. 

  1914.             "psrad                      $8, %%mm2       \n\t"
    1915. 

  1915.             "psrad                      $8, %%mm3       \n\t"
    1916. 

  1916. #endif
    1917. 

  1917.             "packssdw                %%mm2, %%mm4       \n\t"
    1918. 

  1918.             "packssdw                %%mm3, %%mm1       \n\t"
    1919. 

  1919.             "pmaddwd                 %%mm5, %%mm4       \n\t"
    1920. 

  1920.             "pmaddwd                 %%mm5, %%mm1       \n\t"
    1921. 

  1921.             "add                       $24, %%"REG_d"   \n\t"
    1922. 

  1922.             "packssdw                %%mm1, %%mm4       \n\t" // V3 V2 U3 U2
    1923. 

  1923.             "psraw                      $7, %%mm4       \n\t"
    1924. 

  1924. 

  1925.             "movq                    %%mm0, %%mm1           \n\t"
    1926. 

  1926.             "punpckldq               %%mm4, %%mm0           \n\t"
    1927. 

  1927.             "punpckhdq               %%mm4, %%mm1           \n\t"
    1928. 

  1928.             "packsswb                %%mm1, %%mm0           \n\t"
    1929. 

  1929.             "paddb "MANGLE(ff_bgr2UVOffset)", %%mm0         \n\t"
    1930. 

  1930.             "movd                    %%mm0, (%2, %%"REG_a") \n\t"
    1931. 

  1931.             "punpckhdq               %%mm0, %%mm0           \n\t"
    1932. 

  1932.             "movd                    %%mm0, (%3, %%"REG_a") \n\t"
    1933. 

  1933.             "add                        $4, %%"REG_a"       \n\t"
    1934. 

  1934.             " js                        1b                  \n\t"
    1935. 

  1935.             : : "r" (src+chromWidth*6), "r" (src+srcStride+chromWidth*6), "r" (udst+chromWidth), "r" (vdst+chromWidth), "g" (-chromWidth)
    1936. 

  1936.             : "%"REG_a, "%"REG_d
    1937. 

  1937.         );
    1938. 

  1938. 

  1939.         udst += chromStride;
    1940. 

  1940.         vdst += chromStride;
    1941. 

  1941.         src  += srcStride*2;
    1942. 

  1942.     }
    1943. 

  1943. 

  1944.     __asm__ volatile(EMMS"       \n\t"
    1945. 

  1945.                      SFENCE"     \n\t"
    1946. 

  1946.                      :::"memory");
    1947. 

  1947. 

  1948.      rgb24toyv12_c(src, ydst, udst, vdst, width, height-y, lumStride, chromStride, srcStride);
    1949. 

  1949. }
    1950. 

  1950. #endif /* !COMPILE_TEMPLATE_SSE2 */
    1951. 

  1951. 

  1952. #if !COMPILE_TEMPLATE_AMD3DNOW
    1953. 

  1953. static void RENAME(interleaveBytes)(const uint8_t *src1, const uint8_t *src2, uint8_t *dest,
    1954. 

  1954.                                     int width, int height, int src1Stride,
    1955. 

  1955.                                     int src2Stride, int dstStride)
    1956. 

  1956. {
    1957. 

  1957.     int h;
    1958. 

  1958. 

  1959.     for (h=0; h < height; h++) {
    1960. 

  1960.         int w;
    1961. 

  1961. 

  1962. #if COMPILE_TEMPLATE_SSE2
    1963. 

  1963.         __asm__(
    1964. 

  1964.             "xor              %%"REG_a", %%"REG_a"  \n\t"
    1965. 

  1965.             "1:                                     \n\t"
    1966. 

  1966.             PREFETCH" 64(%1, %%"REG_a")             \n\t"
    1967. 

  1967.             PREFETCH" 64(%2, %%"REG_a")             \n\t"
    1968. 

  1968.             "movdqa     (%1, %%"REG_a"), %%xmm0     \n\t"
    1969. 

  1969.             "movdqa     (%1, %%"REG_a"), %%xmm1     \n\t"
    1970. 

  1970.             "movdqa     (%2, %%"REG_a"), %%xmm2     \n\t"
    1971. 

  1971.             "punpcklbw           %%xmm2, %%xmm0     \n\t"
    1972. 

  1972.             "punpckhbw           %%xmm2, %%xmm1     \n\t"
    1973. 

  1973.             "movntdq             %%xmm0,   (%0, %%"REG_a", 2)   \n\t"
    1974. 

  1974.             "movntdq             %%xmm1, 16(%0, %%"REG_a", 2)   \n\t"
    1975. 

  1975.             "add                    $16, %%"REG_a"  \n\t"
    1976. 

  1976.             "cmp                     %3, %%"REG_a"  \n\t"
    1977. 

  1977.             " jb                     1b             \n\t"
    1978. 

  1978.             ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
    1979. 

  1979.             : "memory", "%"REG_a""
    1980. 

  1980.         );
    1981. 

  1981. #else
    1982. 

  1982.         __asm__(
    1983. 

  1983.             "xor %%"REG_a", %%"REG_a"               \n\t"
    1984. 

  1984.             "1:                                     \n\t"
    1985. 

  1985.             PREFETCH" 64(%1, %%"REG_a")             \n\t"
    1986. 

  1986.             PREFETCH" 64(%2, %%"REG_a")             \n\t"
    1987. 

  1987.             "movq       (%1, %%"REG_a"), %%mm0      \n\t"
    1988. 

  1988.             "movq      8(%1, %%"REG_a"), %%mm2      \n\t"
    1989. 

  1989.             "movq                 %%mm0, %%mm1      \n\t"
    1990. 

  1990.             "movq                 %%mm2, %%mm3      \n\t"
    1991. 

  1991.             "movq       (%2, %%"REG_a"), %%mm4      \n\t"
    1992. 

  1992.             "movq      8(%2, %%"REG_a"), %%mm5      \n\t"
    1993. 

  1993.             "punpcklbw            %%mm4, %%mm0      \n\t"
    1994. 

  1994.             "punpckhbw            %%mm4, %%mm1      \n\t"
    1995. 

  1995.             "punpcklbw            %%mm5, %%mm2      \n\t"
    1996. 

  1996.             "punpckhbw            %%mm5, %%mm3      \n\t"
    1997. 

  1997.             MOVNTQ"               %%mm0,   (%0, %%"REG_a", 2)   \n\t"
    1998. 

  1998.             MOVNTQ"               %%mm1,  8(%0, %%"REG_a", 2)   \n\t"
    1999. 

  1999.             MOVNTQ"               %%mm2, 16(%0, %%"REG_a", 2)   \n\t"
    2000. 

  2000.             MOVNTQ"               %%mm3, 24(%0, %%"REG_a", 2)   \n\t"
    2001. 

  2001.             "add                    $16, %%"REG_a"  \n\t"
    2002. 

  2002.             "cmp                     %3, %%"REG_a"  \n\t"
    2003. 

  2003.             " jb                     1b             \n\t"
    2004. 

  2004.             ::"r"(dest), "r"(src1), "r"(src2), "r" ((x86_reg)width-15)
    2005. 

  2005.             : "memory", "%"REG_a
    2006. 

  2006.         );
    2007. 

  2007. #endif
    2008. 

  2008.         for (w= (width&(~15)); w < width; w++) {
    2009. 

  2009.             dest[2*w+0] = src1[w];
    2010. 

  2010.             dest[2*w+1] = src2[w];
    2011. 

  2011.         }
    2012. 

  2012.         dest += dstStride;
    2013. 

  2013.         src1 += src1Stride;
    2014. 

  2014.         src2 += src2Stride;
    2015. 

  2015.     }
    2016. 

  2016.     __asm__(
    2017. 

  2017.             EMMS"       \n\t"
    2018. 

  2018.             SFENCE"     \n\t"
    2019. 

  2019.             ::: "memory"
    2020. 

  2020.             );
    2021. 

  2021. }
    2022. 

  2022. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2023. 

  2023. 

  2024. #if !COMPILE_TEMPLATE_SSE2
    2025. 

  2025. #if !COMPILE_TEMPLATE_AMD3DNOW
    2026. 

  2026. static inline void RENAME(vu9_to_vu12)(const uint8_t *src1, const uint8_t *src2,
    2027. 

  2027.                                        uint8_t *dst1, uint8_t *dst2,
    2028. 

  2028.                                        int width, int height,
    2029. 

  2029.                                        int srcStride1, int srcStride2,
    2030. 

  2030.                                        int dstStride1, int dstStride2)
    2031. 

  2031. {
    2032. 

  2032.     x86_reg y;
    2033. 

  2033.     int x,w,h;
    2034. 

  2034.     w=width/2; h=height/2;
    2035. 

  2035.     __asm__ volatile(
    2036. 

  2036.         PREFETCH" %0    \n\t"
    2037. 

  2037.         PREFETCH" %1    \n\t"
    2038. 

  2038.         ::"m"(*(src1+srcStride1)),"m"(*(src2+srcStride2)):"memory");
    2039. 

  2039.     for (y=0;y<h;y++) {
    2040. 

  2040.         const uint8_t* s1=src1+srcStride1*(y>>1);
    2041. 

  2041.         uint8_t* d=dst1+dstStride1*y;
    2042. 

  2042.         x=0;
    2043. 

  2043.         for (;x<w-31;x+=32) {
    2044. 

  2044.             __asm__ volatile(
    2045. 

  2045.                 PREFETCH"   32%1        \n\t"
    2046. 

  2046.                 "movq         %1, %%mm0 \n\t"
    2047. 

  2047.                 "movq        8%1, %%mm2 \n\t"
    2048. 

  2048.                 "movq       16%1, %%mm4 \n\t"
    2049. 

  2049.                 "movq       24%1, %%mm6 \n\t"
    2050. 

  2050.                 "movq      %%mm0, %%mm1 \n\t"
    2051. 

  2051.                 "movq      %%mm2, %%mm3 \n\t"
    2052. 

  2052.                 "movq      %%mm4, %%mm5 \n\t"
    2053. 

  2053.                 "movq      %%mm6, %%mm7 \n\t"
    2054. 

  2054.                 "punpcklbw %%mm0, %%mm0 \n\t"
    2055. 

  2055.                 "punpckhbw %%mm1, %%mm1 \n\t"
    2056. 

  2056.                 "punpcklbw %%mm2, %%mm2 \n\t"
    2057. 

  2057.                 "punpckhbw %%mm3, %%mm3 \n\t"
    2058. 

  2058.                 "punpcklbw %%mm4, %%mm4 \n\t"
    2059. 

  2059.                 "punpckhbw %%mm5, %%mm5 \n\t"
    2060. 

  2060.                 "punpcklbw %%mm6, %%mm6 \n\t"
    2061. 

  2061.                 "punpckhbw %%mm7, %%mm7 \n\t"
    2062. 

  2062.                 MOVNTQ"    %%mm0,   %0  \n\t"
    2063. 

  2063.                 MOVNTQ"    %%mm1,  8%0  \n\t"
    2064. 

  2064.                 MOVNTQ"    %%mm2, 16%0  \n\t"
    2065. 

  2065.                 MOVNTQ"    %%mm3, 24%0  \n\t"
    2066. 

  2066.                 MOVNTQ"    %%mm4, 32%0  \n\t"
    2067. 

  2067.                 MOVNTQ"    %%mm5, 40%0  \n\t"
    2068. 

  2068.                 MOVNTQ"    %%mm6, 48%0  \n\t"
    2069. 

  2069.                 MOVNTQ"    %%mm7, 56%0"
    2070. 

  2070.                 :"=m"(d[2*x])
    2071. 

  2071.                 :"m"(s1[x])
    2072. 

  2072.                 :"memory");
    2073. 

  2073.         }
    2074. 

  2074.         for (;x<w;x++) d[2*x]=d[2*x+1]=s1[x];
    2075. 

  2075.     }
    2076. 

  2076.     for (y=0;y<h;y++) {
    2077. 

  2077.         const uint8_t* s2=src2+srcStride2*(y>>1);
    2078. 

  2078.         uint8_t* d=dst2+dstStride2*y;
    2079. 

  2079.         x=0;
    2080. 

  2080.         for (;x<w-31;x+=32) {
    2081. 

  2081.             __asm__ volatile(
    2082. 

  2082.                 PREFETCH"   32%1        \n\t"
    2083. 

  2083.                 "movq         %1, %%mm0 \n\t"
    2084. 

  2084.                 "movq        8%1, %%mm2 \n\t"
    2085. 

  2085.                 "movq       16%1, %%mm4 \n\t"
    2086. 

  2086.                 "movq       24%1, %%mm6 \n\t"
    2087. 

  2087.                 "movq      %%mm0, %%mm1 \n\t"
    2088. 

  2088.                 "movq      %%mm2, %%mm3 \n\t"
    2089. 

  2089.                 "movq      %%mm4, %%mm5 \n\t"
    2090. 

  2090.                 "movq      %%mm6, %%mm7 \n\t"
    2091. 

  2091.                 "punpcklbw %%mm0, %%mm0 \n\t"
    2092. 

  2092.                 "punpckhbw %%mm1, %%mm1 \n\t"
    2093. 

  2093.                 "punpcklbw %%mm2, %%mm2 \n\t"
    2094. 

  2094.                 "punpckhbw %%mm3, %%mm3 \n\t"
    2095. 

  2095.                 "punpcklbw %%mm4, %%mm4 \n\t"
    2096. 

  2096.                 "punpckhbw %%mm5, %%mm5 \n\t"
    2097. 

  2097.                 "punpcklbw %%mm6, %%mm6 \n\t"
    2098. 

  2098.                 "punpckhbw %%mm7, %%mm7 \n\t"
    2099. 

  2099.                 MOVNTQ"    %%mm0,   %0  \n\t"
    2100. 

  2100.                 MOVNTQ"    %%mm1,  8%0  \n\t"
    2101. 

  2101.                 MOVNTQ"    %%mm2, 16%0  \n\t"
    2102. 

  2102.                 MOVNTQ"    %%mm3, 24%0  \n\t"
    2103. 

  2103.                 MOVNTQ"    %%mm4, 32%0  \n\t"
    2104. 

  2104.                 MOVNTQ"    %%mm5, 40%0  \n\t"
    2105. 

  2105.                 MOVNTQ"    %%mm6, 48%0  \n\t"
    2106. 

  2106.                 MOVNTQ"    %%mm7, 56%0"
    2107. 

  2107.                 :"=m"(d[2*x])
    2108. 

  2108.                 :"m"(s2[x])
    2109. 

  2109.                 :"memory");
    2110. 

  2110.         }
    2111. 

  2111.         for (;x<w;x++) d[2*x]=d[2*x+1]=s2[x];
    2112. 

  2112.     }
    2113. 

  2113.     __asm__(
    2114. 

  2114.             EMMS"       \n\t"
    2115. 

  2115.             SFENCE"     \n\t"
    2116. 

  2116.             ::: "memory"
    2117. 

  2117.         );
    2118. 

  2118. }
    2119. 

  2119. 

  2120. static inline void RENAME(yvu9_to_yuy2)(const uint8_t *src1, const uint8_t *src2, const uint8_t *src3,
    2121. 

  2121.                                         uint8_t *dst,
    2122. 

  2122.                                         int width, int height,
    2123. 

  2123.                                         int srcStride1, int srcStride2,
    2124. 

  2124.                                         int srcStride3, int dstStride)
    2125. 

  2125. {
    2126. 

  2126.     x86_reg x;
    2127. 

  2127.     int y,w,h;
    2128. 

  2128.     w=width/2; h=height;
    2129. 

  2129.     for (y=0;y<h;y++) {
    2130. 

  2130.         const uint8_t* yp=src1+srcStride1*y;
    2131. 

  2131.         const uint8_t* up=src2+srcStride2*(y>>2);
    2132. 

  2132.         const uint8_t* vp=src3+srcStride3*(y>>2);
    2133. 

  2133.         uint8_t* d=dst+dstStride*y;
    2134. 

  2134.         x=0;
    2135. 

  2135.         for (;x<w-7;x+=8) {
    2136. 

  2136.             __asm__ volatile(
    2137. 

  2137.                 PREFETCH"   32(%1, %0)          \n\t"
    2138. 

  2138.                 PREFETCH"   32(%2, %0)          \n\t"
    2139. 

  2139.                 PREFETCH"   32(%3, %0)          \n\t"
    2140. 

  2140.                 "movq      (%1, %0, 4), %%mm0   \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
    2141. 

  2141.                 "movq         (%2, %0), %%mm1   \n\t" /* U0U1U2U3U4U5U6U7 */
    2142. 

  2142.                 "movq         (%3, %0), %%mm2   \n\t" /* V0V1V2V3V4V5V6V7 */
    2143. 

  2143.                 "movq            %%mm0, %%mm3   \n\t" /* Y0Y1Y2Y3Y4Y5Y6Y7 */
    2144. 

  2144.                 "movq            %%mm1, %%mm4   \n\t" /* U0U1U2U3U4U5U6U7 */
    2145. 

  2145.                 "movq            %%mm2, %%mm5   \n\t" /* V0V1V2V3V4V5V6V7 */
    2146. 

  2146.                 "punpcklbw       %%mm1, %%mm1   \n\t" /* U0U0 U1U1 U2U2 U3U3 */
    2147. 

  2147.                 "punpcklbw       %%mm2, %%mm2   \n\t" /* V0V0 V1V1 V2V2 V3V3 */
    2148. 

  2148.                 "punpckhbw       %%mm4, %%mm4   \n\t" /* U4U4 U5U5 U6U6 U7U7 */
    2149. 

  2149.                 "punpckhbw       %%mm5, %%mm5   \n\t" /* V4V4 V5V5 V6V6 V7V7 */
    2150. 

  2150. 

  2151.                 "movq            %%mm1, %%mm6   \n\t"
    2152. 

  2152.                 "punpcklbw       %%mm2, %%mm1   \n\t" /* U0V0 U0V0 U1V1 U1V1*/
    2153. 

  2153.                 "punpcklbw       %%mm1, %%mm0   \n\t" /* Y0U0 Y1V0 Y2U0 Y3V0*/
    2154. 

  2154.                 "punpckhbw       %%mm1, %%mm3   \n\t" /* Y4U1 Y5V1 Y6U1 Y7V1*/
    2155. 

  2155.                 MOVNTQ"          %%mm0,  (%4, %0, 8)    \n\t"
    2156. 

  2156.                 MOVNTQ"          %%mm3, 8(%4, %0, 8)    \n\t"
    2157. 

  2157. 

  2158.                 "punpckhbw       %%mm2, %%mm6   \n\t" /* U2V2 U2V2 U3V3 U3V3*/
    2159. 

  2159.                 "movq     8(%1, %0, 4), %%mm0   \n\t"
    2160. 

  2160.                 "movq            %%mm0, %%mm3   \n\t"
    2161. 

  2161.                 "punpcklbw       %%mm6, %%mm0   \n\t" /* Y U2 Y V2 Y U2 Y V2*/
    2162. 

  2162.                 "punpckhbw       %%mm6, %%mm3   \n\t" /* Y U3 Y V3 Y U3 Y V3*/
    2163. 

  2163.                 MOVNTQ"          %%mm0, 16(%4, %0, 8)   \n\t"
    2164. 

  2164.                 MOVNTQ"          %%mm3, 24(%4, %0, 8)   \n\t"
    2165. 

  2165. 

  2166.                 "movq            %%mm4, %%mm6   \n\t"
    2167. 

  2167.                 "movq    16(%1, %0, 4), %%mm0   \n\t"
    2168. 

  2168.                 "movq            %%mm0, %%mm3   \n\t"
    2169. 

  2169.                 "punpcklbw       %%mm5, %%mm4   \n\t"
    2170. 

  2170.                 "punpcklbw       %%mm4, %%mm0   \n\t" /* Y U4 Y V4 Y U4 Y V4*/
    2171. 

  2171.                 "punpckhbw       %%mm4, %%mm3   \n\t" /* Y U5 Y V5 Y U5 Y V5*/
    2172. 

  2172.                 MOVNTQ"          %%mm0, 32(%4, %0, 8)   \n\t"
    2173. 

  2173.                 MOVNTQ"          %%mm3, 40(%4, %0, 8)   \n\t"
    2174. 

  2174. 

  2175.                 "punpckhbw       %%mm5, %%mm6   \n\t"
    2176. 

  2176.                 "movq    24(%1, %0, 4), %%mm0   \n\t"
    2177. 

  2177.                 "movq            %%mm0, %%mm3   \n\t"
    2178. 

  2178.                 "punpcklbw       %%mm6, %%mm0   \n\t" /* Y U6 Y V6 Y U6 Y V6*/
    2179. 

  2179.                 "punpckhbw       %%mm6, %%mm3   \n\t" /* Y U7 Y V7 Y U7 Y V7*/
    2180. 

  2180.                 MOVNTQ"          %%mm0, 48(%4, %0, 8)   \n\t"
    2181. 

  2181.                 MOVNTQ"          %%mm3, 56(%4, %0, 8)   \n\t"
    2182. 

  2182. 

  2183.                 : "+r" (x)
    2184. 

  2184.                 : "r"(yp), "r" (up), "r"(vp), "r"(d)
    2185. 

  2185.                 :"memory");
    2186. 

  2186.         }
    2187. 

  2187.         for (; x<w; x++) {
    2188. 

  2188.             const int x2 = x<<2;
    2189. 

  2189.             d[8*x+0] = yp[x2];
    2190. 

  2190.             d[8*x+1] = up[x];
    2191. 

  2191.             d[8*x+2] = yp[x2+1];
    2192. 

  2192.             d[8*x+3] = vp[x];
    2193. 

  2193.             d[8*x+4] = yp[x2+2];
    2194. 

  2194.             d[8*x+5] = up[x];
    2195. 

  2195.             d[8*x+6] = yp[x2+3];
    2196. 

  2196.             d[8*x+7] = vp[x];
    2197. 

  2197.         }
    2198. 

  2198.     }
    2199. 

  2199.     __asm__(
    2200. 

  2200.             EMMS"       \n\t"
    2201. 

  2201.             SFENCE"     \n\t"
    2202. 

  2202.             ::: "memory"
    2203. 

  2203.         );
    2204. 

  2204. }
    2205. 

  2205. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2206. 

  2206. 

  2207. static void RENAME(extract_even)(const uint8_t *src, uint8_t *dst, x86_reg count)
    2208. 

  2208. {
    2209. 

  2209.     dst +=   count;
    2210. 

  2210.     src += 2*count;
    2211. 

  2211.     count= - count;
    2212. 

  2212. 

  2213.     if(count <= -16) {
    2214. 

  2214.         count += 15;
    2215. 

  2215.         __asm__ volatile(
    2216. 

  2216.             "pcmpeqw       %%mm7, %%mm7        \n\t"
    2217. 

  2217.             "psrlw            $8, %%mm7        \n\t"
    2218. 

  2218.             "1:                                \n\t"
    2219. 

  2219.             "movq -30(%1, %0, 2), %%mm0        \n\t"
    2220. 

  2220.             "movq -22(%1, %0, 2), %%mm1        \n\t"
    2221. 

  2221.             "movq -14(%1, %0, 2), %%mm2        \n\t"
    2222. 

  2222.             "movq  -6(%1, %0, 2), %%mm3        \n\t"
    2223. 

  2223.             "pand          %%mm7, %%mm0        \n\t"
    2224. 

  2224.             "pand          %%mm7, %%mm1        \n\t"
    2225. 

  2225.             "pand          %%mm7, %%mm2        \n\t"
    2226. 

  2226.             "pand          %%mm7, %%mm3        \n\t"
    2227. 

  2227.             "packuswb      %%mm1, %%mm0        \n\t"
    2228. 

  2228.             "packuswb      %%mm3, %%mm2        \n\t"
    2229. 

  2229.             MOVNTQ"        %%mm0,-15(%2, %0)   \n\t"
    2230. 

  2230.             MOVNTQ"        %%mm2,- 7(%2, %0)   \n\t"
    2231. 

  2231.             "add             $16, %0           \n\t"
    2232. 

  2232.             " js 1b                            \n\t"
    2233. 

  2233.             : "+r"(count)
    2234. 

  2234.             : "r"(src), "r"(dst)
    2235. 

  2235.         );
    2236. 

  2236.         count -= 15;
    2237. 

  2237.     }
    2238. 

  2238.     while(count<0) {
    2239. 

  2239.         dst[count]= src[2*count];
    2240. 

  2240.         count++;
    2241. 

  2241.     }
    2242. 

  2242. }
    2243. 

  2243. 

  2244. #if !COMPILE_TEMPLATE_AMD3DNOW
    2245. 

  2245. static void RENAME(extract_even2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
    2246. 

  2246. {
    2247. 

  2247.     dst0+=   count;
    2248. 

  2248.     dst1+=   count;
    2249. 

  2249.     src += 4*count;
    2250. 

  2250.     count= - count;
    2251. 

  2251.     if(count <= -8) {
    2252. 

  2252.         count += 7;
    2253. 

  2253.         __asm__ volatile(
    2254. 

  2254.             "pcmpeqw       %%mm7, %%mm7        \n\t"
    2255. 

  2255.             "psrlw            $8, %%mm7        \n\t"
    2256. 

  2256.             "1:                                \n\t"
    2257. 

  2257.             "movq -28(%1, %0, 4), %%mm0        \n\t"
    2258. 

  2258.             "movq -20(%1, %0, 4), %%mm1        \n\t"
    2259. 

  2259.             "movq -12(%1, %0, 4), %%mm2        \n\t"
    2260. 

  2260.             "movq  -4(%1, %0, 4), %%mm3        \n\t"
    2261. 

  2261.             "pand          %%mm7, %%mm0        \n\t"
    2262. 

  2262.             "pand          %%mm7, %%mm1        \n\t"
    2263. 

  2263.             "pand          %%mm7, %%mm2        \n\t"
    2264. 

  2264.             "pand          %%mm7, %%mm3        \n\t"
    2265. 

  2265.             "packuswb      %%mm1, %%mm0        \n\t"
    2266. 

  2266.             "packuswb      %%mm3, %%mm2        \n\t"
    2267. 

  2267.             "movq          %%mm0, %%mm1        \n\t"
    2268. 

  2268.             "movq          %%mm2, %%mm3        \n\t"
    2269. 

  2269.             "psrlw            $8, %%mm0        \n\t"
    2270. 

  2270.             "psrlw            $8, %%mm2        \n\t"
    2271. 

  2271.             "pand          %%mm7, %%mm1        \n\t"
    2272. 

  2272.             "pand          %%mm7, %%mm3        \n\t"
    2273. 

  2273.             "packuswb      %%mm2, %%mm0        \n\t"
    2274. 

  2274.             "packuswb      %%mm3, %%mm1        \n\t"
    2275. 

  2275.             MOVNTQ"        %%mm0,- 7(%3, %0)   \n\t"
    2276. 

  2276.             MOVNTQ"        %%mm1,- 7(%2, %0)   \n\t"
    2277. 

  2277.             "add              $8, %0           \n\t"
    2278. 

  2278.             " js 1b                            \n\t"
    2279. 

  2279.             : "+r"(count)
    2280. 

  2280.             : "r"(src), "r"(dst0), "r"(dst1)
    2281. 

  2281.         );
    2282. 

  2282.         count -= 7;
    2283. 

  2283.     }
    2284. 

  2284.     while(count<0) {
    2285. 

  2285.         dst0[count]= src[4*count+0];
    2286. 

  2286.         dst1[count]= src[4*count+2];
    2287. 

  2287.         count++;
    2288. 

  2288.     }
    2289. 

  2289. }
    2290. 

  2290. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2291. 

  2291. 

  2292. static void RENAME(extract_even2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
    2293. 

  2293. {
    2294. 

  2294.     dst0 +=   count;
    2295. 

  2295.     dst1 +=   count;
    2296. 

  2296.     src0 += 4*count;
    2297. 

  2297.     src1 += 4*count;
    2298. 

  2298.     count= - count;
    2299. 

  2299. #ifdef PAVGB
    2300. 

  2300.     if(count <= -8) {
    2301. 

  2301.         count += 7;
    2302. 

  2302.         __asm__ volatile(
    2303. 

  2303.             "pcmpeqw        %%mm7, %%mm7        \n\t"
    2304. 

  2304.             "psrlw             $8, %%mm7        \n\t"
    2305. 

  2305.             "1:                                \n\t"
    2306. 

  2306.             "movq  -28(%1, %0, 4), %%mm0        \n\t"
    2307. 

  2307.             "movq  -20(%1, %0, 4), %%mm1        \n\t"
    2308. 

  2308.             "movq  -12(%1, %0, 4), %%mm2        \n\t"
    2309. 

  2309.             "movq   -4(%1, %0, 4), %%mm3        \n\t"
    2310. 

  2310.             PAVGB" -28(%2, %0, 4), %%mm0        \n\t"
    2311. 

  2311.             PAVGB" -20(%2, %0, 4), %%mm1        \n\t"
    2312. 

  2312.             PAVGB" -12(%2, %0, 4), %%mm2        \n\t"
    2313. 

  2313.             PAVGB" - 4(%2, %0, 4), %%mm3        \n\t"
    2314. 

  2314.             "pand           %%mm7, %%mm0        \n\t"
    2315. 

  2315.             "pand           %%mm7, %%mm1        \n\t"
    2316. 

  2316.             "pand           %%mm7, %%mm2        \n\t"
    2317. 

  2317.             "pand           %%mm7, %%mm3        \n\t"
    2318. 

  2318.             "packuswb       %%mm1, %%mm0        \n\t"
    2319. 

  2319.             "packuswb       %%mm3, %%mm2        \n\t"
    2320. 

  2320.             "movq           %%mm0, %%mm1        \n\t"
    2321. 

  2321.             "movq           %%mm2, %%mm3        \n\t"
    2322. 

  2322.             "psrlw             $8, %%mm0        \n\t"
    2323. 

  2323.             "psrlw             $8, %%mm2        \n\t"
    2324. 

  2324.             "pand           %%mm7, %%mm1        \n\t"
    2325. 

  2325.             "pand           %%mm7, %%mm3        \n\t"
    2326. 

  2326.             "packuswb       %%mm2, %%mm0        \n\t"
    2327. 

  2327.             "packuswb       %%mm3, %%mm1        \n\t"
    2328. 

  2328.             MOVNTQ"         %%mm0,- 7(%4, %0)   \n\t"
    2329. 

  2329.             MOVNTQ"         %%mm1,- 7(%3, %0)   \n\t"
    2330. 

  2330.             "add               $8, %0           \n\t"
    2331. 

  2331.             " js 1b                            \n\t"
    2332. 

  2332.             : "+r"(count)
    2333. 

  2333.             : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
    2334. 

  2334.         );
    2335. 

  2335.         count -= 7;
    2336. 

  2336.     }
    2337. 

  2337. #endif
    2338. 

  2338.     while(count<0) {
    2339. 

  2339.         dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
    2340. 

  2340.         dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
    2341. 

  2341.         count++;
    2342. 

  2342.     }
    2343. 

  2343. }
    2344. 

  2344. 

  2345. #if !COMPILE_TEMPLATE_AMD3DNOW
    2346. 

  2346. static void RENAME(extract_odd2)(const uint8_t *src, uint8_t *dst0, uint8_t *dst1, x86_reg count)
    2347. 

  2347. {
    2348. 

  2348.     dst0+=   count;
    2349. 

  2349.     dst1+=   count;
    2350. 

  2350.     src += 4*count;
    2351. 

  2351.     count= - count;
    2352. 

  2352.     if(count <= -8) {
    2353. 

  2353.         count += 7;
    2354. 

  2354.         __asm__ volatile(
    2355. 

  2355.             "pcmpeqw       %%mm7, %%mm7        \n\t"
    2356. 

  2356.             "psrlw            $8, %%mm7        \n\t"
    2357. 

  2357.             "1:                                \n\t"
    2358. 

  2358.             "movq -28(%1, %0, 4), %%mm0        \n\t"
    2359. 

  2359.             "movq -20(%1, %0, 4), %%mm1        \n\t"
    2360. 

  2360.             "movq -12(%1, %0, 4), %%mm2        \n\t"
    2361. 

  2361.             "movq  -4(%1, %0, 4), %%mm3        \n\t"
    2362. 

  2362.             "psrlw            $8, %%mm0        \n\t"
    2363. 

  2363.             "psrlw            $8, %%mm1        \n\t"
    2364. 

  2364.             "psrlw            $8, %%mm2        \n\t"
    2365. 

  2365.             "psrlw            $8, %%mm3        \n\t"
    2366. 

  2366.             "packuswb      %%mm1, %%mm0        \n\t"
    2367. 

  2367.             "packuswb      %%mm3, %%mm2        \n\t"
    2368. 

  2368.             "movq          %%mm0, %%mm1        \n\t"
    2369. 

  2369.             "movq          %%mm2, %%mm3        \n\t"
    2370. 

  2370.             "psrlw            $8, %%mm0        \n\t"
    2371. 

  2371.             "psrlw            $8, %%mm2        \n\t"
    2372. 

  2372.             "pand          %%mm7, %%mm1        \n\t"
    2373. 

  2373.             "pand          %%mm7, %%mm3        \n\t"
    2374. 

  2374.             "packuswb      %%mm2, %%mm0        \n\t"
    2375. 

  2375.             "packuswb      %%mm3, %%mm1        \n\t"
    2376. 

  2376.             MOVNTQ"        %%mm0,- 7(%3, %0)   \n\t"
    2377. 

  2377.             MOVNTQ"        %%mm1,- 7(%2, %0)   \n\t"
    2378. 

  2378.             "add              $8, %0           \n\t"
    2379. 

  2379.             " js 1b                            \n\t"
    2380. 

  2380.             : "+r"(count)
    2381. 

  2381.             : "r"(src), "r"(dst0), "r"(dst1)
    2382. 

  2382.         );
    2383. 

  2383.         count -= 7;
    2384. 

  2384.     }
    2385. 

  2385.     src++;
    2386. 

  2386.     while(count<0) {
    2387. 

  2387.         dst0[count]= src[4*count+0];
    2388. 

  2388.         dst1[count]= src[4*count+2];
    2389. 

  2389.         count++;
    2390. 

  2390.     }
    2391. 

  2391. }
    2392. 

  2392. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2393. 

  2393. 

  2394. static void RENAME(extract_odd2avg)(const uint8_t *src0, const uint8_t *src1, uint8_t *dst0, uint8_t *dst1, x86_reg count)
    2395. 

  2395. {
    2396. 

  2396.     dst0 +=   count;
    2397. 

  2397.     dst1 +=   count;
    2398. 

  2398.     src0 += 4*count;
    2399. 

  2399.     src1 += 4*count;
    2400. 

  2400.     count= - count;
    2401. 

  2401. #ifdef PAVGB
    2402. 

  2402.     if(count <= -8) {
    2403. 

  2403.         count += 7;
    2404. 

  2404.         __asm__ volatile(
    2405. 

  2405.             "pcmpeqw        %%mm7, %%mm7        \n\t"
    2406. 

  2406.             "psrlw             $8, %%mm7        \n\t"
    2407. 

  2407.             "1:                                \n\t"
    2408. 

  2408.             "movq  -28(%1, %0, 4), %%mm0        \n\t"
    2409. 

  2409.             "movq  -20(%1, %0, 4), %%mm1        \n\t"
    2410. 

  2410.             "movq  -12(%1, %0, 4), %%mm2        \n\t"
    2411. 

  2411.             "movq   -4(%1, %0, 4), %%mm3        \n\t"
    2412. 

  2412.             PAVGB" -28(%2, %0, 4), %%mm0        \n\t"
    2413. 

  2413.             PAVGB" -20(%2, %0, 4), %%mm1        \n\t"
    2414. 

  2414.             PAVGB" -12(%2, %0, 4), %%mm2        \n\t"
    2415. 

  2415.             PAVGB" - 4(%2, %0, 4), %%mm3        \n\t"
    2416. 

  2416.             "psrlw             $8, %%mm0        \n\t"
    2417. 

  2417.             "psrlw             $8, %%mm1        \n\t"
    2418. 

  2418.             "psrlw             $8, %%mm2        \n\t"
    2419. 

  2419.             "psrlw             $8, %%mm3        \n\t"
    2420. 

  2420.             "packuswb       %%mm1, %%mm0        \n\t"
    2421. 

  2421.             "packuswb       %%mm3, %%mm2        \n\t"
    2422. 

  2422.             "movq           %%mm0, %%mm1        \n\t"
    2423. 

  2423.             "movq           %%mm2, %%mm3        \n\t"
    2424. 

  2424.             "psrlw             $8, %%mm0        \n\t"
    2425. 

  2425.             "psrlw             $8, %%mm2        \n\t"
    2426. 

  2426.             "pand           %%mm7, %%mm1        \n\t"
    2427. 

  2427.             "pand           %%mm7, %%mm3        \n\t"
    2428. 

  2428.             "packuswb       %%mm2, %%mm0        \n\t"
    2429. 

  2429.             "packuswb       %%mm3, %%mm1        \n\t"
    2430. 

  2430.             MOVNTQ"         %%mm0,- 7(%4, %0)   \n\t"
    2431. 

  2431.             MOVNTQ"         %%mm1,- 7(%3, %0)   \n\t"
    2432. 

  2432.             "add               $8, %0           \n\t"
    2433. 

  2433.             " js 1b                            \n\t"
    2434. 

  2434.             : "+r"(count)
    2435. 

  2435.             : "r"(src0), "r"(src1), "r"(dst0), "r"(dst1)
    2436. 

  2436.         );
    2437. 

  2437.         count -= 7;
    2438. 

  2438.     }
    2439. 

  2439. #endif
    2440. 

  2440.     src0++;
    2441. 

  2441.     src1++;
    2442. 

  2442.     while(count<0) {
    2443. 

  2443.         dst0[count]= (src0[4*count+0]+src1[4*count+0])>>1;
    2444. 

  2444.         dst1[count]= (src0[4*count+2]+src1[4*count+2])>>1;
    2445. 

  2445.         count++;
    2446. 

  2446.     }
    2447. 

  2447. }
    2448. 

  2448. 

  2449. static void RENAME(yuyvtoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
    2450. 

  2450.                                  int width, int height,
    2451. 

  2451.                                  int lumStride, int chromStride, int srcStride)
    2452. 

  2452. {
    2453. 

  2453.     int y;
    2454. 

  2454.     const int chromWidth= -((-width)>>1);
    2455. 

  2455. 

  2456.     for (y=0; y<height; y++) {
    2457. 

  2457.         RENAME(extract_even)(src, ydst, width);
    2458. 

  2458.         if(y&1) {
    2459. 

  2459.             RENAME(extract_odd2avg)(src-srcStride, src, udst, vdst, chromWidth);
    2460. 

  2460.             udst+= chromStride;
    2461. 

  2461.             vdst+= chromStride;
    2462. 

  2462.         }
    2463. 

  2463. 

  2464.         src += srcStride;
    2465. 

  2465.         ydst+= lumStride;
    2466. 

  2466.     }
    2467. 

  2467.     __asm__(
    2468. 

  2468.             EMMS"       \n\t"
    2469. 

  2469.             SFENCE"     \n\t"
    2470. 

  2470.             ::: "memory"
    2471. 

  2471.         );
    2472. 

  2472. }
    2473. 

  2473. 

  2474. #if !COMPILE_TEMPLATE_AMD3DNOW
    2475. 

  2475. static void RENAME(yuyvtoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
    2476. 

  2476.                                  int width, int height,
    2477. 

  2477.                                  int lumStride, int chromStride, int srcStride)
    2478. 

  2478. {
    2479. 

  2479.     int y;
    2480. 

  2480.     const int chromWidth= -((-width)>>1);
    2481. 

  2481. 

  2482.     for (y=0; y<height; y++) {
    2483. 

  2483.         RENAME(extract_even)(src, ydst, width);
    2484. 

  2484.         RENAME(extract_odd2)(src, udst, vdst, chromWidth);
    2485. 

  2485. 

  2486.         src += srcStride;
    2487. 

  2487.         ydst+= lumStride;
    2488. 

  2488.         udst+= chromStride;
    2489. 

  2489.         vdst+= chromStride;
    2490. 

  2490.     }
    2491. 

  2491.     __asm__(
    2492. 

  2492.             EMMS"       \n\t"
    2493. 

  2493.             SFENCE"     \n\t"
    2494. 

  2494.             ::: "memory"
    2495. 

  2495.         );
    2496. 

  2496. }
    2497. 

  2497. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2498. 

  2498. 

  2499. static void RENAME(uyvytoyuv420)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
    2500. 

  2500.                                  int width, int height,
    2501. 

  2501.                                  int lumStride, int chromStride, int srcStride)
    2502. 

  2502. {
    2503. 

  2503.     int y;
    2504. 

  2504.     const int chromWidth= -((-width)>>1);
    2505. 

  2505. 

  2506.     for (y=0; y<height; y++) {
    2507. 

  2507.         RENAME(extract_even)(src+1, ydst, width);
    2508. 

  2508.         if(y&1) {
    2509. 

  2509.             RENAME(extract_even2avg)(src-srcStride, src, udst, vdst, chromWidth);
    2510. 

  2510.             udst+= chromStride;
    2511. 

  2511.             vdst+= chromStride;
    2512. 

  2512.         }
    2513. 

  2513. 

  2514.         src += srcStride;
    2515. 

  2515.         ydst+= lumStride;
    2516. 

  2516.     }
    2517. 

  2517.     __asm__(
    2518. 

  2518.             EMMS"       \n\t"
    2519. 

  2519.             SFENCE"     \n\t"
    2520. 

  2520.             ::: "memory"
    2521. 

  2521.         );
    2522. 

  2522. }
    2523. 

  2523. 

  2524. #if !COMPILE_TEMPLATE_AMD3DNOW
    2525. 

  2525. static void RENAME(uyvytoyuv422)(uint8_t *ydst, uint8_t *udst, uint8_t *vdst, const uint8_t *src,
    2526. 

  2526.                                  int width, int height,
    2527. 

  2527.                                  int lumStride, int chromStride, int srcStride)
    2528. 

  2528. {
    2529. 

  2529.     int y;
    2530. 

  2530.     const int chromWidth= -((-width)>>1);
    2531. 

  2531. 

  2532.     for (y=0; y<height; y++) {
    2533. 

  2533.         RENAME(extract_even)(src+1, ydst, width);
    2534. 

  2534.         RENAME(extract_even2)(src, udst, vdst, chromWidth);
    2535. 

  2535. 

  2536.         src += srcStride;
    2537. 

  2537.         ydst+= lumStride;
    2538. 

  2538.         udst+= chromStride;
    2539. 

  2539.         vdst+= chromStride;
    2540. 

  2540.     }
    2541. 

  2541.     __asm__(
    2542. 

  2542.             EMMS"       \n\t"
    2543. 

  2543.             SFENCE"     \n\t"
    2544. 

  2544.             ::: "memory"
    2545. 

  2545.         );
    2546. 

  2546. }
    2547. 

  2547. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2548. 

  2548. #endif /* !COMPILE_TEMPLATE_SSE2 */
    2549. 

  2549. 

  2550. static inline void RENAME(rgb2rgb_init)(void)
    2551. 

  2551. {
    2552. 

  2552. #if !COMPILE_TEMPLATE_SSE2
    2553. 

  2553. #if !COMPILE_TEMPLATE_AMD3DNOW
    2554. 

  2554.     rgb15to16          = RENAME(rgb15to16);
    2555. 

  2555.     rgb15tobgr24       = RENAME(rgb15tobgr24);
    2556. 

  2556.     rgb15to32          = RENAME(rgb15to32);
    2557. 

  2557.     rgb16tobgr24       = RENAME(rgb16tobgr24);
    2558. 

  2558.     rgb16to32          = RENAME(rgb16to32);
    2559. 

  2559.     rgb16to15          = RENAME(rgb16to15);
    2560. 

  2560.     rgb24tobgr16       = RENAME(rgb24tobgr16);
    2561. 

  2561.     rgb24tobgr15       = RENAME(rgb24tobgr15);
    2562. 

  2562.     rgb24tobgr32       = RENAME(rgb24tobgr32);
    2563. 

  2563.     rgb32to16          = RENAME(rgb32to16);
    2564. 

  2564.     rgb32to15          = RENAME(rgb32to15);
    2565. 

  2565.     rgb32tobgr24       = RENAME(rgb32tobgr24);
    2566. 

  2566.     rgb24to15          = RENAME(rgb24to15);
    2567. 

  2567.     rgb24to16          = RENAME(rgb24to16);
    2568. 

  2568.     rgb24tobgr24       = RENAME(rgb24tobgr24);
    2569. 

  2569.     shuffle_bytes_2103 = RENAME(shuffle_bytes_2103);
    2570. 

  2570.     rgb32tobgr16       = RENAME(rgb32tobgr16);
    2571. 

  2571.     rgb32tobgr15       = RENAME(rgb32tobgr15);
    2572. 

  2572.     yv12toyuy2         = RENAME(yv12toyuy2);
    2573. 

  2573.     yv12touyvy         = RENAME(yv12touyvy);
    2574. 

  2574.     yuv422ptoyuy2      = RENAME(yuv422ptoyuy2);
    2575. 

  2575.     yuv422ptouyvy      = RENAME(yuv422ptouyvy);
    2576. 

  2576.     yuy2toyv12         = RENAME(yuy2toyv12);
    2577. 

  2577.     vu9_to_vu12        = RENAME(vu9_to_vu12);
    2578. 

  2578.     yvu9_to_yuy2       = RENAME(yvu9_to_yuy2);
    2579. 

  2579.     uyvytoyuv422       = RENAME(uyvytoyuv422);
    2580. 

  2580.     yuyvtoyuv422       = RENAME(yuyvtoyuv422);
    2581. 

  2581. #endif /* !COMPILE_TEMPLATE_SSE2 */
    2582. 

  2582. 

  2583. #if COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW
    2584. 

  2584.     planar2x           = RENAME(planar2x);
    2585. 

  2585. #endif /* COMPILE_TEMPLATE_MMX2 || COMPILE_TEMPLATE_AMD3DNOW */
    2586. 

  2586.     rgb24toyv12        = RENAME(rgb24toyv12);
    2587. 

  2587. 

  2588.     yuyvtoyuv420       = RENAME(yuyvtoyuv420);
    2589. 

  2589.     uyvytoyuv420       = RENAME(uyvytoyuv420);
    2590. 

  2590. #endif /* COMPILE_TEMPLATE_SSE2 */
    2591. 

  2591. 

  2592. #if !COMPILE_TEMPLATE_AMD3DNOW
    2593. 

  2593.     interleaveBytes    = RENAME(interleaveBytes);
    2594. 

  2594. #endif /* !COMPILE_TEMPLATE_AMD3DNOW */
    2595. 

  2595. }
    2596. 

  2596.