PageRenderTime 62ms CodeModel.GetById 28ms RepoModel.GetById 1ms app.codeStats 0ms

/lib/ffmpeg/libswscale/x86/rgb2rgb_template.c

https://bitbucket.org/bgiorgini/xbmc
C | 2595 lines | 2386 code | 131 blank | 78 comment | 78 complexity | 69133a8698389e86b7d9121a15e4a22d MD5 | raw file
Possible License(s): GPL-3.0, CC-BY-SA-3.0, BSD-3-Clause, GPL-2.0, LGPL-3.0, 0BSD, LGPL-2.0, AGPL-1.0, LGPL-2.1

Large files files are truncated, but you can click here to view the full file

    

  1. /*
    2. 

  2.  * software RGB to RGB converter
    3. 

  3.  * pluralize by software PAL8 to RGB converter
    4. 

  4.  *              software YUV to YUV converter
    5. 

  5.  *              software YUV to RGB converter
    6. 

  6.  * Written by Nick Kurshev.
    7. 

  7.  * palette & YUV & runtime CPU stuff by Michael (michaelni@gmx.at)
    8. 

  8.  * lot of big-endian byte order fixes by Alex Beregszaszi
    9. 

  9.  *
    10. 

  10.  * This file is part of FFmpeg.
    11. 

  11.  *
    12. 

  12.  * FFmpeg is free software; you can redistribute it and/or
    13. 

  13.  * modify it under the terms of the GNU Lesser General Public
    14. 

  14.  * License as published by the Free Software Foundation; either
    15. 

  15.  * version 2.1 of the License, or (at your option) any later version.
    16. 

  16.  *
    17. 

  17.  * FFmpeg is distributed in the hope that it will be useful,
    18. 

  18.  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    19. 

  19.  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    20. 

  20.  * Lesser General Public License for more details.
    21. 

  21.  *
    22. 

  22.  * You should have received a copy of the GNU Lesser General Public
    23. 

  23.  * License along with FFmpeg; if not, write to the Free Software
    24. 

  24.  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
    25. 

  25.  */
    26. 

  26. 

  27. #include <stddef.h>
    28. 

  28. 

  29. #undef PREFETCH
    30. 

  30. #undef MOVNTQ
    31. 

  31. #undef EMMS
    32. 

  32. #undef SFENCE
    33. 

  33. #undef PAVGB
    34. 

  34. 

  35. #if COMPILE_TEMPLATE_AMD3DNOW
    36. 

  36. #define PREFETCH  "prefetch"
    37. 

  37. #define PAVGB     "pavgusb"
    38. 

  38. #elif COMPILE_TEMPLATE_MMX2
    39. 

  39. #define PREFETCH "prefetchnta"
    40. 

  40. #define PAVGB     "pavgb"
    41. 

  41. #else
    42. 

  42. #define PREFETCH  " # nop"
    43. 

  43. #endif
    44. 

  44. 

  45. #if COMPILE_TEMPLATE_AMD3DNOW
    46. 

  46. /* On K6 femms is faster than emms. On K7 femms is directly mapped to emms. */
    47. 

  47. #define EMMS     "femms"
    48. 

  48. #else
    49. 

  49. #define EMMS     "emms"
    50. 

  50. #endif
    51. 

  51. 

  52. #if COMPILE_TEMPLATE_MMX2
    53. 

  53. #define MOVNTQ "movntq"
    54. 

  54. #define SFENCE "sfence"
    55. 

  55. #else
    56. 

  56. #define MOVNTQ "movq"
    57. 

  57. #define SFENCE " # nop"
    58. 

  58. #endif
    59. 

  59. 

  60. #if !COMPILE_TEMPLATE_SSE2
    61. 

  61. 

  62. #if !COMPILE_TEMPLATE_AMD3DNOW
    63. 

  63. 

  64. static inline void RENAME(rgb24tobgr32)(const uint8_t *src, uint8_t *dst, int src_size)
    65. 

  65. {
    66. 

  66.     uint8_t *dest = dst;
    67. 

  67.     const uint8_t *s = src;
    68. 

  68.     const uint8_t *end;
    69. 

  69.     const uint8_t *mm_end;
    70. 

  70.     end = s + src_size;
    71. 

  71.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    72. 

  72.     mm_end = end - 23;
    73. 

  73.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask32a):"memory");
    74. 

  74.     while (s < mm_end) {
    75. 

  75.         __asm__ volatile(
    76. 

  76.             PREFETCH"    32%1           \n\t"
    77. 

  77.             "movd          %1, %%mm0    \n\t"
    78. 

  78.             "punpckldq    3%1, %%mm0    \n\t"
    79. 

  79.             "movd         6%1, %%mm1    \n\t"
    80. 

  80.             "punpckldq    9%1, %%mm1    \n\t"
    81. 

  81.             "movd        12%1, %%mm2    \n\t"
    82. 

  82.             "punpckldq   15%1, %%mm2    \n\t"
    83. 

  83.             "movd        18%1, %%mm3    \n\t"
    84. 

  84.             "punpckldq   21%1, %%mm3    \n\t"
    85. 

  85.             "por        %%mm7, %%mm0    \n\t"
    86. 

  86.             "por        %%mm7, %%mm1    \n\t"
    87. 

  87.             "por        %%mm7, %%mm2    \n\t"
    88. 

  88.             "por        %%mm7, %%mm3    \n\t"
    89. 

  89.             MOVNTQ"     %%mm0,   %0     \n\t"
    90. 

  90.             MOVNTQ"     %%mm1,  8%0     \n\t"
    91. 

  91.             MOVNTQ"     %%mm2, 16%0     \n\t"
    92. 

  92.             MOVNTQ"     %%mm3, 24%0"
    93. 

  93.             :"=m"(*dest)
    94. 

  94.             :"m"(*s)
    95. 

  95.             :"memory");
    96. 

  96.         dest += 32;
    97. 

  97.         s += 24;
    98. 

  98.     }
    99. 

  99.     __asm__ volatile(SFENCE:::"memory");
    100. 

  100.     __asm__ volatile(EMMS:::"memory");
    101. 

  101.     while (s < end) {
    102. 

  102.         *dest++ = *s++;
    103. 

  103.         *dest++ = *s++;
    104. 

  104.         *dest++ = *s++;
    105. 

  105.         *dest++ = 255;
    106. 

  106.     }
    107. 

  107. }
    108. 

  108. 

  109. #define STORE_BGR24_MMX \
    110. 

  110.             "psrlq         $8, %%mm2    \n\t" \
    111. 

  111.             "psrlq         $8, %%mm3    \n\t" \
    112. 

  112.             "psrlq         $8, %%mm6    \n\t" \
    113. 

  113.             "psrlq         $8, %%mm7    \n\t" \
    114. 

  114.             "pand "MANGLE(mask24l)", %%mm0\n\t" \
    115. 

  115.             "pand "MANGLE(mask24l)", %%mm1\n\t" \
    116. 

  116.             "pand "MANGLE(mask24l)", %%mm4\n\t" \
    117. 

  117.             "pand "MANGLE(mask24l)", %%mm5\n\t" \
    118. 

  118.             "pand "MANGLE(mask24h)", %%mm2\n\t" \
    119. 

  119.             "pand "MANGLE(mask24h)", %%mm3\n\t" \
    120. 

  120.             "pand "MANGLE(mask24h)", %%mm6\n\t" \
    121. 

  121.             "pand "MANGLE(mask24h)", %%mm7\n\t" \
    122. 

  122.             "por        %%mm2, %%mm0    \n\t" \
    123. 

  123.             "por        %%mm3, %%mm1    \n\t" \
    124. 

  124.             "por        %%mm6, %%mm4    \n\t" \
    125. 

  125.             "por        %%mm7, %%mm5    \n\t" \
    126. 

  126.  \
    127. 

  127.             "movq       %%mm1, %%mm2    \n\t" \
    128. 

  128.             "movq       %%mm4, %%mm3    \n\t" \
    129. 

  129.             "psllq        $48, %%mm2    \n\t" \
    130. 

  130.             "psllq        $32, %%mm3    \n\t" \
    131. 

  131.             "pand "MANGLE(mask24hh)", %%mm2\n\t" \
    132. 

  132.             "pand "MANGLE(mask24hhh)", %%mm3\n\t" \
    133. 

  133.             "por        %%mm2, %%mm0    \n\t" \
    134. 

  134.             "psrlq        $16, %%mm1    \n\t" \
    135. 

  135.             "psrlq        $32, %%mm4    \n\t" \
    136. 

  136.             "psllq        $16, %%mm5    \n\t" \
    137. 

  137.             "por        %%mm3, %%mm1    \n\t" \
    138. 

  138.             "pand  "MANGLE(mask24hhhh)", %%mm5\n\t" \
    139. 

  139.             "por        %%mm5, %%mm4    \n\t" \
    140. 

  140.  \
    141. 

  141.             MOVNTQ"     %%mm0,   %0     \n\t" \
    142. 

  142.             MOVNTQ"     %%mm1,  8%0     \n\t" \
    143. 

  143.             MOVNTQ"     %%mm4, 16%0"
    144. 

  144. 

  145. 

  146. static inline void RENAME(rgb32tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    147. 

  147. {
    148. 

  148.     uint8_t *dest = dst;
    149. 

  149.     const uint8_t *s = src;
    150. 

  150.     const uint8_t *end;
    151. 

  151.     const uint8_t *mm_end;
    152. 

  152.     end = s + src_size;
    153. 

  153.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    154. 

  154.     mm_end = end - 31;
    155. 

  155.     while (s < mm_end) {
    156. 

  156.         __asm__ volatile(
    157. 

  157.             PREFETCH"    32%1           \n\t"
    158. 

  158.             "movq          %1, %%mm0    \n\t"
    159. 

  159.             "movq         8%1, %%mm1    \n\t"
    160. 

  160.             "movq        16%1, %%mm4    \n\t"
    161. 

  161.             "movq        24%1, %%mm5    \n\t"
    162. 

  162.             "movq       %%mm0, %%mm2    \n\t"
    163. 

  163.             "movq       %%mm1, %%mm3    \n\t"
    164. 

  164.             "movq       %%mm4, %%mm6    \n\t"
    165. 

  165.             "movq       %%mm5, %%mm7    \n\t"
    166. 

  166.             STORE_BGR24_MMX
    167. 

  167.             :"=m"(*dest)
    168. 

  168.             :"m"(*s)
    169. 

  169.             :"memory");
    170. 

  170.         dest += 24;
    171. 

  171.         s += 32;
    172. 

  172.     }
    173. 

  173.     __asm__ volatile(SFENCE:::"memory");
    174. 

  174.     __asm__ volatile(EMMS:::"memory");
    175. 

  175.     while (s < end) {
    176. 

  176.         *dest++ = *s++;
    177. 

  177.         *dest++ = *s++;
    178. 

  178.         *dest++ = *s++;
    179. 

  179.         s++;
    180. 

  180.     }
    181. 

  181. }
    182. 

  182. 

  183. /*
    184. 

  184.  original by Strepto/Astral
    185. 

  185.  ported to gcc & bugfixed: A'rpi
    186. 

  186.  MMX2, 3DNOW optimization by Nick Kurshev
    187. 

  187.  32-bit C version, and and&add trick by Michael Niedermayer
    188. 

  188. */
    189. 

  189. static inline void RENAME(rgb15to16)(const uint8_t *src, uint8_t *dst, int src_size)
    190. 

  190. {
    191. 

  191.     register const uint8_t* s=src;
    192. 

  192.     register uint8_t* d=dst;
    193. 

  193.     register const uint8_t *end;
    194. 

  194.     const uint8_t *mm_end;
    195. 

  195.     end = s + src_size;
    196. 

  196.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));
    197. 

  197.     __asm__ volatile("movq        %0, %%mm4"::"m"(mask15s));
    198. 

  198.     mm_end = end - 15;
    199. 

  199.     while (s<mm_end) {
    200. 

  200.         __asm__ volatile(
    201. 

  201.             PREFETCH"  32%1         \n\t"
    202. 

  202.             "movq        %1, %%mm0  \n\t"
    203. 

  203.             "movq       8%1, %%mm2  \n\t"
    204. 

  204.             "movq     %%mm0, %%mm1  \n\t"
    205. 

  205.             "movq     %%mm2, %%mm3  \n\t"
    206. 

  206.             "pand     %%mm4, %%mm0  \n\t"
    207. 

  207.             "pand     %%mm4, %%mm2  \n\t"
    208. 

  208.             "paddw    %%mm1, %%mm0  \n\t"
    209. 

  209.             "paddw    %%mm3, %%mm2  \n\t"
    210. 

  210.             MOVNTQ"   %%mm0,  %0    \n\t"
    211. 

  211.             MOVNTQ"   %%mm2, 8%0"
    212. 

  212.             :"=m"(*d)
    213. 

  213.             :"m"(*s)
    214. 

  214.         );
    215. 

  215.         d+=16;
    216. 

  216.         s+=16;
    217. 

  217.     }
    218. 

  218.     __asm__ volatile(SFENCE:::"memory");
    219. 

  219.     __asm__ volatile(EMMS:::"memory");
    220. 

  220.     mm_end = end - 3;
    221. 

  221.     while (s < mm_end) {
    222. 

  222.         register unsigned x= *((const uint32_t *)s);
    223. 

  223.         *((uint32_t *)d) = (x&0x7FFF7FFF) + (x&0x7FE07FE0);
    224. 

  224.         d+=4;
    225. 

  225.         s+=4;
    226. 

  226.     }
    227. 

  227.     if (s < end) {
    228. 

  228.         register unsigned short x= *((const uint16_t *)s);
    229. 

  229.         *((uint16_t *)d) = (x&0x7FFF) + (x&0x7FE0);
    230. 

  230.     }
    231. 

  231. }
    232. 

  232. 

  233. static inline void RENAME(rgb16to15)(const uint8_t *src, uint8_t *dst, int src_size)
    234. 

  234. {
    235. 

  235.     register const uint8_t* s=src;
    236. 

  236.     register uint8_t* d=dst;
    237. 

  237.     register const uint8_t *end;
    238. 

  238.     const uint8_t *mm_end;
    239. 

  239.     end = s + src_size;
    240. 

  240.     __asm__ volatile(PREFETCH"    %0"::"m"(*s));
    241. 

  241.     __asm__ volatile("movq        %0, %%mm7"::"m"(mask15rg));
    242. 

  242.     __asm__ volatile("movq        %0, %%mm6"::"m"(mask15b));
    243. 

  243.     mm_end = end - 15;
    244. 

  244.     while (s<mm_end) {
    245. 

  245.         __asm__ volatile(
    246. 

  246.             PREFETCH"  32%1         \n\t"
    247. 

  247.             "movq        %1, %%mm0  \n\t"
    248. 

  248.             "movq       8%1, %%mm2  \n\t"
    249. 

  249.             "movq     %%mm0, %%mm1  \n\t"
    250. 

  250.             "movq     %%mm2, %%mm3  \n\t"
    251. 

  251.             "psrlq       $1, %%mm0  \n\t"
    252. 

  252.             "psrlq       $1, %%mm2  \n\t"
    253. 

  253.             "pand     %%mm7, %%mm0  \n\t"
    254. 

  254.             "pand     %%mm7, %%mm2  \n\t"
    255. 

  255.             "pand     %%mm6, %%mm1  \n\t"
    256. 

  256.             "pand     %%mm6, %%mm3  \n\t"
    257. 

  257.             "por      %%mm1, %%mm0  \n\t"
    258. 

  258.             "por      %%mm3, %%mm2  \n\t"
    259. 

  259.             MOVNTQ"   %%mm0,  %0    \n\t"
    260. 

  260.             MOVNTQ"   %%mm2, 8%0"
    261. 

  261.             :"=m"(*d)
    262. 

  262.             :"m"(*s)
    263. 

  263.         );
    264. 

  264.         d+=16;
    265. 

  265.         s+=16;
    266. 

  266.     }
    267. 

  267.     __asm__ volatile(SFENCE:::"memory");
    268. 

  268.     __asm__ volatile(EMMS:::"memory");
    269. 

  269.     mm_end = end - 3;
    270. 

  270.     while (s < mm_end) {
    271. 

  271.         register uint32_t x= *((const uint32_t*)s);
    272. 

  272.         *((uint32_t *)d) = ((x>>1)&0x7FE07FE0) | (x&0x001F001F);
    273. 

  273.         s+=4;
    274. 

  274.         d+=4;
    275. 

  275.     }
    276. 

  276.     if (s < end) {
    277. 

  277.         register uint16_t x= *((const uint16_t*)s);
    278. 

  278.         *((uint16_t *)d) = ((x>>1)&0x7FE0) | (x&0x001F);
    279. 

  279.     }
    280. 

  280. }
    281. 

  281. 

  282. static inline void RENAME(rgb32to16)(const uint8_t *src, uint8_t *dst, int src_size)
    283. 

  283. {
    284. 

  284.     const uint8_t *s = src;
    285. 

  285.     const uint8_t *end;
    286. 

  286.     const uint8_t *mm_end;
    287. 

  287.     uint16_t *d = (uint16_t *)dst;
    288. 

  288.     end = s + src_size;
    289. 

  289.     mm_end = end - 15;
    290. 

  290. #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster)
    291. 

  291.     __asm__ volatile(
    292. 

  292.         "movq           %3, %%mm5   \n\t"
    293. 

  293.         "movq           %4, %%mm6   \n\t"
    294. 

  294.         "movq           %5, %%mm7   \n\t"
    295. 

  295.         "jmp 2f                     \n\t"
    296. 

  296.         ".p2align        4          \n\t"
    297. 

  297.         "1:                         \n\t"
    298. 

  298.         PREFETCH"   32(%1)          \n\t"
    299. 

  299.         "movd         (%1), %%mm0   \n\t"
    300. 

  300.         "movd        4(%1), %%mm3   \n\t"
    301. 

  301.         "punpckldq   8(%1), %%mm0   \n\t"
    302. 

  302.         "punpckldq  12(%1), %%mm3   \n\t"
    303. 

  303.         "movq        %%mm0, %%mm1   \n\t"
    304. 

  304.         "movq        %%mm3, %%mm4   \n\t"
    305. 

  305.         "pand        %%mm6, %%mm0   \n\t"
    306. 

  306.         "pand        %%mm6, %%mm3   \n\t"
    307. 

  307.         "pmaddwd     %%mm7, %%mm0   \n\t"
    308. 

  308.         "pmaddwd     %%mm7, %%mm3   \n\t"
    309. 

  309.         "pand        %%mm5, %%mm1   \n\t"
    310. 

  310.         "pand        %%mm5, %%mm4   \n\t"
    311. 

  311.         "por         %%mm1, %%mm0   \n\t"
    312. 

  312.         "por         %%mm4, %%mm3   \n\t"
    313. 

  313.         "psrld          $5, %%mm0   \n\t"
    314. 

  314.         "pslld         $11, %%mm3   \n\t"
    315. 

  315.         "por         %%mm3, %%mm0   \n\t"
    316. 

  316.         MOVNTQ"      %%mm0, (%0)    \n\t"
    317. 

  317.         "add           $16,  %1     \n\t"
    318. 

  318.         "add            $8,  %0     \n\t"
    319. 

  319.         "2:                         \n\t"
    320. 

  320.         "cmp            %2,  %1     \n\t"
    321. 

  321.         " jb            1b          \n\t"
    322. 

  322.         : "+r" (d), "+r"(s)
    323. 

  323.         : "r" (mm_end), "m" (mask3216g), "m" (mask3216br), "m" (mul3216)
    324. 

  324.     );
    325. 

  325. #else
    326. 

  326.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    327. 

  327.     __asm__ volatile(
    328. 

  328.         "movq    %0, %%mm7    \n\t"
    329. 

  329.         "movq    %1, %%mm6    \n\t"
    330. 

  330.         ::"m"(red_16mask),"m"(green_16mask));
    331. 

  331.     while (s < mm_end) {
    332. 

  332.         __asm__ volatile(
    333. 

  333.             PREFETCH"    32%1           \n\t"
    334. 

  334.             "movd          %1, %%mm0    \n\t"
    335. 

  335.             "movd         4%1, %%mm3    \n\t"
    336. 

  336.             "punpckldq    8%1, %%mm0    \n\t"
    337. 

  337.             "punpckldq   12%1, %%mm3    \n\t"
    338. 

  338.             "movq       %%mm0, %%mm1    \n\t"
    339. 

  339.             "movq       %%mm0, %%mm2    \n\t"
    340. 

  340.             "movq       %%mm3, %%mm4    \n\t"
    341. 

  341.             "movq       %%mm3, %%mm5    \n\t"
    342. 

  342.             "psrlq         $3, %%mm0    \n\t"
    343. 

  343.             "psrlq         $3, %%mm3    \n\t"
    344. 

  344.             "pand          %2, %%mm0    \n\t"
    345. 

  345.             "pand          %2, %%mm3    \n\t"
    346. 

  346.             "psrlq         $5, %%mm1    \n\t"
    347. 

  347.             "psrlq         $5, %%mm4    \n\t"
    348. 

  348.             "pand       %%mm6, %%mm1    \n\t"
    349. 

  349.             "pand       %%mm6, %%mm4    \n\t"
    350. 

  350.             "psrlq         $8, %%mm2    \n\t"
    351. 

  351.             "psrlq         $8, %%mm5    \n\t"
    352. 

  352.             "pand       %%mm7, %%mm2    \n\t"
    353. 

  353.             "pand       %%mm7, %%mm5    \n\t"
    354. 

  354.             "por        %%mm1, %%mm0    \n\t"
    355. 

  355.             "por        %%mm4, %%mm3    \n\t"
    356. 

  356.             "por        %%mm2, %%mm0    \n\t"
    357. 

  357.             "por        %%mm5, %%mm3    \n\t"
    358. 

  358.             "psllq        $16, %%mm3    \n\t"
    359. 

  359.             "por        %%mm3, %%mm0    \n\t"
    360. 

  360.             MOVNTQ"     %%mm0, %0       \n\t"
    361. 

  361.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    362. 

  362.         d += 4;
    363. 

  363.         s += 16;
    364. 

  364.     }
    365. 

  365. #endif
    366. 

  366.     __asm__ volatile(SFENCE:::"memory");
    367. 

  367.     __asm__ volatile(EMMS:::"memory");
    368. 

  368.     while (s < end) {
    369. 

  369.         register int rgb = *(const uint32_t*)s; s += 4;
    370. 

  370.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>8);
    371. 

  371.     }
    372. 

  372. }
    373. 

  373. 

  374. static inline void RENAME(rgb32tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
    375. 

  375. {
    376. 

  376.     const uint8_t *s = src;
    377. 

  377.     const uint8_t *end;
    378. 

  378.     const uint8_t *mm_end;
    379. 

  379.     uint16_t *d = (uint16_t *)dst;
    380. 

  380.     end = s + src_size;
    381. 

  381.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    382. 

  382.     __asm__ volatile(
    383. 

  383.         "movq          %0, %%mm7    \n\t"
    384. 

  384.         "movq          %1, %%mm6    \n\t"
    385. 

  385.         ::"m"(red_16mask),"m"(green_16mask));
    386. 

  386.     mm_end = end - 15;
    387. 

  387.     while (s < mm_end) {
    388. 

  388.         __asm__ volatile(
    389. 

  389.             PREFETCH"    32%1           \n\t"
    390. 

  390.             "movd          %1, %%mm0    \n\t"
    391. 

  391.             "movd         4%1, %%mm3    \n\t"
    392. 

  392.             "punpckldq    8%1, %%mm0    \n\t"
    393. 

  393.             "punpckldq   12%1, %%mm3    \n\t"
    394. 

  394.             "movq       %%mm0, %%mm1    \n\t"
    395. 

  395.             "movq       %%mm0, %%mm2    \n\t"
    396. 

  396.             "movq       %%mm3, %%mm4    \n\t"
    397. 

  397.             "movq       %%mm3, %%mm5    \n\t"
    398. 

  398.             "psllq         $8, %%mm0    \n\t"
    399. 

  399.             "psllq         $8, %%mm3    \n\t"
    400. 

  400.             "pand       %%mm7, %%mm0    \n\t"
    401. 

  401.             "pand       %%mm7, %%mm3    \n\t"
    402. 

  402.             "psrlq         $5, %%mm1    \n\t"
    403. 

  403.             "psrlq         $5, %%mm4    \n\t"
    404. 

  404.             "pand       %%mm6, %%mm1    \n\t"
    405. 

  405.             "pand       %%mm6, %%mm4    \n\t"
    406. 

  406.             "psrlq        $19, %%mm2    \n\t"
    407. 

  407.             "psrlq        $19, %%mm5    \n\t"
    408. 

  408.             "pand          %2, %%mm2    \n\t"
    409. 

  409.             "pand          %2, %%mm5    \n\t"
    410. 

  410.             "por        %%mm1, %%mm0    \n\t"
    411. 

  411.             "por        %%mm4, %%mm3    \n\t"
    412. 

  412.             "por        %%mm2, %%mm0    \n\t"
    413. 

  413.             "por        %%mm5, %%mm3    \n\t"
    414. 

  414.             "psllq        $16, %%mm3    \n\t"
    415. 

  415.             "por        %%mm3, %%mm0    \n\t"
    416. 

  416.             MOVNTQ"     %%mm0, %0       \n\t"
    417. 

  417.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    418. 

  418.         d += 4;
    419. 

  419.         s += 16;
    420. 

  420.     }
    421. 

  421.     __asm__ volatile(SFENCE:::"memory");
    422. 

  422.     __asm__ volatile(EMMS:::"memory");
    423. 

  423.     while (s < end) {
    424. 

  424.         register int rgb = *(const uint32_t*)s; s += 4;
    425. 

  425.         *d++ = ((rgb&0xF8)<<8) + ((rgb&0xFC00)>>5) + ((rgb&0xF80000)>>19);
    426. 

  426.     }
    427. 

  427. }
    428. 

  428. 

  429. static inline void RENAME(rgb32to15)(const uint8_t *src, uint8_t *dst, int src_size)
    430. 

  430. {
    431. 

  431.     const uint8_t *s = src;
    432. 

  432.     const uint8_t *end;
    433. 

  433.     const uint8_t *mm_end;
    434. 

  434.     uint16_t *d = (uint16_t *)dst;
    435. 

  435.     end = s + src_size;
    436. 

  436.     mm_end = end - 15;
    437. 

  437. #if 1 //is faster only if multiplies are reasonably fast (FIXME figure out on which CPUs this is faster, on Athlon it is slightly faster)
    438. 

  438.     __asm__ volatile(
    439. 

  439.         "movq           %3, %%mm5   \n\t"
    440. 

  440.         "movq           %4, %%mm6   \n\t"
    441. 

  441.         "movq           %5, %%mm7   \n\t"
    442. 

  442.         "jmp            2f          \n\t"
    443. 

  443.         ".p2align        4          \n\t"
    444. 

  444.         "1:                         \n\t"
    445. 

  445.         PREFETCH"   32(%1)          \n\t"
    446. 

  446.         "movd         (%1), %%mm0   \n\t"
    447. 

  447.         "movd        4(%1), %%mm3   \n\t"
    448. 

  448.         "punpckldq   8(%1), %%mm0   \n\t"
    449. 

  449.         "punpckldq  12(%1), %%mm3   \n\t"
    450. 

  450.         "movq        %%mm0, %%mm1   \n\t"
    451. 

  451.         "movq        %%mm3, %%mm4   \n\t"
    452. 

  452.         "pand        %%mm6, %%mm0   \n\t"
    453. 

  453.         "pand        %%mm6, %%mm3   \n\t"
    454. 

  454.         "pmaddwd     %%mm7, %%mm0   \n\t"
    455. 

  455.         "pmaddwd     %%mm7, %%mm3   \n\t"
    456. 

  456.         "pand        %%mm5, %%mm1   \n\t"
    457. 

  457.         "pand        %%mm5, %%mm4   \n\t"
    458. 

  458.         "por         %%mm1, %%mm0   \n\t"
    459. 

  459.         "por         %%mm4, %%mm3   \n\t"
    460. 

  460.         "psrld          $6, %%mm0   \n\t"
    461. 

  461.         "pslld         $10, %%mm3   \n\t"
    462. 

  462.         "por         %%mm3, %%mm0   \n\t"
    463. 

  463.         MOVNTQ"      %%mm0, (%0)    \n\t"
    464. 

  464.         "add           $16,  %1     \n\t"
    465. 

  465.         "add            $8,  %0     \n\t"
    466. 

  466.         "2:                         \n\t"
    467. 

  467.         "cmp            %2,  %1     \n\t"
    468. 

  468.         " jb            1b          \n\t"
    469. 

  469.         : "+r" (d), "+r"(s)
    470. 

  470.         : "r" (mm_end), "m" (mask3215g), "m" (mask3216br), "m" (mul3215)
    471. 

  471.     );
    472. 

  472. #else
    473. 

  473.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    474. 

  474.     __asm__ volatile(
    475. 

  475.         "movq          %0, %%mm7    \n\t"
    476. 

  476.         "movq          %1, %%mm6    \n\t"
    477. 

  477.         ::"m"(red_15mask),"m"(green_15mask));
    478. 

  478.     while (s < mm_end) {
    479. 

  479.         __asm__ volatile(
    480. 

  480.             PREFETCH"    32%1           \n\t"
    481. 

  481.             "movd          %1, %%mm0    \n\t"
    482. 

  482.             "movd         4%1, %%mm3    \n\t"
    483. 

  483.             "punpckldq    8%1, %%mm0    \n\t"
    484. 

  484.             "punpckldq   12%1, %%mm3    \n\t"
    485. 

  485.             "movq       %%mm0, %%mm1    \n\t"
    486. 

  486.             "movq       %%mm0, %%mm2    \n\t"
    487. 

  487.             "movq       %%mm3, %%mm4    \n\t"
    488. 

  488.             "movq       %%mm3, %%mm5    \n\t"
    489. 

  489.             "psrlq         $3, %%mm0    \n\t"
    490. 

  490.             "psrlq         $3, %%mm3    \n\t"
    491. 

  491.             "pand          %2, %%mm0    \n\t"
    492. 

  492.             "pand          %2, %%mm3    \n\t"
    493. 

  493.             "psrlq         $6, %%mm1    \n\t"
    494. 

  494.             "psrlq         $6, %%mm4    \n\t"
    495. 

  495.             "pand       %%mm6, %%mm1    \n\t"
    496. 

  496.             "pand       %%mm6, %%mm4    \n\t"
    497. 

  497.             "psrlq         $9, %%mm2    \n\t"
    498. 

  498.             "psrlq         $9, %%mm5    \n\t"
    499. 

  499.             "pand       %%mm7, %%mm2    \n\t"
    500. 

  500.             "pand       %%mm7, %%mm5    \n\t"
    501. 

  501.             "por        %%mm1, %%mm0    \n\t"
    502. 

  502.             "por        %%mm4, %%mm3    \n\t"
    503. 

  503.             "por        %%mm2, %%mm0    \n\t"
    504. 

  504.             "por        %%mm5, %%mm3    \n\t"
    505. 

  505.             "psllq        $16, %%mm3    \n\t"
    506. 

  506.             "por        %%mm3, %%mm0    \n\t"
    507. 

  507.             MOVNTQ"     %%mm0, %0       \n\t"
    508. 

  508.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    509. 

  509.         d += 4;
    510. 

  510.         s += 16;
    511. 

  511.     }
    512. 

  512. #endif
    513. 

  513.     __asm__ volatile(SFENCE:::"memory");
    514. 

  514.     __asm__ volatile(EMMS:::"memory");
    515. 

  515.     while (s < end) {
    516. 

  516.         register int rgb = *(const uint32_t*)s; s += 4;
    517. 

  517.         *d++ = ((rgb&0xFF)>>3) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>9);
    518. 

  518.     }
    519. 

  519. }
    520. 

  520. 

  521. static inline void RENAME(rgb32tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
    522. 

  522. {
    523. 

  523.     const uint8_t *s = src;
    524. 

  524.     const uint8_t *end;
    525. 

  525.     const uint8_t *mm_end;
    526. 

  526.     uint16_t *d = (uint16_t *)dst;
    527. 

  527.     end = s + src_size;
    528. 

  528.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    529. 

  529.     __asm__ volatile(
    530. 

  530.         "movq          %0, %%mm7    \n\t"
    531. 

  531.         "movq          %1, %%mm6    \n\t"
    532. 

  532.         ::"m"(red_15mask),"m"(green_15mask));
    533. 

  533.     mm_end = end - 15;
    534. 

  534.     while (s < mm_end) {
    535. 

  535.         __asm__ volatile(
    536. 

  536.             PREFETCH"    32%1           \n\t"
    537. 

  537.             "movd          %1, %%mm0    \n\t"
    538. 

  538.             "movd         4%1, %%mm3    \n\t"
    539. 

  539.             "punpckldq    8%1, %%mm0    \n\t"
    540. 

  540.             "punpckldq   12%1, %%mm3    \n\t"
    541. 

  541.             "movq       %%mm0, %%mm1    \n\t"
    542. 

  542.             "movq       %%mm0, %%mm2    \n\t"
    543. 

  543.             "movq       %%mm3, %%mm4    \n\t"
    544. 

  544.             "movq       %%mm3, %%mm5    \n\t"
    545. 

  545.             "psllq         $7, %%mm0    \n\t"
    546. 

  546.             "psllq         $7, %%mm3    \n\t"
    547. 

  547.             "pand       %%mm7, %%mm0    \n\t"
    548. 

  548.             "pand       %%mm7, %%mm3    \n\t"
    549. 

  549.             "psrlq         $6, %%mm1    \n\t"
    550. 

  550.             "psrlq         $6, %%mm4    \n\t"
    551. 

  551.             "pand       %%mm6, %%mm1    \n\t"
    552. 

  552.             "pand       %%mm6, %%mm4    \n\t"
    553. 

  553.             "psrlq        $19, %%mm2    \n\t"
    554. 

  554.             "psrlq        $19, %%mm5    \n\t"
    555. 

  555.             "pand          %2, %%mm2    \n\t"
    556. 

  556.             "pand          %2, %%mm5    \n\t"
    557. 

  557.             "por        %%mm1, %%mm0    \n\t"
    558. 

  558.             "por        %%mm4, %%mm3    \n\t"
    559. 

  559.             "por        %%mm2, %%mm0    \n\t"
    560. 

  560.             "por        %%mm5, %%mm3    \n\t"
    561. 

  561.             "psllq        $16, %%mm3    \n\t"
    562. 

  562.             "por        %%mm3, %%mm0    \n\t"
    563. 

  563.             MOVNTQ"     %%mm0, %0       \n\t"
    564. 

  564.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    565. 

  565.         d += 4;
    566. 

  566.         s += 16;
    567. 

  567.     }
    568. 

  568.     __asm__ volatile(SFENCE:::"memory");
    569. 

  569.     __asm__ volatile(EMMS:::"memory");
    570. 

  570.     while (s < end) {
    571. 

  571.         register int rgb = *(const uint32_t*)s; s += 4;
    572. 

  572.         *d++ = ((rgb&0xF8)<<7) + ((rgb&0xF800)>>6) + ((rgb&0xF80000)>>19);
    573. 

  573.     }
    574. 

  574. }
    575. 

  575. 

  576. static inline void RENAME(rgb24tobgr16)(const uint8_t *src, uint8_t *dst, int src_size)
    577. 

  577. {
    578. 

  578.     const uint8_t *s = src;
    579. 

  579.     const uint8_t *end;
    580. 

  580.     const uint8_t *mm_end;
    581. 

  581.     uint16_t *d = (uint16_t *)dst;
    582. 

  582.     end = s + src_size;
    583. 

  583.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    584. 

  584.     __asm__ volatile(
    585. 

  585.         "movq         %0, %%mm7     \n\t"
    586. 

  586.         "movq         %1, %%mm6     \n\t"
    587. 

  587.         ::"m"(red_16mask),"m"(green_16mask));
    588. 

  588.     mm_end = end - 11;
    589. 

  589.     while (s < mm_end) {
    590. 

  590.         __asm__ volatile(
    591. 

  591.             PREFETCH"    32%1           \n\t"
    592. 

  592.             "movd          %1, %%mm0    \n\t"
    593. 

  593.             "movd         3%1, %%mm3    \n\t"
    594. 

  594.             "punpckldq    6%1, %%mm0    \n\t"
    595. 

  595.             "punpckldq    9%1, %%mm3    \n\t"
    596. 

  596.             "movq       %%mm0, %%mm1    \n\t"
    597. 

  597.             "movq       %%mm0, %%mm2    \n\t"
    598. 

  598.             "movq       %%mm3, %%mm4    \n\t"
    599. 

  599.             "movq       %%mm3, %%mm5    \n\t"
    600. 

  600.             "psrlq         $3, %%mm0    \n\t"
    601. 

  601.             "psrlq         $3, %%mm3    \n\t"
    602. 

  602.             "pand          %2, %%mm0    \n\t"
    603. 

  603.             "pand          %2, %%mm3    \n\t"
    604. 

  604.             "psrlq         $5, %%mm1    \n\t"
    605. 

  605.             "psrlq         $5, %%mm4    \n\t"
    606. 

  606.             "pand       %%mm6, %%mm1    \n\t"
    607. 

  607.             "pand       %%mm6, %%mm4    \n\t"
    608. 

  608.             "psrlq         $8, %%mm2    \n\t"
    609. 

  609.             "psrlq         $8, %%mm5    \n\t"
    610. 

  610.             "pand       %%mm7, %%mm2    \n\t"
    611. 

  611.             "pand       %%mm7, %%mm5    \n\t"
    612. 

  612.             "por        %%mm1, %%mm0    \n\t"
    613. 

  613.             "por        %%mm4, %%mm3    \n\t"
    614. 

  614.             "por        %%mm2, %%mm0    \n\t"
    615. 

  615.             "por        %%mm5, %%mm3    \n\t"
    616. 

  616.             "psllq        $16, %%mm3    \n\t"
    617. 

  617.             "por        %%mm3, %%mm0    \n\t"
    618. 

  618.             MOVNTQ"     %%mm0, %0       \n\t"
    619. 

  619.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    620. 

  620.         d += 4;
    621. 

  621.         s += 12;
    622. 

  622.     }
    623. 

  623.     __asm__ volatile(SFENCE:::"memory");
    624. 

  624.     __asm__ volatile(EMMS:::"memory");
    625. 

  625.     while (s < end) {
    626. 

  626.         const int b = *s++;
    627. 

  627.         const int g = *s++;
    628. 

  628.         const int r = *s++;
    629. 

  629.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
    630. 

  630.     }
    631. 

  631. }
    632. 

  632. 

  633. static inline void RENAME(rgb24to16)(const uint8_t *src, uint8_t *dst, int src_size)
    634. 

  634. {
    635. 

  635.     const uint8_t *s = src;
    636. 

  636.     const uint8_t *end;
    637. 

  637.     const uint8_t *mm_end;
    638. 

  638.     uint16_t *d = (uint16_t *)dst;
    639. 

  639.     end = s + src_size;
    640. 

  640.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    641. 

  641.     __asm__ volatile(
    642. 

  642.         "movq         %0, %%mm7     \n\t"
    643. 

  643.         "movq         %1, %%mm6     \n\t"
    644. 

  644.         ::"m"(red_16mask),"m"(green_16mask));
    645. 

  645.     mm_end = end - 15;
    646. 

  646.     while (s < mm_end) {
    647. 

  647.         __asm__ volatile(
    648. 

  648.             PREFETCH"    32%1           \n\t"
    649. 

  649.             "movd          %1, %%mm0    \n\t"
    650. 

  650.             "movd         3%1, %%mm3    \n\t"
    651. 

  651.             "punpckldq    6%1, %%mm0    \n\t"
    652. 

  652.             "punpckldq    9%1, %%mm3    \n\t"
    653. 

  653.             "movq       %%mm0, %%mm1    \n\t"
    654. 

  654.             "movq       %%mm0, %%mm2    \n\t"
    655. 

  655.             "movq       %%mm3, %%mm4    \n\t"
    656. 

  656.             "movq       %%mm3, %%mm5    \n\t"
    657. 

  657.             "psllq         $8, %%mm0    \n\t"
    658. 

  658.             "psllq         $8, %%mm3    \n\t"
    659. 

  659.             "pand       %%mm7, %%mm0    \n\t"
    660. 

  660.             "pand       %%mm7, %%mm3    \n\t"
    661. 

  661.             "psrlq         $5, %%mm1    \n\t"
    662. 

  662.             "psrlq         $5, %%mm4    \n\t"
    663. 

  663.             "pand       %%mm6, %%mm1    \n\t"
    664. 

  664.             "pand       %%mm6, %%mm4    \n\t"
    665. 

  665.             "psrlq        $19, %%mm2    \n\t"
    666. 

  666.             "psrlq        $19, %%mm5    \n\t"
    667. 

  667.             "pand          %2, %%mm2    \n\t"
    668. 

  668.             "pand          %2, %%mm5    \n\t"
    669. 

  669.             "por        %%mm1, %%mm0    \n\t"
    670. 

  670.             "por        %%mm4, %%mm3    \n\t"
    671. 

  671.             "por        %%mm2, %%mm0    \n\t"
    672. 

  672.             "por        %%mm5, %%mm3    \n\t"
    673. 

  673.             "psllq        $16, %%mm3    \n\t"
    674. 

  674.             "por        %%mm3, %%mm0    \n\t"
    675. 

  675.             MOVNTQ"     %%mm0, %0       \n\t"
    676. 

  676.             :"=m"(*d):"m"(*s),"m"(blue_16mask):"memory");
    677. 

  677.         d += 4;
    678. 

  678.         s += 12;
    679. 

  679.     }
    680. 

  680.     __asm__ volatile(SFENCE:::"memory");
    681. 

  681.     __asm__ volatile(EMMS:::"memory");
    682. 

  682.     while (s < end) {
    683. 

  683.         const int r = *s++;
    684. 

  684.         const int g = *s++;
    685. 

  685.         const int b = *s++;
    686. 

  686.         *d++ = (b>>3) | ((g&0xFC)<<3) | ((r&0xF8)<<8);
    687. 

  687.     }
    688. 

  688. }
    689. 

  689. 

  690. static inline void RENAME(rgb24tobgr15)(const uint8_t *src, uint8_t *dst, int src_size)
    691. 

  691. {
    692. 

  692.     const uint8_t *s = src;
    693. 

  693.     const uint8_t *end;
    694. 

  694.     const uint8_t *mm_end;
    695. 

  695.     uint16_t *d = (uint16_t *)dst;
    696. 

  696.     end = s + src_size;
    697. 

  697.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    698. 

  698.     __asm__ volatile(
    699. 

  699.         "movq          %0, %%mm7    \n\t"
    700. 

  700.         "movq          %1, %%mm6    \n\t"
    701. 

  701.         ::"m"(red_15mask),"m"(green_15mask));
    702. 

  702.     mm_end = end - 11;
    703. 

  703.     while (s < mm_end) {
    704. 

  704.         __asm__ volatile(
    705. 

  705.             PREFETCH"    32%1           \n\t"
    706. 

  706.             "movd          %1, %%mm0    \n\t"
    707. 

  707.             "movd         3%1, %%mm3    \n\t"
    708. 

  708.             "punpckldq    6%1, %%mm0    \n\t"
    709. 

  709.             "punpckldq    9%1, %%mm3    \n\t"
    710. 

  710.             "movq       %%mm0, %%mm1    \n\t"
    711. 

  711.             "movq       %%mm0, %%mm2    \n\t"
    712. 

  712.             "movq       %%mm3, %%mm4    \n\t"
    713. 

  713.             "movq       %%mm3, %%mm5    \n\t"
    714. 

  714.             "psrlq         $3, %%mm0    \n\t"
    715. 

  715.             "psrlq         $3, %%mm3    \n\t"
    716. 

  716.             "pand          %2, %%mm0    \n\t"
    717. 

  717.             "pand          %2, %%mm3    \n\t"
    718. 

  718.             "psrlq         $6, %%mm1    \n\t"
    719. 

  719.             "psrlq         $6, %%mm4    \n\t"
    720. 

  720.             "pand       %%mm6, %%mm1    \n\t"
    721. 

  721.             "pand       %%mm6, %%mm4    \n\t"
    722. 

  722.             "psrlq         $9, %%mm2    \n\t"
    723. 

  723.             "psrlq         $9, %%mm5    \n\t"
    724. 

  724.             "pand       %%mm7, %%mm2    \n\t"
    725. 

  725.             "pand       %%mm7, %%mm5    \n\t"
    726. 

  726.             "por        %%mm1, %%mm0    \n\t"
    727. 

  727.             "por        %%mm4, %%mm3    \n\t"
    728. 

  728.             "por        %%mm2, %%mm0    \n\t"
    729. 

  729.             "por        %%mm5, %%mm3    \n\t"
    730. 

  730.             "psllq        $16, %%mm3    \n\t"
    731. 

  731.             "por        %%mm3, %%mm0    \n\t"
    732. 

  732.             MOVNTQ"     %%mm0, %0       \n\t"
    733. 

  733.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    734. 

  734.         d += 4;
    735. 

  735.         s += 12;
    736. 

  736.     }
    737. 

  737.     __asm__ volatile(SFENCE:::"memory");
    738. 

  738.     __asm__ volatile(EMMS:::"memory");
    739. 

  739.     while (s < end) {
    740. 

  740.         const int b = *s++;
    741. 

  741.         const int g = *s++;
    742. 

  742.         const int r = *s++;
    743. 

  743.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
    744. 

  744.     }
    745. 

  745. }
    746. 

  746. 

  747. static inline void RENAME(rgb24to15)(const uint8_t *src, uint8_t *dst, int src_size)
    748. 

  748. {
    749. 

  749.     const uint8_t *s = src;
    750. 

  750.     const uint8_t *end;
    751. 

  751.     const uint8_t *mm_end;
    752. 

  752.     uint16_t *d = (uint16_t *)dst;
    753. 

  753.     end = s + src_size;
    754. 

  754.     __asm__ volatile(PREFETCH"    %0"::"m"(*src):"memory");
    755. 

  755.     __asm__ volatile(
    756. 

  756.         "movq         %0, %%mm7     \n\t"
    757. 

  757.         "movq         %1, %%mm6     \n\t"
    758. 

  758.         ::"m"(red_15mask),"m"(green_15mask));
    759. 

  759.     mm_end = end - 15;
    760. 

  760.     while (s < mm_end) {
    761. 

  761.         __asm__ volatile(
    762. 

  762.             PREFETCH"   32%1            \n\t"
    763. 

  763.             "movd         %1, %%mm0     \n\t"
    764. 

  764.             "movd        3%1, %%mm3     \n\t"
    765. 

  765.             "punpckldq   6%1, %%mm0     \n\t"
    766. 

  766.             "punpckldq   9%1, %%mm3     \n\t"
    767. 

  767.             "movq      %%mm0, %%mm1     \n\t"
    768. 

  768.             "movq      %%mm0, %%mm2     \n\t"
    769. 

  769.             "movq      %%mm3, %%mm4     \n\t"
    770. 

  770.             "movq      %%mm3, %%mm5     \n\t"
    771. 

  771.             "psllq        $7, %%mm0     \n\t"
    772. 

  772.             "psllq        $7, %%mm3     \n\t"
    773. 

  773.             "pand      %%mm7, %%mm0     \n\t"
    774. 

  774.             "pand      %%mm7, %%mm3     \n\t"
    775. 

  775.             "psrlq        $6, %%mm1     \n\t"
    776. 

  776.             "psrlq        $6, %%mm4     \n\t"
    777. 

  777.             "pand      %%mm6, %%mm1     \n\t"
    778. 

  778.             "pand      %%mm6, %%mm4     \n\t"
    779. 

  779.             "psrlq       $19, %%mm2     \n\t"
    780. 

  780.             "psrlq       $19, %%mm5     \n\t"
    781. 

  781.             "pand         %2, %%mm2     \n\t"
    782. 

  782.             "pand         %2, %%mm5     \n\t"
    783. 

  783.             "por       %%mm1, %%mm0     \n\t"
    784. 

  784.             "por       %%mm4, %%mm3     \n\t"
    785. 

  785.             "por       %%mm2, %%mm0     \n\t"
    786. 

  786.             "por       %%mm5, %%mm3     \n\t"
    787. 

  787.             "psllq       $16, %%mm3     \n\t"
    788. 

  788.             "por       %%mm3, %%mm0     \n\t"
    789. 

  789.             MOVNTQ"    %%mm0, %0        \n\t"
    790. 

  790.             :"=m"(*d):"m"(*s),"m"(blue_15mask):"memory");
    791. 

  791.         d += 4;
    792. 

  792.         s += 12;
    793. 

  793.     }
    794. 

  794.     __asm__ volatile(SFENCE:::"memory");
    795. 

  795.     __asm__ volatile(EMMS:::"memory");
    796. 

  796.     while (s < end) {
    797. 

  797.         const int r = *s++;
    798. 

  798.         const int g = *s++;
    799. 

  799.         const int b = *s++;
    800. 

  800.         *d++ = (b>>3) | ((g&0xF8)<<2) | ((r&0xF8)<<7);
    801. 

  801.     }
    802. 

  802. }
    803. 

  803. 

  804. static inline void RENAME(rgb15tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    805. 

  805. {
    806. 

  806.     const uint16_t *end;
    807. 

  807.     const uint16_t *mm_end;
    808. 

  808.     uint8_t *d = dst;
    809. 

  809.     const uint16_t *s = (const uint16_t*)src;
    810. 

  810.     end = s + src_size/2;
    811. 

  811.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    812. 

  812.     mm_end = end - 7;
    813. 

  813.     while (s < mm_end) {
    814. 

  814.         __asm__ volatile(
    815. 

  815.             PREFETCH"    32%1           \n\t"
    816. 

  816.             "movq          %1, %%mm0    \n\t"
    817. 

  817.             "movq          %1, %%mm1    \n\t"
    818. 

  818.             "movq          %1, %%mm2    \n\t"
    819. 

  819.             "pand          %2, %%mm0    \n\t"
    820. 

  820.             "pand          %3, %%mm1    \n\t"
    821. 

  821.             "pand          %4, %%mm2    \n\t"
    822. 

  822.             "psllq         $5, %%mm0    \n\t"
    823. 

  823.             "pmulhw        %6, %%mm0    \n\t"
    824. 

  824.             "pmulhw        %6, %%mm1    \n\t"
    825. 

  825.             "pmulhw        %7, %%mm2    \n\t"
    826. 

  826.             "movq       %%mm0, %%mm3    \n\t"
    827. 

  827.             "movq       %%mm1, %%mm4    \n\t"
    828. 

  828.             "movq       %%mm2, %%mm5    \n\t"
    829. 

  829.             "punpcklwd     %5, %%mm0    \n\t"
    830. 

  830.             "punpcklwd     %5, %%mm1    \n\t"
    831. 

  831.             "punpcklwd     %5, %%mm2    \n\t"
    832. 

  832.             "punpckhwd     %5, %%mm3    \n\t"
    833. 

  833.             "punpckhwd     %5, %%mm4    \n\t"
    834. 

  834.             "punpckhwd     %5, %%mm5    \n\t"
    835. 

  835.             "psllq         $8, %%mm1    \n\t"
    836. 

  836.             "psllq        $16, %%mm2    \n\t"
    837. 

  837.             "por        %%mm1, %%mm0    \n\t"
    838. 

  838.             "por        %%mm2, %%mm0    \n\t"
    839. 

  839.             "psllq         $8, %%mm4    \n\t"
    840. 

  840.             "psllq        $16, %%mm5    \n\t"
    841. 

  841.             "por        %%mm4, %%mm3    \n\t"
    842. 

  842.             "por        %%mm5, %%mm3    \n\t"
    843. 

  843. 

  844.             "movq       %%mm0, %%mm6    \n\t"
    845. 

  845.             "movq       %%mm3, %%mm7    \n\t"
    846. 

  846. 

  847.             "movq         8%1, %%mm0    \n\t"
    848. 

  848.             "movq         8%1, %%mm1    \n\t"
    849. 

  849.             "movq         8%1, %%mm2    \n\t"
    850. 

  850.             "pand          %2, %%mm0    \n\t"
    851. 

  851.             "pand          %3, %%mm1    \n\t"
    852. 

  852.             "pand          %4, %%mm2    \n\t"
    853. 

  853.             "psllq         $5, %%mm0    \n\t"
    854. 

  854.             "pmulhw        %6, %%mm0    \n\t"
    855. 

  855.             "pmulhw        %6, %%mm1    \n\t"
    856. 

  856.             "pmulhw        %7, %%mm2    \n\t"
    857. 

  857.             "movq       %%mm0, %%mm3    \n\t"
    858. 

  858.             "movq       %%mm1, %%mm4    \n\t"
    859. 

  859.             "movq       %%mm2, %%mm5    \n\t"
    860. 

  860.             "punpcklwd     %5, %%mm0    \n\t"
    861. 

  861.             "punpcklwd     %5, %%mm1    \n\t"
    862. 

  862.             "punpcklwd     %5, %%mm2    \n\t"
    863. 

  863.             "punpckhwd     %5, %%mm3    \n\t"
    864. 

  864.             "punpckhwd     %5, %%mm4    \n\t"
    865. 

  865.             "punpckhwd     %5, %%mm5    \n\t"
    866. 

  866.             "psllq         $8, %%mm1    \n\t"
    867. 

  867.             "psllq        $16, %%mm2    \n\t"
    868. 

  868.             "por        %%mm1, %%mm0    \n\t"
    869. 

  869.             "por        %%mm2, %%mm0    \n\t"
    870. 

  870.             "psllq         $8, %%mm4    \n\t"
    871. 

  871.             "psllq        $16, %%mm5    \n\t"
    872. 

  872.             "por        %%mm4, %%mm3    \n\t"
    873. 

  873.             "por        %%mm5, %%mm3    \n\t"
    874. 

  874. 

  875.             :"=m"(*d)
    876. 

  876.             :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r),"m"(mmx_null),"m"(mul15_mid),"m"(mul15_hi)
    877. 

  877.             :"memory");
    878. 

  878.         /* borrowed 32 to 24 */
    879. 

  879.         __asm__ volatile(
    880. 

  880.             "movq       %%mm0, %%mm4    \n\t"
    881. 

  881.             "movq       %%mm3, %%mm5    \n\t"
    882. 

  882.             "movq       %%mm6, %%mm0    \n\t"
    883. 

  883.             "movq       %%mm7, %%mm1    \n\t"
    884. 

  884. 

  885.             "movq       %%mm4, %%mm6    \n\t"
    886. 

  886.             "movq       %%mm5, %%mm7    \n\t"
    887. 

  887.             "movq       %%mm0, %%mm2    \n\t"
    888. 

  888.             "movq       %%mm1, %%mm3    \n\t"
    889. 

  889. 

  890.             STORE_BGR24_MMX
    891. 

  891. 

  892.             :"=m"(*d)
    893. 

  893.             :"m"(*s)
    894. 

  894.             :"memory");
    895. 

  895.         d += 24;
    896. 

  896.         s += 8;
    897. 

  897.     }
    898. 

  898.     __asm__ volatile(SFENCE:::"memory");
    899. 

  899.     __asm__ volatile(EMMS:::"memory");
    900. 

  900.     while (s < end) {
    901. 

  901.         register uint16_t bgr;
    902. 

  902.         bgr = *s++;
    903. 

  903.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    904. 

  904.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
    905. 

  905.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
    906. 

  906.     }
    907. 

  907. }
    908. 

  908. 

  909. static inline void RENAME(rgb16tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    910. 

  910. {
    911. 

  911.     const uint16_t *end;
    912. 

  912.     const uint16_t *mm_end;
    913. 

  913.     uint8_t *d = (uint8_t *)dst;
    914. 

  914.     const uint16_t *s = (const uint16_t *)src;
    915. 

  915.     end = s + src_size/2;
    916. 

  916.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    917. 

  917.     mm_end = end - 7;
    918. 

  918.     while (s < mm_end) {
    919. 

  919.         __asm__ volatile(
    920. 

  920.             PREFETCH"    32%1           \n\t"
    921. 

  921.             "movq          %1, %%mm0    \n\t"
    922. 

  922.             "movq          %1, %%mm1    \n\t"
    923. 

  923.             "movq          %1, %%mm2    \n\t"
    924. 

  924.             "pand          %2, %%mm0    \n\t"
    925. 

  925.             "pand          %3, %%mm1    \n\t"
    926. 

  926.             "pand          %4, %%mm2    \n\t"
    927. 

  927.             "psllq         $5, %%mm0    \n\t"
    928. 

  928.             "psrlq         $1, %%mm2    \n\t"
    929. 

  929.             "pmulhw        %6, %%mm0    \n\t"
    930. 

  930.             "pmulhw        %8, %%mm1    \n\t"
    931. 

  931.             "pmulhw        %7, %%mm2    \n\t"
    932. 

  932.             "movq       %%mm0, %%mm3    \n\t"
    933. 

  933.             "movq       %%mm1, %%mm4    \n\t"
    934. 

  934.             "movq       %%mm2, %%mm5    \n\t"
    935. 

  935.             "punpcklwd     %5, %%mm0    \n\t"
    936. 

  936.             "punpcklwd     %5, %%mm1    \n\t"
    937. 

  937.             "punpcklwd     %5, %%mm2    \n\t"
    938. 

  938.             "punpckhwd     %5, %%mm3    \n\t"
    939. 

  939.             "punpckhwd     %5, %%mm4    \n\t"
    940. 

  940.             "punpckhwd     %5, %%mm5    \n\t"
    941. 

  941.             "psllq         $8, %%mm1    \n\t"
    942. 

  942.             "psllq        $16, %%mm2    \n\t"
    943. 

  943.             "por        %%mm1, %%mm0    \n\t"
    944. 

  944.             "por        %%mm2, %%mm0    \n\t"
    945. 

  945.             "psllq         $8, %%mm4    \n\t"
    946. 

  946.             "psllq        $16, %%mm5    \n\t"
    947. 

  947.             "por        %%mm4, %%mm3    \n\t"
    948. 

  948.             "por        %%mm5, %%mm3    \n\t"
    949. 

  949. 

  950.             "movq       %%mm0, %%mm6    \n\t"
    951. 

  951.             "movq       %%mm3, %%mm7    \n\t"
    952. 

  952. 

  953.             "movq         8%1, %%mm0    \n\t"
    954. 

  954.             "movq         8%1, %%mm1    \n\t"
    955. 

  955.             "movq         8%1, %%mm2    \n\t"
    956. 

  956.             "pand          %2, %%mm0    \n\t"
    957. 

  957.             "pand          %3, %%mm1    \n\t"
    958. 

  958.             "pand          %4, %%mm2    \n\t"
    959. 

  959.             "psllq         $5, %%mm0    \n\t"
    960. 

  960.             "psrlq         $1, %%mm2    \n\t"
    961. 

  961.             "pmulhw        %6, %%mm0    \n\t"
    962. 

  962.             "pmulhw        %8, %%mm1    \n\t"
    963. 

  963.             "pmulhw        %7, %%mm2    \n\t"
    964. 

  964.             "movq       %%mm0, %%mm3    \n\t"
    965. 

  965.             "movq       %%mm1, %%mm4    \n\t"
    966. 

  966.             "movq       %%mm2, %%mm5    \n\t"
    967. 

  967.             "punpcklwd     %5, %%mm0    \n\t"
    968. 

  968.             "punpcklwd     %5, %%mm1    \n\t"
    969. 

  969.             "punpcklwd     %5, %%mm2    \n\t"
    970. 

  970.             "punpckhwd     %5, %%mm3    \n\t"
    971. 

  971.             "punpckhwd     %5, %%mm4    \n\t"
    972. 

  972.             "punpckhwd     %5, %%mm5    \n\t"
    973. 

  973.             "psllq         $8, %%mm1    \n\t"
    974. 

  974.             "psllq        $16, %%mm2    \n\t"
    975. 

  975.             "por        %%mm1, %%mm0    \n\t"
    976. 

  976.             "por        %%mm2, %%mm0    \n\t"
    977. 

  977.             "psllq         $8, %%mm4    \n\t"
    978. 

  978.             "psllq        $16, %%mm5    \n\t"
    979. 

  979.             "por        %%mm4, %%mm3    \n\t"
    980. 

  980.             "por        %%mm5, %%mm3    \n\t"
    981. 

  981.             :"=m"(*d)
    982. 

  982.             :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mmx_null),"m"(mul15_mid),"m"(mul15_hi),"m"(mul16_mid)
    983. 

  983.             :"memory");
    984. 

  984.         /* borrowed 32 to 24 */
    985. 

  985.         __asm__ volatile(
    986. 

  986.             "movq       %%mm0, %%mm4    \n\t"
    987. 

  987.             "movq       %%mm3, %%mm5    \n\t"
    988. 

  988.             "movq       %%mm6, %%mm0    \n\t"
    989. 

  989.             "movq       %%mm7, %%mm1    \n\t"
    990. 

  990. 

  991.             "movq       %%mm4, %%mm6    \n\t"
    992. 

  992.             "movq       %%mm5, %%mm7    \n\t"
    993. 

  993.             "movq       %%mm0, %%mm2    \n\t"
    994. 

  994.             "movq       %%mm1, %%mm3    \n\t"
    995. 

  995. 

  996.             STORE_BGR24_MMX
    997. 

  997. 

  998.             :"=m"(*d)
    999. 

  999.             :"m"(*s)
    1000. 

  1000.             :"memory");
    1001. 

  1001.         d += 24;
    1002. 

  1002.         s += 8;
    1003. 

  1003.     }
    1004. 

  1004.     __asm__ volatile(SFENCE:::"memory");
    1005. 

  1005.     __asm__ volatile(EMMS:::"memory");
    1006. 

  1006.     while (s < end) {
    1007. 

  1007.         register uint16_t bgr;
    1008. 

  1008.         bgr = *s++;
    1009. 

  1009.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1010. 

  1010.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
    1011. 

  1011.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
    1012. 

  1012.     }
    1013. 

  1013. }
    1014. 

  1014. 

  1015. /*
    1016. 

  1016.  * mm0 = 00 B3 00 B2 00 B1 00 B0
    1017. 

  1017.  * mm1 = 00 G3 00 G2 00 G1 00 G0
    1018. 

  1018.  * mm2 = 00 R3 00 R2 00 R1 00 R0
    1019. 

  1019.  * mm6 = FF FF FF FF FF FF FF FF
    1020. 

  1020.  * mm7 = 00 00 00 00 00 00 00 00
    1021. 

  1021.  */
    1022. 

  1022. #define PACK_RGB32 \
    1023. 

  1023.     "packuswb   %%mm7, %%mm0    \n\t" /* 00 00 00 00 B3 B2 B1 B0 */ \
    1024. 

  1024.     "packuswb   %%mm7, %%mm1    \n\t" /* 00 00 00 00 G3 G2 G1 G0 */ \
    1025. 

  1025.     "packuswb   %%mm7, %%mm2    \n\t" /* 00 00 00 00 R3 R2 R1 R0 */ \
    1026. 

  1026.     "punpcklbw  %%mm1, %%mm0    \n\t" /* G3 B3 G2 B2 G1 B1 G0 B0 */ \
    1027. 

  1027.     "punpcklbw  %%mm6, %%mm2    \n\t" /* FF R3 FF R2 FF R1 FF R0 */ \
    1028. 

  1028.     "movq       %%mm0, %%mm3    \n\t"                               \
    1029. 

  1029.     "punpcklwd  %%mm2, %%mm0    \n\t" /* FF R1 G1 B1 FF R0 G0 B0 */ \
    1030. 

  1030.     "punpckhwd  %%mm2, %%mm3    \n\t" /* FF R3 G3 B3 FF R2 G2 B2 */ \
    1031. 

  1031.     MOVNTQ"     %%mm0,  %0      \n\t"                               \
    1032. 

  1032.     MOVNTQ"     %%mm3, 8%0      \n\t"                               \
    1033. 

  1033. 

  1034. static inline void RENAME(rgb15to32)(const uint8_t *src, uint8_t *dst, int src_size)
    1035. 

  1035. {
    1036. 

  1036.     const uint16_t *end;
    1037. 

  1037.     const uint16_t *mm_end;
    1038. 

  1038.     uint8_t *d = dst;
    1039. 

  1039.     const uint16_t *s = (const uint16_t *)src;
    1040. 

  1040.     end = s + src_size/2;
    1041. 

  1041.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    1042. 

  1042.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");
    1043. 

  1043.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");
    1044. 

  1044.     mm_end = end - 3;
    1045. 

  1045.     while (s < mm_end) {
    1046. 

  1046.         __asm__ volatile(
    1047. 

  1047.             PREFETCH"    32%1           \n\t"
    1048. 

  1048.             "movq          %1, %%mm0    \n\t"
    1049. 

  1049.             "movq          %1, %%mm1    \n\t"
    1050. 

  1050.             "movq          %1, %%mm2    \n\t"
    1051. 

  1051.             "pand          %2, %%mm0    \n\t"
    1052. 

  1052.             "pand          %3, %%mm1    \n\t"
    1053. 

  1053.             "pand          %4, %%mm2    \n\t"
    1054. 

  1054.             "psllq         $5, %%mm0    \n\t"
    1055. 

  1055.             "pmulhw        %5, %%mm0    \n\t"
    1056. 

  1056.             "pmulhw        %5, %%mm1    \n\t"
    1057. 

  1057.             "pmulhw        %6, %%mm2    \n\t"
    1058. 

  1058.             PACK_RGB32
    1059. 

  1059.             :"=m"(*d)
    1060. 

  1060.             :"m"(*s),"m"(mask15b),"m"(mask15g),"m"(mask15r),"m"(mul15_mid),"m"(mul15_hi)
    1061. 

  1061.             :"memory");
    1062. 

  1062.         d += 16;
    1063. 

  1063.         s += 4;
    1064. 

  1064.     }
    1065. 

  1065.     __asm__ volatile(SFENCE:::"memory");
    1066. 

  1066.     __asm__ volatile(EMMS:::"memory");
    1067. 

  1067.     while (s < end) {
    1068. 

  1068.         register uint16_t bgr;
    1069. 

  1069.         bgr = *s++;
    1070. 

  1070.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1071. 

  1071.         *d++ = ((bgr&0x3E0)>>2) | ((bgr&0x3E0)>>7);
    1072. 

  1072.         *d++ = ((bgr&0x7C00)>>7) | ((bgr&0x7C00)>>12);
    1073. 

  1073.         *d++ = 255;
    1074. 

  1074.     }
    1075. 

  1075. }
    1076. 

  1076. 

  1077. static inline void RENAME(rgb16to32)(const uint8_t *src, uint8_t *dst, int src_size)
    1078. 

  1078. {
    1079. 

  1079.     const uint16_t *end;
    1080. 

  1080.     const uint16_t *mm_end;
    1081. 

  1081.     uint8_t *d = dst;
    1082. 

  1082.     const uint16_t *s = (const uint16_t*)src;
    1083. 

  1083.     end = s + src_size/2;
    1084. 

  1084.     __asm__ volatile(PREFETCH"    %0"::"m"(*s):"memory");
    1085. 

  1085.     __asm__ volatile("pxor    %%mm7,%%mm7    \n\t":::"memory");
    1086. 

  1086.     __asm__ volatile("pcmpeqd %%mm6,%%mm6    \n\t":::"memory");
    1087. 

  1087.     mm_end = end - 3;
    1088. 

  1088.     while (s < mm_end) {
    1089. 

  1089.         __asm__ volatile(
    1090. 

  1090.             PREFETCH"    32%1           \n\t"
    1091. 

  1091.             "movq          %1, %%mm0    \n\t"
    1092. 

  1092.             "movq          %1, %%mm1    \n\t"
    1093. 

  1093.             "movq          %1, %%mm2    \n\t"
    1094. 

  1094.             "pand          %2, %%mm0    \n\t"
    1095. 

  1095.             "pand          %3, %%mm1    \n\t"
    1096. 

  1096.             "pand          %4, %%mm2    \n\t"
    1097. 

  1097.             "psllq         $5, %%mm0    \n\t"
    1098. 

  1098.             "psrlq         $1, %%mm2    \n\t"
    1099. 

  1099.             "pmulhw        %5, %%mm0    \n\t"
    1100. 

  1100.             "pmulhw        %7, %%mm1    \n\t"
    1101. 

  1101.             "pmulhw        %6, %%mm2    \n\t"
    1102. 

  1102.             PACK_RGB32
    1103. 

  1103.             :"=m"(*d)
    1104. 

  1104.             :"m"(*s),"m"(mask16b),"m"(mask16g),"m"(mask16r),"m"(mul15_mid),"m"(mul15_hi),"m"(mul16_mid)
    1105. 

  1105.             :"memory");
    1106. 

  1106.         d += 16;
    1107. 

  1107.         s += 4;
    1108. 

  1108.     }
    1109. 

  1109.     __asm__ volatile(SFENCE:::"memory");
    1110. 

  1110.     __asm__ volatile(EMMS:::"memory");
    1111. 

  1111.     while (s < end) {
    1112. 

  1112.         register uint16_t bgr;
    1113. 

  1113.         bgr = *s++;
    1114. 

  1114.         *d++ = ((bgr&0x1F)<<3) | ((bgr&0x1F)>>2);
    1115. 

  1115.         *d++ = ((bgr&0x7E0)>>3) | ((bgr&0x7E0)>>9);
    1116. 

  1116.         *d++ = ((bgr&0xF800)>>8) | ((bgr&0xF800)>>13);
    1117. 

  1117.         *d++ = 255;
    1118. 

  1118.     }
    1119. 

  1119. }
    1120. 

  1120. 

  1121. static inline void RENAME(shuffle_bytes_2103)(const uint8_t *src, uint8_t *dst, int src_size)
    1122. 

  1122. {
    1123. 

  1123.     x86_reg idx = 15 - src_size;
    1124. 

  1124.     const uint8_t *s = src-idx;
    1125. 

  1125.     uint8_t *d = dst-idx;
    1126. 

  1126.     __asm__ volatile(
    1127. 

  1127.         "test          %0, %0           \n\t"
    1128. 

  1128.         "jns           2f               \n\t"
    1129. 

  1129.         PREFETCH"       (%1, %0)        \n\t"
    1130. 

  1130.         "movq          %3, %%mm7        \n\t"
    1131. 

  1131.         "pxor          %4, %%mm7        \n\t"
    1132. 

  1132.         "movq       %%mm7, %%mm6        \n\t"
    1133. 

  1133.         "pxor          %5, %%mm7        \n\t"
    1134. 

  1134.         ".p2align       4               \n\t"
    1135. 

  1135.         "1:                             \n\t"
    1136. 

  1136.         PREFETCH"     32(%1, %0)        \n\t"
    1137. 

  1137.         "movq           (%1, %0), %%mm0 \n\t"
    1138. 

  1138.         "movq          8(%1, %0), %%mm1 \n\t"
    1139. 

  1139. # if COMPILE_TEMPLATE_MMX2
    1140. 

  1140.         "pshufw      $177, %%mm0, %%mm3 \n\t"
    1141. 

  1141.         "pshufw      $177, %%mm1, %%mm5 \n\t"
    1142. 

  1142.         "pand       %%mm7, %%mm0        \n\t"
    1143. 

  1143.         "pand       %%mm6, %%mm3        \n\t"
    1144. 

  1144.         "pand       %%mm7, %%mm1        \n\t"
    1145. 

  1145.         "pand       %%mm6, %%mm5        \n\t"
    1146. 

  1146.         "por        %%mm3, %%mm0        \n\t"
    1147. 

  1147.         "por        %%mm5, %%mm1        \n\t"
    1148. 

  1148. # else
    1149. 

  1149.         "movq       %%mm0, %%mm2        \n\t"
    1150. 

  1150.         "movq       %%mm1, %%mm4        \n\t"
    1151. 

  1151.         "pand       %%mm7, %%mm0        \n\t"
    1152. 

  1152.         "pand       %%mm6, %%mm2        \n\t"
    1153. 

  1153.         "pand       %%mm7, %%mm1        \n\t"
    1154. 

  1154.         "pand       %%mm6, %%mm4        \n\t"
    1155. 

  1155.         "movq       %%mm2, %%mm3        \n\t"
    1156. 

  1156.         "movq       %%mm4, %%mm5        \n\t"
    1157. 

  1157.         "pslld        $16, %%mm2        \n\t"
    1158. 

  1158.         "psrld        $16, %%mm3        \n\t"
    1159. 

  1159.         "pslld        $16, %%mm4        \n\t"
    1160. 

  1160.         "psrld        $16, %%mm5        \n\t"
    1161. 

  1161.         "por        %%mm2, %%mm0        \n\t"
    1162. 

  1162.         "por        %%mm4, %%mm1        \n\t"
    1163. 

  1163.         "por        %%mm3, %%mm0        \n\t"
    1164. 

  1164.         "por        %%mm5, %%mm1        \n\t"
    1165. 

  1165. # endif
    1166. 

  1166.         MOVNTQ"     %%mm0,  (%2, %0)    \n\t"
    1167. 

  1167.         MOVNTQ"     %%mm1, 8(%2, %0)    \n\t"
    1168. 

  1168.         "add          $16, %0           \n\t"
    1169. 

  1169.         "js            1b               \n\t"
    1170. 

  1170.         SFENCE"                         \n\t"
    1171. 

  1171.         EMMS"                           \n\t"
    1172. 

  1172.         "2:                             \n\t"
    1173. 

  1173.         : "+&r"(idx)
    1174. 

  1174.         : "r" (s), "r" (d), "m" (mask32b), "m" (mask32r), "m" (mmx_one)
    1175. 

  1175.         : "memory");
    1176. 

  1176.     for (; idx<15; idx+=4) {
    1177. 

  1177.         register int v = *(const uint32_t *)&s[idx], g = v & 0xff00ff00;
    1178. 

  1178.         v &= 0xff00ff;
    1179. 

  1179.         *(uint32_t *)&d[idx] = (v>>16) + g + (v<<16);
    1180. 

  1180.     }
    1181. 

  1181. }
    1182. 

  1182. 

  1183. static inline void RENAME(rgb24tobgr24)(const uint8_t *src, uint8_t *dst, int src_size)
    1184. 

  1184. {
    1185. 

  1185.     unsigned i;
    1186. 

  1186.     x86_reg mmx_size= 23 - src_size;
    1187. 

  1187.     __asm__ volatile (
    1188. 

  1188.         "test             %%"REG_a", %%"REG_a"          \n\t"
    1189. 

  1189.         "jns                     2f                     \n\t"
    1190. 

  1190.         "movq     "MANGLE(mask24r)", %%mm5              \n\t"
    1191. 

  1191.         "movq     "MANGLE(mask24g)", %%mm6              \n\t"
    1192. 

  1192.         "movq     "MANGLE(mask24b)", %%mm7              \n\t"
    1193. 

  1193.         ".p2align                 4                     \n\t"
    1194. 

  1194.         "1:                                             \n\t"
    1195. 

  1195.         PREFETCH" 32(%1, %%"REG_a")                     \n\t"
    1196. 

  1196.         "movq       (%1, %%"REG_a"), %%mm0              \n\t" // BGR BGR BG
    1197. 

  1197.         "movq       (%1, %%"REG_a"), %%mm1              \n\t" // BGR BGR BG
    1198. 

  1198.         "movq      2(%1, %%"REG_a"), %%mm2              \n\t" // R BGR BGR B
    1199. 

  1199.         "psllq                  $16, %%mm0              \n\t" // 00 BGR BGR
    1200. 

  1200.         "pand                 %%mm5, %%mm0              \n\t"
    1201. 

  1201.         "pand                 %%mm6, %%mm1              \n\t"
    1202. 

  1202.         "pand                 %%mm7, %%mm2              \n\t"
    1203. 

  1203.         "por                  %%mm0, %%mm1              \n\t"
    1204. 

  1204.         "por                  %%mm2, %%mm1              \n\t"
    1205. 

  1205.         "movq      6(%1, %%"REG_a"), %%mm0              \n\t" // BGR BGR BG
    1206. 

  1206.         MOVNTQ"               %%mm1,   (%2, %%"REG_a")  \n\t" // RGB RGB RG
    1207. 

  1207.         "movq      8(%1, %%"REG_a"), %%mm1              \n\t" // R BGR BGR B
    1208. 

  1208.         "movq     10(%1, %%"REG_a"), %%mm2              \n\t" // GR BGR BGR
    1209. 

  1209.         "pand                 %%mm7, %%mm0              \n\t"
    1210. 

  1210.         "pand                 %%mm5, %%mm1              \n\t"
    1211. 

  1211.         "pand                 %%mm6, %%mm2              \n\t"
    1212. 

  1212.         "por                  %%mm0, %%mm1              \n\t"
    1213. 

  1213.         "por                  %%mm2, %%mm1              \n\t"
    1214. 

  1214.         "movq     14(%1, %%"REG_a"), %%mm0              \n\t" // R BGR BGR B
    1215. 

  1215.         MOVNTQ"               %%mm1,  8(%2, %%"REG_a")  \n\t" // B RGB RGB R
    1216. 

  1216.         "movq     16(%1, %%"REG_a"), %%mm1              \n\t" // GR BGR BGR
    1217. 

  1217.         "movq     18(%1, %%"REG_a"), %%mm2              \n\t" // BGR BGR BG
    1218. 

  1218.         "pand                 %%mm6, %%mm0              \n\t"
    1219. 

  1219.         "pand                 %%mm7, %%mm1              \n\t"
    1220. 

  1220.         "pand                 %%mm5, %%mm2              \n\t"
    1221. 

  1221.         "por                  %%mm0, %%mm1              \n\t"
    1222. 

  1222.         "por                  %%mm2, %%mm1              \n\t"
    1223. 

  1223.         MOVNTQ"               %%mm1, 16(%2, %%"REG_a")  \n\t"
    1224. 

  1224.         "add                    $24, %%"REG_a"          \n\t"
    1225. 

  1225.         " js                     1b                     \n\t"
    1226. 

  1226.         "2:                                             \n\t"
    1227. 

  1227.         : "+a" (mmx_size)
    1228. 

  1228.         : "r" (src-mmx_size), "r"(dst-mmx_size)
    1229. 

  1229.     );
    1230. 

  1230. 

  1231.     __asm__ volatile(SFENCE:::"memory");
    1232. 

  1232.     __asm__ volatile(EMMS:::"memory");
    1233. 

  1233. 

  1234.     if (mmx_size==23) return; //finished, was multiple of 8
    1235. 

  1235. 

  1236.     src+= src_size;
    1237. 

  1237.     dst+= src_size;
    1238. 

  1238.     src_size= 23-mmx_size;
    1239. 

  1239.     src-= src_size;
    1240. 

  1240.     dst-= src_size;
    1241. 

  1241.     for (i=0; i<src_size; i+=3) {
    1242. 

  1242.         register uint8_t x;
    1243. 

  1243.         x          = src[i + 2];
    1244. 

  1244.         dst[i + 1] = src[i + 1];
    1245. 

  1245.         dst[i + 2] = src[i + 0];
    1246. 

  1246.         dst[i + 0] = x;
    1247. 

  1247.     }
    1248. 

  1248. }
    1249. 

  1249. 

  1250. static inline void RENAME(yuvPlanartoyuy2)(const uint8_t *ysrc, const uint8_t *usrc, const uint8_t *vsrc, uint8_t *dst,
    1251. 

  1251.                                            int width, int height,
    1252. 

  1252.                                            int lumStride, int chromStride, int dstStride, int vertLumPerChroma)
    1253. 

  1253. {
    1254. 

  1254.     int y;
    1255. 

  1255.     const x86_reg chromWidth= width>>1;
    1256. 

  1256.     for (y=0; y<height; y++) {
    1257. 

  1257.         //FIXME handle 2 lines at once (fewer prefetches, reuse some chroma, but very likely memory-limited anyway)
    1258. 

  1258.         __asm__ volatile(
    1259. 

  1259.             "xor                 %%"REG_a", %%"REG_a"   \n\t"
    1260. 

  1260.             ".p2align                    4              \n\t"
    1261. 

  1261.             "1:                                         \n\t"
    1262. 

  1262.             PREFETCH"    32(%1, %%"REG_a", 2)           \n\t"
    1263. 

  1263.             PREFETCH"    32(%2, %%"REG_a")              \n\t"
    1264. 

  1264.             PREFETCH"    32(%3, %%"REG_a")              \n\t"
    1265. 

  1265.             "movq          (%2, %%"REG_a"), %%mm0       \n\t" // U(0)
    1266. 

  1266.             "movq                    %%mm0, %%mm2       \n\t" // U(0)
    1267. 

  1267.             "movq          (%3, %%"REG_a"), %%mm1       \n\t" // V(0)
    1268. 

  1268.             "punpcklbw               %%mm1, %%mm0       \n\t" // UVUV UVUV(0)
    1269. 

  1269.             "punpckhbw               %%mm1, %%mm2       \n\t" // UVUV UVUV(8)
    1270. 

  1270. 

  1271.             "movq        (%1, %%"REG_a",2), %%mm3       \n\t" // Y(0)
    1272. 

  1272.             "movq       8(%1, %%"REG_a",2), %%mm5       \n\t" // Y(8)
    1273. 

  1273.             "movq                    %%mm3, %%mm4       \n\t" // Y(0)
    1274. 

  1274.             "movq                    %%mm5, %%mm6       \n\t" // Y(8)
    1275. 

  1275.             "punpcklbw               %%mm0, %%mm3       \n\t" // YUYV YUYV(0)
    1276. 

  1276.             "punpckhbw               %%mm0, %%mm4       \n\t" // YUYV YUYV(4)
    1277. 

  1277.             "punpcklbw               %%mm2, %%mm5       \n\t" // YUYV YUYV(8)
    1278. 

  1278.             "punpckhbw               %%mm2, %%mm6       \n\t" // YUYV YUYV(12)
    1279. 

  1279. 

  1280.             MOVNTQ"                  %%mm3,   (%0, %%"REG_a", 4)    \n\t"
    1281. 

  1281.             MOVNTQ"                  %%mm4,  8(%0, %%"REG_a", 4)    \n\t"
    1282. 

  1282.             MOVNTQ"                  %%mm5, 16(%0, %%"REG_a", 4)    \n\t"
    1283. 

  1283.             MOVNTQ"                  %%mm6, 24(%0, %%"REG_a", 4)    \n\t"
    1284. 

  1284. 

  1285.             "add                        $8, %%"REG_a"   \n\t"
    1286. 

  1286.             "cmp                        %4, %%"REG_a"   \n\t"
    1287. 

  1287.             " jb                        1b              \n\t"
    1288. 

  1288.             ::"r"(dst), "r"(ysrc), "r"(usrc), "r"(vsrc), "g" (chromWidth)
    1289. 

  1289.             : "%"REG_a
    1290. 

  1290.         );
    1291. 

  1291.         if ((y&(vertLumPerChroma-1)) == vertLumPerChroma-1) {
    1292. 

  1292.             usrc += chromStride;
    1293. 

  1293.             vsrc += chromStride;
    1294. 

  1294.         }
    1295. 

  1295.         ysrc += lumStride;
    1296. 

  1296.         dst  += dstStride;
    1297. 

  1297.     }
    1298. 

  1298.     __asm__(EMMS"     …
    1299.

Large files files are truncated, but you can click here to view the full file