PageRenderTime 30ms CodeModel.GetById 21ms RepoModel.GetById 0ms app.codeStats 1ms

/apps/desktop/libvncserver/tight.c

http://ftk.googlecode.com/
C | 1731 lines | 1354 code | 261 blank | 116 comment | 305 complexity | 95bdc96328834640e8a56ddbddb4ca7e MD5 | raw file
Possible License(s): LGPL-3.0
  1. /*
  2. * tight.c
  3. *
  4. * Routines to implement Tight Encoding
  5. */
  6. /*
  7. * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved.
  8. * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
  9. *
  10. * This is free software; you can redistribute it and/or modify
  11. * it under the terms of the GNU General Public License as published by
  12. * the Free Software Foundation; either version 2 of the License, or
  13. * (at your option) any later version.
  14. *
  15. * This software is distributed in the hope that it will be useful,
  16. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18. * GNU General Public License for more details.
  19. *
  20. * You should have received a copy of the GNU General Public License
  21. * along with this software; if not, write to the Free Software
  22. * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
  23. * USA.
  24. */
  25. /*#include <stdio.h>*/
  26. #include <rfb/rfb.h>
  27. #include "private.h"
  28. #ifdef WIN32
  29. #define XMD_H
  30. #undef FAR
  31. #define NEEDFAR_POINTERS
  32. #endif
  33. #include <jpeglib.h>
  34. /* Note: The following constant should not be changed. */
  35. #define TIGHT_MIN_TO_COMPRESS 12
  36. /* The parameters below may be adjusted. */
  37. #define MIN_SPLIT_RECT_SIZE 4096
  38. #define MIN_SOLID_SUBRECT_SIZE 2048
  39. #define MAX_SPLIT_TILE_SIZE 16
  40. /* May be set to TRUE with "-lazytight" Xvnc option. */
  41. rfbBool rfbTightDisableGradient = FALSE;
  42. /* This variable is set on every rfbSendRectEncodingTight() call. */
  43. static rfbBool usePixelFormat24;
  44. /* Compression level stuff. The following array contains various
  45. encoder parameters for each of 10 compression levels (0..9).
  46. Last three parameters correspond to JPEG quality levels (0..9). */
  47. typedef struct TIGHT_CONF_s {
  48. int maxRectSize, maxRectWidth;
  49. int monoMinRectSize, gradientMinRectSize;
  50. int idxZlibLevel, monoZlibLevel, rawZlibLevel, gradientZlibLevel;
  51. int gradientThreshold, gradientThreshold24;
  52. int idxMaxColorsDivisor;
  53. int jpegQuality, jpegThreshold, jpegThreshold24;
  54. } TIGHT_CONF;
  55. static TIGHT_CONF tightConf[10] = {
  56. { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 },
  57. { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 },
  58. { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 },
  59. { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 },
  60. { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 },
  61. { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 },
  62. { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 },
  63. { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
  64. { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
  65. { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
  66. };
  67. static int compressLevel;
  68. static int qualityLevel;
  69. /* Stuff dealing with palettes. */
  70. typedef struct COLOR_LIST_s {
  71. struct COLOR_LIST_s *next;
  72. int idx;
  73. uint32_t rgb;
  74. } COLOR_LIST;
  75. typedef struct PALETTE_ENTRY_s {
  76. COLOR_LIST *listNode;
  77. int numPixels;
  78. } PALETTE_ENTRY;
  79. typedef struct PALETTE_s {
  80. PALETTE_ENTRY entry[256];
  81. COLOR_LIST *hash[256];
  82. COLOR_LIST list[256];
  83. } PALETTE;
  84. /* TODO: move into rfbScreen struct */
  85. static int paletteNumColors, paletteMaxColors;
  86. static uint32_t monoBackground, monoForeground;
  87. static PALETTE palette;
  88. /* Pointers to dynamically-allocated buffers. */
  89. static int tightBeforeBufSize = 0;
  90. static char *tightBeforeBuf = NULL;
  91. static int tightAfterBufSize = 0;
  92. static char *tightAfterBuf = NULL;
  93. static int *prevRowBuf = NULL;
  94. void rfbTightCleanup(rfbScreenInfoPtr screen)
  95. {
  96. if(tightBeforeBufSize) {
  97. free(tightBeforeBuf);
  98. tightBeforeBufSize=0;
  99. }
  100. if(tightAfterBufSize) {
  101. free(tightAfterBuf);
  102. tightAfterBufSize=0;
  103. }
  104. }
  105. /* Prototypes for static functions. */
  106. static void FindBestSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
  107. uint32_t colorValue, int *w_ptr, int *h_ptr);
  108. static void ExtendSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
  109. uint32_t colorValue,
  110. int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr);
  111. static rfbBool CheckSolidTile (rfbClientPtr cl, int x, int y, int w, int h,
  112. uint32_t *colorPtr, rfbBool needSameColor);
  113. static rfbBool CheckSolidTile8 (rfbClientPtr cl, int x, int y, int w, int h,
  114. uint32_t *colorPtr, rfbBool needSameColor);
  115. static rfbBool CheckSolidTile16 (rfbClientPtr cl, int x, int y, int w, int h,
  116. uint32_t *colorPtr, rfbBool needSameColor);
  117. static rfbBool CheckSolidTile32 (rfbClientPtr cl, int x, int y, int w, int h,
  118. uint32_t *colorPtr, rfbBool needSameColor);
  119. static rfbBool SendRectSimple (rfbClientPtr cl, int x, int y, int w, int h);
  120. static rfbBool SendSubrect (rfbClientPtr cl, int x, int y, int w, int h);
  121. static rfbBool SendTightHeader (rfbClientPtr cl, int x, int y, int w, int h);
  122. static rfbBool SendSolidRect (rfbClientPtr cl);
  123. static rfbBool SendMonoRect (rfbClientPtr cl, int w, int h);
  124. static rfbBool SendIndexedRect (rfbClientPtr cl, int w, int h);
  125. static rfbBool SendFullColorRect (rfbClientPtr cl, int w, int h);
  126. static rfbBool SendGradientRect (rfbClientPtr cl, int w, int h);
  127. static rfbBool CompressData(rfbClientPtr cl, int streamId, int dataLen,
  128. int zlibLevel, int zlibStrategy);
  129. static rfbBool SendCompressedData(rfbClientPtr cl, int compressedLen);
  130. static void FillPalette8(int count);
  131. static void FillPalette16(int count);
  132. static void FillPalette32(int count);
  133. static void PaletteReset(void);
  134. static int PaletteInsert(uint32_t rgb, int numPixels, int bpp);
  135. static void Pack24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int count);
  136. static void EncodeIndexedRect16(uint8_t *buf, int count);
  137. static void EncodeIndexedRect32(uint8_t *buf, int count);
  138. static void EncodeMonoRect8(uint8_t *buf, int w, int h);
  139. static void EncodeMonoRect16(uint8_t *buf, int w, int h);
  140. static void EncodeMonoRect32(uint8_t *buf, int w, int h);
  141. static void FilterGradient24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int w, int h);
  142. static void FilterGradient16(rfbClientPtr cl, uint16_t *buf, rfbPixelFormat *fmt, int w, int h);
  143. static void FilterGradient32(rfbClientPtr cl, uint32_t *buf, rfbPixelFormat *fmt, int w, int h);
  144. static int DetectSmoothImage(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
  145. static unsigned long DetectSmoothImage24(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
  146. static unsigned long DetectSmoothImage16(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
  147. static unsigned long DetectSmoothImage32(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
  148. static rfbBool SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h,
  149. int quality);
  150. static void PrepareRowForJpeg(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
  151. static void PrepareRowForJpeg24(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
  152. static void PrepareRowForJpeg16(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
  153. static void PrepareRowForJpeg32(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
  154. static void JpegInitDestination(j_compress_ptr cinfo);
  155. static boolean JpegEmptyOutputBuffer(j_compress_ptr cinfo);
  156. static void JpegTermDestination(j_compress_ptr cinfo);
  157. static void JpegSetDstManager(j_compress_ptr cinfo);
  158. /*
  159. * Tight encoding implementation.
  160. */
  161. int
  162. rfbNumCodedRectsTight(rfbClientPtr cl,
  163. int x,
  164. int y,
  165. int w,
  166. int h)
  167. {
  168. int maxRectSize, maxRectWidth;
  169. int subrectMaxWidth, subrectMaxHeight;
  170. /* No matter how many rectangles we will send if LastRect markers
  171. are used to terminate rectangle stream. */
  172. if (cl->enableLastRectEncoding && w * h >= MIN_SPLIT_RECT_SIZE)
  173. return 0;
  174. maxRectSize = tightConf[cl->tightCompressLevel].maxRectSize;
  175. maxRectWidth = tightConf[cl->tightCompressLevel].maxRectWidth;
  176. if (w > maxRectWidth || w * h > maxRectSize) {
  177. subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
  178. subrectMaxHeight = maxRectSize / subrectMaxWidth;
  179. return (((w - 1) / maxRectWidth + 1) *
  180. ((h - 1) / subrectMaxHeight + 1));
  181. } else {
  182. return 1;
  183. }
  184. }
  185. rfbBool
  186. rfbSendRectEncodingTight(rfbClientPtr cl,
  187. int x,
  188. int y,
  189. int w,
  190. int h)
  191. {
  192. int nMaxRows;
  193. uint32_t colorValue;
  194. int dx, dy, dw, dh;
  195. int x_best, y_best, w_best, h_best;
  196. char *fbptr;
  197. rfbSendUpdateBuf(cl);
  198. compressLevel = cl->tightCompressLevel;
  199. qualityLevel = cl->tightQualityLevel;
  200. if ( cl->format.depth == 24 && cl->format.redMax == 0xFF &&
  201. cl->format.greenMax == 0xFF && cl->format.blueMax == 0xFF ) {
  202. usePixelFormat24 = TRUE;
  203. } else {
  204. usePixelFormat24 = FALSE;
  205. }
  206. if (!cl->enableLastRectEncoding || w * h < MIN_SPLIT_RECT_SIZE)
  207. return SendRectSimple(cl, x, y, w, h);
  208. /* Make sure we can write at least one pixel into tightBeforeBuf. */
  209. if (tightBeforeBufSize < 4) {
  210. tightBeforeBufSize = 4;
  211. if (tightBeforeBuf == NULL)
  212. tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
  213. else
  214. tightBeforeBuf = (char *)realloc(tightBeforeBuf,
  215. tightBeforeBufSize);
  216. }
  217. /* Calculate maximum number of rows in one non-solid rectangle. */
  218. {
  219. int maxRectSize, maxRectWidth, nMaxWidth;
  220. maxRectSize = tightConf[compressLevel].maxRectSize;
  221. maxRectWidth = tightConf[compressLevel].maxRectWidth;
  222. nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
  223. nMaxRows = maxRectSize / nMaxWidth;
  224. }
  225. /* Try to find large solid-color areas and send them separately. */
  226. for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
  227. /* If a rectangle becomes too large, send its upper part now. */
  228. if (dy - y >= nMaxRows) {
  229. if (!SendRectSimple(cl, x, y, w, nMaxRows))
  230. return 0;
  231. y += nMaxRows;
  232. h -= nMaxRows;
  233. }
  234. dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
  235. MAX_SPLIT_TILE_SIZE : (y + h - dy);
  236. for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {
  237. dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
  238. MAX_SPLIT_TILE_SIZE : (x + w - dx);
  239. if (CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, FALSE)) {
  240. /* Get dimensions of solid-color area. */
  241. FindBestSolidArea(cl, dx, dy, w - (dx - x), h - (dy - y),
  242. colorValue, &w_best, &h_best);
  243. /* Make sure a solid rectangle is large enough
  244. (or the whole rectangle is of the same color). */
  245. if ( w_best * h_best != w * h &&
  246. w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
  247. continue;
  248. /* Try to extend solid rectangle to maximum size. */
  249. x_best = dx; y_best = dy;
  250. ExtendSolidArea(cl, x, y, w, h, colorValue,
  251. &x_best, &y_best, &w_best, &h_best);
  252. /* Send rectangles at top and left to solid-color area. */
  253. if ( y_best != y &&
  254. !SendRectSimple(cl, x, y, w, y_best-y) )
  255. return FALSE;
  256. if ( x_best != x &&
  257. !rfbSendRectEncodingTight(cl, x, y_best,
  258. x_best-x, h_best) )
  259. return FALSE;
  260. /* Send solid-color rectangle. */
  261. if (!SendTightHeader(cl, x_best, y_best, w_best, h_best))
  262. return FALSE;
  263. fbptr = (cl->scaledScreen->frameBuffer +
  264. (cl->scaledScreen->paddedWidthInBytes * y_best) +
  265. (x_best * (cl->scaledScreen->bitsPerPixel / 8)));
  266. (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
  267. &cl->format, fbptr, tightBeforeBuf,
  268. cl->scaledScreen->paddedWidthInBytes, 1, 1);
  269. if (!SendSolidRect(cl))
  270. return FALSE;
  271. /* Send remaining rectangles (at right and bottom). */
  272. if ( x_best + w_best != x + w &&
  273. !rfbSendRectEncodingTight(cl, x_best+w_best, y_best,
  274. w-(x_best-x)-w_best, h_best) )
  275. return FALSE;
  276. if ( y_best + h_best != y + h &&
  277. !rfbSendRectEncodingTight(cl, x, y_best+h_best,
  278. w, h-(y_best-y)-h_best) )
  279. return FALSE;
  280. /* Return after all recursive calls are done. */
  281. return TRUE;
  282. }
  283. }
  284. }
  285. /* No suitable solid-color rectangles found. */
  286. return SendRectSimple(cl, x, y, w, h);
  287. }
  288. static void
  289. FindBestSolidArea(rfbClientPtr cl,
  290. int x,
  291. int y,
  292. int w,
  293. int h,
  294. uint32_t colorValue,
  295. int *w_ptr,
  296. int *h_ptr)
  297. {
  298. int dx, dy, dw, dh;
  299. int w_prev;
  300. int w_best = 0, h_best = 0;
  301. w_prev = w;
  302. for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
  303. dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
  304. MAX_SPLIT_TILE_SIZE : (y + h - dy);
  305. dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
  306. MAX_SPLIT_TILE_SIZE : w_prev;
  307. if (!CheckSolidTile(cl, x, dy, dw, dh, &colorValue, TRUE))
  308. break;
  309. for (dx = x + dw; dx < x + w_prev;) {
  310. dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
  311. MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
  312. if (!CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, TRUE))
  313. break;
  314. dx += dw;
  315. }
  316. w_prev = dx - x;
  317. if (w_prev * (dy + dh - y) > w_best * h_best) {
  318. w_best = w_prev;
  319. h_best = dy + dh - y;
  320. }
  321. }
  322. *w_ptr = w_best;
  323. *h_ptr = h_best;
  324. }
  325. static void
  326. ExtendSolidArea(rfbClientPtr cl,
  327. int x,
  328. int y,
  329. int w,
  330. int h,
  331. uint32_t colorValue,
  332. int *x_ptr,
  333. int *y_ptr,
  334. int *w_ptr,
  335. int *h_ptr)
  336. {
  337. int cx, cy;
  338. /* Try to extend the area upwards. */
  339. for ( cy = *y_ptr - 1;
  340. cy >= y && CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
  341. cy-- );
  342. *h_ptr += *y_ptr - (cy + 1);
  343. *y_ptr = cy + 1;
  344. /* ... downwards. */
  345. for ( cy = *y_ptr + *h_ptr;
  346. cy < y + h &&
  347. CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
  348. cy++ );
  349. *h_ptr += cy - (*y_ptr + *h_ptr);
  350. /* ... to the left. */
  351. for ( cx = *x_ptr - 1;
  352. cx >= x && CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
  353. cx-- );
  354. *w_ptr += *x_ptr - (cx + 1);
  355. *x_ptr = cx + 1;
  356. /* ... to the right. */
  357. for ( cx = *x_ptr + *w_ptr;
  358. cx < x + w &&
  359. CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
  360. cx++ );
  361. *w_ptr += cx - (*x_ptr + *w_ptr);
  362. }
  363. /*
  364. * Check if a rectangle is all of the same color. If needSameColor is
  365. * set to non-zero, then also check that its color equals to the
  366. * *colorPtr value. The result is 1 if the test is successfull, and in
  367. * that case new color will be stored in *colorPtr.
  368. */
  369. static rfbBool CheckSolidTile(rfbClientPtr cl, int x, int y, int w, int h, uint32_t* colorPtr, rfbBool needSameColor)
  370. {
  371. switch(cl->screen->serverFormat.bitsPerPixel) {
  372. case 32:
  373. return CheckSolidTile32(cl, x, y, w, h, colorPtr, needSameColor);
  374. case 16:
  375. return CheckSolidTile16(cl, x, y, w, h, colorPtr, needSameColor);
  376. default:
  377. return CheckSolidTile8(cl, x, y, w, h, colorPtr, needSameColor);
  378. }
  379. }
  380. #define DEFINE_CHECK_SOLID_FUNCTION(bpp) \
  381. \
  382. static rfbBool \
  383. CheckSolidTile##bpp(rfbClientPtr cl, int x, int y, int w, int h, \
  384. uint32_t* colorPtr, rfbBool needSameColor) \
  385. { \
  386. uint##bpp##_t *fbptr; \
  387. uint##bpp##_t colorValue; \
  388. int dx, dy; \
  389. \
  390. fbptr = (uint##bpp##_t *) \
  391. &cl->scaledScreen->frameBuffer[y * cl->scaledScreen->paddedWidthInBytes + x * (bpp/8)]; \
  392. \
  393. colorValue = *fbptr; \
  394. if (needSameColor && (uint32_t)colorValue != *colorPtr) \
  395. return FALSE; \
  396. \
  397. for (dy = 0; dy < h; dy++) { \
  398. for (dx = 0; dx < w; dx++) { \
  399. if (colorValue != fbptr[dx]) \
  400. return FALSE; \
  401. } \
  402. fbptr = (uint##bpp##_t *)((uint8_t *)fbptr + cl->scaledScreen->paddedWidthInBytes); \
  403. } \
  404. \
  405. *colorPtr = (uint32_t)colorValue; \
  406. return TRUE; \
  407. }
  408. DEFINE_CHECK_SOLID_FUNCTION(8)
  409. DEFINE_CHECK_SOLID_FUNCTION(16)
  410. DEFINE_CHECK_SOLID_FUNCTION(32)
  411. static rfbBool
  412. SendRectSimple(rfbClientPtr cl, int x, int y, int w, int h)
  413. {
  414. int maxBeforeSize, maxAfterSize;
  415. int maxRectSize, maxRectWidth;
  416. int subrectMaxWidth, subrectMaxHeight;
  417. int dx, dy;
  418. int rw, rh;
  419. maxRectSize = tightConf[compressLevel].maxRectSize;
  420. maxRectWidth = tightConf[compressLevel].maxRectWidth;
  421. maxBeforeSize = maxRectSize * (cl->format.bitsPerPixel / 8);
  422. maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
  423. if (tightBeforeBufSize < maxBeforeSize) {
  424. tightBeforeBufSize = maxBeforeSize;
  425. if (tightBeforeBuf == NULL)
  426. tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
  427. else
  428. tightBeforeBuf = (char *)realloc(tightBeforeBuf,
  429. tightBeforeBufSize);
  430. }
  431. if (tightAfterBufSize < maxAfterSize) {
  432. tightAfterBufSize = maxAfterSize;
  433. if (tightAfterBuf == NULL)
  434. tightAfterBuf = (char *)malloc(tightAfterBufSize);
  435. else
  436. tightAfterBuf = (char *)realloc(tightAfterBuf,
  437. tightAfterBufSize);
  438. }
  439. if (w > maxRectWidth || w * h > maxRectSize) {
  440. subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
  441. subrectMaxHeight = maxRectSize / subrectMaxWidth;
  442. for (dy = 0; dy < h; dy += subrectMaxHeight) {
  443. for (dx = 0; dx < w; dx += maxRectWidth) {
  444. rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
  445. rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
  446. if (!SendSubrect(cl, x+dx, y+dy, rw, rh))
  447. return FALSE;
  448. }
  449. }
  450. } else {
  451. if (!SendSubrect(cl, x, y, w, h))
  452. return FALSE;
  453. }
  454. return TRUE;
  455. }
  456. static rfbBool
  457. SendSubrect(rfbClientPtr cl,
  458. int x,
  459. int y,
  460. int w,
  461. int h)
  462. {
  463. char *fbptr;
  464. rfbBool success = FALSE;
  465. /* Send pending data if there is more than 128 bytes. */
  466. if (cl->ublen > 128) {
  467. if (!rfbSendUpdateBuf(cl))
  468. return FALSE;
  469. }
  470. if (!SendTightHeader(cl, x, y, w, h))
  471. return FALSE;
  472. fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y)
  473. + (x * (cl->scaledScreen->bitsPerPixel / 8)));
  474. (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
  475. &cl->format, fbptr, tightBeforeBuf,
  476. cl->scaledScreen->paddedWidthInBytes, w, h);
  477. paletteMaxColors = w * h / tightConf[compressLevel].idxMaxColorsDivisor;
  478. if ( paletteMaxColors < 2 &&
  479. w * h >= tightConf[compressLevel].monoMinRectSize ) {
  480. paletteMaxColors = 2;
  481. }
  482. switch (cl->format.bitsPerPixel) {
  483. case 8:
  484. FillPalette8(w * h);
  485. break;
  486. case 16:
  487. FillPalette16(w * h);
  488. break;
  489. default:
  490. FillPalette32(w * h);
  491. }
  492. switch (paletteNumColors) {
  493. case 0:
  494. /* Truecolor image */
  495. if (DetectSmoothImage(cl, &cl->format, w, h)) {
  496. if (qualityLevel != -1) {
  497. success = SendJpegRect(cl, x, y, w, h,
  498. tightConf[qualityLevel].jpegQuality);
  499. } else {
  500. success = SendGradientRect(cl, w, h);
  501. }
  502. } else {
  503. success = SendFullColorRect(cl, w, h);
  504. }
  505. break;
  506. case 1:
  507. /* Solid rectangle */
  508. success = SendSolidRect(cl);
  509. break;
  510. case 2:
  511. /* Two-color rectangle */
  512. success = SendMonoRect(cl, w, h);
  513. break;
  514. default:
  515. /* Up to 256 different colors */
  516. if ( paletteNumColors > 96 &&
  517. qualityLevel != -1 && qualityLevel <= 3 &&
  518. DetectSmoothImage(cl, &cl->format, w, h) ) {
  519. success = SendJpegRect(cl, x, y, w, h,
  520. tightConf[qualityLevel].jpegQuality);
  521. } else {
  522. success = SendIndexedRect(cl, w, h);
  523. }
  524. }
  525. return success;
  526. }
  527. static rfbBool
  528. SendTightHeader(rfbClientPtr cl,
  529. int x,
  530. int y,
  531. int w,
  532. int h)
  533. {
  534. rfbFramebufferUpdateRectHeader rect;
  535. if (cl->ublen + sz_rfbFramebufferUpdateRectHeader > UPDATE_BUF_SIZE) {
  536. if (!rfbSendUpdateBuf(cl))
  537. return FALSE;
  538. }
  539. rect.r.x = Swap16IfLE(x);
  540. rect.r.y = Swap16IfLE(y);
  541. rect.r.w = Swap16IfLE(w);
  542. rect.r.h = Swap16IfLE(h);
  543. rect.encoding = Swap32IfLE(rfbEncodingTight);
  544. memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
  545. sz_rfbFramebufferUpdateRectHeader);
  546. cl->ublen += sz_rfbFramebufferUpdateRectHeader;
  547. rfbStatRecordEncodingSent(cl, rfbEncodingTight, sz_rfbFramebufferUpdateRectHeader,
  548. sz_rfbFramebufferUpdateRectHeader + w * (cl->format.bitsPerPixel / 8) * h);
  549. return TRUE;
  550. }
  551. /*
  552. * Subencoding implementations.
  553. */
  554. static rfbBool
  555. SendSolidRect(rfbClientPtr cl)
  556. {
  557. int len;
  558. if (usePixelFormat24) {
  559. Pack24(cl, tightBeforeBuf, &cl->format, 1);
  560. len = 3;
  561. } else
  562. len = cl->format.bitsPerPixel / 8;
  563. if (cl->ublen + 1 + len > UPDATE_BUF_SIZE) {
  564. if (!rfbSendUpdateBuf(cl))
  565. return FALSE;
  566. }
  567. cl->updateBuf[cl->ublen++] = (char)(rfbTightFill << 4);
  568. memcpy (&cl->updateBuf[cl->ublen], tightBeforeBuf, len);
  569. cl->ublen += len;
  570. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, len+1);
  571. return TRUE;
  572. }
  573. static rfbBool
  574. SendMonoRect(rfbClientPtr cl,
  575. int w,
  576. int h)
  577. {
  578. int streamId = 1;
  579. int paletteLen, dataLen;
  580. if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
  581. 2 * cl->format.bitsPerPixel / 8 > UPDATE_BUF_SIZE ) {
  582. if (!rfbSendUpdateBuf(cl))
  583. return FALSE;
  584. }
  585. /* Prepare tight encoding header. */
  586. dataLen = (w + 7) / 8;
  587. dataLen *= h;
  588. cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
  589. cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
  590. cl->updateBuf[cl->ublen++] = 1;
  591. /* Prepare palette, convert image. */
  592. switch (cl->format.bitsPerPixel) {
  593. case 32:
  594. EncodeMonoRect32((uint8_t *)tightBeforeBuf, w, h);
  595. ((uint32_t *)tightAfterBuf)[0] = monoBackground;
  596. ((uint32_t *)tightAfterBuf)[1] = monoForeground;
  597. if (usePixelFormat24) {
  598. Pack24(cl, tightAfterBuf, &cl->format, 2);
  599. paletteLen = 6;
  600. } else
  601. paletteLen = 8;
  602. memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteLen);
  603. cl->ublen += paletteLen;
  604. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteLen);
  605. break;
  606. case 16:
  607. EncodeMonoRect16((uint8_t *)tightBeforeBuf, w, h);
  608. ((uint16_t *)tightAfterBuf)[0] = (uint16_t)monoBackground;
  609. ((uint16_t *)tightAfterBuf)[1] = (uint16_t)monoForeground;
  610. memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, 4);
  611. cl->ublen += 4;
  612. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 7);
  613. break;
  614. default:
  615. EncodeMonoRect8((uint8_t *)tightBeforeBuf, w, h);
  616. cl->updateBuf[cl->ublen++] = (char)monoBackground;
  617. cl->updateBuf[cl->ublen++] = (char)monoForeground;
  618. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 5);
  619. }
  620. return CompressData(cl, streamId, dataLen,
  621. tightConf[compressLevel].monoZlibLevel,
  622. Z_DEFAULT_STRATEGY);
  623. }
  624. static rfbBool
  625. SendIndexedRect(rfbClientPtr cl,
  626. int w,
  627. int h)
  628. {
  629. int streamId = 2;
  630. int i, entryLen;
  631. if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
  632. paletteNumColors * cl->format.bitsPerPixel / 8 >
  633. UPDATE_BUF_SIZE ) {
  634. if (!rfbSendUpdateBuf(cl))
  635. return FALSE;
  636. }
  637. /* Prepare tight encoding header. */
  638. cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
  639. cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
  640. cl->updateBuf[cl->ublen++] = (char)(paletteNumColors - 1);
  641. /* Prepare palette, convert image. */
  642. switch (cl->format.bitsPerPixel) {
  643. case 32:
  644. EncodeIndexedRect32((uint8_t *)tightBeforeBuf, w * h);
  645. for (i = 0; i < paletteNumColors; i++) {
  646. ((uint32_t *)tightAfterBuf)[i] =
  647. palette.entry[i].listNode->rgb;
  648. }
  649. if (usePixelFormat24) {
  650. Pack24(cl, tightAfterBuf, &cl->format, paletteNumColors);
  651. entryLen = 3;
  652. } else
  653. entryLen = 4;
  654. memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * entryLen);
  655. cl->ublen += paletteNumColors * entryLen;
  656. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteNumColors * entryLen);
  657. break;
  658. case 16:
  659. EncodeIndexedRect16((uint8_t *)tightBeforeBuf, w * h);
  660. for (i = 0; i < paletteNumColors; i++) {
  661. ((uint16_t *)tightAfterBuf)[i] =
  662. (uint16_t)palette.entry[i].listNode->rgb;
  663. }
  664. memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * 2);
  665. cl->ublen += paletteNumColors * 2;
  666. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteNumColors * 2);
  667. break;
  668. default:
  669. return FALSE; /* Should never happen. */
  670. }
  671. return CompressData(cl, streamId, w * h,
  672. tightConf[compressLevel].idxZlibLevel,
  673. Z_DEFAULT_STRATEGY);
  674. }
  675. static rfbBool
  676. SendFullColorRect(rfbClientPtr cl,
  677. int w,
  678. int h)
  679. {
  680. int streamId = 0;
  681. int len;
  682. if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
  683. if (!rfbSendUpdateBuf(cl))
  684. return FALSE;
  685. }
  686. cl->updateBuf[cl->ublen++] = 0x00; /* stream id = 0, no flushing, no filter */
  687. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
  688. if (usePixelFormat24) {
  689. Pack24(cl, tightBeforeBuf, &cl->format, w * h);
  690. len = 3;
  691. } else
  692. len = cl->format.bitsPerPixel / 8;
  693. return CompressData(cl, streamId, w * h * len,
  694. tightConf[compressLevel].rawZlibLevel,
  695. Z_DEFAULT_STRATEGY);
  696. }
  697. static rfbBool
  698. SendGradientRect(rfbClientPtr cl,
  699. int w,
  700. int h)
  701. {
  702. int streamId = 3;
  703. int len;
  704. if (cl->format.bitsPerPixel == 8)
  705. return SendFullColorRect(cl, w, h);
  706. if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 2 > UPDATE_BUF_SIZE) {
  707. if (!rfbSendUpdateBuf(cl))
  708. return FALSE;
  709. }
  710. if (prevRowBuf == NULL)
  711. prevRowBuf = (int *)malloc(2048 * 3 * sizeof(int));
  712. cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
  713. cl->updateBuf[cl->ublen++] = rfbTightFilterGradient;
  714. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 2);
  715. if (usePixelFormat24) {
  716. FilterGradient24(cl, tightBeforeBuf, &cl->format, w, h);
  717. len = 3;
  718. } else if (cl->format.bitsPerPixel == 32) {
  719. FilterGradient32(cl, (uint32_t *)tightBeforeBuf, &cl->format, w, h);
  720. len = 4;
  721. } else {
  722. FilterGradient16(cl, (uint16_t *)tightBeforeBuf, &cl->format, w, h);
  723. len = 2;
  724. }
  725. return CompressData(cl, streamId, w * h * len,
  726. tightConf[compressLevel].gradientZlibLevel,
  727. Z_FILTERED);
  728. }
  729. static rfbBool
  730. CompressData(rfbClientPtr cl,
  731. int streamId,
  732. int dataLen,
  733. int zlibLevel,
  734. int zlibStrategy)
  735. {
  736. z_streamp pz;
  737. int err;
  738. if (dataLen < TIGHT_MIN_TO_COMPRESS) {
  739. memcpy(&cl->updateBuf[cl->ublen], tightBeforeBuf, dataLen);
  740. cl->ublen += dataLen;
  741. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, dataLen);
  742. return TRUE;
  743. }
  744. pz = &cl->zsStruct[streamId];
  745. /* Initialize compression stream if needed. */
  746. if (!cl->zsActive[streamId]) {
  747. pz->zalloc = Z_NULL;
  748. pz->zfree = Z_NULL;
  749. pz->opaque = Z_NULL;
  750. err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
  751. MAX_MEM_LEVEL, zlibStrategy);
  752. if (err != Z_OK)
  753. return FALSE;
  754. cl->zsActive[streamId] = TRUE;
  755. cl->zsLevel[streamId] = zlibLevel;
  756. }
  757. /* Prepare buffer pointers. */
  758. pz->next_in = (Bytef *)tightBeforeBuf;
  759. pz->avail_in = dataLen;
  760. pz->next_out = (Bytef *)tightAfterBuf;
  761. pz->avail_out = tightAfterBufSize;
  762. /* Change compression parameters if needed. */
  763. if (zlibLevel != cl->zsLevel[streamId]) {
  764. if (deflateParams (pz, zlibLevel, zlibStrategy) != Z_OK) {
  765. return FALSE;
  766. }
  767. cl->zsLevel[streamId] = zlibLevel;
  768. }
  769. /* Actual compression. */
  770. if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
  771. pz->avail_in != 0 || pz->avail_out == 0 ) {
  772. return FALSE;
  773. }
  774. return SendCompressedData(cl, tightAfterBufSize - pz->avail_out);
  775. }
  776. static rfbBool SendCompressedData(rfbClientPtr cl,
  777. int compressedLen)
  778. {
  779. int i, portionLen;
  780. cl->updateBuf[cl->ublen++] = compressedLen & 0x7F;
  781. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
  782. if (compressedLen > 0x7F) {
  783. cl->updateBuf[cl->ublen-1] |= 0x80;
  784. cl->updateBuf[cl->ublen++] = compressedLen >> 7 & 0x7F;
  785. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
  786. if (compressedLen > 0x3FFF) {
  787. cl->updateBuf[cl->ublen-1] |= 0x80;
  788. cl->updateBuf[cl->ublen++] = compressedLen >> 14 & 0xFF;
  789. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
  790. }
  791. }
  792. portionLen = UPDATE_BUF_SIZE;
  793. for (i = 0; i < compressedLen; i += portionLen) {
  794. if (i + portionLen > compressedLen) {
  795. portionLen = compressedLen - i;
  796. }
  797. if (cl->ublen + portionLen > UPDATE_BUF_SIZE) {
  798. if (!rfbSendUpdateBuf(cl))
  799. return FALSE;
  800. }
  801. memcpy(&cl->updateBuf[cl->ublen], &tightAfterBuf[i], portionLen);
  802. cl->ublen += portionLen;
  803. }
  804. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, compressedLen);
  805. return TRUE;
  806. }
  807. /*
  808. * Code to determine how many different colors used in rectangle.
  809. */
  810. static void
  811. FillPalette8(int count)
  812. {
  813. uint8_t *data = (uint8_t *)tightBeforeBuf;
  814. uint8_t c0, c1;
  815. int i, n0, n1;
  816. paletteNumColors = 0;
  817. c0 = data[0];
  818. for (i = 1; i < count && data[i] == c0; i++);
  819. if (i == count) {
  820. paletteNumColors = 1;
  821. return; /* Solid rectangle */
  822. }
  823. if (paletteMaxColors < 2)
  824. return;
  825. n0 = i;
  826. c1 = data[i];
  827. n1 = 0;
  828. for (i++; i < count; i++) {
  829. if (data[i] == c0) {
  830. n0++;
  831. } else if (data[i] == c1) {
  832. n1++;
  833. } else
  834. break;
  835. }
  836. if (i == count) {
  837. if (n0 > n1) {
  838. monoBackground = (uint32_t)c0;
  839. monoForeground = (uint32_t)c1;
  840. } else {
  841. monoBackground = (uint32_t)c1;
  842. monoForeground = (uint32_t)c0;
  843. }
  844. paletteNumColors = 2; /* Two colors */
  845. }
  846. }
  847. #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
  848. \
  849. static void \
  850. FillPalette##bpp(int count) { \
  851. uint##bpp##_t *data = (uint##bpp##_t *)tightBeforeBuf; \
  852. uint##bpp##_t c0, c1, ci; \
  853. int i, n0, n1, ni; \
  854. \
  855. c0 = data[0]; \
  856. for (i = 1; i < count && data[i] == c0; i++); \
  857. if (i >= count) { \
  858. paletteNumColors = 1; /* Solid rectangle */ \
  859. return; \
  860. } \
  861. \
  862. if (paletteMaxColors < 2) { \
  863. paletteNumColors = 0; /* Full-color encoding preferred */ \
  864. return; \
  865. } \
  866. \
  867. n0 = i; \
  868. c1 = data[i]; \
  869. n1 = 0; \
  870. for (i++; i < count; i++) { \
  871. ci = data[i]; \
  872. if (ci == c0) { \
  873. n0++; \
  874. } else if (ci == c1) { \
  875. n1++; \
  876. } else \
  877. break; \
  878. } \
  879. if (i >= count) { \
  880. if (n0 > n1) { \
  881. monoBackground = (uint32_t)c0; \
  882. monoForeground = (uint32_t)c1; \
  883. } else { \
  884. monoBackground = (uint32_t)c1; \
  885. monoForeground = (uint32_t)c0; \
  886. } \
  887. paletteNumColors = 2; /* Two colors */ \
  888. return; \
  889. } \
  890. \
  891. PaletteReset(); \
  892. PaletteInsert (c0, (uint32_t)n0, bpp); \
  893. PaletteInsert (c1, (uint32_t)n1, bpp); \
  894. \
  895. ni = 1; \
  896. for (i++; i < count; i++) { \
  897. if (data[i] == ci) { \
  898. ni++; \
  899. } else { \
  900. if (!PaletteInsert (ci, (uint32_t)ni, bpp)) \
  901. return; \
  902. ci = data[i]; \
  903. ni = 1; \
  904. } \
  905. } \
  906. PaletteInsert (ci, (uint32_t)ni, bpp); \
  907. }
  908. DEFINE_FILL_PALETTE_FUNCTION(16)
  909. DEFINE_FILL_PALETTE_FUNCTION(32)
  910. /*
  911. * Functions to operate with palette structures.
  912. */
  913. #define HASH_FUNC16(rgb) ((int)(((rgb >> 8) + rgb) & 0xFF))
  914. #define HASH_FUNC32(rgb) ((int)(((rgb >> 16) + (rgb >> 8)) & 0xFF))
  915. static void
  916. PaletteReset(void)
  917. {
  918. paletteNumColors = 0;
  919. memset(palette.hash, 0, 256 * sizeof(COLOR_LIST *));
  920. }
  921. static int
  922. PaletteInsert(uint32_t rgb,
  923. int numPixels,
  924. int bpp)
  925. {
  926. COLOR_LIST *pnode;
  927. COLOR_LIST *prev_pnode = NULL;
  928. int hash_key, idx, new_idx, count;
  929. hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);
  930. pnode = palette.hash[hash_key];
  931. while (pnode != NULL) {
  932. if (pnode->rgb == rgb) {
  933. /* Such palette entry already exists. */
  934. new_idx = idx = pnode->idx;
  935. count = palette.entry[idx].numPixels + numPixels;
  936. if (new_idx && palette.entry[new_idx-1].numPixels < count) {
  937. do {
  938. palette.entry[new_idx] = palette.entry[new_idx-1];
  939. palette.entry[new_idx].listNode->idx = new_idx;
  940. new_idx--;
  941. }
  942. while (new_idx && palette.entry[new_idx-1].numPixels < count);
  943. palette.entry[new_idx].listNode = pnode;
  944. pnode->idx = new_idx;
  945. }
  946. palette.entry[new_idx].numPixels = count;
  947. return paletteNumColors;
  948. }
  949. prev_pnode = pnode;
  950. pnode = pnode->next;
  951. }
  952. /* Check if palette is full. */
  953. if (paletteNumColors == 256 || paletteNumColors == paletteMaxColors) {
  954. paletteNumColors = 0;
  955. return 0;
  956. }
  957. /* Move palette entries with lesser pixel counts. */
  958. for ( idx = paletteNumColors;
  959. idx > 0 && palette.entry[idx-1].numPixels < numPixels;
  960. idx-- ) {
  961. palette.entry[idx] = palette.entry[idx-1];
  962. palette.entry[idx].listNode->idx = idx;
  963. }
  964. /* Add new palette entry into the freed slot. */
  965. pnode = &palette.list[paletteNumColors];
  966. if (prev_pnode != NULL) {
  967. prev_pnode->next = pnode;
  968. } else {
  969. palette.hash[hash_key] = pnode;
  970. }
  971. pnode->next = NULL;
  972. pnode->idx = idx;
  973. pnode->rgb = rgb;
  974. palette.entry[idx].listNode = pnode;
  975. palette.entry[idx].numPixels = numPixels;
  976. return (++paletteNumColors);
  977. }
  978. /*
  979. * Converting 32-bit color samples into 24-bit colors.
  980. * Should be called only when redMax, greenMax and blueMax are 255.
  981. * Color components assumed to be byte-aligned.
  982. */
  983. static void Pack24(rfbClientPtr cl,
  984. char *buf,
  985. rfbPixelFormat *fmt,
  986. int count)
  987. {
  988. uint32_t *buf32;
  989. uint32_t pix;
  990. int r_shift, g_shift, b_shift;
  991. buf32 = (uint32_t *)buf;
  992. if (!cl->screen->serverFormat.bigEndian == !fmt->bigEndian) {
  993. r_shift = fmt->redShift;
  994. g_shift = fmt->greenShift;
  995. b_shift = fmt->blueShift;
  996. } else {
  997. r_shift = 24 - fmt->redShift;
  998. g_shift = 24 - fmt->greenShift;
  999. b_shift = 24 - fmt->blueShift;
  1000. }
  1001. while (count--) {
  1002. pix = *buf32++;
  1003. *buf++ = (char)(pix >> r_shift);
  1004. *buf++ = (char)(pix >> g_shift);
  1005. *buf++ = (char)(pix >> b_shift);
  1006. }
  1007. }
  1008. /*
  1009. * Converting truecolor samples into palette indices.
  1010. */
  1011. #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
  1012. \
  1013. static void \
  1014. EncodeIndexedRect##bpp(uint8_t *buf, int count) { \
  1015. COLOR_LIST *pnode; \
  1016. uint##bpp##_t *src; \
  1017. uint##bpp##_t rgb; \
  1018. int rep = 0; \
  1019. \
  1020. src = (uint##bpp##_t *) buf; \
  1021. \
  1022. while (count--) { \
  1023. rgb = *src++; \
  1024. while (count && *src == rgb) { \
  1025. rep++, src++, count--; \
  1026. } \
  1027. pnode = palette.hash[HASH_FUNC##bpp(rgb)]; \
  1028. while (pnode != NULL) { \
  1029. if ((uint##bpp##_t)pnode->rgb == rgb) { \
  1030. *buf++ = (uint8_t)pnode->idx; \
  1031. while (rep) { \
  1032. *buf++ = (uint8_t)pnode->idx; \
  1033. rep--; \
  1034. } \
  1035. break; \
  1036. } \
  1037. pnode = pnode->next; \
  1038. } \
  1039. } \
  1040. }
  1041. DEFINE_IDX_ENCODE_FUNCTION(16)
  1042. DEFINE_IDX_ENCODE_FUNCTION(32)
  1043. #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
  1044. \
  1045. static void \
  1046. EncodeMonoRect##bpp(uint8_t *buf, int w, int h) { \
  1047. uint##bpp##_t *ptr; \
  1048. uint##bpp##_t bg; \
  1049. unsigned int value, mask; \
  1050. int aligned_width; \
  1051. int x, y, bg_bits; \
  1052. \
  1053. ptr = (uint##bpp##_t *) buf; \
  1054. bg = (uint##bpp##_t) monoBackground; \
  1055. aligned_width = w - w % 8; \
  1056. \
  1057. for (y = 0; y < h; y++) { \
  1058. for (x = 0; x < aligned_width; x += 8) { \
  1059. for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
  1060. if (*ptr++ != bg) \
  1061. break; \
  1062. } \
  1063. if (bg_bits == 8) { \
  1064. *buf++ = 0; \
  1065. continue; \
  1066. } \
  1067. mask = 0x80 >> bg_bits; \
  1068. value = mask; \
  1069. for (bg_bits++; bg_bits < 8; bg_bits++) { \
  1070. mask >>= 1; \
  1071. if (*ptr++ != bg) { \
  1072. value |= mask; \
  1073. } \
  1074. } \
  1075. *buf++ = (uint8_t)value; \
  1076. } \
  1077. \
  1078. mask = 0x80; \
  1079. value = 0; \
  1080. if (x >= w) \
  1081. continue; \
  1082. \
  1083. for (; x < w; x++) { \
  1084. if (*ptr++ != bg) { \
  1085. value |= mask; \
  1086. } \
  1087. mask >>= 1; \
  1088. } \
  1089. *buf++ = (uint8_t)value; \
  1090. } \
  1091. }
  1092. DEFINE_MONO_ENCODE_FUNCTION(8)
  1093. DEFINE_MONO_ENCODE_FUNCTION(16)
  1094. DEFINE_MONO_ENCODE_FUNCTION(32)
  1095. /*
  1096. * ``Gradient'' filter for 24-bit color samples.
  1097. * Should be called only when redMax, greenMax and blueMax are 255.
  1098. * Color components assumed to be byte-aligned.
  1099. */
  1100. static void
  1101. FilterGradient24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int w, int h)
  1102. {
  1103. uint32_t *buf32;
  1104. uint32_t pix32;
  1105. int *prevRowPtr;
  1106. int shiftBits[3];
  1107. int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];
  1108. int prediction;
  1109. int x, y, c;
  1110. buf32 = (uint32_t *)buf;
  1111. memset (prevRowBuf, 0, w * 3 * sizeof(int));
  1112. if (!cl->screen->serverFormat.bigEndian == !fmt->bigEndian) {
  1113. shiftBits[0] = fmt->redShift;
  1114. shiftBits[1] = fmt->greenShift;
  1115. shiftBits[2] = fmt->blueShift;
  1116. } else {
  1117. shiftBits[0] = 24 - fmt->redShift;
  1118. shiftBits[1] = 24 - fmt->greenShift;
  1119. shiftBits[2] = 24 - fmt->blueShift;
  1120. }
  1121. for (y = 0; y < h; y++) {
  1122. for (c = 0; c < 3; c++) {
  1123. pixUpper[c] = 0;
  1124. pixHere[c] = 0;
  1125. }
  1126. prevRowPtr = prevRowBuf;
  1127. for (x = 0; x < w; x++) {
  1128. pix32 = *buf32++;
  1129. for (c = 0; c < 3; c++) {
  1130. pixUpperLeft[c] = pixUpper[c];
  1131. pixLeft[c] = pixHere[c];
  1132. pixUpper[c] = *prevRowPtr;
  1133. pixHere[c] = (int)(pix32 >> shiftBits[c] & 0xFF);
  1134. *prevRowPtr++ = pixHere[c];
  1135. prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];
  1136. if (prediction < 0) {
  1137. prediction = 0;
  1138. } else if (prediction > 0xFF) {
  1139. prediction = 0xFF;
  1140. }
  1141. *buf++ = (char)(pixHere[c] - prediction);
  1142. }
  1143. }
  1144. }
  1145. }
  1146. /*
  1147. * ``Gradient'' filter for other color depths.
  1148. */
  1149. #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
  1150. \
  1151. static void \
  1152. FilterGradient##bpp(rfbClientPtr cl, uint##bpp##_t *buf, \
  1153. rfbPixelFormat *fmt, int w, int h) { \
  1154. uint##bpp##_t pix, diff; \
  1155. rfbBool endianMismatch; \
  1156. int *prevRowPtr; \
  1157. int maxColor[3], shiftBits[3]; \
  1158. int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3]; \
  1159. int prediction; \
  1160. int x, y, c; \
  1161. \
  1162. memset (prevRowBuf, 0, w * 3 * sizeof(int)); \
  1163. \
  1164. endianMismatch = (!cl->screen->serverFormat.bigEndian != !fmt->bigEndian); \
  1165. \
  1166. maxColor[0] = fmt->redMax; \
  1167. maxColor[1] = fmt->greenMax; \
  1168. maxColor[2] = fmt->blueMax; \
  1169. shiftBits[0] = fmt->redShift; \
  1170. shiftBits[1] = fmt->greenShift; \
  1171. shiftBits[2] = fmt->blueShift; \
  1172. \
  1173. for (y = 0; y < h; y++) { \
  1174. for (c = 0; c < 3; c++) { \
  1175. pixUpper[c] = 0; \
  1176. pixHere[c] = 0; \
  1177. } \
  1178. prevRowPtr = prevRowBuf; \
  1179. for (x = 0; x < w; x++) { \
  1180. pix = *buf; \
  1181. if (endianMismatch) { \
  1182. pix = Swap##bpp(pix); \
  1183. } \
  1184. diff = 0; \
  1185. for (c = 0; c < 3; c++) { \
  1186. pixUpperLeft[c] = pixUpper[c]; \
  1187. pixLeft[c] = pixHere[c]; \
  1188. pixUpper[c] = *prevRowPtr; \
  1189. pixHere[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
  1190. *prevRowPtr++ = pixHere[c]; \
  1191. \
  1192. prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c]; \
  1193. if (prediction < 0) { \
  1194. prediction = 0; \
  1195. } else if (prediction > maxColor[c]) { \
  1196. prediction = maxColor[c]; \
  1197. } \
  1198. diff |= ((pixHere[c] - prediction) & maxColor[c]) \
  1199. << shiftBits[c]; \
  1200. } \
  1201. if (endianMismatch) { \
  1202. diff = Swap##bpp(diff); \
  1203. } \
  1204. *buf++ = diff; \
  1205. } \
  1206. } \
  1207. }
  1208. DEFINE_GRADIENT_FILTER_FUNCTION(16)
  1209. DEFINE_GRADIENT_FILTER_FUNCTION(32)
  1210. /*
  1211. * Code to guess if given rectangle is suitable for smooth image
  1212. * compression (by applying "gradient" filter or JPEG coder).
  1213. */
  1214. #define JPEG_MIN_RECT_SIZE 4096
  1215. #define DETECT_SUBROW_WIDTH 7
  1216. #define DETECT_MIN_WIDTH 8
  1217. #define DETECT_MIN_HEIGHT 8
  1218. static int
  1219. DetectSmoothImage (rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h)
  1220. {
  1221. long avgError;
  1222. if ( cl->screen->serverFormat.bitsPerPixel == 8 || fmt->bitsPerPixel == 8 ||
  1223. w < DETECT_MIN_WIDTH || h < DETECT_MIN_HEIGHT ) {
  1224. return 0;
  1225. }
  1226. if (qualityLevel != -1) {
  1227. if (w * h < JPEG_MIN_RECT_SIZE) {
  1228. return 0;
  1229. }
  1230. } else {
  1231. if ( rfbTightDisableGradient ||
  1232. w * h < tightConf[compressLevel].gradientMinRectSize ) {
  1233. return 0;
  1234. }
  1235. }
  1236. if (fmt->bitsPerPixel == 32) {
  1237. if (usePixelFormat24) {
  1238. avgError = DetectSmoothImage24(cl, fmt, w, h);
  1239. if (qualityLevel != -1) {
  1240. return (avgError < tightConf[qualityLevel].jpegThreshold24);
  1241. }
  1242. return (avgError < tightConf[compressLevel].gradientThreshold24);
  1243. } else {
  1244. avgError = DetectSmoothImage32(cl, fmt, w, h);
  1245. }
  1246. } else {
  1247. avgError = DetectSmoothImage16(cl, fmt, w, h);
  1248. }
  1249. if (qualityLevel != -1) {
  1250. return (avgError < tightConf[qualityLevel].jpegThreshold);
  1251. }
  1252. return (avgError < tightConf[compressLevel].gradientThreshold);
  1253. }
  1254. static unsigned long
  1255. DetectSmoothImage24 (rfbClientPtr cl,
  1256. rfbPixelFormat *fmt,
  1257. int w,
  1258. int h)
  1259. {
  1260. int off;
  1261. int x, y, d, dx, c;
  1262. int diffStat[256];
  1263. int pixelCount = 0;
  1264. int pix, left[3];
  1265. unsigned long avgError;
  1266. /* If client is big-endian, color samples begin from the second
  1267. byte (offset 1) of a 32-bit pixel value. */
  1268. off = (fmt->bigEndian != 0);
  1269. memset(diffStat, 0, 256*sizeof(int));
  1270. y = 0, x = 0;
  1271. while (y < h && x < w) {
  1272. for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) {
  1273. for (c = 0; c < 3; c++) {
  1274. left[c] = (int)tightBeforeBuf[((y+d)*w+x+d)*4+off+c] & 0xFF;
  1275. }
  1276. for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) {
  1277. for (c = 0; c < 3; c++) {
  1278. pix = (int)tightBeforeBuf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
  1279. diffStat[abs(pix - left[c])]++;
  1280. left[c] = pix;
  1281. }
  1282. pixelCount++;
  1283. }
  1284. }
  1285. if (w > h) {
  1286. x += h;
  1287. y = 0;
  1288. } else {
  1289. x = 0;
  1290. y += w;
  1291. }
  1292. }
  1293. if (diffStat[0] * 33 / pixelCount >= 95)
  1294. return 0;
  1295. avgError = 0;
  1296. for (c = 1; c < 8; c++) {
  1297. avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
  1298. if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2)
  1299. return 0;
  1300. }
  1301. for (; c < 256; c++) {
  1302. avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
  1303. }
  1304. avgError /= (pixelCount * 3 - diffStat[0]);
  1305. return avgError;
  1306. }
  1307. #define DEFINE_DETECT_FUNCTION(bpp) \
  1308. \
  1309. static unsigned long \
  1310. DetectSmoothImage##bpp (rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h) {\
  1311. rfbBool endianMismatch; \
  1312. uint##bpp##_t pix; \
  1313. int maxColor[3], shiftBits[3]; \
  1314. int x, y, d, dx, c; \
  1315. int diffStat[256]; \
  1316. int pixelCount = 0; \
  1317. int sample, sum, left[3]; \
  1318. unsigned long avgError; \
  1319. \
  1320. endianMismatch = (!cl->screen->serverFormat.bigEndian != !fmt->bigEndian); \
  1321. \
  1322. maxColor[0] = fmt->redMax; \
  1323. maxColor[1] = fmt->greenMax; \
  1324. maxColor[2] = fmt->blueMax; \
  1325. shiftBits[0] = fmt->redShift; \
  1326. shiftBits[1] = fmt->greenShift; \
  1327. shiftBits[2] = fmt->blueShift; \
  1328. \
  1329. memset(diffStat, 0, 256*sizeof(int)); \
  1330. \
  1331. y = 0, x = 0; \
  1332. while (y < h && x < w) { \
  1333. for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) { \
  1334. pix = ((uint##bpp##_t *)tightBeforeBuf)[(y+d)*w+x+d]; \
  1335. if (endianMismatch) { \
  1336. pix = Swap##bpp(pix); \
  1337. } \
  1338. for (c = 0; c < 3; c++) { \
  1339. left[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
  1340. } \
  1341. for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) { \
  1342. pix = ((uint##bpp##_t *)tightBeforeBuf)[(y+d)*w+x+d+dx]; \
  1343. if (endianMismatch) { \
  1344. pix = Swap##bpp(pix); \
  1345. } \
  1346. sum = 0; \
  1347. for (c = 0; c < 3; c++) { \
  1348. sample = (int)(pix >> shiftBits[c] & maxColor[c]); \
  1349. sum += abs(sample - left[c]); \
  1350. left[c] = sample; \
  1351. } \
  1352. if (sum > 255) \
  1353. sum = 255; \
  1354. diffStat[sum]++; \
  1355. pixelCount++; \
  1356. } \
  1357. } \
  1358. if (w > h) { \
  1359. x += h; \
  1360. y = 0; \
  1361. } else { \
  1362. x = 0; \
  1363. y += w; \
  1364. } \
  1365. } \
  1366. \
  1367. if ((diffStat[0] + diffStat[1]) * 100 / pixelCount >= 90) \
  1368. return 0; \
  1369. \
  1370. avgError = 0; \
  1371. for (c = 1; c < 8; c++) { \
  1372. avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
  1373. if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2) \
  1374. return 0; \
  1375. } \
  1376. for (; c < 256; c++) { \
  1377. avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
  1378. } \
  1379. avgError /= (pixelCount - diffStat[0]); \
  1380. \
  1381. return avgError; \
  1382. }
  1383. DEFINE_DETECT_FUNCTION(16)
  1384. DEFINE_DETECT_FUNCTION(32)
  1385. /*
  1386. * JPEG compression stuff.
  1387. */
  1388. static struct jpeg_destination_mgr jpegDstManager;
  1389. static rfbBool jpegError;
  1390. static int jpegDstDataLen;
  1391. static rfbBool
  1392. SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h, int quality)
  1393. {
  1394. struct jpeg_compress_struct cinfo;
  1395. struct jpeg_error_mgr jerr;
  1396. uint8_t *srcBuf;
  1397. JSAMPROW rowPointer[1];
  1398. int dy;
  1399. if (cl->screen->serverFormat.bitsPerPixel == 8)
  1400. return SendFullColorRect(cl, w, h);
  1401. srcBuf = (uint8_t *)malloc(w * 3);
  1402. if (srcBuf == NULL) {
  1403. return SendFullColorRect(cl, w, h);
  1404. }
  1405. rowPointer[0] = srcBuf;
  1406. cinfo.err = jpeg_std_error(&jerr);
  1407. jpeg_create_compress(&cinfo);
  1408. cinfo.image_width = w;
  1409. cinfo.image_height = h;
  1410. cinfo.input_components = 3;
  1411. cinfo.in_color_space = JCS_RGB;
  1412. jpeg_set_defaults(&cinfo);
  1413. jpeg_set_quality(&cinfo, quality, TRUE);
  1414. JpegSetDstManager (&cinfo);
  1415. jpeg_start_compress(&cinfo, TRUE);
  1416. for (dy = 0; dy < h; dy++) {
  1417. PrepareRowForJpeg(cl, srcBuf, x, y + dy, w);
  1418. jpeg_write_scanlines(&cinfo, rowPointer, 1);
  1419. if (jpegError)
  1420. break;
  1421. }
  1422. if (!jpegError)
  1423. jpeg_finish_compress(&cinfo);
  1424. jpeg_destroy_compress(&cinfo);
  1425. free(srcBuf);
  1426. if (jpegError)
  1427. return SendFullColorRect(cl, w, h);
  1428. if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
  1429. if (!rfbSendUpdateBuf(cl))
  1430. return FALSE;
  1431. }
  1432. cl->updateBuf[cl->ublen++] = (char)(rfbTightJpeg << 4);
  1433. rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
  1434. return SendCompressedData(cl, jpegDstDataLen);
  1435. }
  1436. static void
  1437. PrepareRowForJpeg(rfbClientPtr cl,
  1438. uint8_t *dst,
  1439. int x,
  1440. int y,
  1441. int count)
  1442. {
  1443. if (cl->screen->serverFormat.bitsPerPixel == 32) {
  1444. if ( cl->screen->serverFormat.redMax == 0xFF &&
  1445. cl->screen->serverFormat.greenMax == 0xFF &&
  1446. cl->screen->serverFormat.blueMax == 0xFF ) {
  1447. PrepareRowForJpeg24(cl, dst, x, y, count);
  1448. } else {
  1449. PrepareRowForJpeg32(cl, dst, x, y, count);
  1450. }
  1451. } else {
  1452. /* 16 bpp assumed. */
  1453. PrepareRowForJpeg16(cl, dst, x, y, count);
  1454. }
  1455. }
  1456. static void
  1457. PrepareRowForJpeg24(rfbClientPtr cl,
  1458. uint8_t *dst,
  1459. int x,
  1460. int y,
  1461. int count)
  1462. {
  1463. uint32_t *fbptr;
  1464. uint32_t pix;
  1465. fbptr = (uint32_t *)
  1466. &cl->scaledScreen->frameBuffer[y * cl->scaledScreen->paddedWidthInBytes + x * 4];
  1467. while (count--) {
  1468. pix = *fbptr++;
  1469. *dst++ = (uint8_t)(pix >> cl->screen->serverFormat.redShift);
  1470. *dst++ = (uint8_t)(pix >> cl->screen->serverFormat.