PageRenderTime 882ms CodeModel.GetById 162ms app.highlight 514ms RepoModel.GetById 190ms app.codeStats 1ms

/apps/desktop/libvncserver/tight.c

http://ftk.googlecode.com/
C | 1731 lines | 1354 code | 261 blank | 116 comment | 305 complexity | 95bdc96328834640e8a56ddbddb4ca7e MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*
   2 * tight.c
   3 *
   4 * Routines to implement Tight Encoding
   5 */
   6
   7/*
   8 *  Copyright (C) 2000, 2001 Const Kaplinsky.  All Rights Reserved.
   9 *  Copyright (C) 1999 AT&T Laboratories Cambridge.  All Rights Reserved.
  10 *
  11 *  This is free software; you can redistribute it and/or modify
  12 *  it under the terms of the GNU General Public License as published by
  13 *  the Free Software Foundation; either version 2 of the License, or
  14 *  (at your option) any later version.
  15 *
  16 *  This software is distributed in the hope that it will be useful,
  17 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 *  GNU General Public License for more details.
  20 *
  21 *  You should have received a copy of the GNU General Public License
  22 *  along with this software; if not, write to the Free Software
  23 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
  24 *  USA.
  25 */
  26
  27/*#include <stdio.h>*/
  28#include <rfb/rfb.h>
  29#include "private.h"
  30
  31#ifdef WIN32
  32#define XMD_H
  33#undef FAR
  34#define NEEDFAR_POINTERS
  35#endif
  36
  37#include <jpeglib.h>
  38
  39/* Note: The following constant should not be changed. */
  40#define TIGHT_MIN_TO_COMPRESS 12
  41
  42/* The parameters below may be adjusted. */
  43#define MIN_SPLIT_RECT_SIZE     4096
  44#define MIN_SOLID_SUBRECT_SIZE  2048
  45#define MAX_SPLIT_TILE_SIZE       16
  46
  47/* May be set to TRUE with "-lazytight" Xvnc option. */
  48rfbBool rfbTightDisableGradient = FALSE;
  49
  50/* This variable is set on every rfbSendRectEncodingTight() call. */
  51static rfbBool usePixelFormat24;
  52
  53
  54/* Compression level stuff. The following array contains various
  55   encoder parameters for each of 10 compression levels (0..9).
  56   Last three parameters correspond to JPEG quality levels (0..9). */
  57
  58typedef struct TIGHT_CONF_s {
  59    int maxRectSize, maxRectWidth;
  60    int monoMinRectSize, gradientMinRectSize;
  61    int idxZlibLevel, monoZlibLevel, rawZlibLevel, gradientZlibLevel;
  62    int gradientThreshold, gradientThreshold24;
  63    int idxMaxColorsDivisor;
  64    int jpegQuality, jpegThreshold, jpegThreshold24;
  65} TIGHT_CONF;
  66
  67static TIGHT_CONF tightConf[10] = {
  68    {   512,   32,   6, 65536, 0, 0, 0, 0,   0,   0,   4,  5, 10000, 23000 },
  69    {  2048,  128,   6, 65536, 1, 1, 1, 0,   0,   0,   8, 10,  8000, 18000 },
  70    {  6144,  256,   8, 65536, 3, 3, 2, 0,   0,   0,  24, 15,  6500, 15000 },
  71    { 10240, 1024,  12, 65536, 5, 5, 3, 0,   0,   0,  32, 25,  5000, 12000 },
  72    { 16384, 2048,  12, 65536, 6, 6, 4, 0,   0,   0,  32, 37,  4000, 10000 },
  73    { 32768, 2048,  12,  4096, 7, 7, 5, 4, 150, 380,  32, 50,  3000,  8000 },
  74    { 65536, 2048,  16,  4096, 7, 7, 6, 4, 170, 420,  48, 60,  2000,  5000 },
  75    { 65536, 2048,  16,  4096, 8, 8, 7, 5, 180, 450,  64, 70,  1000,  2500 },
  76    { 65536, 2048,  32,  8192, 9, 9, 8, 6, 190, 475,  64, 75,   500,  1200 },
  77    { 65536, 2048,  32,  8192, 9, 9, 9, 6, 200, 500,  96, 80,   200,   500 }
  78};
  79
  80static int compressLevel;
  81static int qualityLevel;
  82
  83/* Stuff dealing with palettes. */
  84
  85typedef struct COLOR_LIST_s {
  86    struct COLOR_LIST_s *next;
  87    int idx;
  88    uint32_t rgb;
  89} COLOR_LIST;
  90
  91typedef struct PALETTE_ENTRY_s {
  92    COLOR_LIST *listNode;
  93    int numPixels;
  94} PALETTE_ENTRY;
  95
  96typedef struct PALETTE_s {
  97    PALETTE_ENTRY entry[256];
  98    COLOR_LIST *hash[256];
  99    COLOR_LIST list[256];
 100} PALETTE;
 101
 102/* TODO: move into rfbScreen struct */
 103static int paletteNumColors, paletteMaxColors;
 104static uint32_t monoBackground, monoForeground;
 105static PALETTE palette;
 106
 107/* Pointers to dynamically-allocated buffers. */
 108
 109static int tightBeforeBufSize = 0;
 110static char *tightBeforeBuf = NULL;
 111
 112static int tightAfterBufSize = 0;
 113static char *tightAfterBuf = NULL;
 114
 115static int *prevRowBuf = NULL;
 116
 117void rfbTightCleanup(rfbScreenInfoPtr screen)
 118{
 119  if(tightBeforeBufSize) {
 120    free(tightBeforeBuf);
 121    tightBeforeBufSize=0;
 122  }
 123  if(tightAfterBufSize) {
 124    free(tightAfterBuf);
 125    tightAfterBufSize=0;
 126  }
 127}
 128
 129/* Prototypes for static functions. */
 130
 131static void FindBestSolidArea (rfbClientPtr cl, int x, int y, int w, int h,
 132                               uint32_t colorValue, int *w_ptr, int *h_ptr);
 133static void ExtendSolidArea   (rfbClientPtr cl, int x, int y, int w, int h,
 134                               uint32_t colorValue,
 135                               int *x_ptr, int *y_ptr, int *w_ptr, int *h_ptr);
 136static rfbBool CheckSolidTile    (rfbClientPtr cl, int x, int y, int w, int h,
 137                               uint32_t *colorPtr, rfbBool needSameColor);
 138static rfbBool CheckSolidTile8   (rfbClientPtr cl, int x, int y, int w, int h,
 139                               uint32_t *colorPtr, rfbBool needSameColor);
 140static rfbBool CheckSolidTile16  (rfbClientPtr cl, int x, int y, int w, int h,
 141                               uint32_t *colorPtr, rfbBool needSameColor);
 142static rfbBool CheckSolidTile32  (rfbClientPtr cl, int x, int y, int w, int h,
 143                               uint32_t *colorPtr, rfbBool needSameColor);
 144
 145static rfbBool SendRectSimple    (rfbClientPtr cl, int x, int y, int w, int h);
 146static rfbBool SendSubrect       (rfbClientPtr cl, int x, int y, int w, int h);
 147static rfbBool SendTightHeader   (rfbClientPtr cl, int x, int y, int w, int h);
 148
 149static rfbBool SendSolidRect     (rfbClientPtr cl);
 150static rfbBool SendMonoRect      (rfbClientPtr cl, int w, int h);
 151static rfbBool SendIndexedRect   (rfbClientPtr cl, int w, int h);
 152static rfbBool SendFullColorRect (rfbClientPtr cl, int w, int h);
 153static rfbBool SendGradientRect  (rfbClientPtr cl, int w, int h);
 154
 155static rfbBool CompressData(rfbClientPtr cl, int streamId, int dataLen,
 156                         int zlibLevel, int zlibStrategy);
 157static rfbBool SendCompressedData(rfbClientPtr cl, int compressedLen);
 158
 159static void FillPalette8(int count);
 160static void FillPalette16(int count);
 161static void FillPalette32(int count);
 162
 163static void PaletteReset(void);
 164static int PaletteInsert(uint32_t rgb, int numPixels, int bpp);
 165
 166static void Pack24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int count);
 167
 168static void EncodeIndexedRect16(uint8_t *buf, int count);
 169static void EncodeIndexedRect32(uint8_t *buf, int count);
 170
 171static void EncodeMonoRect8(uint8_t *buf, int w, int h);
 172static void EncodeMonoRect16(uint8_t *buf, int w, int h);
 173static void EncodeMonoRect32(uint8_t *buf, int w, int h);
 174
 175static void FilterGradient24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int w, int h);
 176static void FilterGradient16(rfbClientPtr cl, uint16_t *buf, rfbPixelFormat *fmt, int w, int h);
 177static void FilterGradient32(rfbClientPtr cl, uint32_t *buf, rfbPixelFormat *fmt, int w, int h);
 178
 179static int DetectSmoothImage(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
 180static unsigned long DetectSmoothImage24(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
 181static unsigned long DetectSmoothImage16(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
 182static unsigned long DetectSmoothImage32(rfbClientPtr cl, rfbPixelFormat *fmt, int w, int h);
 183
 184static rfbBool SendJpegRect(rfbClientPtr cl, int x, int y, int w, int h,
 185                         int quality);
 186static void PrepareRowForJpeg(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
 187static void PrepareRowForJpeg24(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
 188static void PrepareRowForJpeg16(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
 189static void PrepareRowForJpeg32(rfbClientPtr cl, uint8_t *dst, int x, int y, int count);
 190
 191static void JpegInitDestination(j_compress_ptr cinfo);
 192static boolean JpegEmptyOutputBuffer(j_compress_ptr cinfo);
 193static void JpegTermDestination(j_compress_ptr cinfo);
 194static void JpegSetDstManager(j_compress_ptr cinfo);
 195
 196
 197/*
 198 * Tight encoding implementation.
 199 */
 200
 201int
 202rfbNumCodedRectsTight(rfbClientPtr cl,
 203                      int x,
 204                      int y,
 205                      int w,
 206                      int h)
 207{
 208    int maxRectSize, maxRectWidth;
 209    int subrectMaxWidth, subrectMaxHeight;
 210
 211    /* No matter how many rectangles we will send if LastRect markers
 212       are used to terminate rectangle stream. */
 213    if (cl->enableLastRectEncoding && w * h >= MIN_SPLIT_RECT_SIZE)
 214      return 0;
 215
 216    maxRectSize = tightConf[cl->tightCompressLevel].maxRectSize;
 217    maxRectWidth = tightConf[cl->tightCompressLevel].maxRectWidth;
 218
 219    if (w > maxRectWidth || w * h > maxRectSize) {
 220        subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
 221        subrectMaxHeight = maxRectSize / subrectMaxWidth;
 222        return (((w - 1) / maxRectWidth + 1) *
 223                ((h - 1) / subrectMaxHeight + 1));
 224    } else {
 225        return 1;
 226    }
 227}
 228
 229rfbBool
 230rfbSendRectEncodingTight(rfbClientPtr cl,
 231                         int x,
 232                         int y,
 233                         int w,
 234                         int h)
 235{
 236    int nMaxRows;
 237    uint32_t colorValue;
 238    int dx, dy, dw, dh;
 239    int x_best, y_best, w_best, h_best;
 240    char *fbptr;
 241
 242    rfbSendUpdateBuf(cl);
 243
 244    compressLevel = cl->tightCompressLevel;
 245    qualityLevel = cl->tightQualityLevel;
 246
 247    if ( cl->format.depth == 24 && cl->format.redMax == 0xFF &&
 248         cl->format.greenMax == 0xFF && cl->format.blueMax == 0xFF ) {
 249        usePixelFormat24 = TRUE;
 250    } else {
 251        usePixelFormat24 = FALSE;
 252    }
 253
 254    if (!cl->enableLastRectEncoding || w * h < MIN_SPLIT_RECT_SIZE)
 255        return SendRectSimple(cl, x, y, w, h);
 256
 257    /* Make sure we can write at least one pixel into tightBeforeBuf. */
 258
 259    if (tightBeforeBufSize < 4) {
 260        tightBeforeBufSize = 4;
 261        if (tightBeforeBuf == NULL)
 262            tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
 263        else
 264            tightBeforeBuf = (char *)realloc(tightBeforeBuf,
 265                                              tightBeforeBufSize);
 266    }
 267
 268    /* Calculate maximum number of rows in one non-solid rectangle. */
 269
 270    {
 271        int maxRectSize, maxRectWidth, nMaxWidth;
 272
 273        maxRectSize = tightConf[compressLevel].maxRectSize;
 274        maxRectWidth = tightConf[compressLevel].maxRectWidth;
 275        nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
 276        nMaxRows = maxRectSize / nMaxWidth;
 277    }
 278
 279    /* Try to find large solid-color areas and send them separately. */
 280
 281    for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
 282
 283        /* If a rectangle becomes too large, send its upper part now. */
 284
 285        if (dy - y >= nMaxRows) {
 286            if (!SendRectSimple(cl, x, y, w, nMaxRows))
 287                return 0;
 288            y += nMaxRows;
 289            h -= nMaxRows;
 290        }
 291
 292        dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
 293            MAX_SPLIT_TILE_SIZE : (y + h - dy);
 294
 295        for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {
 296
 297            dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
 298                MAX_SPLIT_TILE_SIZE : (x + w - dx);
 299
 300            if (CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, FALSE)) {
 301
 302                /* Get dimensions of solid-color area. */
 303
 304                FindBestSolidArea(cl, dx, dy, w - (dx - x), h - (dy - y),
 305				  colorValue, &w_best, &h_best);
 306
 307                /* Make sure a solid rectangle is large enough
 308                   (or the whole rectangle is of the same color). */
 309
 310                if ( w_best * h_best != w * h &&
 311                     w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
 312                    continue;
 313
 314                /* Try to extend solid rectangle to maximum size. */
 315
 316                x_best = dx; y_best = dy;
 317                ExtendSolidArea(cl, x, y, w, h, colorValue,
 318                                &x_best, &y_best, &w_best, &h_best);
 319
 320                /* Send rectangles at top and left to solid-color area. */
 321
 322                if ( y_best != y &&
 323                     !SendRectSimple(cl, x, y, w, y_best-y) )
 324                    return FALSE;
 325                if ( x_best != x &&
 326                     !rfbSendRectEncodingTight(cl, x, y_best,
 327                                               x_best-x, h_best) )
 328                    return FALSE;
 329
 330                /* Send solid-color rectangle. */
 331
 332                if (!SendTightHeader(cl, x_best, y_best, w_best, h_best))
 333                    return FALSE;
 334
 335                fbptr = (cl->scaledScreen->frameBuffer +
 336                         (cl->scaledScreen->paddedWidthInBytes * y_best) +
 337                         (x_best * (cl->scaledScreen->bitsPerPixel / 8)));
 338
 339                (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
 340                                   &cl->format, fbptr, tightBeforeBuf,
 341                                   cl->scaledScreen->paddedWidthInBytes, 1, 1);
 342
 343                if (!SendSolidRect(cl))
 344                    return FALSE;
 345
 346                /* Send remaining rectangles (at right and bottom). */
 347
 348                if ( x_best + w_best != x + w &&
 349                     !rfbSendRectEncodingTight(cl, x_best+w_best, y_best,
 350                                               w-(x_best-x)-w_best, h_best) )
 351                    return FALSE;
 352                if ( y_best + h_best != y + h &&
 353                     !rfbSendRectEncodingTight(cl, x, y_best+h_best,
 354                                               w, h-(y_best-y)-h_best) )
 355                    return FALSE;
 356
 357                /* Return after all recursive calls are done. */
 358
 359                return TRUE;
 360            }
 361
 362        }
 363
 364    }
 365
 366    /* No suitable solid-color rectangles found. */
 367
 368    return SendRectSimple(cl, x, y, w, h);
 369}
 370
 371static void
 372FindBestSolidArea(rfbClientPtr cl,
 373                  int x,
 374                  int y,
 375                  int w,
 376                  int h,
 377                  uint32_t colorValue,
 378                  int *w_ptr,
 379                  int *h_ptr)
 380{
 381    int dx, dy, dw, dh;
 382    int w_prev;
 383    int w_best = 0, h_best = 0;
 384
 385    w_prev = w;
 386
 387    for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
 388
 389        dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
 390            MAX_SPLIT_TILE_SIZE : (y + h - dy);
 391        dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
 392            MAX_SPLIT_TILE_SIZE : w_prev;
 393
 394        if (!CheckSolidTile(cl, x, dy, dw, dh, &colorValue, TRUE))
 395            break;
 396
 397        for (dx = x + dw; dx < x + w_prev;) {
 398            dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
 399                MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
 400            if (!CheckSolidTile(cl, dx, dy, dw, dh, &colorValue, TRUE))
 401                break;
 402	    dx += dw;
 403        }
 404
 405        w_prev = dx - x;
 406        if (w_prev * (dy + dh - y) > w_best * h_best) {
 407            w_best = w_prev;
 408            h_best = dy + dh - y;
 409        }
 410    }
 411
 412    *w_ptr = w_best;
 413    *h_ptr = h_best;
 414}
 415
 416static void
 417ExtendSolidArea(rfbClientPtr cl,
 418                int x,
 419                int y,
 420                int w,
 421                int h,
 422                uint32_t colorValue,
 423                int *x_ptr,
 424                int *y_ptr,
 425                int *w_ptr,
 426                int *h_ptr)
 427{
 428    int cx, cy;
 429
 430    /* Try to extend the area upwards. */
 431    for ( cy = *y_ptr - 1;
 432          cy >= y && CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
 433          cy-- );
 434    *h_ptr += *y_ptr - (cy + 1);
 435    *y_ptr = cy + 1;
 436
 437    /* ... downwards. */
 438    for ( cy = *y_ptr + *h_ptr;
 439          cy < y + h &&
 440              CheckSolidTile(cl, *x_ptr, cy, *w_ptr, 1, &colorValue, TRUE);
 441          cy++ );
 442    *h_ptr += cy - (*y_ptr + *h_ptr);
 443
 444    /* ... to the left. */
 445    for ( cx = *x_ptr - 1;
 446          cx >= x && CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
 447          cx-- );
 448    *w_ptr += *x_ptr - (cx + 1);
 449    *x_ptr = cx + 1;
 450
 451    /* ... to the right. */
 452    for ( cx = *x_ptr + *w_ptr;
 453          cx < x + w &&
 454              CheckSolidTile(cl, cx, *y_ptr, 1, *h_ptr, &colorValue, TRUE);
 455          cx++ );
 456    *w_ptr += cx - (*x_ptr + *w_ptr);
 457}
 458
 459/*
 460 * Check if a rectangle is all of the same color. If needSameColor is
 461 * set to non-zero, then also check that its color equals to the
 462 * *colorPtr value. The result is 1 if the test is successfull, and in
 463 * that case new color will be stored in *colorPtr.
 464 */
 465
 466static rfbBool CheckSolidTile(rfbClientPtr cl, int x, int y, int w, int h, uint32_t* colorPtr, rfbBool needSameColor)
 467{
 468    switch(cl->screen->serverFormat.bitsPerPixel) {
 469    case 32:
 470        return CheckSolidTile32(cl, x, y, w, h, colorPtr, needSameColor);
 471    case 16:
 472        return CheckSolidTile16(cl, x, y, w, h, colorPtr, needSameColor);
 473    default:
 474        return CheckSolidTile8(cl, x, y, w, h, colorPtr, needSameColor);
 475    }
 476}
 477
 478#define DEFINE_CHECK_SOLID_FUNCTION(bpp)                                      \
 479                                                                              \
 480static rfbBool                                                                \
 481CheckSolidTile##bpp(rfbClientPtr cl, int x, int y, int w, int h,              \
 482		uint32_t* colorPtr, rfbBool needSameColor)                    \
 483{                                                                             \
 484    uint##bpp##_t *fbptr;                                                     \
 485    uint##bpp##_t colorValue;                                                 \
 486    int dx, dy;                                                               \
 487                                                                              \
 488    fbptr = (uint##bpp##_t *)                                                 \
 489        &cl->scaledScreen->frameBuffer[y * cl->scaledScreen->paddedWidthInBytes + x * (bpp/8)]; \
 490                                                                              \
 491    colorValue = *fbptr;                                                      \
 492    if (needSameColor && (uint32_t)colorValue != *colorPtr)                   \
 493        return FALSE;                                                         \
 494                                                                              \
 495    for (dy = 0; dy < h; dy++) {                                              \
 496        for (dx = 0; dx < w; dx++) {                                          \
 497            if (colorValue != fbptr[dx])                                      \
 498                return FALSE;                                                 \
 499        }                                                                     \
 500        fbptr = (uint##bpp##_t *)((uint8_t *)fbptr + cl->scaledScreen->paddedWidthInBytes); \
 501    }                                                                         \
 502                                                                              \
 503    *colorPtr = (uint32_t)colorValue;                                         \
 504    return TRUE;                                                              \
 505}
 506
 507DEFINE_CHECK_SOLID_FUNCTION(8)
 508DEFINE_CHECK_SOLID_FUNCTION(16)
 509DEFINE_CHECK_SOLID_FUNCTION(32)
 510
 511static rfbBool
 512SendRectSimple(rfbClientPtr cl, int x, int y, int w, int h)
 513{
 514    int maxBeforeSize, maxAfterSize;
 515    int maxRectSize, maxRectWidth;
 516    int subrectMaxWidth, subrectMaxHeight;
 517    int dx, dy;
 518    int rw, rh;
 519
 520    maxRectSize = tightConf[compressLevel].maxRectSize;
 521    maxRectWidth = tightConf[compressLevel].maxRectWidth;
 522
 523    maxBeforeSize = maxRectSize * (cl->format.bitsPerPixel / 8);
 524    maxAfterSize = maxBeforeSize + (maxBeforeSize + 99) / 100 + 12;
 525
 526    if (tightBeforeBufSize < maxBeforeSize) {
 527        tightBeforeBufSize = maxBeforeSize;
 528        if (tightBeforeBuf == NULL)
 529            tightBeforeBuf = (char *)malloc(tightBeforeBufSize);
 530        else
 531            tightBeforeBuf = (char *)realloc(tightBeforeBuf,
 532                                              tightBeforeBufSize);
 533    }
 534
 535    if (tightAfterBufSize < maxAfterSize) {
 536        tightAfterBufSize = maxAfterSize;
 537        if (tightAfterBuf == NULL)
 538            tightAfterBuf = (char *)malloc(tightAfterBufSize);
 539        else
 540            tightAfterBuf = (char *)realloc(tightAfterBuf,
 541                                             tightAfterBufSize);
 542    }
 543
 544    if (w > maxRectWidth || w * h > maxRectSize) {
 545        subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
 546        subrectMaxHeight = maxRectSize / subrectMaxWidth;
 547
 548        for (dy = 0; dy < h; dy += subrectMaxHeight) {
 549            for (dx = 0; dx < w; dx += maxRectWidth) {
 550                rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
 551                rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
 552                if (!SendSubrect(cl, x+dx, y+dy, rw, rh))
 553                    return FALSE;
 554            }
 555        }
 556    } else {
 557        if (!SendSubrect(cl, x, y, w, h))
 558            return FALSE;
 559    }
 560
 561    return TRUE;
 562}
 563
 564static rfbBool
 565SendSubrect(rfbClientPtr cl,
 566            int x,
 567            int y,
 568            int w,
 569            int h)
 570{
 571    char *fbptr;
 572    rfbBool success = FALSE;
 573
 574    /* Send pending data if there is more than 128 bytes. */
 575    if (cl->ublen > 128) {
 576        if (!rfbSendUpdateBuf(cl))
 577            return FALSE;
 578    }
 579
 580    if (!SendTightHeader(cl, x, y, w, h))
 581        return FALSE;
 582
 583    fbptr = (cl->scaledScreen->frameBuffer + (cl->scaledScreen->paddedWidthInBytes * y)
 584             + (x * (cl->scaledScreen->bitsPerPixel / 8)));
 585
 586    (*cl->translateFn)(cl->translateLookupTable, &cl->screen->serverFormat,
 587                       &cl->format, fbptr, tightBeforeBuf,
 588                       cl->scaledScreen->paddedWidthInBytes, w, h);
 589
 590    paletteMaxColors = w * h / tightConf[compressLevel].idxMaxColorsDivisor;
 591    if ( paletteMaxColors < 2 &&
 592         w * h >= tightConf[compressLevel].monoMinRectSize ) {
 593        paletteMaxColors = 2;
 594    }
 595    switch (cl->format.bitsPerPixel) {
 596    case 8:
 597        FillPalette8(w * h);
 598        break;
 599    case 16:
 600        FillPalette16(w * h);
 601        break;
 602    default:
 603        FillPalette32(w * h);
 604    }
 605
 606    switch (paletteNumColors) {
 607    case 0:
 608        /* Truecolor image */
 609        if (DetectSmoothImage(cl, &cl->format, w, h)) {
 610            if (qualityLevel != -1) {
 611                success = SendJpegRect(cl, x, y, w, h,
 612                                       tightConf[qualityLevel].jpegQuality);
 613            } else {
 614                success = SendGradientRect(cl, w, h);
 615            }
 616        } else {
 617            success = SendFullColorRect(cl, w, h);
 618        }
 619        break;
 620    case 1:
 621        /* Solid rectangle */
 622        success = SendSolidRect(cl);
 623        break;
 624    case 2:
 625        /* Two-color rectangle */
 626        success = SendMonoRect(cl, w, h);
 627        break;
 628    default:
 629        /* Up to 256 different colors */
 630        if ( paletteNumColors > 96 &&
 631             qualityLevel != -1 && qualityLevel <= 3 &&
 632             DetectSmoothImage(cl, &cl->format, w, h) ) {
 633            success = SendJpegRect(cl, x, y, w, h,
 634                                   tightConf[qualityLevel].jpegQuality);
 635        } else {
 636            success = SendIndexedRect(cl, w, h);
 637        }
 638    }
 639    return success;
 640}
 641
 642static rfbBool
 643SendTightHeader(rfbClientPtr cl,
 644                int x,
 645                int y,
 646                int w,
 647                int h)
 648{
 649    rfbFramebufferUpdateRectHeader rect;
 650
 651    if (cl->ublen + sz_rfbFramebufferUpdateRectHeader > UPDATE_BUF_SIZE) {
 652        if (!rfbSendUpdateBuf(cl))
 653            return FALSE;
 654    }
 655
 656    rect.r.x = Swap16IfLE(x);
 657    rect.r.y = Swap16IfLE(y);
 658    rect.r.w = Swap16IfLE(w);
 659    rect.r.h = Swap16IfLE(h);
 660    rect.encoding = Swap32IfLE(rfbEncodingTight);
 661
 662    memcpy(&cl->updateBuf[cl->ublen], (char *)&rect,
 663           sz_rfbFramebufferUpdateRectHeader);
 664    cl->ublen += sz_rfbFramebufferUpdateRectHeader;
 665
 666    rfbStatRecordEncodingSent(cl, rfbEncodingTight, sz_rfbFramebufferUpdateRectHeader,
 667                              sz_rfbFramebufferUpdateRectHeader + w * (cl->format.bitsPerPixel / 8) * h);
 668
 669    return TRUE;
 670}
 671
 672/*
 673 * Subencoding implementations.
 674 */
 675
 676static rfbBool
 677SendSolidRect(rfbClientPtr cl)
 678{
 679    int len;
 680
 681    if (usePixelFormat24) {
 682        Pack24(cl, tightBeforeBuf, &cl->format, 1);
 683        len = 3;
 684    } else
 685        len = cl->format.bitsPerPixel / 8;
 686
 687    if (cl->ublen + 1 + len > UPDATE_BUF_SIZE) {
 688        if (!rfbSendUpdateBuf(cl))
 689            return FALSE;
 690    }
 691
 692    cl->updateBuf[cl->ublen++] = (char)(rfbTightFill << 4);
 693    memcpy (&cl->updateBuf[cl->ublen], tightBeforeBuf, len);
 694    cl->ublen += len;
 695
 696    rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, len+1);
 697
 698    return TRUE;
 699}
 700
 701static rfbBool
 702SendMonoRect(rfbClientPtr cl,
 703             int w,
 704             int h)
 705{
 706    int streamId = 1;
 707    int paletteLen, dataLen;
 708
 709    if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
 710	 2 * cl->format.bitsPerPixel / 8 > UPDATE_BUF_SIZE ) {
 711        if (!rfbSendUpdateBuf(cl))
 712            return FALSE;
 713    }
 714
 715    /* Prepare tight encoding header. */
 716    dataLen = (w + 7) / 8;
 717    dataLen *= h;
 718
 719    cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
 720    cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
 721    cl->updateBuf[cl->ublen++] = 1;
 722
 723    /* Prepare palette, convert image. */
 724    switch (cl->format.bitsPerPixel) {
 725
 726    case 32:
 727        EncodeMonoRect32((uint8_t *)tightBeforeBuf, w, h);
 728
 729        ((uint32_t *)tightAfterBuf)[0] = monoBackground;
 730        ((uint32_t *)tightAfterBuf)[1] = monoForeground;
 731        if (usePixelFormat24) {
 732            Pack24(cl, tightAfterBuf, &cl->format, 2);
 733            paletteLen = 6;
 734        } else
 735            paletteLen = 8;
 736
 737        memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteLen);
 738        cl->ublen += paletteLen;
 739        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteLen);
 740        break;
 741
 742    case 16:
 743        EncodeMonoRect16((uint8_t *)tightBeforeBuf, w, h);
 744
 745        ((uint16_t *)tightAfterBuf)[0] = (uint16_t)monoBackground;
 746        ((uint16_t *)tightAfterBuf)[1] = (uint16_t)monoForeground;
 747
 748        memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, 4);
 749        cl->ublen += 4;
 750        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 7);
 751        break;
 752
 753    default:
 754        EncodeMonoRect8((uint8_t *)tightBeforeBuf, w, h);
 755
 756        cl->updateBuf[cl->ublen++] = (char)monoBackground;
 757        cl->updateBuf[cl->ublen++] = (char)monoForeground;
 758        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 5);
 759    }
 760
 761    return CompressData(cl, streamId, dataLen,
 762                        tightConf[compressLevel].monoZlibLevel,
 763                        Z_DEFAULT_STRATEGY);
 764}
 765
 766static rfbBool
 767SendIndexedRect(rfbClientPtr cl,
 768                int w,
 769                int h)
 770{
 771    int streamId = 2;
 772    int i, entryLen;
 773
 774    if ( cl->ublen + TIGHT_MIN_TO_COMPRESS + 6 +
 775	 paletteNumColors * cl->format.bitsPerPixel / 8 >
 776         UPDATE_BUF_SIZE ) {
 777        if (!rfbSendUpdateBuf(cl))
 778            return FALSE;
 779    }
 780
 781    /* Prepare tight encoding header. */
 782    cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
 783    cl->updateBuf[cl->ublen++] = rfbTightFilterPalette;
 784    cl->updateBuf[cl->ublen++] = (char)(paletteNumColors - 1);
 785
 786    /* Prepare palette, convert image. */
 787    switch (cl->format.bitsPerPixel) {
 788
 789    case 32:
 790        EncodeIndexedRect32((uint8_t *)tightBeforeBuf, w * h);
 791
 792        for (i = 0; i < paletteNumColors; i++) {
 793            ((uint32_t *)tightAfterBuf)[i] =
 794                palette.entry[i].listNode->rgb;
 795        }
 796        if (usePixelFormat24) {
 797            Pack24(cl, tightAfterBuf, &cl->format, paletteNumColors);
 798            entryLen = 3;
 799        } else
 800            entryLen = 4;
 801
 802        memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * entryLen);
 803        cl->ublen += paletteNumColors * entryLen;
 804        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteNumColors * entryLen);
 805        break;
 806
 807    case 16:
 808        EncodeIndexedRect16((uint8_t *)tightBeforeBuf, w * h);
 809
 810        for (i = 0; i < paletteNumColors; i++) {
 811            ((uint16_t *)tightAfterBuf)[i] =
 812                (uint16_t)palette.entry[i].listNode->rgb;
 813        }
 814
 815        memcpy(&cl->updateBuf[cl->ublen], tightAfterBuf, paletteNumColors * 2);
 816        cl->ublen += paletteNumColors * 2;
 817        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 3 + paletteNumColors * 2);
 818        break;
 819
 820    default:
 821        return FALSE;           /* Should never happen. */
 822    }
 823
 824    return CompressData(cl, streamId, w * h,
 825                        tightConf[compressLevel].idxZlibLevel,
 826                        Z_DEFAULT_STRATEGY);
 827}
 828
 829static rfbBool
 830SendFullColorRect(rfbClientPtr cl,
 831                  int w,
 832                  int h)
 833{
 834    int streamId = 0;
 835    int len;
 836
 837    if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 1 > UPDATE_BUF_SIZE) {
 838        if (!rfbSendUpdateBuf(cl))
 839            return FALSE;
 840    }
 841
 842    cl->updateBuf[cl->ublen++] = 0x00;  /* stream id = 0, no flushing, no filter */
 843    rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
 844
 845    if (usePixelFormat24) {
 846        Pack24(cl, tightBeforeBuf, &cl->format, w * h);
 847        len = 3;
 848    } else
 849        len = cl->format.bitsPerPixel / 8;
 850
 851    return CompressData(cl, streamId, w * h * len,
 852                        tightConf[compressLevel].rawZlibLevel,
 853                        Z_DEFAULT_STRATEGY);
 854}
 855
 856static rfbBool
 857SendGradientRect(rfbClientPtr cl,
 858                 int w,
 859                 int h)
 860{
 861    int streamId = 3;
 862    int len;
 863
 864    if (cl->format.bitsPerPixel == 8)
 865        return SendFullColorRect(cl, w, h);
 866
 867    if (cl->ublen + TIGHT_MIN_TO_COMPRESS + 2 > UPDATE_BUF_SIZE) {
 868        if (!rfbSendUpdateBuf(cl))
 869            return FALSE;
 870    }
 871
 872    if (prevRowBuf == NULL)
 873        prevRowBuf = (int *)malloc(2048 * 3 * sizeof(int));
 874
 875    cl->updateBuf[cl->ublen++] = (streamId | rfbTightExplicitFilter) << 4;
 876    cl->updateBuf[cl->ublen++] = rfbTightFilterGradient;
 877    rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 2);
 878
 879    if (usePixelFormat24) {
 880        FilterGradient24(cl, tightBeforeBuf, &cl->format, w, h);
 881        len = 3;
 882    } else if (cl->format.bitsPerPixel == 32) {
 883        FilterGradient32(cl, (uint32_t *)tightBeforeBuf, &cl->format, w, h);
 884        len = 4;
 885    } else {
 886        FilterGradient16(cl, (uint16_t *)tightBeforeBuf, &cl->format, w, h);
 887        len = 2;
 888    }
 889
 890    return CompressData(cl, streamId, w * h * len,
 891                        tightConf[compressLevel].gradientZlibLevel,
 892                        Z_FILTERED);
 893}
 894
 895static rfbBool
 896CompressData(rfbClientPtr cl,
 897             int streamId,
 898             int dataLen,
 899             int zlibLevel,
 900             int zlibStrategy)
 901{
 902    z_streamp pz;
 903    int err;
 904
 905    if (dataLen < TIGHT_MIN_TO_COMPRESS) {
 906        memcpy(&cl->updateBuf[cl->ublen], tightBeforeBuf, dataLen);
 907        cl->ublen += dataLen;
 908        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, dataLen);
 909        return TRUE;
 910    }
 911
 912    pz = &cl->zsStruct[streamId];
 913
 914    /* Initialize compression stream if needed. */
 915    if (!cl->zsActive[streamId]) {
 916        pz->zalloc = Z_NULL;
 917        pz->zfree = Z_NULL;
 918        pz->opaque = Z_NULL;
 919
 920        err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
 921                            MAX_MEM_LEVEL, zlibStrategy);
 922        if (err != Z_OK)
 923            return FALSE;
 924
 925        cl->zsActive[streamId] = TRUE;
 926        cl->zsLevel[streamId] = zlibLevel;
 927    }
 928
 929    /* Prepare buffer pointers. */
 930    pz->next_in = (Bytef *)tightBeforeBuf;
 931    pz->avail_in = dataLen;
 932    pz->next_out = (Bytef *)tightAfterBuf;
 933    pz->avail_out = tightAfterBufSize;
 934
 935    /* Change compression parameters if needed. */
 936    if (zlibLevel != cl->zsLevel[streamId]) {
 937        if (deflateParams (pz, zlibLevel, zlibStrategy) != Z_OK) {
 938            return FALSE;
 939        }
 940        cl->zsLevel[streamId] = zlibLevel;
 941    }
 942
 943    /* Actual compression. */
 944    if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
 945         pz->avail_in != 0 || pz->avail_out == 0 ) {
 946        return FALSE;
 947    }
 948
 949    return SendCompressedData(cl, tightAfterBufSize - pz->avail_out);
 950}
 951
 952static rfbBool SendCompressedData(rfbClientPtr cl,
 953                                  int compressedLen)
 954{
 955    int i, portionLen;
 956
 957    cl->updateBuf[cl->ublen++] = compressedLen & 0x7F;
 958    rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
 959    if (compressedLen > 0x7F) {
 960        cl->updateBuf[cl->ublen-1] |= 0x80;
 961        cl->updateBuf[cl->ublen++] = compressedLen >> 7 & 0x7F;
 962        rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
 963        if (compressedLen > 0x3FFF) {
 964            cl->updateBuf[cl->ublen-1] |= 0x80;
 965            cl->updateBuf[cl->ublen++] = compressedLen >> 14 & 0xFF;
 966            rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, 1);
 967        }
 968    }
 969
 970    portionLen = UPDATE_BUF_SIZE;
 971    for (i = 0; i < compressedLen; i += portionLen) {
 972        if (i + portionLen > compressedLen) {
 973            portionLen = compressedLen - i;
 974        }
 975        if (cl->ublen + portionLen > UPDATE_BUF_SIZE) {
 976            if (!rfbSendUpdateBuf(cl))
 977                return FALSE;
 978        }
 979        memcpy(&cl->updateBuf[cl->ublen], &tightAfterBuf[i], portionLen);
 980        cl->ublen += portionLen;
 981    }
 982    rfbStatRecordEncodingSentAdd(cl, rfbEncodingTight, compressedLen);
 983
 984    return TRUE;
 985}
 986
 987/*
 988 * Code to determine how many different colors used in rectangle.
 989 */
 990
 991static void
 992FillPalette8(int count)
 993{
 994    uint8_t *data = (uint8_t *)tightBeforeBuf;
 995    uint8_t c0, c1;
 996    int i, n0, n1;
 997
 998    paletteNumColors = 0;
 999
1000    c0 = data[0];
1001    for (i = 1; i < count && data[i] == c0; i++);
1002    if (i == count) {
1003        paletteNumColors = 1;
1004        return;                 /* Solid rectangle */
1005    }
1006
1007    if (paletteMaxColors < 2)
1008        return;
1009
1010    n0 = i;
1011    c1 = data[i];
1012    n1 = 0;
1013    for (i++; i < count; i++) {
1014        if (data[i] == c0) {
1015            n0++;
1016        } else if (data[i] == c1) {
1017            n1++;
1018        } else
1019            break;
1020    }
1021    if (i == count) {
1022        if (n0 > n1) {
1023            monoBackground = (uint32_t)c0;
1024            monoForeground = (uint32_t)c1;
1025        } else {
1026            monoBackground = (uint32_t)c1;
1027            monoForeground = (uint32_t)c0;
1028        }
1029        paletteNumColors = 2;   /* Two colors */
1030    }
1031}
1032
1033#define DEFINE_FILL_PALETTE_FUNCTION(bpp)                               \
1034                                                                        \
1035static void                                                             \
1036FillPalette##bpp(int count) {                                           \
1037    uint##bpp##_t *data = (uint##bpp##_t *)tightBeforeBuf;              \
1038    uint##bpp##_t c0, c1, ci;                                           \
1039    int i, n0, n1, ni;                                                  \
1040                                                                        \
1041    c0 = data[0];                                                       \
1042    for (i = 1; i < count && data[i] == c0; i++);                       \
1043    if (i >= count) {                                                   \
1044        paletteNumColors = 1;   /* Solid rectangle */                   \
1045        return;                                                         \
1046    }                                                                   \
1047                                                                        \
1048    if (paletteMaxColors < 2) {                                         \
1049        paletteNumColors = 0;   /* Full-color encoding preferred */     \
1050        return;                                                         \
1051    }                                                                   \
1052                                                                        \
1053    n0 = i;                                                             \
1054    c1 = data[i];                                                       \
1055    n1 = 0;                                                             \
1056    for (i++; i < count; i++) {                                         \
1057        ci = data[i];                                                   \
1058        if (ci == c0) {                                                 \
1059            n0++;                                                       \
1060        } else if (ci == c1) {                                          \
1061            n1++;                                                       \
1062        } else                                                          \
1063            break;                                                      \
1064    }                                                                   \
1065    if (i >= count) {                                                   \
1066        if (n0 > n1) {                                                  \
1067            monoBackground = (uint32_t)c0;                              \
1068            monoForeground = (uint32_t)c1;                              \
1069        } else {                                                        \
1070            monoBackground = (uint32_t)c1;                              \
1071            monoForeground = (uint32_t)c0;                              \
1072        }                                                               \
1073        paletteNumColors = 2;   /* Two colors */                        \
1074        return;                                                         \
1075    }                                                                   \
1076                                                                        \
1077    PaletteReset();                                                     \
1078    PaletteInsert (c0, (uint32_t)n0, bpp);                              \
1079    PaletteInsert (c1, (uint32_t)n1, bpp);                              \
1080                                                                        \
1081    ni = 1;                                                             \
1082    for (i++; i < count; i++) {                                         \
1083        if (data[i] == ci) {                                            \
1084            ni++;                                                       \
1085        } else {                                                        \
1086            if (!PaletteInsert (ci, (uint32_t)ni, bpp))                 \
1087                return;                                                 \
1088            ci = data[i];                                               \
1089            ni = 1;                                                     \
1090        }                                                               \
1091    }                                                                   \
1092    PaletteInsert (ci, (uint32_t)ni, bpp);                              \
1093}
1094
1095DEFINE_FILL_PALETTE_FUNCTION(16)
1096DEFINE_FILL_PALETTE_FUNCTION(32)
1097
1098
1099/*
1100 * Functions to operate with palette structures.
1101 */
1102
1103#define HASH_FUNC16(rgb) ((int)(((rgb >> 8) + rgb) & 0xFF))
1104#define HASH_FUNC32(rgb) ((int)(((rgb >> 16) + (rgb >> 8)) & 0xFF))
1105
1106static void
1107PaletteReset(void)
1108{
1109    paletteNumColors = 0;
1110    memset(palette.hash, 0, 256 * sizeof(COLOR_LIST *));
1111}
1112
1113static int
1114PaletteInsert(uint32_t rgb,
1115              int numPixels,
1116              int bpp)
1117{
1118    COLOR_LIST *pnode;
1119    COLOR_LIST *prev_pnode = NULL;
1120    int hash_key, idx, new_idx, count;
1121
1122    hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);
1123
1124    pnode = palette.hash[hash_key];
1125
1126    while (pnode != NULL) {
1127        if (pnode->rgb == rgb) {
1128            /* Such palette entry already exists. */
1129            new_idx = idx = pnode->idx;
1130            count = palette.entry[idx].numPixels + numPixels;
1131            if (new_idx && palette.entry[new_idx-1].numPixels < count) {
1132                do {
1133                    palette.entry[new_idx] = palette.entry[new_idx-1];
1134                    palette.entry[new_idx].listNode->idx = new_idx;
1135                    new_idx--;
1136                }
1137                while (new_idx && palette.entry[new_idx-1].numPixels < count);
1138                palette.entry[new_idx].listNode = pnode;
1139                pnode->idx = new_idx;
1140            }
1141            palette.entry[new_idx].numPixels = count;
1142            return paletteNumColors;
1143        }
1144        prev_pnode = pnode;
1145        pnode = pnode->next;
1146    }
1147
1148    /* Check if palette is full. */
1149    if (paletteNumColors == 256 || paletteNumColors == paletteMaxColors) {
1150        paletteNumColors = 0;
1151        return 0;
1152    }
1153
1154    /* Move palette entries with lesser pixel counts. */
1155    for ( idx = paletteNumColors;
1156          idx > 0 && palette.entry[idx-1].numPixels < numPixels;
1157          idx-- ) {
1158        palette.entry[idx] = palette.entry[idx-1];
1159        palette.entry[idx].listNode->idx = idx;
1160    }
1161
1162    /* Add new palette entry into the freed slot. */
1163    pnode = &palette.list[paletteNumColors];
1164    if (prev_pnode != NULL) {
1165        prev_pnode->next = pnode;
1166    } else {
1167        palette.hash[hash_key] = pnode;
1168    }
1169    pnode->next = NULL;
1170    pnode->idx = idx;
1171    pnode->rgb = rgb;
1172    palette.entry[idx].listNode = pnode;
1173    palette.entry[idx].numPixels = numPixels;
1174
1175    return (++paletteNumColors);
1176}
1177
1178
1179/*
1180 * Converting 32-bit color samples into 24-bit colors.
1181 * Should be called only when redMax, greenMax and blueMax are 255.
1182 * Color components assumed to be byte-aligned.
1183 */
1184
1185static void Pack24(rfbClientPtr cl,
1186                   char *buf,
1187                   rfbPixelFormat *fmt,
1188                   int count)
1189{
1190    uint32_t *buf32;
1191    uint32_t pix;
1192    int r_shift, g_shift, b_shift;
1193
1194    buf32 = (uint32_t *)buf;
1195
1196    if (!cl->screen->serverFormat.bigEndian == !fmt->bigEndian) {
1197        r_shift = fmt->redShift;
1198        g_shift = fmt->greenShift;
1199        b_shift = fmt->blueShift;
1200    } else {
1201        r_shift = 24 - fmt->redShift;
1202        g_shift = 24 - fmt->greenShift;
1203        b_shift = 24 - fmt->blueShift;
1204    }
1205
1206    while (count--) {
1207        pix = *buf32++;
1208        *buf++ = (char)(pix >> r_shift);
1209        *buf++ = (char)(pix >> g_shift);
1210        *buf++ = (char)(pix >> b_shift);
1211    }
1212}
1213
1214
1215/*
1216 * Converting truecolor samples into palette indices.
1217 */
1218
1219#define DEFINE_IDX_ENCODE_FUNCTION(bpp)                                 \
1220                                                                        \
1221static void                                                             \
1222EncodeIndexedRect##bpp(uint8_t *buf, int count) {                       \
1223    COLOR_LIST *pnode;                                                  \
1224    uint##bpp##_t *src;                                                 \
1225    uint##bpp##_t rgb;                                                  \
1226    int rep = 0;                                                        \
1227                                                                        \
1228    src = (uint##bpp##_t *) buf;                                        \
1229                                                                        \
1230    while (count--) {                                                   \
1231        rgb = *src++;                                                   \
1232        while (count && *src == rgb) {                                  \
1233            rep++, src++, count--;                                      \
1234        }                                                               \
1235        pnode = palette.hash[HASH_FUNC##bpp(rgb)];                      \
1236        while (pnode != NULL) {                                         \
1237            if ((uint##bpp##_t)pnode->rgb == rgb) {                     \
1238                *buf++ = (uint8_t)pnode->idx;                           \
1239                while (rep) {                                           \
1240                    *buf++ = (uint8_t)pnode->idx;                       \
1241                    rep--;                                              \
1242                }                                                       \
1243                break;                                                  \
1244            }                                                           \
1245            pnode = pnode->next;                                        \
1246        }                                                               \
1247    }                                                                   \
1248}
1249
1250DEFINE_IDX_ENCODE_FUNCTION(16)
1251DEFINE_IDX_ENCODE_FUNCTION(32)
1252
1253#define DEFINE_MONO_ENCODE_FUNCTION(bpp)                                \
1254                                                                        \
1255static void                                                             \
1256EncodeMonoRect##bpp(uint8_t *buf, int w, int h) {                       \
1257    uint##bpp##_t *ptr;                                                 \
1258    uint##bpp##_t bg;                                                   \
1259    unsigned int value, mask;                                           \
1260    int aligned_width;                                                  \
1261    int x, y, bg_bits;                                                  \
1262                                                                        \
1263    ptr = (uint##bpp##_t *) buf;                                        \
1264    bg = (uint##bpp##_t) monoBackground;                                \
1265    aligned_width = w - w % 8;                                          \
1266                                                                        \
1267    for (y = 0; y < h; y++) {                                           \
1268        for (x = 0; x < aligned_width; x += 8) {                        \
1269            for (bg_bits = 0; bg_bits < 8; bg_bits++) {                 \
1270                if (*ptr++ != bg)                                       \
1271                    break;                                              \
1272            }                                                           \
1273            if (bg_bits == 8) {                                         \
1274                *buf++ = 0;                                             \
1275                continue;                                               \
1276            }                                                           \
1277            mask = 0x80 >> bg_bits;                                     \
1278            value = mask;                                               \
1279            for (bg_bits++; bg_bits < 8; bg_bits++) {                   \
1280                mask >>= 1;                                             \
1281                if (*ptr++ != bg) {                                     \
1282                    value |= mask;                                      \
1283                }                                                       \
1284            }                                                           \
1285            *buf++ = (uint8_t)value;                                    \
1286        }                                                               \
1287                                                                        \
1288        mask = 0x80;                                                    \
1289        value = 0;                                                      \
1290        if (x >= w)                                                     \
1291            continue;                                                   \
1292                                                                        \
1293        for (; x < w; x++) {                                            \
1294            if (*ptr++ != bg) {                                         \
1295                value |= mask;                                          \
1296            }                                                           \
1297            mask >>= 1;                                                 \
1298        }                                                               \
1299        *buf++ = (uint8_t)value;                                        \
1300    }                                                                   \
1301}
1302
1303DEFINE_MONO_ENCODE_FUNCTION(8)
1304DEFINE_MONO_ENCODE_FUNCTION(16)
1305DEFINE_MONO_ENCODE_FUNCTION(32)
1306
1307
1308/*
1309 * ``Gradient'' filter for 24-bit color samples.
1310 * Should be called only when redMax, greenMax and blueMax are 255.
1311 * Color components assumed to be byte-aligned.
1312 */
1313
1314static void
1315FilterGradient24(rfbClientPtr cl, char *buf, rfbPixelFormat *fmt, int w, int h)
1316{
1317    uint32_t *buf32;
1318    uint32_t pix32;
1319    int *prevRowPtr;
1320    int shiftBits[3];
1321    int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];
1322    int prediction;
1323    int x, y, c;
1324
1325    buf32 = (uint32_t *)buf;
1326    memset (prevRowBuf, 0, w * 3 * sizeof(int));
1327
1328    if (!cl->screen->serverFormat.bigEndian == !fmt->bigEndian) {
1329        shiftBits[0] = fmt->redShift;
1330        shiftBits[1] = fmt->greenShift;
1331        shiftBits[2] = fmt->blueShift;
1332    } else {
1333        shiftBits[0] = 24 - fmt->redShift;
1334        shiftBits[1] = 24 - fmt->greenShift;
1335        shiftBits[2] = 24 - fmt->blueShift;
1336    }
1337
1338    for (y = 0; y < h; y++) {
1339        for (c = 0; c < 3; c++) {
1340            pixUpper[c] = 0;
1341            pixHere[c] = 0;
1342        }
1343        prevRowPtr = prevRowBuf;
1344        for (x = 0; x < w; x++) {
1345            pix32 = *buf32++;
1346            for (c = 0; c < 3; c++) {
1347                pixUpperLeft[c] = pixUpper[c];
1348                pixLeft[c] = pixHere[c];
1349                pixUpper[c] = *prevRowPtr;
1350                pixHere[c] = (int)(pix32 >> shiftBits[c] & 0xFF);
1351                *prevRowPtr++ = pixHere[c];
1352
1353                prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];
1354                if (prediction < 0) {
1355                    prediction = 0;
1356                } else if (prediction > 0xFF) {
1357                    prediction = 0xFF;
1358                }
1359                *buf++ = (char)(pixHere[c] - prediction);
1360            }
1361        }
1362    }
1363}
1364
1365
1366/*
1367 * ``Gradient'' filter for other color depths.
1368 */
1369
1370#define DEFINE_GRADIENT_FILTER_FUNCTION(bpp)                             \
1371                                                                         \
1372static void                                                              \
1373FilterGradient##bpp(rfbClientPtr cl, uint##bpp##_t *buf,                 \
1374		rfbPixelFormat *fmt, int w, int h) {                     \
1375    uint##bpp##_t pix, diff;                                             \
1376    rfbBool endianMismatch;                                              \
1377    int *prevRowPtr;                                                     \
1378    int maxColor[3], shiftBits[3];                                       \
1379    int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];            \
1380    int prediction;           …

Large files files are truncated, but you can click here to view the full file