PageRenderTime 112ms CodeModel.GetById 27ms app.highlight 66ms RepoModel.GetById 2ms app.codeStats 0ms

/H264Dec/source/h264bsd_deblocking.c

http://github.com/mbebenita/Broadway
C | 2417 lines | 1698 code | 295 blank | 424 comment | 323 complexity | 7c34d152f359d1bdba24d54c513ce489 MD5 | raw file

Large files files are truncated, but you can click here to view the full file

   1/*
   2 * Copyright (C) 2009 The Android Open Source Project
   3 *
   4 * Licensed under the Apache License, Version 2.0 (the "License");
   5 * you may not use this file except in compliance with the License.
   6 * You may obtain a copy of the License at
   7 *
   8 *      http://www.apache.org/licenses/LICENSE-2.0
   9 *
  10 * Unless required by applicable law or agreed to in writing, software
  11 * distributed under the License is distributed on an "AS IS" BASIS,
  12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 * See the License for the specific language governing permissions and
  14 * limitations under the License.
  15 */
  16
  17/*------------------------------------------------------------------------------
  18
  19    Table of contents
  20
  21     1. Include headers
  22     2. External compiler flags
  23     3. Module defines
  24     4. Local function prototypes
  25     5. Functions
  26          h264bsdFilterPicture
  27          FilterVerLumaEdge
  28          FilterHorLumaEdge
  29          FilterHorLuma
  30          FilterVerChromaEdge
  31          FilterHorChromaEdge
  32          FilterHorChroma
  33          InnerBoundaryStrength
  34          EdgeBoundaryStrength
  35          GetBoundaryStrengths
  36          IsSliceBoundaryOnLeft
  37          IsSliceBoundaryOnTop
  38          GetMbFilteringFlags
  39          GetLumaEdgeThresholds
  40          GetChromaEdgeThresholds
  41          FilterLuma
  42          FilterChroma
  43
  44------------------------------------------------------------------------------*/
  45
  46/*------------------------------------------------------------------------------
  47    1. Include headers
  48------------------------------------------------------------------------------*/
  49
  50#include "basetype.h"
  51#include "h264bsd_util.h"
  52#include "h264bsd_macroblock_layer.h"
  53#include "h264bsd_deblocking.h"
  54#include "h264bsd_dpb.h"
  55
  56#ifdef H264DEC_OMXDL
  57#include "omxtypes.h"
  58#include "omxVC.h"
  59#include "armVC.h"
  60#endif /* H264DEC_OMXDL */
  61
  62/*------------------------------------------------------------------------------
  63    2. External compiler flags
  64--------------------------------------------------------------------------------
  65
  66--------------------------------------------------------------------------------
  67    3. Module defines
  68------------------------------------------------------------------------------*/
  69
  70/* Switch off the following Lint messages for this file:
  71 * Info 701: Shift left of signed quantity (int)
  72 * Info 702: Shift right of signed quantity (int)
  73 */
  74/*lint -e701 -e702 */
  75
  76/* array of alpha values, from the standard */
  77static const u8 alphas[52] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,5,6,7,8,9,10,
  78    12,13,15,17,20,22,25,28,32,36,40,45,50,56,63,71,80,90,101,113,127,144,162,
  79    182,203,226,255,255};
  80
  81/* array of beta values, from the standard */
  82static const u8 betas[52] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,3,3,3,3,4,4,
  83    4,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18};
  84
  85
  86
  87#ifndef H264DEC_OMXDL
  88/* array of tc0 values, from the standard, each triplet corresponds to a
  89 * column in the table. Indexing goes as tc0[indexA][bS-1] */
  90static const u8 tc0[52][3] = {
  91    {0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},
  92    {0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},
  93    {0,0,0},{0,0,1},{0,0,1},{0,0,1},{0,0,1},{0,1,1},{0,1,1},{1,1,1},
  94    {1,1,1},{1,1,1},{1,1,1},{1,1,2},{1,1,2},{1,1,2},{1,1,2},{1,2,3},
  95    {1,2,3},{2,2,3},{2,2,4},{2,3,4},{2,3,4},{3,3,5},{3,4,6},{3,4,6},
  96    {4,5,7},{4,5,8},{4,6,9},{5,7,10},{6,8,11},{6,8,13},{7,10,14},{8,11,16},
  97    {9,12,18},{10,13,20},{11,15,23},{13,17,25}
  98};
  99#else
 100/* array of tc0 values, from the standard, each triplet corresponds to a
 101 * column in the table. Indexing goes as tc0[indexA][bS] */
 102static const u8 tc0[52][5] = {
 103    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 104    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 105    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 106    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 107    {0, 0, 0, 0, 0}, {0, 0, 0, 1, 0}, {0, 0, 0, 1, 0}, {0, 0, 0, 1, 0},
 108    {0, 0, 0, 1, 0}, {0, 0, 1, 1, 0}, {0, 0, 1, 1, 0}, {0, 1, 1, 1, 0},
 109    {0, 1, 1, 1, 0}, {0, 1, 1, 1, 0}, {0, 1, 1, 1, 0}, {0, 1, 1, 2, 0},
 110    {0, 1, 1, 2, 0}, {0, 1, 1, 2, 0}, {0, 1, 1, 2, 0}, {0, 1, 2, 3, 0},
 111    {0, 1, 2, 3, 0}, {0, 2, 2, 3, 0}, {0, 2, 2, 4, 0}, {0, 2, 3, 4, 0},
 112    {0, 2, 3, 4, 0}, {0, 3, 3, 5, 0}, {0, 3, 4, 6, 0}, {0, 3, 4, 6, 0},
 113    {0, 4, 5, 7, 0}, {0, 4, 5, 8, 0}, {0, 4, 6, 9, 0}, {0, 5, 7, 10, 0},
 114    {0, 6, 8, 11, 0}, {0, 6, 8, 13, 0}, {0, 7, 10, 14, 0},
 115    {0, 8, 11, 16, 0}, {0, 9, 12, 18, 0}, {0, 10, 13, 20, 0},
 116    {0, 11, 15, 23, 0}, {0, 13, 17, 25, 0}
 117};
 118#endif
 119
 120
 121#ifndef H264DEC_OMXDL
 122/* mapping of raster scan block index to 4x4 block index */
 123static const u32 mb4x4Index[16] =
 124    {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
 125
 126typedef struct {
 127    const u8 *tc0;
 128    u32 alpha;
 129    u32 beta;
 130} edgeThreshold_t;
 131
 132typedef struct {
 133    u32 top;
 134    u32 left;
 135} bS_t;
 136
 137enum { TOP = 0, LEFT = 1, INNER = 2 };
 138#endif /* H264DEC_OMXDL */
 139
 140#define FILTER_LEFT_EDGE    0x04
 141#define FILTER_TOP_EDGE     0x02
 142#define FILTER_INNER_EDGE   0x01
 143
 144
 145/* clipping table defined in intra_prediction.c */
 146extern const u8 h264bsdClip[];
 147
 148/*------------------------------------------------------------------------------
 149    4. Local function prototypes
 150------------------------------------------------------------------------------*/
 151
 152static u32 InnerBoundaryStrength(mbStorage_t *mb1, u32 i1, u32 i2);
 153
 154#ifndef H264DEC_OMXDL
 155static u32 EdgeBoundaryStrength(mbStorage_t *mb1, mbStorage_t *mb2,
 156    u32 i1, u32 i2);
 157#else
 158static u32 InnerBoundaryStrength2(mbStorage_t *mb1, u32 i1, u32 i2);
 159static u32 EdgeBoundaryStrengthLeft(mbStorage_t *mb1, mbStorage_t *mb2);
 160static u32 EdgeBoundaryStrengthTop(mbStorage_t *mb1, mbStorage_t *mb2);
 161#endif
 162
 163static u32 IsSliceBoundaryOnLeft(mbStorage_t *mb);
 164
 165static u32 IsSliceBoundaryOnTop(mbStorage_t *mb);
 166
 167static u32 GetMbFilteringFlags(mbStorage_t *mb);
 168
 169#ifndef H264DEC_OMXDL
 170
 171static u32 GetBoundaryStrengths(mbStorage_t *mb, bS_t *bs, u32 flags);
 172
 173static void FilterLuma(u8 *data, bS_t *bS, edgeThreshold_t *thresholds,
 174        u32 imageWidth);
 175
 176static void FilterChroma(u8 *cb, u8 *cr, bS_t *bS, edgeThreshold_t *thresholds,
 177        u32 imageWidth);
 178
 179static void FilterVerLumaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 180        u32 imageWidth);
 181static void FilterHorLumaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 182        i32 imageWidth);
 183static void FilterHorLuma( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 184        i32 imageWidth);
 185
 186static void FilterVerChromaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 187  u32 imageWidth);
 188static void FilterHorChromaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 189  i32 imageWidth);
 190static void FilterHorChroma( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 191  i32 imageWidth);
 192
 193static void GetLumaEdgeThresholds(
 194  edgeThreshold_t *thresholds,
 195  mbStorage_t *mb,
 196  u32 filteringFlags);
 197
 198static void GetChromaEdgeThresholds(
 199  edgeThreshold_t *thresholds,
 200  mbStorage_t *mb,
 201  u32 filteringFlags,
 202  i32 chromaQpIndexOffset);
 203
 204#else /* H264DEC_OMXDL */
 205
 206static u32 GetBoundaryStrengths(mbStorage_t *mb, u8 (*bs)[16], u32 flags);
 207
 208static void GetLumaEdgeThresholds(
 209    mbStorage_t *mb,
 210    u8 (*alpha)[2],
 211    u8 (*beta)[2],
 212    u8 (*threshold)[16],
 213    u8 (*bs)[16],
 214    u32 filteringFlags );
 215
 216static void GetChromaEdgeThresholds(
 217    mbStorage_t *mb,
 218    u8 (*alpha)[2],
 219    u8 (*beta)[2],
 220    u8 (*threshold)[8],
 221    u8 (*bs)[16],
 222    u32 filteringFlags,
 223    i32 chromaQpIndexOffset);
 224
 225#endif /* H264DEC_OMXDL */
 226
 227/*------------------------------------------------------------------------------
 228
 229    Function: IsSliceBoundaryOnLeft
 230
 231        Functional description:
 232            Function to determine if there is a slice boundary on the left side
 233            of a macroblock.
 234
 235------------------------------------------------------------------------------*/
 236u32 IsSliceBoundaryOnLeft(mbStorage_t *mb)
 237{
 238
 239/* Variables */
 240
 241/* Code */
 242
 243    ASSERT(mb && mb->mbA);
 244
 245    if (mb->sliceId != mb->mbA->sliceId)
 246        return(HANTRO_TRUE);
 247    else
 248        return(HANTRO_FALSE);
 249
 250}
 251
 252/*------------------------------------------------------------------------------
 253
 254    Function: IsSliceBoundaryOnTop
 255
 256        Functional description:
 257            Function to determine if there is a slice boundary above the
 258            current macroblock.
 259
 260------------------------------------------------------------------------------*/
 261u32 IsSliceBoundaryOnTop(mbStorage_t *mb)
 262{
 263
 264/* Variables */
 265
 266/* Code */
 267
 268    ASSERT(mb && mb->mbB);
 269
 270    if (mb->sliceId != mb->mbB->sliceId)
 271        return(HANTRO_TRUE);
 272    else
 273        return(HANTRO_FALSE);
 274
 275}
 276
 277/*------------------------------------------------------------------------------
 278
 279    Function: GetMbFilteringFlags
 280
 281        Functional description:
 282          Function to determine which edges of a macroblock has to be
 283          filtered. Output is a bit-wise OR of FILTER_LEFT_EDGE,
 284          FILTER_TOP_EDGE and FILTER_INNER_EDGE, depending on which edges
 285          shall be filtered.
 286
 287------------------------------------------------------------------------------*/
 288u32 GetMbFilteringFlags(mbStorage_t *mb)
 289{
 290
 291/* Variables */
 292
 293    u32 flags = 0;
 294
 295/* Code */
 296
 297    ASSERT(mb);
 298
 299    /* nothing will be filtered if disableDeblockingFilterIdc == 1 */
 300    if (mb->disableDeblockingFilterIdc != 1)
 301    {
 302        flags |= FILTER_INNER_EDGE;
 303
 304        /* filterLeftMbEdgeFlag, left mb is MB_A */
 305        if (mb->mbA &&
 306            ((mb->disableDeblockingFilterIdc != 2) ||
 307             !IsSliceBoundaryOnLeft(mb)))
 308            flags |= FILTER_LEFT_EDGE;
 309
 310        /* filterTopMbEdgeFlag */
 311        if (mb->mbB &&
 312            ((mb->disableDeblockingFilterIdc != 2) ||
 313             !IsSliceBoundaryOnTop(mb)))
 314            flags |= FILTER_TOP_EDGE;
 315    }
 316
 317    return(flags);
 318
 319}
 320
 321/*------------------------------------------------------------------------------
 322
 323    Function: InnerBoundaryStrength
 324
 325        Functional description:
 326            Function to calculate boundary strength value bs for an inner
 327            edge of a macroblock. Macroblock type is checked before this is
 328            called -> no intra mb condition here.
 329
 330------------------------------------------------------------------------------*/
 331u32 InnerBoundaryStrength(mbStorage_t *mb1, u32 ind1, u32 ind2)
 332{
 333    i32 tmp1, tmp2;
 334    i32 mv1, mv2, mv3, mv4;
 335
 336    tmp1 = mb1->totalCoeff[ind1];
 337    tmp2 = mb1->totalCoeff[ind2];
 338    mv1 = mb1->mv[ind1].hor;
 339    mv2 = mb1->mv[ind2].hor;
 340    mv3 = mb1->mv[ind1].ver;
 341    mv4 = mb1->mv[ind2].ver;
 342
 343    if (tmp1 || tmp2)
 344    {
 345        return 2;
 346    }
 347    else if ( (ABS(mv1 - mv2) >= 4) || (ABS(mv3 - mv4) >= 4) ||
 348              (mb1->refAddr[ind1 >> 2] != mb1->refAddr[ind2 >> 2]) )
 349    {
 350        return 1;
 351    }
 352    else
 353        return 0;
 354}
 355
 356/*------------------------------------------------------------------------------
 357
 358    Function: InnerBoundaryStrength2
 359
 360        Functional description:
 361            Function to calculate boundary strength value bs for an inner
 362            edge of a macroblock. The function is the same as
 363            InnerBoundaryStrength but without checking totalCoeff.
 364
 365------------------------------------------------------------------------------*/
 366u32 InnerBoundaryStrength2(mbStorage_t *mb1, u32 ind1, u32 ind2)
 367{
 368    i32 tmp1, tmp2, tmp3, tmp4;
 369
 370    tmp1 = mb1->mv[ind1].hor;
 371    tmp2 = mb1->mv[ind2].hor;
 372    tmp3 = mb1->mv[ind1].ver;
 373    tmp4 = mb1->mv[ind2].ver;
 374
 375    if ( (ABS(tmp1 - tmp2) >= 4) || (ABS(tmp3 - tmp4) >= 4) ||
 376         (mb1->refAddr[ind1 >> 2] != mb1->refAddr[ind2 >> 2]))
 377    {
 378        return 1;
 379    }
 380    else
 381        return 0;
 382}
 383#ifndef H264DEC_OMXDL
 384/*------------------------------------------------------------------------------
 385
 386    Function: EdgeBoundaryStrength
 387
 388        Functional description:
 389            Function to calculate boundary strength value bs for left- or
 390            top-most edge of a macroblock. Macroblock types are checked
 391            before this is called -> no intra mb conditions here.
 392
 393------------------------------------------------------------------------------*/
 394u32 EdgeBoundaryStrength(mbStorage_t *mb1, mbStorage_t *mb2,
 395    u32 ind1, u32 ind2)
 396{
 397
 398    if (mb1->totalCoeff[ind1] || mb2->totalCoeff[ind2])
 399    {
 400        return 2;
 401    }
 402    else if ((mb1->refAddr[ind1 >> 2] != mb2->refAddr[ind2 >> 2]) ||
 403             (ABS(mb1->mv[ind1].hor - mb2->mv[ind2].hor) >= 4) ||
 404             (ABS(mb1->mv[ind1].ver - mb2->mv[ind2].ver) >= 4))
 405    {
 406        return 1;
 407    }
 408    else
 409        return 0;
 410}
 411
 412#else /* H264DEC_OMXDL */
 413
 414/*------------------------------------------------------------------------------
 415
 416    Function: EdgeBoundaryStrengthTop
 417
 418        Functional description:
 419            Function to calculate boundary strength value bs for
 420            top-most edge of a macroblock. Macroblock types are checked
 421            before this is called -> no intra mb conditions here.
 422
 423------------------------------------------------------------------------------*/
 424u32 EdgeBoundaryStrengthTop(mbStorage_t *mb1, mbStorage_t *mb2)
 425{
 426    u32 topBs = 0;
 427    u32 tmp1, tmp2, tmp3, tmp4;
 428
 429    tmp1 = mb1->totalCoeff[0];
 430    tmp2 = mb2->totalCoeff[10];
 431    tmp3 = mb1->totalCoeff[1];
 432    tmp4 = mb2->totalCoeff[11];
 433    if (tmp1 || tmp2)
 434    {
 435        topBs = 2<<0;
 436    }
 437    else if ((ABS(mb1->mv[0].hor - mb2->mv[10].hor) >= 4) ||
 438             (ABS(mb1->mv[0].ver - mb2->mv[10].ver) >= 4) ||
 439             (mb1->refAddr[0] != mb2->refAddr[10 >> 2]))
 440    {
 441        topBs = 1<<0;
 442    }
 443    tmp1 = mb1->totalCoeff[4];
 444    tmp2 = mb2->totalCoeff[14];
 445    if (tmp3 || tmp4)
 446    {
 447        topBs += 2<<8;
 448    }
 449    else if ((ABS(mb1->mv[1].hor - mb2->mv[11].hor) >= 4) ||
 450             (ABS(mb1->mv[1].ver - mb2->mv[11].ver) >= 4) ||
 451             (mb1->refAddr[0] != mb2->refAddr[11 >> 2]))
 452    {
 453        topBs += 1<<8;
 454    }
 455    tmp3 = mb1->totalCoeff[5];
 456    tmp4 = mb2->totalCoeff[15];
 457    if (tmp1 || tmp2)
 458    {
 459        topBs += 2<<16;
 460    }
 461    else if ((ABS(mb1->mv[4].hor - mb2->mv[14].hor) >= 4) ||
 462             (ABS(mb1->mv[4].ver - mb2->mv[14].ver) >= 4) ||
 463             (mb1->refAddr[4 >> 2] != mb2->refAddr[14 >> 2]))
 464    {
 465        topBs += 1<<16;
 466    }
 467    if (tmp3 || tmp4)
 468    {
 469        topBs += 2<<24;
 470    }
 471    else if ((ABS(mb1->mv[5].hor - mb2->mv[15].hor) >= 4) ||
 472             (ABS(mb1->mv[5].ver - mb2->mv[15].ver) >= 4) ||
 473             (mb1->refAddr[5 >> 2] != mb2->refAddr[15 >> 2]))
 474    {
 475        topBs += 1<<24;
 476    }
 477
 478    return topBs;
 479}
 480
 481/*------------------------------------------------------------------------------
 482
 483    Function: EdgeBoundaryStrengthLeft
 484
 485        Functional description:
 486            Function to calculate boundary strength value bs for left-
 487            edge of a macroblock. Macroblock types are checked
 488            before this is called -> no intra mb conditions here.
 489
 490------------------------------------------------------------------------------*/
 491u32 EdgeBoundaryStrengthLeft(mbStorage_t *mb1, mbStorage_t *mb2)
 492{
 493    u32 leftBs = 0;
 494    u32 tmp1, tmp2, tmp3, tmp4;
 495
 496    tmp1 = mb1->totalCoeff[0];
 497    tmp2 = mb2->totalCoeff[5];
 498    tmp3 = mb1->totalCoeff[2];
 499    tmp4 = mb2->totalCoeff[7];
 500
 501    if (tmp1 || tmp2)
 502    {
 503        leftBs = 2<<0;
 504    }
 505    else if ((ABS(mb1->mv[0].hor - mb2->mv[5].hor) >= 4) ||
 506             (ABS(mb1->mv[0].ver - mb2->mv[5].ver) >= 4) ||
 507             (mb1->refAddr[0] != mb2->refAddr[5 >> 2]))
 508    {
 509        leftBs = 1<<0;
 510    }
 511    tmp1 = mb1->totalCoeff[8];
 512    tmp2 = mb2->totalCoeff[13];
 513    if (tmp3 || tmp4)
 514    {
 515        leftBs += 2<<8;
 516    }
 517    else if ((ABS(mb1->mv[2].hor - mb2->mv[7].hor) >= 4) ||
 518             (ABS(mb1->mv[2].ver - mb2->mv[7].ver) >= 4) ||
 519             (mb1->refAddr[0] != mb2->refAddr[7 >> 2]))
 520    {
 521        leftBs += 1<<8;
 522    }
 523    tmp3 = mb1->totalCoeff[10];
 524    tmp4 = mb2->totalCoeff[15];
 525    if (tmp1 || tmp2)
 526    {
 527        leftBs += 2<<16;
 528    }
 529    else if ((ABS(mb1->mv[8].hor - mb2->mv[13].hor) >= 4) ||
 530             (ABS(mb1->mv[8].ver - mb2->mv[13].ver) >= 4) ||
 531             (mb1->refAddr[8 >> 2] != mb2->refAddr[13 >> 2]))
 532    {
 533        leftBs += 1<<16;
 534    }
 535    if (tmp3 || tmp4)
 536    {
 537        leftBs += 2<<24;
 538    }
 539    else if ((ABS(mb1->mv[10].hor - mb2->mv[15].hor) >= 4) ||
 540             (ABS(mb1->mv[10].ver - mb2->mv[15].ver) >= 4) ||
 541             (mb1->refAddr[10 >> 2] != mb2->refAddr[15 >> 2]))
 542    {
 543        leftBs += 1<<24;
 544    }
 545
 546    return leftBs;
 547}
 548#endif /* H264DEC_OMXDL */
 549/*------------------------------------------------------------------------------
 550
 551    Function: h264bsdFilterPicture
 552
 553        Functional description:
 554          Perform deblocking filtering for a picture. Filter does not copy
 555          the original picture anywhere but filtering is performed directly
 556          on the original image. Parameters controlling the filtering process
 557          are computed based on information in macroblock structures of the
 558          filtered macroblock, macroblock above and macroblock on the left of
 559          the filtered one.
 560
 561        Inputs:
 562          image         pointer to image to be filtered
 563          mb            pointer to macroblock data structure of the top-left
 564                        macroblock of the picture
 565
 566        Outputs:
 567          image         filtered image stored here
 568
 569        Returns:
 570          none
 571
 572------------------------------------------------------------------------------*/
 573#ifndef H264DEC_OMXDL
 574void h264bsdFilterPicture(
 575  image_t *image,
 576  mbStorage_t *mb)
 577{
 578
 579/* Variables */
 580
 581    u32 flags;
 582    u32 picSizeInMbs, mbRow, mbCol;
 583    u32 picWidthInMbs;
 584    u8 *data;
 585    mbStorage_t *pMb;
 586    bS_t bS[16];
 587    edgeThreshold_t thresholds[3];
 588
 589/* Code */
 590
 591    ASSERT(image);
 592    ASSERT(mb);
 593    ASSERT(image->data);
 594    ASSERT(image->width);
 595    ASSERT(image->height);
 596
 597    picWidthInMbs = image->width;
 598    data = image->data;
 599    picSizeInMbs = picWidthInMbs * image->height;
 600
 601    pMb = mb;
 602
 603    for (mbRow = 0, mbCol = 0; mbRow < image->height; pMb++)
 604    {
 605        flags = GetMbFilteringFlags(pMb);
 606
 607        if (flags)
 608        {
 609            /* GetBoundaryStrengths function returns non-zero value if any of
 610             * the bS values for the macroblock being processed was non-zero */
 611            if (GetBoundaryStrengths(pMb, bS, flags))
 612            {
 613                /* luma */
 614                GetLumaEdgeThresholds(thresholds, pMb, flags);
 615                data = image->data + mbRow * picWidthInMbs * 256 + mbCol * 16;
 616
 617                FilterLuma((u8*)data, bS, thresholds, picWidthInMbs*16);
 618
 619                /* chroma */
 620                GetChromaEdgeThresholds(thresholds, pMb, flags,
 621                    pMb->chromaQpIndexOffset);
 622                data = image->data + picSizeInMbs * 256 +
 623                    mbRow * picWidthInMbs * 64 + mbCol * 8;
 624
 625                FilterChroma((u8*)data, data + 64*picSizeInMbs, bS,
 626                        thresholds, picWidthInMbs*8);
 627
 628            }
 629        }
 630
 631        mbCol++;
 632        if (mbCol == picWidthInMbs)
 633        {
 634            mbCol = 0;
 635            mbRow++;
 636        }
 637    }
 638
 639}
 640
 641/*------------------------------------------------------------------------------
 642
 643    Function: FilterVerLumaEdge
 644
 645        Functional description:
 646            Filter one vertical 4-pixel luma edge.
 647
 648------------------------------------------------------------------------------*/
 649void FilterVerLumaEdge(
 650  u8 *data,
 651  u32 bS,
 652  edgeThreshold_t *thresholds,
 653  u32 imageWidth)
 654{
 655
 656/* Variables */
 657
 658    i32 delta, tc, tmp;
 659    u32 i;
 660    u8 p0, q0, p1, q1, p2, q2;
 661    u32 tmpFlag;
 662    const u8 *clp = h264bsdClip + 512;
 663
 664/* Code */
 665
 666    ASSERT(data);
 667    ASSERT(bS && bS <= 4);
 668    ASSERT(thresholds);
 669
 670    if (bS < 4)
 671    {
 672        tc = thresholds->tc0[bS-1];
 673        tmp = tc;
 674        for (i = 4; i; i--, data += imageWidth)
 675        {
 676            p1 = data[-2]; p0 = data[-1];
 677            q0 = data[0]; q1 = data[1];
 678            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 679                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 680                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 681            {
 682                p2 = data[-3];
 683                q2 = data[2];
 684
 685                if ((unsigned)ABS(p2-p0) < thresholds->beta)
 686                {
 687                    data[-2] = (u8)(p1 + CLIP3(-tc,tc,
 688                        (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 689                    tmp++;
 690                }
 691
 692                if ((unsigned)ABS(q2-q0) < thresholds->beta)
 693                {
 694                    data[1] = (u8)(q1 + CLIP3(-tc,tc,
 695                        (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 696                    tmp++;
 697                }
 698
 699                delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 700                          (p1 - q1) + 4) >> 3));
 701
 702                p0 = clp[p0 + delta];
 703                q0 = clp[q0 - delta];
 704                tmp = tc;
 705                data[-1] = p0;
 706                data[ 0] = q0;
 707            }
 708        }
 709    }
 710    else
 711    {
 712        for (i = 4; i; i--, data += imageWidth)
 713        {
 714            p1 = data[-2]; p0 = data[-1];
 715            q0 = data[0]; q1 = data[1];
 716            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 717                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 718                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 719            {
 720                tmpFlag =
 721                    ((unsigned)ABS(p0-q0) < ((thresholds->alpha >> 2) +2)) ?
 722                        HANTRO_TRUE : HANTRO_FALSE;
 723
 724                p2 = data[-3];
 725                q2 = data[2];
 726
 727                if (tmpFlag && (unsigned)ABS(p2-p0) < thresholds->beta)
 728                {
 729                    tmp = p1 + p0 + q0;
 730                    data[-1] = (u8)((p2 + 2 * tmp + q1 + 4) >> 3);
 731                    data[-2] = (u8)((p2 + tmp + 2) >> 2);
 732                    data[-3] = (u8)((2 * data[-4] + 3 * p2 + tmp + 4) >> 3);
 733                }
 734                else
 735                    data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
 736
 737                if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta)
 738                {
 739                    tmp = p0 + q0 + q1;
 740                    data[0] = (u8)((p1 + 2 * tmp + q2 + 4) >> 3);
 741                    data[1] = (u8)((tmp + q2 + 2) >> 2);
 742                    data[2] = (u8)((2 * data[3] + 3 * q2 + tmp + 4) >> 3);
 743                }
 744                else
 745                    data[0] = (u8)((2 * q1 + q0 + p1 + 2) >> 2);
 746            }
 747        }
 748    }
 749
 750}
 751
 752/*------------------------------------------------------------------------------
 753
 754    Function: FilterHorLumaEdge
 755
 756        Functional description:
 757            Filter one horizontal 4-pixel luma edge
 758
 759------------------------------------------------------------------------------*/
 760void FilterHorLumaEdge(
 761  u8 *data,
 762  u32 bS,
 763  edgeThreshold_t *thresholds,
 764  i32 imageWidth)
 765{
 766
 767/* Variables */
 768
 769    i32 delta, tc, tmp;
 770    u32 i;
 771    u8 p0, q0, p1, q1, p2, q2;
 772    const u8 *clp = h264bsdClip + 512;
 773
 774/* Code */
 775
 776    ASSERT(data);
 777    ASSERT(bS < 4);
 778    ASSERT(thresholds);
 779
 780    tc = thresholds->tc0[bS-1];
 781    tmp = tc;
 782    for (i = 4; i; i--, data++)
 783    {
 784        p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 785        q0 = data[0]; q1 = data[imageWidth];
 786        if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 787             ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 788             ((unsigned)ABS(q1-q0) < thresholds->beta) )
 789        {
 790            p2 = data[-imageWidth*3];
 791
 792            if ((unsigned)ABS(p2-p0) < thresholds->beta)
 793            {
 794                data[-imageWidth*2] = (u8)(p1 + CLIP3(-tc,tc,
 795                    (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 796                tmp++;
 797            }
 798
 799            q2 = data[imageWidth*2];
 800
 801            if ((unsigned)ABS(q2-q0) < thresholds->beta)
 802            {
 803                data[imageWidth] = (u8)(q1 + CLIP3(-tc,tc,
 804                    (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 805                tmp++;
 806            }
 807
 808            delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 809                      (p1 - q1) + 4) >> 3));
 810
 811            p0 = clp[p0 + delta];
 812            q0 = clp[q0 - delta];
 813            tmp = tc;
 814            data[-imageWidth] = p0;
 815            data[  0] = q0;
 816        }
 817    }
 818}
 819
 820/*------------------------------------------------------------------------------
 821
 822    Function: FilterHorLuma
 823
 824        Functional description:
 825            Filter all four successive horizontal 4-pixel luma edges. This can
 826            be done when bS is equal to all four edges.
 827
 828------------------------------------------------------------------------------*/
 829void FilterHorLuma(
 830  u8 *data,
 831  u32 bS,
 832  edgeThreshold_t *thresholds,
 833  i32 imageWidth)
 834{
 835
 836/* Variables */
 837
 838    i32 delta, tc, tmp;
 839    u32 i;
 840    u8 p0, q0, p1, q1, p2, q2;
 841    u32 tmpFlag;
 842    const u8 *clp = h264bsdClip + 512;
 843
 844/* Code */
 845
 846    ASSERT(data);
 847    ASSERT(bS <= 4);
 848    ASSERT(thresholds);
 849
 850    if (bS < 4)
 851    {
 852        tc = thresholds->tc0[bS-1];
 853        tmp = tc;
 854        for (i = 16; i; i--, data++)
 855        {
 856            p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 857            q0 = data[0]; q1 = data[imageWidth];
 858            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 859                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 860                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 861            {
 862                p2 = data[-imageWidth*3];
 863
 864                if ((unsigned)ABS(p2-p0) < thresholds->beta)
 865                {
 866                    data[-imageWidth*2] = (u8)(p1 + CLIP3(-tc,tc,
 867                        (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 868                    tmp++;
 869                }
 870
 871                q2 = data[imageWidth*2];
 872
 873                if ((unsigned)ABS(q2-q0) < thresholds->beta)
 874                {
 875                    data[imageWidth] = (u8)(q1 + CLIP3(-tc,tc,
 876                        (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 877                    tmp++;
 878                }
 879
 880                delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 881                          (p1 - q1) + 4) >> 3));
 882
 883                p0 = clp[p0 + delta];
 884                q0 = clp[q0 - delta];
 885                tmp = tc;
 886                data[-imageWidth] = p0;
 887                data[  0] = q0;
 888            }
 889        }
 890    }
 891    else
 892    {
 893        for (i = 16; i; i--, data++)
 894        {
 895            p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 896            q0 = data[0]; q1 = data[imageWidth];
 897            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 898                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 899                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 900            {
 901                tmpFlag = ((unsigned)ABS(p0-q0) < ((thresholds->alpha >> 2) +2))
 902                            ? HANTRO_TRUE : HANTRO_FALSE;
 903
 904                p2 = data[-imageWidth*3];
 905                q2 = data[imageWidth*2];
 906
 907                if (tmpFlag && (unsigned)ABS(p2-p0) < thresholds->beta)
 908                {
 909                    tmp = p1 + p0 + q0;
 910                    data[-imageWidth] = (u8)((p2 + 2 * tmp + q1 + 4) >> 3);
 911                    data[-imageWidth*2] = (u8)((p2 + tmp + 2) >> 2);
 912                    data[-imageWidth*3] = (u8)((2 * data[-imageWidth*4] +
 913                                           3 * p2 + tmp + 4) >> 3);
 914                }
 915                else
 916                    data[-imageWidth] = (u8)((2 * p1 + p0 + q1 + 2) >> 2);
 917
 918                if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta)
 919                {
 920                    tmp = p0 + q0 + q1;
 921                    data[ 0] = (u8)((p1 + 2 * tmp + q2 + 4) >> 3);
 922                    data[imageWidth] = (u8)((tmp + q2 + 2) >> 2);
 923                    data[imageWidth*2] = (u8)((2 * data[imageWidth*3] +
 924                                          3 * q2 + tmp + 4) >> 3);
 925                }
 926                else
 927                    data[0] = (2 * q1 + q0 + p1 + 2) >> 2;
 928            }
 929        }
 930    }
 931
 932}
 933
 934/*------------------------------------------------------------------------------
 935
 936    Function: FilterVerChromaEdge
 937
 938        Functional description:
 939            Filter one vertical 2-pixel chroma edge
 940
 941------------------------------------------------------------------------------*/
 942void FilterVerChromaEdge(
 943  u8 *data,
 944  u32 bS,
 945  edgeThreshold_t *thresholds,
 946  u32 width)
 947{
 948
 949/* Variables */
 950
 951    i32 delta, tc;
 952    u8 p0, q0, p1, q1;
 953    const u8 *clp = h264bsdClip + 512;
 954
 955/* Code */
 956
 957    ASSERT(data);
 958    ASSERT(bS <= 4);
 959    ASSERT(thresholds);
 960
 961    p1 = data[-2]; p0 = data[-1];
 962    q0 = data[0]; q1 = data[1];
 963    if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 964         ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 965         ((unsigned)ABS(q1-q0) < thresholds->beta) )
 966    {
 967        if (bS < 4)
 968        {
 969            tc = thresholds->tc0[bS-1] + 1;
 970            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
 971                      (p1 - q1) + 4) >> 3));
 972            p0 = clp[p0 + delta];
 973            q0 = clp[q0 - delta];
 974            data[-1] = p0;
 975            data[ 0] = q0;
 976        }
 977        else
 978        {
 979            data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
 980            data[ 0] = (2 * q1 + q0 + p1 + 2) >> 2;
 981        }
 982    }
 983    data += width;
 984    p1 = data[-2]; p0 = data[-1];
 985    q0 = data[0]; q1 = data[1];
 986    if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 987         ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 988         ((unsigned)ABS(q1-q0) < thresholds->beta) )
 989    {
 990        if (bS < 4)
 991        {
 992            tc = thresholds->tc0[bS-1] + 1;
 993            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
 994                      (p1 - q1) + 4) >> 3));
 995            p0 = clp[p0 + delta];
 996            q0 = clp[q0 - delta];
 997            data[-1] = p0;
 998            data[ 0] = q0;
 999        }
1000        else
1001        {
1002            data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
1003            data[ 0] = (2 * q1 + q0 + p1 + 2) >> 2;
1004        }
1005    }
1006
1007}
1008
1009/*------------------------------------------------------------------------------
1010
1011    Function: FilterHorChromaEdge
1012
1013        Functional description:
1014            Filter one horizontal 2-pixel chroma edge
1015
1016------------------------------------------------------------------------------*/
1017void FilterHorChromaEdge(
1018  u8 *data,
1019  u32 bS,
1020  edgeThreshold_t *thresholds,
1021  i32 width)
1022{
1023
1024/* Variables */
1025
1026    i32 delta, tc;
1027    u32 i;
1028    u8 p0, q0, p1, q1;
1029    const u8 *clp = h264bsdClip + 512;
1030
1031/* Code */
1032
1033    ASSERT(data);
1034    ASSERT(bS < 4);
1035    ASSERT(thresholds);
1036
1037    tc = thresholds->tc0[bS-1] + 1;
1038    for (i = 2; i; i--, data++)
1039    {
1040        p1 = data[-width*2]; p0 = data[-width];
1041        q0 = data[0]; q1 = data[width];
1042        if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1043             ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1044             ((unsigned)ABS(q1-q0) < thresholds->beta) )
1045        {
1046            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
1047                      (p1 - q1) + 4) >> 3));
1048            p0 = clp[p0 + delta];
1049            q0 = clp[q0 - delta];
1050            data[-width] = p0;
1051            data[  0] = q0;
1052        }
1053    }
1054}
1055
1056/*------------------------------------------------------------------------------
1057
1058    Function: FilterHorChroma
1059
1060        Functional description:
1061            Filter all four successive horizontal 2-pixel chroma edges. This
1062            can be done if bS is equal for all four edges.
1063
1064------------------------------------------------------------------------------*/
1065void FilterHorChroma(
1066  u8 *data,
1067  u32 bS,
1068  edgeThreshold_t *thresholds,
1069  i32 width)
1070{
1071
1072/* Variables */
1073
1074    i32 delta, tc;
1075    u32 i;
1076    u8 p0, q0, p1, q1;
1077    const u8 *clp = h264bsdClip + 512;
1078
1079/* Code */
1080
1081    ASSERT(data);
1082    ASSERT(bS <= 4);
1083    ASSERT(thresholds);
1084
1085    if (bS < 4)
1086    {
1087        tc = thresholds->tc0[bS-1] + 1;
1088        for (i = 8; i; i--, data++)
1089        {
1090            p1 = data[-width*2]; p0 = data[-width];
1091            q0 = data[0]; q1 = data[width];
1092            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1093                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1094                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
1095            {
1096                delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
1097                          (p1 - q1) + 4) >> 3));
1098                p0 = clp[p0 + delta];
1099                q0 = clp[q0 - delta];
1100                data[-width] = p0;
1101                data[  0] = q0;
1102            }
1103        }
1104    }
1105    else
1106    {
1107        for (i = 8; i; i--, data++)
1108        {
1109            p1 = data[-width*2]; p0 = data[-width];
1110            q0 = data[0]; q1 = data[width];
1111            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1112                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1113                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
1114            {
1115                    data[-width] = (2 * p1 + p0 + q1 + 2) >> 2;
1116                    data[  0] = (2 * q1 + q0 + p1 + 2) >> 2;
1117            }
1118        }
1119    }
1120
1121}
1122
1123
1124/*------------------------------------------------------------------------------
1125
1126    Function: GetBoundaryStrengths
1127
1128        Functional description:
1129            Function to calculate boundary strengths for all edges of a
1130            macroblock. Function returns HANTRO_TRUE if any of the bS values for
1131            the macroblock had non-zero value, HANTRO_FALSE otherwise.
1132
1133------------------------------------------------------------------------------*/
1134u32 GetBoundaryStrengths(mbStorage_t *mb, bS_t *bS, u32 flags)
1135{
1136
1137/* Variables */
1138
1139    /* this flag is set HANTRO_TRUE as soon as any boundary strength value is
1140     * non-zero */
1141    u32 nonZeroBs = HANTRO_FALSE;
1142
1143/* Code */
1144
1145    ASSERT(mb);
1146    ASSERT(bS);
1147    ASSERT(flags);
1148
1149    /* top edges */
1150    if (flags & FILTER_TOP_EDGE)
1151    {
1152        if (IS_INTRA_MB(*mb) || IS_INTRA_MB(*mb->mbB))
1153        {
1154            bS[0].top = bS[1].top = bS[2].top = bS[3].top = 4;
1155            nonZeroBs = HANTRO_TRUE;
1156        }
1157        else
1158        {
1159            bS[0].top = EdgeBoundaryStrength(mb, mb->mbB, 0, 10);
1160            bS[1].top = EdgeBoundaryStrength(mb, mb->mbB, 1, 11);
1161            bS[2].top = EdgeBoundaryStrength(mb, mb->mbB, 4, 14);
1162            bS[3].top = EdgeBoundaryStrength(mb, mb->mbB, 5, 15);
1163            if (bS[0].top || bS[1].top || bS[2].top || bS[3].top)
1164                nonZeroBs = HANTRO_TRUE;
1165        }
1166    }
1167    else
1168    {
1169        bS[0].top = bS[1].top = bS[2].top = bS[3].top = 0;
1170    }
1171
1172    /* left edges */
1173    if (flags & FILTER_LEFT_EDGE)
1174    {
1175        if (IS_INTRA_MB(*mb) || IS_INTRA_MB(*mb->mbA))
1176        {
1177            bS[0].left = bS[4].left = bS[8].left = bS[12].left = 4;
1178            nonZeroBs = HANTRO_TRUE;
1179        }
1180        else
1181        {
1182            bS[0].left = EdgeBoundaryStrength(mb, mb->mbA, 0, 5);
1183            bS[4].left = EdgeBoundaryStrength(mb, mb->mbA, 2, 7);
1184            bS[8].left = EdgeBoundaryStrength(mb, mb->mbA, 8, 13);
1185            bS[12].left = EdgeBoundaryStrength(mb, mb->mbA, 10, 15);
1186            if (!nonZeroBs &&
1187                (bS[0].left || bS[4].left || bS[8].left || bS[12].left))
1188                nonZeroBs = HANTRO_TRUE;
1189        }
1190    }
1191    else
1192    {
1193        bS[0].left = bS[4].left = bS[8].left = bS[12].left = 0;
1194    }
1195
1196    /* inner edges */
1197    if (IS_INTRA_MB(*mb))
1198    {
1199        bS[4].top  = bS[5].top  = bS[6].top  = bS[7].top  =
1200        bS[8].top  = bS[9].top  = bS[10].top = bS[11].top =
1201        bS[12].top = bS[13].top = bS[14].top = bS[15].top = 3;
1202
1203        bS[1].left  = bS[2].left  = bS[3].left  =
1204        bS[5].left  = bS[6].left  = bS[7].left  =
1205        bS[9].left  = bS[10].left = bS[11].left =
1206        bS[13].left = bS[14].left = bS[15].left = 3;
1207        nonZeroBs = HANTRO_TRUE;
1208    }
1209    else
1210    {
1211        /* 16x16 inter mb -> ref addresses or motion vectors cannot differ,
1212         * only check if either of the blocks contain coefficients */
1213        if (h264bsdNumMbPart(mb->mbType) == 1)
1214        {
1215            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1216            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1217            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1218            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1219            bS[8].top = mb->totalCoeff[8] || mb->totalCoeff[2] ? 2 : 0;
1220            bS[9].top = mb->totalCoeff[9] || mb->totalCoeff[3] ? 2 : 0;
1221            bS[10].top = mb->totalCoeff[12] || mb->totalCoeff[6] ? 2 : 0;
1222            bS[11].top = mb->totalCoeff[13] || mb->totalCoeff[7] ? 2 : 0;
1223            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1224            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1225            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1226            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1227
1228            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1229            bS[2].left = mb->totalCoeff[4] || mb->totalCoeff[1] ? 2 : 0;
1230            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1231            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1232            bS[6].left = mb->totalCoeff[6] || mb->totalCoeff[3] ? 2 : 0;
1233            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1234            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1235            bS[10].left = mb->totalCoeff[12] || mb->totalCoeff[9] ? 2 : 0;
1236            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1237            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1238            bS[14].left = mb->totalCoeff[14] || mb->totalCoeff[11] ? 2 : 0;
1239            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1240        }
1241        /* 16x8 inter mb -> ref addresses and motion vectors can be different
1242         * only for the middle horizontal edge, for the other top edges it is
1243         * enough to check whether the blocks contain coefficients or not. The
1244         * same applies to all internal left edges. */
1245        else if (mb->mbType == P_L0_L0_16x8)
1246        {
1247            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1248            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1249            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1250            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1251            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1252            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1253            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1254            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1255            bS[8].top = InnerBoundaryStrength(mb, 8, 2);
1256            bS[9].top = InnerBoundaryStrength(mb, 9, 3);
1257            bS[10].top = InnerBoundaryStrength(mb, 12, 6);
1258            bS[11].top = InnerBoundaryStrength(mb, 13, 7);
1259
1260            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1261            bS[2].left = mb->totalCoeff[4] || mb->totalCoeff[1] ? 2 : 0;
1262            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1263            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1264            bS[6].left = mb->totalCoeff[6] || mb->totalCoeff[3] ? 2 : 0;
1265            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1266            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1267            bS[10].left = mb->totalCoeff[12] || mb->totalCoeff[9] ? 2 : 0;
1268            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1269            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1270            bS[14].left = mb->totalCoeff[14] || mb->totalCoeff[11] ? 2 : 0;
1271            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1272        }
1273        /* 8x16 inter mb -> ref addresses and motion vectors can be different
1274         * only for the middle vertical edge, for the other left edges it is
1275         * enough to check whether the blocks contain coefficients or not. The
1276         * same applies to all internal top edges. */
1277        else if (mb->mbType == P_L0_L0_8x16)
1278        {
1279            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1280            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1281            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1282            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1283            bS[8].top = mb->totalCoeff[8] || mb->totalCoeff[2] ? 2 : 0;
1284            bS[9].top = mb->totalCoeff[9] || mb->totalCoeff[3] ? 2 : 0;
1285            bS[10].top = mb->totalCoeff[12] || mb->totalCoeff[6] ? 2 : 0;
1286            bS[11].top = mb->totalCoeff[13] || mb->totalCoeff[7] ? 2 : 0;
1287            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1288            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1289            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1290            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1291
1292            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1293            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1294            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1295            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1296            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1297            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1298            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1299            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1300            bS[2].left = InnerBoundaryStrength(mb, 4, 1);
1301            bS[6].left = InnerBoundaryStrength(mb, 6, 3);
1302            bS[10].left = InnerBoundaryStrength(mb, 12, 9);
1303            bS[14].left = InnerBoundaryStrength(mb, 14, 11);
1304        }
1305        else
1306        {
1307            bS[4].top =
1308                InnerBoundaryStrength(mb, mb4x4Index[4], mb4x4Index[0]);
1309            bS[5].top =
1310                InnerBoundaryStrength(mb, mb4x4Index[5], mb4x4Index[1]);
1311            bS[6].top =
1312                InnerBoundaryStrength(mb, mb4x4Index[6], mb4x4Index[2]);
1313            bS[7].top =
1314                InnerBoundaryStrength(mb, mb4x4Index[7], mb4x4Index[3]);
1315            bS[8].top =
1316                InnerBoundaryStrength(mb, mb4x4Index[8], mb4x4Index[4]);
1317            bS[9].top =
1318                InnerBoundaryStrength(mb, mb4x4Index[9], mb4x4Index[5]);
1319            bS[10].top =
1320                InnerBoundaryStrength(mb, mb4x4Index[10], mb4x4Index[6]);
1321            bS[11].top =
1322                InnerBoundaryStrength(mb, mb4x4Index[11], mb4x4Index[7]);
1323            bS[12].top =
1324                InnerBoundaryStrength(mb, mb4x4Index[12], mb4x4Index[8]);
1325            bS[13].top =
1326                InnerBoundaryStrength(mb, mb4x4Index[13], mb4x4Index[9]);
1327            bS[14].top =
1328                InnerBoundaryStrength(mb, mb4x4Index[14], mb4x4Index[10]);
1329            bS[15].top =
1330                InnerBoundaryStrength(mb, mb4x4Index[15], mb4x4Index[11]);
1331
1332            bS[1].left =
1333                InnerBoundaryStrength(mb, mb4x4Index[1], mb4x4Index[0]);
1334            bS[2].left =
1335                InnerBoundaryStrength(mb, mb4x4Index[2], mb4x4Index[1]);
1336            bS[3].left =
1337                InnerBoundaryStrength(mb, mb4x4Index[3], mb4x4Index[2]);
1338            bS[5].left =
1339                InnerBoundaryStrength(mb, mb4x4Index[5], mb4x4Index[4]);
1340            bS[6].left =
1341                InnerBoundaryStrength(mb, mb4x4Index[6], mb4x4Index[5]);
1342            bS[7].left =
1343                InnerBoundaryStrength(mb, mb4x4Index[7], mb4x4Index[6]);
1344            bS[9].left =
1345                InnerBoundaryStrength(mb, mb4x4Index[9], mb4x4Index[8]);
1346            bS[10].left =
1347                InnerBoundaryStrength(mb, mb4x4Index[10], mb4x4Index[9]);
1348            bS[11].left =
1349                InnerBoundaryStrength(mb, mb4x4Index[11], mb4x4Index[10]);
1350            bS[13].left =
1351                InnerBoundaryStrength(mb, mb4x4Index[13], mb4x4Index[12]);
1352            bS[14].left =
1353                InnerBoundaryStrength(mb, mb4x4Index[14], mb4x4Index[13]);
1354            bS[15].left =
1355                InnerBoundaryStrength(mb, mb4x4Index[15], mb4x4Index[14]);
1356        }
1357        if (!nonZeroBs &&
1358            (bS[4].top || bS[5].top || bS[6].top || bS[7].top ||
1359             bS[8].top || bS[9].top || bS[10].top || bS[11].top ||
1360             bS[12].top || bS[13].top || bS[14].top || bS[15].top ||
1361             bS[1].left || bS[2].left || bS[3].left ||
1362             bS[5].left || bS[6].left || bS[7].left ||
1363             bS[9].left || bS[10].left || bS[11].left ||
1364             bS[13].left || bS[14].left || bS[15].left))
1365            nonZeroBs = HANTRO_TRUE;
1366    }
1367
1368    return(nonZeroBs);
1369
1370}
1371
1372/*------------------------------------------------------------------------------
1373
1374    Function: GetLumaEdgeThresholds
1375
1376        Functional description:
1377            Compute alpha, beta and tc0 thresholds for inner, left and top
1378            luma edges of a macroblock.
1379
1380------------------------------------------------------------------------------*/
1381void GetLumaEdgeThresholds(
1382  edgeThreshold_t *thresholds,
1383  mbStorage_t *mb,
1384  u32 filteringFlags)
1385{
1386
1387/* Variables */
1388
1389    u32 indexA, indexB;
1390    u32 qpAv, qp, qpTmp;
1391
1392/* Code */
1393
1394    ASSERT(thresholds);
1395    ASSERT(mb);
1396
1397    qp = mb->qpY;
1398
1399    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
1400    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
1401
1402    thresholds[INNER].alpha = alphas[indexA];
1403    thresholds[INNER].beta = betas[indexB];
1404    thresholds[INNER].tc0 = tc0[indexA];
1405
1406    if (filteringFlags & FILTER_TOP_EDGE)
1407    {
1408        qpTmp = mb->mbB->qpY;
1409        if (qpTmp != qp)
1410        {
1411            qpAv = (qp + qpTmp + 1) >> 1;
1412
1413            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1414            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1415
1416            thresholds[TOP].alpha = alphas[indexA];
1417            thresholds[TOP].beta = betas[indexB];
1418            thresholds[TOP].tc0 = tc0[indexA];
1419        }
1420        else
1421        {
1422            thresholds[TOP].alpha = thresholds[INNER].alpha;
1423            thresholds[TOP].beta = thresholds[INNER].beta;
1424            thresholds[TOP].tc0 = thresholds[INNER].tc0;
1425        }
1426    }
1427    if (filteringFlags & FILTER_LEFT_EDGE)
1428    {
1429        qpTmp = mb->mbA->qpY;
1430        if (qpTmp != qp)
1431        {
1432            qpAv = (qp + qpTmp + 1) >> 1;
1433
1434            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1435            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1436
1437            thresholds[LEFT].alpha = alphas[indexA];
1438            thresholds[LEFT].beta = betas[indexB];
1439            thresholds[LEFT].tc0 = tc0[indexA];
1440        }
1441        else
1442        {
1443            thresholds[LEFT].alpha = thresholds[INNER].alpha;
1444            thresholds[LEFT].beta = thresholds[INNER].beta;
1445            thresholds[LEFT].tc0 = thresholds[INNER].tc0;
1446        }
1447    }
1448
1449}
1450
1451/*------------------------------------------------------------------------------
1452
1453    Function: GetChromaEdgeThresholds
1454
1455        Functional description:
1456            Compute alpha, beta and tc0 thresholds for inner, left and top
1457            chroma edges of a macroblock.
1458
1459------------------------------------------------------------------------------*/
1460void GetChromaEdgeThresholds(
1461  edgeThreshold_t *thresholds,
1462  mbStorage_t *mb,
1463  u32 filteringFlags,
1464  i32 chromaQpIndexOffset)
1465{
1466
1467/* Variables */
1468
1469    u32 indexA, indexB;
1470    u32 qpAv, qp, qpTmp;
1471
1472/* Code */
1473
1474    ASSERT(thresholds);
1475    ASSERT(mb);
1476
1477    qp = mb->qpY;
1478    qp = h264bsdQpC[CLIP3(0, 51, (i32)qp + chromaQpIndexOffset)];
1479
1480    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
1481    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
1482
1483    thresholds[INNER].alpha = alphas[indexA];
1484    thresholds[INNER].beta = betas[indexB];
1485    thresholds[INNER].tc0 = tc0[indexA];
1486
1487    if (filteringFlags & FILTER_TOP_EDGE)
1488    {
1489        qpTmp = mb->mbB->qpY;
1490        if (qpTmp != mb->qpY)
1491        {
1492            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
1493            qpAv = (qp + qpTmp + 1) >> 1;
1494
1495            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1496            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1497
1498            thresholds[TOP].alpha = alphas[indexA];
1499            thresholds[TOP].beta = betas[indexB];
1500            thresholds[TOP].tc0 = tc0[indexA];
1501        }
1502        else
1503        {
1504            thresholds[TOP].alpha = thresholds[INNER].alpha;
1505            thresholds[TOP].beta = thresholds[INNER].beta;
1506            thresholds[TOP].tc0 = thresholds[INNER].tc0;
1507        }
1508    }
1509    if (filteringFlags & FILTER_LEFT_EDGE)
1510    {
1511        qpTmp = mb->mbA->qpY;
1512        if (qpTmp != mb->qpY)
1513        {
1514            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
1515            qpAv = (qp + qpTmp + 1) >> 1;
1516
1517            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1518            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1519
1520            thresholds[LEFT].alpha = alphas[indexA];
1521            thresholds[LEFT].beta = betas[indexB];
1522            thresholds[LEFT].tc0 = tc0[indexA];
1523        }
1524        else
1525        {
1526            thresholds[LEFT].alpha = thresholds[INNER].alpha;
1527            thresholds[LEFT].beta = thresholds[INNER].beta;
1528            thresholds[LEFT].tc0 = thresholds[INNER].tc0;
1529        }
1530    }
1531
1532}
1533
1534/*------------------------------------------------------------------------------
1535
1536    Function: FilterLuma
1537
1538        Functional description:
1539            Function to filter all luma edges of a macroblock
1540
1541------------------------------------------------------------------------------*/
1542void FilterLuma(
1543  u8 *data,
1544  bS_t *bS,
1545  edgeThreshold_t *thresholds,
1546  u32 width)
1547{
1548
1549/* Variables */
1550
1551    u32 vblock;
1552    bS_t *tmp;
1553    u8 *ptr;
1554    u32 offset;
1555
1556/* Code */
1557
1558    ASSERT(data);
1559    ASSERT(bS);
1560    ASSERT(thresholds);
1561
1562    ptr = data;
1563    tmp = bS;
1564
1565    offset  = TOP;
1566
1567    /* loop block rows, perform filtering for all vertica…

Large files files are truncated, but you can click here to view the full file