PageRenderTime 262ms CodeModel.GetById 67ms app.highlight 180ms RepoModel.GetById 1ms app.codeStats 0ms

/H264Dec/source/h264bsd_deblocking.c

http://github.com/mbebenita/Broadway
C | 2417 lines | 1698 code | 295 blank | 424 comment | 323 complexity | 7c34d152f359d1bdba24d54c513ce489 MD5 | raw file
   1/*
   2 * Copyright (C) 2009 The Android Open Source Project
   3 *
   4 * Licensed under the Apache License, Version 2.0 (the "License");
   5 * you may not use this file except in compliance with the License.
   6 * You may obtain a copy of the License at
   7 *
   8 *      http://www.apache.org/licenses/LICENSE-2.0
   9 *
  10 * Unless required by applicable law or agreed to in writing, software
  11 * distributed under the License is distributed on an "AS IS" BASIS,
  12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 * See the License for the specific language governing permissions and
  14 * limitations under the License.
  15 */
  16
  17/*------------------------------------------------------------------------------
  18
  19    Table of contents
  20
  21     1. Include headers
  22     2. External compiler flags
  23     3. Module defines
  24     4. Local function prototypes
  25     5. Functions
  26          h264bsdFilterPicture
  27          FilterVerLumaEdge
  28          FilterHorLumaEdge
  29          FilterHorLuma
  30          FilterVerChromaEdge
  31          FilterHorChromaEdge
  32          FilterHorChroma
  33          InnerBoundaryStrength
  34          EdgeBoundaryStrength
  35          GetBoundaryStrengths
  36          IsSliceBoundaryOnLeft
  37          IsSliceBoundaryOnTop
  38          GetMbFilteringFlags
  39          GetLumaEdgeThresholds
  40          GetChromaEdgeThresholds
  41          FilterLuma
  42          FilterChroma
  43
  44------------------------------------------------------------------------------*/
  45
  46/*------------------------------------------------------------------------------
  47    1. Include headers
  48------------------------------------------------------------------------------*/
  49
  50#include "basetype.h"
  51#include "h264bsd_util.h"
  52#include "h264bsd_macroblock_layer.h"
  53#include "h264bsd_deblocking.h"
  54#include "h264bsd_dpb.h"
  55
  56#ifdef H264DEC_OMXDL
  57#include "omxtypes.h"
  58#include "omxVC.h"
  59#include "armVC.h"
  60#endif /* H264DEC_OMXDL */
  61
  62/*------------------------------------------------------------------------------
  63    2. External compiler flags
  64--------------------------------------------------------------------------------
  65
  66--------------------------------------------------------------------------------
  67    3. Module defines
  68------------------------------------------------------------------------------*/
  69
  70/* Switch off the following Lint messages for this file:
  71 * Info 701: Shift left of signed quantity (int)
  72 * Info 702: Shift right of signed quantity (int)
  73 */
  74/*lint -e701 -e702 */
  75
  76/* array of alpha values, from the standard */
  77static const u8 alphas[52] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,4,5,6,7,8,9,10,
  78    12,13,15,17,20,22,25,28,32,36,40,45,50,56,63,71,80,90,101,113,127,144,162,
  79    182,203,226,255,255};
  80
  81/* array of beta values, from the standard */
  82static const u8 betas[52] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,2,2,3,3,3,3,4,4,
  83    4,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,16,16,17,17,18,18};
  84
  85
  86
  87#ifndef H264DEC_OMXDL
  88/* array of tc0 values, from the standard, each triplet corresponds to a
  89 * column in the table. Indexing goes as tc0[indexA][bS-1] */
  90static const u8 tc0[52][3] = {
  91    {0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},
  92    {0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},{0,0,0},
  93    {0,0,0},{0,0,1},{0,0,1},{0,0,1},{0,0,1},{0,1,1},{0,1,1},{1,1,1},
  94    {1,1,1},{1,1,1},{1,1,1},{1,1,2},{1,1,2},{1,1,2},{1,1,2},{1,2,3},
  95    {1,2,3},{2,2,3},{2,2,4},{2,3,4},{2,3,4},{3,3,5},{3,4,6},{3,4,6},
  96    {4,5,7},{4,5,8},{4,6,9},{5,7,10},{6,8,11},{6,8,13},{7,10,14},{8,11,16},
  97    {9,12,18},{10,13,20},{11,15,23},{13,17,25}
  98};
  99#else
 100/* array of tc0 values, from the standard, each triplet corresponds to a
 101 * column in the table. Indexing goes as tc0[indexA][bS] */
 102static const u8 tc0[52][5] = {
 103    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 104    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 105    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 106    {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0}, {0, 0, 0, 0, 0},
 107    {0, 0, 0, 0, 0}, {0, 0, 0, 1, 0}, {0, 0, 0, 1, 0}, {0, 0, 0, 1, 0},
 108    {0, 0, 0, 1, 0}, {0, 0, 1, 1, 0}, {0, 0, 1, 1, 0}, {0, 1, 1, 1, 0},
 109    {0, 1, 1, 1, 0}, {0, 1, 1, 1, 0}, {0, 1, 1, 1, 0}, {0, 1, 1, 2, 0},
 110    {0, 1, 1, 2, 0}, {0, 1, 1, 2, 0}, {0, 1, 1, 2, 0}, {0, 1, 2, 3, 0},
 111    {0, 1, 2, 3, 0}, {0, 2, 2, 3, 0}, {0, 2, 2, 4, 0}, {0, 2, 3, 4, 0},
 112    {0, 2, 3, 4, 0}, {0, 3, 3, 5, 0}, {0, 3, 4, 6, 0}, {0, 3, 4, 6, 0},
 113    {0, 4, 5, 7, 0}, {0, 4, 5, 8, 0}, {0, 4, 6, 9, 0}, {0, 5, 7, 10, 0},
 114    {0, 6, 8, 11, 0}, {0, 6, 8, 13, 0}, {0, 7, 10, 14, 0},
 115    {0, 8, 11, 16, 0}, {0, 9, 12, 18, 0}, {0, 10, 13, 20, 0},
 116    {0, 11, 15, 23, 0}, {0, 13, 17, 25, 0}
 117};
 118#endif
 119
 120
 121#ifndef H264DEC_OMXDL
 122/* mapping of raster scan block index to 4x4 block index */
 123static const u32 mb4x4Index[16] =
 124    {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
 125
 126typedef struct {
 127    const u8 *tc0;
 128    u32 alpha;
 129    u32 beta;
 130} edgeThreshold_t;
 131
 132typedef struct {
 133    u32 top;
 134    u32 left;
 135} bS_t;
 136
 137enum { TOP = 0, LEFT = 1, INNER = 2 };
 138#endif /* H264DEC_OMXDL */
 139
 140#define FILTER_LEFT_EDGE    0x04
 141#define FILTER_TOP_EDGE     0x02
 142#define FILTER_INNER_EDGE   0x01
 143
 144
 145/* clipping table defined in intra_prediction.c */
 146extern const u8 h264bsdClip[];
 147
 148/*------------------------------------------------------------------------------
 149    4. Local function prototypes
 150------------------------------------------------------------------------------*/
 151
 152static u32 InnerBoundaryStrength(mbStorage_t *mb1, u32 i1, u32 i2);
 153
 154#ifndef H264DEC_OMXDL
 155static u32 EdgeBoundaryStrength(mbStorage_t *mb1, mbStorage_t *mb2,
 156    u32 i1, u32 i2);
 157#else
 158static u32 InnerBoundaryStrength2(mbStorage_t *mb1, u32 i1, u32 i2);
 159static u32 EdgeBoundaryStrengthLeft(mbStorage_t *mb1, mbStorage_t *mb2);
 160static u32 EdgeBoundaryStrengthTop(mbStorage_t *mb1, mbStorage_t *mb2);
 161#endif
 162
 163static u32 IsSliceBoundaryOnLeft(mbStorage_t *mb);
 164
 165static u32 IsSliceBoundaryOnTop(mbStorage_t *mb);
 166
 167static u32 GetMbFilteringFlags(mbStorage_t *mb);
 168
 169#ifndef H264DEC_OMXDL
 170
 171static u32 GetBoundaryStrengths(mbStorage_t *mb, bS_t *bs, u32 flags);
 172
 173static void FilterLuma(u8 *data, bS_t *bS, edgeThreshold_t *thresholds,
 174        u32 imageWidth);
 175
 176static void FilterChroma(u8 *cb, u8 *cr, bS_t *bS, edgeThreshold_t *thresholds,
 177        u32 imageWidth);
 178
 179static void FilterVerLumaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 180        u32 imageWidth);
 181static void FilterHorLumaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 182        i32 imageWidth);
 183static void FilterHorLuma( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 184        i32 imageWidth);
 185
 186static void FilterVerChromaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 187  u32 imageWidth);
 188static void FilterHorChromaEdge( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 189  i32 imageWidth);
 190static void FilterHorChroma( u8 *data, u32 bS, edgeThreshold_t *thresholds,
 191  i32 imageWidth);
 192
 193static void GetLumaEdgeThresholds(
 194  edgeThreshold_t *thresholds,
 195  mbStorage_t *mb,
 196  u32 filteringFlags);
 197
 198static void GetChromaEdgeThresholds(
 199  edgeThreshold_t *thresholds,
 200  mbStorage_t *mb,
 201  u32 filteringFlags,
 202  i32 chromaQpIndexOffset);
 203
 204#else /* H264DEC_OMXDL */
 205
 206static u32 GetBoundaryStrengths(mbStorage_t *mb, u8 (*bs)[16], u32 flags);
 207
 208static void GetLumaEdgeThresholds(
 209    mbStorage_t *mb,
 210    u8 (*alpha)[2],
 211    u8 (*beta)[2],
 212    u8 (*threshold)[16],
 213    u8 (*bs)[16],
 214    u32 filteringFlags );
 215
 216static void GetChromaEdgeThresholds(
 217    mbStorage_t *mb,
 218    u8 (*alpha)[2],
 219    u8 (*beta)[2],
 220    u8 (*threshold)[8],
 221    u8 (*bs)[16],
 222    u32 filteringFlags,
 223    i32 chromaQpIndexOffset);
 224
 225#endif /* H264DEC_OMXDL */
 226
 227/*------------------------------------------------------------------------------
 228
 229    Function: IsSliceBoundaryOnLeft
 230
 231        Functional description:
 232            Function to determine if there is a slice boundary on the left side
 233            of a macroblock.
 234
 235------------------------------------------------------------------------------*/
 236u32 IsSliceBoundaryOnLeft(mbStorage_t *mb)
 237{
 238
 239/* Variables */
 240
 241/* Code */
 242
 243    ASSERT(mb && mb->mbA);
 244
 245    if (mb->sliceId != mb->mbA->sliceId)
 246        return(HANTRO_TRUE);
 247    else
 248        return(HANTRO_FALSE);
 249
 250}
 251
 252/*------------------------------------------------------------------------------
 253
 254    Function: IsSliceBoundaryOnTop
 255
 256        Functional description:
 257            Function to determine if there is a slice boundary above the
 258            current macroblock.
 259
 260------------------------------------------------------------------------------*/
 261u32 IsSliceBoundaryOnTop(mbStorage_t *mb)
 262{
 263
 264/* Variables */
 265
 266/* Code */
 267
 268    ASSERT(mb && mb->mbB);
 269
 270    if (mb->sliceId != mb->mbB->sliceId)
 271        return(HANTRO_TRUE);
 272    else
 273        return(HANTRO_FALSE);
 274
 275}
 276
 277/*------------------------------------------------------------------------------
 278
 279    Function: GetMbFilteringFlags
 280
 281        Functional description:
 282          Function to determine which edges of a macroblock has to be
 283          filtered. Output is a bit-wise OR of FILTER_LEFT_EDGE,
 284          FILTER_TOP_EDGE and FILTER_INNER_EDGE, depending on which edges
 285          shall be filtered.
 286
 287------------------------------------------------------------------------------*/
 288u32 GetMbFilteringFlags(mbStorage_t *mb)
 289{
 290
 291/* Variables */
 292
 293    u32 flags = 0;
 294
 295/* Code */
 296
 297    ASSERT(mb);
 298
 299    /* nothing will be filtered if disableDeblockingFilterIdc == 1 */
 300    if (mb->disableDeblockingFilterIdc != 1)
 301    {
 302        flags |= FILTER_INNER_EDGE;
 303
 304        /* filterLeftMbEdgeFlag, left mb is MB_A */
 305        if (mb->mbA &&
 306            ((mb->disableDeblockingFilterIdc != 2) ||
 307             !IsSliceBoundaryOnLeft(mb)))
 308            flags |= FILTER_LEFT_EDGE;
 309
 310        /* filterTopMbEdgeFlag */
 311        if (mb->mbB &&
 312            ((mb->disableDeblockingFilterIdc != 2) ||
 313             !IsSliceBoundaryOnTop(mb)))
 314            flags |= FILTER_TOP_EDGE;
 315    }
 316
 317    return(flags);
 318
 319}
 320
 321/*------------------------------------------------------------------------------
 322
 323    Function: InnerBoundaryStrength
 324
 325        Functional description:
 326            Function to calculate boundary strength value bs for an inner
 327            edge of a macroblock. Macroblock type is checked before this is
 328            called -> no intra mb condition here.
 329
 330------------------------------------------------------------------------------*/
 331u32 InnerBoundaryStrength(mbStorage_t *mb1, u32 ind1, u32 ind2)
 332{
 333    i32 tmp1, tmp2;
 334    i32 mv1, mv2, mv3, mv4;
 335
 336    tmp1 = mb1->totalCoeff[ind1];
 337    tmp2 = mb1->totalCoeff[ind2];
 338    mv1 = mb1->mv[ind1].hor;
 339    mv2 = mb1->mv[ind2].hor;
 340    mv3 = mb1->mv[ind1].ver;
 341    mv4 = mb1->mv[ind2].ver;
 342
 343    if (tmp1 || tmp2)
 344    {
 345        return 2;
 346    }
 347    else if ( (ABS(mv1 - mv2) >= 4) || (ABS(mv3 - mv4) >= 4) ||
 348              (mb1->refAddr[ind1 >> 2] != mb1->refAddr[ind2 >> 2]) )
 349    {
 350        return 1;
 351    }
 352    else
 353        return 0;
 354}
 355
 356/*------------------------------------------------------------------------------
 357
 358    Function: InnerBoundaryStrength2
 359
 360        Functional description:
 361            Function to calculate boundary strength value bs for an inner
 362            edge of a macroblock. The function is the same as
 363            InnerBoundaryStrength but without checking totalCoeff.
 364
 365------------------------------------------------------------------------------*/
 366u32 InnerBoundaryStrength2(mbStorage_t *mb1, u32 ind1, u32 ind2)
 367{
 368    i32 tmp1, tmp2, tmp3, tmp4;
 369
 370    tmp1 = mb1->mv[ind1].hor;
 371    tmp2 = mb1->mv[ind2].hor;
 372    tmp3 = mb1->mv[ind1].ver;
 373    tmp4 = mb1->mv[ind2].ver;
 374
 375    if ( (ABS(tmp1 - tmp2) >= 4) || (ABS(tmp3 - tmp4) >= 4) ||
 376         (mb1->refAddr[ind1 >> 2] != mb1->refAddr[ind2 >> 2]))
 377    {
 378        return 1;
 379    }
 380    else
 381        return 0;
 382}
 383#ifndef H264DEC_OMXDL
 384/*------------------------------------------------------------------------------
 385
 386    Function: EdgeBoundaryStrength
 387
 388        Functional description:
 389            Function to calculate boundary strength value bs for left- or
 390            top-most edge of a macroblock. Macroblock types are checked
 391            before this is called -> no intra mb conditions here.
 392
 393------------------------------------------------------------------------------*/
 394u32 EdgeBoundaryStrength(mbStorage_t *mb1, mbStorage_t *mb2,
 395    u32 ind1, u32 ind2)
 396{
 397
 398    if (mb1->totalCoeff[ind1] || mb2->totalCoeff[ind2])
 399    {
 400        return 2;
 401    }
 402    else if ((mb1->refAddr[ind1 >> 2] != mb2->refAddr[ind2 >> 2]) ||
 403             (ABS(mb1->mv[ind1].hor - mb2->mv[ind2].hor) >= 4) ||
 404             (ABS(mb1->mv[ind1].ver - mb2->mv[ind2].ver) >= 4))
 405    {
 406        return 1;
 407    }
 408    else
 409        return 0;
 410}
 411
 412#else /* H264DEC_OMXDL */
 413
 414/*------------------------------------------------------------------------------
 415
 416    Function: EdgeBoundaryStrengthTop
 417
 418        Functional description:
 419            Function to calculate boundary strength value bs for
 420            top-most edge of a macroblock. Macroblock types are checked
 421            before this is called -> no intra mb conditions here.
 422
 423------------------------------------------------------------------------------*/
 424u32 EdgeBoundaryStrengthTop(mbStorage_t *mb1, mbStorage_t *mb2)
 425{
 426    u32 topBs = 0;
 427    u32 tmp1, tmp2, tmp3, tmp4;
 428
 429    tmp1 = mb1->totalCoeff[0];
 430    tmp2 = mb2->totalCoeff[10];
 431    tmp3 = mb1->totalCoeff[1];
 432    tmp4 = mb2->totalCoeff[11];
 433    if (tmp1 || tmp2)
 434    {
 435        topBs = 2<<0;
 436    }
 437    else if ((ABS(mb1->mv[0].hor - mb2->mv[10].hor) >= 4) ||
 438             (ABS(mb1->mv[0].ver - mb2->mv[10].ver) >= 4) ||
 439             (mb1->refAddr[0] != mb2->refAddr[10 >> 2]))
 440    {
 441        topBs = 1<<0;
 442    }
 443    tmp1 = mb1->totalCoeff[4];
 444    tmp2 = mb2->totalCoeff[14];
 445    if (tmp3 || tmp4)
 446    {
 447        topBs += 2<<8;
 448    }
 449    else if ((ABS(mb1->mv[1].hor - mb2->mv[11].hor) >= 4) ||
 450             (ABS(mb1->mv[1].ver - mb2->mv[11].ver) >= 4) ||
 451             (mb1->refAddr[0] != mb2->refAddr[11 >> 2]))
 452    {
 453        topBs += 1<<8;
 454    }
 455    tmp3 = mb1->totalCoeff[5];
 456    tmp4 = mb2->totalCoeff[15];
 457    if (tmp1 || tmp2)
 458    {
 459        topBs += 2<<16;
 460    }
 461    else if ((ABS(mb1->mv[4].hor - mb2->mv[14].hor) >= 4) ||
 462             (ABS(mb1->mv[4].ver - mb2->mv[14].ver) >= 4) ||
 463             (mb1->refAddr[4 >> 2] != mb2->refAddr[14 >> 2]))
 464    {
 465        topBs += 1<<16;
 466    }
 467    if (tmp3 || tmp4)
 468    {
 469        topBs += 2<<24;
 470    }
 471    else if ((ABS(mb1->mv[5].hor - mb2->mv[15].hor) >= 4) ||
 472             (ABS(mb1->mv[5].ver - mb2->mv[15].ver) >= 4) ||
 473             (mb1->refAddr[5 >> 2] != mb2->refAddr[15 >> 2]))
 474    {
 475        topBs += 1<<24;
 476    }
 477
 478    return topBs;
 479}
 480
 481/*------------------------------------------------------------------------------
 482
 483    Function: EdgeBoundaryStrengthLeft
 484
 485        Functional description:
 486            Function to calculate boundary strength value bs for left-
 487            edge of a macroblock. Macroblock types are checked
 488            before this is called -> no intra mb conditions here.
 489
 490------------------------------------------------------------------------------*/
 491u32 EdgeBoundaryStrengthLeft(mbStorage_t *mb1, mbStorage_t *mb2)
 492{
 493    u32 leftBs = 0;
 494    u32 tmp1, tmp2, tmp3, tmp4;
 495
 496    tmp1 = mb1->totalCoeff[0];
 497    tmp2 = mb2->totalCoeff[5];
 498    tmp3 = mb1->totalCoeff[2];
 499    tmp4 = mb2->totalCoeff[7];
 500
 501    if (tmp1 || tmp2)
 502    {
 503        leftBs = 2<<0;
 504    }
 505    else if ((ABS(mb1->mv[0].hor - mb2->mv[5].hor) >= 4) ||
 506             (ABS(mb1->mv[0].ver - mb2->mv[5].ver) >= 4) ||
 507             (mb1->refAddr[0] != mb2->refAddr[5 >> 2]))
 508    {
 509        leftBs = 1<<0;
 510    }
 511    tmp1 = mb1->totalCoeff[8];
 512    tmp2 = mb2->totalCoeff[13];
 513    if (tmp3 || tmp4)
 514    {
 515        leftBs += 2<<8;
 516    }
 517    else if ((ABS(mb1->mv[2].hor - mb2->mv[7].hor) >= 4) ||
 518             (ABS(mb1->mv[2].ver - mb2->mv[7].ver) >= 4) ||
 519             (mb1->refAddr[0] != mb2->refAddr[7 >> 2]))
 520    {
 521        leftBs += 1<<8;
 522    }
 523    tmp3 = mb1->totalCoeff[10];
 524    tmp4 = mb2->totalCoeff[15];
 525    if (tmp1 || tmp2)
 526    {
 527        leftBs += 2<<16;
 528    }
 529    else if ((ABS(mb1->mv[8].hor - mb2->mv[13].hor) >= 4) ||
 530             (ABS(mb1->mv[8].ver - mb2->mv[13].ver) >= 4) ||
 531             (mb1->refAddr[8 >> 2] != mb2->refAddr[13 >> 2]))
 532    {
 533        leftBs += 1<<16;
 534    }
 535    if (tmp3 || tmp4)
 536    {
 537        leftBs += 2<<24;
 538    }
 539    else if ((ABS(mb1->mv[10].hor - mb2->mv[15].hor) >= 4) ||
 540             (ABS(mb1->mv[10].ver - mb2->mv[15].ver) >= 4) ||
 541             (mb1->refAddr[10 >> 2] != mb2->refAddr[15 >> 2]))
 542    {
 543        leftBs += 1<<24;
 544    }
 545
 546    return leftBs;
 547}
 548#endif /* H264DEC_OMXDL */
 549/*------------------------------------------------------------------------------
 550
 551    Function: h264bsdFilterPicture
 552
 553        Functional description:
 554          Perform deblocking filtering for a picture. Filter does not copy
 555          the original picture anywhere but filtering is performed directly
 556          on the original image. Parameters controlling the filtering process
 557          are computed based on information in macroblock structures of the
 558          filtered macroblock, macroblock above and macroblock on the left of
 559          the filtered one.
 560
 561        Inputs:
 562          image         pointer to image to be filtered
 563          mb            pointer to macroblock data structure of the top-left
 564                        macroblock of the picture
 565
 566        Outputs:
 567          image         filtered image stored here
 568
 569        Returns:
 570          none
 571
 572------------------------------------------------------------------------------*/
 573#ifndef H264DEC_OMXDL
 574void h264bsdFilterPicture(
 575  image_t *image,
 576  mbStorage_t *mb)
 577{
 578
 579/* Variables */
 580
 581    u32 flags;
 582    u32 picSizeInMbs, mbRow, mbCol;
 583    u32 picWidthInMbs;
 584    u8 *data;
 585    mbStorage_t *pMb;
 586    bS_t bS[16];
 587    edgeThreshold_t thresholds[3];
 588
 589/* Code */
 590
 591    ASSERT(image);
 592    ASSERT(mb);
 593    ASSERT(image->data);
 594    ASSERT(image->width);
 595    ASSERT(image->height);
 596
 597    picWidthInMbs = image->width;
 598    data = image->data;
 599    picSizeInMbs = picWidthInMbs * image->height;
 600
 601    pMb = mb;
 602
 603    for (mbRow = 0, mbCol = 0; mbRow < image->height; pMb++)
 604    {
 605        flags = GetMbFilteringFlags(pMb);
 606
 607        if (flags)
 608        {
 609            /* GetBoundaryStrengths function returns non-zero value if any of
 610             * the bS values for the macroblock being processed was non-zero */
 611            if (GetBoundaryStrengths(pMb, bS, flags))
 612            {
 613                /* luma */
 614                GetLumaEdgeThresholds(thresholds, pMb, flags);
 615                data = image->data + mbRow * picWidthInMbs * 256 + mbCol * 16;
 616
 617                FilterLuma((u8*)data, bS, thresholds, picWidthInMbs*16);
 618
 619                /* chroma */
 620                GetChromaEdgeThresholds(thresholds, pMb, flags,
 621                    pMb->chromaQpIndexOffset);
 622                data = image->data + picSizeInMbs * 256 +
 623                    mbRow * picWidthInMbs * 64 + mbCol * 8;
 624
 625                FilterChroma((u8*)data, data + 64*picSizeInMbs, bS,
 626                        thresholds, picWidthInMbs*8);
 627
 628            }
 629        }
 630
 631        mbCol++;
 632        if (mbCol == picWidthInMbs)
 633        {
 634            mbCol = 0;
 635            mbRow++;
 636        }
 637    }
 638
 639}
 640
 641/*------------------------------------------------------------------------------
 642
 643    Function: FilterVerLumaEdge
 644
 645        Functional description:
 646            Filter one vertical 4-pixel luma edge.
 647
 648------------------------------------------------------------------------------*/
 649void FilterVerLumaEdge(
 650  u8 *data,
 651  u32 bS,
 652  edgeThreshold_t *thresholds,
 653  u32 imageWidth)
 654{
 655
 656/* Variables */
 657
 658    i32 delta, tc, tmp;
 659    u32 i;
 660    u8 p0, q0, p1, q1, p2, q2;
 661    u32 tmpFlag;
 662    const u8 *clp = h264bsdClip + 512;
 663
 664/* Code */
 665
 666    ASSERT(data);
 667    ASSERT(bS && bS <= 4);
 668    ASSERT(thresholds);
 669
 670    if (bS < 4)
 671    {
 672        tc = thresholds->tc0[bS-1];
 673        tmp = tc;
 674        for (i = 4; i; i--, data += imageWidth)
 675        {
 676            p1 = data[-2]; p0 = data[-1];
 677            q0 = data[0]; q1 = data[1];
 678            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 679                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 680                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 681            {
 682                p2 = data[-3];
 683                q2 = data[2];
 684
 685                if ((unsigned)ABS(p2-p0) < thresholds->beta)
 686                {
 687                    data[-2] = (u8)(p1 + CLIP3(-tc,tc,
 688                        (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 689                    tmp++;
 690                }
 691
 692                if ((unsigned)ABS(q2-q0) < thresholds->beta)
 693                {
 694                    data[1] = (u8)(q1 + CLIP3(-tc,tc,
 695                        (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 696                    tmp++;
 697                }
 698
 699                delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 700                          (p1 - q1) + 4) >> 3));
 701
 702                p0 = clp[p0 + delta];
 703                q0 = clp[q0 - delta];
 704                tmp = tc;
 705                data[-1] = p0;
 706                data[ 0] = q0;
 707            }
 708        }
 709    }
 710    else
 711    {
 712        for (i = 4; i; i--, data += imageWidth)
 713        {
 714            p1 = data[-2]; p0 = data[-1];
 715            q0 = data[0]; q1 = data[1];
 716            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 717                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 718                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 719            {
 720                tmpFlag =
 721                    ((unsigned)ABS(p0-q0) < ((thresholds->alpha >> 2) +2)) ?
 722                        HANTRO_TRUE : HANTRO_FALSE;
 723
 724                p2 = data[-3];
 725                q2 = data[2];
 726
 727                if (tmpFlag && (unsigned)ABS(p2-p0) < thresholds->beta)
 728                {
 729                    tmp = p1 + p0 + q0;
 730                    data[-1] = (u8)((p2 + 2 * tmp + q1 + 4) >> 3);
 731                    data[-2] = (u8)((p2 + tmp + 2) >> 2);
 732                    data[-3] = (u8)((2 * data[-4] + 3 * p2 + tmp + 4) >> 3);
 733                }
 734                else
 735                    data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
 736
 737                if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta)
 738                {
 739                    tmp = p0 + q0 + q1;
 740                    data[0] = (u8)((p1 + 2 * tmp + q2 + 4) >> 3);
 741                    data[1] = (u8)((tmp + q2 + 2) >> 2);
 742                    data[2] = (u8)((2 * data[3] + 3 * q2 + tmp + 4) >> 3);
 743                }
 744                else
 745                    data[0] = (u8)((2 * q1 + q0 + p1 + 2) >> 2);
 746            }
 747        }
 748    }
 749
 750}
 751
 752/*------------------------------------------------------------------------------
 753
 754    Function: FilterHorLumaEdge
 755
 756        Functional description:
 757            Filter one horizontal 4-pixel luma edge
 758
 759------------------------------------------------------------------------------*/
 760void FilterHorLumaEdge(
 761  u8 *data,
 762  u32 bS,
 763  edgeThreshold_t *thresholds,
 764  i32 imageWidth)
 765{
 766
 767/* Variables */
 768
 769    i32 delta, tc, tmp;
 770    u32 i;
 771    u8 p0, q0, p1, q1, p2, q2;
 772    const u8 *clp = h264bsdClip + 512;
 773
 774/* Code */
 775
 776    ASSERT(data);
 777    ASSERT(bS < 4);
 778    ASSERT(thresholds);
 779
 780    tc = thresholds->tc0[bS-1];
 781    tmp = tc;
 782    for (i = 4; i; i--, data++)
 783    {
 784        p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 785        q0 = data[0]; q1 = data[imageWidth];
 786        if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 787             ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 788             ((unsigned)ABS(q1-q0) < thresholds->beta) )
 789        {
 790            p2 = data[-imageWidth*3];
 791
 792            if ((unsigned)ABS(p2-p0) < thresholds->beta)
 793            {
 794                data[-imageWidth*2] = (u8)(p1 + CLIP3(-tc,tc,
 795                    (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 796                tmp++;
 797            }
 798
 799            q2 = data[imageWidth*2];
 800
 801            if ((unsigned)ABS(q2-q0) < thresholds->beta)
 802            {
 803                data[imageWidth] = (u8)(q1 + CLIP3(-tc,tc,
 804                    (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 805                tmp++;
 806            }
 807
 808            delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 809                      (p1 - q1) + 4) >> 3));
 810
 811            p0 = clp[p0 + delta];
 812            q0 = clp[q0 - delta];
 813            tmp = tc;
 814            data[-imageWidth] = p0;
 815            data[  0] = q0;
 816        }
 817    }
 818}
 819
 820/*------------------------------------------------------------------------------
 821
 822    Function: FilterHorLuma
 823
 824        Functional description:
 825            Filter all four successive horizontal 4-pixel luma edges. This can
 826            be done when bS is equal to all four edges.
 827
 828------------------------------------------------------------------------------*/
 829void FilterHorLuma(
 830  u8 *data,
 831  u32 bS,
 832  edgeThreshold_t *thresholds,
 833  i32 imageWidth)
 834{
 835
 836/* Variables */
 837
 838    i32 delta, tc, tmp;
 839    u32 i;
 840    u8 p0, q0, p1, q1, p2, q2;
 841    u32 tmpFlag;
 842    const u8 *clp = h264bsdClip + 512;
 843
 844/* Code */
 845
 846    ASSERT(data);
 847    ASSERT(bS <= 4);
 848    ASSERT(thresholds);
 849
 850    if (bS < 4)
 851    {
 852        tc = thresholds->tc0[bS-1];
 853        tmp = tc;
 854        for (i = 16; i; i--, data++)
 855        {
 856            p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 857            q0 = data[0]; q1 = data[imageWidth];
 858            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 859                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 860                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 861            {
 862                p2 = data[-imageWidth*3];
 863
 864                if ((unsigned)ABS(p2-p0) < thresholds->beta)
 865                {
 866                    data[-imageWidth*2] = (u8)(p1 + CLIP3(-tc,tc,
 867                        (p2 + ((p0 + q0 + 1) >> 1) - (p1 << 1)) >> 1));
 868                    tmp++;
 869                }
 870
 871                q2 = data[imageWidth*2];
 872
 873                if ((unsigned)ABS(q2-q0) < thresholds->beta)
 874                {
 875                    data[imageWidth] = (u8)(q1 + CLIP3(-tc,tc,
 876                        (q2 + ((p0 + q0 + 1) >> 1) - (q1 << 1)) >> 1));
 877                    tmp++;
 878                }
 879
 880                delta = CLIP3(-tmp, tmp, ((((q0 - p0) << 2) +
 881                          (p1 - q1) + 4) >> 3));
 882
 883                p0 = clp[p0 + delta];
 884                q0 = clp[q0 - delta];
 885                tmp = tc;
 886                data[-imageWidth] = p0;
 887                data[  0] = q0;
 888            }
 889        }
 890    }
 891    else
 892    {
 893        for (i = 16; i; i--, data++)
 894        {
 895            p1 = data[-imageWidth*2]; p0 = data[-imageWidth];
 896            q0 = data[0]; q1 = data[imageWidth];
 897            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 898                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 899                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
 900            {
 901                tmpFlag = ((unsigned)ABS(p0-q0) < ((thresholds->alpha >> 2) +2))
 902                            ? HANTRO_TRUE : HANTRO_FALSE;
 903
 904                p2 = data[-imageWidth*3];
 905                q2 = data[imageWidth*2];
 906
 907                if (tmpFlag && (unsigned)ABS(p2-p0) < thresholds->beta)
 908                {
 909                    tmp = p1 + p0 + q0;
 910                    data[-imageWidth] = (u8)((p2 + 2 * tmp + q1 + 4) >> 3);
 911                    data[-imageWidth*2] = (u8)((p2 + tmp + 2) >> 2);
 912                    data[-imageWidth*3] = (u8)((2 * data[-imageWidth*4] +
 913                                           3 * p2 + tmp + 4) >> 3);
 914                }
 915                else
 916                    data[-imageWidth] = (u8)((2 * p1 + p0 + q1 + 2) >> 2);
 917
 918                if (tmpFlag && (unsigned)ABS(q2-q0) < thresholds->beta)
 919                {
 920                    tmp = p0 + q0 + q1;
 921                    data[ 0] = (u8)((p1 + 2 * tmp + q2 + 4) >> 3);
 922                    data[imageWidth] = (u8)((tmp + q2 + 2) >> 2);
 923                    data[imageWidth*2] = (u8)((2 * data[imageWidth*3] +
 924                                          3 * q2 + tmp + 4) >> 3);
 925                }
 926                else
 927                    data[0] = (2 * q1 + q0 + p1 + 2) >> 2;
 928            }
 929        }
 930    }
 931
 932}
 933
 934/*------------------------------------------------------------------------------
 935
 936    Function: FilterVerChromaEdge
 937
 938        Functional description:
 939            Filter one vertical 2-pixel chroma edge
 940
 941------------------------------------------------------------------------------*/
 942void FilterVerChromaEdge(
 943  u8 *data,
 944  u32 bS,
 945  edgeThreshold_t *thresholds,
 946  u32 width)
 947{
 948
 949/* Variables */
 950
 951    i32 delta, tc;
 952    u8 p0, q0, p1, q1;
 953    const u8 *clp = h264bsdClip + 512;
 954
 955/* Code */
 956
 957    ASSERT(data);
 958    ASSERT(bS <= 4);
 959    ASSERT(thresholds);
 960
 961    p1 = data[-2]; p0 = data[-1];
 962    q0 = data[0]; q1 = data[1];
 963    if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 964         ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 965         ((unsigned)ABS(q1-q0) < thresholds->beta) )
 966    {
 967        if (bS < 4)
 968        {
 969            tc = thresholds->tc0[bS-1] + 1;
 970            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
 971                      (p1 - q1) + 4) >> 3));
 972            p0 = clp[p0 + delta];
 973            q0 = clp[q0 - delta];
 974            data[-1] = p0;
 975            data[ 0] = q0;
 976        }
 977        else
 978        {
 979            data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
 980            data[ 0] = (2 * q1 + q0 + p1 + 2) >> 2;
 981        }
 982    }
 983    data += width;
 984    p1 = data[-2]; p0 = data[-1];
 985    q0 = data[0]; q1 = data[1];
 986    if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
 987         ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
 988         ((unsigned)ABS(q1-q0) < thresholds->beta) )
 989    {
 990        if (bS < 4)
 991        {
 992            tc = thresholds->tc0[bS-1] + 1;
 993            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
 994                      (p1 - q1) + 4) >> 3));
 995            p0 = clp[p0 + delta];
 996            q0 = clp[q0 - delta];
 997            data[-1] = p0;
 998            data[ 0] = q0;
 999        }
1000        else
1001        {
1002            data[-1] = (2 * p1 + p0 + q1 + 2) >> 2;
1003            data[ 0] = (2 * q1 + q0 + p1 + 2) >> 2;
1004        }
1005    }
1006
1007}
1008
1009/*------------------------------------------------------------------------------
1010
1011    Function: FilterHorChromaEdge
1012
1013        Functional description:
1014            Filter one horizontal 2-pixel chroma edge
1015
1016------------------------------------------------------------------------------*/
1017void FilterHorChromaEdge(
1018  u8 *data,
1019  u32 bS,
1020  edgeThreshold_t *thresholds,
1021  i32 width)
1022{
1023
1024/* Variables */
1025
1026    i32 delta, tc;
1027    u32 i;
1028    u8 p0, q0, p1, q1;
1029    const u8 *clp = h264bsdClip + 512;
1030
1031/* Code */
1032
1033    ASSERT(data);
1034    ASSERT(bS < 4);
1035    ASSERT(thresholds);
1036
1037    tc = thresholds->tc0[bS-1] + 1;
1038    for (i = 2; i; i--, data++)
1039    {
1040        p1 = data[-width*2]; p0 = data[-width];
1041        q0 = data[0]; q1 = data[width];
1042        if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1043             ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1044             ((unsigned)ABS(q1-q0) < thresholds->beta) )
1045        {
1046            delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
1047                      (p1 - q1) + 4) >> 3));
1048            p0 = clp[p0 + delta];
1049            q0 = clp[q0 - delta];
1050            data[-width] = p0;
1051            data[  0] = q0;
1052        }
1053    }
1054}
1055
1056/*------------------------------------------------------------------------------
1057
1058    Function: FilterHorChroma
1059
1060        Functional description:
1061            Filter all four successive horizontal 2-pixel chroma edges. This
1062            can be done if bS is equal for all four edges.
1063
1064------------------------------------------------------------------------------*/
1065void FilterHorChroma(
1066  u8 *data,
1067  u32 bS,
1068  edgeThreshold_t *thresholds,
1069  i32 width)
1070{
1071
1072/* Variables */
1073
1074    i32 delta, tc;
1075    u32 i;
1076    u8 p0, q0, p1, q1;
1077    const u8 *clp = h264bsdClip + 512;
1078
1079/* Code */
1080
1081    ASSERT(data);
1082    ASSERT(bS <= 4);
1083    ASSERT(thresholds);
1084
1085    if (bS < 4)
1086    {
1087        tc = thresholds->tc0[bS-1] + 1;
1088        for (i = 8; i; i--, data++)
1089        {
1090            p1 = data[-width*2]; p0 = data[-width];
1091            q0 = data[0]; q1 = data[width];
1092            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1093                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1094                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
1095            {
1096                delta = CLIP3(-tc, tc, ((((q0 - p0) << 2) +
1097                          (p1 - q1) + 4) >> 3));
1098                p0 = clp[p0 + delta];
1099                q0 = clp[q0 - delta];
1100                data[-width] = p0;
1101                data[  0] = q0;
1102            }
1103        }
1104    }
1105    else
1106    {
1107        for (i = 8; i; i--, data++)
1108        {
1109            p1 = data[-width*2]; p0 = data[-width];
1110            q0 = data[0]; q1 = data[width];
1111            if ( ((unsigned)ABS(p0-q0) < thresholds->alpha) &&
1112                 ((unsigned)ABS(p1-p0) < thresholds->beta)  &&
1113                 ((unsigned)ABS(q1-q0) < thresholds->beta) )
1114            {
1115                    data[-width] = (2 * p1 + p0 + q1 + 2) >> 2;
1116                    data[  0] = (2 * q1 + q0 + p1 + 2) >> 2;
1117            }
1118        }
1119    }
1120
1121}
1122
1123
1124/*------------------------------------------------------------------------------
1125
1126    Function: GetBoundaryStrengths
1127
1128        Functional description:
1129            Function to calculate boundary strengths for all edges of a
1130            macroblock. Function returns HANTRO_TRUE if any of the bS values for
1131            the macroblock had non-zero value, HANTRO_FALSE otherwise.
1132
1133------------------------------------------------------------------------------*/
1134u32 GetBoundaryStrengths(mbStorage_t *mb, bS_t *bS, u32 flags)
1135{
1136
1137/* Variables */
1138
1139    /* this flag is set HANTRO_TRUE as soon as any boundary strength value is
1140     * non-zero */
1141    u32 nonZeroBs = HANTRO_FALSE;
1142
1143/* Code */
1144
1145    ASSERT(mb);
1146    ASSERT(bS);
1147    ASSERT(flags);
1148
1149    /* top edges */
1150    if (flags & FILTER_TOP_EDGE)
1151    {
1152        if (IS_INTRA_MB(*mb) || IS_INTRA_MB(*mb->mbB))
1153        {
1154            bS[0].top = bS[1].top = bS[2].top = bS[3].top = 4;
1155            nonZeroBs = HANTRO_TRUE;
1156        }
1157        else
1158        {
1159            bS[0].top = EdgeBoundaryStrength(mb, mb->mbB, 0, 10);
1160            bS[1].top = EdgeBoundaryStrength(mb, mb->mbB, 1, 11);
1161            bS[2].top = EdgeBoundaryStrength(mb, mb->mbB, 4, 14);
1162            bS[3].top = EdgeBoundaryStrength(mb, mb->mbB, 5, 15);
1163            if (bS[0].top || bS[1].top || bS[2].top || bS[3].top)
1164                nonZeroBs = HANTRO_TRUE;
1165        }
1166    }
1167    else
1168    {
1169        bS[0].top = bS[1].top = bS[2].top = bS[3].top = 0;
1170    }
1171
1172    /* left edges */
1173    if (flags & FILTER_LEFT_EDGE)
1174    {
1175        if (IS_INTRA_MB(*mb) || IS_INTRA_MB(*mb->mbA))
1176        {
1177            bS[0].left = bS[4].left = bS[8].left = bS[12].left = 4;
1178            nonZeroBs = HANTRO_TRUE;
1179        }
1180        else
1181        {
1182            bS[0].left = EdgeBoundaryStrength(mb, mb->mbA, 0, 5);
1183            bS[4].left = EdgeBoundaryStrength(mb, mb->mbA, 2, 7);
1184            bS[8].left = EdgeBoundaryStrength(mb, mb->mbA, 8, 13);
1185            bS[12].left = EdgeBoundaryStrength(mb, mb->mbA, 10, 15);
1186            if (!nonZeroBs &&
1187                (bS[0].left || bS[4].left || bS[8].left || bS[12].left))
1188                nonZeroBs = HANTRO_TRUE;
1189        }
1190    }
1191    else
1192    {
1193        bS[0].left = bS[4].left = bS[8].left = bS[12].left = 0;
1194    }
1195
1196    /* inner edges */
1197    if (IS_INTRA_MB(*mb))
1198    {
1199        bS[4].top  = bS[5].top  = bS[6].top  = bS[7].top  =
1200        bS[8].top  = bS[9].top  = bS[10].top = bS[11].top =
1201        bS[12].top = bS[13].top = bS[14].top = bS[15].top = 3;
1202
1203        bS[1].left  = bS[2].left  = bS[3].left  =
1204        bS[5].left  = bS[6].left  = bS[7].left  =
1205        bS[9].left  = bS[10].left = bS[11].left =
1206        bS[13].left = bS[14].left = bS[15].left = 3;
1207        nonZeroBs = HANTRO_TRUE;
1208    }
1209    else
1210    {
1211        /* 16x16 inter mb -> ref addresses or motion vectors cannot differ,
1212         * only check if either of the blocks contain coefficients */
1213        if (h264bsdNumMbPart(mb->mbType) == 1)
1214        {
1215            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1216            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1217            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1218            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1219            bS[8].top = mb->totalCoeff[8] || mb->totalCoeff[2] ? 2 : 0;
1220            bS[9].top = mb->totalCoeff[9] || mb->totalCoeff[3] ? 2 : 0;
1221            bS[10].top = mb->totalCoeff[12] || mb->totalCoeff[6] ? 2 : 0;
1222            bS[11].top = mb->totalCoeff[13] || mb->totalCoeff[7] ? 2 : 0;
1223            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1224            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1225            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1226            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1227
1228            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1229            bS[2].left = mb->totalCoeff[4] || mb->totalCoeff[1] ? 2 : 0;
1230            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1231            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1232            bS[6].left = mb->totalCoeff[6] || mb->totalCoeff[3] ? 2 : 0;
1233            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1234            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1235            bS[10].left = mb->totalCoeff[12] || mb->totalCoeff[9] ? 2 : 0;
1236            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1237            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1238            bS[14].left = mb->totalCoeff[14] || mb->totalCoeff[11] ? 2 : 0;
1239            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1240        }
1241        /* 16x8 inter mb -> ref addresses and motion vectors can be different
1242         * only for the middle horizontal edge, for the other top edges it is
1243         * enough to check whether the blocks contain coefficients or not. The
1244         * same applies to all internal left edges. */
1245        else if (mb->mbType == P_L0_L0_16x8)
1246        {
1247            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1248            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1249            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1250            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1251            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1252            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1253            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1254            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1255            bS[8].top = InnerBoundaryStrength(mb, 8, 2);
1256            bS[9].top = InnerBoundaryStrength(mb, 9, 3);
1257            bS[10].top = InnerBoundaryStrength(mb, 12, 6);
1258            bS[11].top = InnerBoundaryStrength(mb, 13, 7);
1259
1260            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1261            bS[2].left = mb->totalCoeff[4] || mb->totalCoeff[1] ? 2 : 0;
1262            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1263            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1264            bS[6].left = mb->totalCoeff[6] || mb->totalCoeff[3] ? 2 : 0;
1265            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1266            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1267            bS[10].left = mb->totalCoeff[12] || mb->totalCoeff[9] ? 2 : 0;
1268            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1269            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1270            bS[14].left = mb->totalCoeff[14] || mb->totalCoeff[11] ? 2 : 0;
1271            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1272        }
1273        /* 8x16 inter mb -> ref addresses and motion vectors can be different
1274         * only for the middle vertical edge, for the other left edges it is
1275         * enough to check whether the blocks contain coefficients or not. The
1276         * same applies to all internal top edges. */
1277        else if (mb->mbType == P_L0_L0_8x16)
1278        {
1279            bS[4].top = mb->totalCoeff[2] || mb->totalCoeff[0] ? 2 : 0;
1280            bS[5].top = mb->totalCoeff[3] || mb->totalCoeff[1] ? 2 : 0;
1281            bS[6].top = mb->totalCoeff[6] || mb->totalCoeff[4] ? 2 : 0;
1282            bS[7].top = mb->totalCoeff[7] || mb->totalCoeff[5] ? 2 : 0;
1283            bS[8].top = mb->totalCoeff[8] || mb->totalCoeff[2] ? 2 : 0;
1284            bS[9].top = mb->totalCoeff[9] || mb->totalCoeff[3] ? 2 : 0;
1285            bS[10].top = mb->totalCoeff[12] || mb->totalCoeff[6] ? 2 : 0;
1286            bS[11].top = mb->totalCoeff[13] || mb->totalCoeff[7] ? 2 : 0;
1287            bS[12].top = mb->totalCoeff[10] || mb->totalCoeff[8] ? 2 : 0;
1288            bS[13].top = mb->totalCoeff[11] || mb->totalCoeff[9] ? 2 : 0;
1289            bS[14].top = mb->totalCoeff[14] || mb->totalCoeff[12] ? 2 : 0;
1290            bS[15].top = mb->totalCoeff[15] || mb->totalCoeff[13] ? 2 : 0;
1291
1292            bS[1].left = mb->totalCoeff[1] || mb->totalCoeff[0] ? 2 : 0;
1293            bS[3].left = mb->totalCoeff[5] || mb->totalCoeff[4] ? 2 : 0;
1294            bS[5].left = mb->totalCoeff[3] || mb->totalCoeff[2] ? 2 : 0;
1295            bS[7].left = mb->totalCoeff[7] || mb->totalCoeff[6] ? 2 : 0;
1296            bS[9].left = mb->totalCoeff[9] || mb->totalCoeff[8] ? 2 : 0;
1297            bS[11].left = mb->totalCoeff[13] || mb->totalCoeff[12] ? 2 : 0;
1298            bS[13].left = mb->totalCoeff[11] || mb->totalCoeff[10] ? 2 : 0;
1299            bS[15].left = mb->totalCoeff[15] || mb->totalCoeff[14] ? 2 : 0;
1300            bS[2].left = InnerBoundaryStrength(mb, 4, 1);
1301            bS[6].left = InnerBoundaryStrength(mb, 6, 3);
1302            bS[10].left = InnerBoundaryStrength(mb, 12, 9);
1303            bS[14].left = InnerBoundaryStrength(mb, 14, 11);
1304        }
1305        else
1306        {
1307            bS[4].top =
1308                InnerBoundaryStrength(mb, mb4x4Index[4], mb4x4Index[0]);
1309            bS[5].top =
1310                InnerBoundaryStrength(mb, mb4x4Index[5], mb4x4Index[1]);
1311            bS[6].top =
1312                InnerBoundaryStrength(mb, mb4x4Index[6], mb4x4Index[2]);
1313            bS[7].top =
1314                InnerBoundaryStrength(mb, mb4x4Index[7], mb4x4Index[3]);
1315            bS[8].top =
1316                InnerBoundaryStrength(mb, mb4x4Index[8], mb4x4Index[4]);
1317            bS[9].top =
1318                InnerBoundaryStrength(mb, mb4x4Index[9], mb4x4Index[5]);
1319            bS[10].top =
1320                InnerBoundaryStrength(mb, mb4x4Index[10], mb4x4Index[6]);
1321            bS[11].top =
1322                InnerBoundaryStrength(mb, mb4x4Index[11], mb4x4Index[7]);
1323            bS[12].top =
1324                InnerBoundaryStrength(mb, mb4x4Index[12], mb4x4Index[8]);
1325            bS[13].top =
1326                InnerBoundaryStrength(mb, mb4x4Index[13], mb4x4Index[9]);
1327            bS[14].top =
1328                InnerBoundaryStrength(mb, mb4x4Index[14], mb4x4Index[10]);
1329            bS[15].top =
1330                InnerBoundaryStrength(mb, mb4x4Index[15], mb4x4Index[11]);
1331
1332            bS[1].left =
1333                InnerBoundaryStrength(mb, mb4x4Index[1], mb4x4Index[0]);
1334            bS[2].left =
1335                InnerBoundaryStrength(mb, mb4x4Index[2], mb4x4Index[1]);
1336            bS[3].left =
1337                InnerBoundaryStrength(mb, mb4x4Index[3], mb4x4Index[2]);
1338            bS[5].left =
1339                InnerBoundaryStrength(mb, mb4x4Index[5], mb4x4Index[4]);
1340            bS[6].left =
1341                InnerBoundaryStrength(mb, mb4x4Index[6], mb4x4Index[5]);
1342            bS[7].left =
1343                InnerBoundaryStrength(mb, mb4x4Index[7], mb4x4Index[6]);
1344            bS[9].left =
1345                InnerBoundaryStrength(mb, mb4x4Index[9], mb4x4Index[8]);
1346            bS[10].left =
1347                InnerBoundaryStrength(mb, mb4x4Index[10], mb4x4Index[9]);
1348            bS[11].left =
1349                InnerBoundaryStrength(mb, mb4x4Index[11], mb4x4Index[10]);
1350            bS[13].left =
1351                InnerBoundaryStrength(mb, mb4x4Index[13], mb4x4Index[12]);
1352            bS[14].left =
1353                InnerBoundaryStrength(mb, mb4x4Index[14], mb4x4Index[13]);
1354            bS[15].left =
1355                InnerBoundaryStrength(mb, mb4x4Index[15], mb4x4Index[14]);
1356        }
1357        if (!nonZeroBs &&
1358            (bS[4].top || bS[5].top || bS[6].top || bS[7].top ||
1359             bS[8].top || bS[9].top || bS[10].top || bS[11].top ||
1360             bS[12].top || bS[13].top || bS[14].top || bS[15].top ||
1361             bS[1].left || bS[2].left || bS[3].left ||
1362             bS[5].left || bS[6].left || bS[7].left ||
1363             bS[9].left || bS[10].left || bS[11].left ||
1364             bS[13].left || bS[14].left || bS[15].left))
1365            nonZeroBs = HANTRO_TRUE;
1366    }
1367
1368    return(nonZeroBs);
1369
1370}
1371
1372/*------------------------------------------------------------------------------
1373
1374    Function: GetLumaEdgeThresholds
1375
1376        Functional description:
1377            Compute alpha, beta and tc0 thresholds for inner, left and top
1378            luma edges of a macroblock.
1379
1380------------------------------------------------------------------------------*/
1381void GetLumaEdgeThresholds(
1382  edgeThreshold_t *thresholds,
1383  mbStorage_t *mb,
1384  u32 filteringFlags)
1385{
1386
1387/* Variables */
1388
1389    u32 indexA, indexB;
1390    u32 qpAv, qp, qpTmp;
1391
1392/* Code */
1393
1394    ASSERT(thresholds);
1395    ASSERT(mb);
1396
1397    qp = mb->qpY;
1398
1399    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
1400    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
1401
1402    thresholds[INNER].alpha = alphas[indexA];
1403    thresholds[INNER].beta = betas[indexB];
1404    thresholds[INNER].tc0 = tc0[indexA];
1405
1406    if (filteringFlags & FILTER_TOP_EDGE)
1407    {
1408        qpTmp = mb->mbB->qpY;
1409        if (qpTmp != qp)
1410        {
1411            qpAv = (qp + qpTmp + 1) >> 1;
1412
1413            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1414            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1415
1416            thresholds[TOP].alpha = alphas[indexA];
1417            thresholds[TOP].beta = betas[indexB];
1418            thresholds[TOP].tc0 = tc0[indexA];
1419        }
1420        else
1421        {
1422            thresholds[TOP].alpha = thresholds[INNER].alpha;
1423            thresholds[TOP].beta = thresholds[INNER].beta;
1424            thresholds[TOP].tc0 = thresholds[INNER].tc0;
1425        }
1426    }
1427    if (filteringFlags & FILTER_LEFT_EDGE)
1428    {
1429        qpTmp = mb->mbA->qpY;
1430        if (qpTmp != qp)
1431        {
1432            qpAv = (qp + qpTmp + 1) >> 1;
1433
1434            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1435            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1436
1437            thresholds[LEFT].alpha = alphas[indexA];
1438            thresholds[LEFT].beta = betas[indexB];
1439            thresholds[LEFT].tc0 = tc0[indexA];
1440        }
1441        else
1442        {
1443            thresholds[LEFT].alpha = thresholds[INNER].alpha;
1444            thresholds[LEFT].beta = thresholds[INNER].beta;
1445            thresholds[LEFT].tc0 = thresholds[INNER].tc0;
1446        }
1447    }
1448
1449}
1450
1451/*------------------------------------------------------------------------------
1452
1453    Function: GetChromaEdgeThresholds
1454
1455        Functional description:
1456            Compute alpha, beta and tc0 thresholds for inner, left and top
1457            chroma edges of a macroblock.
1458
1459------------------------------------------------------------------------------*/
1460void GetChromaEdgeThresholds(
1461  edgeThreshold_t *thresholds,
1462  mbStorage_t *mb,
1463  u32 filteringFlags,
1464  i32 chromaQpIndexOffset)
1465{
1466
1467/* Variables */
1468
1469    u32 indexA, indexB;
1470    u32 qpAv, qp, qpTmp;
1471
1472/* Code */
1473
1474    ASSERT(thresholds);
1475    ASSERT(mb);
1476
1477    qp = mb->qpY;
1478    qp = h264bsdQpC[CLIP3(0, 51, (i32)qp + chromaQpIndexOffset)];
1479
1480    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
1481    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
1482
1483    thresholds[INNER].alpha = alphas[indexA];
1484    thresholds[INNER].beta = betas[indexB];
1485    thresholds[INNER].tc0 = tc0[indexA];
1486
1487    if (filteringFlags & FILTER_TOP_EDGE)
1488    {
1489        qpTmp = mb->mbB->qpY;
1490        if (qpTmp != mb->qpY)
1491        {
1492            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
1493            qpAv = (qp + qpTmp + 1) >> 1;
1494
1495            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1496            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1497
1498            thresholds[TOP].alpha = alphas[indexA];
1499            thresholds[TOP].beta = betas[indexB];
1500            thresholds[TOP].tc0 = tc0[indexA];
1501        }
1502        else
1503        {
1504            thresholds[TOP].alpha = thresholds[INNER].alpha;
1505            thresholds[TOP].beta = thresholds[INNER].beta;
1506            thresholds[TOP].tc0 = thresholds[INNER].tc0;
1507        }
1508    }
1509    if (filteringFlags & FILTER_LEFT_EDGE)
1510    {
1511        qpTmp = mb->mbA->qpY;
1512        if (qpTmp != mb->qpY)
1513        {
1514            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
1515            qpAv = (qp + qpTmp + 1) >> 1;
1516
1517            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
1518            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
1519
1520            thresholds[LEFT].alpha = alphas[indexA];
1521            thresholds[LEFT].beta = betas[indexB];
1522            thresholds[LEFT].tc0 = tc0[indexA];
1523        }
1524        else
1525        {
1526            thresholds[LEFT].alpha = thresholds[INNER].alpha;
1527            thresholds[LEFT].beta = thresholds[INNER].beta;
1528            thresholds[LEFT].tc0 = thresholds[INNER].tc0;
1529        }
1530    }
1531
1532}
1533
1534/*------------------------------------------------------------------------------
1535
1536    Function: FilterLuma
1537
1538        Functional description:
1539            Function to filter all luma edges of a macroblock
1540
1541------------------------------------------------------------------------------*/
1542void FilterLuma(
1543  u8 *data,
1544  bS_t *bS,
1545  edgeThreshold_t *thresholds,
1546  u32 width)
1547{
1548
1549/* Variables */
1550
1551    u32 vblock;
1552    bS_t *tmp;
1553    u8 *ptr;
1554    u32 offset;
1555
1556/* Code */
1557
1558    ASSERT(data);
1559    ASSERT(bS);
1560    ASSERT(thresholds);
1561
1562    ptr = data;
1563    tmp = bS;
1564
1565    offset  = TOP;
1566
1567    /* loop block rows, perform filtering for all vertical edges of the block
1568     * row first, then filter each horizontal edge of the row */
1569    for (vblock = 4; vblock--;)
1570    {
1571        /* only perform filtering if bS is non-zero, first of the four
1572         * FilterVerLumaEdge handles the left edge of the macroblock, others
1573         * filter inner edges */
1574        if (tmp[0].left)
1575            FilterVerLumaEdge(ptr, tmp[0].left, thresholds + LEFT, width);
1576        if (tmp[1].left)
1577            FilterVerLumaEdge(ptr+4, tmp[1].left, thresholds + INNER, width);
1578        if (tmp[2].left)
1579            FilterVerLumaEdge(ptr+8, tmp[2].left, thresholds + INNER, width);
1580        if (tmp[3].left)
1581            FilterVerLumaEdge(ptr+12, tmp[3].left, thresholds + INNER, width);
1582
1583        /* if bS is equal for all horizontal edges of the row -> perform
1584         * filtering with FilterHorLuma, otherwise use FilterHorLumaEdge for
1585         * each edge separately. offset variable indicates top macroblock edge
1586         * on the first loop round, inner edge for the other rounds */
1587        if (tmp[0].top == tmp[1].top && tmp[1].top == tmp[2].top &&
1588            tmp[2].top == tmp[3].top)
1589        {
1590            if(tmp[0].top)
1591                FilterHorLuma(ptr, tmp[0].top, thresholds + offset, (i32)width);
1592        }
1593        else
1594        {
1595            if(tmp[0].top)
1596                FilterHorLumaEdge(ptr, tmp[0].top, thresholds+offset,
1597                    (i32)width);
1598            if(tmp[1].top)
1599                FilterHorLumaEdge(ptr+4, tmp[1].top, thresholds+offset,
1600                    (i32)width);
1601            if(tmp[2].top)
1602                FilterHorLumaEdge(ptr+8, tmp[2].top, thresholds+offset,
1603                    (i32)width);
1604            if(tmp[3].top)
1605                FilterHorLumaEdge(ptr+12, tmp[3].top, thresholds+offset,
1606                    (i32)width);
1607        }
1608
1609        /* four pixel rows ahead, i.e. next row of 4x4-blocks */
1610        ptr += width*4;
1611        tmp += 4;
1612        offset = INNER;
1613    }
1614}
1615
1616/*------------------------------------------------------------------------------
1617
1618    Function: FilterChroma
1619
1620        Functional description:
1621            Function to filter all chroma edges of a macroblock
1622
1623------------------------------------------------------------------------------*/
1624void FilterChroma(
1625  u8 *dataCb,
1626  u8 *dataCr,
1627  bS_t *bS,
1628  edgeThreshold_t *thresholds,
1629  u32 width)
1630{
1631
1632/* Variables */
1633
1634    u32 vblock;
1635    bS_t *tmp;
1636    u32 offset;
1637
1638/* Code */
1639
1640    ASSERT(dataCb);
1641    ASSERT(dataCr);
1642    ASSERT(bS);
1643    ASSERT(thresholds);
1644
1645    tmp = bS;
1646    offset = TOP;
1647
1648    /* loop block rows, perform filtering for all vertical edges of the block
1649     * row first, then filter each horizontal edge of the row */
1650    for (vblock = 0; vblock < 2; vblock++)
1651    {
1652        /* only perform filtering if bS is non-zero, first two of the four
1653         * FilterVerChromaEdge calls handle the left edge of the macroblock,
1654         * others filter the inner edge. Note that as chroma uses bS values
1655         * determined for luma edges, each bS is used only for 2 pixels of
1656         * a 4-pixel edge */
1657        if (tmp[0].left)
1658        {
1659            FilterVerChromaEdge(dataCb, tmp[0].left, thresholds + LEFT, width);
1660            FilterVerChromaEdge(dataCr, tmp[0].left, thresholds + LEFT, width);
1661        }
1662        if (tmp[4].left)
1663        {
1664            FilterVerChromaEdge(dataCb+2*width, tmp[4].left, thresholds + LEFT,
1665                width);
1666            FilterVerChromaEdge(dataCr+2*width, tmp[4].left, thresholds + LEFT,
1667                width);
1668        }
1669        if (tmp[2].left)
1670        {
1671            FilterVerChromaEdge(dataCb+4, tmp[2].left, thresholds + INNER,
1672                width);
1673            FilterVerChromaEdge(dataCr+4, tmp[2].left, thresholds + INNER,
1674                width);
1675        }
1676        if (tmp[6].left)
1677        {
1678            FilterVerChromaEdge(dataCb+2*width+4, tmp[6].left,
1679                thresholds + INNER, width);
1680            FilterVerChromaEdge(dataCr+2*width+4, tmp[6].left,
1681                thresholds + INNER, width);
1682        }
1683
1684        /* if bS is equal for all horizontal edges of the row -> perform
1685         * filtering with FilterHorChroma, otherwise use FilterHorChromaEdge
1686         * for each edge separately. offset variable indicates top macroblock
1687         * edge on the first loop round, inner edge for the second */
1688        if (tmp[0].top == tmp[1].top && tmp[1].top == tmp[2].top &&
1689            tmp[2].top == tmp[3].top)
1690        {
1691            if(tmp[0].top)
1692            {
1693                FilterHorChroma(dataCb, tmp[0].top, thresholds+offset,
1694                    (i32)width);
1695                FilterHorChroma(dataCr, tmp[0].top, thresholds+offset,
1696                    (i32)width);
1697            }
1698        }
1699        else
1700        {
1701            if (tmp[0].top)
1702            {
1703                FilterHorChromaEdge(dataCb, tmp[0].top, thresholds+offset,
1704                    (i32)width);
1705                FilterHorChromaEdge(dataCr, tmp[0].top, thresholds+offset,
1706                    (i32)width);
1707            }
1708            if (tmp[1].top)
1709            {
1710                FilterHorChromaEdge(dataCb+2, tmp[1].top, thresholds+offset,
1711                    (i32)width);
1712                FilterHorChromaEdge(dataCr+2, tmp[1].top, thresholds+offset,
1713                    (i32)width);
1714            }
1715            if (tmp[2].top)
1716            {
1717                FilterHorChromaEdge(dataCb+4, tmp[2].top, thresholds+offset,
1718                    (i32)width);
1719                FilterHorChromaEdge(dataCr+4, tmp[2].top, thresholds+offset,
1720                    (i32)width);
1721            }
1722            if (tmp[3].top)
1723            {
1724                FilterHorChromaEdge(dataCb+6, tmp[3].top, thresholds+offset,
1725                    (i32)width);
1726                FilterHorChromaEdge(dataCr+6, tmp[3].top, thresholds+offset,
1727                    (i32)width);
1728            }
1729        }
1730
1731        tmp += 8;
1732        dataCb += width*4;
1733        dataCr += width*4;
1734        offset = INNER;
1735    }
1736}
1737
1738#else /* H264DEC_OMXDL */
1739
1740/*------------------------------------------------------------------------------
1741
1742    Function: h264bsdFilterPicture
1743
1744        Functional description:
1745          Perform deblocking filtering for a picture. Filter does not copy
1746          the original picture anywhere but filtering is performed directly
1747          on the original image. Parameters controlling the filtering process
1748          are computed based on information in macroblock structures of the
1749          filtered macroblock, macroblock above and macroblock on the left of
1750          the filtered one.
1751
1752        Inputs:
1753          image         pointer to image to be filtered
1754          mb            pointer to macroblock data structure of the top-left
1755                        macroblock of the picture
1756
1757        Outputs:
1758          image         filtered image stored here
1759
1760        Returns:
1761          none
1762
1763------------------------------------------------------------------------------*/
1764
1765/*lint --e{550} Symbol not accessed */
1766void h264bsdFilterPicture(
1767  image_t *image,
1768  mbStorage_t *mb)
1769{
1770
1771/* Variables */
1772
1773    u32 flags;
1774    u32 picSizeInMbs, mbRow, mbCol;
1775    u32 picWidthInMbs;
1776    u8 *data;
1777    mbStorage_t *pMb;
1778    u8 bS[2][16];
1779    u8 thresholdLuma[2][16];
1780    u8 thresholdChroma[2][8];
1781    u8 alpha[2][2];
1782    u8 beta[2][2];
1783    OMXResult res;
1784
1785/* Code */
1786
1787    ASSERT(image);
1788    ASSERT(mb);
1789    ASSERT(image->data);
1790    ASSERT(image->width);
1791    ASSERT(image->height);
1792
1793    picWidthInMbs = image->width;
1794    data = image->data;
1795    picSizeInMbs = picWidthInMbs * image->height;
1796
1797    pMb = mb;
1798
1799    for (mbRow = 0, mbCol = 0; mbRow < image->height; pMb++)
1800    {
1801        flags = GetMbFilteringFlags(pMb);
1802
1803        if (flags)
1804        {
1805            /* GetBoundaryStrengths function returns non-zero value if any of
1806             * the bS values for the macroblock being processed was non-zero */
1807            if (GetBoundaryStrengths(pMb, bS, flags))
1808            {
1809
1810                /* Luma */
1811                GetLumaEdgeThresholds(pMb,alpha,beta,thresholdLuma,bS,flags);
1812                data = image->data + mbRow * picWidthInMbs * 256 + mbCol * 16;
1813
1814                res = omxVCM4P10_FilterDeblockingLuma_VerEdge_I( data,
1815                                                (OMX_S32)(picWidthInMbs*16),
1816                                                (const OMX_U8*)alpha,
1817                                                (const OMX_U8*)beta,
1818                                                (const OMX_U8*)thresholdLuma,
1819                                                (const OMX_U8*)bS );
1820
1821                res = omxVCM4P10_FilterDeblockingLuma_HorEdge_I( data,
1822                                                (OMX_S32)(picWidthInMbs*16),
1823                                                (const OMX_U8*)alpha+2,
1824                                                (const OMX_U8*)beta+2,
1825                                                (const OMX_U8*)thresholdLuma+16,
1826                                                (const OMX_U8*)bS+16 );
1827                /* Cb */
1828                GetChromaEdgeThresholds(pMb, alpha, beta, thresholdChroma,
1829                                        bS, flags, pMb->chromaQpIndexOffset);
1830                data = image->data + picSizeInMbs * 256 +
1831                    mbRow * picWidthInMbs * 64 + mbCol * 8;
1832
1833                res = omxVCM4P10_FilterDeblockingChroma_VerEdge_I( data,
1834                                              (OMX_S32)(picWidthInMbs*8),
1835                                              (const OMX_U8*)alpha,
1836                                              (const OMX_U8*)beta,
1837                                              (const OMX_U8*)thresholdChroma,
1838                                              (const OMX_U8*)bS );
1839                res = omxVCM4P10_FilterDeblockingChroma_HorEdge_I( data,
1840                                              (OMX_S32)(picWidthInMbs*8),
1841                                              (const OMX_U8*)alpha+2,
1842                                              (const OMX_U8*)beta+2,
1843                                              (const OMX_U8*)thresholdChroma+8,
1844                                              (const OMX_U8*)bS+16 );
1845                /* Cr */
1846                data += (picSizeInMbs * 64);
1847                res = omxVCM4P10_FilterDeblockingChroma_VerEdge_I( data,
1848                                              (OMX_S32)(picWidthInMbs*8),
1849                                              (const OMX_U8*)alpha,
1850                                              (const OMX_U8*)beta,
1851                                              (const OMX_U8*)thresholdChroma,
1852                                              (const OMX_U8*)bS );
1853                res = omxVCM4P10_FilterDeblockingChroma_HorEdge_I( data,
1854                                              (OMX_S32)(picWidthInMbs*8),
1855                                              (const OMX_U8*)alpha+2,
1856                                              (const OMX_U8*)beta+2,
1857                                              (const OMX_U8*)thresholdChroma+8,
1858                                              (const OMX_U8*)bS+16 );
1859            }
1860        }
1861
1862        mbCol++;
1863        if (mbCol == picWidthInMbs)
1864        {
1865            mbCol = 0;
1866            mbRow++;
1867        }
1868    }
1869
1870}
1871
1872/*------------------------------------------------------------------------------
1873
1874    Function: GetBoundaryStrengths
1875
1876        Functional description:
1877            Function to calculate boundary strengths for all edges of a
1878            macroblock. Function returns HANTRO_TRUE if any of the bS values for
1879            the macroblock had non-zero value, HANTRO_FALSE otherwise.
1880
1881------------------------------------------------------------------------------*/
1882u32 GetBoundaryStrengths(mbStorage_t *mb, u8 (*bS)[16], u32 flags)
1883{
1884
1885/* Variables */
1886
1887    /* this flag is set HANTRO_TRUE as soon as any boundary strength value is
1888     * non-zero */
1889    u32 nonZeroBs = HANTRO_FALSE;
1890    u32 *pTmp;
1891    u32 tmp1, tmp2, isIntraMb;
1892
1893/* Code */
1894
1895    ASSERT(mb);
1896    ASSERT(bS);
1897    ASSERT(flags);
1898
1899    isIntraMb = IS_INTRA_MB(*mb);
1900
1901    /* top edges */
1902    pTmp = (u32*)&bS[1][0];
1903    if (flags & FILTER_TOP_EDGE)
1904    {
1905        if (isIntraMb || IS_INTRA_MB(*mb->mbB))
1906        {
1907            *pTmp = 0x04040404;
1908            nonZeroBs = HANTRO_TRUE;
1909        }
1910        else
1911        {
1912            *pTmp = EdgeBoundaryStrengthTop(mb, mb->mbB);
1913            if (*pTmp)
1914                nonZeroBs = HANTRO_TRUE;
1915        }
1916    }
1917    else
1918    {
1919        *pTmp = 0;
1920    }
1921
1922    /* left edges */
1923    pTmp = (u32*)&bS[0][0];
1924    if (flags & FILTER_LEFT_EDGE)
1925    {
1926        if (isIntraMb || IS_INTRA_MB(*mb->mbA))
1927        {
1928            /*bS[0][0] = bS[0][1] = bS[0][2] = bS[0][3] = 4;*/
1929            *pTmp = 0x04040404;
1930            nonZeroBs = HANTRO_TRUE;
1931        }
1932        else
1933        {
1934            *pTmp = EdgeBoundaryStrengthLeft(mb, mb->mbA);
1935            if (!nonZeroBs && *pTmp)
1936                nonZeroBs = HANTRO_TRUE;
1937        }
1938    }
1939    else
1940    {
1941        *pTmp = 0;
1942    }
1943
1944    /* inner edges */
1945    if (isIntraMb)
1946    {
1947        pTmp++;
1948        *pTmp++ = 0x03030303;
1949        *pTmp++ = 0x03030303;
1950        *pTmp++ = 0x03030303;
1951        pTmp++;
1952        *pTmp++ = 0x03030303;
1953        *pTmp++ = 0x03030303;
1954        *pTmp = 0x03030303;
1955
1956        nonZeroBs = HANTRO_TRUE;
1957    }
1958    else
1959    {
1960        pTmp = (u32*)mb->totalCoeff;
1961
1962        /* 16x16 inter mb -> ref addresses or motion vectors cannot differ,
1963         * only check if either of the blocks contain coefficients */
1964        if (h264bsdNumMbPart(mb->mbType) == 1)
1965        {
1966            tmp1 = *pTmp++;
1967            tmp2 = *pTmp++;
1968            bS[1][4]  = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [2]  || [0] */
1969            bS[1][5]  = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [3]  || [1] */
1970            bS[0][4]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [1]  || [0] */
1971            bS[0][5]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [3]  || [2] */
1972
1973            tmp1 = *pTmp++;
1974            bS[1][6]  = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [6]  || [4] */
1975            bS[1][7]  = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [7]  || [5] */
1976            bS[0][12] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [5]  || [4] */
1977            bS[0][13] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [7]  || [6] */
1978            tmp2 = *pTmp;
1979            bS[1][12] = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [10] || [8] */
1980            bS[1][13] = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [11] || [9] */
1981            bS[0][6]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [9]  || [8] */
1982            bS[0][7]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [11] || [10] */
1983
1984            bS[1][14] = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [14] || [12] */
1985            bS[1][15] = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [15] || [13] */
1986            bS[0][14] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [13] || [12] */
1987            bS[0][15] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [15] || [14] */
1988
1989            {
1990            u32 tmp3, tmp4;
1991
1992            tmp1 = mb->totalCoeff[8];
1993            tmp2 = mb->totalCoeff[2];
1994            tmp3 = mb->totalCoeff[9];
1995            tmp4 = mb->totalCoeff[3];
1996
1997            bS[1][8] = tmp1 || tmp2 ? 2 : 0;
1998            tmp1 = mb->totalCoeff[12];
1999            tmp2 = mb->totalCoeff[6];
2000            bS[1][9] = tmp3 || tmp4 ? 2 : 0;
2001            tmp3 = mb->totalCoeff[13];
2002            tmp4 = mb->totalCoeff[7];
2003            bS[1][10] = tmp1 || tmp2 ? 2 : 0;
2004            tmp1 = mb->totalCoeff[4];
2005            tmp2 = mb->totalCoeff[1];
2006            bS[1][11] = tmp3 || tmp4 ? 2 : 0;
2007            tmp3 = mb->totalCoeff[6];
2008            tmp4 = mb->totalCoeff[3];
2009            bS[0][8] = tmp1 || tmp2 ? 2 : 0;
2010            tmp1 = mb->totalCoeff[12];
2011            tmp2 = mb->totalCoeff[9];
2012            bS[0][9] = tmp3 || tmp4 ? 2 : 0;
2013            tmp3 = mb->totalCoeff[14];
2014            tmp4 = mb->totalCoeff[11];
2015            bS[0][10] = tmp1 || tmp2 ? 2 : 0;
2016            bS[0][11] = tmp3 || tmp4 ? 2 : 0;
2017            }
2018        }
2019
2020        /* 16x8 inter mb -> ref addresses and motion vectors can be different
2021         * only for the middle horizontal edge, for the other top edges it is
2022         * enough to check whether the blocks contain coefficients or not. The
2023         * same applies to all internal left edges. */
2024        else if (mb->mbType == P_L0_L0_16x8)
2025        {
2026            tmp1 = *pTmp++;
2027            tmp2 = *pTmp++;
2028            bS[1][4]  = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [2]  || [0] */
2029            bS[1][5]  = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [3]  || [1] */
2030            bS[0][4]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [1]  || [0] */
2031            bS[0][5]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [3]  || [2] */
2032            tmp1 = *pTmp++;
2033            bS[1][6]  = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [6]  || [4] */
2034            bS[1][7]  = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [7]  || [5] */
2035            bS[0][12] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [5]  || [4] */
2036            bS[0][13] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [7]  || [6] */
2037            tmp2 = *pTmp;
2038            bS[1][12] = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [10] || [8] */
2039            bS[1][13] = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [11] || [9] */
2040            bS[0][6]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [9]  || [8] */
2041            bS[0][7]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [11] || [10] */
2042
2043            bS[1][14] = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [14] || [12] */
2044            bS[1][15] = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [15] || [13] */
2045            bS[0][14] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [13] || [12] */
2046            bS[0][15] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [15] || [14] */
2047
2048            bS[1][8] = (u8)InnerBoundaryStrength(mb, 8, 2);
2049            bS[1][9] = (u8)InnerBoundaryStrength(mb, 9, 3);
2050            bS[1][10] = (u8)InnerBoundaryStrength(mb, 12, 6);
2051            bS[1][11] = (u8)InnerBoundaryStrength(mb, 13, 7);
2052
2053            {
2054            u32 tmp3, tmp4;
2055
2056            tmp1 = mb->totalCoeff[4];
2057            tmp2 = mb->totalCoeff[1];
2058            tmp3 = mb->totalCoeff[6];
2059            tmp4 = mb->totalCoeff[3];
2060            bS[0][8] = tmp1 || tmp2 ? 2 : 0;
2061            tmp1 = mb->totalCoeff[12];
2062            tmp2 = mb->totalCoeff[9];
2063            bS[0][9] = tmp3 || tmp4 ? 2 : 0;
2064            tmp3 = mb->totalCoeff[14];
2065            tmp4 = mb->totalCoeff[11];
2066            bS[0][10] = tmp1 || tmp2 ? 2 : 0;
2067            bS[0][11] = tmp3 || tmp4 ? 2 : 0;
2068            }
2069        }
2070        /* 8x16 inter mb -> ref addresses and motion vectors can be different
2071         * only for the middle vertical edge, for the other left edges it is
2072         * enough to check whether the blocks contain coefficients or not. The
2073         * same applies to all internal top edges. */
2074        else if (mb->mbType == P_L0_L0_8x16)
2075        {
2076            tmp1 = *pTmp++;
2077            tmp2 = *pTmp++;
2078            bS[1][4]  = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [2]  || [0] */
2079            bS[1][5]  = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [3]  || [1] */
2080            bS[0][4]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [1]  || [0] */
2081            bS[0][5]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [3]  || [2] */
2082            tmp1 = *pTmp++;
2083            bS[1][6]  = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [6]  || [4] */
2084            bS[1][7]  = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [7]  || [5] */
2085            bS[0][12] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [5]  || [4] */
2086            bS[0][13] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [7]  || [6] */
2087            tmp2 = *pTmp;
2088            bS[1][12] = (tmp1 & 0x00FF00FF) ? 2 : 0; /* [10] || [8] */
2089            bS[1][13] = (tmp1 & 0xFF00FF00) ? 2 : 0; /* [11] || [9] */
2090            bS[0][6]  = (tmp1 & 0x0000FFFF) ? 2 : 0; /* [9]  || [8] */
2091            bS[0][7]  = (tmp1 & 0xFFFF0000) ? 2 : 0; /* [11] || [10] */
2092
2093            bS[1][14] = (tmp2 & 0x00FF00FF) ? 2 : 0; /* [14] || [12] */
2094            bS[1][15] = (tmp2 & 0xFF00FF00) ? 2 : 0; /* [15] || [13] */
2095            bS[0][14] = (tmp2 & 0x0000FFFF) ? 2 : 0; /* [13] || [12] */
2096            bS[0][15] = (tmp2 & 0xFFFF0000) ? 2 : 0; /* [15] || [14] */
2097
2098            bS[0][8] = (u8)InnerBoundaryStrength(mb, 4, 1);
2099            bS[0][9] = (u8)InnerBoundaryStrength(mb, 6, 3);
2100            bS[0][10] = (u8)InnerBoundaryStrength(mb, 12, 9);
2101            bS[0][11] = (u8)InnerBoundaryStrength(mb, 14, 11);
2102
2103            {
2104            u32 tmp3, tmp4;
2105
2106            tmp1 = mb->totalCoeff[8];
2107            tmp2 = mb->totalCoeff[2];
2108            tmp3 = mb->totalCoeff[9];
2109            tmp4 = mb->totalCoeff[3];
2110            bS[1][8] = tmp1 || tmp2 ? 2 : 0;
2111            tmp1 = mb->totalCoeff[12];
2112            tmp2 = mb->totalCoeff[6];
2113            bS[1][9] = tmp3 || tmp4 ? 2 : 0;
2114            tmp3 = mb->totalCoeff[13];
2115            tmp4 = mb->totalCoeff[7];
2116            bS[1][10] = tmp1 || tmp2 ? 2 : 0;
2117            bS[1][11] = tmp3 || tmp4 ? 2 : 0;
2118            }
2119        }
2120        else
2121        {
2122            tmp1 = *pTmp++;
2123            bS[1][4] = (tmp1 & 0x00FF00FF) ? 2 : (u8)InnerBoundaryStrength2(mb, 2, 0);
2124            bS[1][5] = (tmp1 & 0xFF00FF00) ? 2 : (u8)InnerBoundaryStrength2(mb, 3, 1);
2125            bS[0][4] = (tmp1 & 0x0000FFFF) ? 2 : (u8)InnerBoundaryStrength2(mb, 1, 0);
2126            bS[0][5] = (tmp1 & 0xFFFF0000) ? 2 : (u8)InnerBoundaryStrength2(mb, 3, 2);
2127            tmp1 = *pTmp++;
2128            bS[1][6]  = (tmp1 & 0x00FF00FF) ? 2 : (u8)InnerBoundaryStrength2(mb, 6, 4);
2129            bS[1][7]  = (tmp1 & 0xFF00FF00) ? 2 : (u8)InnerBoundaryStrength2(mb, 7, 5);
2130            bS[0][12] = (tmp1 & 0x0000FFFF) ? 2 : (u8)InnerBoundaryStrength2(mb, 5, 4);
2131            bS[0][13] = (tmp1 & 0xFFFF0000) ? 2 : (u8)InnerBoundaryStrength2(mb, 7, 6);
2132            tmp1 = *pTmp++;
2133            bS[1][12] = (tmp1 & 0x00FF00FF) ? 2 : (u8)InnerBoundaryStrength2(mb, 10, 8);
2134            bS[1][13] = (tmp1 & 0xFF00FF00) ? 2 : (u8)InnerBoundaryStrength2(mb, 11, 9);
2135            bS[0][6]  = (tmp1 & 0x0000FFFF) ? 2 : (u8)InnerBoundaryStrength2(mb, 9, 8);
2136            bS[0][7]  = (tmp1 & 0xFFFF0000) ? 2 : (u8)InnerBoundaryStrength2(mb, 11, 10);
2137            tmp1 = *pTmp;
2138            bS[1][14] = (tmp1 & 0x00FF00FF) ? 2 : (u8)InnerBoundaryStrength2(mb, 14, 12);
2139            bS[1][15] = (tmp1 & 0xFF00FF00) ? 2 : (u8)InnerBoundaryStrength2(mb, 15, 13);
2140            bS[0][14] = (tmp1 & 0x0000FFFF) ? 2 : (u8)InnerBoundaryStrength2(mb, 13, 12);
2141            bS[0][15] = (tmp1 & 0xFFFF0000) ? 2 : (u8)InnerBoundaryStrength2(mb, 15, 14);
2142
2143            bS[1][8] = (u8)InnerBoundaryStrength(mb, 8, 2);
2144            bS[1][9] = (u8)InnerBoundaryStrength(mb, 9, 3);
2145            bS[1][10] = (u8)InnerBoundaryStrength(mb, 12, 6);
2146            bS[1][11] = (u8)InnerBoundaryStrength(mb, 13, 7);
2147
2148            bS[0][8] = (u8)InnerBoundaryStrength(mb, 4, 1);
2149            bS[0][9] = (u8)InnerBoundaryStrength(mb, 6, 3);
2150            bS[0][10] = (u8)InnerBoundaryStrength(mb, 12, 9);
2151            bS[0][11] = (u8)InnerBoundaryStrength(mb, 14, 11);
2152        }
2153        pTmp = (u32*)&bS[0][0];
2154        if (!nonZeroBs && (pTmp[1] || pTmp[2] || pTmp[3] ||
2155                           pTmp[5] || pTmp[6] || pTmp[7]) )
2156        {
2157            nonZeroBs = HANTRO_TRUE;
2158        }
2159    }
2160
2161    return(nonZeroBs);
2162
2163}
2164
2165/*------------------------------------------------------------------------------
2166
2167    Function: GetLumaEdgeThresholds
2168
2169        Functional description:
2170            Compute alpha, beta and tc0 thresholds for inner, left and top
2171            luma edges of a macroblock.
2172
2173------------------------------------------------------------------------------*/
2174void GetLumaEdgeThresholds(
2175    mbStorage_t *mb,
2176    u8 (*alpha)[2],
2177    u8 (*beta)[2],
2178    u8 (*threshold)[16],
2179    u8 (*bs)[16],
2180    u32 filteringFlags )
2181{
2182
2183/* Variables */
2184
2185    u32 indexA, indexB;
2186    u32 qpAv, qp, qpTmp;
2187    u32 i;
2188
2189/* Code */
2190
2191    ASSERT(threshold);
2192    ASSERT(bs);
2193    ASSERT(beta);
2194    ASSERT(alpha);
2195    ASSERT(mb);
2196
2197    qp = mb->qpY;
2198
2199    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
2200    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
2201
2202    /* Internal edge values */
2203    alpha[0][1] = alphas[indexA];
2204    alpha[1][1] = alphas[indexA];
2205    alpha[1][0] = alphas[indexA];
2206    alpha[0][0] = alphas[indexA];
2207    beta[0][1] = betas[indexB];
2208    beta[1][1] = betas[indexB];
2209    beta[1][0] = betas[indexB];
2210    beta[0][0] = betas[indexB];
2211
2212    /* vertical scan order */
2213    for (i = 0; i < 2; i++)
2214    {
2215        u32 t1, t2;
2216
2217        t1 = bs[i][0];
2218        t2 = bs[i][1];
2219        threshold[i][0]  = (t1) ? tc0[indexA][t1] : 0;
2220        t1 = bs[i][2];
2221        threshold[i][1]  = (t2) ? tc0[indexA][t2] : 0;
2222        t2 = bs[i][3];
2223        threshold[i][2]  = (t1) ? tc0[indexA][t1] : 0;
2224        t1 = bs[i][4];
2225        threshold[i][3]  = (t2) ? tc0[indexA][t2] : 0;
2226        t2 = bs[i][5];
2227        threshold[i][4]  = (t1) ? tc0[indexA][t1] : 0;
2228        t1 = bs[i][6];
2229        threshold[i][5]  = (t2) ? tc0[indexA][t2] : 0;
2230        t2 = bs[i][7];
2231        threshold[i][6]  = (t1) ? tc0[indexA][t1] : 0;
2232        t1 = bs[i][8];
2233        threshold[i][7]  = (t2) ? tc0[indexA][t2] : 0;
2234        t2 = bs[i][9];
2235        threshold[i][8]  = (t1) ? tc0[indexA][t1] : 0;
2236        t1 = bs[i][10];
2237        threshold[i][9]  = (t2) ? tc0[indexA][t2] : 0;
2238        t2 = bs[i][11];
2239        threshold[i][10] = (t1) ? tc0[indexA][t1] : 0;
2240        t1 = bs[i][12];
2241        threshold[i][11] = (t2) ? tc0[indexA][t2] : 0;
2242        t2 = bs[i][13];
2243        threshold[i][12] = (t1) ? tc0[indexA][t1] : 0;
2244        t1 = bs[i][14];
2245        threshold[i][13] = (t2) ? tc0[indexA][t2] : 0;
2246        t2 = bs[i][15];
2247        threshold[i][14] = (t1) ? tc0[indexA][t1] : 0;
2248        threshold[i][15] = (t2) ? tc0[indexA][t2] : 0;
2249    }
2250
2251    if (filteringFlags & FILTER_TOP_EDGE)
2252    {
2253        qpTmp = mb->mbB->qpY;
2254        if (qpTmp != qp)
2255        {
2256            u32 t1, t2, t3, t4;
2257            qpAv = (qp + qpTmp + 1) >> 1;
2258
2259            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
2260            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
2261
2262            alpha[1][0] = alphas[indexA];
2263            beta[1][0] = betas[indexB];
2264            t1 = bs[1][0];
2265            t2 = bs[1][1];
2266            t3 = bs[1][2];
2267            t4 = bs[1][3];
2268            threshold[1][0] = (t1 && (t1 < 4)) ? tc0[indexA][t1] : 0;
2269            threshold[1][1] = (t2 && (t2 < 4)) ? tc0[indexA][t2] : 0;
2270            threshold[1][2] = (t3 && (t3 < 4)) ? tc0[indexA][t3] : 0;
2271            threshold[1][3] = (t4 && (t4 < 4)) ? tc0[indexA][t4] : 0;
2272        }
2273    }
2274    if (filteringFlags & FILTER_LEFT_EDGE)
2275    {
2276        qpTmp = mb->mbA->qpY;
2277        if (qpTmp != qp)
2278        {
2279            qpAv = (qp + qpTmp + 1) >> 1;
2280
2281            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
2282            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
2283
2284            alpha[0][0] = alphas[indexA];
2285            beta[0][0] = betas[indexB];
2286            threshold[0][0] = (bs[0][0] && (bs[0][0] < 4)) ? tc0[indexA][bs[0][0]] : 0;
2287            threshold[0][1] = (bs[0][1] && (bs[0][1] < 4)) ? tc0[indexA][bs[0][1]] : 0;
2288            threshold[0][2] = (bs[0][2] && (bs[0][2] < 4)) ? tc0[indexA][bs[0][2]] : 0;
2289            threshold[0][3] = (bs[0][3] && (bs[0][3] < 4)) ? tc0[indexA][bs[0][3]] : 0;
2290        }
2291    }
2292
2293}
2294
2295/*------------------------------------------------------------------------------
2296
2297    Function: GetChromaEdgeThresholds
2298
2299        Functional description:
2300            Compute alpha, beta and tc0 thresholds for inner, left and top
2301            chroma edges of a macroblock.
2302
2303------------------------------------------------------------------------------*/
2304void GetChromaEdgeThresholds(
2305    mbStorage_t *mb,
2306    u8 (*alpha)[2],
2307    u8 (*beta)[2],
2308    u8 (*threshold)[8],
2309    u8 (*bs)[16],
2310    u32 filteringFlags,
2311    i32 chromaQpIndexOffset)
2312{
2313
2314/* Variables */
2315
2316    u32 indexA, indexB;
2317    u32 qpAv, qp, qpTmp;
2318    u32 i;
2319
2320/* Code */
2321
2322    ASSERT(threshold);
2323    ASSERT(bs);
2324    ASSERT(beta);
2325    ASSERT(alpha);
2326    ASSERT(mb);
2327    ASSERT(mb);
2328
2329    qp = mb->qpY;
2330    qp = h264bsdQpC[CLIP3(0, 51, (i32)qp + chromaQpIndexOffset)];
2331
2332    indexA = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetA);
2333    indexB = (u32)CLIP3(0, 51, (i32)qp + mb->filterOffsetB);
2334
2335    alpha[0][1] = alphas[indexA];
2336    alpha[1][1] = alphas[indexA];
2337    alpha[1][0] = alphas[indexA];
2338    alpha[0][0] = alphas[indexA];
2339    beta[0][1] = betas[indexB];
2340    beta[1][1] = betas[indexB];
2341    beta[1][0] = betas[indexB];
2342    beta[0][0] = betas[indexB];
2343
2344    for (i = 0; i < 2; i++)
2345    {
2346        u32 t1, t2;
2347
2348        t1 = bs[i][0];
2349        t2 = bs[i][1];
2350        threshold[i][0]  = (t1) ? tc0[indexA][t1] : 0;
2351        t1 = bs[i][2];
2352        threshold[i][1]  = (t2) ? tc0[indexA][t2] : 0;
2353        t2 = bs[i][3];
2354        threshold[i][2]  = (t1) ? tc0[indexA][t1] : 0;
2355        t1 = bs[i][8];
2356        threshold[i][3]  = (t2) ? tc0[indexA][t2] : 0;
2357        t2 = bs[i][9];
2358        threshold[i][4]  = (t1) ? tc0[indexA][t1] : 0;
2359        t1 = bs[i][10];
2360        threshold[i][5]  = (t2) ? tc0[indexA][t2] : 0;
2361        t2 = bs[i][11];
2362        threshold[i][6]  = (t1) ? tc0[indexA][t1] : 0;
2363        threshold[i][7]  = (t2) ? tc0[indexA][t2] : 0;
2364    }
2365
2366    if (filteringFlags & FILTER_TOP_EDGE)
2367    {
2368        qpTmp = mb->mbB->qpY;
2369        if (qpTmp != mb->qpY)
2370        {
2371            u32 t1, t2, t3, t4;
2372            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
2373            qpAv = (qp + qpTmp + 1) >> 1;
2374
2375            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
2376            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
2377
2378            alpha[1][0] = alphas[indexA];
2379            beta[1][0] = betas[indexB];
2380
2381            t1 = bs[1][0];
2382            t2 = bs[1][1];
2383            t3 = bs[1][2];
2384            t4 = bs[1][3];
2385            threshold[1][0] = (t1) ? tc0[indexA][t1] : 0;
2386            threshold[1][1] = (t2) ? tc0[indexA][t2] : 0;
2387            threshold[1][2] = (t3) ? tc0[indexA][t3] : 0;
2388            threshold[1][3] = (t4) ? tc0[indexA][t4] : 0;
2389        }
2390    }
2391    if (filteringFlags & FILTER_LEFT_EDGE)
2392    {
2393        qpTmp = mb->mbA->qpY;
2394        if (qpTmp != mb->qpY)
2395        {
2396
2397            qpTmp = h264bsdQpC[CLIP3(0, 51, (i32)qpTmp + chromaQpIndexOffset)];
2398            qpAv = (qp + qpTmp + 1) >> 1;
2399
2400            indexA = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetA);
2401            indexB = (u32)CLIP3(0, 51, (i32)qpAv + mb->filterOffsetB);
2402
2403            alpha[0][0] = alphas[indexA];
2404            beta[0][0] = betas[indexB];
2405            threshold[0][0] = (bs[0][0]) ? tc0[indexA][bs[0][0]] : 0;
2406            threshold[0][1] = (bs[0][1]) ? tc0[indexA][bs[0][1]] : 0;
2407            threshold[0][2] = (bs[0][2]) ? tc0[indexA][bs[0][2]] : 0;
2408            threshold[0][3] = (bs[0][3]) ? tc0[indexA][bs[0][3]] : 0;
2409        }
2410    }
2411
2412}
2413
2414#endif /* H264DEC_OMXDL */
2415
2416/*lint +e701 +e702 */
2417