/ocr/ocrservice/jni/hydrogen/src/clusterer.cpp

  1/*
  2 * Copyright 2011, Google Inc.
  3 *
  4 * Licensed under the Apache License, Version 2.0 (the "License");
  5 * you may not use this file except in compliance with the License.
  6 * You may obtain a copy of the License at
  7 *
  8 *     http://www.apache.org/licenses/LICENSE-2.0
  9 *
 10 * Unless required by applicable law or agreed to in writing, software
 11 * distributed under the License is distributed on an "AS IS" BASIS,
 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 13 * See the License for the specific language governing permissions and
 14 * limitations under the License.
 15 */
 16
 17#include <malloc.h>
 18#include "leptonica.h"
 19#include "clusterer.h"
 20#include "validator.h"
 21
 22/* Type of connected components: 4 is up/down/left/right. 8 includes diagonals */
 23#define CONN_COMP 8
 24
 25l_int32 ConnCompValidPixa(PIX *pix8, PIX *pix, PIXA **ppixa, NUMA **pconfs,
 26                          HydrogenTextDetector::TextDetectorParameters &params) {
 27  l_int32 h, iszero;
 28  l_int32 x, y, xstart, ystart;
 29  l_float32 singleton_conf;
 30  PIX *pixt1, *pixt2, *pixt3, *pixt4, *pixt5;
 31  PIXA *pixa, *pixasort;
 32  NUMA *confs, *confsort;
 33  BOX *box;
 34  BOXA *boxa;
 35  L_STACK *lstack, *auxstack;
 36
 37  PROCNAME("pixConnCompValidPixa");
 38
 39  if (!ppixa)
 40    return ERROR_INT("&pixa not defined", procName, 1);
 41  if (!pconfs)
 42    return ERROR_INT("&confs not defined", procName, 1);
 43  *ppixa = NULL;
 44  *pconfs = NULL;
 45  if (!pix || pixGetDepth(pix) != 1)
 46    return ERROR_INT("pixs undefined or not 1 bpp", procName, 1);
 47
 48  pixa = pixaCreate(0);
 49  confs = numaCreate(0);
 50
 51  pixZero(pix, &iszero);
 52  if (iszero) {
 53    *ppixa = pixa;
 54    return 0;
 55  }
 56
 57  if ((pixt1 = pixCopy(NULL, pix)) == NULL)
 58    return ERROR_INT("pixt1 not made", procName, 1);
 59  if ((pixt2 = pixCopy(NULL, pix)) == NULL)
 60    return ERROR_INT("pixt2 not made", procName, 1);
 61
 62  h = pixGetHeight(pix);
 63  if ((lstack = lstackCreate(h)) == NULL)
 64    return ERROR_INT("lstack not made", procName, 1);
 65  if ((auxstack = lstackCreate(0)) == NULL)
 66    return ERROR_INT("auxstack not made", procName, 1);
 67  lstack->auxstack = auxstack;
 68  if ((boxa = boxaCreate(0)) == NULL)
 69    return ERROR_INT("boxa not made", procName, 1);
 70
 71  xstart = 0;
 72  ystart = 0;
 73  while (1) {
 74    if (!nextOnPixelInRaster(pixt1, xstart, ystart, &x, &y))
 75      break;
 76
 77    if ((box = pixSeedfillBB(pixt1, lstack, x, y, CONN_COMP)) == NULL)
 78      return ERROR_INT("box not made", procName, 1);
 79
 80    /* Save the c.c. and remove from pixt2 as well */
 81    pixt3 = pixClipRectangle(pixt1, box, NULL);
 82    pixt4 = pixClipRectangle(pixt2, box, NULL);
 83    pixt5 = pixClipRectangle(pix8, box, NULL);
 84    pixXor(pixt3, pixt3, pixt4);
 85    pixRasterop(pixt2, box->x, box->y, box->w, box->h, PIX_SRC ^ PIX_DST, pixt3, 0, 0);
 86    pixDestroy(&pixt4);
 87
 88    if (ValidateSingleton(pixt3, box, pixt5, &singleton_conf, params)) {
 89      boxaAddBox(boxa, box, L_INSERT);
 90      pixaAddPix(pixa, pixt3, L_INSERT);
 91      numaAddNumber(confs, singleton_conf);
 92    } else {
 93      boxDestroy(&box);
 94      pixDestroy(&pixt3);
 95    }
 96
 97    pixDestroy(&pixt5);
 98
 99    xstart = x;
100    ystart = y;
101  }
102
103  /* Remove old boxa of pixa and replace with a clone copy */
104  boxaDestroy(&pixa->boxa);
105  pixa->boxa = boxaCopy(boxa, L_CLONE);
106
107  /* Sort pixa, then destroy old pixa */
108  NUMA *naindex;
109  if ((pixasort = pixaSort(pixa, L_SORT_BY_X, L_SORT_INCREASING, &naindex, L_CLONE)) == NULL)
110    return ERROR_INT("pixasort not made", procName, 1);
111  confsort = numaSortByIndex(confs, naindex);
112
113  /* Cleanup, freeing the fillsegs on each stack */
114  lstackDestroy(&lstack, TRUE);
115  pixDestroy(&pixt1);
116  pixDestroy(&pixt2);
117  boxaDestroy(&boxa);
118  pixaDestroy(&pixa);
119
120  *ppixa = pixasort;
121  *pconfs = confsort;
122
123  return 0;
124}
125
126l_int32 MergePix(PIXA *pixad, l_int32 d_idx, PIXA *pixas, l_int32 s_idx) {
127  l_int32 op;
128  l_int32 x, y, w, h;
129  l_int32 dx, dy, dw, dh;
130  PIX *pixd, *pixs, *pixmerge;
131  BOX *boxd, *boxs, *boxmerge;
132
133  PROCNAME("pixMergePix");
134
135  if (!pixad)
136    return ERROR_INT("pixad not defined", procName, 1);
137  if (!pixas)
138    return ERROR_INT("pixas not defined", procName, 1);
139
140  boxd = pixaGetBox(pixad, d_idx, L_CLONE);
141  boxs = pixaGetBox(pixas, s_idx, L_CLONE);
142  boxmerge = boxBoundingRegion(boxd, boxs);
143
144  boxGetGeometry(boxmerge, &x, &y, &w, &h);
145  pixmerge = pixCreate(w, h, 1);
146
147  op = PIX_SRC | PIX_DST;
148
149  pixs = pixaGetPix(pixas, s_idx, L_CLONE);
150
151  if (!pixs)
152    return ERROR_INT("s_idx not valid", procName, 1);
153
154  boxGetGeometry(boxs, &dx, &dy, &dw, &dh);
155  pixRasterop(pixmerge, dx - x, dy - y, dw, dh, op, pixs, 0, 0);
156  pixDestroy(&pixs);
157  boxDestroy(&boxs);
158
159  pixd = pixaGetPix(pixad, d_idx, L_CLONE);
160
161  if (!pixd)
162    return ERROR_INT("d_idx not valid", procName, 1);
163
164  boxGetGeometry(boxd, &dx, &dy, &dw, &dh);
165  pixRasterop(pixmerge, dx - x, dy - y, dw, dh, op, pixd, 0, 0);
166  pixDestroy(&pixd);
167  boxDestroy(&boxd);
168
169  pixaReplacePix(pixad, d_idx, pixmerge, boxmerge);
170
171  return 0;
172}
173
174l_int32 MergePairFragments(PIX *pix8, PIXA *clusters, PIXA *pixa, l_uint8 *remove) {
175  l_uint8 setj;
176  l_int32 i, j, real_j, contains, n, count, num_clusters, initj;
177  l_int32 xi, yi, wi, hi;
178  l_int32 xj, yj, wj, hj;
179  BOX *boxi, *boxj;
180  PIXA *pixasort;
181  NUMA *numa;
182
183  PROCNAME("pixMergePairFragments");
184
185  if (!pixa)
186    return ERROR_INT("pixa not defined", procName, -1);
187  if (!remove)
188    return ERROR_INT("remove not defined", procName, -1);
189
190  n = pixaGetCount(pixa);
191  num_clusters = pixaGetCount(clusters);
192
193  if (!n) {
194    L_INFO("pixa contained 0 pix", procName);
195    return 0;
196  }
197  if (!num_clusters) {
198    L_INFO("clusters contained 0 pix", procName);
199    return 0;
200  }
201  if ((pixasort = pixaSort(pixa, L_SORT_BY_Y, L_SORT_INCREASING, &numa, L_CLONE)) == NULL)
202    return ERROR_INT("failed to sort pixa", procName, -1);
203
204  count = 0;
205  initj = 0;
206  setj = 0;
207
208  for (i = 0; i < num_clusters; i++) {
209    pixaGetBoxGeometry(clusters, i, &xi, &yi, &wi, &hi);
210    boxi = pixaGetBox(clusters, i, L_CLONE);
211
212    setj = 0;
213
214    for (j = initj; j < n; j++) {
215      numaGetIValue(numa, j, &real_j);
216
217      // Only consider removed pix
218      if (!remove[real_j])
219        continue;
220
221      pixaGetBoxGeometry(pixasort, j, &xj, &yj, &wj, &hj);
222
223      // If the top of this pix is above the top of the cluster, skip
224      if (yj < yi)
225        continue;
226
227      if (!setj) {
228        initj = j;
229        setj = 1;
230      }
231
232      // If the bottom of this pix is below the bottom of the cluster, stop
233      if (yj > yi + hi)
234        break;
235
236      boxj = pixaGetBox(pixasort, j, L_CLONE);
237
238      boxIntersects(boxi, boxj, &contains);
239
240      if (contains) {
241        MergePix(clusters, i, pixasort, j);
242        //remove[real_j] = 0; // TODO eliminates duplicates
243        count++;
244      }
245
246      boxDestroy(&boxj);
247    }
248
249    boxDestroy(&boxi);
250  }
251
252  pixaDestroy(&pixasort);
253  numaDestroy(&numa);
254
255  return count;
256}
257
258l_int32 RemoveInvalidPairs(PIX *pix8, PIXA *pixa, NUMA *confs, l_uint8 *remove,
259                           HydrogenTextDetector::TextDetectorParameters &params) {
260  l_int32 i, j, n, count;
261  l_float32 pair_conf;
262  l_uint8 *has_partner;
263  BOX *b1, *b2;
264
265  PROCNAME("pixRemoveInvalidPairs");
266
267  if (!pixa)
268    return ERROR_INT("pixa not defined", procName, -1);
269  if (!remove)
270    return ERROR_INT("remove not defined", procName, -1);
271
272  n = pixaGetCount(pixa);
273
274  if (!n) {
275    L_INFO("pixa contained 0 pix", procName);
276    return 0;
277  }
278
279  has_partner = (l_uint8 *) calloc(n, sizeof(l_uint8));
280  count = 0;
281
282  for (i = 0; i < n; i++) {
283    if (remove[i])
284      continue;
285
286    b1 = pixaGetBox(pixa, i, L_CLONE);
287
288    /* Search right for a partner for i */
289    for (j = i + 1; j < n; j++) {
290      if (remove[j])
291        continue;
292
293      b2 = pixaGetBox(pixa, j, L_CLONE);
294
295      /* Check whether this is a valid pair */
296      if (!ValidatePair(b1, b2, &pair_conf, params)) {
297        boxDestroy(&b2);
298        continue;
299      }
300
301      // We don't need to adjust confidence values here, since we'll
302      // generate cluster pairs and use those later.
303
304      boxDestroy(&b2);
305
306      has_partner[i] = 1;
307      has_partner[j] = 1;
308      break;
309    }
310
311    boxDestroy(&b1);
312  }
313
314  for (i = 0; i < n; i++) {
315    if (!has_partner[i]) {
316      remove[i] = 1;
317
318      count++;
319    }
320  }
321
322  free(has_partner);
323
324  return count;
325}
326
327// Clustering pass
328
329l_int32 GenerateClusterPartners(PIX *pix8, PIXA *pixa, NUMA *confs, l_uint8 *remove, l_int32 **pleft,
330                                l_int32 **pright, HydrogenTextDetector::TextDetectorParameters &params) {
331  l_int32 n, i, j;
332  l_int32 xi, yi, wi, hi, maxd;
333  l_int32 xj, yj, wj, hj;
334  l_int32 dx, dy, d, mind, minj;
335  l_int32 *left, *right;
336  l_float32 clusterpair_conf, minconf;
337  BOX *b1, *b2;
338  bool too_far;
339
340  PROCNAME("GenerateClusterPartners");
341
342  if (!pixa)
343    return ERROR_INT("pixa not defined", procName, -1);
344  if (!pright)
345    return ERROR_INT("&right not defined", procName, -1);
346  if (!pleft)
347    return ERROR_INT("&left not defined", procName, -1);
348
349  n = pixaGetCount(pixa);
350
351  if (!n) {
352    L_INFO("pixa contained 0 pix", procName);
353    return 0;
354  }
355
356  /* If n == 0, remove may be NULL. Since we have already checked for that,
357   * any NULL arrays signal an error condition.
358   */
359  if (!remove)
360    return ERROR_INT("remove not defined", procName, -1);
361
362  left = (l_int32 *) malloc(n * sizeof(l_int32));
363  right = (l_int32 *) malloc(n * sizeof(l_int32));
364
365  /* Initialize left and right arrays */
366  for (i = 0; i < n; i++) {
367    left[i] = -2;
368    right[i] = -2;
369  }
370
371  /* For each component, check all possible neighbors to find the most likely
372   * right neighbor. If that right neighbor already has a left neighbor, insert
373   * the component to the right of the existing neighbor and the left of the
374   * right neighbor.
375   */
376  for (i = 0; i < n; i++) {
377    if (remove[i])
378      continue;
379
380    pixaGetBoxGeometry(pixa, i, &xi, &yi, &wi, &hi);
381    b1 = pixaGetBox(pixa, i, L_CLONE);
382    mind = -1;
383    minj = -1;
384    maxd = L_MAX(wi, hi);
385    minconf = 0.0;
386
387    /* Search for closest right neighbor */
388    for (j = i + 1; j < n; j++) {
389      if (remove[j])
390        continue;
391
392      pixaGetBoxGeometry(pixa, j, &xj, &yj, &wj, &hj);
393      b2 = pixaGetBox(pixa, j, L_CLONE);
394
395      if (!ValidateClusterPair(b1, b2, &too_far, &clusterpair_conf, params)) {
396        if (too_far)
397          break;
398        else
399          continue;
400      }
401
402      /* calculate spacing between i and j */
403      dx = xj - (xi + wi);
404      dy = (yj + hj) - (yi + hi);
405      d = dx * dx + dy * dy;
406
407      /* If we haven't found a neighbor OR we're the closest neighbor, update
408       * i's record for most likely neighbor.
409       */
410      if (mind < 0 || d < mind) {
411        mind = d;
412        minj = j;
413        minconf = clusterpair_conf;
414      }
415    }
416
417    /* If we found a valid neighbor, go ahead and use it. */
418    if (mind >= 0) {
419      j = left[minj];
420
421      /* If minj already had a left neighbor, replace it with i */
422      // TODO(alanv): Insertion fudges the partner confidence value
423      if (j >= 0) {
424        left[i] = j;
425        right[j] = i;
426      }
427
428      left[minj] = i;
429      right[i] = minj;
430
431      // Adjust confidence to reflect partner confidence
432      l_float32 conf;
433      numaGetFValue(confs, i, &conf);
434      conf *= minconf;
435      numaReplaceNumber(confs, i, conf);
436    }
437  }
438
439  *pleft = left;
440  *pright = right;
441
442  return 0;
443}
444
445l_int32 MergeClusterPartners(PIX *pix8, PIXA *pixa, NUMA *confs, l_uint8 *remove, l_int32 *left, l_int32 *right,
446                             PIXA **ppixad, NUMA **pclusterconfs, HydrogenTextDetector::TextDetectorParameters &params) {
447  l_int32 n, count, i, j, temp;
448  l_uint32 x, y, w, h;
449  l_int32 xi, yi, wi, hi;
450  l_int32 xj, yj, wj, hj;
451  PIXA *pixad, *pixa_cluster;
452  NUMA *confd, *numa_cluster;
453  PIX *pix, *pixd, *pix_cluster;
454  BOX *box, *boxd;
455
456  PROCNAME("ClusterValidComponents");
457
458  if (!ppixad)
459    return ERROR_INT("&pixad not defined", procName, -1);
460  if (!pclusterconfs)
461    return ERROR_INT("&clusterconfs not defined", procName, -1);
462
463  n = pixaGetCount(pixa);
464
465  pixad = pixaCreate(0);
466  confd = numaCreate(0);
467  *ppixad = pixad;
468  *pclusterconfs = confd;
469
470  if (!n) {
471    L_INFO("pixa contained 0 pix", procName);
472    return 0;
473  }
474
475  /* If n == 0, then left, right, and remove may be NULL. Since we have
476   * already checked for that, any NULL arrays signal an error condition.
477   */
478  if (!left)
479    return ERROR_INT("left not defined", procName, -1);
480  if (!right)
481    return ERROR_INT("right not defined", procName, -1);
482  if (!remove)
483    return ERROR_INT("remove not defined", procName, -1);
484
485  count = 0;
486
487  /* Starting from the first component, generate a cluster by traveling
488   * left and right as far as possible. Ignore components that have no
489   * neighbors.
490   */
491  for (i = 0; i < n; i++) {
492    if (remove[i])
493      continue;
494    if (left[i] < -1 && right[i] < -1)
495      remove[i] = 1;
496    if (left[i] < 0 && right[i] < 0)
497      continue;
498
499    pixa_cluster = pixaCreate(1);
500    numa_cluster = numaCreate(1);
501
502    /* We don't need to destroy this pix and box since pixa_cluster
503     * takes ownership with L_INSERT.
504     */
505    pix = pixaGetPix(pixa, i, L_CLONE);
506    box = pixaGetBox(pixa, i, L_CLONE);
507    pixaAddPix(pixa_cluster, pix, L_INSERT);
508    pixaAddBox(pixa_cluster, box, L_INSERT);
509    numaAddNumber(numa_cluster, i);
510
511    boxGetGeometry(box, &xi, &yi, &wi, &hi);
512    x = xi;
513    y = yi;
514    w = xi + wi;
515    h = yi + hi;
516
517    /* Move along left neighbors */
518    j = left[i];
519    left[i] = -1;
520    while (j >= 0) {
521      pix = pixaGetPix(pixa, j, L_CLONE);
522      box = pixaGetBox(pixa, j, L_CLONE);
523      pixaAddPix(pixa_cluster, pix, L_INSERT);
524      pixaAddBox(pixa_cluster, box, L_INSERT);
525      numaAddNumber(numa_cluster, j);
526
527      boxGetGeometry(box, &xj, &yj, &wj, &hj);
528      x = L_MIN(x, (l_uint32) xj);
529      y = L_MIN(y, (l_uint32) yj);
530      w = L_MAX(w, (l_uint32) (xj + wj));
531      h = L_MAX(h, (l_uint32) (yj + hj));
532
533      right[j] = -1;
534      temp = left[j];
535      left[j] = -1;
536      j = temp;
537    }
538
539    /* Move along right neighbors */
540    j = right[i];
541    right[i] = -1;
542    while (j >= 0) {
543      pix = pixaGetPix(pixa, j, L_CLONE);
544      box = pixaGetBox(pixa, j, L_CLONE);
545      pixaAddPix(pixa_cluster, pix, L_INSERT);
546      pixaAddBox(pixa_cluster, box, L_INSERT);
547      numaAddNumber(numa_cluster, j);
548
549      boxGetGeometry(box, &xj, &yj, &wj, &hj);
550      x = L_MIN(x, (l_uint32) xj);
551      y = L_MIN(y, (l_uint32) yj);
552      w = L_MAX(w, (l_uint32) xj + wj);
553      h = L_MAX(h, (l_uint32) (yj + hj));
554
555      left[j] = -1;
556      temp = right[j];
557      right[j] = -1;
558      j = temp;
559    }
560
561    w = w - x;
562    h = h - y;
563
564    boxd = boxCreate(x, y, w, h);
565    pix_cluster = pixClipRectangle(pix8, boxd, NULL);
566
567    l_float32 temp_conf;
568    l_float32 cluster_conf;
569
570    /* If pixa seems valid, collapse its components to a single pix */
571    if (ValidateCluster(pix_cluster, pixa_cluster, boxd, &cluster_conf, params)) {
572      l_int32 num_comps = pixaGetCount(pixa_cluster);
573      l_float32 avg_conf = 0.0;
574
575      pixd = pixCreate(w, h, 1);
576
577      for (int i = 0; i < num_comps; i++) {
578        pix = pixaGetPix(pixa_cluster, i, L_CLONE);
579        pixaGetBoxGeometry(pixa_cluster, i, &xj, &yj, &wj, &hj);
580        pixRasterop(pixd, xj - x, yj - y, wj, hj, PIX_PAINT, pix, 0, 0);
581        pixDestroy(&pix);
582
583        numaGetFValue(confs, i, &temp_conf);
584        avg_conf += temp_conf;
585      }
586
587      // Adjust average confidence to reflect overall cluster confidence
588      avg_conf /= num_comps;
589      avg_conf *= cluster_conf;
590
591      pixaAddPix(pixad, pixd, L_INSERT);
592      pixaAddBox(pixad, boxd, L_INSERT);
593      numaAddNumber(confd, avg_conf);
594
595      count++;
596    } else {
597      l_int32 num_nums = numaGetCount(numa_cluster);
598
599      // Otherwise, mark its components as removed
600      for (int i = 0; i < num_nums; i++) {
601        if (!numaGetIValue(numa_cluster, i, &temp)) {
602          remove[temp] = 1;
603        }
604      }
605
606      boxDestroy(&boxd);
607    }
608
609    pixDestroy(&pix_cluster);
610    pixaDestroy(&pixa_cluster);
611    numaDestroy(&numa_cluster);
612  }
613
614  free(left);
615  free(right);
616
617  PIXA *pixasort;
618  NUMA *confsort;
619
620  /* Sort pixa, then destroy old pixa */
621  NUMA *naindex;
622  if ((pixasort = pixaSort(pixad, L_SORT_BY_Y, L_SORT_INCREASING, &naindex, L_CLONE)) == NULL)
623    return ERROR_INT("pixasort not made", procName, 1);
624  confsort = numaSortByIndex(confd, naindex);
625
626  *ppixad = pixasort;
627  *pclusterconfs = confsort;
628
629  pixaDestroy(&pixad);
630  numaDestroy(&confd);
631
632  return count;
633}
634
635l_int32 ClusterValidComponents(PIX *pix8, PIXA *pixa, NUMA *confs, l_uint8 *remove, PIXA **ppixad,
636                               NUMA **pclusterconfs, HydrogenTextDetector::TextDetectorParameters &params) {
637  l_int32 *left, *right;
638  PIXA *pixad;
639  NUMA *clusterconfs;
640
641  if (GenerateClusterPartners(pix8, pixa, confs, remove, &left, &right, params))
642    return -1;
643
644  int count = MergeClusterPartners(pix8, pixa, confs, remove, left, right, &pixad, &clusterconfs, params);
645
646  *ppixad = pixad;
647  *pclusterconfs = clusterconfs;
648
649  return count;
650}