PageRenderTime 100ms CodeModel.GetById 21ms app.highlight 67ms RepoModel.GetById 1ms app.codeStats 0ms

/alaspatial/src/main/java/org/ala/spatial/analysis/index/FilteringIndex.java

http://alageospatialportal.googlecode.com/
Java | 1570 lines | 897 code | 237 blank | 436 comment | 154 complexity | ea063ab2f923b344ef1edc32284f6cee MD5 | raw file
   1package org.ala.spatial.analysis.index;
   2
   3//import com.vividsolutions.jts.geom.Geometry;
   4//import com.vividsolutions.jts.geom.GeometryFactory;
   5//import com.vividsolutions.jts.io.WKTReader;
   6import java.io.BufferedInputStream;
   7import java.io.BufferedOutputStream;
   8import java.io.File;
   9import java.io.FileInputStream;
  10import java.io.FileOutputStream;
  11import java.io.ObjectInputStream;
  12import java.io.ObjectOutputStream;
  13import java.io.RandomAccessFile;
  14import java.nio.ByteBuffer;
  15import java.util.ArrayList;
  16import java.util.Comparator;
  17import java.util.HashMap;
  18import java.util.Map;
  19import java.util.concurrent.LinkedBlockingQueue;
  20import org.ala.spatial.analysis.cluster.SpatialCluster3;
  21import org.ala.spatial.util.Grid;
  22import org.ala.spatial.util.Layer;
  23import org.ala.spatial.util.Layers;
  24import org.ala.spatial.util.SimpleShapeFile;
  25import org.ala.spatial.util.SpatialLogger;
  26import org.ala.spatial.util.TabulationSettings;
  27import org.ala.spatial.util.Tile;
  28//import org.geotools.geometry.jts.JTS;
  29//import org.geotools.geometry.jts.JTSFactoryFinder;
  30//import org.geotools.referencing.CRS;
  31//import org.opengis.referencing.crs.CoordinateReferenceSystem;
  32//import org.opengis.referencing.operation.MathTransform;
  33
  34/**
  35 * builder for species list index.
  36 *
  37 * requires OccurrencesIndex to be up to date.
  38 *
  39 * operates on GridFiles
  40 * operates on ShapeFiles
  41 *
  42 * TODO: incremental and partial update mechanism
  43 *
  44 * @author adam
  45 *
  46 */
  47public class FilteringIndex extends Object {
  48
  49    /**
  50     * length of a grid record in bytes
  51     */
  52    static final int GRID_RECORD_LENGTH = 4 * 3 + 4; //3 floats + 1 int
  53    /**
  54     * length of a catagory record in bytes
  55     */
  56    static final int CATAGORY_RECORD_LENGTH = 4 * 2 + 4;//1 short + 2 floats + 1 int
  57    /**
  58     * mapping for LayerFilters copies, init since static
  59     */
  60    static Map<String, LayerFilter> map_layerfilters = new HashMap<String, LayerFilter>();
  61    /**
  62     * destination of loaded occurrences
  63     */
  64    ArrayList<String[]> occurrences;
  65    String index_path;
  66
  67    /**
  68     * default constructor
  69     */
  70    public FilteringIndex(String directoryPath) {
  71        index_path = directoryPath;
  72
  73        /* load settings file */
  74        TabulationSettings.load();
  75    }
  76    
  77    /**
  78     * performs update of 'indexing' for a new layer (grid or shapefile)
  79     *
  80     * @param layername name of the layer to update as String.  To update
  81     * all layers use null.
  82     */
  83    public void layersUpdate(String layername) {
  84        /* placeholder for patial update mechanism */
  85        makeSPL_GRID(layername);
  86
  87        makeSPL_CATAGORIES(layername);
  88
  89        /* for onscreen filtering server or client side */
  90        makeAllScaledShortImages(layername);
  91    }
  92
  93    /**
  94     * method to determine if the index is up to date
  95     *
  96     * @return true if index is up to date
  97     */
  98    public boolean isUpToDate() {
  99        /* placeholder for patial update mechanism */
 100        return true;
 101    }
 102
 103    /**
 104     * make species list index for environmental files
 105     *
 106     * for each grid file intersect and export in points order
 107     *
 108     * uses OccurrencesIndex generated SAM_D_ files
 109     */
 110    void makeSPL_GRID(String layername) {
 111
 112        Layer layer;
 113        int i;
 114        /* iterate for each environmental file */
 115        for (i = 0; i < TabulationSettings.environmental_data_files.length; i++) {
 116            layer = TabulationSettings.environmental_data_files[i];
 117
 118            if (layername != null
 119                    && !layername.equalsIgnoreCase(TabulationSettings.environmental_data_files[i].name)) {
 120                continue;
 121            }
 122
 123            System.out.println("makeSPL_GRID: " + layer.display_name + " target?: " + layername);
 124
 125            try {
 126
 127                /* open input file for this layer's values in order of records*/
 128                RandomAccessFile sam_d_gridfile = new RandomAccessFile(
 129                        index_path + "SAM_D_" + layer.name + ".dat", "r");
 130
 131                int number_of_records = (int) (sam_d_gridfile.length() / 4); //4 bytes in float
 132                byte[] b = new byte[(number_of_records) * 4];
 133                sam_d_gridfile.read(b);
 134                ByteBuffer bb = ByteBuffer.wrap(b);
 135                sam_d_gridfile.close();
 136
 137                /* temporary variables */
 138                double value;
 139
 140                ArrayList<SPLGridRecord> records = new ArrayList<SPLGridRecord>(number_of_records);
 141
 142                /* maintain original record index number
 143                 * for unqiueness
 144                 */
 145                int p = 0;
 146
 147                /* load all valid values and add to records with species number and
 148                 * original record index
 149                 */
 150                for (i = 0; i < number_of_records; i++) {
 151                    /* split up record line for extraction of:
 152                     *  species name (column at idx ofu.onetwoCount-1)
 153                     *  longitude (before last value)
 154                     *  latitude (last value)
 155                     */
 156
 157                    value = bb.getFloat();
 158
 159                    /* do not want NaN */
 160                    if (!Double.isNaN(value)) {
 161                        /* put it in a map to sort later */
 162                        records.add(new SPLGridRecord(value, p));
 163                    }
 164
 165                    p++;	//increment for next original record
 166                }
 167
 168                /* sort records by environmental value */
 169                java.util.Collections.sort(records,
 170                        new Comparator<SPLGridRecord>() {
 171
 172                            public int compare(SPLGridRecord r1, SPLGridRecord r2) {
 173                                if (r2.value == r1.value) {
 174                                    return 0;
 175                                } else if (r1.value < r2.value) {
 176                                    return -1;
 177                                }
 178                                return 1;
 179                            }
 180                        });
 181
 182                /* open output file and write */
 183                RandomAccessFile outputvalues = new RandomAccessFile(
 184                        index_path + "SPL_V_" + layer.name + ".dat", "rw");
 185
 186                RandomAccessFile outputrecords = new RandomAccessFile(
 187                        index_path + "SPL_R_" + layer.name + ".dat", "rw");
 188
 189                byte[] b1 = new byte[records.size() * 4]; //for float
 190                ByteBuffer bb1 = ByteBuffer.wrap(b1);
 191                byte[] b2 = new byte[records.size() * 8]; //for int
 192                ByteBuffer bb2 = ByteBuffer.wrap(b2);
 193
 194                for (SPLGridRecord r : records) {
 195                    bb1.putFloat((float) r.value);
 196                    bb2.putInt(r.record_number);
 197                }
 198                outputvalues.write(b1);
 199                outputrecords.write(b2);
 200
 201                outputvalues.close();
 202                outputrecords.close();
 203
 204                SpatialLogger.log("makeSPL_GRID writing done > " + layer.name);
 205
 206            } catch (Exception e) {
 207                SpatialLogger.log("makeSPL_GRID writing", ">" + layer.name + "> " + e.toString());
 208            }
 209        }
 210
 211    }
 212
 213    /**
 214     * make species list index for catagorical files
 215     *
 216     * for each catagory file, split by field index value
 217     *
 218     */
 219    void makeSPL_CATAGORIES(String layername) {
 220        Layer layer;
 221        int i;
 222        /* iterate for each shape file */
 223        for (i = 0; i < TabulationSettings.geo_tables.length; i++) {
 224            if (layername != null
 225                    && !layername.equalsIgnoreCase(TabulationSettings.geo_tables[i].name)) {
 226                continue;
 227
 228            }
 229
 230            layer = TabulationSettings.geo_tables[i];
 231            try {
 232                /* open catagorical value file */
 233                RandomAccessFile sam_i_layerfile = new RandomAccessFile(
 234                        index_path
 235                        + SamplingIndex.CATAGORICAL_PREFIX + layer.name
 236                        + SamplingIndex.VALUE_POSTFIX, "r");
 237
 238                int number_of_records = (int) (sam_i_layerfile.length() / 2); //2 bytes in short
 239                byte[] b = new byte[(number_of_records) * 2];
 240                sam_i_layerfile.read(b);
 241                ByteBuffer bb = ByteBuffer.wrap(b);
 242                sam_i_layerfile.close();
 243
 244
 245                /* temporary variables */
 246                int value;
 247                ArrayList<SPLGridRecord> records = new ArrayList<SPLGridRecord>(number_of_records);
 248
 249                int p = 0;			/* maintain original record index number
 250                 * for unqiueness
 251                 */
 252
 253                /* load all valid values and add to records with species number and
 254                 * original record index*/
 255                for (int j = 0; j < number_of_records; j++) {
 256
 257                    /* split up record line for extraction of:
 258                     *  species name (2 before last value)
 259                     *  longitude (before last value)
 260                     *  latitude (last value)
 261                     */
 262                    value = bb.getShort();
 263
 264                    /* put it in a map to sort later */
 265                    records.add(new SPLGridRecord(value, p));
 266
 267                    p++; 		//maintain original record number
 268                }
 269
 270                /* sort by catagorical value */
 271                java.util.Collections.sort(records,
 272                        new Comparator<SPLGridRecord>() {
 273
 274                            public int compare(SPLGridRecord r1, SPLGridRecord r2) {
 275                                if (r2.value == r1.value) {
 276                                    return r1.record_number - r2.record_number;
 277                                } else if (r1.value < r2.value) {
 278                                    return -1;
 279                                }
 280                                return 1;
 281                            }
 282                        });
 283
 284
 285                /* open output file and write */
 286                int idxpos = 0;
 287                int startpos = 0;
 288
 289                /* iterate across records, exporting each group of records
 290                 * with the same value into a separate file with serialization
 291                 *
 292                 * exports as species_number and original_sorted_index (key)
 293                 *
 294                 * usage is whole of file
 295                 */
 296                int len = records.size();
 297                for (idxpos = 0; idxpos < len; idxpos++) {
 298                    if (idxpos + 1 == len || records.get(idxpos).value != records.get(idxpos + 1).value) {
 299                        //output the previous group of records into a new file
 300
 301                        String filename =
 302                                index_path
 303                                + "SPL_" + layer.name + "_"
 304                                + records.get(idxpos).value + ".dat";
 305
 306
 307                        /* open output stream */
 308                        FileOutputStream fos = new FileOutputStream(filename);
 309                        BufferedOutputStream bos = new BufferedOutputStream(fos);
 310                        ObjectOutputStream oos = new ObjectOutputStream(bos);
 311
 312                        /* build set */
 313                        int[] set = new int[idxpos - startpos + 1];
 314                        for (p = startpos; p <= idxpos; p++) {
 315                            set[p - startpos] = records.get(p).record_number;
 316                        }
 317
 318                        /* sort by record_number (key) */
 319                        java.util.Arrays.sort(set);
 320
 321                        /* write */
 322                        oos.writeObject(set);
 323                        oos.close();
 324
 325                        /* preserve starting pos for end of next value */
 326                        startpos = idxpos + 1;
 327                    }
 328                }
 329
 330                sam_i_layerfile.close();
 331
 332                SpatialLogger.log("makeSPL_CATAGORIES writing done > " + layer.name);
 333
 334            } catch (Exception e) {
 335                SpatialLogger.log("makeSPL_CATAGORIES writing", ">" + layer.name + "> " + e.toString());
 336            }
 337        }
 338    }
 339
 340    /**
 341     * gets a default LayerFilter for a layer by layer name
 342     * @param layer name of layer
 343     * @return new default LayerFilter or null if does not exist
 344     */
 345    public static LayerFilter getLayerFilter(String layer) {
 346        /* if template already loaded copy it */
 347        LayerFilter layerfilter = map_layerfilters.get(layer);
 348        if (layerfilter == null) {
 349            layerfilter = getLayerFilter(Layers.getLayer(layer));
 350            map_layerfilters.put(layer, layerfilter);
 351        }
 352
 353        /* if still not loaded, reutrn null */
 354        if (layerfilter == null) {
 355            SpatialLogger.info("getLayerFilter(" + layer + ")", ">layer does not exist");
 356            return null;
 357        }
 358
 359        return layerfilter.copy();
 360    }
 361
 362    /**
 363     * gets a default LayerFilter for a layer by layer object
 364     *
 365     * note: use getLayerFilter(String) for public function
 366     *
 367     * @param layer as Layer
 368     * @return new default LayerFilter
 369     */
 370    static private LayerFilter getLayerFilter(Layer layer) {
 371        if (layer == null) {
 372            return null;
 373        }
 374
 375        // check that the tabulationsettings is loaded.
 376        // if not, load it.
 377        if (!TabulationSettings.loaded) {
 378            TabulationSettings.load();
 379        }
 380
 381        /* is it continous (grid file) */
 382        File continous_file = new File(TabulationSettings.environmental_data_path
 383                + layer.name + ".gri");
 384        if (!continous_file.exists()) {
 385            continous_file = new File(TabulationSettings.environmental_data_path
 386                    + layer.name + ".GRI");
 387        }
 388        if (continous_file.exists()) {
 389            /* load grid file and fill layerfilter with min/max */
 390            try {
 391                Grid grid = new Grid(TabulationSettings.environmental_data_path
 392                        + layer.name);
 393
 394                double minimum = grid.minval;
 395                double maximum = grid.maxval;
 396
 397                return new LayerFilter(layer, null, null, minimum, maximum);
 398
 399            } catch (Exception e) {
 400                SpatialLogger.info("getLayerFilter(" + layer + ")", ">error opening grid file");
 401                return null;
 402            }
 403        }
 404
 405        /* otherwise, catagorical */
 406
 407        /*
 408         * TODO: handle multiple fieldnames
 409         */
 410        String fieldname = layer.fields[0].name;
 411        int i;
 412
 413        File catagories_file = new File(
 414                TabulationSettings.index_path
 415                + SamplingIndex.CATAGORY_LIST_PREFIX
 416                + layer.name + "_" + fieldname
 417                + SamplingIndex.CATAGORY_LIST_POSTFIX);
 418        if (catagories_file.exists()) {
 419            /* load catagories names and fill layerfilter with it */
 420            byte[] data = new byte[(int) catagories_file.length()];
 421            try {
 422                FileInputStream fis = new FileInputStream(catagories_file);
 423                fis.read(data);
 424                fis.close();
 425
 426                /* insert (row number) as beginning of each line */
 427                String str = new String(data);
 428                data = null;
 429
 430                String[] catagory_names = str.split("\n");
 431                int[] catagories = new int[catagory_names.length];
 432
 433                for (i = 0; i < catagories.length; i++) {
 434                    catagories[i] = i;
 435                }
 436
 437                return new LayerFilter(layer, catagories, catagory_names, 0, 0);
 438
 439            } catch (Exception e) {
 440                SpatialLogger.log("getLayerExtents(" + layer.name + "), catagorical",
 441                        e.toString());
 442            }
 443        }
 444
 445        return null;
 446    }
 447
 448    /**
 449     * gets layer extents for a layer
 450     *
 451     * TODO: improve this function, move to LayerService.java
 452     *
 453     * @param layer_name layer name as String
 454     * @return formatted layer extents as String,
 455     * - min and max for grid
 456     * - list of values for shape
 457     */
 458    static public String getLayerExtents(String layer_name) {
 459        /* is it continous (grid file) */
 460        /*
 461         * TODO: handle multiple fieldnames
 462         */
 463        String fieldname = "";
 464        int i;
 465        for (i = 0; i < TabulationSettings.geo_tables.length; i++) {
 466            Layer l = TabulationSettings.geo_tables[i];
 467            if (l.name.equalsIgnoreCase(layer_name)) {
 468                fieldname = l.fields[0].name;
 469                break;
 470            }
 471        }
 472        File catagories_file = new File(
 473                TabulationSettings.index_path
 474                + SamplingIndex.CATAGORY_LIST_PREFIX
 475                + layer_name + "_" + fieldname
 476                + SamplingIndex.CATAGORY_LIST_POSTFIX);
 477
 478        if (!catagories_file.exists()) {
 479            /* load grid file and get min/max */
 480            try {
 481                Grid grid = new Grid(TabulationSettings.environmental_data_path
 482                        + layer_name);
 483
 484                double minimum = grid.minval;
 485                double maximum = grid.maxval;
 486
 487                return ((float) minimum) + " to " + ((float) maximum);
 488            } catch (Exception e) {
 489                /* log error */
 490                //return nothing
 491                return "";
 492            }
 493        }
 494
 495        /* otherwise, catagorical */
 496        if (catagories_file.exists()) {
 497            /* load catagories values file and format */
 498            byte[] data = new byte[(int) catagories_file.length()];
 499            try {
 500                FileInputStream fis = new FileInputStream(catagories_file);
 501                fis.read(data);
 502                fis.close();
 503
 504                /* insert (row number) as beginning of each line */
 505                String str = new String(data);
 506                data = null;
 507
 508                String[] lines = str.split("\n");
 509                str = null;
 510
 511                StringBuffer output = new StringBuffer();
 512
 513                /*
 514                 * TODO: add -1, missing
 515                 */
 516                int row = 0;					// 0..n indexing
 517                for (String s : lines) {
 518                    if (s.length() > 0) {
 519                        output.append(row + "," + s + "\r\n");
 520                        row = row + 1;
 521                    }
 522                }
 523
 524                return output.toString().replace("\r\n", "<br>\r\n");
 525            } catch (Exception e) {
 526                SpatialLogger.log("getLayerExtents(" + layer_name + "), catagorical",
 527                        e.toString());
 528            }
 529        }
 530
 531        return "";
 532    }
 533
 534    /**
 535     * gets list of SpeciesRecord (species number and record number) from
 536     * a layer filter (LayerFilter), grid/environmental/continous data source
 537     * @param filter layer filter as LayerFilter
 538     * @return records within the filtered region defined as ArrayList<RecordKey>
 539     */
 540    public int[] getGridSampleSet(LayerFilter filter) {
 541        /* grid/env filter, binary searching by seeking */
 542
 543        double dblvalue;
 544
 545        int[] set = null;
 546
 547        try {
 548            /* open VALUES file to determine records positions */
 549            RandomAccessFile raf = new RandomAccessFile(
 550                    index_path + "SPL_V_" + filter.layername + ".dat", "r");
 551
 552            /* seek to first position */
 553            int length = (int) raf.length() / 4;                       //4 = sizeof float
 554            int recordpos = length / 2;
 555            int step = recordpos / 2;
 556            while (step > 1) {
 557                raf.seek(recordpos * 4);				//4 = sizeof float
 558                dblvalue = raf.readFloat();
 559
 560                /* determine direction */
 561                if (dblvalue > filter.minimum_value) {
 562                    recordpos -= step;
 563                } else {
 564                    recordpos += step;
 565                }
 566                step /= 2;
 567            }
 568            /* roll back to before the list */
 569            raf.seek(recordpos * 4);					//4 = sizeof float
 570            while (recordpos > 0 && raf.readFloat() > filter.minimum_value) {
 571                recordpos--;
 572                raf.seek(recordpos * 4);				//4 = sizeof float
 573            }
 574            /* roll forwards to first record */
 575            raf.seek(recordpos * 4);					//4 = sizeof float
 576            while (recordpos < length && raf.readFloat() < filter.minimum_value) {
 577                recordpos++;
 578                raf.seek(recordpos * 4);				//4 = sizeof float
 579            }
 580            /* seek to actual position */
 581            raf.seek(recordpos * 4);					//4 = sizeof float
 582
 583            int start_recordpos = recordpos;
 584            double over_max = filter.maximum_value + 0.000001; //TODO fix this as an actual small value
 585
 586            /* seek to last position */
 587            length = (int) raf.length() / 4;                             //4 = sizeof float
 588            recordpos = length / 2;
 589            step = recordpos / 2;
 590            while (step > 1) {
 591                raf.seek(recordpos * 4);				//4 = sizeof float
 592                dblvalue = raf.readFloat();
 593
 594                /* determine direction */
 595                if (dblvalue > over_max) {
 596                    recordpos -= step;
 597                } else {
 598                    recordpos += step;
 599                }
 600                step /= 2;
 601            }
 602            /* roll back to before the list */
 603            raf.seek(recordpos * 4);					//4 = sizeof float
 604            while (recordpos > 0 && raf.readFloat() > over_max) {
 605                recordpos--;
 606                raf.seek(recordpos * 4);				//4 = sizeof float
 607
 608            }
 609            /* roll forwards to first record */
 610            raf.seek(recordpos * 4);					//4 = sizeof float
 611            while (recordpos < length && raf.readFloat() < over_max) {
 612                recordpos++;
 613                raf.seek(recordpos * 4);				//4 = sizeof float
 614            }
 615            /* seek to actual position */
 616            raf.seek(recordpos * 4);					//4 = sizeof float
 617
 618            int end_recordpos = recordpos - 1;
 619            System.out.println("size: " + end_recordpos + " " + start_recordpos);
 620
 621            raf.close();        //done with VALUES file
 622
 623
 624            /* open RECORDS file to get records numbers */
 625            raf = new RandomAccessFile(
 626                    index_path + "SPL_R_" + filter.layername + ".dat", "r");
 627
 628            /* read between start_recordpos and end_recordpos */
 629            raf.seek(start_recordpos * 4);				//4 = sizeof int
 630            byte[] records = new byte[(end_recordpos - start_recordpos + 1) * 4]; //TODO: setup for correct use of +1 //4 = sizeof int
 631
 632            raf.read(records);
 633
 634            /* byte buffer */
 635            ByteBuffer bb = ByteBuffer.wrap(records);
 636
 637            /* convert records to keys & add to system */
 638            long len = records.length / 4;                          //4 = sizeof int
 639            //float longitude, latitude;
 640            int record;
 641            int j;
 642            set = new int[(int) len];
 643            for (j = 0; j < len; j++) {
 644                set[j] = bb.getInt();
 645            }
 646
 647            /* sort by key (record number) */
 648            java.util.Arrays.sort(set);
 649
 650            raf.close();
 651
 652        } catch (Exception e) {
 653            SpatialLogger.log("apply continous species list file",
 654                    "layer:" + filter.layername
 655                    + "> " + e.toString());
 656        }
 657
 658        return set;
 659
 660    }
 661
 662    /**
 663     * gets list of SpeciesRecord (species number and record number) from
 664     * a layer filter (LayerFilter), catagorical/contextual/shapefile data source
 665     * @param filter layer filter as LayerFilter
 666     * @return records within the filtered region defined as ArrayList<SpeciesRecord>
 667     */
 668    public int[] getCatagorySampleSet(LayerFilter filter) {
 669
 670        int[] set = null;
 671
 672        /* catagory filter, iterate the whole file for each catagory */
 673        int value;
 674        int k = 0;
 675        int j;
 676        for (j = 0; j < filter.catagories.length; j++) {
 677            value = filter.catagories[j];
 678            try {
 679                String filename =
 680                        index_path + "SPL_" + filter.layername + "_"
 681                        + ((double) value) + ".dat";
 682
 683                FileInputStream fos = new FileInputStream(filename);
 684                BufferedInputStream bos = new BufferedInputStream(fos);
 685                ObjectInputStream oos = new ObjectInputStream(bos);
 686
 687                int[] newset;
 688                newset = (int[]) oos.readObject();
 689                oos.close();
 690
 691                if (set != null) {
 692                    int[] joined = new int[set.length + newset.length];
 693                    System.arraycopy(set, 0, joined, 0, set.length);
 694                    System.arraycopy(newset, 0, joined, set.length, newset.length);
 695                } else {
 696                    set = newset;
 697                }
 698            } catch (Exception e) {
 699                SpatialLogger.log("apply catagory species list file",
 700                        "layer:" + filter.layername + " cat:" + value
 701                        + ": species_mask_layer len:"
 702                        + " : " + k + ">" + e.toString());
 703            }
 704        }
 705        if (filter.catagories.length > 1) {
 706            java.util.Arrays.sort(set);
 707        }
 708        return set;
 709    }
 710
 711    /**
 712     * makes indexed tiles for filtering by a region
 713     *
 714     * TODO: dynamic
 715     */
 716    public void makeAllScaledShortImages(String layername) {
 717        /* ?? default long lat & dist */
 718        double longitude_start = 112;
 719        double longitude_end = 154;
 720        double latitude_start = -44;//-44;
 721        double latitude_end = -9;
 722        int height = 840; //210 42/210
 723        int width = 1008; //252
 724
 725        int i, j;
 726
 727        /* get all layers */
 728        int size = 0;
 729        Layer[] all_layers = new Layer[TabulationSettings.environmental_data_files.length
 730                + TabulationSettings.geo_tables.length];
 731        j = TabulationSettings.environmental_data_files.length;
 732        for (i = 0; i < j; i++) {
 733            if (layername != null
 734                    && !layername.equalsIgnoreCase(TabulationSettings.environmental_data_files[i].name)) {
 735                continue;
 736
 737            }
 738            all_layers[size++] = TabulationSettings.environmental_data_files[i];
 739        }
 740        for (i = 0; i < TabulationSettings.geo_tables.length; i++) {
 741            if (layername != null
 742                    && !layername.equalsIgnoreCase(TabulationSettings.geo_tables[i].name)) {
 743                continue;
 744            }
 745            all_layers[size++] = TabulationSettings.geo_tables[i];
 746        }
 747
 748        /* process all layers */
 749        for (i = 0; i < size; i++) {
 750            //makeScaledShortImageFromGrid(all_layers[i], longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 751            makeScaledShortImageFromGridToMetresGrid(all_layers[i], longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 752            i++;
 753        }
 754    }
 755
 756    /**
 757     * filter tile creation...
 758     *
 759     * more in docs now
 760     *
 761     * @param l	layer as Layer
 762     * @param longitude_start longitude extent as double
 763     * @param longitude_end other longitude extent as double
 764     * @param latitude_start latitude extent as double
 765     * @param latitude_end other latitude extent as double
 766     * @param width width resoluion as int
 767     * @param height height resolution as in
 768     */
 769    public void makeScaledShortImageFromGrid(Layer l, double longitude_start, double longitude_end,
 770            double latitude_start, double latitude_end, int width, int height) {
 771
 772        /* output data */
 773        Tile[] data;
 774
 775        /* load raw */
 776        if (l.type.equals("environmental")) {
 777            data = getTileFromGrid(l.name, longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 778        } else {
 779            data = getTileFromShape(l, longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 780        }
 781
 782        /* sort */
 783        java.util.Arrays.sort(data,
 784                new Comparator<Tile>() {
 785
 786                    public int compare(Tile i1, Tile i2) {
 787                        if (i1.value_ < i2.value_) {
 788                            return -1;
 789                        } else if (i1.value_ > i2.value_) {
 790                            return 1;
 791                        }
 792                        return 0;
 793                    }
 794                });
 795
 796        /* index, only used for non-enviornmental layers */
 797        boolean has_index = true;
 798        if (l.type.equals("environmental")) {
 799            has_index = false;
 800        }
 801        int[] index = null;
 802        if (has_index) {
 803            int i, j;
 804            int max = (int) data[data.length - 1].value_;
 805
 806            index = new int[max + 2];
 807            int last_idx = 1;
 808            for (i = 1; i < data.length; i++) {
 809                if (data[i].value_ != data[i - 1].value_) {
 810                    for (j = last_idx; j <= data[i].value_; j++) {
 811                        index[j] = i;
 812                    }
 813                    last_idx = (int) data[i].value_ + 1;
 814                }
 815            }
 816            index[max + 1] = data.length;
 817        }
 818
 819        /* write as object*/
 820        try {
 821            FileOutputStream fos = new FileOutputStream(
 822                    TabulationSettings.index_path + "SPL_IMG_T_" + l.name + ".dat");
 823            BufferedOutputStream bos = new BufferedOutputStream(fos);
 824            ObjectOutputStream oos = new ObjectOutputStream(bos);
 825            oos.writeObject(data);
 826            oos.writeObject(new Boolean(has_index));
 827            if (has_index) {
 828                oos.writeObject(index);
 829            }
 830            oos.close();
 831
 832        } catch (Exception e) {
 833            /* TODO: log error */
 834        }
 835    }
 836
 837    /**
 838     * filter tile creation...
 839     *
 840     * more in docs now, meters grid from provided grid,
 841     *
 842     * TODO: better solution for projection change
 843     *
 844     * @param l	layer as Layer
 845     * @param longitude_start longitude extent as double
 846     * @param longitude_end other longitude extent as double
 847     * @param latitude_start latitude extent as double
 848     * @param latitude_end other latitude extent as double
 849     * @param width width resoluion as int
 850     * @param height height resolution as in
 851     */
 852    public void makeScaledShortImageFromGridToMetresGrid(Layer l, double longitude_start, double longitude_end,
 853            double latitude_start, double latitude_end, int width, int height) {
 854
 855        /* output data */
 856        Tile[] data;
 857
 858        /* load raw */
 859        if (l.type.equals("environmental")) {
 860            data = getTileFromGridToMetresGrid(l.name, longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 861        } else {
 862            data = getTileFromShape(l, longitude_start, longitude_end, latitude_start, latitude_end, width, height);
 863        }
 864
 865        /* sort */
 866        java.util.Arrays.sort(data,
 867                new Comparator<Tile>() {
 868
 869                    public int compare(Tile i1, Tile i2) {
 870                        if (i1.value_ < i2.value_) {
 871                            return -1;
 872                        } else if (i1.value_ > i2.value_) {
 873                            return 1;
 874                        }
 875                        return 0;
 876                    }
 877                });
 878
 879        /* index, only used for non-enviornmental layers */
 880        boolean has_index = true;
 881        if (l.type.equals("environmental")) {
 882            has_index = false;
 883        }
 884        int[] index = null;
 885        if (has_index) {
 886            int i, j;
 887            int max = (int) data[data.length - 1].value_;
 888
 889            index = new int[max + 2];
 890            int last_idx = 1;
 891            for (i = 1; i < data.length; i++) {
 892                if (data[i].value_ != data[i - 1].value_) {
 893                    for (j = last_idx; j <= data[i].value_; j++) {
 894                        index[j] = i;
 895                    }
 896                    last_idx = (int) data[i].value_ + 1;
 897                }
 898            }
 899            index[max + 1] = data.length;
 900        }
 901
 902        /* write as object*/
 903        try {
 904            FileOutputStream fos = new FileOutputStream(
 905                    TabulationSettings.index_path + "SPL_IMG_T_" + l.name + ".dat");
 906            BufferedOutputStream bos = new BufferedOutputStream(fos);
 907            ObjectOutputStream oos = new ObjectOutputStream(bos);
 908            oos.writeObject(data);
 909            oos.writeObject(new Boolean(has_index));
 910            if (has_index) {
 911                oos.writeObject(index);
 912            }
 913            oos.close();
 914
 915        } catch (Exception e) {
 916            /* TODO: log error */
 917        }
 918    }
 919
 920    /**
 921     * gets Tile data from a grid file onto specified extents
 922     *
 923     * @param layer_name
 924     * @param longitude_start
 925     * @param longitude_end
 926     * @param latitude_start
 927     * @param latitude_end
 928     * @param longitude_steps
 929     * @param latitude_steps
 930     * @return Tile[]
 931     */
 932    public Tile[] getTileFromGrid(String layer_name,
 933            double longitude_start, double longitude_end,
 934            double latitude_start, double latitude_end,
 935            int longitude_steps, int latitude_steps) {
 936
 937        Grid grid = new Grid(
 938                TabulationSettings.environmental_data_path
 939                + layer_name);
 940
 941        /* make points to interrogate */
 942        double[][] points = new double[longitude_steps * latitude_steps][2];
 943
 944        for (int j = 0; j < latitude_steps; j++) {
 945            for (int i = 0; i < longitude_steps; i++) {
 946                points[j * longitude_steps + i][0] = longitude_start
 947                        + i / (double) (longitude_steps - 1) * (longitude_end - longitude_start);
 948                points[j * longitude_steps + i][1] = latitude_end
 949                        - j / (double) (latitude_steps - 1) * (latitude_end - latitude_start);
 950            }
 951        }
 952
 953        /* get layer data */
 954        float[] values = grid.getValues2(points);
 955
 956        if (values != null && values.length > 0) {
 957            int i;
 958
 959            /* copy values back to byte data */
 960            int countvalues = 0;
 961            for (i = 0; i < values.length; i++) {
 962                if (!Double.isNaN(values[i])) {
 963                    countvalues++;
 964                }
 965            }
 966
 967            Tile[] data = new Tile[countvalues];
 968
 969            int p = 0;
 970            for (i = 0; i < values.length; i++) {
 971                if (!Double.isNaN(values[i])) {
 972                    data[p++] = new Tile((float) values[i], i);
 973                }
 974            }
 975
 976            /* return data */
 977            return data;
 978        }
 979
 980        return null;
 981    }
 982
 983    /**
 984     * gets Tile data from a grid file onto specified extents
 985     *
 986     * @param layer_name
 987     * @param longitude_start
 988     * @param longitude_end
 989     * @param latitude_start
 990     * @param latitude_end
 991     * @param longitude_steps
 992     * @param latitude_steps
 993     * @return Tile[]
 994     */
 995    public Tile[] getTileFromGridToMetresGrid(String layer_name,
 996            double longitude_start, double longitude_end,
 997            double latitude_start, double latitude_end,
 998            int longitude_steps, int latitude_steps) {
 999
1000        Grid grid = new Grid(
1001                TabulationSettings.environmental_data_path
1002                + layer_name);
1003
1004        /* make points to interrogate */
1005        double[][] points = new double[longitude_steps * latitude_steps][2];
1006
1007        //get latproj and longproj between display projection and data projection
1008        SpatialCluster3 sc = new SpatialCluster3();
1009        int[] px_boundary = new int[4];
1010        px_boundary[0] = sc.convertLngToPixel(longitude_start);
1011        px_boundary[2] = sc.convertLngToPixel(longitude_end);
1012        px_boundary[1] = sc.convertLatToPixel(latitude_start);
1013        px_boundary[3] = sc.convertLatToPixel(latitude_end);
1014
1015        double[] latproj = new double[latitude_steps];
1016        double[] longproj = new double[longitude_steps];
1017        for (int i = 0; i < latproj.length; i++) {
1018            latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length))));
1019        }
1020        for (int i = 0; i < longproj.length; i++) {
1021            longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length))));
1022        }
1023
1024        //add half cell for sample center
1025        double latoffset = (latproj[1] - latproj[0]) / 2.0;
1026        for (int i = 0; i < latproj.length; i++) {
1027            latproj[i] += latoffset;
1028        }
1029        double longoffset = (longproj[1] - longproj[0]) / 2.0;
1030        for (int i = 0; i < longproj.length; i++) {
1031            longproj[i] += longoffset;
1032        }
1033
1034        for (int j = 0; j < latitude_steps; j++) {
1035            for (int i = 0; i < longitude_steps; i++) {
1036                points[j * longitude_steps + i][0] = longproj[i];
1037                points[j * longitude_steps + i][1] = latproj[latitude_steps - 1 - j];
1038            }
1039        }
1040
1041        /* get layer data */
1042        float[] values = grid.getValues2(points);
1043
1044        if (values != null && values.length > 0) {
1045            int i;
1046
1047            /* copy values back to byte data */
1048            int countvalues = 0;
1049            for (i = 0; i < values.length; i++) {
1050                if (!Double.isNaN(values[i])) {
1051                    countvalues++;
1052                }
1053            }
1054
1055            Tile[] data = new Tile[countvalues];
1056
1057            int p = 0;
1058            for (i = 0; i < values.length; i++) {
1059                if (!Double.isNaN(values[i])) {
1060                    data[p++] = new Tile((float) values[i], i);
1061                }
1062            }
1063
1064            /* return data */
1065            return data;
1066        }
1067
1068        return null;
1069    }
1070
1071    /**
1072     * get Tiles from shape file layer
1073     *
1074     * @param l layer as Layer
1075     * @param longitude_start
1076     * @param longitude_end
1077     * @param latitude_start
1078     * @param latitude_end
1079     * @param longitude_steps
1080     * @param latitude_steps
1081     * @return Tile []
1082     */
1083    public Tile[] getTileFromShape(Layer l,
1084            double longitude_start, double longitude_end,
1085            double latitude_start, double latitude_end,
1086            int longitude_steps, int latitude_steps) {
1087
1088        SimpleShapeFile ssf = new SimpleShapeFile(
1089                TabulationSettings.environmental_data_path
1090                + l.name);
1091
1092        int column_idx = ssf.getColumnIdx(l.fields[0].name);
1093
1094        Tile[] data = ssf.getTileList(column_idx, longitude_start, latitude_start, longitude_end, latitude_end, longitude_steps, latitude_steps);
1095
1096        return data;
1097    }
1098
1099    void occurrencesUpdate(boolean forceUpdate) {
1100        /* threaded building, needs more ram than one at a time */
1101        int threadcount = TabulationSettings.analysis_threads;
1102        ArrayList<String> layers = new ArrayList();
1103        int i;
1104        for (i = 0; i < TabulationSettings.geo_tables.length; i++) {
1105            if (forceUpdate || !isUpToDateCatagorical(TabulationSettings.geo_tables[i].name)) {
1106            //    layers.add(TabulationSettings.geo_tables[i].name);
1107            }
1108        }
1109        for (i = 0; i < TabulationSettings.environmental_data_files.length; i++) {
1110            if (forceUpdate || !isUpToDateContinous(TabulationSettings.environmental_data_files[i].name)) {
1111                layers.add(TabulationSettings.environmental_data_files[i].name);
1112            }
1113        }
1114
1115        if(layers.size() == 0) {
1116            return;
1117        }
1118
1119        LinkedBlockingQueue<String> lbq = new LinkedBlockingQueue(layers);
1120
1121        FilteringIndexThread[] it = new FilteringIndexThread[threadcount];
1122
1123        for (i = 0; i < threadcount; i++) {
1124            it[i] = new FilteringIndexThread(lbq, index_path);
1125        }
1126
1127        System.out.println("Start FilteringIndex.build_all (" + threadcount + " threads): " + System.currentTimeMillis());
1128
1129        //height mapping here while it is running
1130        startBuildAreaSize();
1131
1132        //wait until all done
1133        try {
1134            boolean alive = true;
1135            while (alive) {
1136                Thread.sleep(2000);
1137                alive = false;
1138                for (i = 0; i < threadcount; i++) {
1139                    if (it[i].isAlive()) {
1140                        alive = true;
1141                        break;
1142                    }
1143                }
1144            }
1145        } catch (Exception e) {
1146            e.printStackTrace();
1147        }
1148    }
1149
1150    //build vertical area file, for image in epsg900913
1151//    static void buildAreaSize(double longitude_start, double longitude_end,
1152//            double latitude_start, double latitude_end,
1153//            int longitude_steps, int latitude_steps,
1154//            String filename) {
1155//
1156//        //get latproj between display projection and data projection
1157//        SpatialCluster3 sc = new SpatialCluster3();
1158//        int[] px_boundary = new int[4];
1159//        px_boundary[0] = sc.convertLngToPixel(longitude_start);
1160//        px_boundary[2] = sc.convertLngToPixel(longitude_end);
1161//        px_boundary[1] = sc.convertLatToPixel(latitude_start);
1162//        px_boundary[3] = sc.convertLatToPixel(latitude_end);
1163//
1164//        double[] latproj = new double[latitude_steps + 1];
1165//        double[] longproj = new double[longitude_steps];
1166//        for (int i = 0; i < latproj.length; i++) {
1167//            latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length))));
1168//        }
1169//        for (int i = 0; i < longproj.length; i++) {
1170//            longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length))));
1171//        }
1172//
1173//        //get area's
1174//        double[] areaSize = new double[latitude_steps];
1175//
1176//        double north, south, east, west;
1177//        east = longproj[longproj.length - 2];
1178//        west = longproj[longproj.length - 1];
1179//
1180//        try {
1181//            String wkt4326 = "GEOGCS[" + "\"WGS 84\"," + "  DATUM[" + "    \"WGS_1984\","
1182//                    + "    SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]],"
1183//                    + "    TOWGS84[0,0,0,0,0,0,0]," + "    AUTHORITY[\"EPSG\",\"6326\"]],"
1184//                    + "  PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]],"
1185//                    + "  UNIT[\"DMSH\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9108\"]],"
1186//                    + "  AXIS[\"Lat\",NORTH]," + "  AXIS[\"Long\",EAST],"
1187//                    + "  AUTHORITY[\"EPSG\",\"4326\"]]";
1188//            String wkt3577 = "PROJCS[\"GDA94 / Australian Albers\","
1189//                    + "    GEOGCS[\"GDA94\","
1190//                    + "        DATUM[\"Geocentric_Datum_of_Australia_1994\","
1191//                    + "            SPHEROID[\"GRS 1980\",6378137,298.257222101,"
1192//                    + "                AUTHORITY[\"EPSG\",\"7019\"]],"
1193//                    + "            TOWGS84[0,0,0,0,0,0,0],"
1194//                    + "            AUTHORITY[\"EPSG\",\"6283\"]],"
1195//                    + "        PRIMEM[\"Greenwich\",0,"
1196//                    + "            AUTHORITY[\"EPSG\",\"8901\"]],"
1197//                    + "        UNIT[\"degree\",0.01745329251994328,"
1198//                    + "            AUTHORITY[\"EPSG\",\"9122\"]],"
1199//                    + "        AUTHORITY[\"EPSG\",\"4283\"]],"
1200//                    + "    UNIT[\"metre\",1,"
1201//                    + "        AUTHORITY[\"EPSG\",\"9001\"]],"
1202//                    + "    PROJECTION[\"Albers_Conic_Equal_Area\"],"
1203//                    + "    PARAMETER[\"standard_parallel_1\",-18],"
1204//                    + "    PARAMETER[\"standard_parallel_2\",-36],"
1205//                    + "    PARAMETER[\"false_northing\",0],"
1206//                    + "    PARAMETER[\"latitude_of_center\",0],"
1207//                    + "    PARAMETER[\"longitude_of_center\",132],"
1208//                    + "    PARAMETER[\"false_easting\",0],"
1209//                    + "    AUTHORITY[\"EPSG\",\"3577\"],"
1210//                    + "    AXIS[\"Easting\",EAST],"
1211//                    + "    AXIS[\"Northing\",NORTH]]";
1212//
1213//            CoordinateReferenceSystem wgsCRS = CRS.parseWKT(wkt4326);
1214//            CoordinateReferenceSystem GDA94CRS = CRS.parseWKT(wkt3577);
1215//            MathTransform transform = CRS.findMathTransform(wgsCRS, GDA94CRS);
1216//
1217//            GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null);
1218//            WKTReader reader = new WKTReader(geometryFactory);
1219//
1220//            for (int i = 0; i < latitude_steps; i++) {
1221//                south = latproj[i];
1222//                north = latproj[i + 1];     //latproj is len latitude_steps+1
1223//
1224//                //backwards otherwise projection fails (?)
1225//                String wkt = "POLYGON((" + south + " " + east
1226//                        + ", " + north + " " + east
1227//                        + ", " + north + " " + west
1228//                        + ", " + south + " " + west
1229//                        + ", " + south + " " + east + "))";
1230//
1231//                Geometry geom = reader.read(wkt);
1232//                Geometry geomT = JTS.transform(geom, transform);
1233//
1234//                //images are top down, this is south to north.
1235//                areaSize[latitude_steps - 1 - i] = geomT.getArea();
1236//            }
1237//        } catch (Exception ex) {
1238//            ex.printStackTrace();
1239//        }
1240//
1241//        //export
1242//        try {
1243//            String fname = TabulationSettings.index_path + filename;
1244//
1245//            FileOutputStream fos = new FileOutputStream(fname);
1246//            BufferedOutputStream bos = new BufferedOutputStream(fos);
1247//            ObjectOutputStream oos = new ObjectOutputStream(bos);
1248//
1249//            oos.writeObject(areaSize);
1250//
1251//            oos.close();
1252//        } catch (Exception e) {
1253//            e.printStackTrace();
1254//        }
1255//    }
1256
1257    static void buildAreaSizeG(double longitude_start, double longitude_end,
1258            double latitude_start, double latitude_end,
1259            int longitude_steps, int latitude_steps,
1260            String filename, boolean imageSpacing) {
1261
1262        //get area's
1263        double[] areaSize = new double[latitude_steps];
1264
1265        double [][] points = new double[4][2];
1266        
1267        if(imageSpacing) {
1268            //get latproj between display projection and data projection
1269            SpatialCluster3 sc = new SpatialCluster3();
1270            int[] px_boundary = new int[4];
1271            px_boundary[0] = sc.convertLngToPixel(longitude_start);
1272            px_boundary[2] = sc.convertLngToPixel(longitude_end);
1273            px_boundary[1] = sc.convertLatToPixel(latitude_start);
1274            px_boundary[3] = sc.convertLatToPixel(latitude_end);
1275
1276            double[] latproj = new double[latitude_steps + 1];
1277            double[] longproj = new double[longitude_steps];
1278            for (int i = 0; i < latproj.length; i++) {
1279                latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length))));
1280            }
1281            for (int i = 0; i < longproj.length; i++) {
1282                longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length))));
1283            }
1284
1285            double east = longproj[longproj.length - 2];
1286            double west = longproj[longproj.length - 1];
1287            points[0][0] = east;
1288            points[1][0] = west;
1289            points[2][0] = west;
1290            points[3][0] = east;
1291
1292            for (int i = 0; i < latitude_steps; i++) {
1293                double south = latproj[i];
1294                double north = latproj[i + 1];     //latproj is len latitude_steps+1
1295
1296                points[0][1] = north;
1297                points[1][1] = north;
1298                points[2][1] = south;
1299                points[3][1] = south;
1300
1301                //images are top down, this is south to north.
1302                areaSize[latitude_steps - 1 - i] = calculateArea(points);
1303            }
1304        } else {
1305            double east = longitude_start;
1306            double west = east + (longitude_end - longitude_start) / (double) longitude_steps;
1307            double yrange = latitude_end - latitude_start;
1308            points[0][0] = east;
1309            points[1][0] = west;
1310            points[2][0] = west;
1311            points[3][0] = east;
1312
1313            for (int i = 0; i < latitude_steps; i++) {
1314                double south = latitude_start + i / (double) latitude_steps * yrange;
1315                double north = latitude_start + (i+1) / (double) latitude_steps * yrange;
1316                if(i == latitude_steps - 1) {
1317                    north = latitude_end;
1318                }
1319
1320                points[0][1] = north;
1321                points[1][1] = north;
1322                points[2][1] = south;
1323                points[3][1] = south;
1324
1325                //images are top down, this is south to north.
1326                areaSize[latitude_steps - 1 - i] = calculateArea(points);
1327            }
1328        }
1329
1330        //export
1331        try {
1332            String fname = TabulationSettings.index_path + filename;
1333
1334            FileOutputStream fos = new FileOutputStream(fname);
1335            BufferedOutputStream bos = new BufferedOutputStream(fos);
1336            ObjectOutputStream oos = new ObjectOutputStream(bos);
1337
1338            oos.writeObject(areaSize);
1339
1340            oos.close();
1341        } catch (Exception e) {
1342            e.printStackTrace();
1343        }
1344    }
1345
1346    static public double[] getImageLatitudeArea() {
1347        String filename = TabulationSettings.index_path + "IMAGE_LATITUDE_AREA";
1348
1349        //create it if it does not exist
1350        File file = new File(filename);
1351        if (!file.exists()) {
1352            startBuildAreaSize();
1353        }
1354
1355        //import
1356        double[] areaSize = null;
1357        try {
1358            FileInputStream fis = new FileInputStream(filename);
1359            BufferedInputStream bis = new BufferedInputStream(fis);
1360            ObjectInputStream ois = new ObjectInputStream(bis);
1361
1362            areaSize = (double[]) ois.readObject();
1363
1364            ois.close();
1365        } catch (Exception e) {
1366            e.printStackTrace();
1367        }
1368
1369        return areaSize;
1370    }
1371
1372    static void startBuildAreaSize() {
1373        double longitude_start = 112;
1374        double longitude_end = 154;
1375        double latitude_start = -44;//-44;
1376        double latitude_end = -9;
1377        int height = 840; //210 42/210
1378        int width = 1008; //252
1379
1380        buildAreaSizeG(longitude_start, longitude_end, latitude_start, latitude_end, width, height, "IMAGE_LATITUDE_AREA", true);
1381    }
1382
1383    static public double[] getCommonGridLatitudeArea() {
1384        String filename = TabulationSettings.index_path + "IMAGE_LATITUDE_AREA_COMMON_GRID";
1385
1386        //create it if it does not exist
1387        File file = new File(filename);
1388        if (!file.exists()) {
1389            buildAreaSizeG(TabulationSettings.grd_xmin,
1390                    TabulationSettings.grd_xmax,
1391                    TabulationSettings.grd_ymin,
1392                    TabulationSettings.grd_ymax,
1393                    TabulationSettings.grd_ncols,
1394                    TabulationSettings.grd_nrows,
1395                    "IMAGE_LATITUDE_AREA_COMMON_GRID", false);
1396        }
1397
1398        //import
1399        double[] areaSize = null;
1400        try {
1401            FileInputStream fis = new FileInputStream(filename);
1402            BufferedInputStream bis = new BufferedInputStream(fis);
1403            ObjectInputStream ois = new ObjectInputStream(bis);
1404
1405            areaSize = (double[]) ois.readObject();
1406
1407            ois.close();
1408        } catch (Exception e) {
1409            e.printStackTrace();
1410        }
1411
1412        return areaSize;
1413    }
1414
1415    /**
1416     * determine if a GRD/GRI layer's filtering index is up to date.
1417     *
1418     * index_path + "SAM_D_" + layer.name + ".dat"
1419     * grid file
1420     *
1421     * both dated earlier than
1422     *
1423     * index_path + "SPL_V_" + layer.name + ".dat"
1424     * index_path + "SPL_R_" + layer.name + ".dat"
1425     *
1426     * @param fileName
1427     * @return true if up to date
1428     */
1429    private boolean isUpToDateContinous(String fileName) {
1430        //is it continous (grd/gri)
1431        File gri = new File(TabulationSettings.environmental_data_path
1432                + fileName + ".gri");
1433        File grd = new File(TabulationSettings.environmental_data_path
1434                + fileName + ".grd");
1435
1436        //extension case alternative
1437        if (!gri.exists()) {
1438            gri = new File(TabulationSettings.environmental_data_path
1439                    + fileName + ".GRI");
1440        }
1441        if (!grd.exists()) {
1442            grd = new File(TabulationSettings.environmental_data_path
1443                    + fileName + ".GRD");
1444        }
1445
1446        File samD = new File(index_path + "SAM_D_" + fileName + ".dat");
1447
1448        File splV = new File(index_path + "SPL_V_" + fileName + ".dat");
1449        File splR = new File(index_path + "SPL_R_" + fileName + ".dat");
1450
1451        if (gri.exists() && grd.exists() && samD.exists()
1452                && splV.exists() && splR.exists()) {
1453            long latestData = Math.max(Math.max(grd.lastModified(), gri.lastModified()), samD.lastModified());
1454
1455            return splV.lastModified() > latestData
1456                    && splR.lastModified() > latestData;
1457        }
1458
1459        return false;
1460    }
1461
1462    /**
1463     * determine if a SHP/DBF layer's sampling index is up to date.
1464     *
1465     * index_path + "SAM_D_" + layer.name + ".dat"
1466     * index_path + SamplingIndex.CATAGORICAL_PREFIX + layer.name + SamplingIndex.VALUE_POSTFIX
1467     *
1468     * both dated earlier than
1469     *
1470     * index_path + "SPL_V_" + layer.name + ".dat"
1471     * index_path + "SPL_R_" + layer.name + ".dat"
1472     *
1473     * @param fileName
1474     * @return true if up to date
1475     */
1476    private boolean isUpToDateCatagorical(String fileName) {
1477        //is it continous (grd/gri)
1478        File sam = new File(index_path + SamplingIndex.CATAGORICAL_PREFIX + fileName + SamplingIndex.VALUE_POSTFIX);
1479
1480        File spl0 = new File(index_path
1481                + "SPL_" + fileName + "_"
1482                + "0.0" + ".dat");
1483
1484        if (spl0.exists() && sam.exists()) {
1485            return spl0.lastModified() > sam.lastModified();
1486        }
1487
1488        return false;
1489    }
1490
1491    //calculateArea from FilteringResultsWCController
1492    static private double calculateArea(double [][] areaarr) {
1493        try {
1494            double totalarea = 0.0;
1495            double [] d = areaarr[0];
1496            for (int f = 1; f < areaarr.length-2; ++f) {
1497                totalarea += Mh(d, areaarr[f], areaarr[f + 1]);
1498            }
1499
1500            totalarea = Math.abs(totalarea*6378137*6378137);
1501
1502            //return as sq km
1503            return totalarea / 1000.0 / 1000.0;
1504
1505        } catch (Exception e) {
1506            System.out.println("Error in calculateArea");
1507            e.printStackTrace(System.out);
1508        }
1509
1510        return 0;
1511    }
1512
1513    static private double Mh(double [] a, double [] b, double [] c) {
1514        return Nh(a, b, c) * hi(a, b, c);
1515    }
1516
1517    static private double Nh(double [] a, double [] b, double [] c) {
1518        double [][] poly = {a, b, c, a};
1519        double[] area = new double[3];
1520        int i = 0;
1521        double j = 0.0;
1522        for (i=0; i < 3; ++i) {
1523            area[i] = vd(poly[i], poly[i + 1]);
1524            j += area[i];
1525        }
1526        j /= 2;
1527        double f = Math.tan(j / 2);
1528        for (i = 0; i < 3; ++i) {
1529            f *= Math.tan((j - area[i]) / 2);
1530        }
1531        return 4 * Math.atan(Math.sqrt(Math.abs(f)));
1532    }
1533
1534    static private double hi(double [] a, double [] b, double [] c) {
1535        double [][] d = {a, b, c};
1536
1537        int i = 0;
1538        double[][] bb = new double[3][3];
1539        for (i = 0; i < 3; ++i) {
1540            double lng = d[i][0];
1541            double lat = d[i][1];
1542
1543            double y = Uc(lat);
1544            double x = Uc(lng);
1545
1546            bb[i][0] = Math.cos(y) * Math.cos(x);
1547            bb[i][1] = Math.cos(y) * Math.sin(x);
1548            bb[i][2] = Math.sin(y);
1549        }
1550
1551        return (bb[0][0] * bb[1][1] * bb[2][2] + bb[1][0] * bb[2][1] * bb[0][2] + bb[2][0] * bb[0][1] * bb[1][2] - bb[0][0] * bb[2][1] * bb[1][2] - bb[1][0] * bb[0][1] * bb[2][2] - bb[2][0] * bb[1][1] * bb[0][2] > 0) ? 1 : -1;
1552    }
1553
1554    static private double vd(double [] a, double [] b) {
1555        double lng1 = a[0];
1556        double lat1 = a[1];
1557
1558        double lng2 = b[0];
1559        double lat2 = b[1];
1560
1561        double c = Uc(lat1);
1562        double d = Uc(lat2);
1563
1564        return 2 * Math.asin(Math.sqrt(Math.pow(Math.sin((c - d) / 2), 2) + Math.cos(c) * Math.cos(d) * Math.pow(Math.sin((Uc(lng1) - Uc(lng2)) / 2), 2)));
1565    }
1566
1567    static private double Uc(double a) {
1568        return a * (Math.PI / 180);
1569    }
1570};