/alaspatial/src/main/java/org/ala/spatial/analysis/index/FilteringIndex.java
Java | 1570 lines | 897 code | 237 blank | 436 comment | 154 complexity | ea063ab2f923b344ef1edc32284f6cee MD5 | raw file
Large files files are truncated, but you can click here to view the full file
1package org.ala.spatial.analysis.index; 2 3//import com.vividsolutions.jts.geom.Geometry; 4//import com.vividsolutions.jts.geom.GeometryFactory; 5//import com.vividsolutions.jts.io.WKTReader; 6import java.io.BufferedInputStream; 7import java.io.BufferedOutputStream; 8import java.io.File; 9import java.io.FileInputStream; 10import java.io.FileOutputStream; 11import java.io.ObjectInputStream; 12import java.io.ObjectOutputStream; 13import java.io.RandomAccessFile; 14import java.nio.ByteBuffer; 15import java.util.ArrayList; 16import java.util.Comparator; 17import java.util.HashMap; 18import java.util.Map; 19import java.util.concurrent.LinkedBlockingQueue; 20import org.ala.spatial.analysis.cluster.SpatialCluster3; 21import org.ala.spatial.util.Grid; 22import org.ala.spatial.util.Layer; 23import org.ala.spatial.util.Layers; 24import org.ala.spatial.util.SimpleShapeFile; 25import org.ala.spatial.util.SpatialLogger; 26import org.ala.spatial.util.TabulationSettings; 27import org.ala.spatial.util.Tile; 28//import org.geotools.geometry.jts.JTS; 29//import org.geotools.geometry.jts.JTSFactoryFinder; 30//import org.geotools.referencing.CRS; 31//import org.opengis.referencing.crs.CoordinateReferenceSystem; 32//import org.opengis.referencing.operation.MathTransform; 33 34/** 35 * builder for species list index. 36 * 37 * requires OccurrencesIndex to be up to date. 38 * 39 * operates on GridFiles 40 * operates on ShapeFiles 41 * 42 * TODO: incremental and partial update mechanism 43 * 44 * @author adam 45 * 46 */ 47public class FilteringIndex extends Object { 48 49 /** 50 * length of a grid record in bytes 51 */ 52 static final int GRID_RECORD_LENGTH = 4 * 3 + 4; //3 floats + 1 int 53 /** 54 * length of a catagory record in bytes 55 */ 56 static final int CATAGORY_RECORD_LENGTH = 4 * 2 + 4;//1 short + 2 floats + 1 int 57 /** 58 * mapping for LayerFilters copies, init since static 59 */ 60 static Map<String, LayerFilter> map_layerfilters = new HashMap<String, LayerFilter>(); 61 /** 62 * destination of loaded occurrences 63 */ 64 ArrayList<String[]> occurrences; 65 String index_path; 66 67 /** 68 * default constructor 69 */ 70 public FilteringIndex(String directoryPath) { 71 index_path = directoryPath; 72 73 /* load settings file */ 74 TabulationSettings.load(); 75 } 76 77 /** 78 * performs update of 'indexing' for a new layer (grid or shapefile) 79 * 80 * @param layername name of the layer to update as String. To update 81 * all layers use null. 82 */ 83 public void layersUpdate(String layername) { 84 /* placeholder for patial update mechanism */ 85 makeSPL_GRID(layername); 86 87 makeSPL_CATAGORIES(layername); 88 89 /* for onscreen filtering server or client side */ 90 makeAllScaledShortImages(layername); 91 } 92 93 /** 94 * method to determine if the index is up to date 95 * 96 * @return true if index is up to date 97 */ 98 public boolean isUpToDate() { 99 /* placeholder for patial update mechanism */ 100 return true; 101 } 102 103 /** 104 * make species list index for environmental files 105 * 106 * for each grid file intersect and export in points order 107 * 108 * uses OccurrencesIndex generated SAM_D_ files 109 */ 110 void makeSPL_GRID(String layername) { 111 112 Layer layer; 113 int i; 114 /* iterate for each environmental file */ 115 for (i = 0; i < TabulationSettings.environmental_data_files.length; i++) { 116 layer = TabulationSettings.environmental_data_files[i]; 117 118 if (layername != null 119 && !layername.equalsIgnoreCase(TabulationSettings.environmental_data_files[i].name)) { 120 continue; 121 } 122 123 System.out.println("makeSPL_GRID: " + layer.display_name + " target?: " + layername); 124 125 try { 126 127 /* open input file for this layer's values in order of records*/ 128 RandomAccessFile sam_d_gridfile = new RandomAccessFile( 129 index_path + "SAM_D_" + layer.name + ".dat", "r"); 130 131 int number_of_records = (int) (sam_d_gridfile.length() / 4); //4 bytes in float 132 byte[] b = new byte[(number_of_records) * 4]; 133 sam_d_gridfile.read(b); 134 ByteBuffer bb = ByteBuffer.wrap(b); 135 sam_d_gridfile.close(); 136 137 /* temporary variables */ 138 double value; 139 140 ArrayList<SPLGridRecord> records = new ArrayList<SPLGridRecord>(number_of_records); 141 142 /* maintain original record index number 143 * for unqiueness 144 */ 145 int p = 0; 146 147 /* load all valid values and add to records with species number and 148 * original record index 149 */ 150 for (i = 0; i < number_of_records; i++) { 151 /* split up record line for extraction of: 152 * species name (column at idx ofu.onetwoCount-1) 153 * longitude (before last value) 154 * latitude (last value) 155 */ 156 157 value = bb.getFloat(); 158 159 /* do not want NaN */ 160 if (!Double.isNaN(value)) { 161 /* put it in a map to sort later */ 162 records.add(new SPLGridRecord(value, p)); 163 } 164 165 p++; //increment for next original record 166 } 167 168 /* sort records by environmental value */ 169 java.util.Collections.sort(records, 170 new Comparator<SPLGridRecord>() { 171 172 public int compare(SPLGridRecord r1, SPLGridRecord r2) { 173 if (r2.value == r1.value) { 174 return 0; 175 } else if (r1.value < r2.value) { 176 return -1; 177 } 178 return 1; 179 } 180 }); 181 182 /* open output file and write */ 183 RandomAccessFile outputvalues = new RandomAccessFile( 184 index_path + "SPL_V_" + layer.name + ".dat", "rw"); 185 186 RandomAccessFile outputrecords = new RandomAccessFile( 187 index_path + "SPL_R_" + layer.name + ".dat", "rw"); 188 189 byte[] b1 = new byte[records.size() * 4]; //for float 190 ByteBuffer bb1 = ByteBuffer.wrap(b1); 191 byte[] b2 = new byte[records.size() * 8]; //for int 192 ByteBuffer bb2 = ByteBuffer.wrap(b2); 193 194 for (SPLGridRecord r : records) { 195 bb1.putFloat((float) r.value); 196 bb2.putInt(r.record_number); 197 } 198 outputvalues.write(b1); 199 outputrecords.write(b2); 200 201 outputvalues.close(); 202 outputrecords.close(); 203 204 SpatialLogger.log("makeSPL_GRID writing done > " + layer.name); 205 206 } catch (Exception e) { 207 SpatialLogger.log("makeSPL_GRID writing", ">" + layer.name + "> " + e.toString()); 208 } 209 } 210 211 } 212 213 /** 214 * make species list index for catagorical files 215 * 216 * for each catagory file, split by field index value 217 * 218 */ 219 void makeSPL_CATAGORIES(String layername) { 220 Layer layer; 221 int i; 222 /* iterate for each shape file */ 223 for (i = 0; i < TabulationSettings.geo_tables.length; i++) { 224 if (layername != null 225 && !layername.equalsIgnoreCase(TabulationSettings.geo_tables[i].name)) { 226 continue; 227 228 } 229 230 layer = TabulationSettings.geo_tables[i]; 231 try { 232 /* open catagorical value file */ 233 RandomAccessFile sam_i_layerfile = new RandomAccessFile( 234 index_path 235 + SamplingIndex.CATAGORICAL_PREFIX + layer.name 236 + SamplingIndex.VALUE_POSTFIX, "r"); 237 238 int number_of_records = (int) (sam_i_layerfile.length() / 2); //2 bytes in short 239 byte[] b = new byte[(number_of_records) * 2]; 240 sam_i_layerfile.read(b); 241 ByteBuffer bb = ByteBuffer.wrap(b); 242 sam_i_layerfile.close(); 243 244 245 /* temporary variables */ 246 int value; 247 ArrayList<SPLGridRecord> records = new ArrayList<SPLGridRecord>(number_of_records); 248 249 int p = 0; /* maintain original record index number 250 * for unqiueness 251 */ 252 253 /* load all valid values and add to records with species number and 254 * original record index*/ 255 for (int j = 0; j < number_of_records; j++) { 256 257 /* split up record line for extraction of: 258 * species name (2 before last value) 259 * longitude (before last value) 260 * latitude (last value) 261 */ 262 value = bb.getShort(); 263 264 /* put it in a map to sort later */ 265 records.add(new SPLGridRecord(value, p)); 266 267 p++; //maintain original record number 268 } 269 270 /* sort by catagorical value */ 271 java.util.Collections.sort(records, 272 new Comparator<SPLGridRecord>() { 273 274 public int compare(SPLGridRecord r1, SPLGridRecord r2) { 275 if (r2.value == r1.value) { 276 return r1.record_number - r2.record_number; 277 } else if (r1.value < r2.value) { 278 return -1; 279 } 280 return 1; 281 } 282 }); 283 284 285 /* open output file and write */ 286 int idxpos = 0; 287 int startpos = 0; 288 289 /* iterate across records, exporting each group of records 290 * with the same value into a separate file with serialization 291 * 292 * exports as species_number and original_sorted_index (key) 293 * 294 * usage is whole of file 295 */ 296 int len = records.size(); 297 for (idxpos = 0; idxpos < len; idxpos++) { 298 if (idxpos + 1 == len || records.get(idxpos).value != records.get(idxpos + 1).value) { 299 //output the previous group of records into a new file 300 301 String filename = 302 index_path 303 + "SPL_" + layer.name + "_" 304 + records.get(idxpos).value + ".dat"; 305 306 307 /* open output stream */ 308 FileOutputStream fos = new FileOutputStream(filename); 309 BufferedOutputStream bos = new BufferedOutputStream(fos); 310 ObjectOutputStream oos = new ObjectOutputStream(bos); 311 312 /* build set */ 313 int[] set = new int[idxpos - startpos + 1]; 314 for (p = startpos; p <= idxpos; p++) { 315 set[p - startpos] = records.get(p).record_number; 316 } 317 318 /* sort by record_number (key) */ 319 java.util.Arrays.sort(set); 320 321 /* write */ 322 oos.writeObject(set); 323 oos.close(); 324 325 /* preserve starting pos for end of next value */ 326 startpos = idxpos + 1; 327 } 328 } 329 330 sam_i_layerfile.close(); 331 332 SpatialLogger.log("makeSPL_CATAGORIES writing done > " + layer.name); 333 334 } catch (Exception e) { 335 SpatialLogger.log("makeSPL_CATAGORIES writing", ">" + layer.name + "> " + e.toString()); 336 } 337 } 338 } 339 340 /** 341 * gets a default LayerFilter for a layer by layer name 342 * @param layer name of layer 343 * @return new default LayerFilter or null if does not exist 344 */ 345 public static LayerFilter getLayerFilter(String layer) { 346 /* if template already loaded copy it */ 347 LayerFilter layerfilter = map_layerfilters.get(layer); 348 if (layerfilter == null) { 349 layerfilter = getLayerFilter(Layers.getLayer(layer)); 350 map_layerfilters.put(layer, layerfilter); 351 } 352 353 /* if still not loaded, reutrn null */ 354 if (layerfilter == null) { 355 SpatialLogger.info("getLayerFilter(" + layer + ")", ">layer does not exist"); 356 return null; 357 } 358 359 return layerfilter.copy(); 360 } 361 362 /** 363 * gets a default LayerFilter for a layer by layer object 364 * 365 * note: use getLayerFilter(String) for public function 366 * 367 * @param layer as Layer 368 * @return new default LayerFilter 369 */ 370 static private LayerFilter getLayerFilter(Layer layer) { 371 if (layer == null) { 372 return null; 373 } 374 375 // check that the tabulationsettings is loaded. 376 // if not, load it. 377 if (!TabulationSettings.loaded) { 378 TabulationSettings.load(); 379 } 380 381 /* is it continous (grid file) */ 382 File continous_file = new File(TabulationSettings.environmental_data_path 383 + layer.name + ".gri"); 384 if (!continous_file.exists()) { 385 continous_file = new File(TabulationSettings.environmental_data_path 386 + layer.name + ".GRI"); 387 } 388 if (continous_file.exists()) { 389 /* load grid file and fill layerfilter with min/max */ 390 try { 391 Grid grid = new Grid(TabulationSettings.environmental_data_path 392 + layer.name); 393 394 double minimum = grid.minval; 395 double maximum = grid.maxval; 396 397 return new LayerFilter(layer, null, null, minimum, maximum); 398 399 } catch (Exception e) { 400 SpatialLogger.info("getLayerFilter(" + layer + ")", ">error opening grid file"); 401 return null; 402 } 403 } 404 405 /* otherwise, catagorical */ 406 407 /* 408 * TODO: handle multiple fieldnames 409 */ 410 String fieldname = layer.fields[0].name; 411 int i; 412 413 File catagories_file = new File( 414 TabulationSettings.index_path 415 + SamplingIndex.CATAGORY_LIST_PREFIX 416 + layer.name + "_" + fieldname 417 + SamplingIndex.CATAGORY_LIST_POSTFIX); 418 if (catagories_file.exists()) { 419 /* load catagories names and fill layerfilter with it */ 420 byte[] data = new byte[(int) catagories_file.length()]; 421 try { 422 FileInputStream fis = new FileInputStream(catagories_file); 423 fis.read(data); 424 fis.close(); 425 426 /* insert (row number) as beginning of each line */ 427 String str = new String(data); 428 data = null; 429 430 String[] catagory_names = str.split("\n"); 431 int[] catagories = new int[catagory_names.length]; 432 433 for (i = 0; i < catagories.length; i++) { 434 catagories[i] = i; 435 } 436 437 return new LayerFilter(layer, catagories, catagory_names, 0, 0); 438 439 } catch (Exception e) { 440 SpatialLogger.log("getLayerExtents(" + layer.name + "), catagorical", 441 e.toString()); 442 } 443 } 444 445 return null; 446 } 447 448 /** 449 * gets layer extents for a layer 450 * 451 * TODO: improve this function, move to LayerService.java 452 * 453 * @param layer_name layer name as String 454 * @return formatted layer extents as String, 455 * - min and max for grid 456 * - list of values for shape 457 */ 458 static public String getLayerExtents(String layer_name) { 459 /* is it continous (grid file) */ 460 /* 461 * TODO: handle multiple fieldnames 462 */ 463 String fieldname = ""; 464 int i; 465 for (i = 0; i < TabulationSettings.geo_tables.length; i++) { 466 Layer l = TabulationSettings.geo_tables[i]; 467 if (l.name.equalsIgnoreCase(layer_name)) { 468 fieldname = l.fields[0].name; 469 break; 470 } 471 } 472 File catagories_file = new File( 473 TabulationSettings.index_path 474 + SamplingIndex.CATAGORY_LIST_PREFIX 475 + layer_name + "_" + fieldname 476 + SamplingIndex.CATAGORY_LIST_POSTFIX); 477 478 if (!catagories_file.exists()) { 479 /* load grid file and get min/max */ 480 try { 481 Grid grid = new Grid(TabulationSettings.environmental_data_path 482 + layer_name); 483 484 double minimum = grid.minval; 485 double maximum = grid.maxval; 486 487 return ((float) minimum) + " to " + ((float) maximum); 488 } catch (Exception e) { 489 /* log error */ 490 //return nothing 491 return ""; 492 } 493 } 494 495 /* otherwise, catagorical */ 496 if (catagories_file.exists()) { 497 /* load catagories values file and format */ 498 byte[] data = new byte[(int) catagories_file.length()]; 499 try { 500 FileInputStream fis = new FileInputStream(catagories_file); 501 fis.read(data); 502 fis.close(); 503 504 /* insert (row number) as beginning of each line */ 505 String str = new String(data); 506 data = null; 507 508 String[] lines = str.split("\n"); 509 str = null; 510 511 StringBuffer output = new StringBuffer(); 512 513 /* 514 * TODO: add -1, missing 515 */ 516 int row = 0; // 0..n indexing 517 for (String s : lines) { 518 if (s.length() > 0) { 519 output.append(row + "," + s + "\r\n"); 520 row = row + 1; 521 } 522 } 523 524 return output.toString().replace("\r\n", "<br>\r\n"); 525 } catch (Exception e) { 526 SpatialLogger.log("getLayerExtents(" + layer_name + "), catagorical", 527 e.toString()); 528 } 529 } 530 531 return ""; 532 } 533 534 /** 535 * gets list of SpeciesRecord (species number and record number) from 536 * a layer filter (LayerFilter), grid/environmental/continous data source 537 * @param filter layer filter as LayerFilter 538 * @return records within the filtered region defined as ArrayList<RecordKey> 539 */ 540 public int[] getGridSampleSet(LayerFilter filter) { 541 /* grid/env filter, binary searching by seeking */ 542 543 double dblvalue; 544 545 int[] set = null; 546 547 try { 548 /* open VALUES file to determine records positions */ 549 RandomAccessFile raf = new RandomAccessFile( 550 index_path + "SPL_V_" + filter.layername + ".dat", "r"); 551 552 /* seek to first position */ 553 int length = (int) raf.length() / 4; //4 = sizeof float 554 int recordpos = length / 2; 555 int step = recordpos / 2; 556 while (step > 1) { 557 raf.seek(recordpos * 4); //4 = sizeof float 558 dblvalue = raf.readFloat(); 559 560 /* determine direction */ 561 if (dblvalue > filter.minimum_value) { 562 recordpos -= step; 563 } else { 564 recordpos += step; 565 } 566 step /= 2; 567 } 568 /* roll back to before the list */ 569 raf.seek(recordpos * 4); //4 = sizeof float 570 while (recordpos > 0 && raf.readFloat() > filter.minimum_value) { 571 recordpos--; 572 raf.seek(recordpos * 4); //4 = sizeof float 573 } 574 /* roll forwards to first record */ 575 raf.seek(recordpos * 4); //4 = sizeof float 576 while (recordpos < length && raf.readFloat() < filter.minimum_value) { 577 recordpos++; 578 raf.seek(recordpos * 4); //4 = sizeof float 579 } 580 /* seek to actual position */ 581 raf.seek(recordpos * 4); //4 = sizeof float 582 583 int start_recordpos = recordpos; 584 double over_max = filter.maximum_value + 0.000001; //TODO fix this as an actual small value 585 586 /* seek to last position */ 587 length = (int) raf.length() / 4; //4 = sizeof float 588 recordpos = length / 2; 589 step = recordpos / 2; 590 while (step > 1) { 591 raf.seek(recordpos * 4); //4 = sizeof float 592 dblvalue = raf.readFloat(); 593 594 /* determine direction */ 595 if (dblvalue > over_max) { 596 recordpos -= step; 597 } else { 598 recordpos += step; 599 } 600 step /= 2; 601 } 602 /* roll back to before the list */ 603 raf.seek(recordpos * 4); //4 = sizeof float 604 while (recordpos > 0 && raf.readFloat() > over_max) { 605 recordpos--; 606 raf.seek(recordpos * 4); //4 = sizeof float 607 608 } 609 /* roll forwards to first record */ 610 raf.seek(recordpos * 4); //4 = sizeof float 611 while (recordpos < length && raf.readFloat() < over_max) { 612 recordpos++; 613 raf.seek(recordpos * 4); //4 = sizeof float 614 } 615 /* seek to actual position */ 616 raf.seek(recordpos * 4); //4 = sizeof float 617 618 int end_recordpos = recordpos - 1; 619 System.out.println("size: " + end_recordpos + " " + start_recordpos); 620 621 raf.close(); //done with VALUES file 622 623 624 /* open RECORDS file to get records numbers */ 625 raf = new RandomAccessFile( 626 index_path + "SPL_R_" + filter.layername + ".dat", "r"); 627 628 /* read between start_recordpos and end_recordpos */ 629 raf.seek(start_recordpos * 4); //4 = sizeof int 630 byte[] records = new byte[(end_recordpos - start_recordpos + 1) * 4]; //TODO: setup for correct use of +1 //4 = sizeof int 631 632 raf.read(records); 633 634 /* byte buffer */ 635 ByteBuffer bb = ByteBuffer.wrap(records); 636 637 /* convert records to keys & add to system */ 638 long len = records.length / 4; //4 = sizeof int 639 //float longitude, latitude; 640 int record; 641 int j; 642 set = new int[(int) len]; 643 for (j = 0; j < len; j++) { 644 set[j] = bb.getInt(); 645 } 646 647 /* sort by key (record number) */ 648 java.util.Arrays.sort(set); 649 650 raf.close(); 651 652 } catch (Exception e) { 653 SpatialLogger.log("apply continous species list file", 654 "layer:" + filter.layername 655 + "> " + e.toString()); 656 } 657 658 return set; 659 660 } 661 662 /** 663 * gets list of SpeciesRecord (species number and record number) from 664 * a layer filter (LayerFilter), catagorical/contextual/shapefile data source 665 * @param filter layer filter as LayerFilter 666 * @return records within the filtered region defined as ArrayList<SpeciesRecord> 667 */ 668 public int[] getCatagorySampleSet(LayerFilter filter) { 669 670 int[] set = null; 671 672 /* catagory filter, iterate the whole file for each catagory */ 673 int value; 674 int k = 0; 675 int j; 676 for (j = 0; j < filter.catagories.length; j++) { 677 value = filter.catagories[j]; 678 try { 679 String filename = 680 index_path + "SPL_" + filter.layername + "_" 681 + ((double) value) + ".dat"; 682 683 FileInputStream fos = new FileInputStream(filename); 684 BufferedInputStream bos = new BufferedInputStream(fos); 685 ObjectInputStream oos = new ObjectInputStream(bos); 686 687 int[] newset; 688 newset = (int[]) oos.readObject(); 689 oos.close(); 690 691 if (set != null) { 692 int[] joined = new int[set.length + newset.length]; 693 System.arraycopy(set, 0, joined, 0, set.length); 694 System.arraycopy(newset, 0, joined, set.length, newset.length); 695 } else { 696 set = newset; 697 } 698 } catch (Exception e) { 699 SpatialLogger.log("apply catagory species list file", 700 "layer:" + filter.layername + " cat:" + value 701 + ": species_mask_layer len:" 702 + " : " + k + ">" + e.toString()); 703 } 704 } 705 if (filter.catagories.length > 1) { 706 java.util.Arrays.sort(set); 707 } 708 return set; 709 } 710 711 /** 712 * makes indexed tiles for filtering by a region 713 * 714 * TODO: dynamic 715 */ 716 public void makeAllScaledShortImages(String layername) { 717 /* ?? default long lat & dist */ 718 double longitude_start = 112; 719 double longitude_end = 154; 720 double latitude_start = -44;//-44; 721 double latitude_end = -9; 722 int height = 840; //210 42/210 723 int width = 1008; //252 724 725 int i, j; 726 727 /* get all layers */ 728 int size = 0; 729 Layer[] all_layers = new Layer[TabulationSettings.environmental_data_files.length 730 + TabulationSettings.geo_tables.length]; 731 j = TabulationSettings.environmental_data_files.length; 732 for (i = 0; i < j; i++) { 733 if (layername != null 734 && !layername.equalsIgnoreCase(TabulationSettings.environmental_data_files[i].name)) { 735 continue; 736 737 } 738 all_layers[size++] = TabulationSettings.environmental_data_files[i]; 739 } 740 for (i = 0; i < TabulationSettings.geo_tables.length; i++) { 741 if (layername != null 742 && !layername.equalsIgnoreCase(TabulationSettings.geo_tables[i].name)) { 743 continue; 744 } 745 all_layers[size++] = TabulationSettings.geo_tables[i]; 746 } 747 748 /* process all layers */ 749 for (i = 0; i < size; i++) { 750 //makeScaledShortImageFromGrid(all_layers[i], longitude_start, longitude_end, latitude_start, latitude_end, width, height); 751 makeScaledShortImageFromGridToMetresGrid(all_layers[i], longitude_start, longitude_end, latitude_start, latitude_end, width, height); 752 i++; 753 } 754 } 755 756 /** 757 * filter tile creation... 758 * 759 * more in docs now 760 * 761 * @param l layer as Layer 762 * @param longitude_start longitude extent as double 763 * @param longitude_end other longitude extent as double 764 * @param latitude_start latitude extent as double 765 * @param latitude_end other latitude extent as double 766 * @param width width resoluion as int 767 * @param height height resolution as in 768 */ 769 public void makeScaledShortImageFromGrid(Layer l, double longitude_start, double longitude_end, 770 double latitude_start, double latitude_end, int width, int height) { 771 772 /* output data */ 773 Tile[] data; 774 775 /* load raw */ 776 if (l.type.equals("environmental")) { 777 data = getTileFromGrid(l.name, longitude_start, longitude_end, latitude_start, latitude_end, width, height); 778 } else { 779 data = getTileFromShape(l, longitude_start, longitude_end, latitude_start, latitude_end, width, height); 780 } 781 782 /* sort */ 783 java.util.Arrays.sort(data, 784 new Comparator<Tile>() { 785 786 public int compare(Tile i1, Tile i2) { 787 if (i1.value_ < i2.value_) { 788 return -1; 789 } else if (i1.value_ > i2.value_) { 790 return 1; 791 } 792 return 0; 793 } 794 }); 795 796 /* index, only used for non-enviornmental layers */ 797 boolean has_index = true; 798 if (l.type.equals("environmental")) { 799 has_index = false; 800 } 801 int[] index = null; 802 if (has_index) { 803 int i, j; 804 int max = (int) data[data.length - 1].value_; 805 806 index = new int[max + 2]; 807 int last_idx = 1; 808 for (i = 1; i < data.length; i++) { 809 if (data[i].value_ != data[i - 1].value_) { 810 for (j = last_idx; j <= data[i].value_; j++) { 811 index[j] = i; 812 } 813 last_idx = (int) data[i].value_ + 1; 814 } 815 } 816 index[max + 1] = data.length; 817 } 818 819 /* write as object*/ 820 try { 821 FileOutputStream fos = new FileOutputStream( 822 TabulationSettings.index_path + "SPL_IMG_T_" + l.name + ".dat"); 823 BufferedOutputStream bos = new BufferedOutputStream(fos); 824 ObjectOutputStream oos = new ObjectOutputStream(bos); 825 oos.writeObject(data); 826 oos.writeObject(new Boolean(has_index)); 827 if (has_index) { 828 oos.writeObject(index); 829 } 830 oos.close(); 831 832 } catch (Exception e) { 833 /* TODO: log error */ 834 } 835 } 836 837 /** 838 * filter tile creation... 839 * 840 * more in docs now, meters grid from provided grid, 841 * 842 * TODO: better solution for projection change 843 * 844 * @param l layer as Layer 845 * @param longitude_start longitude extent as double 846 * @param longitude_end other longitude extent as double 847 * @param latitude_start latitude extent as double 848 * @param latitude_end other latitude extent as double 849 * @param width width resoluion as int 850 * @param height height resolution as in 851 */ 852 public void makeScaledShortImageFromGridToMetresGrid(Layer l, double longitude_start, double longitude_end, 853 double latitude_start, double latitude_end, int width, int height) { 854 855 /* output data */ 856 Tile[] data; 857 858 /* load raw */ 859 if (l.type.equals("environmental")) { 860 data = getTileFromGridToMetresGrid(l.name, longitude_start, longitude_end, latitude_start, latitude_end, width, height); 861 } else { 862 data = getTileFromShape(l, longitude_start, longitude_end, latitude_start, latitude_end, width, height); 863 } 864 865 /* sort */ 866 java.util.Arrays.sort(data, 867 new Comparator<Tile>() { 868 869 public int compare(Tile i1, Tile i2) { 870 if (i1.value_ < i2.value_) { 871 return -1; 872 } else if (i1.value_ > i2.value_) { 873 return 1; 874 } 875 return 0; 876 } 877 }); 878 879 /* index, only used for non-enviornmental layers */ 880 boolean has_index = true; 881 if (l.type.equals("environmental")) { 882 has_index = false; 883 } 884 int[] index = null; 885 if (has_index) { 886 int i, j; 887 int max = (int) data[data.length - 1].value_; 888 889 index = new int[max + 2]; 890 int last_idx = 1; 891 for (i = 1; i < data.length; i++) { 892 if (data[i].value_ != data[i - 1].value_) { 893 for (j = last_idx; j <= data[i].value_; j++) { 894 index[j] = i; 895 } 896 last_idx = (int) data[i].value_ + 1; 897 } 898 } 899 index[max + 1] = data.length; 900 } 901 902 /* write as object*/ 903 try { 904 FileOutputStream fos = new FileOutputStream( 905 TabulationSettings.index_path + "SPL_IMG_T_" + l.name + ".dat"); 906 BufferedOutputStream bos = new BufferedOutputStream(fos); 907 ObjectOutputStream oos = new ObjectOutputStream(bos); 908 oos.writeObject(data); 909 oos.writeObject(new Boolean(has_index)); 910 if (has_index) { 911 oos.writeObject(index); 912 } 913 oos.close(); 914 915 } catch (Exception e) { 916 /* TODO: log error */ 917 } 918 } 919 920 /** 921 * gets Tile data from a grid file onto specified extents 922 * 923 * @param layer_name 924 * @param longitude_start 925 * @param longitude_end 926 * @param latitude_start 927 * @param latitude_end 928 * @param longitude_steps 929 * @param latitude_steps 930 * @return Tile[] 931 */ 932 public Tile[] getTileFromGrid(String layer_name, 933 double longitude_start, double longitude_end, 934 double latitude_start, double latitude_end, 935 int longitude_steps, int latitude_steps) { 936 937 Grid grid = new Grid( 938 TabulationSettings.environmental_data_path 939 + layer_name); 940 941 /* make points to interrogate */ 942 double[][] points = new double[longitude_steps * latitude_steps][2]; 943 944 for (int j = 0; j < latitude_steps; j++) { 945 for (int i = 0; i < longitude_steps; i++) { 946 points[j * longitude_steps + i][0] = longitude_start 947 + i / (double) (longitude_steps - 1) * (longitude_end - longitude_start); 948 points[j * longitude_steps + i][1] = latitude_end 949 - j / (double) (latitude_steps - 1) * (latitude_end - latitude_start); 950 } 951 } 952 953 /* get layer data */ 954 float[] values = grid.getValues2(points); 955 956 if (values != null && values.length > 0) { 957 int i; 958 959 /* copy values back to byte data */ 960 int countvalues = 0; 961 for (i = 0; i < values.length; i++) { 962 if (!Double.isNaN(values[i])) { 963 countvalues++; 964 } 965 } 966 967 Tile[] data = new Tile[countvalues]; 968 969 int p = 0; 970 for (i = 0; i < values.length; i++) { 971 if (!Double.isNaN(values[i])) { 972 data[p++] = new Tile((float) values[i], i); 973 } 974 } 975 976 /* return data */ 977 return data; 978 } 979 980 return null; 981 } 982 983 /** 984 * gets Tile data from a grid file onto specified extents 985 * 986 * @param layer_name 987 * @param longitude_start 988 * @param longitude_end 989 * @param latitude_start 990 * @param latitude_end 991 * @param longitude_steps 992 * @param latitude_steps 993 * @return Tile[] 994 */ 995 public Tile[] getTileFromGridToMetresGrid(String layer_name, 996 double longitude_start, double longitude_end, 997 double latitude_start, double latitude_end, 998 int longitude_steps, int latitude_steps) { 999 1000 Grid grid = new Grid( 1001 TabulationSettings.environmental_data_path 1002 + layer_name); 1003 1004 /* make points to interrogate */ 1005 double[][] points = new double[longitude_steps * latitude_steps][2]; 1006 1007 //get latproj and longproj between display projection and data projection 1008 SpatialCluster3 sc = new SpatialCluster3(); 1009 int[] px_boundary = new int[4]; 1010 px_boundary[0] = sc.convertLngToPixel(longitude_start); 1011 px_boundary[2] = sc.convertLngToPixel(longitude_end); 1012 px_boundary[1] = sc.convertLatToPixel(latitude_start); 1013 px_boundary[3] = sc.convertLatToPixel(latitude_end); 1014 1015 double[] latproj = new double[latitude_steps]; 1016 double[] longproj = new double[longitude_steps]; 1017 for (int i = 0; i < latproj.length; i++) { 1018 latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length)))); 1019 } 1020 for (int i = 0; i < longproj.length; i++) { 1021 longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length)))); 1022 } 1023 1024 //add half cell for sample center 1025 double latoffset = (latproj[1] - latproj[0]) / 2.0; 1026 for (int i = 0; i < latproj.length; i++) { 1027 latproj[i] += latoffset; 1028 } 1029 double longoffset = (longproj[1] - longproj[0]) / 2.0; 1030 for (int i = 0; i < longproj.length; i++) { 1031 longproj[i] += longoffset; 1032 } 1033 1034 for (int j = 0; j < latitude_steps; j++) { 1035 for (int i = 0; i < longitude_steps; i++) { 1036 points[j * longitude_steps + i][0] = longproj[i]; 1037 points[j * longitude_steps + i][1] = latproj[latitude_steps - 1 - j]; 1038 } 1039 } 1040 1041 /* get layer data */ 1042 float[] values = grid.getValues2(points); 1043 1044 if (values != null && values.length > 0) { 1045 int i; 1046 1047 /* copy values back to byte data */ 1048 int countvalues = 0; 1049 for (i = 0; i < values.length; i++) { 1050 if (!Double.isNaN(values[i])) { 1051 countvalues++; 1052 } 1053 } 1054 1055 Tile[] data = new Tile[countvalues]; 1056 1057 int p = 0; 1058 for (i = 0; i < values.length; i++) { 1059 if (!Double.isNaN(values[i])) { 1060 data[p++] = new Tile((float) values[i], i); 1061 } 1062 } 1063 1064 /* return data */ 1065 return data; 1066 } 1067 1068 return null; 1069 } 1070 1071 /** 1072 * get Tiles from shape file layer 1073 * 1074 * @param l layer as Layer 1075 * @param longitude_start 1076 * @param longitude_end 1077 * @param latitude_start 1078 * @param latitude_end 1079 * @param longitude_steps 1080 * @param latitude_steps 1081 * @return Tile [] 1082 */ 1083 public Tile[] getTileFromShape(Layer l, 1084 double longitude_start, double longitude_end, 1085 double latitude_start, double latitude_end, 1086 int longitude_steps, int latitude_steps) { 1087 1088 SimpleShapeFile ssf = new SimpleShapeFile( 1089 TabulationSettings.environmental_data_path 1090 + l.name); 1091 1092 int column_idx = ssf.getColumnIdx(l.fields[0].name); 1093 1094 Tile[] data = ssf.getTileList(column_idx, longitude_start, latitude_start, longitude_end, latitude_end, longitude_steps, latitude_steps); 1095 1096 return data; 1097 } 1098 1099 void occurrencesUpdate(boolean forceUpdate) { 1100 /* threaded building, needs more ram than one at a time */ 1101 int threadcount = TabulationSettings.analysis_threads; 1102 ArrayList<String> layers = new ArrayList(); 1103 int i; 1104 for (i = 0; i < TabulationSettings.geo_tables.length; i++) { 1105 if (forceUpdate || !isUpToDateCatagorical(TabulationSettings.geo_tables[i].name)) { 1106 // layers.add(TabulationSettings.geo_tables[i].name); 1107 } 1108 } 1109 for (i = 0; i < TabulationSettings.environmental_data_files.length; i++) { 1110 if (forceUpdate || !isUpToDateContinous(TabulationSettings.environmental_data_files[i].name)) { 1111 layers.add(TabulationSettings.environmental_data_files[i].name); 1112 } 1113 } 1114 1115 if(layers.size() == 0) { 1116 return; 1117 } 1118 1119 LinkedBlockingQueue<String> lbq = new LinkedBlockingQueue(layers); 1120 1121 FilteringIndexThread[] it = new FilteringIndexThread[threadcount]; 1122 1123 for (i = 0; i < threadcount; i++) { 1124 it[i] = new FilteringIndexThread(lbq, index_path); 1125 } 1126 1127 System.out.println("Start FilteringIndex.build_all (" + threadcount + " threads): " + System.currentTimeMillis()); 1128 1129 //height mapping here while it is running 1130 startBuildAreaSize(); 1131 1132 //wait until all done 1133 try { 1134 boolean alive = true; 1135 while (alive) { 1136 Thread.sleep(2000); 1137 alive = false; 1138 for (i = 0; i < threadcount; i++) { 1139 if (it[i].isAlive()) { 1140 alive = true; 1141 break; 1142 } 1143 } 1144 } 1145 } catch (Exception e) { 1146 e.printStackTrace(); 1147 } 1148 } 1149 1150 //build vertical area file, for image in epsg900913 1151// static void buildAreaSize(double longitude_start, double longitude_end, 1152// double latitude_start, double latitude_end, 1153// int longitude_steps, int latitude_steps, 1154// String filename) { 1155// 1156// //get latproj between display projection and data projection 1157// SpatialCluster3 sc = new SpatialCluster3(); 1158// int[] px_boundary = new int[4]; 1159// px_boundary[0] = sc.convertLngToPixel(longitude_start); 1160// px_boundary[2] = sc.convertLngToPixel(longitude_end); 1161// px_boundary[1] = sc.convertLatToPixel(latitude_start); 1162// px_boundary[3] = sc.convertLatToPixel(latitude_end); 1163// 1164// double[] latproj = new double[latitude_steps + 1]; 1165// double[] longproj = new double[longitude_steps]; 1166// for (int i = 0; i < latproj.length; i++) { 1167// latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length)))); 1168// } 1169// for (int i = 0; i < longproj.length; i++) { 1170// longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length)))); 1171// } 1172// 1173// //get area's 1174// double[] areaSize = new double[latitude_steps]; 1175// 1176// double north, south, east, west; 1177// east = longproj[longproj.length - 2]; 1178// west = longproj[longproj.length - 1]; 1179// 1180// try { 1181// String wkt4326 = "GEOGCS[" + "\"WGS 84\"," + " DATUM[" + " \"WGS_1984\"," 1182// + " SPHEROID[\"WGS 84\",6378137,298.257223563,AUTHORITY[\"EPSG\",\"7030\"]]," 1183// + " TOWGS84[0,0,0,0,0,0,0]," + " AUTHORITY[\"EPSG\",\"6326\"]]," 1184// + " PRIMEM[\"Greenwich\",0,AUTHORITY[\"EPSG\",\"8901\"]]," 1185// + " UNIT[\"DMSH\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9108\"]]," 1186// + " AXIS[\"Lat\",NORTH]," + " AXIS[\"Long\",EAST]," 1187// + " AUTHORITY[\"EPSG\",\"4326\"]]"; 1188// String wkt3577 = "PROJCS[\"GDA94 / Australian Albers\"," 1189// + " GEOGCS[\"GDA94\"," 1190// + " DATUM[\"Geocentric_Datum_of_Australia_1994\"," 1191// + " SPHEROID[\"GRS 1980\",6378137,298.257222101," 1192// + " AUTHORITY[\"EPSG\",\"7019\"]]," 1193// + " TOWGS84[0,0,0,0,0,0,0]," 1194// + " AUTHORITY[\"EPSG\",\"6283\"]]," 1195// + " PRIMEM[\"Greenwich\",0," 1196// + " AUTHORITY[\"EPSG\",\"8901\"]]," 1197// + " UNIT[\"degree\",0.01745329251994328," 1198// + " AUTHORITY[\"EPSG\",\"9122\"]]," 1199// + " AUTHORITY[\"EPSG\",\"4283\"]]," 1200// + " UNIT[\"metre\",1," 1201// + " AUTHORITY[\"EPSG\",\"9001\"]]," 1202// + " PROJECTION[\"Albers_Conic_Equal_Area\"]," 1203// + " PARAMETER[\"standard_parallel_1\",-18]," 1204// + " PARAMETER[\"standard_parallel_2\",-36]," 1205// + " PARAMETER[\"false_northing\",0]," 1206// + " PARAMETER[\"latitude_of_center\",0]," 1207// + " PARAMETER[\"longitude_of_center\",132]," 1208// + " PARAMETER[\"false_easting\",0]," 1209// + " AUTHORITY[\"EPSG\",\"3577\"]," 1210// + " AXIS[\"Easting\",EAST]," 1211// + " AXIS[\"Northing\",NORTH]]"; 1212// 1213// CoordinateReferenceSystem wgsCRS = CRS.parseWKT(wkt4326); 1214// CoordinateReferenceSystem GDA94CRS = CRS.parseWKT(wkt3577); 1215// MathTransform transform = CRS.findMathTransform(wgsCRS, GDA94CRS); 1216// 1217// GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null); 1218// WKTReader reader = new WKTReader(geometryFactory); 1219// 1220// for (int i = 0; i < latitude_steps; i++) { 1221// south = latproj[i]; 1222// north = latproj[i + 1]; //latproj is len latitude_steps+1 1223// 1224// //backwards otherwise projection fails (?) 1225// String wkt = "POLYGON((" + south + " " + east 1226// + ", " + north + " " + east 1227// + ", " + north + " " + west 1228// + ", " + south + " " + west 1229// + ", " + south + " " + east + "))"; 1230// 1231// Geometry geom = reader.read(wkt); 1232// Geometry geomT = JTS.transform(geom, transform); 1233// 1234// //images are top down, this is south to north. 1235// areaSize[latitude_steps - 1 - i] = geomT.getArea(); 1236// } 1237// } catch (Exception ex) { 1238// ex.printStackTrace(); 1239// } 1240// 1241// //export 1242// try { 1243// String fname = TabulationSettings.index_path + filename; 1244// 1245// FileOutputStream fos = new FileOutputStream(fname); 1246// BufferedOutputStream bos = new BufferedOutputStream(fos); 1247// ObjectOutputStream oos = new ObjectOutputStream(bos); 1248// 1249// oos.writeObject(areaSize); 1250// 1251// oos.close(); 1252// } catch (Exception e) { 1253// e.printStackTrace(); 1254// } 1255// } 1256 1257 static void buildAreaSizeG(double longitude_start, double longitude_end, 1258 double latitude_start, double latitude_end, 1259 int longitude_steps, int latitude_steps, 1260 String filename, boolean imageSpacing) { 1261 1262 //get area's 1263 double[] areaSize = new double[latitude_steps]; 1264 1265 double [][] points = new double[4][2]; 1266 1267 if(imageSpacing) { 1268 //get latproj between display projection and data projection 1269 SpatialCluster3 sc = new SpatialCluster3(); 1270 int[] px_boundary = new int[4]; 1271 px_boundary[0] = sc.convertLngToPixel(longitude_start); 1272 px_boundary[2] = sc.convertLngToPixel(longitude_end); 1273 px_boundary[1] = sc.convertLatToPixel(latitude_start); 1274 px_boundary[3] = sc.convertLatToPixel(latitude_end); 1275 1276 double[] latproj = new double[latitude_steps + 1]; 1277 double[] longproj = new double[longitude_steps]; 1278 for (int i = 0; i < latproj.length; i++) { 1279 latproj[i] = sc.convertPixelToLat((int) (px_boundary[1] + (px_boundary[3] - px_boundary[1]) * (i / (double) (latproj.length)))); 1280 } 1281 for (int i = 0; i < longproj.length; i++) { 1282 longproj[i] = sc.convertPixelToLng((int) (px_boundary[0] + (px_boundary[2] - px_boundary[0]) * (i / (double) (longproj.length)))); 1283 } 1284 1285 double east = longproj[longproj.length - 2]; 1286 double west = longproj[longproj.length - 1]; 1287 points[0][0] = east; 1288 points[1][0] = west; 1289 points[2][0] = west; 1290 points[3][0] = east; 1291 1292 for (int i = 0; i < latitude_steps; i++) { 1293 double south = latproj[i]; 1294 double north = latproj[i + 1]; //latproj is len latitude_steps+1 1295 1296 points[0][1] = north; 1297 points[1][1] = north; 1298 points[2][1] = south; 1299 points[3][1] = south; 1300 1301 //images are top down, this is south to north. 1302 areaSize[latitude_steps - 1 - i] = calculateArea(points); 1303 } 1304 } else { 1305 double east = longitude_start; 1306 double west = east + (longitude_end - longitude_start) / (double) longitude_steps; 1307 double yrange = latitude_end - latitude_start; 1308 points[0][0] = east; 1309 points[1][0] = west; 1310 points[2][0] = west; 1311 points[3][0] = east; 1312 1313 for (int i = 0; i < latitude_steps; i++) { 1314 double south = latitude_start + i / (double) latitude_steps * yrange; 1315 double north = latitude_start + (i+1) / (double) latitude_steps * yrange; 1316 if(i == latitude_steps - 1) { 1317 north = latitude_end; 1318 } 1319 1320 points[0][1] = north; 1321 points[1][1] = north; 1322 points[2][1] = south; 1323 points[3][1] = south; 1324 1325 //images are top down, this is south to north. 1326 areaSize[latitude_steps - 1 - i] = calculateArea(points); 1327 } 1328 } 1329 1330 //export 1331 try { 1332 String fname = TabulationSettings.index_path + filename; 1333 1334 FileOutputStream fos = new FileOutputStream(fname); 1335 BufferedOutputStream bos = new BufferedOutputStream(fos); 1336 ObjectOutputStream oos = new ObjectOutputStream(bos); 1337 1338 oos.writeObject(areaSize); 1339 1340 oos.close(); 1341 } catch (Exception e) { 1342 e.printStackTrace(); 1343 } 1344 } 1345 1346 static public double[] getImageLatitudeArea() { 1347 String filename = TabulationSettings.index_path + "IMAGE_LATITUDE_AREA"; 1348 1349 //create it if it does not exist 1350 File file = new File(filename); 1351 if (!file.exists()) { 1352 startBuildAreaSize(); 1353 } 1354 1355 //import 1356 double[] areaSize = n…
Large files files are truncated, but you can click here to view the full file