/alaspatial/src/main/java/org/ala/spatial/analysis/service/SamplingService.java
Java | 613 lines | 367 code | 94 blank | 152 comment | 111 complexity | a85e5ad0c15892522f62b4bd7bcc212c MD5 | raw file
1package org.ala.spatial.analysis.service; 2 3import java.io.File; 4import java.io.FileWriter; 5import java.util.ArrayList; 6import org.ala.spatial.analysis.index.BoundingBoxes; 7 8import org.ala.spatial.analysis.index.IndexedRecord; 9import org.ala.spatial.analysis.index.OccurrenceRecordNumbers; 10import org.ala.spatial.analysis.index.OccurrencesCollection; 11import org.ala.spatial.analysis.index.OccurrencesFilter; 12import org.ala.spatial.analysis.index.SpeciesColourOption; 13import org.ala.spatial.util.AnalysisJobSampling; 14import org.ala.spatial.util.Layers; 15import org.ala.spatial.util.OccurrencesFieldsUtil; 16import org.ala.spatial.util.SimpleRegion; 17import org.ala.spatial.util.SpatialLogger; 18import org.ala.spatial.util.TabulationSettings; 19 20/** 21 * service for returning occurrences + optional values from layer intersections 22 * 23 * @author adam 24 * 25 */ 26public class SamplingService { 27 28 public static SamplingService newForLSID(String lsid) { 29 if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) { 30 return new SamplingLoadedPointsService(); 31 } else { 32 return new SamplingService(); 33 } 34 } 35 36 /** 37 * constructor init 38 */ 39 SamplingService() { 40 TabulationSettings.load(); 41 } 42 43 /** 44 * gets samples; occurrences records + optional intersecting layer values, 45 * 46 * 47 * limit output 48 * 49 * @param filter species name as String 50 * @param layers list of layer names of additional data to include as String [] 51 * @param region region to restrict results as SimpleRegion 52 * @param records sorted pool of records to intersect with as ArrayList<Integer> 53 * @param max_rows upper limit of records to return as int 54 * @return samples as grid, String [][] 55 */ 56 public String sampleSpeciesAsCSV(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows) { 57 return sampleSpeciesAsCSV(filter, layers, region, records, max_rows, null); 58 } 59 60 /** 61 * gets samples; occurrences records + optional intersecting layer values, 62 * 63 * 64 * limit output 65 * 66 * @param filter species name as String 67 * @param layers list of layer names of additional data to include as String [] 68 * @param region region to restrict results as SimpleRegion 69 * @param records sorted pool of records to intersect with as ArrayList<Integer> 70 * @param max_rows upper limit of records to return as int 71 * @return samples as grid, String [][] 72 */ 73 public String[][] sampleSpecies(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows) { 74 return sampleSpecies(filter, layers, region, records, max_rows, null); 75 } 76 77 public String getHeader(String[] layers) { 78 StringBuffer header = new StringBuffer(); 79 OccurrencesFieldsUtil ofu = new OccurrencesFieldsUtil(); 80 for (String s : ofu.getOutputColumnNames()) { 81 header.append(s).append(","); 82 } 83 if (layers != null) { 84 for (String l : layers) { 85 header.append(Layers.layerNameToDisplayName(l)).append(","); 86 } 87 } 88 header.deleteCharAt(header.length() - 1); //take off end ',' 89 90 return header.toString(); 91 } 92 93 public String[][] sampleSpecies(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows, AnalysisJobSampling job) { 94 ArrayList<String> as = OccurrencesCollection.getFullRecords(new OccurrencesFilter(filter, region, records, layers, max_rows)); 95 96 //split records and append header 97 if (as.size() > 0) { 98 String[] header = getHeader(layers).split(","); 99 100 int numCols = header.length; 101 102 String[][] output = new String[as.size() + 1][numCols]; 103 104 //header 105 for (int j = 0; j < header.length && j < numCols; j++) { 106 output[0][j] = header[j]; 107 } 108 109 //records 110 for (int i = 0; i < as.size(); i++) { 111 String[] s = as.get(i).split(","); 112 for (int j = 0; j < s.length && j < numCols; j++) { 113 output[i + 1][j] = s[j]; 114 } 115 } 116 117 return output; 118 } 119 120 return null; 121 } 122 123 /** 124 * gets array of points for species (genus, etc) name matches within 125 * a specified region 126 * 127 * @param filter species (genus, etc) name 128 * @param region region to filter results by 129 * @param records sorted pool of records to intersect with as int [] 130 * @return points as double[], first is longitude, every second is latitude. 131 */ 132 public double[] sampleSpeciesPoints(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records) { 133 134 //test on bounding box 135 double[] bb = BoundingBoxes.getLsidBoundingBoxDouble(filter); 136 double[][] regionbb = region.getBoundingBox(); 137 if (bb != null && regionbb != null 138 && bb[0] <= regionbb[1][0] && bb[2] >= regionbb[0][0] 139 && bb[1] <= regionbb[1][1] && bb[3] >= regionbb[0][1]) { 140 141 return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER)); 142 } 143 144 return null; 145 } 146 147 /** 148 * gets array of points for species (genus, etc) name matches within 149 * a specified region 150 * 151 * can return other field or sampling for points returned 152 * 153 * @param filter species (genus, etc) name 154 * @param region region to filter results by 155 * @param records sorted pool of records to intersect with as ArrayList<Integer> 156 * @return points as double[], first is longitude, every second is latitude. 157 */ 158 public double[] sampleSpeciesPoints(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, ArrayList<SpeciesColourOption> extra) { 159 //test on bounding box 160 double[] bb = BoundingBoxes.getLsidBoundingBoxDouble(filter); 161 162 if (region == null) { 163 return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), extra); 164 } 165 166 double[][] regionbb = region.getBoundingBox(); 167 if (bb != null && bb[0] <= regionbb[1][0] && bb[2] >= regionbb[0][0] 168 && bb[1] <= regionbb[1][1] && bb[3] >= regionbb[0][1]) { 169 170 /* get points */ 171 return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), extra); 172 } 173 174 return null; 175 } 176 177 /** 178 * for Sensitive Coordinates 179 * 180 * gets array of points for species (genus, etc) name matches within 181 * a specified region 182 * 183 * @param filter species (genus, etc) name 184 * @param region region to filter results by 185 * @param records sorted pool of records to intersect with as ArrayList<Integer> 186 * @return points as double[], first is longitude, every second is latitude. 187 */ 188 public double[] sampleSpeciesPointsSensitive(String filter, SimpleRegion region, int[] records) { 189 IndexedRecord[] ir = null;// OccurrencesIndex.filterSpeciesRecords(filter); 190 191 if (ir != null && ir.length > 0) { 192 193 /* get points */ 194 double[] points = null;//OccurrencesIndex.getPointsSensitive(ir[0].record_start, ir[0].record_end); 195 196 /* test for region absence */ 197 if (region == null) { 198 return points; 199 } 200 201 int i; 202 int count = 0; 203 204 int recordsPos = 0; //for test on records 205 206 /* return all valid points within the region */ 207 for (i = 0; i < points.length; i += 2) { 208 //do not add if does not intersect with records list 209 if (records != null) { 210 int currentRecord = i + ir[0].record_start; 211 //increment recordsPos as required 212 while (recordsPos < records.length 213 && records[recordsPos] < currentRecord) { 214 recordsPos++; 215 } 216 //test for intersect 217 if (recordsPos >= records.length 218 || currentRecord != records[recordsPos]) { 219 continue; 220 } 221 } 222 //region test 223 if (region.isWithin(points[i], points[i + 1])) { 224 count += 2; 225 } else { 226 points[i] = Double.NaN; 227 } 228 } 229 //move into 'output' 230 if (count > 0) { 231 double[] output = new double[count]; 232 int p = 0; 233 for (i = 0; i < points.length; i += 2) { 234 if (!Double.isNaN(points[i])) { 235 output[p++] = points[i]; 236 output[p++] = points[i + 1]; 237 } 238 } 239 return output; 240 } 241 242 } 243 244 return null; 245 } 246 247 /** 248 * for Sensitive Coordinates 249 * 250 * gets array of points for species (genus, etc) name matches within 251 * a specified region 252 * 253 * removes points for all species that are sensitive 254 * 255 * @param filter species (genus, etc) name 256 * @param region region to filter results by 257 * @param records sorted pool of records to intersect with as ArrayList<Integer> 258 * @return points as double[], first is longitude, every second is latitude. 259 */ 260 public double[] sampleSpeciesPointsMinusSensitiveSpecies(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, StringBuffer removedSpecies) { 261 /* get points */ 262 return OccurrencesCollection.getPointsMinusSensitiveSpecies(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), removedSpecies); 263 } 264 265 /** 266 * for Sensitive Records 267 * 268 * Checks if the records are sensitive within filter range 269 * 270 * @param filter species (genus, etc) name 271 * @param region region to filter results by 272 * @param records sorted pool of records to intersect with as ArrayList<Integer> 273 * @return int 274 * 0 when non-sensitive and has records, 275 * 1 when sensitive or no records, 276 * -1 when cannot be determined 277 */ 278 public static int isSensitiveRecord(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records) { 279 StringBuffer sb = new StringBuffer(); 280 try { 281 double[] d = OccurrencesCollection.getPointsMinusSensitiveSpecies(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), sb); 282 if (d == null) { 283 return 1; 284 } else { 285 return 0; 286 } 287 } catch (Exception e) { 288 e.printStackTrace(); 289 } 290 291 return -1; 292 } 293 294 /** 295 * gets samples; occurrences records + optional intersecting layer values, 296 * 297 * 298 * limit output 299 * 300 * @param filter species name as String 301 * @param layers list of layer names of additional data to include as String [] 302 * @param region region to restrict results as SimpleRegion 303 * @param records sorted pool of records to intersect with as ArrayList<Integer> 304 * @param max_rows upper limit of records to return as int 305 * @return samples as grid, String [][] 306 */ 307 public String sampleSpeciesAsCSV(String species, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows, AnalysisJobSampling job) { 308 try { 309 310 System.out.println("Limiting sampling to : " + max_rows); 311 312 String[][] results = sampleSpecies(species, layers, region, records, max_rows); 313 StringBuilder sbResults = new StringBuilder(); 314 315 for (int i = 0; i < results.length; i++) { 316 for (int j = 0; j < results[i].length; j++) { 317 if (results[i][j] != null) { 318 sbResults.append(results[i][j]); 319 } 320 if (j < results[i].length - 1) { 321 sbResults.append(","); 322 } 323 } 324 sbResults.append("\r\n"); 325 } 326 327 /* open output file */ 328 File temporary_file = java.io.File.createTempFile("sample", ".csv"); 329 FileWriter fw = new FileWriter(temporary_file); 330 331 fw.append(sbResults.toString()); 332 fw.close(); 333 return temporary_file.getPath(); 334 335 336 } catch (Exception e) { 337 System.out.println("error with samplesSpeciesAsCSV:"); 338 e.printStackTrace(System.out); 339 } 340 341 return ""; 342 } 343 344 public static String getLSIDAsGeoJSON(String lsid, File outputpath) { 345 if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) { 346 return getLSIDAsGeoJSON(lsid, outputpath); 347 } 348 349 int i; 350 351 /* get samples records from records indexes */ 352 String[][] samples = (new SamplingService()).sampleSpecies(lsid, null, null, null, TabulationSettings.MAX_RECORD_COUNT_DOWNLOAD, null); 353 354 StringBuffer sbGeoJSON = new StringBuffer(); 355 sbGeoJSON.append("{"); 356 sbGeoJSON.append(" \"type\": \"FeatureCollection\","); 357 sbGeoJSON.append(" \"features\": ["); 358 for (i = 1; i < samples.length; i++) { 359 String s = getRecordAsGeoJSON(samples, i); 360 if (s != null) { 361 sbGeoJSON.append(s); 362 if (i < samples.length - 1) { 363 sbGeoJSON.append(","); 364 } 365 } 366 } 367 sbGeoJSON.append(" ],"); 368 sbGeoJSON.append(" \"crs\": {"); 369 sbGeoJSON.append(" \"type\": \"EPSG\","); 370 sbGeoJSON.append(" \"properties\": {"); 371 sbGeoJSON.append(" \"code\": \"4326\""); 372 sbGeoJSON.append(" }"); 373 sbGeoJSON.append(" }"); 374 //sbGeoJSON.append(", \"bbox\": ["); 375 //sbGeoJSON.append(" ").append(bbox[0][0]).append(",").append(bbox[0][1]).append(",").append(bbox[1][0]).append(",").append(bbox[1][1]); 376 //sbGeoJSON.append(" ]"); 377 sbGeoJSON.append("}"); 378 379 /* write samples to a file */ 380 try { 381 File temporary_file = java.io.File.createTempFile("filter_sample", ".csv", outputpath); 382 FileWriter fw = new FileWriter(temporary_file); 383 384 fw.write(sbGeoJSON.toString()); 385 386 fw.close(); 387 388 return temporary_file.getName(); //return location of temp file 389 390 } catch (Exception e) { 391 SpatialLogger.log("SamplingService: getLSIDAsGeoJSON()", e.toString()); 392 e.printStackTrace(); 393 } 394 return ""; 395 396 } 397 398 /** 399 * creates a file with geojson for lsid at outputpath 400 * 401 * returns filename (first line), number of parts (2nd line) 402 * 403 * @param lsid 404 * @param outputpath 405 * @return 406 */ 407 public static String getLSIDAsGeoJSONIntoParts(String lsid, File outputpath) { 408 if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) { 409 return getLSIDAsGeoJSONIntoParts(lsid, outputpath); 410 } 411 412 int i; 413 414 /* get samples records from records indexes */ 415 String[][] samples = (new SamplingService()).sampleSpecies(lsid, null, null, null, TabulationSettings.MAX_RECORD_COUNT_DOWNLOAD, null); 416 417 int max_parts_size = 2000; 418 419 int count = 0; 420 421 //-1 on samples.length for header 422 int partCount = (int) Math.ceil((samples.length - 1) / (double) max_parts_size); 423 424 //test for filename, return if it exists 425 File file; 426 String filename = outputpath + File.separator + lsid.replace(":", "_").replace(".", "_"); 427 try { 428 file = new File(filename + "_" + (partCount - 1)); 429 if (file.exists()) { 430 return lsid.replace(":", "_").replace(".", "_") + "\n" + partCount; 431 } 432 } catch (Exception e) { 433 e.printStackTrace(); 434 } 435 436 for (int j = 1; j < samples.length; j += max_parts_size) { 437 438 StringBuffer sbGeoJSON = new StringBuffer(); 439 sbGeoJSON.append("{"); 440 sbGeoJSON.append("\"type\": \"FeatureCollection\","); 441 sbGeoJSON.append("\"features\": ["); 442 int len = j + max_parts_size; 443 if (len > samples.length) { 444 len = samples.length; 445 } 446 for (i = j; i < len; i++) { 447 String s = getRecordAsGeoJSON(samples, i); 448 if (s != null) { 449 sbGeoJSON.append(s); 450 if (i < len - 1) { 451 sbGeoJSON.append(","); 452 } 453 } 454 } 455 sbGeoJSON.append("],"); 456 sbGeoJSON.append("\"crs\": {"); 457 sbGeoJSON.append("\"type\": \"EPSG\","); 458 sbGeoJSON.append("\"properties\": {"); 459 sbGeoJSON.append("\"code\": \"4326\""); 460 sbGeoJSON.append("}"); 461 sbGeoJSON.append("}"); 462 sbGeoJSON.append("}"); 463 464 /* write samples to a file */ 465 try { 466 //File temporary_file = java.io.File.createTempFile("filter_sample", ".csv", outputpath); 467 FileWriter fw = new FileWriter( 468 filename + "_" + count); 469 count++; 470 471 fw.write(sbGeoJSON.toString()); 472 473 fw.close(); 474 475 //return temporary_file.getName(); //return location of temp file 476 477 } catch (Exception e) { 478 SpatialLogger.log("SamplingService: getLSIDAsGeoJSON()", e.toString()); 479 e.printStackTrace(); 480 } 481 } 482 return lsid.replace(":", "_").replace(".", "_") + "\n" + partCount; 483 484 } 485 486 private static String getRecordAsGeoJSON(String[][] rec, int rw) { 487 //String[] recdata = rec.split(","); 488 489 if (rec == null || rec.length <= rw || rec[rw].length <= TabulationSettings.geojson_latitude) { 490 return null; 491 } 492 493 for (int i = 0; i < TabulationSettings.geojson_latitude; i++) { 494 if (rec[rw][i] == null) { 495 return null; 496 } 497 } 498 499 StringBuffer sbRec = new StringBuffer(); 500 sbRec.append("{"); 501 sbRec.append(" \"type\":\"Feature\","); 502 sbRec.append(" \"id\":\"occurrences.data.").append(rec[rw][TabulationSettings.geojson_id]).append("\","); 503 sbRec.append(" \"geometry\":{"); 504 sbRec.append(" \"type\":\"Point\","); 505 sbRec.append(" \"coordinates\":[\"").append(rec[rw][TabulationSettings.geojson_longitude]).append("\",\"").append(rec[rw][TabulationSettings.geojson_latitude].trim()).append("\"]"); 506 sbRec.append(" },"); 507 sbRec.append(" \"geometry_name\":\"the_geom\","); 508 sbRec.append(" \"properties\":{"); 509 for (int i = 0; i < TabulationSettings.geojson_property_names.length; i++) { 510 sbRec.append(" \"").append(TabulationSettings.geojson_property_names[i]).append("\":\"").append(rec[rw][TabulationSettings.geojson_property_fields[i]]).append("\""); 511 if (i < TabulationSettings.geojson_property_names.length - 1) { 512 sbRec.append(","); 513 } 514 } 515 sbRec.append(" }"); 516 sbRec.append("}"); 517 518 return sbRec.toString(); 519 520 } 521 522 private String[][] sampleSpeciesSmall(String filter, String[] layers, SimpleRegion region, ArrayList<Integer> records, int max_rows, AnalysisJobSampling job) { 523 IndexedRecord[] ir = null;// OccurrencesIndex.filterSpeciesRecords(filter); 524 525 if (ir != null && ir.length > 0) { 526 if (records != null) { 527 java.util.Collections.sort(records); 528 } 529 530 /* get points */ 531 double[] points = null;//OccurrencesIndex.getPoints(ir[0].record_start, ir[0].record_end); 532 533 /* test for region absence */ 534 int i; 535 536 int alen = 0; 537 int[] a = new int[max_rows]; 538 539 int recordsPos = 0; //for test on records 540 541 /* return all valid points within the region */ 542 for (i = 0; i < points.length && alen < max_rows; i += 2) { 543 int currentRecord = (i / 2) + ir[0].record_start; 544 545 //do not add if does not intersect with records list 546 if (records != null) { 547 //increment recordsPos as required 548 while (recordsPos < records.size() 549 && records.get(recordsPos).intValue() < currentRecord) { 550 recordsPos++; 551 } 552 //test for intersect 553 if (recordsPos >= records.size() 554 || currentRecord != records.get(recordsPos).intValue()) { 555 continue; 556 } 557 } 558 //region test 559 if (region == null || region.isWithin(points[i], points[i + 1])) { 560 a[alen++] = currentRecord; 561 } 562 } 563 564 if (alen == 0) { 565 return null; 566 } 567 568 //filled a up to alen, get the data 569 if (alen < max_rows) { 570 a = java.util.Arrays.copyOf(a, alen); 571 } 572 String[] oi = null; 573 ;//OccurrencesIndex.getSortedRecords(a); 574 575 int layerscount = (layers == null) ? 0 : layers.length; 576 int headercount = oi[0].split(",").length; 577 String[][] output = new String[oi.length + 1][headercount + layerscount];//+1 for header 578 579 //fill 580 for (i = 0; i < oi.length; i++) { 581 String[] line = oi[i].split(","); 582 for (int j = 0; j < line.length && j < headercount; j++) { 583 output[i + 1][j] = line[j]; //+1 for header 584 } 585 } 586 587 for (i = 0; layers != null && i < layers.length; i++) { 588 String[] si = null;//SamplingIndex.getRecords(layers[i], a); 589 if (si != null) { 590 for (int j = 0; j < si.length && j < output.length; j++) { 591 output[j][headercount + i] = si[j]; 592 } 593 } 594 } 595 596 //header 597 OccurrencesFieldsUtil ofu = new OccurrencesFieldsUtil(); 598 i = 0; 599 for (String s : ofu.getOutputColumnNames()) { 600 output[0][i++] = s.trim(); 601 } 602 if (layers != null) { 603 for (String l : layers) { 604 output[0][i++] = Layers.layerNameToDisplayName(l).trim(); 605 } 606 } 607 608 return output; 609 } 610 611 return null; 612 } 613}