/webportal/src/main/java/org/ala/spatial/data/UploadQuery.java
Java | 669 lines | 495 code | 86 blank | 88 comment | 101 complexity | e651f83644ecb49dcae340e42d3e0ac8 MD5 | raw file
1/* 2 * To change this template, choose Tools | Templates 3 * and open the template in the editor. 4 */ 5package org.ala.spatial.data; 6 7import java.io.ByteArrayOutputStream; 8import java.io.Serializable; 9import java.util.ArrayList; 10import java.util.HashSet; 11import java.util.List; 12import java.util.logging.Level; 13import java.util.logging.Logger; 14import java.util.zip.ZipEntry; 15import java.util.zip.ZipOutputStream; 16import org.ala.spatial.sampling.Sampling; 17import org.ala.spatial.sampling.SimpleRegion; 18import org.ala.spatial.sampling.SimpleShapeFile; 19import org.ala.spatial.util.CommonData; 20import org.ala.spatial.wms.RecordsLookup; 21 22/** 23 * 24 * @author Adam 25 */ 26public class UploadQuery implements Query, Serializable { 27 28 ArrayList<QueryField> data; 29 double[] points; 30 String name; 31 String uniqueId; 32 String metadata; 33 int originalFieldCount; 34 //for history 35 String wkt = ""; 36 ArrayList<Facet> facets = new ArrayList<Facet>(); 37 HashSet<String> flaggedRecords = new HashSet<String>(); 38 39 /** 40 * 41 * @param uniqueId 42 * @param name 43 * @param points 44 * @param fields uploaded column data in ArrayList<QueryField>. First 45 * three fields must contain record id, longitude, latitude. 46 */ 47 public UploadQuery(String uniqueId, String name, double[] points, ArrayList<QueryField> fields, String metadata) { 48 this.points = points; 49 this.data = fields; 50 this.name = name; 51 this.uniqueId = uniqueId; 52 this.metadata = metadata; 53 this.originalFieldCount = fields.size(); 54 } 55 56 public void resetOriginalFieldCount(int count) { 57 if (count == -1) { 58 this.originalFieldCount = data.size(); 59 } else { 60 this.originalFieldCount = count; 61 } 62 } 63 64 /** 65 * Get records for this query for the provided fields. 66 * 67 * @param fields QueryFields to return in the sample. 68 * @return records as String in JSON format. 69 */ 70 @Override 71 public String sample(ArrayList<QueryField> fields) { 72 StringBuilder sb = new StringBuilder(); 73 74 //do sampling 75 layerSampling(fields); 76 77 //identify fields to data for return 78 int[] validFields = null; 79 if (fields == null) { 80 validFields = new int[data.size()]; 81 for (int i = 0; i < data.size(); i++) { 82 validFields[i] = i; 83 } 84 } else { 85 validFields = new int[fields.size()]; 86 for (int i = 0; i < fields.size(); i++) { 87 validFields[i] = -1; 88 for (int j = 0; j < data.size(); j++) { 89 if (fields.get(i) != null 90 && fields.get(i).getName() != null 91 && data.get(j).getName().equals(fields.get(i).getName())) { 92 validFields[i] = j; 93 break; 94 } 95 } 96 } 97 } 98 99 //header 100 for (int i = 0; i < validFields.length; i++) { 101 if (i > 0) { 102 sb.append(","); 103 } 104 if (validFields[i] >= 0) { 105 sb.append(data.get(validFields[i]).getDisplayName()); 106 } else { 107 sb.append(fields.get(i).getDisplayName()); 108 } 109 } 110 111 //data 112 int recordCount = points.length / 2; 113 for (int j = 0; j < recordCount; j++) { 114 sb.append("\n"); 115 for (int i = 0; i < validFields.length; i++) { 116 if (i > 0) { 117 sb.append(","); 118 } 119 if (validFields[i] >= 0 && data.get(validFields[i]).getAsString(j) != null) { 120 sb.append("\""); 121 sb.append(String.valueOf(data.get(validFields[i]).getAsString(j)).replace("\"", "\"\"")); 122 sb.append("\""); 123 } else { 124 sb.append("n/a"); 125 } 126 } 127 } 128 129 return sb.toString(); 130 } 131 132 /** 133 * Get species list for this query. 134 * 135 * @return species list as String containing JSON array. 136 */ 137 @Override 138 public String speciesList() { 139 return null; 140 } 141 142 public String endemicSpeciesList(){ 143 throw new UnsupportedOperationException(); 144 } 145 146 /** 147 * Get number of occurrences in this query. 148 * 149 * @return number of occurrences as int or -1 on error. 150 */ 151 @Override 152 public int getOccurrenceCount() { 153 return points.length / 2; 154 } 155 156 /** 157 * Get number of species in this query. 158 * 159 * @return number of species as int or -1 on error. 160 */ 161 @Override 162 public int getSpeciesCount() { 163 return 1; 164 } 165 166 public int getEndemicSpeciesCount(){ 167 throw new UnsupportedOperationException(); 168 } 169 170 /** 171 * Get parsed coordinates and optional fields for this query. 172 * 173 * @param fields QueryFields to return in the sample as ArrayList<QueryField>. 174 * If a QueryField isStored() it will be populated with the field data. 175 * @return coordinates as double [] like [lng, lat, lng, lat, ...]. 176 */ 177 @Override 178 public double[] getPoints(ArrayList<QueryField> fields) { 179 if (fields != null) { 180 for (int i = 0; i < fields.size(); i++) { 181 for (int j = 0; j < data.size(); j++) { 182 if (fields.get(i).getName().equals(data.get(j).getName())) { 183 fields.get(i).copyData(data.get(j)); 184 break; 185 } 186 } 187 } 188 } 189 190 return points; 191 } 192 193 @Override 194 public String getName() { 195 return name; 196 } 197 198 @Override 199 public String getRank() { 200 return "species"; 201 } 202 203 @Override 204 public String getQ() { 205 return uniqueId; 206 } 207 208 @Override 209 public String getFullQ(boolean encode) { 210 StringBuilder sb = new StringBuilder(); 211 sb.append(metadata).append("\n"); 212 213 if (wkt.length() > 0) { 214 sb.append(wkt).append("\n"); 215 } 216 217 for (int i = 0; i < facets.size(); i++) { 218 sb.append(facets.get(i).toString()).append("\n"); 219 } 220 221 return sb.toString(); 222 } 223 224 @Override 225 public Query newWkt(String wkt, boolean forMapping) { 226 if (wkt == null || wkt.equals(CommonData.WORLD_WKT)) { 227 return this; 228 } 229 230 SimpleRegion sr = SimpleShapeFile.parseWKT(wkt); 231 232 //per record test 233 boolean[] valid = new boolean[points.length / 2]; 234 int count = 0; 235 for (int i = 0; i < valid.length; i++) { 236 valid[i] = sr.isWithin(points[i * 2], points[i * 2 + 1]); 237 if (valid[i]) { 238 count++; 239 } 240 } 241 242 UploadQuery q = newFromValidMapping(valid, count); 243 244 //maintain wkt history 245 if (this.wkt.length() > 0) { 246 q.wkt = this.wkt + " AND " + wkt; 247 } else { 248 q.wkt = wkt; 249 } 250 251 return q; 252 } 253 254 UploadQuery newFromValidMapping(boolean[] valid, int count) { 255 //copy original data 256 ArrayList<QueryField> facetData = new ArrayList<QueryField>(originalFieldCount); 257 for (int i = 0; i < originalFieldCount; i++) { 258 QueryField qf = new QueryField(data.get(i).getName(), data.get(i).getDisplayName(), data.get(i).getFieldType()); 259 qf.ensureCapacity(count); 260 for (int j = 0; j < valid.length; j++) { 261 if (valid[j]) { 262 qf.add(data.get(i).getAsString(j)); 263 } 264 } 265 qf.store(); 266 facetData.add(qf); 267 } 268 269 //copy points 270 double[] facetPoints = new double[count * 2]; 271 int pos = 0; 272 for (int j = 0; j < valid.length; j++) { 273 if (valid[j]) { 274 facetPoints[pos * 2] = points[j * 2]; 275 facetPoints[pos * 2 + 1] = points[j * 2 + 1]; 276 pos++; 277 } 278 } 279 280 281 String uid = String.valueOf(System.currentTimeMillis()); 282 283 RecordsLookup.putData(uid, facetPoints, facetData, metadata); 284 285 return new UploadQuery(uid, name, facetPoints, facetData, metadata); 286 } 287 288 @Override 289 public Query newFacet(Facet facet, boolean forMapping) { 290 ArrayList<Facet> facets = new ArrayList<Facet>(); 291 facets.add(facet); 292 return newFacets(facets, forMapping); 293 } 294 295 @Override 296 public String getWMSpath() { 297 throw new UnsupportedOperationException("Not supported yet."); 298 } 299 300 @Override 301 public ArrayList<QueryField> getFacetFieldList() { 302 ArrayList<QueryField> fields = new ArrayList<QueryField>(); 303 for (int i = 1; i < data.size(); i++) { 304 fields.add(data.get(i)); 305 } 306 return fields; 307 } 308 309 /** 310 * Do sampling on layers that are in fields but not in data. 311 * 312 * @param fields 313 */ 314 private void layerSampling(ArrayList<QueryField> fields) { 315 if (fields == null) { 316 return; 317 } 318 ArrayList<QueryField> forSampling = new ArrayList<QueryField>(); 319 for (int i = 0; i < fields.size(); i++) { 320 boolean found = false; 321 for (int j = 0; j < data.size(); j++) { 322 if (data.get(j).getName().equals(fields.get(i).getName())) { 323 found = true; 324 break; 325 } 326 } 327 if (!found) { 328 forSampling.add(fields.get(i)); 329 } 330 } 331 332 //do sampling 333 double[][] coordinates = new double[points.length / 2][2]; 334 for (int i = 0; i < points.length; i += 2) { 335 coordinates[i / 2][0] = points[i]; 336 coordinates[i / 2][1] = points[i + 1]; 337 } 338 ArrayList<String> facetIds = new ArrayList<String>(); 339 for (int i = 0; i < forSampling.size(); i++) { 340 facetIds.add(forSampling.get(i).getName()); 341 } 342 ArrayList<String[]> output = Sampling.sampling(facetIds, coordinates); 343 344 for (int i = 0; i < forSampling.size(); i++) { 345 String[] s = output.get(i); 346 QueryField qf = forSampling.get(i); 347 if (qf.getName() != null) { 348 qf.ensureCapacity(s.length); 349 for (int j = 0; j < s.length; j++) { 350 qf.add(s[j]); 351 } 352 qf.store(); 353 354 data.add(qf); 355 } 356 } 357 358 //update RecordsLookup for new fields, if it is already in RecordsLookup 359 Object[] recordData = (Object[]) RecordsLookup.getData(getQ()); 360 if (recordData != null) { 361 double[] points = (double[]) recordData[0]; 362 RecordsLookup.putData(getQ(), points, data, metadata); 363 } 364 } 365 366 @Override 367 public String getSpeciesIdFieldName() { 368 return null; 369 } 370 371 @Override 372 public String getRecordIdFieldName() { 373 return data.get(0).getName(); 374 } 375 376 @Override 377 public String getRecordLongitudeFieldName() { 378 return data.get(1).getName(); 379 } 380 381 @Override 382 public String getRecordLatitudeFieldName() { 383 return data.get(2).getName(); 384 } 385 386 @Override 387 public String getRecordIdFieldDisplayName() { 388 return data.get(0).getDisplayName(); 389 } 390 391 @Override 392 public String getRecordLongitudeFieldDisplayName() { 393 return data.get(1).getDisplayName(); 394 } 395 396 @Override 397 public String getRecordLatitudeFieldDisplayName() { 398 return data.get(2).getDisplayName(); 399 } 400 401 @Override 402 public LegendObject getLegend(String colourmode) { 403 //get existing 404 for (int i = 0; i < data.size(); i++) { 405 if (data.get(i).getName().equals(colourmode)) { 406 return data.get(i).getLegend(); 407 } 408 } 409 410 //create 411 ArrayList<QueryField> fields = new ArrayList<QueryField>(); 412 QueryField qf = new QueryField(colourmode, CommonData.getFacetLayerDisplayName(colourmode), QueryField.FieldType.AUTO); 413 qf.setStored(true); 414 fields.add(qf); 415 layerSampling(fields); 416 417 //get existing 418 for (int i = 0; i < data.size(); i++) { 419 if (data.get(i).getName().equals(colourmode)) { 420 return data.get(i).getLegend(); 421 } 422 } 423 424 //fail 425 return null; 426 } 427 428 @Override 429 public Query newFacets(List<Facet> facet, boolean forMapping) { 430 //copy all the data through the list of facets (AND) 431 432 //setup 433 ArrayList<List<QueryField>> facetFields = new ArrayList<List<QueryField>>(); 434 for (int k = 0; k < facet.size(); k++) { 435 Facet f = facet.get(k); 436 String[] fields = f.getFields(); 437 List<QueryField> qf = new ArrayList<QueryField>(); 438 for (int j = 0; j < fields.length; j++) { 439 int i = 0; 440 for (i = 0; i < data.size(); i++) { 441 if (fields[j].equals(data.get(i).getName())) { 442 qf.add(data.get(i)); 443 break; 444 } 445 } 446 } 447 facetFields.add(qf); 448 } 449 450 //per record test 451 boolean[] valid = new boolean[points.length / 2]; 452 int count = 0; 453 for (int i = 0; i < valid.length; i++) { 454 int sum = 0; 455 for (int j = 0; j < facet.size(); j++) { 456 if (facet.get(j).isValid(facetFields.get(j), i)) { 457 sum++; 458 } 459 } 460 461 valid[i] = sum == facet.size(); 462 if (valid[i]) { 463 count++; 464 } 465 } 466 467 UploadQuery uq = newFromValidMapping(valid, count); 468 469 //maintain facet history 470 uq.facets.addAll(this.facets); 471 uq.facets.addAll(facet); 472 473 return uq; 474 } 475 476 @Override 477 public String getUrl() { 478 return CommonData.webportalServer + "/ws/wms/reflect?"; 479 } 480 List<Double> bbox = null; 481 482 @Override 483 public List<Double> getBBox() { 484 if (bbox == null) { 485 bbox = new ArrayList<Double>(); 486 487 double minx, miny, maxx, maxy; 488 minx = points[0]; 489 maxx = points[0]; 490 miny = points[1]; 491 maxy = points[1]; 492 for (int i = 0; i < points.length; i += 2) { 493 if (minx > points[i]) { 494 minx = points[i]; 495 } 496 if (maxx < points[i]) { 497 maxx = points[i]; 498 } 499 if (miny > points[i + 1]) { 500 miny = points[i + 1]; 501 } 502 if (maxy < points[i + 1]) { 503 maxy = points[i + 1]; 504 } 505 } 506 507 bbox.add(minx); 508 bbox.add(miny); 509 bbox.add(maxx); 510 bbox.add(maxy); 511 } 512 513 return bbox; 514 } 515 516 @Override 517 public String getMetadataHtml() { 518 String[] m = metadata.replace("<br />", "").split("\n"); 519 String name = m[1].substring(m[1].indexOf(':') + 1).trim(); 520 String description = m[2].substring(m[2].indexOf(':') + 1).trim(); 521 String date = m[3].substring(m[3].indexOf(':') + 1).trim(); 522 523 StringBuilder fieldsList = new StringBuilder(); 524 for (int i = 0; i < data.size(); i++) { 525 if (i > 0) { 526 fieldsList.append(", "); 527 } 528 fieldsList.append(data.get(i).getDisplayName()); 529 } 530 531 String html = "User uploaded points\n"; 532 html += "<table class='md_table'>"; 533 html += "<tr class='md_grey-bg'><td class='md_th'>Name: </td><td class='md_spacer'/><td class='md_value'>" + name + "</td></tr>"; 534 html += "<tr><td class='md_th'>Description: </td><td class='md_spacer'/><td class='md_value'>" + description + "</td></tr>"; 535 html += "<tr class='md_grey-bg'><td class='md_th'>Date: </td><td class='md_spacer'/><td class='md_value'>" + date + "</td></tr>"; 536 html += "<tr><td class='md_th'>Number of coordinates: </td><td class='md_spacer'/><td class='md_value'>" + getOccurrenceCount() + "</td></tr>"; 537 html += "<tr class='md_grey-bg'><td class='md_th'>Number of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + data.size() + "</td></tr>"; 538 html += "<tr><td class='md_th'>List of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + fieldsList.toString() + "</td></tr>"; 539 html += "</table>"; 540 541 return html; 542 } 543 544 @Override 545 public String getDownloadUrl(String[] extraFields) { 546 return null; 547 } 548 549 @Override 550 public byte[] getDownloadBytes(String[] extraFields, String[] displayNames) { 551 ArrayList<QueryField> fields = new ArrayList<QueryField>(); 552 if (getFacetFieldList() != null) { 553 fields.add(data.get(0)); //id column (1st) is not in getFacetFieldList() 554 fields.addAll(getFacetFieldList()); 555 } 556 if (extraFields != null && extraFields.length > 0) { 557 for (int i = 0; i < extraFields.length; i++) { 558 String s = extraFields[i]; 559 if (displayNames == null) { 560 fields.add(new QueryField(s, CommonData.getFacetLayerDisplayName(s), QueryField.FieldType.AUTO)); 561 } else { 562 fields.add(new QueryField(s, displayNames[i], QueryField.FieldType.AUTO)); 563 } 564 } 565 } 566 try { 567 //zip it 568 ByteArrayOutputStream bos = new ByteArrayOutputStream(); 569 ZipOutputStream zos = new ZipOutputStream(bos); 570 ZipEntry ze = new ZipEntry(getName() + ".csv"); 571 zos.putNextEntry(ze); 572 zos.write(sample(fields).getBytes("UTF-8")); 573 zos.close(); 574 575 return bos.toByteArray(); 576 } catch (Exception ex) { 577 Logger.getLogger(UploadQuery.class.getName()).log(Level.SEVERE, null, ex); 578 } 579 return null; 580 } 581 582 @Override 583 public String getQc() { 584 return null; 585 } 586 587 @Override 588 public void setQc(String qc) { 589 } 590 591 /** 592 * Get the column header name for a column in the output of sampling. 593 * 594 * @param colourMode 595 * @return 596 */ 597 @Override 598 public String getRecordFieldDisplayName(String colourMode) { 599 for (int i = 0; i < data.size(); i++) { 600 if (data.get(i).getName().equals(colourMode)) { 601 return data.get(i).getDisplayName(); 602 } 603 } 604 return colourMode; 605 } 606 607 /** 608 * Add or remove one record to an internal group. 609 */ 610 @Override 611 public void flagRecord(String id, boolean set) { 612 if (set) { 613 flaggedRecords.add(id); 614 } else { 615 flaggedRecords.remove(id); 616 } 617 } 618 619 /** 620 * Get the number of flagged records. 621 */ 622 @Override 623 public int flagRecordCount() { 624 return flaggedRecords.size(); 625 } 626 627 /** 628 * Get the list of flagged records as '\n' separated String. 629 */ 630 @Override 631 public String getFlaggedRecords() { 632 StringBuilder sb = new StringBuilder(); 633 for (String s : flaggedRecords) { 634 if (sb.length() > 0) { 635 sb.append("\n"); 636 } 637 sb.append(s); 638 } 639 return sb.toString(); 640 } 641 642 /** 643 * Create a new Query that contains only or excludes all flagged records 644 */ 645 @Override 646 public Query newFlaggedRecords(boolean includeOnlyFlaggedRecords) { 647 if (flagRecordCount() == 0) { 648 return this; 649 } 650 651 //per record test 652 boolean[] valid = new boolean[points.length / 2]; 653 int count = 0; 654 for (int i = 0; i < valid.length; i++) { 655 valid[i] = includeOnlyFlaggedRecords != flaggedRecords.contains(data.get(0).getString(i)); 656 657 if (valid[i]) { 658 count++; 659 } 660 } 661 662 return newFromValidMapping(valid, count); 663 } 664 665 @Override 666 public String getAutoComplete(String facet, String value, int limit) { 667 throw new UnsupportedOperationException("Upload Query does not support the ability to perform an autocomplete."); 668 } 669}