/webportal/src/main/java/org/ala/spatial/data/UploadQuery.java

http://alageospatialportal.googlecode.com/ · Java · 669 lines · 495 code · 86 blank · 88 comment · 101 complexity · e651f83644ecb49dcae340e42d3e0ac8 MD5 · raw file

  1. /*
  2. * To change this template, choose Tools | Templates
  3. * and open the template in the editor.
  4. */
  5. package org.ala.spatial.data;
  6. import java.io.ByteArrayOutputStream;
  7. import java.io.Serializable;
  8. import java.util.ArrayList;
  9. import java.util.HashSet;
  10. import java.util.List;
  11. import java.util.logging.Level;
  12. import java.util.logging.Logger;
  13. import java.util.zip.ZipEntry;
  14. import java.util.zip.ZipOutputStream;
  15. import org.ala.spatial.sampling.Sampling;
  16. import org.ala.spatial.sampling.SimpleRegion;
  17. import org.ala.spatial.sampling.SimpleShapeFile;
  18. import org.ala.spatial.util.CommonData;
  19. import org.ala.spatial.wms.RecordsLookup;
  20. /**
  21. *
  22. * @author Adam
  23. */
  24. public class UploadQuery implements Query, Serializable {
  25. ArrayList<QueryField> data;
  26. double[] points;
  27. String name;
  28. String uniqueId;
  29. String metadata;
  30. int originalFieldCount;
  31. //for history
  32. String wkt = "";
  33. ArrayList<Facet> facets = new ArrayList<Facet>();
  34. HashSet<String> flaggedRecords = new HashSet<String>();
  35. /**
  36. *
  37. * @param uniqueId
  38. * @param name
  39. * @param points
  40. * @param fields uploaded column data in ArrayList<QueryField>. First
  41. * three fields must contain record id, longitude, latitude.
  42. */
  43. public UploadQuery(String uniqueId, String name, double[] points, ArrayList<QueryField> fields, String metadata) {
  44. this.points = points;
  45. this.data = fields;
  46. this.name = name;
  47. this.uniqueId = uniqueId;
  48. this.metadata = metadata;
  49. this.originalFieldCount = fields.size();
  50. }
  51. public void resetOriginalFieldCount(int count) {
  52. if (count == -1) {
  53. this.originalFieldCount = data.size();
  54. } else {
  55. this.originalFieldCount = count;
  56. }
  57. }
  58. /**
  59. * Get records for this query for the provided fields.
  60. *
  61. * @param fields QueryFields to return in the sample.
  62. * @return records as String in JSON format.
  63. */
  64. @Override
  65. public String sample(ArrayList<QueryField> fields) {
  66. StringBuilder sb = new StringBuilder();
  67. //do sampling
  68. layerSampling(fields);
  69. //identify fields to data for return
  70. int[] validFields = null;
  71. if (fields == null) {
  72. validFields = new int[data.size()];
  73. for (int i = 0; i < data.size(); i++) {
  74. validFields[i] = i;
  75. }
  76. } else {
  77. validFields = new int[fields.size()];
  78. for (int i = 0; i < fields.size(); i++) {
  79. validFields[i] = -1;
  80. for (int j = 0; j < data.size(); j++) {
  81. if (fields.get(i) != null
  82. && fields.get(i).getName() != null
  83. && data.get(j).getName().equals(fields.get(i).getName())) {
  84. validFields[i] = j;
  85. break;
  86. }
  87. }
  88. }
  89. }
  90. //header
  91. for (int i = 0; i < validFields.length; i++) {
  92. if (i > 0) {
  93. sb.append(",");
  94. }
  95. if (validFields[i] >= 0) {
  96. sb.append(data.get(validFields[i]).getDisplayName());
  97. } else {
  98. sb.append(fields.get(i).getDisplayName());
  99. }
  100. }
  101. //data
  102. int recordCount = points.length / 2;
  103. for (int j = 0; j < recordCount; j++) {
  104. sb.append("\n");
  105. for (int i = 0; i < validFields.length; i++) {
  106. if (i > 0) {
  107. sb.append(",");
  108. }
  109. if (validFields[i] >= 0 && data.get(validFields[i]).getAsString(j) != null) {
  110. sb.append("\"");
  111. sb.append(String.valueOf(data.get(validFields[i]).getAsString(j)).replace("\"", "\"\""));
  112. sb.append("\"");
  113. } else {
  114. sb.append("n/a");
  115. }
  116. }
  117. }
  118. return sb.toString();
  119. }
  120. /**
  121. * Get species list for this query.
  122. *
  123. * @return species list as String containing JSON array.
  124. */
  125. @Override
  126. public String speciesList() {
  127. return null;
  128. }
  129. public String endemicSpeciesList(){
  130. throw new UnsupportedOperationException();
  131. }
  132. /**
  133. * Get number of occurrences in this query.
  134. *
  135. * @return number of occurrences as int or -1 on error.
  136. */
  137. @Override
  138. public int getOccurrenceCount() {
  139. return points.length / 2;
  140. }
  141. /**
  142. * Get number of species in this query.
  143. *
  144. * @return number of species as int or -1 on error.
  145. */
  146. @Override
  147. public int getSpeciesCount() {
  148. return 1;
  149. }
  150. public int getEndemicSpeciesCount(){
  151. throw new UnsupportedOperationException();
  152. }
  153. /**
  154. * Get parsed coordinates and optional fields for this query.
  155. *
  156. * @param fields QueryFields to return in the sample as ArrayList<QueryField>.
  157. * If a QueryField isStored() it will be populated with the field data.
  158. * @return coordinates as double [] like [lng, lat, lng, lat, ...].
  159. */
  160. @Override
  161. public double[] getPoints(ArrayList<QueryField> fields) {
  162. if (fields != null) {
  163. for (int i = 0; i < fields.size(); i++) {
  164. for (int j = 0; j < data.size(); j++) {
  165. if (fields.get(i).getName().equals(data.get(j).getName())) {
  166. fields.get(i).copyData(data.get(j));
  167. break;
  168. }
  169. }
  170. }
  171. }
  172. return points;
  173. }
  174. @Override
  175. public String getName() {
  176. return name;
  177. }
  178. @Override
  179. public String getRank() {
  180. return "species";
  181. }
  182. @Override
  183. public String getQ() {
  184. return uniqueId;
  185. }
  186. @Override
  187. public String getFullQ(boolean encode) {
  188. StringBuilder sb = new StringBuilder();
  189. sb.append(metadata).append("\n");
  190. if (wkt.length() > 0) {
  191. sb.append(wkt).append("\n");
  192. }
  193. for (int i = 0; i < facets.size(); i++) {
  194. sb.append(facets.get(i).toString()).append("\n");
  195. }
  196. return sb.toString();
  197. }
  198. @Override
  199. public Query newWkt(String wkt, boolean forMapping) {
  200. if (wkt == null || wkt.equals(CommonData.WORLD_WKT)) {
  201. return this;
  202. }
  203. SimpleRegion sr = SimpleShapeFile.parseWKT(wkt);
  204. //per record test
  205. boolean[] valid = new boolean[points.length / 2];
  206. int count = 0;
  207. for (int i = 0; i < valid.length; i++) {
  208. valid[i] = sr.isWithin(points[i * 2], points[i * 2 + 1]);
  209. if (valid[i]) {
  210. count++;
  211. }
  212. }
  213. UploadQuery q = newFromValidMapping(valid, count);
  214. //maintain wkt history
  215. if (this.wkt.length() > 0) {
  216. q.wkt = this.wkt + " AND " + wkt;
  217. } else {
  218. q.wkt = wkt;
  219. }
  220. return q;
  221. }
  222. UploadQuery newFromValidMapping(boolean[] valid, int count) {
  223. //copy original data
  224. ArrayList<QueryField> facetData = new ArrayList<QueryField>(originalFieldCount);
  225. for (int i = 0; i < originalFieldCount; i++) {
  226. QueryField qf = new QueryField(data.get(i).getName(), data.get(i).getDisplayName(), data.get(i).getFieldType());
  227. qf.ensureCapacity(count);
  228. for (int j = 0; j < valid.length; j++) {
  229. if (valid[j]) {
  230. qf.add(data.get(i).getAsString(j));
  231. }
  232. }
  233. qf.store();
  234. facetData.add(qf);
  235. }
  236. //copy points
  237. double[] facetPoints = new double[count * 2];
  238. int pos = 0;
  239. for (int j = 0; j < valid.length; j++) {
  240. if (valid[j]) {
  241. facetPoints[pos * 2] = points[j * 2];
  242. facetPoints[pos * 2 + 1] = points[j * 2 + 1];
  243. pos++;
  244. }
  245. }
  246. String uid = String.valueOf(System.currentTimeMillis());
  247. RecordsLookup.putData(uid, facetPoints, facetData, metadata);
  248. return new UploadQuery(uid, name, facetPoints, facetData, metadata);
  249. }
  250. @Override
  251. public Query newFacet(Facet facet, boolean forMapping) {
  252. ArrayList<Facet> facets = new ArrayList<Facet>();
  253. facets.add(facet);
  254. return newFacets(facets, forMapping);
  255. }
  256. @Override
  257. public String getWMSpath() {
  258. throw new UnsupportedOperationException("Not supported yet.");
  259. }
  260. @Override
  261. public ArrayList<QueryField> getFacetFieldList() {
  262. ArrayList<QueryField> fields = new ArrayList<QueryField>();
  263. for (int i = 1; i < data.size(); i++) {
  264. fields.add(data.get(i));
  265. }
  266. return fields;
  267. }
  268. /**
  269. * Do sampling on layers that are in fields but not in data.
  270. *
  271. * @param fields
  272. */
  273. private void layerSampling(ArrayList<QueryField> fields) {
  274. if (fields == null) {
  275. return;
  276. }
  277. ArrayList<QueryField> forSampling = new ArrayList<QueryField>();
  278. for (int i = 0; i < fields.size(); i++) {
  279. boolean found = false;
  280. for (int j = 0; j < data.size(); j++) {
  281. if (data.get(j).getName().equals(fields.get(i).getName())) {
  282. found = true;
  283. break;
  284. }
  285. }
  286. if (!found) {
  287. forSampling.add(fields.get(i));
  288. }
  289. }
  290. //do sampling
  291. double[][] coordinates = new double[points.length / 2][2];
  292. for (int i = 0; i < points.length; i += 2) {
  293. coordinates[i / 2][0] = points[i];
  294. coordinates[i / 2][1] = points[i + 1];
  295. }
  296. ArrayList<String> facetIds = new ArrayList<String>();
  297. for (int i = 0; i < forSampling.size(); i++) {
  298. facetIds.add(forSampling.get(i).getName());
  299. }
  300. ArrayList<String[]> output = Sampling.sampling(facetIds, coordinates);
  301. for (int i = 0; i < forSampling.size(); i++) {
  302. String[] s = output.get(i);
  303. QueryField qf = forSampling.get(i);
  304. if (qf.getName() != null) {
  305. qf.ensureCapacity(s.length);
  306. for (int j = 0; j < s.length; j++) {
  307. qf.add(s[j]);
  308. }
  309. qf.store();
  310. data.add(qf);
  311. }
  312. }
  313. //update RecordsLookup for new fields, if it is already in RecordsLookup
  314. Object[] recordData = (Object[]) RecordsLookup.getData(getQ());
  315. if (recordData != null) {
  316. double[] points = (double[]) recordData[0];
  317. RecordsLookup.putData(getQ(), points, data, metadata);
  318. }
  319. }
  320. @Override
  321. public String getSpeciesIdFieldName() {
  322. return null;
  323. }
  324. @Override
  325. public String getRecordIdFieldName() {
  326. return data.get(0).getName();
  327. }
  328. @Override
  329. public String getRecordLongitudeFieldName() {
  330. return data.get(1).getName();
  331. }
  332. @Override
  333. public String getRecordLatitudeFieldName() {
  334. return data.get(2).getName();
  335. }
  336. @Override
  337. public String getRecordIdFieldDisplayName() {
  338. return data.get(0).getDisplayName();
  339. }
  340. @Override
  341. public String getRecordLongitudeFieldDisplayName() {
  342. return data.get(1).getDisplayName();
  343. }
  344. @Override
  345. public String getRecordLatitudeFieldDisplayName() {
  346. return data.get(2).getDisplayName();
  347. }
  348. @Override
  349. public LegendObject getLegend(String colourmode) {
  350. //get existing
  351. for (int i = 0; i < data.size(); i++) {
  352. if (data.get(i).getName().equals(colourmode)) {
  353. return data.get(i).getLegend();
  354. }
  355. }
  356. //create
  357. ArrayList<QueryField> fields = new ArrayList<QueryField>();
  358. QueryField qf = new QueryField(colourmode, CommonData.getFacetLayerDisplayName(colourmode), QueryField.FieldType.AUTO);
  359. qf.setStored(true);
  360. fields.add(qf);
  361. layerSampling(fields);
  362. //get existing
  363. for (int i = 0; i < data.size(); i++) {
  364. if (data.get(i).getName().equals(colourmode)) {
  365. return data.get(i).getLegend();
  366. }
  367. }
  368. //fail
  369. return null;
  370. }
  371. @Override
  372. public Query newFacets(List<Facet> facet, boolean forMapping) {
  373. //copy all the data through the list of facets (AND)
  374. //setup
  375. ArrayList<List<QueryField>> facetFields = new ArrayList<List<QueryField>>();
  376. for (int k = 0; k < facet.size(); k++) {
  377. Facet f = facet.get(k);
  378. String[] fields = f.getFields();
  379. List<QueryField> qf = new ArrayList<QueryField>();
  380. for (int j = 0; j < fields.length; j++) {
  381. int i = 0;
  382. for (i = 0; i < data.size(); i++) {
  383. if (fields[j].equals(data.get(i).getName())) {
  384. qf.add(data.get(i));
  385. break;
  386. }
  387. }
  388. }
  389. facetFields.add(qf);
  390. }
  391. //per record test
  392. boolean[] valid = new boolean[points.length / 2];
  393. int count = 0;
  394. for (int i = 0; i < valid.length; i++) {
  395. int sum = 0;
  396. for (int j = 0; j < facet.size(); j++) {
  397. if (facet.get(j).isValid(facetFields.get(j), i)) {
  398. sum++;
  399. }
  400. }
  401. valid[i] = sum == facet.size();
  402. if (valid[i]) {
  403. count++;
  404. }
  405. }
  406. UploadQuery uq = newFromValidMapping(valid, count);
  407. //maintain facet history
  408. uq.facets.addAll(this.facets);
  409. uq.facets.addAll(facet);
  410. return uq;
  411. }
  412. @Override
  413. public String getUrl() {
  414. return CommonData.webportalServer + "/ws/wms/reflect?";
  415. }
  416. List<Double> bbox = null;
  417. @Override
  418. public List<Double> getBBox() {
  419. if (bbox == null) {
  420. bbox = new ArrayList<Double>();
  421. double minx, miny, maxx, maxy;
  422. minx = points[0];
  423. maxx = points[0];
  424. miny = points[1];
  425. maxy = points[1];
  426. for (int i = 0; i < points.length; i += 2) {
  427. if (minx > points[i]) {
  428. minx = points[i];
  429. }
  430. if (maxx < points[i]) {
  431. maxx = points[i];
  432. }
  433. if (miny > points[i + 1]) {
  434. miny = points[i + 1];
  435. }
  436. if (maxy < points[i + 1]) {
  437. maxy = points[i + 1];
  438. }
  439. }
  440. bbox.add(minx);
  441. bbox.add(miny);
  442. bbox.add(maxx);
  443. bbox.add(maxy);
  444. }
  445. return bbox;
  446. }
  447. @Override
  448. public String getMetadataHtml() {
  449. String[] m = metadata.replace("<br />", "").split("\n");
  450. String name = m[1].substring(m[1].indexOf(':') + 1).trim();
  451. String description = m[2].substring(m[2].indexOf(':') + 1).trim();
  452. String date = m[3].substring(m[3].indexOf(':') + 1).trim();
  453. StringBuilder fieldsList = new StringBuilder();
  454. for (int i = 0; i < data.size(); i++) {
  455. if (i > 0) {
  456. fieldsList.append(", ");
  457. }
  458. fieldsList.append(data.get(i).getDisplayName());
  459. }
  460. String html = "User uploaded points\n";
  461. html += "<table class='md_table'>";
  462. html += "<tr class='md_grey-bg'><td class='md_th'>Name: </td><td class='md_spacer'/><td class='md_value'>" + name + "</td></tr>";
  463. html += "<tr><td class='md_th'>Description: </td><td class='md_spacer'/><td class='md_value'>" + description + "</td></tr>";
  464. html += "<tr class='md_grey-bg'><td class='md_th'>Date: </td><td class='md_spacer'/><td class='md_value'>" + date + "</td></tr>";
  465. html += "<tr><td class='md_th'>Number of coordinates: </td><td class='md_spacer'/><td class='md_value'>" + getOccurrenceCount() + "</td></tr>";
  466. html += "<tr class='md_grey-bg'><td class='md_th'>Number of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + data.size() + "</td></tr>";
  467. html += "<tr><td class='md_th'>List of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + fieldsList.toString() + "</td></tr>";
  468. html += "</table>";
  469. return html;
  470. }
  471. @Override
  472. public String getDownloadUrl(String[] extraFields) {
  473. return null;
  474. }
  475. @Override
  476. public byte[] getDownloadBytes(String[] extraFields, String[] displayNames) {
  477. ArrayList<QueryField> fields = new ArrayList<QueryField>();
  478. if (getFacetFieldList() != null) {
  479. fields.add(data.get(0)); //id column (1st) is not in getFacetFieldList()
  480. fields.addAll(getFacetFieldList());
  481. }
  482. if (extraFields != null && extraFields.length > 0) {
  483. for (int i = 0; i < extraFields.length; i++) {
  484. String s = extraFields[i];
  485. if (displayNames == null) {
  486. fields.add(new QueryField(s, CommonData.getFacetLayerDisplayName(s), QueryField.FieldType.AUTO));
  487. } else {
  488. fields.add(new QueryField(s, displayNames[i], QueryField.FieldType.AUTO));
  489. }
  490. }
  491. }
  492. try {
  493. //zip it
  494. ByteArrayOutputStream bos = new ByteArrayOutputStream();
  495. ZipOutputStream zos = new ZipOutputStream(bos);
  496. ZipEntry ze = new ZipEntry(getName() + ".csv");
  497. zos.putNextEntry(ze);
  498. zos.write(sample(fields).getBytes("UTF-8"));
  499. zos.close();
  500. return bos.toByteArray();
  501. } catch (Exception ex) {
  502. Logger.getLogger(UploadQuery.class.getName()).log(Level.SEVERE, null, ex);
  503. }
  504. return null;
  505. }
  506. @Override
  507. public String getQc() {
  508. return null;
  509. }
  510. @Override
  511. public void setQc(String qc) {
  512. }
  513. /**
  514. * Get the column header name for a column in the output of sampling.
  515. *
  516. * @param colourMode
  517. * @return
  518. */
  519. @Override
  520. public String getRecordFieldDisplayName(String colourMode) {
  521. for (int i = 0; i < data.size(); i++) {
  522. if (data.get(i).getName().equals(colourMode)) {
  523. return data.get(i).getDisplayName();
  524. }
  525. }
  526. return colourMode;
  527. }
  528. /**
  529. * Add or remove one record to an internal group.
  530. */
  531. @Override
  532. public void flagRecord(String id, boolean set) {
  533. if (set) {
  534. flaggedRecords.add(id);
  535. } else {
  536. flaggedRecords.remove(id);
  537. }
  538. }
  539. /**
  540. * Get the number of flagged records.
  541. */
  542. @Override
  543. public int flagRecordCount() {
  544. return flaggedRecords.size();
  545. }
  546. /**
  547. * Get the list of flagged records as '\n' separated String.
  548. */
  549. @Override
  550. public String getFlaggedRecords() {
  551. StringBuilder sb = new StringBuilder();
  552. for (String s : flaggedRecords) {
  553. if (sb.length() > 0) {
  554. sb.append("\n");
  555. }
  556. sb.append(s);
  557. }
  558. return sb.toString();
  559. }
  560. /**
  561. * Create a new Query that contains only or excludes all flagged records
  562. */
  563. @Override
  564. public Query newFlaggedRecords(boolean includeOnlyFlaggedRecords) {
  565. if (flagRecordCount() == 0) {
  566. return this;
  567. }
  568. //per record test
  569. boolean[] valid = new boolean[points.length / 2];
  570. int count = 0;
  571. for (int i = 0; i < valid.length; i++) {
  572. valid[i] = includeOnlyFlaggedRecords != flaggedRecords.contains(data.get(0).getString(i));
  573. if (valid[i]) {
  574. count++;
  575. }
  576. }
  577. return newFromValidMapping(valid, count);
  578. }
  579. @Override
  580. public String getAutoComplete(String facet, String value, int limit) {
  581. throw new UnsupportedOperationException("Upload Query does not support the ability to perform an autocomplete.");
  582. }
  583. }