/webportal/src/main/java/org/ala/spatial/data/UploadQuery.java

  1/*
  2 * To change this template, choose Tools | Templates
  3 * and open the template in the editor.
  4 */
  5package org.ala.spatial.data;
  6
  7import java.io.ByteArrayOutputStream;
  8import java.io.Serializable;
  9import java.util.ArrayList;
 10import java.util.HashSet;
 11import java.util.List;
 12import java.util.logging.Level;
 13import java.util.logging.Logger;
 14import java.util.zip.ZipEntry;
 15import java.util.zip.ZipOutputStream;
 16import org.ala.spatial.sampling.Sampling;
 17import org.ala.spatial.sampling.SimpleRegion;
 18import org.ala.spatial.sampling.SimpleShapeFile;
 19import org.ala.spatial.util.CommonData;
 20import org.ala.spatial.wms.RecordsLookup;
 21
 22/**
 23 *
 24 * @author Adam
 25 */
 26public class UploadQuery implements Query, Serializable {
 27
 28    ArrayList<QueryField> data;
 29    double[] points;
 30    String name;
 31    String uniqueId;
 32    String metadata;
 33    int originalFieldCount;
 34    //for history
 35    String wkt = "";
 36    ArrayList<Facet> facets = new ArrayList<Facet>();
 37    HashSet<String> flaggedRecords = new HashSet<String>();
 38
 39    /**
 40     *
 41     * @param uniqueId
 42     * @param name
 43     * @param points
 44     * @param fields uploaded column data in ArrayList<QueryField>.  First
 45     * three fields must contain record id, longitude, latitude.
 46     */
 47    public UploadQuery(String uniqueId, String name, double[] points, ArrayList<QueryField> fields, String metadata) {
 48        this.points = points;
 49        this.data = fields;
 50        this.name = name;
 51        this.uniqueId = uniqueId;
 52        this.metadata = metadata;
 53        this.originalFieldCount = fields.size();
 54    }
 55
 56    public void resetOriginalFieldCount(int count) {
 57        if (count == -1) {
 58            this.originalFieldCount = data.size();
 59        } else {
 60            this.originalFieldCount = count;
 61        }
 62    }
 63
 64    /**
 65     * Get records for this query for the provided fields.
 66     *
 67     * @param fields QueryFields to return in the sample.
 68     * @return records as String in JSON format.
 69     */
 70    @Override
 71    public String sample(ArrayList<QueryField> fields) {
 72        StringBuilder sb = new StringBuilder();
 73
 74        //do sampling
 75        layerSampling(fields);
 76
 77        //identify fields to data for return
 78        int[] validFields = null;
 79        if (fields == null) {
 80            validFields = new int[data.size()];
 81            for (int i = 0; i < data.size(); i++) {
 82                validFields[i] = i;
 83            }
 84        } else {
 85            validFields = new int[fields.size()];
 86            for (int i = 0; i < fields.size(); i++) {
 87                validFields[i] = -1;
 88                for (int j = 0; j < data.size(); j++) {
 89                    if (fields.get(i) != null
 90                            && fields.get(i).getName() != null
 91                            && data.get(j).getName().equals(fields.get(i).getName())) {
 92                        validFields[i] = j;
 93                        break;
 94                    }
 95                }
 96            }
 97        }
 98
 99        //header
100        for (int i = 0; i < validFields.length; i++) {
101            if (i > 0) {
102                sb.append(",");
103            }
104            if (validFields[i] >= 0) {
105                sb.append(data.get(validFields[i]).getDisplayName());
106            } else {
107                sb.append(fields.get(i).getDisplayName());
108            }
109        }
110
111        //data
112        int recordCount = points.length / 2;
113        for (int j = 0; j < recordCount; j++) {
114            sb.append("\n");
115            for (int i = 0; i < validFields.length; i++) {
116                if (i > 0) {
117                    sb.append(",");
118                }
119                if (validFields[i] >= 0 && data.get(validFields[i]).getAsString(j) != null) {
120                    sb.append("\"");
121                    sb.append(String.valueOf(data.get(validFields[i]).getAsString(j)).replace("\"", "\"\""));
122                    sb.append("\"");
123                } else {
124                    sb.append("n/a");
125                }
126            }
127        }
128
129        return sb.toString();
130    }
131
132    /**
133     * Get species list for this query.
134     *
135     * @return species list as String containing JSON array.
136     */
137    @Override
138    public String speciesList() {
139        return null;
140    }
141    
142    public String endemicSpeciesList(){
143        throw new UnsupportedOperationException();
144    }
145
146    /**
147     * Get number of occurrences in this query.
148     *
149     * @return number of occurrences as int or -1 on error.
150     */
151    @Override
152    public int getOccurrenceCount() {
153        return points.length / 2;
154    }
155
156    /**
157     * Get number of species in this query.
158     *
159     * @return number of species as int or -1 on error.
160     */
161    @Override
162    public int getSpeciesCount() {
163        return 1;
164    }
165
166    public int getEndemicSpeciesCount(){
167        throw new UnsupportedOperationException();
168    }
169    
170    /**
171     * Get parsed coordinates and optional fields for this query.
172     *
173     * @param fields QueryFields to return in the sample as ArrayList<QueryField>.
174     * If a QueryField isStored() it will be populated with the field data.
175     * @return coordinates as double [] like [lng, lat, lng, lat, ...].
176     */
177    @Override
178    public double[] getPoints(ArrayList<QueryField> fields) {
179        if (fields != null) {
180            for (int i = 0; i < fields.size(); i++) {
181                for (int j = 0; j < data.size(); j++) {
182                    if (fields.get(i).getName().equals(data.get(j).getName())) {
183                        fields.get(i).copyData(data.get(j));
184                        break;
185                    }
186                }
187            }
188        }
189
190        return points;
191    }
192
193    @Override
194    public String getName() {
195        return name;
196    }
197
198    @Override
199    public String getRank() {
200        return "species";
201    }
202
203    @Override
204    public String getQ() {
205        return uniqueId;
206    }
207
208    @Override
209    public String getFullQ(boolean encode) {
210        StringBuilder sb = new StringBuilder();
211        sb.append(metadata).append("\n");
212
213        if (wkt.length() > 0) {
214            sb.append(wkt).append("\n");
215        }
216
217        for (int i = 0; i < facets.size(); i++) {
218            sb.append(facets.get(i).toString()).append("\n");
219        }
220
221        return sb.toString();
222    }
223
224    @Override
225    public Query newWkt(String wkt, boolean forMapping) {
226        if (wkt == null || wkt.equals(CommonData.WORLD_WKT)) {
227            return this;
228        }
229
230        SimpleRegion sr = SimpleShapeFile.parseWKT(wkt);
231
232        //per record test
233        boolean[] valid = new boolean[points.length / 2];
234        int count = 0;
235        for (int i = 0; i < valid.length; i++) {
236            valid[i] = sr.isWithin(points[i * 2], points[i * 2 + 1]);
237            if (valid[i]) {
238                count++;
239            }
240        }
241
242        UploadQuery q = newFromValidMapping(valid, count);
243
244        //maintain wkt history
245        if (this.wkt.length() > 0) {
246            q.wkt = this.wkt + " AND " + wkt;
247        } else {
248            q.wkt = wkt;
249        }
250
251        return q;
252    }
253
254    UploadQuery newFromValidMapping(boolean[] valid, int count) {
255        //copy original data
256        ArrayList<QueryField> facetData = new ArrayList<QueryField>(originalFieldCount);
257        for (int i = 0; i < originalFieldCount; i++) {
258            QueryField qf = new QueryField(data.get(i).getName(), data.get(i).getDisplayName(), data.get(i).getFieldType());
259            qf.ensureCapacity(count);
260            for (int j = 0; j < valid.length; j++) {
261                if (valid[j]) {
262                    qf.add(data.get(i).getAsString(j));
263                }
264            }
265            qf.store();
266            facetData.add(qf);
267        }
268
269        //copy points
270        double[] facetPoints = new double[count * 2];
271        int pos = 0;
272        for (int j = 0; j < valid.length; j++) {
273            if (valid[j]) {
274                facetPoints[pos * 2] = points[j * 2];
275                facetPoints[pos * 2 + 1] = points[j * 2 + 1];
276                pos++;
277            }
278        }
279
280
281        String uid = String.valueOf(System.currentTimeMillis());
282
283        RecordsLookup.putData(uid, facetPoints, facetData, metadata);
284
285        return new UploadQuery(uid, name, facetPoints, facetData, metadata);
286    }
287
288    @Override
289    public Query newFacet(Facet facet, boolean forMapping) {
290        ArrayList<Facet> facets = new ArrayList<Facet>();
291        facets.add(facet);
292        return newFacets(facets, forMapping);
293    }
294
295    @Override
296    public String getWMSpath() {
297        throw new UnsupportedOperationException("Not supported yet.");
298    }
299
300    @Override
301    public ArrayList<QueryField> getFacetFieldList() {
302        ArrayList<QueryField> fields = new ArrayList<QueryField>();
303        for (int i = 1; i < data.size(); i++) {
304            fields.add(data.get(i));
305        }
306        return fields;
307    }
308
309    /**
310     * Do sampling on layers that are in fields but not in data.
311     * 
312     * @param fields
313     */
314    private void layerSampling(ArrayList<QueryField> fields) {
315        if (fields == null) {
316            return;
317        }
318        ArrayList<QueryField> forSampling = new ArrayList<QueryField>();
319        for (int i = 0; i < fields.size(); i++) {
320            boolean found = false;
321            for (int j = 0; j < data.size(); j++) {
322                if (data.get(j).getName().equals(fields.get(i).getName())) {
323                    found = true;
324                    break;
325                }
326            }
327            if (!found) {
328                forSampling.add(fields.get(i));
329            }
330        }
331
332        //do sampling
333        double[][] coordinates = new double[points.length / 2][2];
334        for (int i = 0; i < points.length; i += 2) {
335            coordinates[i / 2][0] = points[i];
336            coordinates[i / 2][1] = points[i + 1];
337        }
338        ArrayList<String> facetIds = new ArrayList<String>();
339        for (int i = 0; i < forSampling.size(); i++) {
340            facetIds.add(forSampling.get(i).getName());
341        }
342        ArrayList<String[]> output = Sampling.sampling(facetIds, coordinates);
343
344        for (int i = 0; i < forSampling.size(); i++) {
345            String[] s = output.get(i);
346            QueryField qf = forSampling.get(i);
347            if (qf.getName() != null) {
348                qf.ensureCapacity(s.length);
349                for (int j = 0; j < s.length; j++) {
350                    qf.add(s[j]);
351                }
352                qf.store();
353
354                data.add(qf);
355            }
356        }
357
358        //update RecordsLookup for new fields, if it is already in RecordsLookup
359        Object[] recordData = (Object[]) RecordsLookup.getData(getQ());
360        if (recordData != null) {
361            double[] points = (double[]) recordData[0];
362            RecordsLookup.putData(getQ(), points, data, metadata);
363        }
364    }
365
366    @Override
367    public String getSpeciesIdFieldName() {
368        return null;
369    }
370
371    @Override
372    public String getRecordIdFieldName() {
373        return data.get(0).getName();
374    }
375
376    @Override
377    public String getRecordLongitudeFieldName() {
378        return data.get(1).getName();
379    }
380
381    @Override
382    public String getRecordLatitudeFieldName() {
383        return data.get(2).getName();
384    }
385
386    @Override
387    public String getRecordIdFieldDisplayName() {
388        return data.get(0).getDisplayName();
389    }
390
391    @Override
392    public String getRecordLongitudeFieldDisplayName() {
393        return data.get(1).getDisplayName();
394    }
395
396    @Override
397    public String getRecordLatitudeFieldDisplayName() {
398        return data.get(2).getDisplayName();
399    }
400
401    @Override
402    public LegendObject getLegend(String colourmode) {
403        //get existing
404        for (int i = 0; i < data.size(); i++) {
405            if (data.get(i).getName().equals(colourmode)) {
406                return data.get(i).getLegend();
407            }
408        }
409
410        //create
411        ArrayList<QueryField> fields = new ArrayList<QueryField>();
412        QueryField qf = new QueryField(colourmode, CommonData.getFacetLayerDisplayName(colourmode), QueryField.FieldType.AUTO);
413        qf.setStored(true);
414        fields.add(qf);
415        layerSampling(fields);
416
417        //get existing
418        for (int i = 0; i < data.size(); i++) {
419            if (data.get(i).getName().equals(colourmode)) {
420                return data.get(i).getLegend();
421            }
422        }
423
424        //fail
425        return null;
426    }
427
428    @Override
429    public Query newFacets(List<Facet> facet, boolean forMapping) {
430        //copy all the data through the list of facets (AND)
431
432        //setup
433        ArrayList<List<QueryField>> facetFields = new ArrayList<List<QueryField>>();
434        for (int k = 0; k < facet.size(); k++) {
435            Facet f = facet.get(k);
436            String[] fields = f.getFields();
437            List<QueryField> qf = new ArrayList<QueryField>();
438            for (int j = 0; j < fields.length; j++) {
439                int i = 0;
440                for (i = 0; i < data.size(); i++) {
441                    if (fields[j].equals(data.get(i).getName())) {
442                        qf.add(data.get(i));
443                        break;
444                    }
445                }
446            }
447            facetFields.add(qf);
448        }
449
450        //per record test
451        boolean[] valid = new boolean[points.length / 2];
452        int count = 0;
453        for (int i = 0; i < valid.length; i++) {
454            int sum = 0;
455            for (int j = 0; j < facet.size(); j++) {
456                if (facet.get(j).isValid(facetFields.get(j), i)) {
457                    sum++;
458                }
459            }
460
461            valid[i] = sum == facet.size();
462            if (valid[i]) {
463                count++;
464            }
465        }
466
467        UploadQuery uq = newFromValidMapping(valid, count);
468
469        //maintain facet history
470        uq.facets.addAll(this.facets);
471        uq.facets.addAll(facet);
472
473        return uq;
474    }
475
476    @Override
477    public String getUrl() {
478        return CommonData.webportalServer + "/ws/wms/reflect?";
479    }
480    List<Double> bbox = null;
481
482    @Override
483    public List<Double> getBBox() {
484        if (bbox == null) {
485            bbox = new ArrayList<Double>();
486
487            double minx, miny, maxx, maxy;
488            minx = points[0];
489            maxx = points[0];
490            miny = points[1];
491            maxy = points[1];
492            for (int i = 0; i < points.length; i += 2) {
493                if (minx > points[i]) {
494                    minx = points[i];
495                }
496                if (maxx < points[i]) {
497                    maxx = points[i];
498                }
499                if (miny > points[i + 1]) {
500                    miny = points[i + 1];
501                }
502                if (maxy < points[i + 1]) {
503                    maxy = points[i + 1];
504                }
505            }
506
507            bbox.add(minx);
508            bbox.add(miny);
509            bbox.add(maxx);
510            bbox.add(maxy);
511        }
512
513        return bbox;
514    }
515
516    @Override
517    public String getMetadataHtml() {
518        String[] m = metadata.replace("<br />", "").split("\n");
519        String name = m[1].substring(m[1].indexOf(':') + 1).trim();
520        String description = m[2].substring(m[2].indexOf(':') + 1).trim();
521        String date = m[3].substring(m[3].indexOf(':') + 1).trim();
522
523        StringBuilder fieldsList = new StringBuilder();
524        for (int i = 0; i < data.size(); i++) {
525            if (i > 0) {
526                fieldsList.append(", ");
527            }
528            fieldsList.append(data.get(i).getDisplayName());
529        }
530
531        String html = "User uploaded points\n";
532        html += "<table class='md_table'>";
533        html += "<tr class='md_grey-bg'><td class='md_th'>Name: </td><td class='md_spacer'/><td class='md_value'>" + name + "</td></tr>";
534        html += "<tr><td class='md_th'>Description: </td><td class='md_spacer'/><td class='md_value'>" + description + "</td></tr>";
535        html += "<tr class='md_grey-bg'><td class='md_th'>Date: </td><td class='md_spacer'/><td class='md_value'>" + date + "</td></tr>";
536        html += "<tr><td class='md_th'>Number of coordinates: </td><td class='md_spacer'/><td class='md_value'>" + getOccurrenceCount() + "</td></tr>";
537        html += "<tr class='md_grey-bg'><td class='md_th'>Number of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + data.size() + "</td></tr>";
538        html += "<tr><td class='md_th'>List of fields: </td><td class='md_spacer'/><td class='md_spacer'>" + fieldsList.toString() + "</td></tr>";
539        html += "</table>";
540
541        return html;
542    }
543
544    @Override
545    public String getDownloadUrl(String[] extraFields) {
546        return null;
547    }
548
549    @Override
550    public byte[] getDownloadBytes(String[] extraFields, String[] displayNames) {
551        ArrayList<QueryField> fields = new ArrayList<QueryField>();
552        if (getFacetFieldList() != null) {
553            fields.add(data.get(0));    //id column (1st) is not in getFacetFieldList()
554            fields.addAll(getFacetFieldList());
555        }
556        if (extraFields != null && extraFields.length > 0) {
557            for (int i = 0; i < extraFields.length; i++) {
558                String s = extraFields[i];
559                if (displayNames == null) {
560                    fields.add(new QueryField(s, CommonData.getFacetLayerDisplayName(s), QueryField.FieldType.AUTO));
561                } else {
562                    fields.add(new QueryField(s, displayNames[i], QueryField.FieldType.AUTO));
563                }
564            }
565        }
566        try {
567            //zip it
568            ByteArrayOutputStream bos = new ByteArrayOutputStream();
569            ZipOutputStream zos = new ZipOutputStream(bos);
570            ZipEntry ze = new ZipEntry(getName() + ".csv");
571            zos.putNextEntry(ze);
572            zos.write(sample(fields).getBytes("UTF-8"));
573            zos.close();
574
575            return bos.toByteArray();
576        } catch (Exception ex) {
577            Logger.getLogger(UploadQuery.class.getName()).log(Level.SEVERE, null, ex);
578        }
579        return null;
580    }
581
582    @Override
583    public String getQc() {
584        return null;
585    }
586
587    @Override
588    public void setQc(String qc) {
589    }
590
591    /**
592     * Get the column header name for a column in the output of sampling.
593     *
594     * @param colourMode
595     * @return
596     */
597    @Override
598    public String getRecordFieldDisplayName(String colourMode) {
599        for (int i = 0; i < data.size(); i++) {
600            if (data.get(i).getName().equals(colourMode)) {
601                return data.get(i).getDisplayName();
602            }
603        }
604        return colourMode;
605    }
606
607    /**
608     * Add or remove one record to an internal group.
609     */
610    @Override
611    public void flagRecord(String id, boolean set) {
612        if (set) {
613            flaggedRecords.add(id);
614        } else {
615            flaggedRecords.remove(id);
616        }
617    }
618
619    /**
620     * Get the number of flagged records.
621     */
622    @Override
623    public int flagRecordCount() {
624        return flaggedRecords.size();
625    }
626
627    /**
628     * Get the list of flagged records as '\n' separated String.
629     */
630    @Override
631    public String getFlaggedRecords() {
632        StringBuilder sb = new StringBuilder();
633        for (String s : flaggedRecords) {
634            if (sb.length() > 0) {
635                sb.append("\n");
636            }
637            sb.append(s);
638        }
639        return sb.toString();
640    }
641
642    /**
643     * Create a new Query that contains only or excludes all flagged records
644     */
645    @Override
646    public Query newFlaggedRecords(boolean includeOnlyFlaggedRecords) {
647        if (flagRecordCount() == 0) {
648            return this;
649        }
650
651        //per record test
652        boolean[] valid = new boolean[points.length / 2];
653        int count = 0;
654        for (int i = 0; i < valid.length; i++) {
655            valid[i] = includeOnlyFlaggedRecords != flaggedRecords.contains(data.get(0).getString(i));
656
657            if (valid[i]) {
658                count++;
659            }
660        }
661
662        return newFromValidMapping(valid, count);
663    }
664
665    @Override
666    public String getAutoComplete(String facet, String value, int limit) {
667        throw new UnsupportedOperationException("Upload Query does not support the ability to perform an autocomplete.");
668    }
669}