/alaspatial/src/main/java/org/ala/spatial/analysis/service/SamplingService.java

  1package org.ala.spatial.analysis.service;
  2
  3import java.io.File;
  4import java.io.FileWriter;
  5import java.util.ArrayList;
  6import org.ala.spatial.analysis.index.BoundingBoxes;
  7
  8import org.ala.spatial.analysis.index.IndexedRecord;
  9import org.ala.spatial.analysis.index.OccurrenceRecordNumbers;
 10import org.ala.spatial.analysis.index.OccurrencesCollection;
 11import org.ala.spatial.analysis.index.OccurrencesFilter;
 12import org.ala.spatial.analysis.index.SpeciesColourOption;
 13import org.ala.spatial.util.AnalysisJobSampling;
 14import org.ala.spatial.util.Layers;
 15import org.ala.spatial.util.OccurrencesFieldsUtil;
 16import org.ala.spatial.util.SimpleRegion;
 17import org.ala.spatial.util.SpatialLogger;
 18import org.ala.spatial.util.TabulationSettings;
 19
 20/**
 21 * service for returning occurrences + optional values from layer intersections
 22 *
 23 * @author adam
 24 *
 25 */
 26public class SamplingService {
 27
 28    public static SamplingService newForLSID(String lsid) {
 29        if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) {
 30            return new SamplingLoadedPointsService();
 31        } else {
 32            return new SamplingService();
 33        }
 34    }
 35
 36    /**
 37     * constructor init
 38     */
 39    SamplingService() {
 40        TabulationSettings.load();
 41    }
 42
 43    /**
 44     * gets samples; occurrences records + optional intersecting layer values,
 45     *
 46     *
 47     * limit output
 48     *
 49     * @param filter species name as String
 50     * @param layers list of layer names of additional data to include as String []
 51     * @param region region to restrict results as SimpleRegion
 52     * @param records sorted pool of records to intersect with as ArrayList<Integer>
 53     * @param max_rows upper limit of records to return as int
 54     * @return samples as grid, String [][]
 55     */
 56    public String sampleSpeciesAsCSV(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows) {
 57        return sampleSpeciesAsCSV(filter, layers, region, records, max_rows, null);
 58    }
 59
 60    /**
 61     * gets samples; occurrences records + optional intersecting layer values,
 62     *
 63     *
 64     * limit output
 65     *
 66     * @param filter species name as String
 67     * @param layers list of layer names of additional data to include as String []
 68     * @param region region to restrict results as SimpleRegion
 69     * @param records sorted pool of records to intersect with as ArrayList<Integer>
 70     * @param max_rows upper limit of records to return as int
 71     * @return samples as grid, String [][]
 72     */
 73    public String[][] sampleSpecies(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows) {
 74        return sampleSpecies(filter, layers, region, records, max_rows, null);
 75    }
 76
 77    public String getHeader(String[] layers) {
 78        StringBuffer header = new StringBuffer();
 79        OccurrencesFieldsUtil ofu = new OccurrencesFieldsUtil();
 80        for (String s : ofu.getOutputColumnNames()) {
 81            header.append(s).append(",");
 82        }
 83        if (layers != null) {
 84            for (String l : layers) {
 85                header.append(Layers.layerNameToDisplayName(l)).append(",");
 86            }
 87        }
 88        header.deleteCharAt(header.length() - 1); //take off end ','
 89
 90        return header.toString();
 91    }
 92
 93    public String[][] sampleSpecies(String filter, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows, AnalysisJobSampling job) {
 94        ArrayList<String> as = OccurrencesCollection.getFullRecords(new OccurrencesFilter(filter, region, records, layers, max_rows));
 95
 96        //split records and append header
 97        if (as.size() > 0) {
 98            String[] header = getHeader(layers).split(",");
 99
100            int numCols = header.length;
101
102            String[][] output = new String[as.size() + 1][numCols];
103
104            //header
105            for (int j = 0; j < header.length && j < numCols; j++) {
106                output[0][j] = header[j];
107            }
108
109            //records
110            for (int i = 0; i < as.size(); i++) {
111                String[] s = as.get(i).split(",");
112                for (int j = 0; j < s.length && j < numCols; j++) {
113                    output[i + 1][j] = s[j];
114                }
115            }
116
117            return output;
118        }
119
120        return null;
121    }
122
123    /**
124     * gets array of points for species (genus, etc) name matches within
125     * a specified region
126     *
127     * @param filter species (genus, etc) name
128     * @param region region to filter results by
129     * @param records sorted pool of records to intersect with as int []
130     * @return points as double[], first is longitude, every second is latitude.
131     */
132    public double[] sampleSpeciesPoints(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records) {
133
134        //test on bounding box
135        double[] bb = BoundingBoxes.getLsidBoundingBoxDouble(filter);
136        double[][] regionbb = region.getBoundingBox();
137        if (bb != null && regionbb != null
138                && bb[0] <= regionbb[1][0] && bb[2] >= regionbb[0][0]
139                && bb[1] <= regionbb[1][1] && bb[3] >= regionbb[0][1]) {
140
141            return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER));
142        }
143
144        return null;
145    }
146
147    /**
148     * gets array of points for species (genus, etc) name matches within
149     * a specified region
150     *
151     * can return other field or sampling for points returned
152     *
153     * @param filter species (genus, etc) name
154     * @param region region to filter results by
155     * @param records sorted pool of records to intersect with as ArrayList<Integer>
156     * @return points as double[], first is longitude, every second is latitude.
157     */
158    public double[] sampleSpeciesPoints(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, ArrayList<SpeciesColourOption> extra) {
159        //test on bounding box
160        double[] bb = BoundingBoxes.getLsidBoundingBoxDouble(filter);
161
162        if (region == null) {
163            return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), extra);
164        }
165
166        double[][] regionbb = region.getBoundingBox();
167        if (bb != null && bb[0] <= regionbb[1][0] && bb[2] >= regionbb[0][0]
168                && bb[1] <= regionbb[1][1] && bb[3] >= regionbb[0][1]) {
169
170            /* get points */
171            return OccurrencesCollection.getPoints(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), extra);
172        }
173
174        return null;
175    }
176
177    /**
178     * for Sensitive Coordinates
179     *
180     * gets array of points for species (genus, etc) name matches within
181     * a specified region
182     *
183     * @param filter species (genus, etc) name
184     * @param region region to filter results by
185     * @param records sorted pool of records to intersect with as ArrayList<Integer>
186     * @return points as double[], first is longitude, every second is latitude.
187     */
188    public double[] sampleSpeciesPointsSensitive(String filter, SimpleRegion region, int[] records) {
189        IndexedRecord[] ir = null;// OccurrencesIndex.filterSpeciesRecords(filter);
190
191        if (ir != null && ir.length > 0) {
192
193            /* get points */
194            double[] points = null;//OccurrencesIndex.getPointsSensitive(ir[0].record_start, ir[0].record_end);
195
196            /* test for region absence */
197            if (region == null) {
198                return points;
199            }
200
201            int i;
202            int count = 0;
203
204            int recordsPos = 0; //for test on records
205
206            /* return all valid points within the region */
207            for (i = 0; i < points.length; i += 2) {
208                //do not add if does not intersect with records list
209                if (records != null) {
210                    int currentRecord = i + ir[0].record_start;
211                    //increment recordsPos as required
212                    while (recordsPos < records.length
213                            && records[recordsPos] < currentRecord) {
214                        recordsPos++;
215                    }
216                    //test for intersect
217                    if (recordsPos >= records.length
218                            || currentRecord != records[recordsPos]) {
219                        continue;
220                    }
221                }
222                //region test
223                if (region.isWithin(points[i], points[i + 1])) {
224                    count += 2;
225                } else {
226                    points[i] = Double.NaN;
227                }
228            }
229            //move into 'output'
230            if (count > 0) {
231                double[] output = new double[count];
232                int p = 0;
233                for (i = 0; i < points.length; i += 2) {
234                    if (!Double.isNaN(points[i])) {
235                        output[p++] = points[i];
236                        output[p++] = points[i + 1];
237                    }
238                }
239                return output;
240            }
241
242        }
243
244        return null;
245    }
246
247    /**
248     * for Sensitive Coordinates
249     *
250     * gets array of points for species (genus, etc) name matches within
251     * a specified region
252     *
253     * removes points for all species that are sensitive
254     *
255     * @param filter species (genus, etc) name
256     * @param region region to filter results by
257     * @param records sorted pool of records to intersect with as ArrayList<Integer>
258     * @return points as double[], first is longitude, every second is latitude.
259     */
260    public double[] sampleSpeciesPointsMinusSensitiveSpecies(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, StringBuffer removedSpecies) {
261        /* get points */
262        return OccurrencesCollection.getPointsMinusSensitiveSpecies(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), removedSpecies);
263    }
264
265    /**
266     * for Sensitive Records
267     *
268     * Checks if the records are sensitive within filter range
269     *
270     * @param filter species (genus, etc) name
271     * @param region region to filter results by
272     * @param records sorted pool of records to intersect with as ArrayList<Integer>
273     * @return int 
274     *      0 when non-sensitive and has records,
275     *      1 when sensitive or no records,
276     *      -1 when cannot be determined
277     */
278    public static int isSensitiveRecord(String filter, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records) {
279        StringBuffer sb = new StringBuffer();
280        try {
281            double[] d = OccurrencesCollection.getPointsMinusSensitiveSpecies(new OccurrencesFilter(filter, region, records, TabulationSettings.MAX_RECORD_COUNT_CLUSTER), sb);
282            if (d == null) {
283                return 1;
284            } else {
285                return 0;
286            }
287        } catch (Exception e) {
288            e.printStackTrace();
289        }
290
291        return -1;
292    }
293
294    /**
295     * gets samples; occurrences records + optional intersecting layer values,
296     *
297     *
298     * limit output
299     *
300     * @param filter species name as String
301     * @param layers list of layer names of additional data to include as String []
302     * @param region region to restrict results as SimpleRegion
303     * @param records sorted pool of records to intersect with as ArrayList<Integer>
304     * @param max_rows upper limit of records to return as int
305     * @return samples as grid, String [][]
306     */
307    public String sampleSpeciesAsCSV(String species, String[] layers, SimpleRegion region, ArrayList<OccurrenceRecordNumbers> records, int max_rows, AnalysisJobSampling job) {
308        try {
309
310            System.out.println("Limiting sampling to : " + max_rows);
311
312            String[][] results = sampleSpecies(species, layers, region, records, max_rows);
313            StringBuilder sbResults = new StringBuilder();
314
315            for (int i = 0; i < results.length; i++) {
316                for (int j = 0; j < results[i].length; j++) {
317                    if (results[i][j] != null) {
318                        sbResults.append(results[i][j]);
319                    }
320                    if (j < results[i].length - 1) {
321                        sbResults.append(",");
322                    }
323                }
324                sbResults.append("\r\n");
325            }
326
327            /* open output file */
328            File temporary_file = java.io.File.createTempFile("sample", ".csv");
329            FileWriter fw = new FileWriter(temporary_file);
330
331            fw.append(sbResults.toString());
332            fw.close();
333            return temporary_file.getPath();
334
335
336        } catch (Exception e) {
337            System.out.println("error with samplesSpeciesAsCSV:");
338            e.printStackTrace(System.out);
339        }
340
341        return "";
342    }
343
344    public static String getLSIDAsGeoJSON(String lsid, File outputpath) {
345        if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) {
346            return getLSIDAsGeoJSON(lsid, outputpath);
347        }
348
349        int i;
350
351        /* get samples records from records indexes */
352        String[][] samples = (new SamplingService()).sampleSpecies(lsid, null, null, null, TabulationSettings.MAX_RECORD_COUNT_DOWNLOAD, null);
353
354        StringBuffer sbGeoJSON = new StringBuffer();
355        sbGeoJSON.append("{");
356        sbGeoJSON.append("  \"type\": \"FeatureCollection\",");
357        sbGeoJSON.append("  \"features\": [");
358        for (i = 1; i < samples.length; i++) {
359            String s = getRecordAsGeoJSON(samples, i);
360            if (s != null) {
361                sbGeoJSON.append(s);
362                if (i < samples.length - 1) {
363                    sbGeoJSON.append(",");
364                }
365            }
366        }
367        sbGeoJSON.append("  ],");
368        sbGeoJSON.append("  \"crs\": {");
369        sbGeoJSON.append("    \"type\": \"EPSG\",");
370        sbGeoJSON.append("    \"properties\": {");
371        sbGeoJSON.append("      \"code\": \"4326\"");
372        sbGeoJSON.append("    }");
373        sbGeoJSON.append("  }");
374        //sbGeoJSON.append(",  \"bbox\": [");
375        //sbGeoJSON.append("    ").append(bbox[0][0]).append(",").append(bbox[0][1]).append(",").append(bbox[1][0]).append(",").append(bbox[1][1]);
376        //sbGeoJSON.append("  ]");
377        sbGeoJSON.append("}");
378
379        /* write samples to a file */
380        try {
381            File temporary_file = java.io.File.createTempFile("filter_sample", ".csv", outputpath);
382            FileWriter fw = new FileWriter(temporary_file);
383
384            fw.write(sbGeoJSON.toString());
385
386            fw.close();
387
388            return temporary_file.getName();	//return location of temp file
389
390        } catch (Exception e) {
391            SpatialLogger.log("SamplingService: getLSIDAsGeoJSON()", e.toString());
392            e.printStackTrace();
393        }
394        return "";
395
396    }
397
398    /**
399     * creates a file with geojson for lsid at outputpath
400     *
401     * returns filename (first line), number of parts (2nd line)
402     *
403     * @param lsid
404     * @param outputpath
405     * @return
406     */
407    public static String getLSIDAsGeoJSONIntoParts(String lsid, File outputpath) {
408        if (SamplingLoadedPointsService.isLoadedPointsLSID(lsid)) {
409            return getLSIDAsGeoJSONIntoParts(lsid, outputpath);
410        }
411
412        int i;
413
414        /* get samples records from records indexes */
415        String[][] samples = (new SamplingService()).sampleSpecies(lsid, null, null, null, TabulationSettings.MAX_RECORD_COUNT_DOWNLOAD, null);
416
417        int max_parts_size = 2000;
418
419        int count = 0;
420
421        //-1 on samples.length for header
422        int partCount = (int) Math.ceil((samples.length - 1) / (double) max_parts_size);
423
424        //test for filename, return if it exists
425        File file;
426        String filename = outputpath + File.separator + lsid.replace(":", "_").replace(".", "_");
427        try {
428            file = new File(filename + "_" + (partCount - 1));
429            if (file.exists()) {
430                return lsid.replace(":", "_").replace(".", "_") + "\n" + partCount;
431            }
432        } catch (Exception e) {
433            e.printStackTrace();
434        }
435
436        for (int j = 1; j < samples.length; j += max_parts_size) {
437
438            StringBuffer sbGeoJSON = new StringBuffer();
439            sbGeoJSON.append("{");
440            sbGeoJSON.append("\"type\": \"FeatureCollection\",");
441            sbGeoJSON.append("\"features\": [");
442            int len = j + max_parts_size;
443            if (len > samples.length) {
444                len = samples.length;
445            }
446            for (i = j; i < len; i++) {
447                String s = getRecordAsGeoJSON(samples, i);
448                if (s != null) {
449                    sbGeoJSON.append(s);
450                    if (i < len - 1) {
451                        sbGeoJSON.append(",");
452                    }
453                }
454            }
455            sbGeoJSON.append("],");
456            sbGeoJSON.append("\"crs\": {");
457            sbGeoJSON.append("\"type\": \"EPSG\",");
458            sbGeoJSON.append("\"properties\": {");
459            sbGeoJSON.append("\"code\": \"4326\"");
460            sbGeoJSON.append("}");
461            sbGeoJSON.append("}");
462            sbGeoJSON.append("}");
463
464            /* write samples to a file */
465            try {
466                //File temporary_file = java.io.File.createTempFile("filter_sample", ".csv", outputpath);
467                FileWriter fw = new FileWriter(
468                        filename + "_" + count);
469                count++;
470
471                fw.write(sbGeoJSON.toString());
472
473                fw.close();
474
475                //return temporary_file.getName();	//return location of temp file
476
477            } catch (Exception e) {
478                SpatialLogger.log("SamplingService: getLSIDAsGeoJSON()", e.toString());
479                e.printStackTrace();
480            }
481        }
482        return lsid.replace(":", "_").replace(".", "_") + "\n" + partCount;
483
484    }
485
486    private static String getRecordAsGeoJSON(String[][] rec, int rw) {
487        //String[] recdata = rec.split(",");
488
489        if (rec == null || rec.length <= rw || rec[rw].length <= TabulationSettings.geojson_latitude) {
490            return null;
491        }
492
493        for (int i = 0; i < TabulationSettings.geojson_latitude; i++) {
494            if (rec[rw][i] == null) {
495                return null;
496            }
497        }
498
499        StringBuffer sbRec = new StringBuffer();
500        sbRec.append("{");
501        sbRec.append("  \"type\":\"Feature\",");
502        sbRec.append("  \"id\":\"occurrences.data.").append(rec[rw][TabulationSettings.geojson_id]).append("\",");
503        sbRec.append("  \"geometry\":{");
504        sbRec.append("      \"type\":\"Point\",");
505        sbRec.append("      \"coordinates\":[\"").append(rec[rw][TabulationSettings.geojson_longitude]).append("\",\"").append(rec[rw][TabulationSettings.geojson_latitude].trim()).append("\"]");
506        sbRec.append("   },");
507        sbRec.append("  \"geometry_name\":\"the_geom\",");
508        sbRec.append("  \"properties\":{");
509        for (int i = 0; i < TabulationSettings.geojson_property_names.length; i++) {
510            sbRec.append("      \"").append(TabulationSettings.geojson_property_names[i]).append("\":\"").append(rec[rw][TabulationSettings.geojson_property_fields[i]]).append("\"");
511            if (i < TabulationSettings.geojson_property_names.length - 1) {
512                sbRec.append(",");
513            }
514        }
515        sbRec.append("  }");
516        sbRec.append("}");
517
518        return sbRec.toString();
519
520    }
521
522    private String[][] sampleSpeciesSmall(String filter, String[] layers, SimpleRegion region, ArrayList<Integer> records, int max_rows, AnalysisJobSampling job) {
523        IndexedRecord[] ir = null;// OccurrencesIndex.filterSpeciesRecords(filter);
524
525        if (ir != null && ir.length > 0) {
526            if (records != null) {
527                java.util.Collections.sort(records);
528            }
529
530            /* get points */
531            double[] points = null;//OccurrencesIndex.getPoints(ir[0].record_start, ir[0].record_end);
532
533            /* test for region absence */
534            int i;
535
536            int alen = 0;
537            int[] a = new int[max_rows];
538
539            int recordsPos = 0; //for test on records
540
541            /* return all valid points within the region */
542            for (i = 0; i < points.length && alen < max_rows; i += 2) {
543                int currentRecord = (i / 2) + ir[0].record_start;
544
545                //do not add if does not intersect with records list
546                if (records != null) {
547                    //increment recordsPos as required
548                    while (recordsPos < records.size()
549                            && records.get(recordsPos).intValue() < currentRecord) {
550                        recordsPos++;
551                    }
552                    //test for intersect
553                    if (recordsPos >= records.size()
554                            || currentRecord != records.get(recordsPos).intValue()) {
555                        continue;
556                    }
557                }
558                //region test
559                if (region == null || region.isWithin(points[i], points[i + 1])) {
560                    a[alen++] = currentRecord;
561                }
562            }
563
564            if (alen == 0) {
565                return null;
566            }
567
568            //filled a up to alen, get the data
569            if (alen < max_rows) {
570                a = java.util.Arrays.copyOf(a, alen);
571            }
572            String[] oi = null;
573            ;//OccurrencesIndex.getSortedRecords(a);
574
575            int layerscount = (layers == null) ? 0 : layers.length;
576            int headercount = oi[0].split(",").length;
577            String[][] output = new String[oi.length + 1][headercount + layerscount];//+1 for header
578
579            //fill
580            for (i = 0; i < oi.length; i++) {
581                String[] line = oi[i].split(",");
582                for (int j = 0; j < line.length && j < headercount; j++) {
583                    output[i + 1][j] = line[j];   //+1 for header
584                }
585            }
586
587            for (i = 0; layers != null && i < layers.length; i++) {
588                String[] si = null;//SamplingIndex.getRecords(layers[i], a);
589                if (si != null) {
590                    for (int j = 0; j < si.length && j < output.length; j++) {
591                        output[j][headercount + i] = si[j];
592                    }
593                }
594            }
595
596            //header
597            OccurrencesFieldsUtil ofu = new OccurrencesFieldsUtil();
598            i = 0;
599            for (String s : ofu.getOutputColumnNames()) {
600                output[0][i++] = s.trim();
601            }
602            if (layers != null) {
603                for (String l : layers) {
604                    output[0][i++] = Layers.layerNameToDisplayName(l).trim();
605                }
606            }
607
608            return output;
609        }
610
611        return null;
612    }
613}