/geonode/layers/tests.py
Python | 1121 lines | 989 code | 56 blank | 76 comment | 29 complexity | 2d7d270408c393fecd43091b0439015b MD5 | raw file
Possible License(s): BSD-3-Clause, Apache-2.0
- #########################################################################
- #
- # Copyright (C) 2016 OSGeo
- #
- # This program is free software: you can redistribute it and/or modify
- # it under the terms of the GNU General Public License as published by
- # the Free Software Foundation, either version 3 of the License, or
- # (at your option) any later version.
- #
- # This program is distributed in the hope that it will be useful,
- # but WITHOUT ANY WARRANTY; without even the implied warranty of
- # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- # GNU General Public License for more details.
- #
- # You should have received a copy of the GNU General Public License
- # along with this program. If not, see <http://www.gnu.org/licenses/>.
- #
- #########################################################################
- import io
- import os
- import shutil
- import gisdata
- import logging
- import zipfile
- from uuid import uuid4
- from unittest.mock import MagicMock, patch
- from collections import namedtuple
- from pinax.ratings.models import OverallRating
- from django.urls import reverse
- from django.test import TestCase
- from django.forms import ValidationError
- from django.test.client import RequestFactory
- from django.contrib.contenttypes.models import ContentType
- from django.core.files.uploadedfile import SimpleUploadedFile
- from django.contrib.auth.models import Group
- from django.contrib.gis.geos import Polygon
- from django.db.models import Count
- from django.contrib.auth import get_user_model
- from django.conf import settings
- from django.test.utils import override_settings
- from django.contrib.admin.sites import AdminSite
- from geonode.geoserver.createlayer.utils import create_dataset
- from geonode.layers import utils
- from geonode.base import enumerations
- from geonode.layers import DatasetAppConfig
- from geonode.layers.admin import DatasetAdmin
- from geonode.decorators import on_ogc_backend
- from geonode.maps.models import Map, MapLayer
- from geonode.utils import DisableDjangoSignals, mkdtemp
- from geonode.layers.views import _resolve_dataset
- from geonode import GeoNodeException, geoserver
- from geonode.people.utils import get_valid_user
- from guardian.shortcuts import get_anonymous_user
- from geonode.tests.base import GeoNodeBaseTestSupport
- from geonode.resource.manager import resource_manager
- from geonode.tests.utils import NotificationsTestsHelper
- from geonode.layers.models import Dataset, Style, Attribute
- from geonode.layers.forms import DatasetForm, JSONField, LayerUploadForm
- from geonode.layers.populate_datasets_data import create_dataset_data
- from geonode.base.models import TopicCategory, License, Region, Link
- from geonode.utils import check_ogc_backend, set_resource_default_links
- from geonode.layers.metadata import convert_keyword, set_metadata, parse_metadata
- from geonode.layers.utils import (
- is_sld_upload_only,
- is_xml_upload_only,
- dataset_type,
- get_files,
- get_valid_name,
- get_valid_dataset_name,
- surrogate_escape_string, validate_input_source)
- from geonode.base.populate_test_data import (
- all_public,
- create_models,
- remove_models,
- create_single_dataset)
- logger = logging.getLogger(__name__)
- class DatasetsTest(GeoNodeBaseTestSupport):
- """Tests geonode.layers app/module
- """
- type = 'dataset'
- fixtures = [
- 'initial_data.json',
- 'group_test_data.json',
- 'default_oauth_apps.json'
- ]
- @classmethod
- def setUpClass(cls):
- super().setUpClass()
- create_models(type=cls.get_type, integration=cls.get_integration)
- all_public()
- @classmethod
- def tearDownClass(cls):
- super().tearDownClass()
- remove_models(cls.get_obj_ids, type=cls.get_type, integration=cls.get_integration)
- def setUp(self):
- super().setUp()
- self.user = 'admin'
- self.passwd = 'admin'
- self.anonymous_user = get_anonymous_user()
- self.exml_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml"
- self.sld_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_sld.sld"
- self.maxDiff = None
- self.sut = create_single_dataset("single_point")
- create_dataset_data(self.sut.resourcebase_ptr_id)
- create_dataset_data(Dataset.objects.first().resourcebase_ptr_id)
- self.r = namedtuple('GSCatalogRes', ['resource'])
- site = AdminSite()
- self.admin = DatasetAdmin(Dataset, site)
- self.request_admin = RequestFactory().get('/admin')
- self.request_admin.user = get_user_model().objects.get(username='admin')
- # Admin Tests
- def test_admin_save_model(self):
- obj = Dataset.objects.first()
- self.assertEqual(len(obj.keywords.all()), 2)
- form = self.admin.get_form(self.request_admin, obj=obj, change=True)
- self.admin.save_model(self.request_admin, obj, form, True)
- def test_default_sourcetype(self):
- obj = Dataset.objects.first()
- self.assertEqual(obj.sourcetype, enumerations.SOURCE_TYPE_LOCAL)
- # Data Tests
- def test_describe_data_2(self):
- '''/data/geonode:CA/metadata -> Test accessing the description of a layer '''
- self.assertEqual(10, get_user_model().objects.all().count())
- response = self.client.get(reverse('dataset_metadata', args=('geonode:CA',)))
- # Since we are not authenticated, we should not be able to access it
- self.assertEqual(response.status_code, 302)
- # but if we log in ...
- self.client.login(username='admin', password='admin')
- # ... all should be good
- response = self.client.get(reverse('dataset_metadata', args=('geonode:CA',)))
- self.assertEqual(response.status_code, 200)
- def test_describe_data_3(self):
- '''/data/geonode:CA/metadata_detail -> Test accessing the description of a layer '''
- self.client.login(username='admin', password='admin')
- # ... all should be good
- response = self.client.get(reverse('dataset_metadata_detail', args=('geonode:CA',)))
- self.assertEqual(response.status_code, 200)
- self.assertContains(response, "Approved", count=1, status_code=200, msg_prefix='', html=False)
- self.assertContains(response, "Published", count=1, status_code=200, msg_prefix='', html=False)
- self.assertContains(response, "Featured", count=3, status_code=200, msg_prefix='', html=False)
- self.assertContains(response, "<dt>Group</dt>", count=0, status_code=200, msg_prefix='', html=False)
- # ... now assigning a Group to the Dataset
- lyr = Dataset.objects.get(alternate='geonode:CA')
- group = Group.objects.first()
- lyr.group = group
- lyr.save()
- response = self.client.get(reverse('dataset_metadata_detail', args=('geonode:CA',)))
- self.assertEqual(response.status_code, 200)
- self.assertContains(response, "<dt>Group</dt>", count=1, status_code=200, msg_prefix='', html=False)
- lyr.group = None
- lyr.save()
- # Dataset Tests
- def test_dataset_name_clash(self):
- _ll_1 = Dataset.objects.create(
- uuid=str(uuid4()),
- owner=get_user_model().objects.get(username=self.user),
- name='states',
- store='geonode_data',
- subtype="vector",
- alternate="geonode:states"
- )
- _ll_2 = Dataset.objects.create(
- uuid=str(uuid4()),
- owner=get_user_model().objects.get(username=self.user),
- name='geonode:states',
- store='httpfooremoteservce',
- subtype="remote",
- alternate="geonode:states"
- )
- _ll_1.set_permissions({'users': {"bobby": ['base.view_resourcebase']}})
- _ll_2.set_permissions({'users': {"bobby": ['base.view_resourcebase']}})
- self.client.login(username="bobby", password="bob")
- _request = self.client.request()
- _request.user = get_user_model().objects.get(username="bobby")
- _ll = _resolve_dataset(_request, alternate="geonode:states")
- self.assertIsNotNone(_ll)
- self.assertEqual(_ll.name, _ll_1.name)
- def test_describe_data(self):
- '''/data/geonode:CA/metadata -> Test accessing the description of a layer '''
- self.assertEqual(10, get_user_model().objects.all().count())
- response = self.client.get(reverse('dataset_metadata', args=('geonode:CA',)))
- # Since we are not authenticated, we should not be able to access it
- self.assertEqual(response.status_code, 302)
- # but if we log in ...
- self.client.login(username='admin', password='admin')
- # ... all should be good
- response = self.client.get(reverse('dataset_metadata', args=('geonode:CA',)))
- self.assertEqual(response.status_code, 200)
- def test_dataset_attributes(self):
- lyr = Dataset.objects.all().first()
- # There should be a total of 3 attributes
- self.assertEqual(len(lyr.attribute_set.all()), 4)
- # 2 out of 3 attributes should be visible
- custom_attributes = lyr.attribute_set.visible()
- self.assertEqual(len(custom_attributes), 3)
- # place_ name should come before description
- self.assertEqual(custom_attributes[0].attribute_label, "Place Name")
- self.assertEqual(custom_attributes[1].attribute_label, "Description")
- self.assertEqual(
- custom_attributes[2].attribute,
- 'N\xfamero_De_M\xe9dicos')
- # TODO: do test against layer with actual attribute statistics
- self.assertEqual(custom_attributes[1].count, 1)
- self.assertEqual(custom_attributes[1].min, "NA")
- self.assertEqual(custom_attributes[1].max, "NA")
- self.assertEqual(custom_attributes[1].average, "NA")
- self.assertEqual(custom_attributes[1].median, "NA")
- self.assertEqual(custom_attributes[1].stddev, "NA")
- self.assertEqual(custom_attributes[1].sum, "NA")
- self.assertEqual(custom_attributes[1].unique_values, "NA")
- def test_dataset_bbox(self):
- lyr = Dataset.objects.all().first()
- dataset_bbox = lyr.bbox[0:4]
- logger.debug(dataset_bbox)
- def decimal_encode(bbox):
- _bbox = [float(o) for o in bbox]
- # Must be in the form : [x0, x1, y0, y1
- return [_bbox[0], _bbox[2], _bbox[1], _bbox[3]]
- from geonode.utils import bbox_to_projection
- projected_bbox = decimal_encode(
- bbox_to_projection([float(coord) for coord in dataset_bbox] + [lyr.srid, ],
- target_srid=4326)[:4])
- logger.debug(projected_bbox)
- self.assertEqual(projected_bbox, [-180.0, -90.0, 180.0, 90.0])
- logger.debug(lyr.ll_bbox)
- self.assertEqual(lyr.ll_bbox, [-180.0, 180.0, -90.0, 90.0, 'EPSG:4326'])
- projected_bbox = decimal_encode(
- bbox_to_projection([float(coord) for coord in dataset_bbox] + [lyr.srid, ],
- target_srid=3857)[:4])
- solution = [-20037397.023298454, -74299743.40065672,
- 20037397.02329845, 74299743.40061197]
- logger.debug(projected_bbox)
- for coord, check in zip(projected_bbox, solution):
- self.assertAlmostEqual(coord, check, places=3)
- def test_dataset_attributes_feature_catalogue(self):
- """ Test layer feature catalogue functionality
- """
- self.assertTrue(self.client.login(username='admin', password='admin'))
- # test a non-existing layer
- url = reverse('dataset_feature_catalogue', args=('bad_dataset',))
- response = self.client.get(url)
- self.assertEqual(response.status_code, 404)
- # Get the layer to work with
- layer = Dataset.objects.all()[3]
- url = reverse('dataset_feature_catalogue', args=(layer.alternate,))
- response = self.client.get(url)
- self.assertNotEqual(response.status_code, 404)
- def test_dataset_attribute_config(self):
- lyr = Dataset.objects.all().first()
- attribute_config = lyr.attribute_config()
- custom_attributes = attribute_config["getFeatureInfo"]
- self.assertEqual(
- custom_attributes["fields"], [
- "place_name", "description", 'N\xfamero_De_M\xe9dicos'])
- self.assertEqual(
- custom_attributes["propertyNames"]["description"],
- "Description")
- self.assertEqual(
- custom_attributes["propertyNames"]["place_name"],
- "Place Name")
- attributes = Attribute.objects.filter(dataset=lyr)
- for _att in attributes:
- self.assertEqual(_att.featureinfo_type, 'type_property')
- lyr.featureinfo_custom_template = "<h1>Test HTML</h1>"
- lyr.use_featureinfo_custom_template = True
- lyr.save()
- attribute_config = lyr.attribute_config()
- self.assertTrue("ftInfoTemplate" in attribute_config)
- self.assertEqual(
- attribute_config["ftInfoTemplate"],
- "<h1>Test HTML</h1>")
- lyr.use_featureinfo_custom_template = False
- lyr.save()
- attribute_config = lyr.attribute_config()
- self.assertTrue("ftInfoTemplate" not in attribute_config)
- def test_dataset_styles(self):
- lyr = Dataset.objects.all().first()
- # There should be a total of 3 styles
- self.assertEqual(len(lyr.styles.all()), 4)
- # One of the style is the default one
- self.assertEqual(
- lyr.default_style,
- Style.objects.get(
- id=lyr.default_style.id))
- try:
- [str(style) for style in lyr.styles.all()]
- except UnicodeEncodeError:
- self.fail(
- "str of the Style model throws a UnicodeEncodeError with special characters.")
- def test_dataset_links(self):
- lyr = Dataset.objects.filter(subtype="vector").first()
- self.assertEqual(lyr.subtype, "vector")
- if check_ogc_backend(geoserver.BACKEND_PACKAGE):
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata")
- self.assertIsNotNone(links)
- for ll in links:
- self.assertEqual(ll.link_type, "metadata")
- _def_link_types = (
- 'data', 'image', 'original', 'html', 'OGC:WMS', 'OGC:WFS', 'OGC:WCS')
- Link.objects.filter(resource=lyr.resourcebase_ptr, link_type__in=_def_link_types).delete()
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data")
- self.assertIsNotNone(links)
- set_resource_default_links(lyr, lyr)
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata")
- self.assertIsNotNone(links)
- for ll in links:
- self.assertEqual(ll.link_type, "metadata")
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data")
- self.assertIsNotNone(links)
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image")
- self.assertIsNotNone(links)
- lyr = Dataset.objects.filter(subtype="raster").first()
- self.assertEqual(lyr.subtype, "raster")
- if check_ogc_backend(geoserver.BACKEND_PACKAGE):
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata")
- self.assertIsNotNone(links)
- for ll in links:
- self.assertEqual(ll.link_type, "metadata")
- _def_link_types = (
- 'data', 'image', 'original', 'html', 'OGC:WMS', 'OGC:WFS', 'OGC:WCS')
- Link.objects.filter(resource=lyr.resourcebase_ptr, link_type__in=_def_link_types).delete()
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data")
- self.assertIsNotNone(links)
- set_resource_default_links(lyr, lyr)
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="metadata")
- self.assertIsNotNone(links)
- for ll in links:
- self.assertEqual(ll.link_type, "metadata")
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="data")
- self.assertIsNotNone(links)
- links = Link.objects.filter(resource=lyr.resourcebase_ptr, link_type="image")
- self.assertIsNotNone(links)
- def test_get_valid_user(self):
- # Verify it accepts an admin user
- adminuser = get_user_model().objects.get(is_superuser=True)
- valid_user = get_valid_user(adminuser)
- msg = (f'Passed in a valid admin user "{adminuser}" but got "{valid_user}" in return')
- assert valid_user.id == adminuser.id, msg
- # Verify it returns a valid user after receiving None
- valid_user = get_valid_user(None)
- msg = f'Expected valid user after passing None, got "{valid_user}"'
- assert isinstance(valid_user, get_user_model()), msg
- newuser = get_user_model().objects.create(username='arieluser')
- valid_user = get_valid_user(newuser)
- msg = (f'Passed in a valid user "{newuser}" but got "{valid_user}" in return')
- assert valid_user.id == newuser.id, msg
- valid_user = get_valid_user('arieluser')
- msg = ('Passed in a valid user by username "arieluser" but got'
- f' "{valid_user}" in return')
- assert valid_user.username == 'arieluser', msg
- nn = get_anonymous_user()
- self.assertRaises(GeoNodeException, get_valid_user, nn)
- def testShapefileValidation(self):
- files = dict(
- base_file=SimpleUploadedFile('foo.shp', b' '),
- shx_file=SimpleUploadedFile('foo.shx', b' '),
- dbf_file=SimpleUploadedFile('foo.dbf', b' '),
- prj_file=SimpleUploadedFile('foo.prj', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.SHP', b' '),
- shx_file=SimpleUploadedFile('foo.SHX', b' '),
- dbf_file=SimpleUploadedFile('foo.DBF', b' '),
- prj_file=SimpleUploadedFile('foo.PRJ', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.SHP', b' '),
- shx_file=SimpleUploadedFile('foo.shx', b' '),
- dbf_file=SimpleUploadedFile('foo.dbf', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.SHP', b' '),
- shx_file=SimpleUploadedFile('foo.shx', b' '),
- dbf_file=SimpleUploadedFile('foo.dbf', b' '),
- prj_file=SimpleUploadedFile('foo.PRJ', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.SHP', b' '),
- shx_file=SimpleUploadedFile('bar.shx', b' '),
- dbf_file=SimpleUploadedFile('bar.dbf', b' '),
- prj_file=SimpleUploadedFile('bar.PRJ', b' '))
- self.assertFalse(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.shp', b' '),
- dbf_file=SimpleUploadedFile('foo.dbf', b' '),
- prj_file=SimpleUploadedFile('foo.PRJ', b' '))
- self.assertFalse(LayerUploadForm(dict(), files).is_valid())
- files = dict(
- base_file=SimpleUploadedFile('foo.txt', b' '),
- shx_file=SimpleUploadedFile('foo.shx', b' '),
- dbf_file=SimpleUploadedFile('foo.sld', b' '),
- prj_file=SimpleUploadedFile('foo.prj', b' '))
- self.assertFalse(LayerUploadForm(dict(), files).is_valid())
- def testGeoTiffValidation(self):
- files = dict(base_file=SimpleUploadedFile('foo.tif', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.TIF', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.tiff', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.TIF', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.geotif', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.GEOTIF', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.geotiff', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.GEOTIF', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- def testASCIIValidation(self):
- files = dict(base_file=SimpleUploadedFile('foo.asc', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- files = dict(base_file=SimpleUploadedFile('foo.ASC', b' '))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- def testZipValidation(self):
- the_zip = zipfile.ZipFile('test_upload.zip', 'w')
- in_memory_file = io.StringIO()
- in_memory_file.write('test')
- the_zip.writestr('foo.shp', in_memory_file.getvalue())
- the_zip.writestr('foo.dbf', in_memory_file.getvalue())
- the_zip.writestr('foo.shx', in_memory_file.getvalue())
- the_zip.writestr('foo.prj', in_memory_file.getvalue())
- the_zip.close()
- files = dict(base_file=SimpleUploadedFile('test_upload.zip',
- open('test_upload.zip', mode='rb').read()))
- self.assertTrue(LayerUploadForm(dict(), files).is_valid())
- os.remove('test_upload.zip')
- def testWriteFiles(self):
- files = dict(
- base_file=SimpleUploadedFile('foo.shp', b' '),
- shx_file=SimpleUploadedFile('foo.shx', b' '),
- dbf_file=SimpleUploadedFile('foo.dbf', b' '),
- prj_file=SimpleUploadedFile('foo.prj', b' '))
- form = LayerUploadForm(dict(), files)
- self.assertTrue(form.is_valid())
- tempdir = form.write_files()[0]
- self.assertEqual(set(os.listdir(tempdir)),
- {'foo.shp', 'foo.shx', 'foo.dbf', 'foo.prj'})
- the_zip = zipfile.ZipFile('test_upload.zip', 'w')
- in_memory_file = io.StringIO()
- in_memory_file.write('test')
- the_zip.writestr('foo.shp', in_memory_file.getvalue())
- the_zip.writestr('foo.dbf', in_memory_file.getvalue())
- the_zip.writestr('foo.shx', in_memory_file.getvalue())
- the_zip.writestr('foo.prj', in_memory_file.getvalue())
- the_zip.close()
- files = dict(base_file=SimpleUploadedFile('test_upload.zip',
- open('test_upload.zip', mode='rb').read()))
- form = LayerUploadForm(dict(), files)
- self.assertTrue(form.is_valid())
- tempdir = form.write_files()[0]
- self.assertEqual(set(os.listdir(tempdir)),
- {'foo.shp', 'foo.shx', 'foo.dbf', 'foo.prj'})
- os.remove('test_upload.zip')
- def test_dataset_type(self):
- self.assertEqual(dataset_type('foo.shp'), 'vector')
- self.assertEqual(dataset_type('foo.SHP'), 'vector')
- self.assertEqual(dataset_type('foo.sHp'), 'vector')
- self.assertEqual(dataset_type('foo.tif'), 'raster')
- self.assertEqual(dataset_type('foo.TIF'), 'raster')
- self.assertEqual(dataset_type('foo.TiF'), 'raster')
- self.assertEqual(dataset_type('foo.geotif'), 'raster')
- self.assertEqual(dataset_type('foo.GEOTIF'), 'raster')
- self.assertEqual(dataset_type('foo.gEoTiF'), 'raster')
- self.assertEqual(dataset_type('foo.tiff'), 'raster')
- self.assertEqual(dataset_type('foo.TIFF'), 'raster')
- self.assertEqual(dataset_type('foo.TiFf'), 'raster')
- self.assertEqual(dataset_type('foo.geotiff'), 'raster')
- self.assertEqual(dataset_type('foo.GEOTIFF'), 'raster')
- self.assertEqual(dataset_type('foo.gEoTiFf'), 'raster')
- self.assertEqual(dataset_type('foo.asc'), 'raster')
- self.assertEqual(dataset_type('foo.ASC'), 'raster')
- self.assertEqual(dataset_type('foo.AsC'), 'raster')
- # basically anything else should produce a GeoNodeException
- self.assertRaises(GeoNodeException, lambda: dataset_type('foo.gml'))
- def test_get_files(self):
- def generate_files(*extensions):
- if extensions[0].lower() != 'shp':
- return
- d = None
- expected_files = None
- try:
- d = mkdtemp()
- fnames = [f"foo.{ext}" for ext in extensions]
- expected_files = {ext.lower(): fname for ext, fname in zip(extensions, fnames)}
- for f in fnames:
- path = os.path.join(d, f)
- # open and immediately close to create empty file
- open(path, 'w').close()
- finally:
- return d, expected_files
- # Check that a well-formed Shapefile has its components all picked up
- d = None
- _tmpdir = None
- try:
- d, expected_files = generate_files("shp", "shx", "prj", "dbf")
- gotten_files, _tmpdir = get_files(os.path.join(d, "foo.shp"))
- gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()}
- self.assertEqual(gotten_files, expected_files)
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- if _tmpdir is not None:
- shutil.rmtree(_tmpdir, ignore_errors=True)
- # Check that a Shapefile missing required components raises an
- # exception
- d = None
- try:
- d, expected_files = generate_files("shp", "shx", "prj")
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp")))
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- # Check that including an SLD with a valid shapefile results in the SLD
- # getting picked up
- d = None
- _tmpdir = None
- try:
- if check_ogc_backend(geoserver.BACKEND_PACKAGE):
- d, expected_files = generate_files("shp", "shx", "prj", "dbf", "sld")
- gotten_files, _tmpdir = get_files(os.path.join(d, "foo.shp"))
- gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()}
- self.assertEqual(gotten_files, expected_files)
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- if _tmpdir is not None:
- shutil.rmtree(_tmpdir, ignore_errors=True)
- # Check that capitalized extensions are ok
- d = None
- _tmpdir = None
- try:
- d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF")
- gotten_files, _tmpdir = get_files(os.path.join(d, "foo.SHP"))
- gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()}
- self.assertEqual(gotten_files, expected_files)
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- if _tmpdir is not None:
- shutil.rmtree(_tmpdir, ignore_errors=True)
- # Check that mixed capital and lowercase extensions are ok
- d = None
- _tmpdir = None
- try:
- d, expected_files = generate_files("SHP", "shx", "pRJ", "DBF")
- gotten_files, _tmpdir = get_files(os.path.join(d, "foo.SHP"))
- gotten_files = {k: os.path.basename(v) for k, v in gotten_files.items()}
- self.assertEqual(gotten_files, expected_files)
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- if _tmpdir is not None:
- shutil.rmtree(_tmpdir, ignore_errors=True)
- # Check that including both capital and lowercase extensions raises an
- # exception
- d = None
- try:
- d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "shp", "shx", "prj", "dbf")
- # Only run the tests if this is a case sensitive OS
- if len(os.listdir(d)) == len(expected_files):
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP")))
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp")))
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- # Check that including both capital and lowercase PRJ (this is
- # special-cased in the implementation)
- d = None
- try:
- d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "prj")
- # Only run the tests if this is a case sensitive OS
- if len(os.listdir(d)) == len(expected_files):
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP")))
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp")))
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- # Check that including both capital and lowercase SLD (this is
- # special-cased in the implementation)
- d = None
- try:
- if check_ogc_backend(geoserver.BACKEND_PACKAGE):
- d, expected_files = generate_files("SHP", "SHX", "PRJ", "DBF", "SLD", "sld")
- # Only run the tests if this is a case sensitive OS
- if len(os.listdir(d)) == len(expected_files):
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.SHP")))
- self.assertRaises(GeoNodeException, lambda: get_files(os.path.join(d, "foo.shp")))
- finally:
- if d is not None:
- shutil.rmtree(d, ignore_errors=True)
- def test_get_valid_name(self):
- self.assertEqual(get_valid_name("blug"), "blug")
- self.assertEqual(get_valid_name("<-->"), "_")
- self.assertEqual(get_valid_name("<ab>"), "_ab_")
- self.assertNotEqual(get_valid_name("CA"), "CA_1")
- self.assertNotEqual(get_valid_name("CA"), "CA_1")
- def test_get_valid_dataset_name(self):
- self.assertEqual(get_valid_dataset_name("blug", False), "blug")
- self.assertEqual(get_valid_dataset_name("blug", True), "blug")
- self.assertEqual(get_valid_dataset_name("<ab>", False), "_ab_")
- self.assertEqual(get_valid_dataset_name("<ab>", True), "<ab>")
- self.assertEqual(get_valid_dataset_name("<-->", False), "_")
- self.assertEqual(get_valid_dataset_name("<-->", True), "<-->")
- self.assertNotEqual(get_valid_dataset_name("CA", False), "CA_1")
- self.assertNotEqual(get_valid_dataset_name("CA", False), "CA_1")
- self.assertEqual(get_valid_dataset_name("CA", True), "CA")
- self.assertEqual(get_valid_dataset_name("CA", True), "CA")
- layer = Dataset.objects.get(name="CA")
- self.assertNotEqual(get_valid_dataset_name(layer, False), "CA_1")
- self.assertEqual(get_valid_dataset_name(layer, True), "CA")
- self.assertRaises(GeoNodeException, get_valid_dataset_name, 12, False)
- self.assertRaises(GeoNodeException, get_valid_dataset_name, 12, True)
- # NOTE: we don't care about file content for many of these tests (the
- # forms under test validate based only on file name, and leave actual
- # content inspection to GeoServer) but Django's form validation will omit
- # any files with empty bodies.
- #
- # That is, this leads to mysterious test failures:
- # SimpleUploadedFile('foo', ' '.encode("UTF-8"))
- #
- # And this should be used instead to avoid that:
- # SimpleUploadedFile('foo', ' '.encode("UTF-8"))
- def testJSONField(self):
- field = JSONField()
- # a valid JSON document should pass
- field.clean('{ "users": [] }')
- # text which is not JSON should fail
- self.assertRaises(
- ValidationError,
- lambda: field.clean('<users></users>'))
- def test_rating_dataset_remove(self):
- """ Test layer rating is removed on layer remove
- """
- # Get the layer to work with
- layer = Dataset.objects.all()[3]
- dataset_id = layer.id
- # Create the rating with the correct content type
- ctype = ContentType.objects.get(model='dataset')
- OverallRating.objects.create(
- category=2,
- object_id=dataset_id,
- content_type=ctype,
- rating=3)
- rating = OverallRating.objects.all()
- self.assertEqual(rating.count(), 1)
- # Remove the layer
- resource_manager.delete(layer.uuid)
- # Check there are no ratings matching the remove layer
- rating = OverallRating.objects.all()
- self.assertEqual(rating.count(), 0)
- def test_sld_upload(self):
- """Test layer remove functionality
- """
- layer = Dataset.objects.all().first()
- url = reverse('dataset_sld_upload', args=(layer.alternate,))
- # Now test with a valid user
- self.client.login(username='admin', password='admin')
- # test a method other than POST and GET
- response = self.client.put(url)
- content = response.content.decode('utf-8')
- self.assertEqual(response.status_code, 200)
- self.assertFalse("#modal_perms" in content)
- def test_category_counts(self):
- topics = TopicCategory.objects.all()
- topics = topics.annotate(
- **{'dataset_count': Count('resourcebase__dataset__category')})
- location = topics.get(identifier='location')
- # there are three layers with location category
- self.assertEqual(location.dataset_count, 3)
- # change the category of one layers_count
- layer = Dataset.objects.filter(category=location)[0]
- elevation = topics.get(identifier='elevation')
- layer.category = elevation
- layer.save()
- # reload the categories since it's caching the old count
- topics = topics.annotate(
- **{'dataset_count': Count('resourcebase__dataset__category')})
- location = topics.get(identifier='location')
- elevation = topics.get(identifier='elevation')
- self.assertEqual(location.dataset_count, 2)
- self.assertEqual(elevation.dataset_count, 4)
- # delete a layer and check the count update
- # use the first since it's the only one which has styles
- layer = Dataset.objects.all().first()
- elevation = topics.get(identifier='elevation')
- self.assertEqual(elevation.dataset_count, 4)
- layer.delete()
- topics = topics.annotate(
- **{'dataset_count': Count('resourcebase__dataset__category')})
- elevation = topics.get(identifier='elevation')
- self.assertEqual(elevation.dataset_count, 3)
- def test_assign_change_dataset_data_perm(self):
- """
- Ensure set_permissions supports the change_dataset_data permission.
- """
- layer = Dataset.objects.first()
- user = get_anonymous_user()
- layer.set_permissions({'users': {user.username: ['change_dataset_data']}})
- perms = layer.get_all_level_info()
- self.assertNotIn(user, perms['users'])
- self.assertNotIn(user.username, perms['users'])
- def test_batch_edit(self):
- """
- Test batch editing of metadata fields.
- """
- Model = Dataset
- view = 'dataset_batch_metadata'
- resources = Model.objects.all()[:3]
- ids = ','.join(str(element.pk) for element in resources)
- # test non-admin access
- self.client.login(username="bobby", password="bob")
- response = self.client.get(reverse(view))
- self.assertTrue(response.status_code in (401, 403))
- # test group change
- group = Group.objects.first()
- self.client.login(username='admin', password='admin')
- response = self.client.post(
- reverse(view),
- data={'group': group.pk, 'ids': ids, 'regions': 1},
- )
- self.assertEqual(response.status_code, 302)
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- self.assertEqual(resource.group, group)
- # test owner change
- owner = get_user_model().objects.first()
- response = self.client.post(
- reverse(view),
- data={'owner': owner.pk, 'ids': ids, 'regions': 1},
- )
- self.assertEqual(response.status_code, 302)
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- self.assertEqual(resource.owner, owner)
- # test license change
- license = License.objects.first()
- response = self.client.post(
- reverse(view),
- data={'license': license.pk, 'ids': ids, 'regions': 1},
- )
- self.assertEqual(response.status_code, 302)
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- self.assertEqual(resource.license, license)
- # test regions change
- region = Region.objects.first()
- response = self.client.post(
- reverse(view),
- data={'region': region.pk, 'ids': ids, 'regions': 1},
- )
- self.assertEqual(response.status_code, 302)
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- if resource.regions.all():
- self.assertTrue(region in resource.regions.all())
- # test language change
- language = 'eng'
- response = self.client.post(
- reverse(view),
- data={'language': language, 'ids': ids, 'regions': 1},
- )
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- self.assertEqual(resource.language, language)
- # test keywords change
- keywords = 'some,thing,new'
- response = self.client.post(
- reverse(view),
- data={'keywords': keywords, 'ids': ids, 'regions': 1},
- )
- resources = Model.objects.filter(id__in=[r.pk for r in resources])
- for resource in resources:
- for word in resource.keywords.all():
- self.assertTrue(word.name in keywords.split(','))
- def test_surrogate_escape_string(self):
- surrogate_escape_raw = "Zo\udcc3\udcab"
- surrogate_escape_expected = "Zoƫ"
- surrogate_escape_result = surrogate_escape_string(
- surrogate_escape_raw, 'UTF-8') # add more test cases using different charsets?
- self.assertEqual(
- surrogate_escape_result,
- surrogate_escape_expected,
- "layers.utils.surrogate_escape_string did not produce expected result. "
- f"Expected {surrogate_escape_expected}, received {surrogate_escape_result}")
- @on_ogc_backend(geoserver.BACKEND_PACKAGE)
- def test_assign_remove_permissions(self):
- # Assing
- layer = Dataset.objects.all().first()
- perm_spec = layer.get_all_level_info()
- self.assertNotIn(get_user_model().objects.get(username="norman"), perm_spec["users"])
- utils.set_datasets_permissions("write", resources_names=[layer.name], users_usernames=["norman"], delete_flag=False, verbose=True)
- perm_spec = layer.get_all_level_info()
- _c = 0
- if "users" in perm_spec:
- for _u in perm_spec["users"]:
- if _u == "norman" or _u == get_user_model().objects.get(username="norman"):
- _c += 1
- # "norman" has both read & write permissions
- self.assertEqual(_c, 1)
- # Remove
- utils.set_datasets_permissions("read", resources_names=[layer.name], users_usernames=["norman"], delete_flag=True, verbose=True)
- perm_spec = layer.get_all_level_info()
- _c = 0
- if "users" in perm_spec:
- for _u in perm_spec["users"]:
- if _u == "norman" or _u == get_user_model().objects.get(username="norman"):
- _c += 1
- # "norman" has no permissions
- self.assertEqual(_c, 0)
- def test_xml_form_without_files_should_raise_500(self):
- files = dict()
- files['permissions'] = '{}'
- files['charset'] = 'utf-8'
- self.client.login(username="admin", password="admin")
- resp = self.client.post(reverse('dataset_upload'), data=files)
- self.assertEqual(500, resp.status_code)
- def test_xml_should_return_404_if_the_dataset_does_not_exists(self):
- params = {
- "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
- "base_file": open(self.exml_path),
- "xml_file": open(self.exml_path),
- "dataset_title": "Fake layer title",
- "metadata_upload_form": True,
- "time": False,
- "charset": "UTF-8"
- }
- self.client.login(username="admin", password="admin")
- resp = self.client.post(reverse('dataset_upload'), params)
- self.assertEqual(404, resp.status_code)
- def test_xml_should_update_the_dataset_with_the_expected_values(self):
- params = {
- "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
- "base_file": open(self.exml_path),
- "xml_file": open(self.exml_path),
- "dataset_title": "geonode:single_point",
- "metadata_upload_form": True,
- "time": False,
- "charset": "UTF-8"
- }
- self.client.login(username="admin", password="admin")
- prev_dataset = Dataset.objects.get(typename="geonode:single_point")
- self.assertEqual(0, prev_dataset.keywords.count())
- resp = self.client.post(reverse('dataset_upload'), params)
- self.assertEqual(404, resp.status_code)
- self.assertEqual(resp.json()["errors"], "The UUID identifier from the XML Metadata, is different from the one saved")
- def test_sld_should_raise_500_if_is_invalid(self):
- layer = Dataset.objects.get(typename="geonode:single_point")
- params = {
- "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
- "base_file": open(self.sld_path),
- "sld_file": open(self.sld_path),
- "dataset_title": "random",
- "metadata_upload_form": False,
- "time": False,
- "charset": "UTF-8"
- }
- self.client.login(username="admin", password="admin")
- self.assertGreaterEqual(layer.styles.count(), 1)
- self.assertIsNotNone(layer.styles.first())
- resp = self.client.post(reverse('dataset_upload'), params)
- self.assertEqual(500, resp.status_code)
- self.assertFalse(resp.json().get('success'))
- self.assertEqual('No Dataset matches the given query.', resp.json().get('errors'))
- def test_sld_should_update_the_dataset_with_the_expected_values(self):
- layer = Dataset.objects.get(typename="geonode:single_point")
- params = {
- "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
- "base_file": open(self.sld_path),
- "sld_file": open(self.sld_path),
- "dataset_title": f"geonode:{layer.name}",
- "metadata_upload_form": False,
- "time": False,
- "charset": "UTF-8"
- }
- self.client.login(username="admin", password="admin")
- self.assertGreaterEqual(layer.styles.count(), 1)
- self.assertIsNotNone(layer.styles.first())
- resp = self.client.post(reverse('dataset_upload'), params)
- self.assertEqual(200, resp.status_code)
- updated_dataset = Dataset.objects.get(alternate=f"geonode:{layer.name}")
- # just checking some values if are updated
- self.assertGreaterEqual(updated_dataset.styles.all().count(), 1)
- self.assertIsNotNone(updated_dataset.styles.first())
- self.assertEqual(layer.styles.first().sld_title, updated_dataset.styles.first().sld_title)
- def test_xml_should_raise_an_error_if_the_uuid_is_changed(self):
- '''
- If the UUID coming from the XML and the one saved in the DB are different
- The system should raise an error
- '''
- params = {
- "permissions": '{ "users": {"AnonymousUser": ["view_resourcebase"]} , "groups":{}}',
- "base_file": open(self.exml_path),
- "xml_file": open(self.exml_path),
- "dataset_title": "geonode:single_point",
- "metadata_upload_form": True,
- "time": False,
- "charset": "UTF-8"
- }
- self.client.login(username="admin", password="admin")
- prev_dataset = Dataset.objects.get(typename="geonode:single_point")
- self.assertEqual(0, prev_dataset.keywords.count())
- resp = self.client.post(reverse('dataset_upload'), params)
- self.assertEqual(404, resp.status_code)
- expected = {
- "success": False,
- "errors": "The UUID identifier from the XML Metadata, is different from the one saved"
- }
- self.assertDictEqual(expected, resp.json())
- def test_will_raise_exception_for_replace_vector_dataset_with_raster(self):
- layer = Dataset.objects.get(name="single_point")
- filename = "/tpm/filename.tif"
- files = ["/opt/file1.shp", "/opt/file2.ccc"]
- with self.assertRaises(Exception) as e:
- validate_input_source(layer, filename, files, action_type="append")
- expected = "You are attempting to append a vector dataset with a raster."
- self.assertEqual(expected, e.exception.args[0])
- def test_will_raise_exception_for_replace_dataset_with_unknown_format(self):
- layer = Dataset.objects.get(name="single_point")
- filename = "/tpm/filename.ccc"
- file_path = gisdata.VECTOR_DATA
- files = {
- "shp": filename,
- "dbf": f"{file_path}/san_andres_y_providencia_highway.asd",
- "prj": f"{file_path}/san_andres_y_providencia_highway.asd",
- "shx": f"{file_path}/san_andres_y_providencia_highway.asd",
- }
- with self.assertRaises(Exception) as e:
- validate_input_source(layer, filename, files, action_type="append")
- expected = "You are attempting to append a vector dataset with an unknown format."
- self.assertEqual(expected, e.exception.args[0])
- def test_will_raise_exception_for_replace_dataset_with_different_file_name(self):
- layer = Dataset.objects.get(name="single_point")
- file_path = gisdata.VECTOR_DATA
- filename = os.path.join(file_path, "san_andres_y_providencia_highway.shp")
- files = {
- "shp": filename,
- "dbf": f"{file_path}/san_andres_y_providencia_highway.sbf",
- "prj": f"{file_path}/san_andres_y_providencia_highway.prj",
- "shx": f"{file_path}/san_andres_y_providencia_highway.shx",
- }
- with self.assertRaises(Exception) as e:
- validate_input_source(layer, filename, files, action_type="append")
- expected = (
- "Some error occurred while trying to access the uploaded schema: "
- "Please ensure the name is consistent with the file you are trying to append."
- )
- self.assertEqual(expected, e.exception.args[0])
- @patch("geonode.layers.utils.gs_catalog")
- def test_will_raise_exception_for_not_existing_dataset_in_the_catalog(self, catalog):
- catalog.get_layer.return_value = None
- create_single_dataset("san_andres_y_providencia_water")
- layer = Dataset.objects.get(name="san_andres_y_providencia_water")
- file_path = gisdata.VECTOR_DATA
- filename = os.path.join(file_path, "san_andres_y_providencia_water.shp")
- files = {
- "shp": filename,
- "dbf": f"{file_path}/san_andres_y_providencia_water.sbf",
- "prj": f"{file_path}/san_andres_y_providencia_water.prj",
- "shx": f"{file_path}/san_andres_y_providencia_water.shx",
- }
- with self.assertRaises(Exception) as e:
- validate_input_source(layer, filename, files, action_type="append")
- expected = (
- "Some error occurred while trying to access the uploaded schema: "
- "The selected Dataset does not exists in the catalog."
- )
- self.assertEqual(expected, e.exception.args[0])
- @patch("geonode.layers.utils.gs_catalog")
- def test_will_raise_exception_if_schema_is_not_equal_between_catalog_and_file(self, catalog):
- attr = namedtuple('GSCatalogAttr', ['attributes'])
- attr.attributes = []
- self.r.resource = attr
- catalog.get_layer.return_value = self.r
- create_single_dataset("san_andres_y_providencia_water")
- layer = Dataset.objects.filter(name="san_andres_y_providencia_water")[0]
- file_path = gisdata.VECTOR_DATA
- filename = os.path.join(file_path, "san_andres_y_providencia_water.shp")
- files = {
- "shp": filename,
- "dbf": f"{file_path}/san_andres_y_providencia_water.sbf",
- "prj": f"{file_path}/san_andres_y_providencia_water.prj",
- "shx": f"{file_path}/san_andres_y_providencia_water.shx",
- }
- with self.assertRaises(Exception) as e:
- validate_input_source(layer, filename, files, action_type="append")
- expected = (
- "Some error occurred while trying to access the uploaded schema: "
- "Please ensure that the dataset structure is consistent with the file you are trying to append.