/subsystem/seeddb/src/seeddb.py

Large files files are truncated, but you can click here to view the full file

# -*- coding: utf-8 -*-
#
# Copyright (C) 2003, 2004 Norwegian University of Science and Technology
# Copyright (C) 2009 UNINETT AS
#
# This file is part of Network Administration Visualized (NAV).
#
# NAV is free software: you can redistribute it and/or modify it under the
# terms of the GNU General Public License version 2 as published by the Free
# Software Foundation.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.  You should have received a copy of the GNU General Public
# License along with NAV. If not, see <http://www.gnu.org/licenses/>.
#
"""SeedDB web tool for NAV.

Contains a mod_python handler.

"""
## Imports

import nav.Snmp
import sys
import re
import copy
import initBox
from options import *
import nav.web
from nav.models import manage, cabling, oid, service
import nav.util

try:
    from mod_python import util, apache
except ImportError:
    apache = None
    util = None

from seeddbSQL import *
from socket import gethostbyaddr,gethostbyname,gaierror
from nav.web.serviceHelper import getCheckers,getDescription
from nav.web.selectTree import selectTree,selectTreeLayoutBox
from nav.web.selectTree import simpleSelect,updateSelect

# Temporary fix:
mod = __import__('encodings.utf_8',globals(),locals(),'*')
mod = __import__('encodings.utf_16_be',globals(),locals(),'*')
mod = __import__('encodings.latin_1',globals(),locals(),'*')
mod = __import__('encodings.utf_16',globals(),locals(),'*')

#################################################
## Templates

from nav.web.templates.seeddbTemplate import seeddbTemplate

#################################################
## Constants

BASEPATH = '/seeddb/'
CONFIGFILE = 'seeddb.conf'

EDITPATH = [('Home','/'), ('Seed Database',BASEPATH)]

ADDNEW_ENTRY = 'addnew_entry'
UPDATE_ENTRY = 'update_entry'
IGNORE_BOX = 'ignore_this_box'

# Bulk import images
BULK_IMG_GREEN = '/images/lys/green.png'
BULK_IMG_YELLOW = '/images/lys/yellow.png'
BULK_IMG_RED = '/images/lys/red.png'

# Bulk import status
BULK_STATUS_OK = 1
BULK_STATUS_YELLOW_ERROR = 2
BULK_STATUS_RED_ERROR = 3
# Bulk fieldname for unspecified fields
BULK_UNSPECIFIED_FIELDNAME = 'excess'

# REQ_TRUE: a required field
# REQ_FALSE: not required
# REQ_NONEMPTY: not required, but don't insert empty field into db
REQ_TRUE = 1
REQ_FALSE = 2
REQ_NONEMPTY = 3

# Fieldtypes
FIELD_STRING = 1
FIELD_INTEGER = 2

#################################################
## Functions

def handler(req):
    ''' mod_python handler '''

    path = req.uri
    match = re.search('seeddb/(.+)$',path)
    if match:
        request = match.group(1)
        request = request.split('/')
    else:
        request = ""

    # Read configuration file
    #if not globals().has_key('CONFIG_CACHED'):
    readConfig()
    
    # Get form from request object
    keep_blank_values = True
    fieldStorage = util.FieldStorage(req,keep_blank_values)
    
    form = {}
    for field in fieldStorage.list:
        if form.has_key(field.name):
            # This input name already exists
            if type(form[field.name]) is list:
                # and it's already a list, so just append the
                # the new value
                form[field.name].append(str(field.value))
            else:
                # it's a string, so make a list and append the
                # new value to it
                valueList = []
                valueList.append(form[field.name])
                valueList.append(str(field.value))
                form[field.name] = valueList
        else:
            form[field.name] = str(field.value)

    # Check that all input is in the default encoding (utf8)
    unicodeError = False
    try:
        for key,value in form.items():
            if type(value) is list:
                for field in value:
                    unicode_str = field.decode(DEFAULT_ENCODING)
            else:
                unicode_str = value.decode(DEFAULT_ENCODING)
    except UnicodeError:
        # Some of the input values is not in the default encoding
        unicodeError = True
    
    # Set form in request object
    req.form = form

    output = None
    showHelp = False
    if len(request) == 2:
        if request[0] == 'help':
            showHelp = True
            request = []
            
    if not len(request) > 1:
        output = index(req,showHelp)
    else:        
        table = request[0]
        action = request[1]

        if table == 'bulk':
            output = bulkImport(req,action)
        elif pageList.has_key(table):
            output = editPage(req,pageList[table](),request,unicodeError)

    if output:
        req.content_type = "text/html"
        req.write(output)
        return apache.OK
    else:
        return apache.HTTP_NOT_FOUND

def readConfig():
    ''' Reads configuration from seeddb.conf and sets global
        variables. '''
    global CONFIG_CACHED,DEFAULT_ENCODING,BULK_TRY_ENCODINGS,\
           CATEGORY_LIST,SPLIT_LIST,SPLIT_OPPOSITE

    config = nav.config.readConfig(CONFIGFILE)
    CONFIG_CACHED = True
    
    DEFAULT_ENCODING = config['default_encoding']
    BULK_TRY_ENCODINGS = eval(config['bulk_try_encodings'])

    # Make list of cable categories
    catlist = eval(config['categories'])
    categories = []
    for cat in catlist:
        categories.append(eval(config[cat]))
    CATEGORY_LIST = categories

    # Make list of splits and dict of opposite splits
    splitlist = eval(config['splits'])
    splits = []
    opposite = {}
    for splitName in splitlist:
        split = eval(config[splitName])
        # id=0, descr=1, opposite=2
        splits.append((split[0],split[1]))
        if split[2]:
            # References another split, set id
            opposite[split[0]] = eval(config[split[2]])[0]
        else:
            opposite[split[0]] = None
    SPLIT_LIST = splits
    SPLIT_OPPOSITE = opposite

def index(req,showHelp=False,status=None):
    ''' Generates the index page (main menu) '''

    # Empty body
    class body:
        ''' Empty struct for template '''
        def __init__(self):
            pass
    
    body.status = status

    body.title = 'Seed Database - Modify seed information for the NAV database'
    body.infotext = 'Here you can add, delete or edit seed information ' +\
                    'that are needed for the NAV database. Keep in mind ' +\
                    'that most of the data in  the NAV database are ' +\
                    'collected automatically by NAV background processes.'
    body.showHelp = showHelp
    body.help = [BASEPATH + 'help/','Show help']
    body.nohelp = [BASEPATH,'Hide help']
    
    body.tables = []
    headings = []
    
    # Table for boxes and services
    rows = [['IP devices',
             'Input seed information on the IP devices you want to ' +\
             'monitor',
            [BASEPATH + 'netbox/edit','Add'],
            [BASEPATH + 'netbox/list','Edit'],
            [BASEPATH + 'bulk/netbox','Bulk import']],
            ['Services',
             'Which services on which servers do you want to monitor?',
            [BASEPATH + 'service/edit','Add'],
            [BASEPATH + 'service/list','Edit'],
            [BASEPATH + 'bulk/service','Bulk import']]]
    body.tables.append(Table('IP devices and services','',headings,rows))

    # Table for rooms and locations 
    rows = [['Room',
             'Register all wiring closets and server rooms that contain ' +\
             'IP devices which NAV monitors',
            [BASEPATH + 'room/edit','Add'],
            [BASEPATH + 'room/list','Edit'],
            [BASEPATH + 'bulk/room','Bulk import']],
            ['Location',
             'Rooms are organised in locations',
            [BASEPATH + 'location/edit','Add'],
            [BASEPATH + 'location/list','Edit'],
            [BASEPATH + 'bulk/location','Bulk import']]]
    body.tables.append(Table('Rooms and locations','',headings,rows))

    # Table org and usage cat
    rows = [['Organisation',
             'Register all organisational units that are relevant. I.e. ' +\
             'all units that have their own subnet/server facilities.',
            [BASEPATH + 'org/edit','Add'],
            [BASEPATH + 'org/list','Edit'],
            [BASEPATH + 'bulk/org','Bulk import']],
            ['Usage categories',
            'NAV encourages a structure in the subnet structure. ' +\
             'Typically a subnet has users from an organisational ' +\
             'unit. In addition this may be subdivided into a ' +\
             'category of users, i.e. students, employees, ' +\
             'administration etc.',
            [BASEPATH + 'usage/edit','Add'],
            [BASEPATH + 'usage/list','Edit'],
            [BASEPATH + 'bulk/usage','Bulk import']]]
    body.tables.append(Table('Organisation and usage categories','',
                             headings,rows))

    # Table for types and vendors
    rows = [['Type',
             'The type describes the type of network device, uniquely ' +\
             'described from the SNMP sysobjectID',
            [BASEPATH + 'type/edit','Add'],
            [BASEPATH + 'type/list','Edit'],
            [BASEPATH + 'bulk/type','Bulk import']],
            ['Vendor',
             'Register the vendors that manufacture equipment that are ' +\
             'represented in your network.',
            [BASEPATH + 'vendor/edit','Add'],
            [BASEPATH + 'vendor/list','Edit'],
            [BASEPATH + 'bulk/vendor','Bulk import']],
            ['Snmpoid',
             'Manually add snmpoids (candidates for the cricket collector)',
            [BASEPATH + 'snmpoid/edit','Add'],
            ['',''],
            ['','']],
            ['Subcategory',
             'The main categories of a device are predefined by NAV (i.e. ' +\
             'GW,SW,SRV). You may however create subcategories yourself.',
            [BASEPATH + 'subcat/edit','Add'],
            [BASEPATH + 'subcat/list','Edit'],
            [BASEPATH + 'bulk/subcat','Bulk import']],
]
    body.tables.append(Table('Types and vendors','',headings,rows))

    # Table for vlans and special subnets
    rows = [['Vlan',
             'Register the vlan number that are in use (this info may ' +\
             'also be derived automatically from the routers)',
            None,
            [BASEPATH + 'vlan/list','Edit'],
            None],
            ['Prefix',
             'Register special ip prefixes. Typically reserved prefixes ' +\
             'or prefixes that are not directly connected to monitored ' +\
             'routers/firewalls fall into this category',
            [BASEPATH + 'prefix/edit','Add'],
            [BASEPATH + 'prefix/list','Edit'],
            [BASEPATH + 'bulk/prefix','Bulk import']]]
    body.tables.append(Table('Vlans and special subnets','',headings,rows))

    # Table for cabling and patch
    rows = [['Cabling',
             'Here you may document the horizontal cabling system ',
            [BASEPATH + 'cabling/edit','Add'],
            [BASEPATH + 'cabling/list','Edit'],
            [BASEPATH + 'bulk/cabling','Bulk import']],
            ['Patch',
             'Register the cross connects in the wiring closets ',
            [BASEPATH + 'patch/edit','Add'],
            [BASEPATH + 'patch/list','Edit'],
            [BASEPATH + 'bulk/patch','Bulk import']]]
    body.tables.append(Table('Cabling system','',headings,rows))



    nameSpace = {'entryList': None, 'editList': None, 'editForm': None, 'body': body}
    template = seeddbTemplate(searchList=[nameSpace])
    template.path = [('Home','/'),
                     ('Seed Database',None)]
    return template.respond()


######################
##
## General functions
##
########################

# General function for handling editing
def editPage(req,page,request,unicodeError):
    ''' General handler function for all editpages. Whenever an action
        add, update, list or delete is performed, this function is called. '''

    # Cancel button redirect
    if req.form.has_key(editForm.cnameCancel):
        nav.web.redirect(req,BASEPATH,seeOther=True)
   
    # Make a status object
    status = seeddbStatus()

    # Get action from request (url)
    action = request[1] 

    selected = []
    addedId = None
    # Get editid from url if it is present (external links and list links)
    if len(request) > 1:
        if (len(request) == 3) or (len(request) == 4):
            if request[2]:
                selected = [request[2]]

    # Make a list of selected entries from a posted selectlist
    if req.form.has_key(selectList.cnameChk):
        if type(req.form[selectList.cnameChk]) is str:
            # only one selected
            selected = [req.form[selectList.cnameChk]]
        elif type(req.form[selectList.cnameChk]) is list:
            # more than one selected
            for s in req.form[selectList.cnameChk]:
                selected.append(s)

    # Remember entries which we are already editing
    # Used if editing is interrupted by an error
    if req.form.has_key(UPDATE_ENTRY):
        if type(req.form[UPDATE_ENTRY]) is str:
            # only one selected
            selected = [req.form[UPDATE_ENTRY]]
        elif type(req.form[UPDATE_ENTRY]) is list:
            # more than one selected
            for s in req.form[UPDATE_ENTRY]:
                selected.append(s)

    # Disallow adding (for pageVlan)
    if req.form.has_key(selectList.cnameAdd) and hasattr(page,'disallowAdd'):
        status.errors.append(page.disallowAddReason)
        action = 'list'

    # Check if any entries are selected when action is 'edit' or 'delete'
    if action == 'edit':
        if req.form.has_key(selectList.cnameEdit):
            if not selected:
                status.errors.append('No entries selected for editing')
                action = 'list' 
        elif req.form.has_key(selectList.cnameDelete):
            action = 'delete'
            if not selected:
                status.errors.append('No entries selected')
                action = 'list'
        else:
            if not selected:
                action = 'add'

    # Check for unicode errors in input
    if unicodeError:
        status.errors.append('The data you input was sent in a ' +\
                             'non-recognisible encoding. Make sure your '
                             'browser uses automatic character encoding ' +\
                             'or set it to \'' + str(DEFAULT_ENCODING) + '\'.')
        action = 'list'

    # Set 'current path'
    path = page.pathAdd

    templatebox = page.editbox(page)
    # Copy field defintions from the main templatebox (used by add/update)
    page.fields = templatebox.fields

    # Make form object for template
    outputForm = editForm()
    if hasattr(page,'action'):
        outputForm.action = page.action
    else:
        outputForm.action = page.basePath + 'edit'

    # List definition, get sorting parameter
    sort = None
    if req.form.has_key('sort'):
        sort = req.form['sort']
    listView = page.listDef(req,page,sort)
   
    # Check if the confirm button has been pressed
    if req.form.has_key(outputForm.cnameConfirm):
        missing = templatebox.hasMissing(req)
        if not missing:
            status = templatebox.verifyFields(req,status)
            if not len(status.errors):
                if req.form.has_key(ADDNEW_ENTRY):
                    # add new entry
                    (status,action,outputForm,addedId) = page.add(req,
                                                            outputForm,action)
                elif req.form.has_key(UPDATE_ENTRY):
                    # update entry
                    (status,action,outputForm,selected) = page.update(req,
                                                                     outputForm,
                                                                      selected)
        else:
            status.errors.append("Required field '" + missing + "' missing") 
    # Confirm delete pressed?
    elif req.form.has_key(selectList.cnameDeleteConfirm):
        status = page.delete(selected,status)
        outputForm = None
        selected = None
        action = 'list' 
    
    # Decide what to show 
    if action == 'predefined':
        # Action is predefined by addNetbox() or updateNetbox()
        outputForm.textConfirm = 'Continue'
        outputForm.action = action
        outputForm.status = status
        listView = None
    elif action == 'edit':
        path = page.pathEdit
        title = 'Edit '
        if len(selected) > 1:
            title += page.plural
        else:
            title += page.singular
        outputForm.title = title
        outputForm.action = action
        outputForm.status = status
        outputForm.textConfirm = 'Update'
        if page.editMultipleAllowed:
            # This page can edit multiple entries at a time
            for s in selected:
                outputForm.add(page.editbox(page,req,s,formData=req.form))
        else:
            # This page can only edit one entry at a time (eg. netbox)
            outputForm.add(page.editbox(page,req,selected[0],formData=req.form))
        # preserve path
        #outputForm.action = page.basePath + 'edit/' + selected[0]
        listView = None
    elif action == 'add':
        path = page.pathAdd
        outputForm.action = action
        outputForm.status = status
        outputForm.title = 'Add ' + page.singular
        outputForm.textConfirm = 'Add ' + page.singular
        outputForm.add(page.editbox(page,req,formData=req.form))
        listView = None
    elif action == 'delete':
        path = page.pathDelete
        listView = page.listDef(req,page,sort,selected)
        listView.status = status
        listView.fill(req)
        outputForm = None
    elif action == 'list':
        if addedId:
            listView.selectedId = addedId
        if selected:
            listView.selectedId = selected[0]
        path = page.pathList
        listView.status=status
        listView.fill(req)
        outputForm = None
    elif action == 'redirect':
        # Redirect to main page (snmpoid add)
        return index(req,status=status)

    nameSpace = {'entryList': listView,'editList': None,'editForm': outputForm}
    template = seeddbTemplate(searchList=[nameSpace])
    template.path = path
    return template.respond()

def insertNetbox(ip,sysname,catid,roomid,orgid,
                 ro,rw,deviceid,serial,
                 typeid,snmpversion,subcatlist=None,
                 function=None):
    ''' Inserts a netbox into the database. Used by pageNetbox.add(). '''
    if not deviceid:

        # Make new device first
        if len(serial):
            fields = {'serial': serial}
        else:
            # Don't insert an empty serialnumber (as serialnumbers must be
            # unique in the database) (ie. don't insert '' for serial)
            fields = {}
            
        deviceid = addEntryFields(fields,
                                  'device',
                                  ('deviceid','device_deviceid_seq'))

    fields = {'ip': ip,
              'roomid': roomid,
              'deviceid': deviceid,
              'sysname': sysname,
              'catid': catid,
              'orgid': orgid,
              'ro': ro,
              'rw': rw}
    #uptodate = false per default

    # Get prefixid
    query = "SELECT prefixid FROM prefix WHERE '%s'::inet << netaddr" \
            % (fields['ip'],)
    try:
        result = executeSQLreturn(query) 
        fields['prefixid'] = str(result[0][0])
    except:
        pass        

    if typeid:
        fields['typeid'] = typeid

        # Set uptyodate = false
        # This part is done in netbox now. And for a new box this
        # field defaults to 'f'
        #tifields = {'uptodate': 'f'}
        #updateEntryFields(tifields,'type','typeid',typeid)

    if snmpversion:
        # Only use the first char from initbox, can't insert eg. '2c' in
        # this field
        snmpversion = snmpversion[0]
        fields['snmp_version'] = snmpversion

    netboxid = addEntryFields(fields,
                              'netbox',
                              ('netboxid','netbox_netboxid_seq'))
    # If subcatlist and function is given, insert them
    if subcatlist:
        if type(subcatlist) is list:
            for sc in subcatlist:
                fields = {'netboxid': netboxid,
                          'category': sc}
                addEntryFields(fields,'netboxcategory')
        else:
            fields = {'netboxid': netboxid,
                      'category': subcatlist}
            addEntryFields(fields,'netboxcategory')

    if function:
        fields = {'netboxid': netboxid,
                  'key': '',
                  'var': 'function',
                  'val': function}
        addEntryFields(fields,'netboxinfo')


######################
##
## General classes
##
########################

class Table:
    ''' A general class for html tables used by index(). '''
    def __init__(self,title,infotext,headings,rows):
        self.title = title
        self.infotext = infotext
        self.headings = headings
        self.rows = rows

class seeddbStatus:
    ''' Struct class which holds two lists (messages and errors). Every 
        form object got an instance of this class and uses it to add 
        messages and errors which is then displayed by the template. '''
    # List of status messages, one line per message
    messages = []
    # List of error messages, one line per message
    errors = []

    def __init__(self):
        self.messages = []
        self.errors = []


class entryListCell:
    ''' Represents a cell (TD) in a selectlist object. '''

    CHECKBOX = 'chk'
    RADIO = 'rad'
    HIDDEN = 'hid'

    def __init__(self,text=None,url=None,buttonType=None,
                 image=None,tooltip=None):
        self.text = text
        self.url = url
        self.buttonType = buttonType
        self.image = image
        self.tooltip = tooltip

## class entryList (rename to selectList)
class entryList:
    ''' Flexible class for making lists of entries which can be selected.
        Used by all 'edit' pages. 
        
        Uses the list definitions defined in every page class.

        descriptionFormat = [(text,forgetSQLfield),(text,...] '''
        
    # Constants
    CNAME_SELECT = 'checkbox_id'
    CNAME_ADD = 'submit_add'
    CNAME_EDIT = 'submit_edit'
    CNAME_DELETE = 'submit_delete'
    CNAME_CONFIRM_DELETE = 'confirm_delete'
    CNAME_CANCEL = 'form_cancel'
    
    # Class variables used by the template
    title = None
    status = None
    body = None
    formMethod = 'post'
    formAction = None
    selectCname = CNAME_SELECT
    buttonsTop = [(CNAME_ADD,'Add new'),
                  (CNAME_EDIT,'Edit selected'),
                  (CNAME_DELETE,'Delete selected')]
    buttonsBottom = buttonsTop
    hideFirstHeading = False  # Don't show first column heading (usually 
                              # the select heading) if True
    buttonTypeOverride = None # override the chosen select button type
                              # used for bulk and delete lists where there
                              # is no checkbox/radiobut., only hidden
    headings = []             # list of cell objects
    rows = []                 # tuples of (sortstring,id,cell object)

    # Variables for filling the list
    tableName = None
    basePath = None                 
    sortBy = None                   # Sort by columnumber
    defaultSortBy = None            # Default columnnumber sorted by
    headingDefinition = None        # list of tuples (heading,show sort link)
    cellDefintion = None            # cellDefinition list
    where = None                    # List of id's (strings)
    sortingOn = True                # Show links for sorting the list

    # SQL filters
    filters = None
    filterConfirm = 'cn_filter'
    filterConfirmText = 'Update list'
    selectedId = None

    def __init__(self,req,struct,sort,deleteWhere=None):
        self.headings = []
        self.rows = []

        if sort:
            sort = int(sort)
        self.sortBy = sort
        self.tableName = struct.tableName
        self.tableIdKey = struct.tableIdKey
        self.basePath = struct.basePath
        self.deleteWhere = deleteWhere
        self.formAction = self.basePath + 'edit'
        self.filterAction = self.basePath + 'list'

        if deleteWhere:
            self.buttonTypeOverride = entryListCell.HIDDEN
            self.hideFirstHeading = True
            self.where = deleteWhere
            title = 'Are you sure you want to delete the ' + \
                    'selected '
            if len(deleteWhere) > 1:
                title += struct.plural
            else:
                title += struct.singular
            self.title = title + '?'
            self.sortingOn = False
            self.buttonsTop = None
            self.buttonsBottom = [(self.CNAME_CONFIRM_DELETE, 'Delete'),
                                  (self.CNAME_CANCEL, 'Cancel')]
        else:
            self.title = 'Edit ' + struct.plural
            self.sortingOn = True

    def fill(self,req):
        """ Fill the list with data from the database. """
        
        # No filters if this is a delete list
        if self.deleteWhere:
            self.filters = None
        # Make filters
        if self.filters:
            self.makeFilters(req)

        # Make headings
        i = 0
        for heading,sortlink,sortFunction in self.headingDefinition:
            if self.hideFirstHeading:
                heading = ''
                self.hideFirstHeading = False
            if self.sortBy:
                currentOrder = self.sortBy
            else:
                currentOrder = self.defaultSortBy
            s = i
            if i == currentOrder:
                # Reverse sort?
                s = -i
            url = self.basePath + 'list?sort=' + str(s)
            if sortlink and self.sortingOn:
                self.headings.append(entryListCell(heading,
                                                    url))
            else:
                self.headings.append(entryListCell(heading,
                                                    None))
            i = i + 1

        # Check filters and decide if we're going to show list
        renderList = True
        filterSettings = []
        if self.filters:
            renderList = False
            for filter in self.filters:
                # filter[10] is selected id in filter
                if filter[10]:
                    # Something is selected, show list
                    renderList = True
                    # filter[6] is tableIdKey
                    filterSettings.append((filter[10],filter[6]))

        # Skip filling if no result from filter
        if renderList:
            # Preparse tooltips, etc.
            for sqlQuery,definition in self.cellDefinition:
                for column in definition:
                    for cell in column:
                        if type(cell) is list:
                            # This cell definition is a list, as opposed to
                            # a tuple, so we must prefetch some data for the
                            # parse function
                            # Must prefetch data for this column
                            for tooltipDef in cell:
                                # There can be one or more defintions per cell
                                sql = tooltipDef[0]
                                # column[2] is reserved for data
                                tooltipDef[2] = executeSQLreturn(sql)

            # Make rows
            reverseSort = False
            if self.sortBy:
                if self.sortBy < 0:
                    self.sortBy = self.sortBy * -1
                    reverseSort = True

            for sqlTuple,definition in self.cellDefinition:
                # Create SQL query from tuple
                columns,tablenames,join,where,orderBy = sqlTuple
                sqlQuery = 'SELECT ' + columns + ' FROM ' + tablenames
                if join:
                    sqlQuery += ' %s ' % (join,)
                if where:
                    sqlQuery += ' WHERE ' + where
                # Add where clause if self.where is present
                if self.where:
                    if not where:
                        # No where defined in sqlTuple, so add it now
                        sqlQuery += ' WHERE '
                    else:
                        # Else, these are additional so add AND
                        sqlQuery += ' AND '
                    first = True
                    sqlQuery += ' ('
                    for id in self.where:
                        if not first:
                            sqlQuery += 'OR'
                        sqlQuery += " %s.%s='%s' " % (self.tableName,
                                                      self.tableIdKey,id)
                        if first:
                            first = False
                    sqlQuery += ') '
                # Add where clause if filterSettings is present
                for filter in filterSettings:
                    if not where:
                        # No where defined in sqlTuple, so add it now
                        sqlQuery += ' WHERE '
                    else:
                        # Else, these are additional so add AND
                        sqlQuery += ' AND '
                    sqlQuery += filter[1] + "='" + filter[0] + "' "
                if orderBy:
                    sqlQuery += ' ORDER BY ' + orderBy                                
                fetched = executeSQLreturn(sqlQuery)
                for row in fetched:
                    id = row[0]
                    cells = []
                    for text,url,buttonType,image,tooltip in definition:
                        if buttonType and self.buttonTypeOverride:
                            buttonType = self.buttonTypeOverride
                        cells.append(entryListCell(self.parse(text,row),
                                                   self.parse(url,row,True),
                                                   buttonType,
                                                   image,
                                                   self.parse(tooltip,row))) 
                    
                    sortKey = None
                    if self.sortBy:
                        sortKey = row[self.sortBy]
                    self.rows.append([sortKey,(id,cells)])
            if self.sortBy:
                if self.headingDefinition[self.sortBy][2]:
                    # Optional compare method
                    self.rows.sort(self.headingDefinition[self.sortBy][2])
                else:
                    self.rows.sort()
                if reverseSort:
                    self.rows.reverse()

    def parse(self,parseString,currentRow,url=False):
        """ Parses format strings used by the list definitions. """
        result = None
        if type(parseString) is int:
            # parseString refers to integer column
            result = [currentRow[parseString]]
        elif type(parseString) is str:
            parseString = parseString.replace('{p}',self.basePath)
            parseString = parseString.replace('{id}',str(currentRow[0]))
            parseString = parseString.replace('{descr}',str(currentRow[1]))
            if parseString.find('SELECT') == 0:
                # This string is a sql query (used by vlan)
                resultString = ''
                sqlresult = executeSQLreturn(parseString)
                for row in sqlresult:
                    for col in row:
                        resultString += col + '<br>'
                result = [resultString]    
            else:
                if url:
                    result = parseString
                else:
                    result = [parseString]
        elif type(parseString) is list:
            result = []
            for tooltipDef in parseString:
                data = tooltipDef[2]
                if data:
                    # There is preparsed data (ie. the sql returned something)
                    result.append(tooltipDef[1][0])
                    # [1][0] = Tooltip (can also be other preparsed data) header
                    for row in data:
                        if currentRow[0] == row[0]:
                            # ID's match
                            result.append(row[1])
        return result        

    def makeFilters(self,req):
        for filter in self.filters:
            text,firstEntry,sqlTuple,idFormat,optionFormat,\
            controlName,tableId,table,fields = filter
           
            optionsList = []
            if firstEntry:
                firstEntry = ([firstEntry[0]],[firstEntry[1]])
                optionsList.append(firstEntry)
            # Make sql
            sql = "SELECT " + sqlTuple[0] + " FROM " + sqlTuple[1] + " "
            if sqlTuple[2]:
                sql += sqlTuple[2] + " "
            if sqlTuple[3]:
                sql += "WHERE " + sqlTuple[3] + " "
            if sqlTuple[4]:
                sql += "ORDER BY " + sqlTuple[4]
            result = executeSQLreturn(sql)
            for row in result:
                id = self.parse(idFormat,row)
                text = self.parse(optionFormat,row)
                optionsList.append((id,text))
            filter.append(optionsList)
            # Selected id
            selected = None
            if self.selectedId:
                entry = table.objects.get(id=self.selectedId)
                fieldList = fields.split('.')
                for field in fieldList:
                    entry = getattr(entry,field)
                selected = str(entry) 
            if req.form.has_key(controlName):
                if len(req.form[controlName]):
                    selected = req.form[controlName]
            filter.append(selected)

# Class representing a form, used by the template
class editForm:
    """ Class representing a form element, the main component of every
        edit page. Each form element can have any number of editbox
        objects added to it. """
    
    # For the template
    method = 'post'
    action = None
    title = None
    error = None
    status = None
    backlink = None
    enctype = 'application/x-www-form-urlencoded'

    # Text and controlname
    textConfirm = None
    cnameConfirm = 'form_confirm'
    showConfirm = True

    textCancel = 'Cancel'
    cnameCancel = 'form_cancel'
    showCancel = True
    actionCancel = BASEPATH

    # Only used for netboxes (see template)
    # 'submit_delete' is fetched by the same handler as when deleting multiple
    # entities from a list
    textDelete = 'Delete'
    cnameDelete = 'submit_delete'
    showDelete = True

    # Used by edit netbox in the intermediate
    CNAME_CONTINUE = 'cname_continue'

    # List of editboxes to display
    editboxes = []

    def __init__(self,cnameConfirm=None):
        if cnameConfirm:
            self.cnameConfirm = cnameConfirm
 
        self.editboxes = []

    def add(self,box):
        """ Add an editbox object to this form element. """
        self.editboxes.append(box)

class inputText:
    """ Class representing a textinput html control. """

    type = 'text'
    name = None
    maxlength = None
    
    def __init__(self,value='',size=22,maxlength=None,disabled=False):
        self.value = value
        self.disabled = disabled
        self.size = str(size)
        if maxlength:
            self.maxlength = str(maxlength)

class inputTreeSelect:
    """ Container class for treeselects. Used to get treeselect in the 
        same format as all the other inputs used by the template. """

    type = 'treeselect'
    name = None
    treeselect = None
    disabled = False

    def __init__(self,treeselect):
        self.value = ''
        self.treeselect = treeselect

class inputSelect:
    """ Class representing a select input html control. """

    type = 'select'
    name = None
    
    def __init__(self,options=None,table=None,attribs=None,disabled=False):
        self.value = ''
        self.options = options
        self.attribs = attribs
        self.disabled = disabled

        if table:
            self.options = table.getOptions() 

class inputMultipleSelect:
    """ Class representing a multiple select input html control. """
    type = 'multipleselect'
    name = None
    value = []
    def __init__(self,options=None,table=None,disabled=False):
        self.options = options
        self.disabled = disabled

        if table:
            self.options = table.getOptions() 

class inputFile:
    """ Class representing a file upload input control. """

    type = 'file'
    name = None
    value = ''
    disabled = False
    def __init__(self):
        pass

class inputTextArea:
    """ Class representing a textarea input html control. """

    type = 'textarea'
    name = None

    def __init__(self,rows=20,cols=80):
        self.rows = rows
        self.cols = cols
        self.value = ''
        self.disabled = False

class inputCheckbox:
    """ Class representing a checkbox input html control. """

    type = 'checkbox'
    name = None

    def __init__(self,disabled=False):
        self.value = '0'
        self.disabled = disabled

class inputHidden:
    """ Class representing a hidden input html control. """
    type = 'hidden'
    name = None
    disabled = False

    def __init__(self,value):
        self.value = value


class inputServiceProperties:
    """ Contains a list of inputServiceProperty inputs. 
        (used by pageService) """

    type = 'serviceproperties'
    disabled = False
    
    def __init__(self,propertyList):
        self.propertyList = propertyList

class inputServiceProperty:
    """ Class representing a serviceproperty input box. 
        (used by pageService) """

    type = 'serviceproperty'
    disabled = False

    def __init__(self,title,id,args,optargs,display=True):
        self.title = title
        self.id = id
        self.display = display
        self.args = args
        self.optargs = optargs

class editbox:
    """ Parent class for all the different editboxes which are all added 
        to an editform object. There are normally one editbox per page, but
        for some pages there are more (there are created three editboxes for
        the netbox page for example, editbox(main),editboxserial and
        editboxfunction (which also includes the subcat)). 
        
        The editbox contains field defitions used by the template to render
        the forms and functions to fill the form with data from either the
        database or from a previous http post. """

    boxName = ADDNEW_ENTRY
    boxId = 0

    # Current box number keeps track of the box number when there are
    # more than one editbox in a form. Used by formFill to get correct data
    #currentBoxNumber = 0

    def fill(self):
        """ Fill this form with data from the database (entry = editId). """
        entry = self.table.objects.get(id=self.editId)
       
        # Set the name of this box to reflect that we are
        # updating an entry
        self.boxName = UPDATE_ENTRY
        self.boxId = self.editId

        ## TEMPORARY:
        ## check if this is one of the new pages by checking
        ## for three instead of two entries in the desc list
        if len(self.fields[self.fields.keys()[0]]) > 2:
            # Uses psycopg to fill field values
            page = pageList[self.page]

            select = ''
            first = True
            keyNumber = {}
            i = 0
            for key in self.fields.keys():
                keyNumber[key] = i
                i+=1
                if not first:
                    select += ', '
                select += key
                first = False
            tables = page.tableName
            where = page.tableIdKey + "='" + self.editId + "'"
            # For the benefit of pagePrefix (which must select from vlan too)
            if hasattr(self,'additionalSQL'):
                tables += ', vlan'
                where += self.additionalSQL

            sql = "SELECT %s FROM %s WHERE %s" % (select, tables, where)

            result = executeSQLreturn(sql)
            result = result[0]

            for key,desc in self.fields.items():
                value = result[keyNumber[key]]
                if value:
                    desc[0].value = str(value)
        else:
            # Old style filling with forgetsql
            for fieldname,desc in self.fields.items():
                value = getattr(entry,fieldname)
                if value:
                    desc[0].value = str(value)

    def setControlNames(self,controlList=None):
        """ Set controlnames for the inputs to the same as the fieldnames. """
        if not controlList:
            controlList = self.fields

        for fieldname,desc in controlList.items():
            desc[0].name = fieldname

    def verifyFields(self,req,status):
        """ Verify that data entered into fields are of correct type.
            Eg. integers in FIELD_INTEGER fields. """

        for field,desc in self.fields.items():
            if req.form.has_key(field):
                if type(req.form[field]) is list:
                    # Editing several entries
                    for each in req.form[field]:
                        # Do tests here
                        if desc[3] == FIELD_INTEGER:
                            if len(each):
                                try:
                                    int(each)
                                except ValueError:
                                    error = "Invalid integer: '" +\
                                            str(each) + "'"
                                    status.errors.append(error)
                else:
                    # Editing only one field
                    # Do tests here
                    if desc[3] == FIELD_INTEGER:
                        try:
                            if len(req.form[field]):
                                int(req.form[field])
                        except ValueError:
                            error = "Invalid integer: '" + \
                                    str(req.form[field]) + "'"
                            status.errors.append(error)
        return status

    def hasMissing(self,req):
        """ Check if any of the required fields are missing in the req.form
            Returns the name the first missing field, or False
            
            Note: keep_blank_values (mod_python) must be True or empty fields 
                  won't be present in the form  """
        missing = False
        for field,desc in self.fields.items():
            # Keep blank values must be switched on, or else the next line
            # will fail, could be more robust
            if req.form.has_key(field):
                if type(req.form[field]) is list:
                    # the field is a list, several entries have been edited
                    for each in req.form[field]:
                        if desc[1] == REQ_TRUE:
                            # this field is required
                            if not len(each):
                                if len(desc) > 2:
                                    # desc[2] is real fieldname
                                    missing = desc[2]
                                else:
                                    # cryptic fieldname (remove this later)
                                    missing = field
                                break
                else:
                    if desc[1] == REQ_TRUE:
                        # tihs field is required
                        if not len(req.form[field]):
                            if len(desc) > 2:
                                # desc[2] is real fieldname
                                missing = desc[2]
                            else:
                                # cryptic fieldname (remove this later)
                                missing = field
                            break
        return missing

    def addHidden(self,fieldname,value):
        """ Add hidden html input control to the editbox. """
        self.hiddenFields[fieldname] = [inputHidden(value),False]
        self.hiddenFields[fieldname][0].name = fieldname

    def addDisabled(self):
        """ Since fields which are disabled, aren't posted (stupid HTML)
            we must add them as hidden fields.
           
            This only goes for textinputs (?!) so we must also change
            controlnames to avoid getting double values for selects, etc. """

        for fieldname,definition in self.fields.items():
            if definition[0].disabled and (not definition[0].type=='hidden'):
                self.addHidden(fieldname,definition[0].value)
                definition[0].name = definition[0].name + '_disabled'

    def formFill(self,formData):
        """ Fill this editbox with data from the form.

            This is used by intermediate steps (like register serial)
            to remember field values and for refilling a form if an error
            is encountered and the user has to resubmit a form. """

        if not hasattr(editbox,'currentBoxNumber'):
            editbox.currentBoxNumber = 0

        for field,definition in self.fields.items():
            first = True
            numberOfBoxes = None
            if formData.has_key(field):
                if type(formData[field]) is list:
                    # Remember how many editboxes this form has
                    if first:
                        # NB! ASSUMES THAT THE FIRST FIELDNAME IN A FORM
                        # IS NEVER A CHECKBOX (SINCE UNCHECKED CHECCKBOXES
                        # ARE NOT POSTED). IF IT IS, THEN numberOfBoxes
                        # WILL BE ONE LESS THAN IT SHOULD BE FOR EACH
                        # UNCHECKED CHECKBOX
                        numberOfBoxes = len(formData[field])                    
                        first = False

                    # We are editing more than one entry, pick the
                    # right data from the form
                    definition[0].value = formData[field][editbox.currentBoxNumber]
                else:
                    definition[0].value = formData[field]
        # Update class variable currentFormNumber
        if numberOfBoxes:
            editbox.currentBoxNumber +=1
            if editbox.currentBoxNumber == numberOfBoxes:
                # Reset the currentFormNumber class instance
                # Since it is a class instance, it will be common
                # to all editbox instances for all requests, that's
                # why it has to be reset
                editbox.currentBoxNumber = 0

class editboxHiddenOrMessage(editbox):
    """ This editbox can display a message and contain hidden inputs. """
    page = 'hiddenormessage'

    def __init__(self,message=None):
        self.hiddenFields = {}
        self.message = message

        # The editboxNetbox has UPDATE_ENTRY (which holds the id) or ADDNEW, 
        # don't need to repeat it here (so setting boxname to IGNORE_BOX) 
        self.boxName = IGNORE_BOX


class seeddbPage:
    """ The main editing class. Every edit page inherits from this class.

        Contains functions for adding, updating and describing entries.
        Default functions can be overriden by children to do more specific
        handling of adding or updating.

        The children of this class contains all the information needed
        for handling the different tables.

         class seeddbPage
                 |+-- class listDef
                 |+-- class editbox
                 |+-- (optionally more editboxes)
        
                 |+-- def add
                 |+-- def update
                 |+-- def delete
                 |+-- def describe
    """


    def add(self,req,outputForm,action):
        """ Called when 'Add' is clicked on an edit page.

            Takes the formdata from the request object and inserts
            an entry in the database.

            This is the general function used by almost all the edit
            pages. Some of the edit page classes overrides this function
            for more control over the adding (eg. the netbox page).

            req: request object containing a form
            outputForm: form with editboxes
                        manipulated directly by eg. editNetbox.add() """

        error = None
        status = seeddbStatus()

        id = None
        nextId = None 
        if self.sequence:
            # Get next id from sequence, will need this id to reload
            # entry when making a description of the inserted row
            # In the case that sequence=None, idfield is already
            # present in the field data
            sql = "SELECT nextval('%s')" % (self.sequence,)
            result = executeSQLreturn(sql)
            nextId = str(result[0][0])

        sql = 'INSERT INTO ' + self.tableName + ' ('
        first = True
        for field,descr in self.fields.items():
            if req.form.has_key(field):
                if len(req.form[field]):
                    if not first:
                        sql += ','
                    sql += field
                    first = False
        # Add the idfield if we have queried the sequence
        if nextId:
            sql += "," + self.tableIdKey
        sql += ') VALUES ('
        first = True
        for field,descr in self.fields.items():
            if req.form.has_key(field):
                if len(req.form[field]):    
      …