/lib/msf/core/db.rb

Large files files are truncated, but you can click here to view the full file

# -*- coding: binary -*-

#
# Standard Library
#

require 'csv'
require 'tmpdir'
require 'uri'
require 'zip'

#
#
# Gems
#
#

#
# PacketFu
#

require 'packetfu'

#
# Rex
#


require 'rex/socket'

# Check Rex::Parser.nokogiri_loaded for status of the Nokogiri parsers
require 'rex/parser/acunetix_nokogiri'
require 'rex/parser/appscan_nokogiri'
require 'rex/parser/burp_session_nokogiri'
require 'rex/parser/ci_nokogiri'
require 'rex/parser/foundstone_nokogiri'
require 'rex/parser/fusionvm_nokogiri'
require 'rex/parser/mbsa_nokogiri'
require 'rex/parser/nexpose_raw_nokogiri'
require 'rex/parser/nexpose_simple_nokogiri'
require 'rex/parser/nmap_nokogiri'
require 'rex/parser/openvas_nokogiri'
require 'rex/parser/wapiti_nokogiri'
require 'rex/parser/outpost24_nokogiri'

# Legacy XML parsers -- these will be converted some day
require 'rex/parser/ip360_aspl_xml'
require 'rex/parser/ip360_xml'
require 'rex/parser/nessus_xml'
require 'rex/parser/netsparker_xml'
require 'rex/parser/nexpose_xml'
require 'rex/parser/nmap_xml'
require 'rex/parser/retina_xml'

#
# Project
#

require 'msf/core/db_manager/import_msf_xml'

module Msf

###
#
# The states that a host can be in.
#
###
module HostState
  #
  # The host is alive.
  #
  Alive   = "alive"
  #
  # The host is dead.
  #
  Dead    = "down"
  #
  # The host state is unknown.
  #
  Unknown = "unknown"
end

###
#
# The states that a service can be in.
#
###
module ServiceState
  Open      = "open"
  Closed    = "closed"
  Filtered  = "filtered"
  Unknown   = "unknown"
end

###
#
# Events that can occur in the host/service database.
#
###
module DatabaseEvent

  #
  # Called when an existing host's state changes
  #
  def on_db_host_state(host, ostate)
  end

  #
  # Called when an existing service's state changes
  #
  def on_db_service_state(host, port, ostate)
  end

  #
  # Called when a new host is added to the database.  The host parameter is
  # of type Host.
  #
  def on_db_host(host)
  end

  #
  # Called when a new client is added to the database.  The client
  # parameter is of type Client.
  #
  def on_db_client(client)
  end

  #
  # Called when a new service is added to the database.  The service
  # parameter is of type Service.
  #
  def on_db_service(service)
  end

  #
  # Called when an applicable vulnerability is found for a service.  The vuln
  # parameter is of type Vuln.
  #
  def on_db_vuln(vuln)
  end

  #
  # Called when a new reference is created.
  #
  def on_db_ref(ref)
  end

end

class DBImportError < RuntimeError
end

###
#
# The DB module ActiveRecord definitions for the DBManager
#
###
class DBManager
  include Msf::DBManager::ImportMsfXml

  def rfc3330_reserved(ip)
    case ip.class.to_s
    when "PacketFu::Octets"
      ip_x = ip.to_x
      ip_i = ip.to_i
    when "String"
      if ipv46_validator(ip)
        ip_x = ip
        ip_i = Rex::Socket.addr_atoi(ip)
      else
        raise ArgumentError, "Invalid IP address: #{ip.inspect}"
      end
    when "Fixnum"
      if (0..2**32-1).include? ip
        ip_x = Rex::Socket.addr_itoa(ip)
        ip_i = ip
      else
        raise ArgumentError, "Invalid IP address: #{ip.inspect}"
      end
    else
      raise ArgumentError, "Invalid IP address: #{ip.inspect}"
    end
    return true if Rex::Socket::RangeWalker.new("0.0.0.0-0.255.255.255").include? ip_x
    return true if Rex::Socket::RangeWalker.new("127.0.0.0-127.255.255.255").include? ip_x
    return true if Rex::Socket::RangeWalker.new("169.254.0.0-169.254.255.255").include? ip_x
    return true if Rex::Socket::RangeWalker.new("224.0.0.0-239.255.255.255").include? ip_x
    return true if Rex::Socket::RangeWalker.new("255.255.255.255-255.255.255.255").include? ip_x
    return false
  end

  def ipv46_validator(addr)
    ipv4_validator(addr) or ipv6_validator(addr)
  end

  def ipv4_validator(addr)
    return false unless addr.kind_of? String
    Rex::Socket.is_ipv4?(addr)
  end

  def ipv6_validator(addr)
    Rex::Socket.is_ipv6?(addr)
  end

  # Takes a space-delimited set of ips and ranges, and subjects
  # them to RangeWalker for validation. Returns true or false.
  def validate_ips(ips)
    ret = true
    begin
      ips.split(/\s+/).each {|ip|
        unless Rex::Socket::RangeWalker.new(ip).ranges
          ret = false
          break
        end
        }
    rescue
      ret = false
    end
    return ret
  end


  #
  # Determines if the database is functional
  #
  def check
  ::ActiveRecord::Base.connection_pool.with_connection {
    res = ::Mdm::Host.find(:first)
  }
  end


  def default_workspace
  ::ActiveRecord::Base.connection_pool.with_connection {
    ::Mdm::Workspace.default
  }
  end

  def find_workspace(name)
  ::ActiveRecord::Base.connection_pool.with_connection {
    ::Mdm::Workspace.find_by_name(name)
  }
  end

  #
  # Creates a new workspace in the database
  #
  def add_workspace(name)
  ::ActiveRecord::Base.connection_pool.with_connection {
    ::Mdm::Workspace.find_or_create_by_name(name)
  }
  end

  def workspaces
  ::ActiveRecord::Base.connection_pool.with_connection {
    ::Mdm::Workspace.find(:all)
  }
  end

  #
  # Wait for all pending write to finish
  #
  def sync
    # There is no more queue.
  end

  #
  # Find a host.  Performs no database writes.
  #
  def get_host(opts)
    if opts.kind_of? ::Mdm::Host
      return opts
    elsif opts.kind_of? String
      raise RuntimeError, "This invokation of get_host is no longer supported: #{caller}"
    else
      address = opts[:addr] || opts[:address] || opts[:host] || return
      return address if address.kind_of? ::Mdm::Host
    end
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    if wspace.kind_of? String
      wspace = find_workspace(wspace)
    end

    address = normalize_host(address)
    return wspace.hosts.find_by_address(address)
  }
  end

  #
  # Exactly like report_host but waits for the database to create a host and returns it.
  #
  def find_or_create_host(opts)
    report_host(opts)
  end

  #
  # Report a host's attributes such as operating system and service pack
  #
  # The opts parameter MUST contain
  # +:host+::         -- the host's ip address
  #
  # The opts parameter can contain:
  # +:state+::        -- one of the Msf::HostState constants
  # +:os_name+::      -- one of the Msf::OperatingSystems constants
  # +:os_flavor+::    -- something like "XP" or "Gentoo"
  # +:os_sp+::        -- something like "SP2"
  # +:os_lang+::      -- something like "English", "French", or "en-US"
  # +:arch+::         -- one of the ARCH_* constants
  # +:mac+::          -- the host's MAC address
  # +:scope+::        -- interface identifier for link-local IPv6
  # +:virtual_host+:: -- the name of the VM host software, eg "VMWare", "QEMU", "Xen", etc.
  #
  def report_host(opts)

    return if not active
    addr = opts.delete(:host) || return

    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
    if addr.eql? "Remote Pipe"
      return
    end

  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    if wspace.kind_of? String
      wspace = find_workspace(wspace)
    end

    ret = { }

    if not addr.kind_of? ::Mdm::Host
      addr = normalize_host(addr)
      addr, scope = addr.split('%', 2)
      opts[:scope] = scope if scope

      unless ipv46_validator(addr)
        raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
      end

      if opts[:comm] and opts[:comm].length > 0
        host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm])
      else
        host = wspace.hosts.find_or_initialize_by_address(addr)
      end
    else
      host = addr
    end

    # Truncate the info field at the maximum field length
    if opts[:info]
      opts[:info] = opts[:info][0,65535]
    end

    # Truncate the name field at the maximum field length
    if opts[:name]
      opts[:name] = opts[:name][0,255]
    end

    opts.each { |k,v|
      if (host.attribute_names.include?(k.to_s))
        unless host.attribute_locked?(k.to_s)
          host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
        end
      else
        dlog("Unknown attribute for ::Mdm::Host: #{k}")
      end
    }
    host.info = host.info[0,::Mdm::Host.columns_hash["info"].limit] if host.info

    # Set default fields if needed
    host.state       = HostState::Alive if not host.state
    host.comm        = ''        if not host.comm
    host.workspace   = wspace    if not host.workspace

    if host.changed?
      msf_import_timestamps(opts,host)
      host.save!
    end

    if opts[:task]
      Mdm::TaskHost.create(
          :task => opts[:task],
          :host => host
      )
    end

    host
  }
  end


  #
  # Update a host's attributes via semi-standardized sysinfo hash (Meterpreter)
  #
  # The opts parameter MUST contain the following entries
  # +:host+::           -- the host's ip address
  # +:info+::           -- the information hash
  # * 'Computer'        -- the host name
  # * 'OS'              -- the operating system string
  # * 'Architecture'    -- the hardware architecture
  # * 'System Language' -- the system language
  #
  # The opts parameter can contain:
  # +:workspace+::      -- the workspace for this host
  #
  def update_host_via_sysinfo(opts)

    return if not active
    addr = opts.delete(:host) || return
    info = opts.delete(:info) || return

    # Sometimes a host setup through a pivot will see the address as "Remote Pipe"
    if addr.eql? "Remote Pipe"
      return
    end

  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    if wspace.kind_of? String
      wspace = find_workspace(wspace)
    end

    if not addr.kind_of? ::Mdm::Host
      addr = normalize_host(addr)
      addr, scope = addr.split('%', 2)
      opts[:scope] = scope if scope

      unless ipv46_validator(addr)
        raise ::ArgumentError, "Invalid IP address in report_host(): #{addr}"
      end

      if opts[:comm] and opts[:comm].length > 0
        host = wspace.hosts.find_or_initialize_by_address_and_comm(addr, opts[:comm])
      else
        host = wspace.hosts.find_or_initialize_by_address(addr)
      end
    else
      host = addr
    end

    res = {}

    if info['Computer']
      res[:name] = info['Computer']
    end

    if info['Architecture']
      res[:arch] = info['Architecture'].split(/\s+/).first
    end

    if info['OS'] =~ /^Windows\s*([^\(]+)\(([^\)]+)\)/i
      res[:os_name]   = "Microsoft Windows"
      res[:os_flavor] = $1.strip
      build = $2.strip

      if build =~ /Service Pack (\d+)/
        res[:os_sp] = "SP" + $1
      else
        res[:os_sp] = "SP0"
      end
    end

    if info["System Language"]
      case info["System Language"]
        when /^en_/
          res[:os_lang] = "English"
      end
    end


    # Truncate the info field at the maximum field length
    if res[:info]
      res[:info] = res[:info][0,65535]
    end

    # Truncate the name field at the maximum field length
    if res[:name]
      res[:name] = res[:name][0,255]
    end

    res.each { |k,v|

      if (host.attribute_names.include?(k.to_s))
        unless host.attribute_locked?(k.to_s)
          host[k] = v.to_s.gsub(/[\x00-\x1f]/n, '')
        end
      else
        dlog("Unknown attribute for Host: #{k}")
      end
    }

    # Set default fields if needed
    host.state       = HostState::Alive if not host.state
    host.comm        = ''        if not host.comm
    host.workspace   = wspace    if not host.workspace

    if host.changed?
      host.save!
    end

    host
  }
  end
  #
  # Iterates over the hosts table calling the supplied block with the host
  # instance of each entry.
  #
  def each_host(wspace=workspace, &block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.hosts.each do |host|
      block.call(host)
    end
  }
  end

  #
  # Returns a list of all hosts in the database
  #
  def hosts(wspace = workspace, only_up = false, addresses = nil)
  ::ActiveRecord::Base.connection_pool.with_connection {
    conditions = {}
    conditions[:state] = [Msf::HostState::Alive, Msf::HostState::Unknown] if only_up
    conditions[:address] = addresses if addresses
    wspace.hosts.where(conditions).order(:address)
  }
  end



  def find_or_create_service(opts)
    report_service(opts)
  end

  #
  # Record a service in the database.
  #
  # opts MUST contain
  # +:host+::  the host where this service is running
  # +:port+::  the port where this service listens
  # +:proto+:: the transport layer protocol (e.g. tcp, udp)
  #
  # opts may contain
  # +:name+::  the application layer protocol (e.g. ssh, mssql, smb)
  # +:sname+:: an alias for the above
  #
  def report_service(opts)
    return if not active
  ::ActiveRecord::Base.connection_pool.with_connection { |conn|
    addr  = opts.delete(:host) || return
    hname = opts.delete(:host_name)
    hmac  = opts.delete(:mac)
    host  = nil
    wspace = opts.delete(:workspace) || workspace
    hopts = {:workspace => wspace, :host => addr}
    hopts[:name] = hname if hname
    hopts[:mac]  = hmac  if hmac

    # Other report_* methods take :sname to mean the service name, so we
    # map it here to ensure it ends up in the right place despite not being
    # a real column.
    if opts[:sname]
      opts[:name] = opts.delete(:sname)
    end

    if addr.kind_of? ::Mdm::Host
      host = addr
      addr = host.address
    else
      host = report_host(hopts)
    end

    if opts[:port].to_i.zero?
      dlog("Skipping port zero for service '%s' on host '%s'" % [opts[:name],host.address])
      return nil
    end

    ret  = {}
=begin
    host = get_host(:workspace => wspace, :address => addr)
    if host
      host.updated_at = host.created_at
      host.state      = HostState::Alive
      host.save!
    end
=end

    proto = opts[:proto] || 'tcp'

    service = host.services.find_or_initialize_by_port_and_proto(opts[:port].to_i, proto)
    opts.each { |k,v|
      if (service.attribute_names.include?(k.to_s))
        service[k] = ((v and k == :name) ? v.to_s.downcase : v)
      else
        dlog("Unknown attribute for Service: #{k}")
      end
    }
    service.state ||= ServiceState::Open
    service.info  ||= ""

    if (service and service.changed?)
      msf_import_timestamps(opts,service)
      service.save!
    end

    if opts[:task]
      Mdm::TaskService.create(
          :task => opts[:task],
          :service => service
      )
    end

    ret[:service] = service
  }
  end

  def get_service(wspace, host, proto, port)
  ::ActiveRecord::Base.connection_pool.with_connection {
    host = get_host(:workspace => wspace, :address => host)
    return if not host
    return host.services.find_by_proto_and_port(proto, port)
  }
  end

  #
  # Iterates over the services table calling the supplied block with the
  # service instance of each entry.
  #
  def each_service(wspace=workspace, &block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    services(wspace).each do |service|
      block.call(service)
    end
  }
  end

  #
  # Returns a list of all services in the database
  #
  def services(wspace = workspace, only_up = false, proto = nil, addresses = nil, ports = nil, names = nil)
  ::ActiveRecord::Base.connection_pool.with_connection {
    conditions = {}
    conditions[:state] = [ServiceState::Open] if only_up
    conditions[:proto] = proto if proto
    conditions["hosts.address"] = addresses if addresses
    conditions[:port] = ports if ports
    conditions[:name] = names if names
    wspace.services.includes(:host).where(conditions).order("hosts.address, port")
  }
  end

  # Returns a session based on opened_time, host address, and workspace
  # (or returns nil)
  def get_session(opts)
    return if not active
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts[:workspace] || opts[:wspace] || workspace
    addr   = opts[:addr] || opts[:address] || opts[:host] || return
    host = get_host(:workspace => wspace, :host => addr)
    time = opts[:opened_at] || opts[:created_at] || opts[:time] || return
    ::Mdm::Session.find_by_host_id_and_opened_at(host.id, time)
  }
  end

  # @note The Mdm::Session#desc will be truncated to 255 characters.
  # @todo https://www.pivotaltracker.com/story/show/48249739
  #
  # @overload report_session(opts)
  #   Creates an Mdm::Session from Msf::Session. If +via_exploit+ is set on the
  #   +session+, then an Mdm::Vuln and Mdm::ExploitAttempt is created for the
  #   session's host.  The Mdm::Host for the +session_host+ is created using
  #   The session.session_host, +session.arch+ (if +session+ responds to arch),
  #   and the workspace derived from opts or the +session+.  The Mdm::Session is
  #   assumed to be +last_seen+ and +opened_at+ at the time report_session is
  #   called.  +session.exploit_datastore['ParentModule']+ is used for the
  #   Mdm::Session#via_exploit if +session.via_exploit+ is
  #   'exploit/multi/handler'.
  #
  #   @param opts [Hash{Symbol => Object}] options
  #   @option opt [Msf::Session, #datastore, #platform, #type, #via_exploit, #via_payload] :session
  #     The in-memory session to persist to the database.
  #   @option opts [Mdm::Workspace] :workspace The workspace for in which the
  #     :session host is contained.  Also used as the workspace for the
  #     Mdm::ExploitAttempt and Mdm::Vuln.  Defaults to Mdm::Worksapce with
  #     Mdm::Workspace#name equal to +session.workspace+.
  #   @return [nil] if {Msf::DBManager#active} is +false+.
  #   @return [Mdm::Session] if session is saved
  #   @raise [ArgumentError] if :session is not an {Msf::Session}.
  #   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
  #     saved, in which case, the Mdm::ExploitAttempt and Mdm::Vuln will not be
  #     created, but the Mdm::Host will have been.   (There is no transaction
  #       to rollback the Mdm::Host creation.)
  #   @see #find_or_create_host
  #   @see #normalize_host
  #   @see #report_exploit_success
  #   @see #report_vuln
  #
  # @overload report_session(opts)
  #   Creates an Mdm::Session from Mdm::Host.
  #
  #   @param opts [Hash{Symbol => Object}] options
  #   @option opts [DateTime, Time] :closed_at The date and time the sesion was
  #     closed.
  #   @option opts [String] :close_reason Reason the session was closed.
  #   @option opts [Hash] :datastore {Msf::DataStore#to_h}.
  #   @option opts [String] :desc Session description.  Will be truncated to 255
  #     characters.
  #   @option opts [Mdm::Host] :host The host on which the session was opened.
  #   @option opts [DateTime, Time] :last_seen The last date and time the
  #     session was seen to be open.  Defaults to :closed_at's value.
  #   @option opts [DateTime, Time] :opened_at The date and time that the
  #     session was opened.
  #   @option opts [String] :platform The platform of the host.
  #   @option opts [Array] :routes ([]) The routes through the session for
  #     pivoting.
  #   @option opts [String] :stype Session type.
  #   @option opts [String] :via_exploit The {Msf::Module#fullname} of the
  #     exploit that was used to open the session.
  #   @option option [String] :via_payload the {MSf::Module#fullname} of the
  #     payload sent to the host when the exploit was successful.
  #   @return [nil] if {Msf::DBManager#active} is +false+.
  #   @return [Mdm::Session] if session is saved.
  #   @raise [ArgumentError] if :host is not an Mdm::Host.
  #   @raise [ActiveRecord::RecordInvalid] if session is invalid and cannot be
  #     saved.
  #
  # @raise ArgumentError if :host and :session is +nil+
  def report_session(opts)
    return if not active
  ::ActiveRecord::Base.connection_pool.with_connection {
    if opts[:session]
      raise ArgumentError.new("Invalid :session, expected Msf::Session") unless opts[:session].kind_of? Msf::Session
      session = opts[:session]
      wspace = opts[:workspace] || find_workspace(session.workspace)
      h_opts = { }
      h_opts[:host]      = normalize_host(session)
      h_opts[:arch]      = session.arch if session.respond_to?(:arch) and session.arch
      h_opts[:workspace] = wspace
      host = find_or_create_host(h_opts)
      sess_data = {
          :host_id     => host.id,
          :stype       => session.type,
          :desc        => session.info,
          :platform    => session.platform,
          :via_payload => session.via_payload,
          :via_exploit => session.via_exploit,
          :routes      => [],
          :datastore   => session.exploit_datastore.to_h,
          :port        => session.session_port,
          :opened_at   => Time.now.utc,
          :last_seen   => Time.now.utc,
          :local_id    => session.sid
      }
    elsif opts[:host]
      raise ArgumentError.new("Invalid :host, expected Host object") unless opts[:host].kind_of? ::Mdm::Host
      host = opts[:host]
      sess_data = {
        :host_id => host.id,
        :stype => opts[:stype],
        :desc => opts[:desc],
        :platform => opts[:platform],
        :via_payload => opts[:via_payload],
        :via_exploit => opts[:via_exploit],
        :routes => opts[:routes] || [],
        :datastore => opts[:datastore],
        :opened_at => opts[:opened_at],
        :closed_at => opts[:closed_at],
        :last_seen => opts[:last_seen] || opts[:closed_at],
        :close_reason => opts[:close_reason],
      }
    else
      raise ArgumentError.new("Missing option :session or :host")
    end
    ret = {}

    # Truncate the session data if necessary
    if sess_data[:desc]
      sess_data[:desc] = sess_data[:desc][0,255]
    end

    # In the case of multi handler we cannot yet determine the true
    # exploit responsible. But we can at least show the parent versus
    # just the generic handler:
    if session and session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
      sess_data[:via_exploit] = sess_data[:datastore]['ParentModule']
    end

    s = ::Mdm::Session.new(sess_data)
    s.save!

    if session and session.exploit_task and session.exploit_task.record
      session_task =  session.exploit_task.record
      if session_task.class == Mdm::Task
        Mdm::TaskSession.create(:task => session_task, :session => s )
      end
    end


    if opts[:session]
      session.db_record = s
    end

    # If this is a live session, we know the host is vulnerable to something.
    if opts[:session] and session.via_exploit
      mod = framework.modules.create(session.via_exploit)

      if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
        mod_fullname = sess_data[:datastore]['ParentModule']
        mod_name = ::Mdm::Module::Detail.find_by_fullname(mod_fullname).name
      else
        mod_name = mod.name
        mod_fullname = mod.fullname
      end

      vuln_info = {
        :host => host.address,
        :name => mod_name,
        :refs => mod.references,
        :workspace => wspace,
        :exploited_at => Time.now.utc,
        :info => "Exploited by #{mod_fullname} to create Session #{s.id}"
      }

      port    = session.exploit_datastore["RPORT"]
      service = (port ? host.services.find_by_port(port.to_i) : nil)

      vuln_info[:service] = service if service

      vuln = framework.db.report_vuln(vuln_info)

      if session.via_exploit == "exploit/multi/handler" and sess_data[:datastore]['ParentModule']
        via_exploit = sess_data[:datastore]['ParentModule']
      else
        via_exploit = session.via_exploit
      end
      attempt_info = {
        :timestamp   => Time.now.utc,
        :workspace   => wspace,
        :module      => via_exploit,
        :username    => session.username,
        :refs        => mod.references,
        :session_id  => s.id,
        :host        => host,
        :service     => service,
        :vuln        => vuln
      }

      framework.db.report_exploit_success(attempt_info)

    end

    s
  }
  end

  #
  # Record a session event in the database
  #
  # opts MUST contain one of:
  # +:session+:: the Msf::Session OR the ::Mdm::Session we are reporting
  # +:etype+::   event type, enum: command, output, upload, download, filedelete
  #
  # opts may contain
  # +:output+::      the data for an output event
  # +:command+::     the data for an command event
  # +:remote_path+:: path to the associated file for upload, download, and filedelete events
  # +:local_path+::  path to the associated file for upload, and download
  #
  def report_session_event(opts)
    return if not active
    raise ArgumentError.new("Missing required option :session") if opts[:session].nil?
    raise ArgumentError.new("Expected an :etype") unless opts[:etype]
    session = nil

  ::ActiveRecord::Base.connection_pool.with_connection {
    if opts[:session].respond_to? :db_record
      session = opts[:session].db_record
      if session.nil?
        # The session doesn't have a db_record which means
        #  a) the database wasn't connected at session registration time
        # or
        #  b) something awful happened and the report_session call failed
        #
        # Either way, we can't do anything with this session as is, so
        # log a warning and punt.
        wlog("Warning: trying to report a session_event for a session with no db_record (#{opts[:session].sid})")
        return
      end
      event_data = { :created_at => Time.now }
    else
      session = opts[:session]
      event_data = { :created_at => opts[:created_at] }
    end

    event_data[:session_id] = session.id
    [:remote_path, :local_path, :output, :command, :etype].each do |attr|
      event_data[attr] = opts[attr] if opts[attr]
    end

    s = ::Mdm::SessionEvent.create(event_data)
  }
  end

  def report_session_route(session, route)
    return if not active
    if session.respond_to? :db_record
      s = session.db_record
    else
      s = session
    end
    unless s.respond_to?(:routes)
      raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}")
    end

  ::ActiveRecord::Base.connection_pool.with_connection {

    subnet, netmask = route.split("/")
    s.routes.create(:subnet => subnet, :netmask => netmask)
  }
  end

  def report_session_route_remove(session, route)
    return if not active
    if session.respond_to? :db_record
      s = session.db_record
    else
      s = session
    end
    unless s.respond_to?(:routes)
      raise ArgumentError.new("Invalid :session, expected Session object got #{session.class}")
    end

  ::ActiveRecord::Base.connection_pool.with_connection {
    subnet, netmask = route.split("/")
    r = s.routes.find_by_subnet_and_netmask(subnet, netmask)
    r.destroy if r
  }
  end


  def report_exploit_success(opts)
  ::ActiveRecord::Base.connection_pool.with_connection {

    wspace = opts.delete(:workspace) || workspace
    mrefs  = opts.delete(:refs) || return
    host   = opts.delete(:host)
    port   = opts.delete(:port)
    prot   = opts.delete(:proto)
    svc    = opts.delete(:service)
    vuln   = opts.delete(:vuln)

    timestamp = opts.delete(:timestamp)
    username  = opts.delete(:username)
    mname     = opts.delete(:module)

    # Look up or generate the host as appropriate
    if not (host and host.kind_of? ::Mdm::Host)
      if svc.kind_of? ::Mdm::Service
        host = svc.host
      else
        host = report_host(:workspace => wspace, :address => host )
      end
    end

    # Bail if we dont have a host object
    return if not host

    # Look up or generate the service as appropriate
    if port and svc.nil?
      svc = report_service(:workspace => wspace, :host => host, :port => port, :proto => prot ) if port
    end

    if not vuln
      # Create a references map from the module list
      ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref|
        if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val)
          "#{ref.ctx_id}-#{ref.ctx_val}"
        else
          ref.to_s
        end
      })

      # Try find a matching vulnerability
      vuln = find_vuln_by_refs(ref_objs, host, svc)
    end

    # We have match, lets create a vuln_attempt record
    if vuln
      attempt_info = {
        :vuln_id      => vuln.id,
        :attempted_at => timestamp || Time.now.utc,
        :exploited    => true,
        :username     => username  || "unknown",
        :module       => mname
      }

      attempt_info[:session_id] = opts[:session_id] if opts[:session_id]
      attempt_info[:loot_id]    = opts[:loot_id]    if opts[:loot_id]

      vuln.vuln_attempts.create(attempt_info)

      # Correct the vuln's associated service if necessary
      if svc and vuln.service_id.nil?
        vuln.service = svc
        vuln.save
      end
    end

    # Report an exploit attempt all the same
    attempt_info = {
      :attempted_at => timestamp || Time.now.utc,
      :exploited    => true,
      :username     => username  || "unknown",
      :module       => mname
    }

    attempt_info[:vuln_id]    = vuln.id           if vuln
    attempt_info[:session_id] = opts[:session_id] if opts[:session_id]
    attempt_info[:loot_id]    = opts[:loot_id]    if opts[:loot_id]

    if svc
      attempt_info[:port]  = svc.port
      attempt_info[:proto] = svc.proto
    end

    if port and svc.nil?
      attempt_info[:port]  = port
      attempt_info[:proto] = prot || "tcp"
    end

    host.exploit_attempts.create(attempt_info)
  }
  end

  def report_exploit_failure(opts)

  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    mrefs  = opts.delete(:refs) || return
    host   = opts.delete(:host)
    port   = opts.delete(:port)
    prot   = opts.delete(:proto)
    svc    = opts.delete(:service)
    vuln   = opts.delete(:vuln)

    timestamp  = opts.delete(:timestamp)
    freason    = opts.delete(:fail_reason)
    fdetail    = opts.delete(:fail_detail)
    username   = opts.delete(:username)
    mname      = opts.delete(:module)

    # Look up the host as appropriate
    if not (host and host.kind_of? ::Mdm::Host)
      if svc.kind_of? ::Mdm::Service
        host = svc.host
      else
        host = get_host( :workspace => wspace, :address => host )
      end
    end

    # Bail if we dont have a host object
    return if not host

    # Look up the service as appropriate
    if port and svc.nil?
      prot ||= "tcp"
      svc = get_service(wspace, host, prot, port) if port
    end

    if not vuln
      # Create a references map from the module list
      ref_objs = ::Mdm::Ref.where(:name => mrefs.map { |ref|
        if ref.respond_to?(:ctx_id) and ref.respond_to?(:ctx_val)
          "#{ref.ctx_id}-#{ref.ctx_val}"
        else
          ref.to_s
        end
      })

      # Try find a matching vulnerability
      vuln = find_vuln_by_refs(ref_objs, host, svc)
    end

    # Report a vuln_attempt if we found a match
    if vuln
      attempt_info = {
        :attempted_at => timestamp || Time.now.utc,
        :exploited    => false,
        :fail_reason  => freason,
        :fail_detail  => fdetail,
        :username     => username  || "unknown",
        :module       => mname
      }

      vuln.vuln_attempts.create(attempt_info)
    end

    # Report an exploit attempt all the same
    attempt_info = {
      :attempted_at => timestamp || Time.now.utc,
      :exploited    => false,
      :username     => username  || "unknown",
      :module       => mname,
      :fail_reason  => freason,
      :fail_detail  => fdetail
    }

    attempt_info[:vuln_id] = vuln.id if vuln

    if svc
      attempt_info[:port]  = svc.port
      attempt_info[:proto] = svc.proto
    end

    if port and svc.nil?
      attempt_info[:port]  = port
      attempt_info[:proto] = prot || "tcp"
    end

    host.exploit_attempts.create(attempt_info)
  }
  end


  def report_vuln_attempt(vuln, opts)
  ::ActiveRecord::Base.connection_pool.with_connection {
    return if not vuln
    info = {}

    # Opts can be keyed by strings or symbols
    ::Mdm::VulnAttempt.column_names.each do |kn|
      k = kn.to_sym
      next if ['id', 'vuln_id'].include?(kn)
      info[k] = opts[kn] if opts[kn]
      info[k] = opts[k]  if opts[k]
    end

    return unless info[:attempted_at]

    vuln.vuln_attempts.create(info)
  }
  end

  def report_exploit_attempt(host, opts)
  ::ActiveRecord::Base.connection_pool.with_connection {
    return if not host
    info = {}

    # Opts can be keyed by strings or symbols
    ::Mdm::VulnAttempt.column_names.each do |kn|
      k = kn.to_sym
      next if ['id', 'host_id'].include?(kn)
      info[k] = opts[kn] if opts[kn]
      info[k] = opts[k]  if opts[k]
    end

    host.exploit_attempts.create(info)
  }
  end

  def get_client(opts)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    host   = get_host(:workspace => wspace, :host => opts[:host]) || return
    client = host.clients.where({:ua_string => opts[:ua_string]}).first()
    return client
  }
  end

  def find_or_create_client(opts)
    report_client(opts)
  end

  #
  # Report a client running on a host.
  #
  # opts MUST contain
  # +:ua_string+::  the value of the User-Agent header
  # +:host+::       the host where this client connected from, can be an ip address or a Host object
  #
  # opts can contain
  # +:ua_name+::    one of the Msf::HttpClients constants
  # +:ua_ver+::     detected version of the given client
  # +:campaign+::   an id or Campaign object
  #
  # Returns a Client.
  #
  def report_client(opts)
    return if not active
  ::ActiveRecord::Base.connection_pool.with_connection {
    addr = opts.delete(:host) || return
    wspace = opts.delete(:workspace) || workspace
    report_host(:workspace => wspace, :host => addr)

    ret = {}

    host = get_host(:workspace => wspace, :host => addr)
    client = host.clients.find_or_initialize_by_ua_string(opts[:ua_string])

    opts[:ua_string] = opts[:ua_string].to_s

    campaign = opts.delete(:campaign)
    if campaign
      case campaign
      when Campaign
        opts[:campaign_id] = campaign.id
      else
        opts[:campaign_id] = campaign
      end
    end

    opts.each { |k,v|
      if (client.attribute_names.include?(k.to_s))
        client[k] = v
      else
        dlog("Unknown attribute for Client: #{k}")
      end
    }
    if (client and client.changed?)
      client.save!
    end
    ret[:client] = client
  }
  end

  #
  # This method iterates the vulns table calling the supplied block with the
  # vuln instance of each entry.
  #
  def each_vuln(wspace=workspace,&block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.vulns.each do |vulns|
      block.call(vulns)
    end
  }
  end

  #
  # This methods returns a list of all vulnerabilities in the database
  #
  def vulns(wspace=workspace)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.vulns
  }
  end

  #
  # This methods returns a list of all credentials in the database
  #
  def creds(wspace=workspace)
  ::ActiveRecord::Base.connection_pool.with_connection {
    Mdm::Cred.includes({:service => :host}).where("hosts.workspace_id = ?", wspace.id)
  }
  end

  #
  # This method returns a list of all exploited hosts in the database.
  #
  def exploited_hosts(wspace=workspace)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.exploited_hosts
  }
  end

  #
  # This method iterates the notes table calling the supplied block with the
  # note instance of each entry.
  #
  def each_note(wspace=workspace, &block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.notes.each do |note|
      block.call(note)
    end
  }
  end

  #
  # Find or create a note matching this type/data
  #
  def find_or_create_note(opts)
    report_note(opts)
  end

  #
  # Report a Note to the database.  Notes can be tied to a ::Mdm::Workspace, Host, or Service.
  #
  # opts MUST contain
  # +:type+::  The type of note, e.g. smb_peer_os
  #
  # opts can contain
  # +:workspace+::  the workspace to associate with this Note
  # +:host+::       an IP address or a Host object to associate with this Note
  # +:service+::    a Service object to associate with this Note
  # +:data+::       whatever it is you're making a note of
  # +:port+::       along with +:host+ and +:proto+, a service to associate with this Note
  # +:proto+::      along with +:host+ and +:port+, a service to associate with this Note
  # +:update+::     what to do in case a similar Note exists, see below
  #
  # The +:update+ option can have the following values:
  # +:unique+::       allow only a single Note per +:host+/+:type+ pair
  # +:unique_data+::  like +:uniqe+, but also compare +:data+
  # +:insert+::       always insert a new Note even if one with identical values exists
  #
  # If the provided +:host+ is an IP address and does not exist in the
  # database, it will be created.  If +:workspace+, +:host+ and +:service+
  # are all omitted, the new Note will be associated with the current
  # workspace.
  #
  def report_note(opts)
    return if not active
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace = opts.delete(:workspace) || workspace
    if wspace.kind_of? String
      wspace = find_workspace(wspace)
    end
    seen = opts.delete(:seen) || false
    crit = opts.delete(:critical) || false
    host = nil
    addr = nil
    # Report the host so it's there for the Proc to use below
    if opts[:host]
      if opts[:host].kind_of? ::Mdm::Host
        host = opts[:host]
      else
        addr = normalize_host(opts[:host])
        host = report_host({:workspace => wspace, :host => addr})
      end
      # Do the same for a service if that's also included.
      if (opts[:port])
        proto = nil
        sname = nil
        case opts[:proto].to_s.downcase # Catch incorrect usages
        when 'tcp','udp'
          proto = opts[:proto]
          sname = opts[:sname] if opts[:sname]
        when 'dns','snmp','dhcp'
          proto = 'udp'
          sname = opts[:proto]
        else
          proto = 'tcp'
          sname = opts[:proto]
        end
        sopts = {
          :workspace => wspace,
          :host  => host,
          :port  => opts[:port],
          :proto => proto
        }
        sopts[:name] = sname if sname
        report_service(sopts)
      end
    end
    # Update Modes can be :unique, :unique_data, :insert
    mode = opts[:update] || :unique

    ret = {}

    if addr and not host
      host = get_host(:workspace => wspace, :host => addr)
    end
    if host and (opts[:port] and opts[:proto])
      service = get_service(wspace, host, opts[:proto], opts[:port])
    elsif opts[:service] and opts[:service].kind_of? ::Mdm::Service
      service = opts[:service]
    end
=begin
    if host
      host.updated_at = host.created_at
      host.state      = HostState::Alive
      host.save!
    end
=end
    ntype  = opts.delete(:type) || opts.delete(:ntype) || (raise RuntimeError, "A note :type or :ntype is required")
    data   = opts[:data]
    method = nil
    args   = []
    note   = nil

    conditions = { :ntype => ntype }
    conditions[:host_id] = host[:id] if host
    conditions[:service_id] = service[:id] if service

    case mode
    when :unique
      notes = wspace.notes.where(conditions)

      # Only one note of this type should exist, make a new one if it
      # isn't there. If it is, grab it and overwrite its data.
      if notes.empty?
        note = wspace.notes.new(conditions)
      else
        note = notes[0]
      end
      note.data = data
    when :unique_data
      notes = wspace.notes.where(conditions)

      # Don't make a new Note with the same data as one that already
      # exists for the given: type and (host or service)
      notes.each do |n|
        # Compare the deserialized data from the table to the raw
        # data we're looking for.  Because of the serialization we
        # can't do this easily or reliably in SQL.
        if n.data == data
          note = n
          break
        end
      end
      if not note
        # We didn't find one with the data we're looking for, make
        # a new one.
        note = wspace.notes.new(conditions.merge(:data => data))
      end
    else
      # Otherwise, assume :insert, which means always make a new one
      note = wspace.notes.new
      if host
        note.host_id = host[:id]
      end
      if opts[:service] and opts[:service].kind_of? ::Mdm::Service
        note.service_id = opts[:service][:id]
      end
      note.seen     = seen
      note.critical = crit
      note.ntype    = ntype
      note.data     = data
    end
    msf_import_timestamps(opts,note)
    note.save!
    ret[:note] = note
  }
  end

  #
  # This methods returns a list of all notes in the database
  #
  def notes(wspace=workspace)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.notes
  }
  end

  # This is only exercised by MSF3 XML importing for now. Needs the wait
  # conditions and return hash as well.
  def report_host_tag(opts)
    name = opts.delete(:name)
    raise DBImportError.new("Missing required option :name") unless name
    addr = opts.delete(:addr)
    raise DBImportError.new("Missing required option :addr") unless addr
    wspace = opts.delete(:wspace)
    raise DBImportError.new("Missing required option :wspace") unless wspace
  ::ActiveRecord::Base.connection_pool.with_connection {
    if wspace.kind_of? String
      wspace = find_workspace(wspace)
    end

    host = nil
    report_host(:workspace => wspace, :address => addr)


    host = get_host(:workspace => wspace, :address => addr)
    desc = opts.delete(:desc)
    summary = opts.delete(:summary)
    detail = opts.delete(:detail)
    crit = opts.delete(:crit)
    possible_tags = Mdm::Tag.includes(:hosts).where("hosts.workspace_id = ? and tags.name = ?", wspace.id, name).order("tags.id DESC").limit(1)
    tag = (possible_tags.blank? ? Mdm::Tag.new : possible_tags.first)
    tag.name = name
    tag.desc = desc
    tag.report_summary = !!summary
    tag.report_detail = !!detail
    tag.critical = !!crit
    tag.hosts = tag.hosts | [host]
    tag.save! if tag.changed?
  }
  end

  #
  # Store a set of credentials in the database.
  #
  # report_auth_info used to create a note, now it creates
  # an entry in the creds table. It's much more akin to
  # report_vuln() now.
  #
  # opts MUST contain
  # +:host+::   an IP address or Host object reference
  # +:port+::   a port number
  #
  # opts can contain
  # +:user+::   the username
  # +:pass+::   the password, or path to ssh_key
  # +:ptype+::  the type of password (password(ish), hash, or ssh_key)
  # +:proto+::  a transport name for the port
  # +:sname+::  service name
  # +:active+:: by default, a cred is active, unless explicitly false
  # +:proof+::  data used to prove the account is actually active.
  #
  # Sources: Credentials can be sourced from another credential, or from
  # a vulnerability. For example, if an exploit was used to dump the
  # smb_hashes, and this credential comes from there, the source_id would
  # be the Vuln id (as reported by report_vuln) and the type would be "Vuln".
  #
  # +:source_id+::   The Vuln or Cred id of the source of this cred.
  # +:source_type+:: Either Vuln or Cred
  #
  # TODO: This is written somewhat host-centric, when really the
  # Service is the thing. Need to revisit someday.
  def report_auth_info(opts={})
    return if not active
    raise ArgumentError.new("Missing required option :host") if opts[:host].nil?
    raise ArgumentError.new("Missing required option :port") if (opts[:port].nil? and opts[:service].nil?)

    if (not opts[:host].kind_of?(::Mdm::Host)) and (not validate_ips(opts[:host]))
      raise ArgumentError.new("Invalid address or object for :host (#{opts[:host].inspect})")
    end

  ::ActiveRecord::Base.connection_pool.with_connection {
    host = opts.delete(:host)
    ptype = opts.delete(:type) || "password"
    token = [opts.delete(:user), opts.delete(:pass)]
    sname = opts.delete(:sname)
    port = opts.delete(:port)
    proto = opts.delete(:proto) || "tcp"
    proof = opts.delete(:proof)
    source_id = opts.delete(:source_id)
    source_type = opts.delete(:source_type)
    duplicate_ok = opts.delete(:duplicate_ok)
    # Nil is true for active.
    active = (opts[:active] || opts[:active].nil?) ? true : false

    wspace = opts.delete(:workspace) || workspace

    # Service management; assume the user knows what
    # he's talking about.
    service = opts.delete(:service) || report_service(:host => host, :port => port, :proto => proto, :name => sname, :workspace => wspace)

    # Non-US-ASCII usernames are tripping up the database at the moment, this is a temporary fix until we update the tables
    if (token[0])
      # convert the token to US-ASCII from UTF-8 to prevent an error
      token[0] = token[0].unpack("C*").pack("C*")
      token[0] = token[0].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] }
    end

    if (token[1])
      token[1] = token[1].unpack("C*").pack("C*")
      token[1] = token[1].gsub(/[\x00-\x1f\x7f-\xff]/n){|m| "\\x%.2x" % m.unpack("C")[0] }
    end

    ret = {}

    # Check to see if the creds already exist. We look also for a downcased username with the
    # same password because we can fairly safely assume they are not in fact two seperate creds.
    # this allows us to hedge against duplication of creds in the DB.

    if duplicate_ok
    # If duplicate usernames are okay, find by both user and password (allows
    # for actual duplicates to get modified updated_at, sources, etc)
      if token[0].nil? or token[0].empty?
        cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
      else
        cred = service.creds.find_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
        unless cred
          dcu = token[0].downcase
          cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "")
          unless cred
            cred = service.creds.find_or_initialize_by_user_and_ptype_and_pass(token[0] || "", ptype, token[1] || "")
          end
        end
      end
    else
      # Create the cred by username only (so we can change passwords)
      if token[0].nil? or token[0].empty?
        cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype)
      else
        cred = service.creds.find_by_user_and_ptype(token[0] || "", ptype)
        unless cred
          dcu = token[0].downcase
          cred = service.creds.find_by_user_and_ptype_and_pass( dcu || "", ptype, token[1] || "")
          unless cred
            cred = service.creds.find_or_initialize_by_user_and_ptype(token[0] || "", ptype)
          end
        end
      end
    end

    # Update with the password
    cred.pass = (token[1] || "")

    # Annotate the credential
    cred.ptype = ptype
    cred.active = active

    # Update the source ID only if there wasn't already one.
    if source_id and !cred.source_id
      cred.source_id = source_id
      cred.source_type = source_type if source_type
    end

    # Safe proof (lazy way) -- doesn't chop expanded
    # characters correctly, but shouldn't ever be a problem.
    unless proof.nil?
      proof = Rex::Text.to_hex_ascii(proof)
      proof = proof[0,4096]
    end
    cred.proof = proof

    # Update the timestamp
    if cred.changed?
      msf_import_timestamps(opts,cred)
      cred.save!
    end

    # Ensure the updated_at is touched any time report_auth_info is called
    # except when it's set explicitly (as it is for imports)
    unless opts[:updated_at] || opts["updated_at"]
      cred.updated_at = Time.now.utc
      cred.save!
    end


    if opts[:task]
      Mdm::TaskCred.create(
          :task => opts[:task],
          :cred => cred
      )
    end

    ret[:cred] = cred
  }
  end

  alias :report_cred :report_auth_info
  alias :report_auth :report_auth_info

  #
  # Find or create a credential matching this type/data
  #
  def find_or_create_cred(opts)
    report_auth_info(opts)
  end

  #
  # This method iterates the creds table calling the supplied block with the
  # cred instance of each entry.
  #
  def each_cred(wspace=workspace,&block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.creds.each do |cred|
      block.call(cred)
    end
  }
  end

  def each_exploited_host(wspace=workspace,&block)
  ::ActiveRecord::Base.connection_pool.with_connection {
    wspace.exploited_hosts.each do |eh|
      block.call(eh)
    end
  }
  end

  #
  # Find or create a vuln matching this service/name
  #
  def find_or_create_vuln(opts)
    report_vuln(opts)
  end

  #
  # opts MUST contain
  # +:host+:: the host where this vulnerability resides
  # +:name+:: the friendly name for this vulnerability (title)
  #
  # opts can contain
  # +:info+::   a human readable description of the vuln, free-form text
  # +:refs+::   an array of Ref objects or string names of references
  # +:details:: a hash with :key pointed to a find criteria hash and the rest containing VulnDetail fields
  #
  def report_vuln(opts)
    return if not active
    raise ArgumentError.new("Missing required option :host") if opts[:host].nil?
    raise ArgumentError.new("Deprecated data column for vuln, use .info instead") if opts[:data]
    name = opts[:name] || return
    info = opts[:info]

  ::ActiveRecord::Base.connection_pool.with_connection {

    wspace = opts.delete(:workspace) || workspace
    exploited_at = opts[:exploited_at] || opts["exploited_at"]
    details = opts.delete(:details)
    rids = opts.delete(:ref_ids)

    if opts[:refs]
      rids ||= []
      opts[:refs].each do |r|
        if (r.respond_to?(:ctx_id)) and (r.respon…