/FPEngine.cc

Large files files are truncated, but you can click here to view the full file

/***************************************************************************
 * FPEngine.cc -- Routines used for IPv6 OS detection via TCP/IP           *
 * fingerprinting.  * For more information on how this works in Nmap, see  *
 * http://nmap.org/osdetect/                                               *
 *                                                                         *
 ***********************IMPORTANT NMAP LICENSE TERMS************************
 *                                                                         *
 * The Nmap Security Scanner is (C) 1996-2015 Insecure.Com LLC. Nmap is    *
 * also a registered trademark of Insecure.Com LLC.  This program is free  *
 * software; you may redistribute and/or modify it under the terms of the  *
 * GNU General Public License as published by the Free Software            *
 * Foundation; Version 2 ("GPL"), BUT ONLY WITH ALL OF THE CLARIFICATIONS  *
 * AND EXCEPTIONS DESCRIBED HEREIN.  This guarantees your right to use,    *
 * modify, and redistribute this software under certain conditions.  If    *
 * you wish to embed Nmap technology into proprietary software, we sell    *
 * alternative licenses (contact sales@nmap.com).  Dozens of software      *
 * vendors already license Nmap technology such as host discovery, port    *
 * scanning, OS detection, version detection, and the Nmap Scripting       *
 * Engine.                                                                 *
 *                                                                         *
 * Note that the GPL places important restrictions on "derivative works",  *
 * yet it does not provide a detailed definition of that term.  To avoid   *
 * misunderstandings, we interpret that term as broadly as copyright law   *
 * allows.  For example, we consider an application to constitute a        *
 * derivative work for the purpose of this license if it does any of the   *
 * following with any software or content covered by this license          *
 * ("Covered Software"):                                                   *
 *                                                                         *
 * o Integrates source code from Covered Software.                         *
 *                                                                         *
 * o Reads or includes copyrighted data files, such as Nmap's nmap-os-db   *
 * or nmap-service-probes.                                                 *
 *                                                                         *
 * o Is designed specifically to execute Covered Software and parse the    *
 * results (as opposed to typical shell or execution-menu apps, which will *
 * execute anything you tell them to).                                     *
 *                                                                         *
 * o Includes Covered Software in a proprietary executable installer.  The *
 * installers produced by InstallShield are an example of this.  Including *
 * Nmap with other software in compressed or archival form does not        *
 * trigger this provision, provided appropriate open source decompression  *
 * or de-archiving software is widely available for no charge.  For the    *
 * purposes of this license, an installer is considered to include Covered *
 * Software even if it actually retrieves a copy of Covered Software from  *
 * another source during runtime (such as by downloading it from the       *
 * Internet).                                                              *
 *                                                                         *
 * o Links (statically or dynamically) to a library which does any of the  *
 * above.                                                                  *
 *                                                                         *
 * o Executes a helper program, module, or script to do any of the above.  *
 *                                                                         *
 * This list is not exclusive, but is meant to clarify our interpretation  *
 * of derived works with some common examples.  Other people may interpret *
 * the plain GPL differently, so we consider this a special exception to   *
 * the GPL that we apply to Covered Software.  Works which meet any of     *
 * these conditions must conform to all of the terms of this license,      *
 * particularly including the GPL Section 3 requirements of providing      *
 * source code and allowing free redistribution of the work as a whole.    *
 *                                                                         *
 * As another special exception to the GPL terms, Insecure.Com LLC grants  *
 * permission to link the code of this program with any version of the     *
 * OpenSSL library which is distributed under a license identical to that  *
 * listed in the included docs/licenses/OpenSSL.txt file, and distribute   *
 * linked combinations including the two.                                  *
 *                                                                         *
 * Any redistribution of Covered Software, including any derived works,    *
 * must obey and carry forward all of the terms of this license, including *
 * obeying all GPL rules and restrictions.  For example, source code of    *
 * the whole work must be provided and free redistribution must be         *
 * allowed.  All GPL references to "this License", are to be treated as    *
 * including the terms and conditions of this license text as well.        *
 *                                                                         *
 * Because this license imposes special exceptions to the GPL, Covered     *
 * Work may not be combined (even as part of a larger work) with plain GPL *
 * software.  The terms, conditions, and exceptions of this license must   *
 * be included as well.  This license is incompatible with some other open *
 * source licenses as well.  In some cases we can relicense portions of    *
 * Nmap or grant special permissions to use it in other open source        *
 * software.  Please contact fyodor@nmap.org with any such requests.       *
 * Similarly, we don't incorporate incompatible open source software into  *
 * Covered Software without special permission from the copyright holders. *
 *                                                                         *
 * If you have any questions about the licensing restrictions on using     *
 * Nmap in other works, are happy to help.  As mentioned above, we also    *
 * offer alternative license to integrate Nmap into proprietary            *
 * applications and appliances.  These contracts have been sold to dozens  *
 * of software vendors, and generally include a perpetual license as well  *
 * as providing for priority support and updates.  They also fund the      *
 * continued development of Nmap.  Please email sales@nmap.com for further *
 * information.                                                            *
 *                                                                         *
 * If you have received a written license agreement or contract for        *
 * Covered Software stating terms other than these, you may choose to use  *
 * and redistribute Covered Software under those terms instead of these.   *
 *                                                                         *
 * Source is provided to this software because we believe users have a     *
 * right to know exactly what a program is going to do before they run it. *
 * This also allows you to audit the software for security holes.          *
 *                                                                         *
 * Source code also allows you to port Nmap to new platforms, fix bugs,    *
 * and add new features.  You are highly encouraged to send your changes   *
 * to the dev@nmap.org mailing list for possible incorporation into the    *
 * main distribution.  By sending these changes to Fyodor or one of the    *
 * Insecure.Org development mailing lists, or checking them into the Nmap  *
 * source code repository, it is understood (unless you specify otherwise) *
 * that you are offering the Nmap Project (Insecure.Com LLC) the           *
 * unlimited, non-exclusive right to reuse, modify, and relicense the      *
 * code.  Nmap will always be available Open Source, but this is important *
 * because the inability to relicense code has caused devastating problems *
 * for other Free Software projects (such as KDE and NASM).  We also       *
 * occasionally relicense the code to third parties as discussed above.    *
 * If you wish to specify special license conditions of your               *
 * contributions, just say so when you send them.                          *
 *                                                                         *
 * This program is distributed in the hope that it will be useful, but     *
 * WITHOUT ANY WARRANTY; without even the implied warranty of              *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the Nmap      *
 * license file for more details (it's in a COPYING file included with     *
 * Nmap, and also available from https://svn.nmap.org/nmap/COPYING)        *
 *                                                                         *
 ***************************************************************************/

/* $Id$ */

#include "FPEngine.h"
#include "Target.h"
#include "FingerPrintResults.h"
#include "NmapOps.h"
#include "nmap_error.h"
#include "osscan.h"
#include "linear.h"
#include "FPModel.h"
extern NmapOps o;

#include <math.h>


/******************************************************************************
 * Globals.                                                                   *
 ******************************************************************************/

/* This is the global network controller. FPHost classes use it to request
 * network resources and schedule packet transmissions. */
FPNetworkControl global_netctl;


/******************************************************************************
 * Implementation of class FPNetworkControl.                                  *
 ******************************************************************************/
FPNetworkControl::FPNetworkControl() {
  memset(&this->nsp, 0, sizeof(nsock_pool));
  memset(&this->pcap_nsi, 0, sizeof(pcap_nsi));
  memset(&this->pcap_ev_id, 0, sizeof(nsock_event_id));
  this->nsock_init = false;
  this->rawsd = -1;
  this->probes_sent = 0;
  this->responses_recv = 0;
  this->probes_timedout = 0;
  this->cc_cwnd = 0;
  this->cc_ssthresh = 0;
}


FPNetworkControl::~FPNetworkControl() {
  if (this->nsock_init) {
    nsock_event_cancel(this->nsp, this->pcap_ev_id, 0);
    nsock_pool_delete(this->nsp);
    this->nsock_init = false;
  }
}


/* (Re)-Initialize object's state (default parameter setup and nsock
 * initialization). */
void FPNetworkControl::init(const char *ifname, devtype iftype) {

  /* Init congestion control parameters */
  this->cc_init();

   /* If there was a previous nsock pool, delete it */
  if (this->pcap_nsi) {
    nsock_iod_delete(this->pcap_nsi, NSOCK_PENDING_SILENT);
  }
  if (this->nsock_init) {
    nsock_event_cancel(this->nsp, this->pcap_ev_id, 0);
    nsock_pool_delete(this->nsp);
  }

  /* Create a new nsock pool */
  if ((this->nsp = nsock_pool_new(NULL)) == NULL)
    fatal("Unable to obtain an Nsock pool");

  nsock_set_log_function(nmap_nsock_stderr_logger);
  nmap_adjust_loglevel(o.packetTrace());

  nsock_pool_set_device(nsp, o.device);

  if (o.proxy_chain)
    nsock_pool_set_proxychain(this->nsp, o.proxy_chain);

  /* Allow broadcast addresses */
  nsock_pool_set_broadcast(this->nsp, 1);

  /* Allocate an NSI for packet capture */
  this->pcap_nsi = nsock_iod_new(this->nsp, NULL);
  this->first_pcap_scheduled = false;

  /* Flag it as already initialized so we free this nsp next time */
  this->nsock_init = true;

  /* Obtain raw socket or check that we can obtain an eth descriptor. */
  if ((o.sendpref & PACKET_SEND_ETH) && iftype == devt_ethernet && ifname != NULL) {
    /* We don't need to store the eth handler because FPProbes come with a
     * suitable one (FPProbes::getEthernet()), we just attempt to obtain one
     * to see if it fails. */
    if (eth_open_cached(ifname) == NULL)
      fatal("dnet: failed to open device %s", ifname);
    this->rawsd = -1;
  } else {
#ifdef WIN32
    win32_fatal_raw_sockets(ifname);
#endif
    if (this->rawsd >= 0)
      close(this->rawsd);
    rawsd = nmap_raw_socket();
    if (rawsd < 0)
      pfatal("Couldn't obtain raw socket in %s", __func__);
  }

  /* De-register existing callers */
  while (this->callers.size() > 0) {
    this->callers.pop_back();
  }
  return;
}


/* This function initializes the controller's congestion control parameters.
 * The network controller uses TCP's Slow Start and Congestion Avoidance
 * algorithms from RFC 5681 (slightly modified for convenience).
 *
 * As the OS detection process does not open full TCP connections, we can't just
 * use ACKs (or the lack of ACKs) to increase or decrease the congestion window
 * so we use probe responses. Every time we get a response to an OS detection
 * probe, we treat it as if it was a TCP ACK in TCP's congestion control.
 *
 * Note that the initial Congestion Window is set to the number of timed
 * probes that we send to each target. This is necessary since we need to
 * know for sure that we can send that many packets in order to transmit them.
 * Otherwise, we could fail to deliver the probes 100ms apart. */
int FPNetworkControl::cc_init() {
  this->probes_sent = 0;
  this->responses_recv = 0;
  this->probes_timedout = 0;
  this->cc_cwnd = OSSCAN_INITIAL_CWND;
  this->cc_ssthresh = OSSCAN_INITIAL_SSTHRESH;
  return OP_SUCCESS;
}


/* This method is used to indicate that we have scheduled the transmission of
 * one or more packets. This is used in congestion control to determine the
 * number of outstanding probes (number of probes sent but not answered yet)
 * and therefore, the effective transmission window. @param pkts indicates the
 * number of packets that were scheduled. Returns OP_SUCCESS on success and
 * OP_FAILURE in case of error. */
int FPNetworkControl::cc_update_sent(int pkts = 1) {
  if (pkts <= 0)
    return OP_FAILURE;
  this->probes_sent+=pkts;
  return OP_SUCCESS;
}


/* This method is used to indicate that a drop has occurred. In TCP, drops are
 * detected by the absence of an ACK. However, we can't use that, since it is
 * very likely that our targets do not respond to some of our OS detection
 * probes intentionally. For this reason, we consider that a drop has occurred
 * when we receive a response for a probe that has already suffered one
 * retransmission (first transmission got dropped in transit, some later
 * transmission made it to the host and it responded). So when we detect a drop
 * we do the same as TCP, adjust the congestion window and the slow start
 * threshold. */
int FPNetworkControl::cc_report_drop() {
/* FROM RFC 5681

   When a TCP sender detects segment loss using the retransmission timer
   and the given segment has not yet been resent by way of the
   retransmission timer, the value of ssthresh MUST be set to no more
   than the value given in equation (4):

      ssthresh = max (FlightSize / 2, 2*SMSS)            (4)

   where, as discussed above, FlightSize is the amount of outstanding
   data in the network.

   On the other hand, when a TCP sender detects segment loss using the
   retransmission timer and the given segment has already been
   retransmitted by way of the retransmission timer at least once, the
   value of ssthresh is held constant.
 */
  int probes_outstanding = this->probes_sent - this->responses_recv - this->probes_timedout;
  this->cc_ssthresh = MAX(probes_outstanding, OSSCAN_INITIAL_CWND);
  this->cc_cwnd = OSSCAN_INITIAL_CWND;
  return OP_SUCCESS;
}


/* This method is used to indicate that a response to a previous probe was
 * received. For us this is like getting and ACK in TCP congestion control, so
 * we update the congestion window (increase by one packet if we are in slow
 * start or increase it by a small percentage of a packet if we are in
 * congestion avoidance). */
int FPNetworkControl::cc_update_received() {
  this->responses_recv++;
  /* If we are in Slow Start, increment congestion window by one packet.
   * (Note that we treat probe responses the same way TCP CC treats ACKs). */
  if (this->cc_cwnd < this->cc_ssthresh) {
    this->cc_cwnd += 1;
  /* Otherwise we are in Congestion Avoidance and CWND is incremented slowly,
   * approximately one packet per RTT */
  } else {
    this->cc_cwnd = this->cc_cwnd + 1/this->cc_cwnd;
  }
  if (o.debugging > 3) {
    log_write(LOG_PLAIN, "[FPNetworkControl] Congestion Control Parameters: cwnd=%f ssthresh=%f sent=%d recv=%d tout=%d outstanding=%d\n",
           this->cc_cwnd, this->cc_ssthresh,  this->probes_sent, this->responses_recv, this->probes_timedout,
           this->probes_sent - this->responses_recv - this->probes_timedout);
  }
  return OP_SUCCESS;
}


/* This method is public and can be called by FPHosts to inform the controller
 * that a probe has experienced a final timeout. In other words, that no
 * response was received for the probe after doing the necessary retransmissions
 * and waiting for the RTO. This is used to decrease the number of outstanding
 * probes. Otherwise, if no host responded to the probes, the effective
 * transmission window could reach zero and prevent new probes from being sent,
 * clogging the engine. */
int FPNetworkControl::cc_report_final_timeout() {
  this->probes_timedout++;
  return OP_SUCCESS;
}


/* This method is used by FPHosts to request permission to transmit a number of
 * probes. Permission is granted if the current congestion window allows the
 * transmission of new probes. It returns true if permission is granted and
 * false if it is denied. */
bool FPNetworkControl::request_slots(size_t num_packets) {
  int probes_outstanding = this->probes_sent - this->responses_recv - this->probes_timedout;
  if (o.debugging > 3)
    log_write(LOG_PLAIN, "[FPNetworkControl] Slot request for %u packets. ProbesOutstanding=%d cwnd=%f ssthresh=%f\n",
              (unsigned int)num_packets, probes_outstanding, this->cc_cwnd, this->cc_ssthresh);
  /* If we still have room for more outstanding probes, let the caller
   * schedule transmissions. */
  if ((probes_outstanding + num_packets) <= this->cc_cwnd) {
    this->cc_update_sent(num_packets);
    return true;
  }
  return false;
}


/* This method lets FPHosts register themselves in the network controller so
 * the controller can call them back every time a packet they are interested
 * in is captured.*/
int FPNetworkControl::register_caller(FPHost *newcaller) {
  this->callers.push_back(newcaller);
  return OP_SUCCESS;
}


/* This method lets FPHosts unregister themselves in the network controller so
 * the controller does not call them back again. This is called by hosts that
 * have already finished their OS detection. */
int FPNetworkControl::unregister_caller(FPHost *oldcaller) {
  for (size_t i = 0; i < this->callers.size(); i++) {
    if (this->callers[i] == oldcaller) {
      this->callers.erase(this->callers.begin() + i);
      return OP_SUCCESS;
    }
  }
  return OP_FAILURE;
}


/* This method gets the controller ready for packet capture. Basically it
 * obtains a pcap descriptor from nsock and sets an appropriate BPF filter. */
int FPNetworkControl::setup_sniffer(const char *iface, const char *bpf_filter) {
  char pcapdev[128];
  int rc;

#ifdef WIN32
  /* Nmap normally uses device names obtained through dnet for interfaces, but
     Pcap has its own naming system.  So the conversion is done here */
  if (!DnetName2PcapName(iface, pcapdev, sizeof(pcapdev))) {
    /* Oh crap -- couldn't find the corresponding dev apparently.  Let's just go
       with what we have then ... */
    Strncpy(pcapdev, iface, sizeof(pcapdev));
  }
#else
  Strncpy(pcapdev, iface, sizeof(pcapdev));
#endif

  /* Obtain a pcap descriptor */
  rc = nsock_pcap_open(this->nsp, this->pcap_nsi, pcapdev, 8192, 0, bpf_filter);
  if (rc)
    fatal("Error opening capture device %s\n", pcapdev);

  /* Store the pcap NSI inside the pool so we can retrieve it inside a callback */
  nsock_pool_set_udata(this->nsp, (void *)&(this->pcap_nsi));

  return OP_SUCCESS;
}


/* This method makes the controller process pending events (like packet
 * transmissions or packet captures). */
void FPNetworkControl::handle_events() {
  nmap_adjust_loglevel(o.packetTrace());
  nsock_loop(nsp, 50);
}


/* This method lets FPHosts to schedule the transmission of an OS detection
 * probe. It takes an FPProbe pointer and the amount of milliseconds the
 * controller should wait before injecting the probe into the wire. */
int FPNetworkControl::scheduleProbe(FPProbe *pkt, int in_msecs_time) {
  nsock_timer_create(this->nsp, probe_transmission_handler_wrapper, in_msecs_time, (void*)pkt);
  return OP_SUCCESS;
}


/* This is the handler for packet transmission. It is called by nsock whenever a timer expires,
 * which means that a new packet needs to be transmitted. Note that this method is not
 * called directly by Nsock but by the wrapper function probe_transmission_handler_wrapper().
 * The reason for that is because C++ does not allow to use class methods as callback
 * functions, so this is a small hack to make that happen. */
void FPNetworkControl::probe_transmission_handler(nsock_pool nsp, nsock_event nse, void *arg) {
  assert(nsock_pool_get_udata(nsp) != NULL);
  nsock_iod nsi_pcap = *((nsock_iod *)nsock_pool_get_udata(nsp));
  enum nse_status status = nse_status(nse);
  enum nse_type type = nse_type(nse);
  FPProbe *myprobe = (FPProbe *)arg;
  u8 *buf;
  size_t len;

  if (status == NSE_STATUS_SUCCESS) {
    switch(type) {
    /* Timer events mean that we need to send a packet.  */
    case NSE_TYPE_TIMER:

      /* The first time a packet is sent, we schedule a pcap event. After that
       * we don't have to worry since the response reception handler schedules
       * a new capture event for each captured packet. */
      if (!this->first_pcap_scheduled) {
        this->pcap_ev_id = nsock_pcap_read_packet(nsp, nsi_pcap, response_reception_handler_wrapper, -1, NULL);
        this->first_pcap_scheduled = true;
      }

      buf = myprobe->getPacketBuffer(&len);
      /* Send the packet*/
      assert(myprobe->host != NULL);
      if (send_ip_packet(this->rawsd, myprobe->getEthernet(), myprobe->host->getTargetAddress(), buf, len) == -1) {
        myprobe->setFailed();
        this->cc_report_final_timeout();
        myprobe->host->fail_one_probe();
        gh_perror("Unable to send packet in %s", __func__);
      }
      myprobe->setTimeSent();
      free(buf);
      break;

    default:
      fatal("Unexpected Nsock event in probe_transmission_handler()");
      break;
    } /* switch(type) */
  } else if (status == NSE_STATUS_EOF) {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): EOF\n");
  } else if (status == NSE_STATUS_ERROR || status == NSE_STATUS_PROXYERROR) {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): %s failed: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_TIMEOUT) {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): %s timeout: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_CANCELLED) {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): %s canceled: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_KILL) {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): %s killed: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else {
    if (o.debugging)
      log_write(LOG_PLAIN, "probe_transmission_handler(): Unknown status code %d\n", status);
  }
  return;
}


/* This is the handler for packet capture. It is called by nsock whenever libpcap
 * captures a packet from the network interface. This method basically captures
 * the packet, extracts its source IP address and tries to find an FPHost that
 * is targeting such address. If it does, it passes the packet to that FPHost
 * via callback() so the FPHost can determine if the packet is actually the
 * response to a FPProbe that it sent before. Note that this method is not
 * called directly by Nsock but by the wrapper function
 * response_reception_handler_wrapper(). See doc in probe_transmission_handler()
 * for details. */
void FPNetworkControl::response_reception_handler(nsock_pool nsp, nsock_event nse, void *arg) {
  nsock_iod nsi = nse_iod(nse);
  enum nse_status status = nse_status(nse);
  enum nse_type type = nse_type(nse);
  const u8 *rcvd_pkt = NULL;                    /* Points to the captured packet */
  size_t rcvd_pkt_len = 0;                      /* Length of the captured packet */
  struct timeval pcaptime;                    /* Time the packet was captured  */
  struct sockaddr_storage sent_ss;
  struct sockaddr_storage rcvd_ss;
  struct sockaddr_in *rcvd_ss4 = (struct sockaddr_in *)&rcvd_ss;
  struct sockaddr_in6 *rcvd_ss6 = (struct sockaddr_in6 *)&rcvd_ss;
  memset(&rcvd_ss, 0, sizeof(struct sockaddr_storage));
  IPv4Header ip4;
  IPv6Header ip6;
  int res = -1;

  struct timeval tv;
  gettimeofday(&tv, NULL);

  if (status == NSE_STATUS_SUCCESS) {
    switch(type) {

      case NSE_TYPE_PCAP_READ:

        /* Schedule a new pcap read operation */
        this->pcap_ev_id = nsock_pcap_read_packet(nsp, nsi, response_reception_handler_wrapper, -1, NULL);

        /* Get captured packet */
        nse_readpcap(nse, NULL, NULL, &rcvd_pkt, &rcvd_pkt_len, NULL, &pcaptime);

        /* Extract the packet's source address */
        ip4.storeRecvData(rcvd_pkt, rcvd_pkt_len);
        if (ip4.validate() != OP_FAILURE && ip4.getVersion() == 4) {
          ip4.getSourceAddress(&(rcvd_ss4->sin_addr));
          rcvd_ss4->sin_family = AF_INET;
        } else {
          ip6.storeRecvData(rcvd_pkt, rcvd_pkt_len);
          if (ip6.validate() != OP_FAILURE && ip6.getVersion() == 6) {
            ip6.getSourceAddress(&(rcvd_ss6->sin6_addr));
            rcvd_ss6->sin6_family = AF_INET6;
          } else {
            /* If we get here it means that the received packet is not
             * IPv4 or IPv6 so we just discard it returning. */
            return;
          }
        }

        /* Check if we have a caller that expects packets from this sender */
        for (size_t i = 0; i < this->callers.size(); i++) {

          /* Obtain the target address */
          sent_ss = *this->callers[i]->getTargetAddress();

          /* Check that the received packet is of the same address family */
          if (sent_ss.ss_family != rcvd_ss.ss_family)
            continue;

          /* Check that the captured packet's source address matches the
           * target address. If it matches, pass the received packet
           * to the appropriate FPHost object through callback().  */
          if (sockaddr_storage_equal(&rcvd_ss, &sent_ss)) {
            if ((res = this->callers[i]->callback(rcvd_pkt, rcvd_pkt_len, &tv)) >= 0) {

               /* If callback() returns >=0 it means that the packet we've just
                * passed was successfully matched with a previous probe. Now
                * update the count of received packets (so we can determine how
                * many outstanding packets are out there). Note that we only do
                * that if callback() returned >0 because 0 is a special case: a
                * reply to a retransmitted timed probe that was already replied
                * to in the past. We don't want to count replies to the same probe
                * more than once, so that's why we only update when res > 0. */
                if (res > 0)
                  this->cc_update_received();

               /* When the callback returns more than 1 it means that the packet
                * was sent more than once before being answered. This means that
                * we experienced congestion (first transmission got dropped), so
                * we update our CC parameters to deal with the congestion. */
                if (res > 1) {
                  this->cc_report_drop();
                }
            }
            return;
          }
        }
      break;

      default:
       fatal("Unexpected Nsock event in response_reception_handler()");
      break;

    } /* switch(type) */

  } else if (status == NSE_STATUS_EOF) {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): EOF\n");
  } else if (status == NSE_STATUS_ERROR || status == NSE_STATUS_PROXYERROR) {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): %s failed: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_TIMEOUT) {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): %s timeout: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_CANCELLED) {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): %s canceled: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else if (status == NSE_STATUS_KILL) {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): %s killed: %s\n", nse_type2str(type), strerror(socket_errno()));
  } else {
    if (o.debugging)
      log_write(LOG_PLAIN, "response_reception_handler(): Unknown status code %d\n", status);
  }
  return;
}


/******************************************************************************
 * Implementation of class FPEngine.                                          *
 ******************************************************************************/
FPEngine::FPEngine() {
  this->osgroup_size = OSSCAN_GROUP_SIZE;
}


FPEngine::~FPEngine() {

}


/* Returns a suitable BPF filter for the OS detection. If less than 20 targets
 * are passed, the filter contains an explicit list of target addresses. It
 * looks similar to this:
 *
 * dst host fe80::250:56ff:fec0:1 and (src host fe80::20c:29ff:feb0:2316 or src host fe80::20c:29ff:fe9f:5bc2)
 *
 * When more than 20 targets are passed, a generic filter based on the source
 * address is used. The returned filter looks something like:
 *
 * dst host fe80::250:56ff:fec0:1
 */
const char *FPEngine::bpf_filter(std::vector<Target *> &Targets) {
  static char pcap_filter[2048];
  /* 20 IPv6 addresses is max (46 byte addy + 14 (" or src host ")) * 20 == 1200 */
  char dst_hosts[1220];
  int filterlen = 0;
  int len = 0;
  unsigned int targetno;
  memset(pcap_filter, 0, sizeof(pcap_filter));

  /* If we have 20 or less targets, build a list of addresses so we can set
   * an explicit BPF filter */
  if (Targets.size() <= 20) {
    for (targetno = 0; targetno < Targets.size(); targetno++) {
      len = Snprintf(dst_hosts + filterlen,
                     sizeof(dst_hosts) - filterlen,
                     "%ssrc host %s", (targetno == 0)? "" : " or ",
                     Targets[targetno]->targetipstr());

      if (len < 0 || len + filterlen >= (int) sizeof(dst_hosts))
        fatal("ran out of space in dst_hosts");
      filterlen += len;
    }
    if (len < 0 || len + filterlen >= (int) sizeof(dst_hosts))
      fatal("ran out of space in dst_hosts");

    len = Snprintf(pcap_filter, sizeof(pcap_filter), "dst host %s and (%s)",
                   Targets[0]->sourceipstr(), dst_hosts);
  } else {
    len = Snprintf(pcap_filter, sizeof(pcap_filter), "dst host %s", Targets[0]->sourceipstr());
  }

  if (len < 0 || len >= (int) sizeof(pcap_filter))
    fatal("ran out of space in pcap filter");

  return pcap_filter;
}


/******************************************************************************
 * Implementation of class FPEngine6.                                         *
 ******************************************************************************/
FPEngine6::FPEngine6() {

}


FPEngine6::~FPEngine6() {

}

/* Not all operating systems allow setting the flow label in outgoing packets;
   notably all Unixes other than Linux when using raw sockets. This function
   finds out whether the flow labels we set are likely really being sent.
   Otherwise, the operating system is probably filling in 0. Compare to the
   logic in send_ipv6_packet_eth_or_sd. */
static bool can_set_flow_label(const struct eth_nfo *eth) {
  if (eth != NULL)
    return true;
#if HAVE_IPV6_IPPROTO_RAW
  return true;
#else
  return false;
#endif
}

void FPHost6::fill_FPR(FingerPrintResultsIPv6 *FPR) {
  unsigned int i;

  FPR->begin_time = this->begin_time;

  for (i = 0; i < sizeof(this->fp_responses) / sizeof(this->fp_responses[0]); i++) {
    const FPResponse *resp;

    resp = this->fp_responses[i];
    if (resp != NULL) {
      FPR->fp_responses[i] = new FPResponse(resp->probe_id, resp->buf, resp->len,
        resp->senttime, resp->rcvdtime);
    }
  }

  /* Were we actually able to set the flow label? */
  FPR->flow_label = 0;
  for (i = 0; i < sizeof(this->fp_probes) / sizeof(this->fp_probes[0]); i++) {
    const FPProbe& probe = fp_probes[0];
    if (probe.is_set()) {
      if (can_set_flow_label(probe.getEthernet()))
        FPR->flow_label = OSDETECT_FLOW_LABEL;
      break;
    }
  }

  /* Did we fail to send some probe? */
  FPR->incomplete = this->incomplete_fp;
}

static const IPv6Header *find_ipv6(const PacketElement *pe) {
  while (pe != NULL && pe->protocol_id() != HEADER_TYPE_IPv6)
    pe = pe->getNextElement();

  return (IPv6Header *) pe;
}

static const TCPHeader *find_tcp(const PacketElement *pe) {
  while (pe != NULL && pe->protocol_id() != HEADER_TYPE_TCP)
    pe = pe->getNextElement();

  return (TCPHeader *) pe;
}

static double vectorize_plen(const PacketElement *pe) {
  const IPv6Header *ipv6;

  ipv6 = find_ipv6(pe);
  if (ipv6 == NULL)
    return -1;
  else
    return ipv6->getPayloadLength();
}

static double vectorize_tc(const PacketElement *pe) {
  const IPv6Header *ipv6;

  ipv6 = find_ipv6(pe);
  if (ipv6 == NULL)
    return -1;
  else
    return ipv6->getTrafficClass();
}

/* For reference, the dev@nmap.org email thread which contains the explanations for the
 * design decisions of this vectorization method:
 * http://seclists.org/nmap-dev/2015/q1/218
 */
static int vectorize_hlim(const PacketElement *pe, int target_distance, enum dist_calc_method method) {
  const IPv6Header *ipv6;
  int hlim;
  int er_lim;

  ipv6 = find_ipv6(pe);
  if (ipv6 == NULL)
    return -1;
  hlim = ipv6->getHopLimit();

  if (method != DIST_METHOD_NONE) {
      if (method == DIST_METHOD_TRACEROUTE || method == DIST_METHOD_ICMP) {
        if (target_distance > 0)
          hlim += target_distance - 1;
      }
      er_lim = 5;
  } else
    er_lim = 20;

  if (32 - er_lim <= hlim && hlim <= 32+ 5 )
    hlim = 32;
  else if (64 - er_lim <= hlim && hlim <= 64+ 5 )
    hlim = 64;
  else if (128 - er_lim <= hlim && hlim <= 128+ 5 )
    hlim = 128;
  else if (255 - er_lim <= hlim && hlim <= 255+ 5 )
    hlim = 255;
  else
    hlim = -1;

  return hlim;
}

static double vectorize_isr(std::map<std::string, FPPacket>& resps) {
  const char * const SEQ_PROBE_NAMES[] = {"S1", "S2", "S3", "S4", "S5", "S6"};
  u32 seqs[NELEMS(SEQ_PROBE_NAMES)];
  struct timeval times[NELEMS(SEQ_PROBE_NAMES)];
  unsigned int i, j;
  double sum, t;

  j = 0;
  for (i = 0; i < NELEMS(SEQ_PROBE_NAMES); i++) {
    const char *probe_name;
    const FPPacket *fp;
    const TCPHeader *tcp;
    std::map<std::string, FPPacket>::iterator it;

    probe_name = SEQ_PROBE_NAMES[i];
    it = resps.find(probe_name);
    if (it == resps.end())
      continue;

    fp = &it->second;
    tcp = find_tcp(fp->getPacket());
    if (tcp == NULL)
      continue;

    seqs[j] = tcp->getSeq();
    times[j] = fp->getTime();
    j++;
  }

  if (j < 2)
    return -1;

  sum = 0.0;
  for (i = 0; i < j - 1; i++)
    sum += seqs[i + 1] - seqs[i];
  t = TIMEVAL_FSEC_SUBTRACT(times[j - 1], times[0]);

  return sum / t;
}

static struct feature_node *vectorize(const FingerPrintResultsIPv6 *FPR) {
  const char * const IPV6_PROBE_NAMES[] = {"S1", "S2", "S3", "S4", "S5", "S6", "IE1", "IE2", "NS", "U1", "TECN", "T2", "T3", "T4", "T5", "T6", "T7"};
  const char * const TCP_PROBE_NAMES[] = {"S1", "S2", "S3", "S4", "S5", "S6", "TECN", "T2", "T3", "T4", "T5", "T6", "T7"};
  unsigned int nr_feature, i, idx;
  struct feature_node *features;
  std::map<std::string, FPPacket> resps;

  for (i = 0; i < NUM_FP_PROBES_IPv6; i++) {
    PacketElement *pe;

    if (FPR->fp_responses[i] == NULL)
      continue;
    pe = PacketParser::split(FPR->fp_responses[i]->buf, FPR->fp_responses[i]->len);
    assert(pe != NULL);
    resps[FPR->fp_responses[i]->probe_id].setPacket(pe);
    resps[FPR->fp_responses[i]->probe_id].setTime(&FPR->fp_responses[i]->senttime);
  }

  nr_feature = get_nr_feature(&FPModel);
  features = new feature_node[nr_feature + 1];
  for (i = 0; i < nr_feature; i++) {
    features[i].index = i + 1;
    features[i].value = -1;
  }
  features[i].index = -1;

  idx = 0;
  for (i = 0; i < NELEMS(IPV6_PROBE_NAMES); i++) {
    const char *probe_name;

    probe_name = IPV6_PROBE_NAMES[i];
    features[idx++].value = vectorize_plen(resps[probe_name].getPacket());
    features[idx++].value = vectorize_tc(resps[probe_name].getPacket());
    features[idx++].value = vectorize_hlim(resps[probe_name].getPacket(), FPR->distance, FPR->distance_calculation_method);
  }
  /* TCP features */
  features[idx++].value = vectorize_isr(resps);
  for (i = 0; i < NELEMS(TCP_PROBE_NAMES); i++) {
    const char *probe_name;
    const TCPHeader *tcp;
    u16 flags;
    u16 mask;
    unsigned int j;
    int mss;
    int sackok;
    int wscale;

    probe_name = TCP_PROBE_NAMES[i];

    mss = -1;
    sackok = -1;
    wscale = -1;

    tcp = find_tcp(resps[probe_name].getPacket());
    if (tcp == NULL) {
      /* 48 TCP features. */
      idx += 48;
      continue;
    }
    features[idx++].value = tcp->getWindow();
    flags = tcp->getFlags16();
    for (mask = 0x001; mask <= 0x800; mask <<= 1)
      features[idx++].value = (flags & mask) != 0;

    for (j = 0; j < 16; j++) {
      nping_tcp_opt_t opt;
      opt = tcp->getOption(j);
      if (opt.value == NULL)
        break;
      features[idx++].value = opt.type;
      /* opt.len includes the two (type, len) bytes. */
      if (opt.type == TCPOPT_MSS && opt.len == 4 && mss == -1)
        mss = ntohs(*(u16 *) opt.value);
      else if (opt.type == TCPOPT_SACKOK && opt.len == 2 && sackok == -1)
        sackok = 1;
      else if (opt.type == TCPOPT_WSCALE && opt.len == 3 && wscale == -1)
        wscale = *(u8 *) opt.value;
    }
    for (; j < 16; j++)
      idx++;

    for (j = 0; j < 16; j++) {
      nping_tcp_opt_t opt;
      opt = tcp->getOption(j);
      if (opt.value == NULL)
        break;
      features[idx++].value = opt.len;
    }
    for (; j < 16; j++)
      idx++;

    features[idx++].value = mss;
    features[idx++].value = sackok;
    features[idx++].value = wscale;
  }
  assert(idx == nr_feature);

  if (o.debugging > 2) {
    log_write(LOG_PLAIN, "v = {");
    for (i = 0; i < nr_feature; i++)
      log_write(LOG_PLAIN, "%.16g, ", features[i].value);
    log_write(LOG_PLAIN, "};\n");
  }

  return features;
}

static void apply_scale(struct feature_node *features, unsigned int num_features,
  const double (*scale)[2]) {
  unsigned int i;

  for (i = 0; i < num_features; i++) {
    double val = features[i].value;
    if (val < 0)
      continue;
    val = (val + scale[i][0]) * scale[i][1];
    features[i].value = val;
  }
}

/* (label, prob) pairs for purpose of sorting. */
struct label_prob {
  int label;
  double prob;
};

int label_prob_cmp(const void *a, const void *b) {
  const struct label_prob *la, *lb;

  la = (struct label_prob *) a;
  lb = (struct label_prob *) b;

  /* Sort descending. */
  if (la->prob > lb->prob)
    return -1;
  else if (la->prob < lb->prob)
    return 1;
  else
    return 0;
}

/* Return a measure of how much the given feature vector differs from the other
   members of the class given by label.

   This can be thought of as the distance from the given feature vector to the
   mean of the class in multidimensional space, after scaling. Each dimension is
   further scaled by the inverse of the sample variance of that feature. This is
   an approximation of the Mahalanobis distance
   (https://en.wikipedia.org/wiki/Mahalanobis_distance), which normally uses a
   full covariance matrix of the features. If we take the features to be
   pairwise independent (which they are not), then the covariance matrix is just
   the diagonal matrix containing per-feature variances, leading to the same
   calculation as is done below. Using only the per-feature variances rather
   than covariance matrices is to save space; it requires only n entries per
   class rather than n^2, where n is the length of a feature vector.

   It happens often that a feature's variance is undefined (because there is
   only one example in the class) or zero (because there are two identical
   values for that feature). Both these cases are mapped to zero by train.py,
   and we handle them the same way: by using a small default variance. This will
   tend to make small differences count a lot (because we probably want this
   fingerprint in order to expand the class), while still allowing near-perfect
   matches to match. */
static double novelty_of(const struct feature_node *features, int label) {
  const double *means, *variances;
  int i, nr_feature;
  double sum;

  nr_feature = get_nr_feature(&FPModel);
  assert(0 <= label);
  assert(label < nr_feature);

  means = FPmean[label];
  variances = FPvariance[label];

  sum = 0.0;
  for (i = 0; i < nr_feature; i++) {
    double d, v;

    assert(i + 1 == features[i].index);
    d = features[i].value - means[i];
    v = variances[i];
    if (v == 0.0) {
      /* No variance? It means that samples were identical. Substitute a default
         variance. This will tend to make novelty large in these cases, which
         will hopefully encourage for submissions for this class. */
      v = 0.01;
    }
    sum += d * d / v;
  }

  return sqrt(sum);
}

static void classify(FingerPrintResultsIPv6 *FPR) {
  int nr_class, i;
  struct feature_node *features;
  double *values;
  struct label_prob *labels;

  nr_class = get_nr_class(&FPModel);

  features = vectorize(FPR);
  values = new double[nr_class];
  labels = new struct label_prob[nr_class];

  apply_scale(features, get_nr_feature(&FPModel), FPscale);

  predict_values(&FPModel, features, values);
  for (i = 0; i < nr_class; i++) {
    labels[i].label = i;
    labels[i].prob = 1.0 / (1.0 + exp(-values[i]));
  }
  qsort(labels, nr_class, sizeof(labels[0]), label_prob_cmp);
  for (i = 0; i < nr_class && i < MAX_FP_RESULTS; i++) {
    FPR->matches[i] = &o.os_labels_ipv6[labels[i].label];
    FPR->accuracy[i] = labels[i].prob;
    FPR->num_matches = i + 1;
    if (labels[i].prob >= 0.90 * labels[0].prob)
      FPR->num_perfect_matches = i + 1;
    if (o.debugging > 2) {
      printf("%7.4f %7.4f %3u %s\n", FPR->accuracy[i] * 100,
        novelty_of(features, labels[i].label), labels[i].label, FPR->matches[i]->OS_name);
    }
  }
  if (FPR->num_perfect_matches == 0) {
    FPR->overall_results = OSSCAN_NOMATCHES;
  } else if (FPR->num_perfect_matches == 1) {
    double novelty;

    novelty = novelty_of(features, labels[0].label);
    if (o.debugging > 1)
      log_write(LOG_PLAIN, "Novelty of closest match is %.3f.\n", novelty);

    if (novelty < FP_NOVELTY_THRESHOLD) {
      FPR->overall_results = OSSCAN_SUCCESS;
    } else {
      if (o.debugging > 0) {
        log_write(LOG_PLAIN, "Novelty of closest match is %.3f > %.3f; ignoring.\n",
          novelty, FP_NOVELTY_THRESHOLD);
      }
      FPR->overall_results = OSSCAN_NOMATCHES;
      FPR->num_perfect_matches = 0;
    }
  } else {
    FPR->overall_results = OSSCAN_NOMATCHES;
    FPR->num_perfect_matches = 0;
  }

  delete[] features;
  delete[] values;
  delete[] labels;
}


/* This method is the core of the FPEngine class. It takes a list of IPv6
 * targets that need to be fingerprinted. The method handles the whole
 * fingerprinting process, sending probes, collecting responses, analyzing
 * results and matching fingerprints. If everything goes well, the internal
 * state of the supplied target objects will be modified to reflect the results
 * of the */
int FPEngine6::os_scan(std::vector<Target *> &Targets) {
  bool osscan_done = false;
  const char *bpf_filter = NULL;
  std::vector<FPHost6 *> curr_hosts;  /* Hosts currently doing OS detection      */
  std::vector<FPHost6 *> done_hosts;  /* Hosts for which we already did OSdetect */
  std::vector<FPHost6 *> left_hosts;  /* Hosts we have not yet started with      */
  struct timeval begin_time;

  if (o.debugging)
    log_write(LOG_PLAIN, "Starting IPv6 OS Scan...\n");

  /* Initialize variables, timers, etc. */
  gettimeofday(&begin_time, NULL);
  global_netctl.init(Targets[0]->deviceName(), Targets[0]->ifType());
  for (size_t i = 0; i < Targets.size(); i++) {
    if (o.debugging > 3) {
      log_write(LOG_PLAIN, "[FPEngine] Allocating FPHost6 for %s %s\n",
        Targets[i]->targetipstr(), Targets[i]->sourceipstr());
    }
    FPHost6 *newhost = new FPHost6(Targets[i], &global_netctl);
    newhost->begin_time = begin_time;
    fphosts.push_back(newhost);
  }

  /* Build the BPF filter */
  bpf_filter = this->bpf_filter(Targets);
  if (o.debugging)
    log_write(LOG_PLAIN, "[FPEngine] Interface=%s BPF:%s\n", Targets[0]->deviceName(), bpf_filter);

  /* Set up the sniffer */
  global_netctl.setup_sniffer(Targets[0]->deviceName(), bpf_filter);

  /* Divide the targets into two groups, the ones we are going to start
   * processing, and the ones we leave for later. */
  for (size_t i = 0; i < Targets.size() && i < this->osgroup_size; i++) {
    curr_hosts.push_back(fphosts[i]);
  }
  for (size_t i = curr_hosts.size(); i < Targets.size(); i++) {
    left_hosts.push_back(fphosts[i]);
  }

  /* Do the OS detection rounds */
  while (!osscan_done) {
    osscan_done = true; /* It will remain true only when all hosts are .done() */
    if (o.debugging > 3) {
      log_write(LOG_PLAIN, "[FPEngine] CurrHosts=%d, LeftHosts=%d, DoneHosts=%d\n",
        (int) curr_hosts.size(), (int) left_hosts.size(), (int) done_hosts.size());
    }

    /* Go through the list of hosts and ask them to schedule their probes */
    for (unsigned int i = 0; i < curr_hosts.size(); i++) {

      /* If the host is not done yet, call schedule() to let it schedule
       * new probes, retransmissions, etc. */
      if (!curr_hosts[i]->done()) {
        osscan_done = false;
        curr_hosts[i]->schedule();
        if (o.debugging > 3)
          log_write(LOG_PLAIN, "[FPEngine] CurrHost #%u not done\n", i);

      /* If the host is done, take it out of the curr_hosts group and add it
       * to the done_hosts group. If we still have hosts left in the left_hosts
       * group, take the first one and insert it into curr_hosts. This way we
       * always have a full working group of hosts (unless we ran out of hosts,
       * of course). */
      } else {
        if (o.debugging > 3)
          log_write(LOG_PLAIN, "[FPEngine] CurrHost #%u done\n", i);
        if (o.debugging > 3)
          log_write(LOG_PLAIN, "[FPEngine] Moving done host %u to the done_hosts list\n", i);
        done_hosts.push_back(curr_hosts[i]);
        curr_hosts.erase(curr_hosts.begin() + i);

        /* If we still have hosts left, add one to the current group */
        if (left_hosts.size() > 0) {
          if (o.debugging > 3)
            log_write(LOG_PLAIN, "[FPEngine] Inserting one new hosts in the curr_hosts list.\n");
          curr_hosts.push_back(left_hosts[0]);
          left_hosts.erase(left_hosts.begin());
          osscan_done = false;
        }

        i--; /* Decrement i so we don't miss the host that is now in the
              * position of the host we've just removed from the list */
      }
    }

    /* Handle scheduled events */
    global_netctl.handle_events();

  }

  /* Once we've finished with all fphosts, check which ones were correctly
   * fingerprinted, and update the Target objects. */
  for (size_t i = 0; i < this->fphosts.size(); i++) {
    fphosts[i]->finish();

    fphosts[i]->fill_FPR((FingerPrintResultsIPv6 *) Targets[i]->FPR);
    classify((FingerPrintResultsIPv6 *) Targets[i]->FPR);
  }

  /* Cleanup and return */
  while (this->fphosts.size() > 0) {
    FPHost6 *tmp = fphosts.back();
    delete tmp;
    fphosts.pop_back();
  }

  if (o.debugging)
    log_write(LOG_PLAIN, "IPv6 OS Scan completed.\n");
  return OP_SUCCESS;
}


/******************************************************************************
 * Implementation of class FPHost.                                            *
 ******************************************************************************/
FPHost::FPHost() {
  this->__reset();
}


FPHost::~FPHost() {

}


void FPHost::__reset() {
  this->total_probes = 0;
  this->timed_probes = 0;
  this->probes_sent = 0;
  this->probes_answered = 0;
  this->probes_unanswered = 0;
  this->incomplete_fp = false;
  this->detection_done = false;
  this->timedprobes_sent = false;
  this->target_host = NULL;
  this->netctl = NULL;
  this->netctl_registered = false;
  this->tcpSeqBase = 0;
  this->open_port_tcp = -1;
  this->closed_port_tcp = -1;
  this->closed_port_udp = -1;
  this->tcp_port_base = -1;
  this->udp_port_base = -1;
  /* Retransmission time-out parameters.
   *
   * From RFC 2988:
   * Until a round-trip time (RTT) measurement has been made for a segment
   * sent b…

Large files files are truncated, but you can click here to view the full file