/src/plugins/select/bluegene/ba/block_allocator.c

Large files files are truncated, but you can click here to view the full file

/*****************************************************************************\
 *  block_allocator.c - Assorted functions for layout of bluegene blocks,
 *	 wiring, mapping for smap, etc.
 *  $Id$
 *****************************************************************************
 *  Copyright (C) 2004-2007 The Regents of the University of California.
 *  Copyright (C) 2008-2009 Lawrence Livermore National Security.
 *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
 *  Written by Dan Phung <phung4@llnl.gov>, Danny Auble <da@llnl.gov>
 *
 *  This file is part of SLURM, a resource management program.
 *  For details, see <http://www.schedmd.com/slurmdocs/>.
 *  Please also read the included file: DISCLAIMER.
 *
 *  SLURM is free software; you can redistribute it and/or modify it under
 *  the terms of the GNU General Public License as published by the Free
 *  Software Foundation; either version 2 of the License, or (at your option)
 *  any later version.
 *
 *  In addition, as a special exception, the copyright holders give permission
 *  to link the code of portions of this program with the OpenSSL library under
 *  certain conditions as described in each individual source file, and
 *  distribute linked combinations including the two. You must obey the GNU
 *  General Public License in all respects for all of the code used other than
 *  OpenSSL. If you modify file(s) with this exception, you may extend this
 *  exception to your version of the file(s), but you are not obligated to do
 *  so. If you do not wish to do so, delete this exception statement from your
 *  version.  If you delete this exception statement from all source files in
 *  the program, then also delete it here.
 *
 *  SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
 *  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 *  FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
 *  details.
 *
 *  You should have received a copy of the GNU General Public License along
 *  with SLURM; if not, write to the Free Software Foundation, Inc.,
 *  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
\*****************************************************************************/

#if HAVE_CONFIG_H
#  include "config.h"
#endif

#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include "block_allocator.h"
#include "src/common/slurmdb_defs.h"
#include "src/common/timers.h"
#include "src/common/uid.h"

/*
 * structure that holds switch path information for finding the wiring
 * path without setting the configuration.
 *
 * - dim      - Which Axis it is on
 * - geometry - node location
 * - in       - ingress port.
 * - out      - egress port.
 *
 */
typedef struct {
	int dim;
	uint16_t geometry[HIGHEST_DIMENSIONS];
	int in;
	int out;
} ba_path_switch_t;


#define DEBUG_PA
#define BEST_COUNT_INIT 20

/* Global */
bool _initialized = false;
bool _wires_initialized = false;
bool _mp_map_initialized = false;

/* _ba_system is the "current" system that the structures will work
 *  on */
List path = NULL;
List best_path = NULL;
int best_count;
uint16_t *deny_pass = NULL;
char *p = '\0';

/* extern Global */
my_bluegene_t *bg = NULL;
ba_mp_t ***ba_main_grid = NULL;

typedef enum {
	BLOCK_ALGO_FIRST,
	BLOCK_ALGO_SECOND
} block_algo_t;

/** internal helper functions */

/* */
static int _check_for_options(select_ba_request_t* ba_request);

/* */
static int _append_geo(uint16_t *geo, List geos, int rotate);

/* */
static int _fill_in_coords(List results, List start_list,
			   uint16_t *geometry, int conn_type);

/* */
static int _copy_the_path(List nodes, ba_switch_t *curr_switch,
			  ba_switch_t *mark_switch,
			  int source, int dim);

/* */
static int _find_yz_path(ba_mp_t *ba_node, uint16_t *first,
			 uint16_t *geometry, int conn_type);

#ifndef HAVE_BG_FILES
/* */
static int _emulate_ext_wiring(ba_mp_t ***grid);
#endif

/** */
static int _reset_the_path(ba_switch_t *curr_switch, int source,
			   int target, int dim);
/* */
static void _delete_path_list(void *object);

/* find the first block match in the system */
static int _find_match(select_ba_request_t* ba_request, List results);

/** */
static bool _node_used(ba_mp_t* ba_node, int x_size);

/* */
static void _switch_config(ba_mp_t* source, ba_mp_t* target, int dim,
			   int port_src, int port_tar);

/* */
static int _set_external_wires(int dim, int count, ba_mp_t* source,
			       ba_mp_t* target);

/* */
static char *_set_internal_wires(List nodes, int size, int conn_type);

/* */
static int _find_x_path(List results, ba_mp_t *ba_node, uint16_t *start,
			int x_size, int found, int conn_type,
			block_algo_t algo);

/* */
static int _remove_node(List results, uint16_t *mp_tar);

/* */
static int _find_next_free_using_port_2(ba_switch_t *curr_switch,
					int source_port,
					List nodes, int dim,
					int count);
/* */
/* static int _find_passthrough(ba_switch_t *curr_switch, int source_port,  */
/* 			     List nodes, int dim,  */
/* 			     int count, int highest_phys_x);  */
/* */
static int _finish_torus(List results,
			 ba_switch_t *curr_switch, int source_port,
			 int dim, int count, uint16_t *start);
/* */
static uint16_t *_set_best_path();

/* */
static int _set_one_dim(uint16_t *start, uint16_t *end, uint16_t *coord);

/* */
static void _destroy_geo(void *object);

extern void destroy_ba_node(void *ptr)
{
	ba_mp_t *ba_node = (ba_mp_t *)ptr;
	if (ba_node) {
		xfree(ba_node->loc);
		xfree(ba_node);
	}
}

/*
 * create a block request.  Note that if the geometry is given,
 * then size is ignored.  If elongate is true, the algorithm will try
 * to fit that a block of cubic shape and then it will try other
 * elongated geometries.  (ie, 2x2x2 -> 4x2x1 -> 8x1x1).
 *
 * IN/OUT - ba_request: structure to allocate and fill in.
 *
 * ALL below IN's need to be set within the ba_request before the call
 * if you want them to be used.
 * ALL below OUT's are set and returned within the ba_request.
 * IN - avail_node_bitmap: bitmap of usable midplanes.
 * IN - blrtsimage: BlrtsImage for this block if not default
 * IN - conn_type: connection type of request (TORUS or MESH or SMALL)
 * IN - elongate: if true, will try to fit different geometries of
 *      same size requests
 * IN/OUT - geometry: requested/returned geometry of block
 * IN - linuximage: LinuxImage for this block if not default
 * IN - mloaderimage: MLoaderImage for this block if not default
 * IN - nodecards: Number of nodecards in each block in request only
 *      used of small block allocations.
 * OUT - passthroughs: if there were passthroughs used in the
 *       generation of the block.
 * IN - procs: Number of real processors requested
 * IN - quarters: Number of midplane quarters in each block in request only
 *      used of small block allocations.
 * IN - RamDiskimage: RamDiskImage for this block if not default
 * IN - rotate: if true, allows rotation of block during fit
 * OUT - save_name: hostlist of midplanes used in block
 * IN/OUT - size: requested/returned count of midplanes in block
 * IN - start: geo location of where to start the allocation
 * IN - start_req: if set use the start variable to start at
 * return success of allocation/validation of params
 */
extern int new_ba_request(select_ba_request_t* ba_request)
{
	int i=0;
	float sz=1;
	int i2, picked, total_sz=1, size2=0;
	uint16_t *geo_ptr;
	int messed_with = 0;
	int checked[DIM_SIZE[X]];
	uint16_t geo[cluster_dims];

	memset(geo, 0, sizeof(geo));
	ba_request->save_name= NULL;
	ba_request->rotate_count= 0;
	ba_request->elongate_count = 0;
	ba_request->elongate_geos = list_create(_destroy_geo);
	memcpy(geo, ba_request->geometry, sizeof(geo));

	if (ba_request->deny_pass == (uint16_t)NO_VAL)
		ba_request->deny_pass = ba_deny_pass;

	if (!(cluster_flags & CLUSTER_FLAG_BG)) {
		if (geo[X] != (uint16_t)NO_VAL) {
			for (i=0; i<cluster_dims; i++) {
				if ((geo[i] < 1) || (geo[i] > DIM_SIZE[i])) {
					error("new_ba_request Error, "
					      "request geometry is invalid %d",
					      geo[i]);
					return 0;
				}
			}
			ba_request->size = ba_request->geometry[X];
		} else if (ba_request->size) {
			ba_request->geometry[X] = ba_request->size;
		} else
			return 0;
		return 1;
	}

	if (geo[X] != (uint16_t)NO_VAL) {
		for (i=0; i<cluster_dims; i++){
			if ((geo[i] < 1) || (geo[i] > DIM_SIZE[i])) {
				error("new_ba_request Error, "
				      "request geometry is invalid dim %d "
				      "can't be %c, largest is %c",
				      i,
				      alpha_num[geo[i]],
				      alpha_num[DIM_SIZE[i]]);
				return 0;
			}
		}
		_append_geo(geo, ba_request->elongate_geos, 0);
		sz=1;
		for (i=0; i<cluster_dims; i++)
			sz *= ba_request->geometry[i];
		ba_request->size = sz;
		sz=0;
	}

	deny_pass = &ba_request->deny_pass;

	if (ba_request->elongate || sz) {
		sz=1;
		/* decompose the size into a cubic geometry */
		ba_request->rotate= 1;
		ba_request->elongate = 1;

		for (i=0; i<cluster_dims; i++) {
			total_sz *= DIM_SIZE[i];
			geo[i] = 1;
		}

		if (ba_request->size==1) {
			_append_geo(geo,
				    ba_request->elongate_geos,
				    ba_request->rotate);
			goto endit;
		}

		if (ba_request->size<=DIM_SIZE[Y]) {
			geo[X] = 1;
			geo[Y] = ba_request->size;
			geo[Z] = 1;
			sz=ba_request->size;
			_append_geo(geo,
				    ba_request->elongate_geos,
				    ba_request->rotate);
		}

		i = ba_request->size/4;
		if (!(ba_request->size%2)
		    && i <= DIM_SIZE[Y]
		    && i <= DIM_SIZE[Z]
		    && i*i == ba_request->size) {
			geo[X] = 1;
			geo[Y] = i;
			geo[Z] = i;
			sz=ba_request->size;
			_append_geo(geo,
				    ba_request->elongate_geos,
				    ba_request->rotate);
		}

		if (ba_request->size > total_sz || ba_request->size < 1) {
			return 0;
		}
		sz = ba_request->size % (DIM_SIZE[Y] * DIM_SIZE[Z]);
		if (!sz) {
			i = ba_request->size / (DIM_SIZE[Y] * DIM_SIZE[Z]);
			geo[X] = i;
			geo[Y] = DIM_SIZE[Y];
			geo[Z] = DIM_SIZE[Z];
			sz=ba_request->size;
			if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
				_append_geo(geo,
					    ba_request->elongate_geos,
					    ba_request->rotate);
			else
				error("%d I was just trying to add a "
				      "geo of %d%d%d "
				      "while I am trying to request "
				      "%d midplanes",
				      __LINE__, geo[X], geo[Y], geo[Z],
				      ba_request->size);
		}
//	startagain:
		picked=0;
		for(i=0; i<DIM_SIZE[X]; i++)
			checked[i]=0;

		for (i=0; i<cluster_dims; i++) {
			total_sz *= DIM_SIZE[i];
			geo[i] = 1;
		}

		sz = 1;
		picked=0;
	tryagain:
		size2 = ba_request->size;
		//messedup:
		for (i=picked; i<cluster_dims; i++) {
			if (size2 <= 1)
				break;

			sz = size2 % DIM_SIZE[i];
			if (!sz) {
				geo[i] = DIM_SIZE[i];
				size2 /= DIM_SIZE[i];
			} else if (size2 > DIM_SIZE[i]) {
				for(i2=(DIM_SIZE[i]-1); i2 > 1; i2--) {
					/* go through each number to see if
					   the size is divisable by a smaller
					   number that is
					   good in the other dims. */
					if (!(size2%i2) && !checked[i2]) {
						size2 /= i2;

						if (i==0)
							checked[i2]=1;

						if (i2<DIM_SIZE[i]) {
							geo[i] = i2;
						} else {
							goto tryagain;
						}
						if ((i2-1)!=1 &&
						    i!=(cluster_dims-1))
							break;
					}
				}
				/* This size can not be made into a
				   block return.  If you want to try
				   until we find the next largest block
				   uncomment the code below and the goto
				   above. If a user specifies a max
				   node count the job will never
				   run.
				*/
				if (i2==1) {
					if (!list_count(
						    ba_request->elongate_geos))
						error("Can't make a block of "
						      "%d into a cube.",
						      ba_request->size);
					goto endit;
/* 					ba_request->size +=1; */
/* 					goto startagain; */
				}
			} else {
				geo[i] = sz;
				break;
			}
		}

		if ((geo[X]*geo[Y]) <= DIM_SIZE[Y]) {
			ba_request->geometry[X] = 1;
			ba_request->geometry[Y] = geo[X] * geo[Y];
			ba_request->geometry[Z] = geo[Z];
			_append_geo(ba_request->geometry,
				    ba_request->elongate_geos,
				    ba_request->rotate);

		}
		if ((geo[X]*geo[Z]) <= DIM_SIZE[Y]) {
			ba_request->geometry[X] = 1;
			ba_request->geometry[Y] = geo[Y];
			ba_request->geometry[Z] = geo[X] * geo[Z];
			_append_geo(ba_request->geometry,
				    ba_request->elongate_geos,
				    ba_request->rotate);

		}

		/* Make sure geo[X] is even and then see if we can get
		   it into the Y or Z dim. */
		if (!(geo[X]%2) && ((geo[X]/2) <= DIM_SIZE[Y])) {
			if (geo[Y] == 1) {
				ba_request->geometry[Y] = geo[X]/2;
				messed_with = 1;
			} else
				ba_request->geometry[Y] = geo[Y];
			if (!messed_with && geo[Z] == 1) {
				messed_with = 1;
				ba_request->geometry[Z] = geo[X]/2;
			} else
				ba_request->geometry[Z] = geo[Z];
			if (messed_with) {
				messed_with = 0;
				ba_request->geometry[X] = 2;
				_append_geo(ba_request->geometry,
					    ba_request->elongate_geos,
					    ba_request->rotate);
			}
		}
		if (geo[X] == DIM_SIZE[X]
		    && (geo[Y] < DIM_SIZE[Y]
			|| geo[Z] < DIM_SIZE[Z])) {
			if (DIM_SIZE[Y]<DIM_SIZE[Z]) {
				i = DIM_SIZE[Y];
				DIM_SIZE[Y] = DIM_SIZE[Z];
				DIM_SIZE[Z] = i;
			}
			ba_request->geometry[X] = geo[X];
			ba_request->geometry[Y] = geo[Y];
			ba_request->geometry[Z] = geo[Z];
			if (ba_request->geometry[Y] < DIM_SIZE[Y]) {
				i = (DIM_SIZE[Y] - ba_request->geometry[Y]);
				ba_request->geometry[Y] +=i;
			}
			if (ba_request->geometry[Z] < DIM_SIZE[Z]) {
				i = (DIM_SIZE[Z] - ba_request->geometry[Z]);
				ba_request->geometry[Z] +=i;
			}
			for(i = DIM_SIZE[X]; i>0; i--) {
				ba_request->geometry[X]--;
				i2 = (ba_request->geometry[X]
				      * ba_request->geometry[Y]
				      * ba_request->geometry[Z]);
				if (i2 < ba_request->size) {
					ba_request->geometry[X]++;
					messed_with = 1;
					break;
				}
			}
			if (messed_with) {
				messed_with = 0;
				_append_geo(ba_request->geometry,
					    ba_request->elongate_geos,
					    ba_request->rotate);
			}
		}

		if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
			_append_geo(geo,
				    ba_request->elongate_geos,
				    ba_request->rotate);
		else
			error("%d I was just trying to add a geo of %d%d%d "
			      "while I am trying to request %d midplanes",
			      __LINE__, geo[X], geo[Y], geo[Z],
			      ba_request->size);

/* Having the functions pow and powf on an aix system doesn't seem to
 * link well, so since this is only for aix and this doesn't really
 * need to be there just don't allow this extra calculation.
 */
#ifndef HAVE_AIX
		/* see if We can find a cube or square root of the
		   size to make an easy cube */
		for(i=0; i<cluster_dims-1; i++) {
			sz = powf((float)ba_request->size,
				  (float)1/(cluster_dims-i));
			if (pow(sz,(cluster_dims-i)) == ba_request->size)
				break;
		}

		if (i < (cluster_dims-1)) {
			/* we found something that looks like a cube! */
			int i3 = i;

			for (i=0; i<i3; i++)
				geo[i] = 1;

			for (i=i3; i<cluster_dims; i++)
				if (sz<=DIM_SIZE[i])
					geo[i] = sz;
				else
					goto endit;

			if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
				_append_geo(geo,
					    ba_request->elongate_geos,
					    ba_request->rotate);
			else
				error("%d I was just trying to add "
				      "a geo of %d%d%d "
				      "while I am trying to request "
				      "%d midplanes",
				      __LINE__, geo[X], geo[Y], geo[Z],
				      ba_request->size);
		}
#endif //HAVE_AIX
	}

endit:
	if (!(geo_ptr = list_peek(ba_request->elongate_geos)))
		return 0;

	ba_request->elongate_count++;
	ba_request->geometry[X] = geo_ptr[X];
	ba_request->geometry[Y] = geo_ptr[Y];
	ba_request->geometry[Z] = geo_ptr[Z];
	sz=1;
	for (i=0; i<cluster_dims; i++)
		sz *= ba_request->geometry[i];
	ba_request->size = sz;

	return 1;
}

/**
 * print a block request
 */
extern void print_ba_request(select_ba_request_t* ba_request)
{
	int i;

	if (ba_request == NULL){
		error("print_ba_request Error, request is NULL");
		return;
	}
	debug("  ba_request:");
	debug("    geometry:\t");
	for (i=0; i<cluster_dims; i++){
		debug("%d", ba_request->geometry[i]);
	}
	debug("        size:\t%d", ba_request->size);
	debug("   conn_type:\t%d", ba_request->conn_type[X]);
	debug("      rotate:\t%d", ba_request->rotate);
	debug("    elongate:\t%d", ba_request->elongate);
}

/* If emulating a system set up a known configuration for wires in a
 * system of the size given.
 * If a real bluegene system, query the system and get all wiring
 * information of the system.
 */
extern void init_wires(void)
{
	int x, y, z, i;
	ba_mp_t *source = NULL;
	if (_wires_initialized)
		return;

	for(x=0;x<DIM_SIZE[X];x++) {
		for(y=0;y<DIM_SIZE[Y];y++) {
			for(z=0;z<DIM_SIZE[Z];z++) {
				source = &ba_main_grid[x][y][z];
				for(i=0; i<NUM_PORTS_PER_NODE; i++) {
					_switch_config(source, source,
						       X, i, i);
					_switch_config(source, source,
						       Y, i, i);
					_switch_config(source, source,
						       Z, i, i);
				}
			}
		}
	}
#ifdef HAVE_BG_FILES
	_set_external_wires(0,0,NULL,NULL);
	if (bridge_setup_system() == -1)
		return;
#endif

	_wires_initialized = true;
	return;
}

/*
 * copy the path of the nodes given
 *
 * IN nodes List of ba_mp_t *'s: nodes to be copied
 * OUT dest_nodes List of ba_mp_t *'s: filled in list of nodes
 * wiring.
 * Return on success SLURM_SUCCESS, on error SLURM_ERROR
 */
extern int copy_node_path(List nodes, List *dest_nodes)
{
	int rc = SLURM_ERROR;

#ifdef HAVE_BG_L_P
	ListIterator itr = NULL;
	ListIterator itr2 = NULL;
	ba_mp_t *ba_node = NULL, *new_ba_node = NULL;
	int dim;
	ba_switch_t *curr_switch = NULL, *new_switch = NULL;

	if (!nodes)
		return SLURM_ERROR;
	if (!*dest_nodes)
		*dest_nodes = list_create(destroy_ba_node);

	itr = list_iterator_create(nodes);
	while ((ba_node = list_next(itr))) {
		itr2 = list_iterator_create(*dest_nodes);
		while ((new_ba_node = list_next(itr2))) {
			if (ba_node->coord[X] == new_ba_node->coord[X] &&
			    ba_node->coord[Y] == new_ba_node->coord[Y] &&
			    ba_node->coord[Z] == new_ba_node->coord[Z])
				break;	/* we found it */
		}
		list_iterator_destroy(itr2);

		if (!new_ba_node) {
			if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
				info("adding %c%c%c as a new node",
				     alpha_num[ba_node->coord[X]],
				     alpha_num[ba_node->coord[Y]],
				     alpha_num[ba_node->coord[Z]]);
			new_ba_node = ba_copy_mp(ba_node);
			ba_setup_mp(new_ba_node, false, false);
			list_push(*dest_nodes, new_ba_node);

		}
		new_ba_node->used = true;
		for(dim=0;dim<cluster_dims;dim++) {
			curr_switch = &ba_node->axis_switch[dim];
			new_switch = &new_ba_node->axis_switch[dim];
			if (curr_switch->int_wire[0].used) {
				if (!_copy_the_path(*dest_nodes,
						    curr_switch, new_switch,
						    0, dim)) {
					rc = SLURM_ERROR;
					break;
				}
			}
		}

	}
	list_iterator_destroy(itr);
	rc = SLURM_SUCCESS;
#endif
	return rc;
}

extern ba_mp_t *coord2ba_mp(const uint16_t *coord)
{
	if ((coord[X] >= DIM_SIZE[X]) || (coord[Y] >= DIM_SIZE[Y]) ||
	    (coord[Z] >= DIM_SIZE[Z])) {
		error("Invalid coordinate %d:%d:%d",
		      coord[X], coord[Y], coord[Z]);
		return NULL;
	}
	return &ba_main_grid[coord[X]][coord[Y]][coord[Z]];
}

/*
 * Try to allocate a block.
 *
 * IN - ba_request: allocation request
 * OUT - results: List of results of the allocation request.  Each
 * list entry will be a coordinate.  allocate_block will create the
 * list, but the caller must destroy it.
 *
 * return: success or error of request
 */
extern int allocate_block(select_ba_request_t* ba_request, List results)
{
	if (!ba_initialized){
		error("Error, configuration not initialized, "
		      "calling ba_init(NULL, 1)");
		ba_init(NULL, 1);
	}

	if (!ba_request){
		error("allocate_block Error, request not initialized");
		return 0;
	}

	// _backup_ba_system();
	if (_find_match(ba_request, results)){
		return 1;
	} else {
		return 0;
	}
}


/*
 * Admin wants to remove a previous allocation.
 * will allow Admin to delete a previous allocation retrival by letter code.
 */
extern int remove_block(List nodes, bool is_small)
{
	int dim;
	ba_mp_t* curr_ba_node = NULL;
	ba_mp_t* ba_node = NULL;
	ba_switch_t *curr_switch = NULL;
	ListIterator itr;

	itr = list_iterator_create(nodes);
	while ((curr_ba_node = (ba_mp_t*) list_next(itr))) {
		/* since the list that comes in might not be pointers
		   to the main list we need to point to that main list */
		ba_node = &ba_main_grid[curr_ba_node->coord[X]]
			[curr_ba_node->coord[Y]]
			[curr_ba_node->coord[Z]];
		if (curr_ba_node->used)
			ba_node->used &= (~BA_MP_USED_TRUE);

		/* Small blocks don't use wires, and only have 1 node,
		   so just break. */
		if (is_small)
			break;
		for(dim=0;dim<cluster_dims;dim++) {
			curr_switch = &ba_node->axis_switch[dim];
			if (curr_switch->int_wire[0].used) {
				_reset_the_path(curr_switch, 0, 1, dim);
			}
		}
	}
	list_iterator_destroy(itr);
	return 1;
}

/*
 * Used to set a block into a virtual system.  The system can be
 * cleared first and this function sets all the wires and midplanes
 * used in the nodelist given.  The nodelist is a list of ba_mp_t's
 * that are already set up.  This is very handly to test if there are
 * any passthroughs used by one block when adding another block that
 * also uses those wires, and neither use any overlapping
 * midplanes. Doing a simple bitmap & will not reveal this.
 *
 * Returns SLURM_SUCCESS if nodelist fits into system without
 * conflict, and SLURM_ERROR if nodelist conflicts with something
 * already in the system.
 */
extern int check_and_set_mp_list(List nodes)
{
	int rc = SLURM_ERROR;

#ifdef HAVE_BG_L_P
	int i, j;
	ba_switch_t *ba_switch = NULL, *curr_ba_switch = NULL;
	ba_mp_t *ba_node = NULL, *curr_ba_node = NULL;
	ListIterator itr = NULL;

	if (!nodes)
		return rc;

	itr = list_iterator_create(nodes);
	while ((ba_node = list_next(itr))) {
		/* info("checking %c%c%c", */
/* 		     ba_node->coord[X],  */
/* 		     ba_node->coord[Y], */
/* 		     ba_node->coord[Z]); */

		curr_ba_node = &ba_main_grid[ba_node->coord[X]]
			[ba_node->coord[Y]]
			[ba_node->coord[Z]];

		if (ba_node->used && curr_ba_node->used) {
			/* Only error if the midplane isn't already
			 * marked down or in a error state outside of
			 * the bluegene block.
			 */
			uint16_t base_state, node_flags;
			base_state = curr_ba_node->state & NODE_STATE_BASE;
			node_flags = curr_ba_node->state & NODE_STATE_FLAGS;
			if (!(node_flags & (NODE_STATE_DRAIN | NODE_STATE_FAIL))
			    && (base_state != NODE_STATE_DOWN)) {
				if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
					info("I have already been to "
					     "this node %c%c%c %s",
					     alpha_num[ba_node->coord[X]],
					     alpha_num[ba_node->coord[Y]],
					     alpha_num[ba_node->coord[Z]],
					     node_state_string(
						     curr_ba_node->state));
				rc = SLURM_ERROR;
				goto end_it;
			}
		}

		if (ba_node->used)
			curr_ba_node->used = ba_node->used;
		if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
			info("check_and_set_mp_list: "
			     "%s is used ?= %d %d",
			     curr_ba_node->coord_str,
			     curr_ba_node->used, ba_node->used);
		for(i=0; i<cluster_dims; i++) {
			ba_switch = &ba_node->axis_switch[i];
			curr_ba_switch = &curr_ba_node->axis_switch[i];
			//info("checking dim %d", i);

			for(j=0; j<NUM_PORTS_PER_NODE; j++) {
				//info("checking port %d", j);

				if (ba_switch->int_wire[j].used
				    && curr_ba_switch->int_wire[j].used
				    && j != curr_ba_switch->
				    int_wire[j].port_tar) {
					if (ba_debug_flags
					    & DEBUG_FLAG_BG_ALGO_DEEP)
						info("%c%c%c dim %d port %d "
						     "is already in use to %d",
						     alpha_num[ba_node->
							       coord[X]],
						     alpha_num[ba_node->
							       coord[Y]],
						     alpha_num[ba_node->
							       coord[Z]],
						     i,
						     j,
						     curr_ba_switch->
						     int_wire[j].port_tar);
					rc = SLURM_ERROR;
					goto end_it;
				}
				if (!ba_switch->int_wire[j].used)
					continue;

				/* info("setting %c%c%c dim %d port %d -> %d",*/
/* 				     alpha_num[ba_node->coord[X]],  */
/* 				     alpha_num[ba_node->coord[Y]], */
/* 				     alpha_num[ba_node->coord[Z]],  */
/* 				     i, */
/* 				     j, */
/* 				     ba_switch->int_wire[j].port_tar); */
				curr_ba_switch->int_wire[j].used = 1;
				curr_ba_switch->int_wire[j].port_tar
					= ba_switch->int_wire[j].port_tar;
			}
		}
	}
	rc = SLURM_SUCCESS;
end_it:
	list_iterator_destroy(itr);
#endif
	return rc;
}

/*
 * Used to find, and set up midplanes and the wires in the virtual
 * system and return them in List results
 *
 * IN/OUT results - a list with a NULL destroyer filled in with
 *        midplanes and wires set to create the block with the api. If
 *        only interested in the hostlist NULL can be excepted also.
 * IN ba_request - request for the block
 *
 * To be set in the ba_request
 *    start - where to start the allocation. (optional)
 *    geometry or size - the requested geometry of the block. (required)
 *    conn_type - mesh, torus, or small. (required)
 *
 * RET char * - hostlist of midplanes results represent must be
 *     xfreed.  NULL on failure
 */
extern char *set_bg_block(List results, select_ba_request_t* ba_request)
{
	char *name = NULL;
	ba_mp_t* ba_node = NULL;
	int send_results = 0;
	int found = 0;

	xassert(ba_request);

	if (cluster_dims == 1) {
		if (ba_request->start[X]>=DIM_SIZE[X])
			return NULL;
		ba_request->size = ba_request->geometry[X];
		ba_node = &ba_main_grid[ba_request->start[X]][0][0];
	} else {
		int dim;

		ba_request->size = 1;
		for (dim=0; dim<cluster_dims; dim++) {
			if (ba_request->start[dim] >= DIM_SIZE[dim])
				return NULL;
			if ((int16_t)ba_request->geometry[dim] <= 0) {
				error("problem with geometry of %c in dim %d, "
				      "needs to be at least 1",
				      alpha_num[ba_request->geometry[dim]],
				      dim);
				return NULL;
			}
			ba_request->size *= ba_request->geometry[dim];
		}

		ba_node = coord2ba_mp(ba_request->start);
	}

	if (!ba_node)
		return NULL;

	if (!results)
		results = list_create(NULL);
	else
		send_results = 1;

	/* This midplane should have already been checked if it was in
	   use or not */
	list_append(results, ba_node);

	if (ba_request->conn_type[0] >= SELECT_SMALL) {
		/* adding the ba_node and ending */
		ba_node->used |= BA_MP_USED_TRUE;
		name = xstrdup_printf("%s", ba_node->coord_str);
		goto end_it;
	} else if (ba_request->conn_type[0] == SELECT_NAV)
		ba_request->conn_type[0] = bg_conf->default_conn_type[0];

	found = _find_x_path(results, ba_node,
			     ba_node->coord,
			     ba_request->geometry[X],
			     1,
			     ba_request->conn_type[0], BLOCK_ALGO_FIRST);

	if (!found) {
		bool is_small = 0;
		if (ba_request->conn_type[0] == SELECT_SMALL)
			is_small = 1;
		debug2("trying less efficient code");
		remove_block(results, is_small);
		list_flush(results);
		list_append(results, ba_node);
		found = _find_x_path(results, ba_node,
				     ba_node->coord,
				     ba_request->geometry[X],
				     1,
				     ba_request->conn_type[0],
				     BLOCK_ALGO_SECOND);
	}
	if (found) {
		if (cluster_flags & CLUSTER_FLAG_BG) {
			List start_list = NULL;
			ListIterator itr;

			start_list = list_create(NULL);
			itr = list_iterator_create(results);
			while ((ba_node = (ba_mp_t*) list_next(itr))) {
				list_append(start_list, ba_node);
			}
			list_iterator_destroy(itr);

			if (!_fill_in_coords(results,
					     start_list,
					     ba_request->geometry,
					     ba_request->conn_type[0])) {
				list_destroy(start_list);
				goto end_it;
			}
			list_destroy(start_list);
		}
	} else {
		goto end_it;
	}

	name = _set_internal_wires(results,
				   ba_request->size,
				   ba_request->conn_type[0]);
end_it:
	if (!send_results && results) {
		list_destroy(results);
		results = NULL;
	}
	if (name!=NULL) {
		debug2("name = %s", name);
	} else {
		debug2("can't allocate");
		xfree(name);
	}

	return name;
}

/* Rotate a 3-D geometry array through its six permutations */
extern void ba_rotate_geo(uint16_t *req_geometry, int rot_cnt)
{
	uint16_t tmp;

	switch (rot_cnt) {
	case 0:		/* ABC -> ACB */
	case 2:		/* CAB -> CBA */
	case 4:		/* BCA -> BAC */
		SWAP(req_geometry[Y], req_geometry[Z], tmp);
		break;
	case 1:		/* ACB -> CAB */
	case 3:		/* CBA -> BCA */
	case 5:		/* BAC -> ABC */
		SWAP(req_geometry[X], req_geometry[Y], tmp);
		break;
	}
}

/********************* Local Functions *********************/

/*
 * This function is here to check options for rotating and elongating
 * and set up the request based on the count of each option
 */
static int _check_for_options(select_ba_request_t* ba_request)
{
	int temp;
	int set=0;
	uint16_t *geo = NULL;
	ListIterator itr;

	if (ba_request->rotate) {
	rotate_again:
		debug2("Rotating! %d",ba_request->rotate_count);

		if (ba_request->rotate_count==(cluster_dims-1)) {
			temp=ba_request->geometry[X];
			ba_request->geometry[X]=ba_request->geometry[Z];
			ba_request->geometry[Z]=temp;
			ba_request->rotate_count++;
			set=1;

		} else if (ba_request->rotate_count<(cluster_dims*2)) {
			temp=ba_request->geometry[X];
			ba_request->geometry[X]=ba_request->geometry[Y];
			ba_request->geometry[Y]=ba_request->geometry[Z];
			ba_request->geometry[Z]=temp;
			ba_request->rotate_count++;
			set=1;
		} else
			ba_request->rotate = false;
		if (set) {
			if (ba_request->geometry[X]<=DIM_SIZE[X]
			    && ba_request->geometry[Y]<=DIM_SIZE[Y]
			    && ba_request->geometry[Z]<=DIM_SIZE[Z])
				return 1;
			else {
				set = 0;
				goto rotate_again;
			}
		}
	}
	if (ba_request->elongate) {
	elongate_again:
		debug2("Elongating! %d",ba_request->elongate_count);
		ba_request->rotate_count=0;
		ba_request->rotate = true;

		set = 0;
		itr = list_iterator_create(ba_request->elongate_geos);
		for(set=0; set<=ba_request->elongate_count; set++)
			geo = list_next(itr);
		list_iterator_destroy(itr);
		if (geo == NULL)
			return 0;
		ba_request->elongate_count++;
		ba_request->geometry[X] = geo[X];
		ba_request->geometry[Y] = geo[Y];
		ba_request->geometry[Z] = geo[Z];
		if (ba_request->geometry[X]<=DIM_SIZE[X]
		    && ba_request->geometry[Y]<=DIM_SIZE[Y]
		    && ba_request->geometry[Z]<=DIM_SIZE[Z]) {
			return 1;
		} else
			goto elongate_again;

	}
	return 0;
}

/*
 * grab all the geometries that we can get and append them to the list geos
 */
static int _append_geo(uint16_t *geometry, List geos, int rotate)
{
	ListIterator itr;
	uint16_t *geo_ptr = NULL;
	uint16_t *geo = NULL;
	int temp_geo;
	int i, j;

	if (rotate) {
		for (i = (cluster_dims - 1); i >= 0; i--) {
			for (j = 1; j <= i; j++) {
				if ((geometry[j-1] > geometry[j])
				    && (geometry[j] <= DIM_SIZE[j-i])
				    && (geometry[j-1] <= DIM_SIZE[j])) {
					temp_geo = geometry[j-1];
					geometry[j-1] = geometry[j];
					geometry[j] = temp_geo;
				}
			}
		}
	}
	itr = list_iterator_create(geos);
	while ((geo_ptr = list_next(itr)) != NULL) {
		if (geometry[X] == geo_ptr[X]
		    && geometry[Y] == geo_ptr[Y]
		    && geometry[Z] == geo_ptr[Z])
			break;

	}
	list_iterator_destroy(itr);

	if (geo_ptr == NULL) {
		geo = xmalloc(sizeof(int)*cluster_dims);
		geo[X] = geometry[X];
		geo[Y] = geometry[Y];
		geo[Z] = geometry[Z];
		if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
			info("adding geo %c%c%c",
			     alpha_num[geo[X]], alpha_num[geo[Y]],
			     alpha_num[geo[Z]]);
		list_append(geos, geo);
	}
	return 1;
}

/*
 * Fill in the paths and extra midplanes we need for the block.
 * Basically copy the x path sent in with the start_list in each Y and
 * Z dimension filling in every midplane for the block and then
 * completing the Y and Z wiring, tying the whole block together.
 *
 * IN/OUT results - total list of midplanes after this function
 *        returns successfully.  Should be
 *        an exact copy of the start_list at first.
 * IN start_list - exact copy of results at first, This should only be
 *        a list of midplanes on the X dim.  We will work off this and
 *        the geometry to fill in this wiring for the X dim in all the
 *        Y and Z coords.
 * IN geometry - What the block looks like
 * IN conn_type - Mesh or Torus
 *
 * RET: 0 on failure 1 on success
 */
static int _fill_in_coords(List results, List start_list,
			   uint16_t *geometry, int conn_type)
{
	ba_mp_t *ba_node = NULL;
	ba_mp_t *check_node = NULL;
	int rc = 1;
	ListIterator itr = NULL;
	int y=0, z=0;
	ba_switch_t *curr_switch = NULL;
	ba_switch_t *next_switch = NULL;

	if (!start_list || !results)
		return 0;
	/* go through the start_list and add all the midplanes */
	itr = list_iterator_create(start_list);
	while ((check_node = (ba_mp_t*) list_next(itr))) {
		curr_switch = &check_node->axis_switch[X];

		for(y=0; y<geometry[Y]; y++) {
			if ((check_node->coord[Y]+y) >= DIM_SIZE[Y]) {
				rc = 0;
				goto failed;
			}
			for(z=0; z<geometry[Z]; z++) {
				if ((check_node->coord[Z]+z) >= DIM_SIZE[Z]) {
					rc = 0;
					goto failed;
				}
				ba_node = &ba_main_grid
					[check_node->coord[X]]
					[check_node->coord[Y]+y]
					[check_node->coord[Z]+z];

				if ((ba_node->coord[Y] == check_node->coord[Y])
				    && (ba_node->coord[Z]
					== check_node->coord[Z]))
					continue;

				if (!_node_used(ba_node, geometry[X])) {
					if (ba_debug_flags
					    & DEBUG_FLAG_BG_ALGO_DEEP)
						info("here Adding %c%c%c",
						     alpha_num[ba_node->
							       coord[X]],
						     alpha_num[ba_node->
							       coord[Y]],
						     alpha_num[ba_node->
							       coord[Z]]);
					list_append(results, ba_node);
					next_switch = &ba_node->axis_switch[X];

					/* since we are going off the
					 * main system we can send NULL
					 * here
					 */
					_copy_the_path(NULL, curr_switch,
						       next_switch,
						       0, X);
				} else {
					rc = 0;
					goto failed;
				}
			}
		}

	}
	list_iterator_destroy(itr);
	itr = list_iterator_create(start_list);
	check_node = (ba_mp_t*) list_next(itr);
	list_iterator_destroy(itr);

	itr = list_iterator_create(results);
	while ((ba_node = (ba_mp_t*) list_next(itr))) {
		if (!_find_yz_path(ba_node,
				   check_node->coord,
				   geometry,
				   conn_type)){
			rc = 0;
			goto failed;
		}
	}

	if (deny_pass) {
		if ((*deny_pass & PASS_DENY_Y)
		    && (*deny_pass & PASS_FOUND_Y)) {
			debug("We don't allow Y passthoughs");
			rc = 0;
		} else if ((*deny_pass & PASS_DENY_Z)
			   && (*deny_pass & PASS_FOUND_Z)) {
			debug("We don't allow Z passthoughs");
			rc = 0;
		}
	}

failed:
	list_iterator_destroy(itr);

	return rc;
}

/*
 * Copy a path through the wiring of a switch to another switch on a
 * starting port on a dimension.
 *
 * IN/OUT: nodes - Local list of midplanes you are keeping track of.  If
 *         you visit any new midplanes a copy from ba_main_grid
 *         will be added to the list.  If NULL the path will be
 *         set in mark_switch of the main virtual system (ba_main_grid).
 * IN: curr_switch - The switch you want to copy the path of
 * IN/OUT: mark_switch - The switch you want to fill in.  On success
 *         this switch will contain a complete path from the curr_switch
 *         starting from the source port.
 * IN: source - source port number (If calling for the first time
 *         should be 0 since we are looking for 1 at the end)
 * IN: dim - Dimension XYZ
 *
 * RET: on success 1, on error 0
 */
static int _copy_the_path(List nodes, ba_switch_t *curr_switch,
			  ba_switch_t *mark_switch,
			  int source, int dim)
{
	uint16_t *mp_tar;
	uint16_t *mark_mp_tar;
	uint16_t *node_curr;
	uint16_t port_tar, port_tar1;
	ba_switch_t *next_switch = NULL;
	ba_switch_t *next_mark_switch = NULL;

	/* Copy the source used and port_tar */
	mark_switch->int_wire[source].used =
		curr_switch->int_wire[source].used;
	mark_switch->int_wire[source].port_tar =
		curr_switch->int_wire[source].port_tar;

	port_tar = curr_switch->int_wire[source].port_tar;

	/* Now to the same thing from the other end */
	mark_switch->int_wire[port_tar].used =
		curr_switch->int_wire[port_tar].used;
	mark_switch->int_wire[port_tar].port_tar =
		curr_switch->int_wire[port_tar].port_tar;
	port_tar1 = port_tar;

	/* follow the path */
	node_curr = curr_switch->ext_wire[0].mp_tar;
	mp_tar = curr_switch->ext_wire[port_tar].mp_tar;
	if (mark_switch->int_wire[source].used)
		if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
			info("setting dim %d %c%c%c %d-> %c%c%c %d",
			     dim,
			     alpha_num[node_curr[X]],
			     alpha_num[node_curr[Y]],
			     alpha_num[node_curr[Z]],
			     source,
			     alpha_num[mp_tar[X]],
			     alpha_num[mp_tar[Y]],
			     alpha_num[mp_tar[Z]],
			     port_tar);

	if (port_tar == 1) {
		/* found the end of the line */
		mark_switch->int_wire[1].used =
			curr_switch->int_wire[1].used;
		mark_switch->int_wire[1].port_tar =
			curr_switch->int_wire[1].port_tar;
		return 1;
	}

	mark_mp_tar = mark_switch->ext_wire[port_tar].mp_tar;
	port_tar = curr_switch->ext_wire[port_tar].port_tar;

	if (node_curr[X] == mp_tar[X]
	    && node_curr[Y] == mp_tar[Y]
	    && node_curr[Z] == mp_tar[Z]) {
		/* We are going to the same node! this should never
		   happen */
		debug5("something bad happened!! "
		       "we are on %c%c%c and are going to it "
		       "from port %d - > %d",
		       alpha_num[node_curr[X]],
		       alpha_num[node_curr[Y]],
		       alpha_num[node_curr[Z]],
		       port_tar1, port_tar);
		return 0;
	}

	/* see what the next switch is going to be */
	next_switch = &ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]].
		axis_switch[dim];
	if (!nodes) {
		/* If no nodes then just get the next switch to fill
		   in from the main system */
		next_mark_switch = &ba_main_grid[mark_mp_tar[X]]
			[mark_mp_tar[Y]]
			[mark_mp_tar[Z]]
			.axis_switch[dim];
	} else {
		ba_mp_t *ba_node = NULL;
		ListIterator itr = list_iterator_create(nodes);
		/* see if we have already been to this node */
		while ((ba_node = list_next(itr))) {
			if (ba_node->coord[X] == mark_mp_tar[X] &&
			    ba_node->coord[Y] == mark_mp_tar[Y] &&
			    ba_node->coord[Z] == mark_mp_tar[Z])
				break;	/* we found it */
		}
		list_iterator_destroy(itr);
		if (!ba_node) {
			/* If node grab a copy and add it to the list */
			ba_node = ba_copy_mp(&ba_main_grid[mark_mp_tar[X]]
					     [mark_mp_tar[Y]]
					     [mark_mp_tar[Z]]);
			ba_setup_mp(ba_node, false, false);
			list_push(nodes, ba_node);
			if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
				info("haven't seen %c%c%c adding it",
				     alpha_num[ba_node->coord[X]],
				     alpha_num[ba_node->coord[Y]],
				     alpha_num[ba_node->coord[Z]]);
		}
		next_mark_switch = &ba_node->axis_switch[dim];

	}

	/* Keep going until we reach the end of the line */
	return _copy_the_path(nodes, next_switch, next_mark_switch,
			      port_tar, dim);
}

static int _find_yz_path(ba_mp_t *ba_node, uint16_t *first,
			 uint16_t *geometry, int conn_type)
{
	ba_mp_t *next_node = NULL;
	uint16_t *mp_tar = NULL;
	ba_switch_t *dim_curr_switch = NULL;
	ba_switch_t *dim_next_switch = NULL;
	int i2;
	int count = 0;

	for(i2=1;i2<=2;i2++) {
		if (geometry[i2] > 1) {
			if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
				info("%d node %c%c%c port 2 -> ",
				     i2,
				     alpha_num[ba_node->coord[X]],
				     alpha_num[ba_node->coord[Y]],
				     alpha_num[ba_node->coord[Z]]);

			dim_curr_switch = &ba_node->axis_switch[i2];
			if (dim_curr_switch->int_wire[2].used) {
				debug5("returning here");
				return 0;
			}

			mp_tar = dim_curr_switch->ext_wire[2].mp_tar;

			next_node =
				&ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]];
			dim_next_switch = &next_node->axis_switch[i2];
			if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
				info("%c%c%c port 5",
				     alpha_num[next_node->coord[X]],
				     alpha_num[next_node->coord[Y]],
				     alpha_num[next_node->coord[Z]]);

			if (dim_next_switch->int_wire[5].used) {
				if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
					info("returning here 2");
				return 0;
			}
			debug5("%d %d %d %d",i2, mp_tar[i2],
			       first[i2], geometry[i2]);

			/* Here we need to see where we are in
			 * reference to the geo of this dimension.  If
			 * we have not gotten the number we need in
			 * the direction we just go to the next node
			 * with 5 -> 1.  If we have all the midplanes
			 * we need then we go through and finish the
			 * torus if needed
			 */
			if (mp_tar[i2] < first[i2])
				count = mp_tar[i2]+(DIM_SIZE[i2]-first[i2]);
			else
				count = (mp_tar[i2]-first[i2]);

			if (count == geometry[i2]) {
				debug5("found end of me %c%c%c",
				       alpha_num[mp_tar[X]],
				       alpha_num[mp_tar[Y]],
				       alpha_num[mp_tar[Z]]);
				if (conn_type == SELECT_TORUS) {
					dim_curr_switch->int_wire[0].used = 1;
					dim_curr_switch->int_wire[0].port_tar
						= 2;
					dim_curr_switch->int_wire[2].used = 1;
					dim_curr_switch->int_wire[2].port_tar
						= 0;
					dim_curr_switch = dim_next_switch;

					if (deny_pass
					    && (mp_tar[i2] != first[i2])) {
						if (i2 == 1)
							*deny_pass |=
								PASS_FOUND_Y;
						else
							*deny_pass |=
								PASS_FOUND_Z;
					}
					while (mp_tar[i2] != first[i2]) {
						if (ba_debug_flags
						    & DEBUG_FLAG_BG_ALGO_DEEP)
							info("on dim %d at %d "
							     "looking for %d",
							     i2,
							     mp_tar[i2],
							     first[i2]);

						if (dim_curr_switch->
						    int_wire[2].used) {
							if (ba_debug_flags
							    & DEBUG_FLAG_BG_ALGO_DEEP)
								info("returning"
								     " here 3");
							return 0;
						}

						dim_curr_switch->
							int_wire[2].used = 1;
						dim_curr_switch->
							int_wire[2].port_tar
							= 5;
						dim_curr_switch->
							int_wire[5].used
							= 1;
						dim_curr_switch->
							int_wire[5].
							port_tar = 2;


						mp_tar = dim_curr_switch->
							ext_wire[2].mp_tar;
						next_node = &ba_main_grid
							[mp_tar[X]]
							[mp_tar[Y]]
							[mp_tar[Z]];
						dim_curr_switch =
							&next_node->
							axis_switch[i2];
					}

					if (ba_debug_flags
					    & DEBUG_FLAG_BG_ALGO_DEEP)
						info("back to first on dim %d "
						     "at %d looking for %d",
						     i2,
						     mp_tar[i2],
						     first[i2]);

					dim_curr_switch->
						int_wire[5].used = 1;
					dim_curr_switch->
						int_wire[5].port_tar
						= 1;
					dim_curr_switch->
						int_wire[1].used
						= 1;
					dim_curr_switch->
						int_wire[1].
						port_tar = 5;
				}

			} else if (count < geometry[i2]) {
				if (conn_type == SELECT_TORUS ||
				    (conn_type == SELECT_MESH &&
				     (mp_tar[i2] != first[i2]))) {
					dim_curr_switch->
						int_wire[0].used = 1;
					dim_curr_switch->
						int_wire[0].port_tar
						= 2;
					dim_curr_switch->
						int_wire[2].used
						= 1;
					dim_curr_switch->
						int_wire[2].
						port_tar = 0;

					dim_next_switch->int_wire[5].used
						= 1;
					dim_next_switch->
						int_wire[5].port_tar
						= 1;
					dim_next_switch->
						int_wire[1].used = 1;
					dim_next_switch->
						int_wire[1].port_tar
						= 5;
				}
			} else {
				error("We were only looking for %d "
				      "in the %d dim, but now we have %d",
				      geometry[i2], i2, count);
				return 0;
			}
		} else if ((geometry[i2] == 1) && (conn_type == SELECT_TORUS)) {
			/* FIX ME: This is put here because we got
			   into a state where the Y dim was not being
			   processed correctly.  This will set up the
			   0 -> 1 port correctly.  We should probably
			   find out why this was happening in the
			   first place though.  A reproducer was to
			   have
			   MPs=[310x323] Type=TORUS
			   MPs=[200x233] Type=TORUS
			   MPs=[300x303] Type=TORUS
			   MPs=[100x133] Type=TORUS
			   MPs=[000x033] Type=TORUS
			   MPs=[400x433] Type=TORUS
			   and then add
			   MPs=[330x333] Type=TORUS
			*/

			dim_curr_switch = &ba_node->axis_switch[i2];
			if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
				info("%d node %c%c%c port 0 -> 1",
				     i2,
				     alpha_num[ba_node->coord[X]],
				     alpha_num[ba_node->coord[Y]],
				     alpha_num[ba_node->coord[Z]]);
			dim_curr_switch->int_wire[0].used = 1;
			dim_curr_switch->int_wire[0].port_tar = 1;
			dim_curr_switch->int_wire[1].used = 1;
			dim_curr_switch->int_wire[1].port_tar = 0;
		}
	}
	return 1;
}

#ifndef HAVE_BG_FILES
/** */
static int _emulate_ext_wiring(ba_mp_t ***grid)
{
	int x;
	ba_mp_t *source = NULL, *target = NULL;
	if (cluster_dims == 1) {
		for(x=0;x<DIM_SIZE[X];x++) {
			source = &grid[x][0][0];
			if (x<(DIM_SIZE[X]-1))
				target = &grid[x+1][0][0];
			else
				target = &grid[0][0][0];
			_set_external_wires(X, x, source, target);
		}
	} else {
		int y,z;
		for(x=0;x<DIM_SIZE[X];x++) {
			for(y=0;y<DIM_SIZE[Y];y++) {
				for(z=0;z<DIM_SIZE[Z];z++) {
					source = &grid[x][y][z];

					if (x<(DIM_SIZE[X]-1)) {
						target = &grid[x+1][y][z];
					} else
						target = &grid[0][y][z];

					_set_external_wires(X, x, source,
							    target);

					if (y<(DIM_SIZE[Y]-1))
						target = &grid[x][y+1][z];
					else
						target = &grid[x][0][z];

					_set_external_wires(Y, y, source,
							    target);
					if (z<(DIM_SIZE[Z]-1))
						target = &grid[x][y][z+1];
					else
						target = &grid[x][y][0];

					_set_external_wires(Z, z, source,
							    target);
				}
			}
		}
	}
	return 1;
}
#endif


static int _reset_the_path(ba_switch_t *curr_switch, int source,
			   int target, int dim)
{
	uint16_t *mp_tar;
	uint16_t *node_curr;
	int port_tar, port_tar1;
	ba_switch_t *next_switch = NULL;

	if (source < 0 || source >= NUM_PORTS_PER_NODE) {
		fatal("source port was %d can only be 0->%d",
		      source, NUM_PORTS_PER_NODE);
	}
	if (target < 0 || target >= NUM_PORTS_PER_NODE) {
		fatal("target port was %d can only be 0->%d",
		      target, NUM_PORTS_PER_NODE);
	}
	/*set the switch to not be used */
	if (!curr_switch->int_wire[source].used) {
		/* This means something overlapping the removing block
		   already cleared this, or the path just never was
		   complete in the first place. */
		if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
			info("I reached the end, the source isn't used");
		return 1;
	}
	curr_switch->int_wire[source].used = 0;
	port_tar = curr_switch->int_wire[source].port_tar;
	if (port_tar < 0 || port_tar >= NUM_PORTS_PER_NODE) {
		fatal("port_tar port was %d can only be 0->%d",
		      source, NUM_PORTS_PER_NODE);
		return 1;
	}

	port_tar1 = port_tar;
	curr_switch->int_wire[source].port_tar = source;
	curr_switch->int_wire[port_tar].used = 0;
	curr_switch->int_wire[port_tar].port_tar = port_tar;
	if (port_tar == target) {
		return 1;
	}
	/* follow the path */
	node_curr = curr_switch->ext_wire[0].mp_tar;
	mp_tar = curr_switch->ext_wire[port_tar].mp_tar;
	port_tar = curr_switch->ext_wire[port_tar].port_tar;
	if (source == port_tar1) {
		debug("got this bad one %c%c%c %d %d -> %c%c%c %d",
		      alpha_num[node_curr[X]],
		      alpha_num[node_curr[Y]],
		      alpha_num[node_curr[Z]],
		      source,
		      port_tar1,
		      alpha_num[mp_tar[X]],
		      alpha_num[mp_tar[Y]],
		      alpha_num[mp_tar[Z]],
		      port_tar);
		return 0;
	}
	debug5("from %c%c%c %d %d -> %c%c%c %d",
	       alpha_num[node_curr[X]],
	       alpha_num[node_curr[Y]],
	       alpha_num[node_curr[Z]],
	       source,
	       port_tar1,
	       alpha_num[mp_tar[X]],
	       alpha_num[mp_tar[Y]],
	       alpha_num[mp_tar[Z]],
	       port_tar);
	if (node_curr[X] == mp_tar[X]
	    && node_curr[Y] == mp_tar[Y]
	    && node_curr[Z] == mp_tar[Z]) {
		debug5("%d something bad happened!!", dim);
		return 0;
	}
	next_switch =
		&ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]].axis_switch[dim];

	return _reset_the_path(next_switch, port_tar, target, dim);
//	return 1;
}

extern void ba_create_system()
{
	int x,y,z, i = 0;

	if (ba_main_grid)
		ba_destroy_system();

	best_count=BEST_COUNT_INIT;

	ba_main_grid = (ba_mp_t***)
		xmalloc(sizeof(ba_mp_t**) * DIM_SIZE[X]);
	for (x=0; x<DIM_SIZE[X]; x++) {
		ba_main_grid[x] = (ba_mp_t**)
			xmalloc(sizeof(ba_mp_t*) * DIM_SIZE[Y]);
		for (y=0; y<DIM_SIZE[Y]; y++) {
			ba_main_grid[x][y] = (ba_mp_t*)
				xmalloc(sizeof(ba_mp_t)
					* DIM_SIZE[Z]);
			for (z=0; z<DIM_SIZE[Z]; z++){
				ba_mp_t *ba_mp = &ba_main_grid[x][y][z];
				ba_mp->coord[X] = x;
				ba_mp->coord[Y] = y;
				ba_mp->coord[Z] = z;
				snprintf(ba_mp->coord_str,
					 sizeof(ba_mp->coord_str),
					 "%c%c%c",
					 alpha_num[ba_mp->coord[X]],
					 alpha_num[ba_mp->coord[Y]],
					 alpha_num[ba_mp->coord[Z]]);
				ba_setup_mp(ba_mp, true, false);
				ba_mp->state = NODE_STATE_IDLE;
				/* This might get changed
				   later, but just incase set
				   it up here.
				*/
				ba_mp->index = i++;
			}
		}
	}
	if ((cluster_flags & CLUSTER_FLAG_BGL) ||
	    (cluster_flags & CLUSTER_FLAG_BGP)) {
		init_wires();
#ifndef HAVE_BG_FILES
		_emulate_ext_wiring(ba_main_grid);
#endif
	}

	path = list_create(_delete_path_list);
	best_path = list_create(_delete_path_list);
}

/** */
extern void ba_destroy_system(void)
{
	int x, y;

	if (path) {
		list_destroy(path);
		path = NULL;
	}
	if (best_path) {
		list_destroy(best_path);
		best_path = NULL;
	}

#ifdef HAVE_BG_FILES
	if (bg)
		bridge_free_bg(bg);
#endif
	_mp_map_initialized = false;
	_wires_initialized = true;

	if (ba_main_grid) {
		for (x=0; x<DIM_SIZE[X]; x++) {
			for (y=0; y<DIM_SIZE[Y]; y++)
				xfree(ba_main_grid[x][y]);

			xfree(ba_main_grid[x]);
		}
		xfree(ba_main_grid);
		ba_main_grid = NULL;
	}
}

extern bool ba_sub_block_in_bitmap(select_jobinfo_t *jobinfo,
				   bitstr_t *usable_bitmap, bool step)
{
	/* This shouldn't be called. */
	xassert(0);
	return false;
}

extern int ba_sub_block_in_bitmap_clear(select_jobinfo_t *jobinfo,
					bitstr_t *usable_bitmap)
{
	/* this doesn't do anything since above doesn't. */
	return SLURM_SUCCESS;
}

extern ba_mp_t *ba_sub_block_in_record(
	bg_record_t *bg_record, uint32_t *node_count,
	select_jobinfo_t *jobinfo)
{
	/* This shouldn't be called. */
	xassert(0);
	return false;
}

extern int ba_sub_block_in_record_clear(
	bg_record_t *bg_record, struct step_record *step_ptr)
{
	/* this does…