/src/plugins/select/bluegene/ba/block_allocator.c
C | 3372 lines | 2555 code | 329 blank | 488 comment | 608 complexity | 6c61c1ad98648645d8aa3896e78e8425 MD5 | raw file
Possible License(s): GPL-2.0, AGPL-1.0
- /*****************************************************************************\
- * block_allocator.c - Assorted functions for layout of bluegene blocks,
- * wiring, mapping for smap, etc.
- * $Id$
- *****************************************************************************
- * Copyright (C) 2004-2007 The Regents of the University of California.
- * Copyright (C) 2008-2009 Lawrence Livermore National Security.
- * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
- * Written by Dan Phung <phung4@llnl.gov>, Danny Auble <da@llnl.gov>
- *
- * This file is part of SLURM, a resource management program.
- * For details, see <http://www.schedmd.com/slurmdocs/>.
- * Please also read the included file: DISCLAIMER.
- *
- * SLURM is free software; you can redistribute it and/or modify it under
- * the terms of the GNU General Public License as published by the Free
- * Software Foundation; either version 2 of the License, or (at your option)
- * any later version.
- *
- * In addition, as a special exception, the copyright holders give permission
- * to link the code of portions of this program with the OpenSSL library under
- * certain conditions as described in each individual source file, and
- * distribute linked combinations including the two. You must obey the GNU
- * General Public License in all respects for all of the code used other than
- * OpenSSL. If you modify file(s) with this exception, you may extend this
- * exception to your version of the file(s), but you are not obligated to do
- * so. If you do not wish to do so, delete this exception statement from your
- * version. If you delete this exception statement from all source files in
- * the program, then also delete it here.
- *
- * SLURM is distributed in the hope that it will be useful, but WITHOUT ANY
- * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
- * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
- * details.
- *
- * You should have received a copy of the GNU General Public License along
- * with SLURM; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- \*****************************************************************************/
- #if HAVE_CONFIG_H
- # include "config.h"
- #endif
- #include <stdio.h>
- #include <stdlib.h>
- #include <math.h>
- #include "block_allocator.h"
- #include "src/common/slurmdb_defs.h"
- #include "src/common/timers.h"
- #include "src/common/uid.h"
- /*
- * structure that holds switch path information for finding the wiring
- * path without setting the configuration.
- *
- * - dim - Which Axis it is on
- * - geometry - node location
- * - in - ingress port.
- * - out - egress port.
- *
- */
- typedef struct {
- int dim;
- uint16_t geometry[HIGHEST_DIMENSIONS];
- int in;
- int out;
- } ba_path_switch_t;
- #define DEBUG_PA
- #define BEST_COUNT_INIT 20
- /* Global */
- bool _initialized = false;
- bool _wires_initialized = false;
- bool _mp_map_initialized = false;
- /* _ba_system is the "current" system that the structures will work
- * on */
- List path = NULL;
- List best_path = NULL;
- int best_count;
- uint16_t *deny_pass = NULL;
- char *p = '\0';
- /* extern Global */
- my_bluegene_t *bg = NULL;
- ba_mp_t ***ba_main_grid = NULL;
- typedef enum {
- BLOCK_ALGO_FIRST,
- BLOCK_ALGO_SECOND
- } block_algo_t;
- /** internal helper functions */
- /* */
- static int _check_for_options(select_ba_request_t* ba_request);
- /* */
- static int _append_geo(uint16_t *geo, List geos, int rotate);
- /* */
- static int _fill_in_coords(List results, List start_list,
- uint16_t *geometry, int conn_type);
- /* */
- static int _copy_the_path(List nodes, ba_switch_t *curr_switch,
- ba_switch_t *mark_switch,
- int source, int dim);
- /* */
- static int _find_yz_path(ba_mp_t *ba_node, uint16_t *first,
- uint16_t *geometry, int conn_type);
- #ifndef HAVE_BG_FILES
- /* */
- static int _emulate_ext_wiring(ba_mp_t ***grid);
- #endif
- /** */
- static int _reset_the_path(ba_switch_t *curr_switch, int source,
- int target, int dim);
- /* */
- static void _delete_path_list(void *object);
- /* find the first block match in the system */
- static int _find_match(select_ba_request_t* ba_request, List results);
- /** */
- static bool _node_used(ba_mp_t* ba_node, int x_size);
- /* */
- static void _switch_config(ba_mp_t* source, ba_mp_t* target, int dim,
- int port_src, int port_tar);
- /* */
- static int _set_external_wires(int dim, int count, ba_mp_t* source,
- ba_mp_t* target);
- /* */
- static char *_set_internal_wires(List nodes, int size, int conn_type);
- /* */
- static int _find_x_path(List results, ba_mp_t *ba_node, uint16_t *start,
- int x_size, int found, int conn_type,
- block_algo_t algo);
- /* */
- static int _remove_node(List results, uint16_t *mp_tar);
- /* */
- static int _find_next_free_using_port_2(ba_switch_t *curr_switch,
- int source_port,
- List nodes, int dim,
- int count);
- /* */
- /* static int _find_passthrough(ba_switch_t *curr_switch, int source_port, */
- /* List nodes, int dim, */
- /* int count, int highest_phys_x); */
- /* */
- static int _finish_torus(List results,
- ba_switch_t *curr_switch, int source_port,
- int dim, int count, uint16_t *start);
- /* */
- static uint16_t *_set_best_path();
- /* */
- static int _set_one_dim(uint16_t *start, uint16_t *end, uint16_t *coord);
- /* */
- static void _destroy_geo(void *object);
- extern void destroy_ba_node(void *ptr)
- {
- ba_mp_t *ba_node = (ba_mp_t *)ptr;
- if (ba_node) {
- xfree(ba_node->loc);
- xfree(ba_node);
- }
- }
- /*
- * create a block request. Note that if the geometry is given,
- * then size is ignored. If elongate is true, the algorithm will try
- * to fit that a block of cubic shape and then it will try other
- * elongated geometries. (ie, 2x2x2 -> 4x2x1 -> 8x1x1).
- *
- * IN/OUT - ba_request: structure to allocate and fill in.
- *
- * ALL below IN's need to be set within the ba_request before the call
- * if you want them to be used.
- * ALL below OUT's are set and returned within the ba_request.
- * IN - avail_node_bitmap: bitmap of usable midplanes.
- * IN - blrtsimage: BlrtsImage for this block if not default
- * IN - conn_type: connection type of request (TORUS or MESH or SMALL)
- * IN - elongate: if true, will try to fit different geometries of
- * same size requests
- * IN/OUT - geometry: requested/returned geometry of block
- * IN - linuximage: LinuxImage for this block if not default
- * IN - mloaderimage: MLoaderImage for this block if not default
- * IN - nodecards: Number of nodecards in each block in request only
- * used of small block allocations.
- * OUT - passthroughs: if there were passthroughs used in the
- * generation of the block.
- * IN - procs: Number of real processors requested
- * IN - quarters: Number of midplane quarters in each block in request only
- * used of small block allocations.
- * IN - RamDiskimage: RamDiskImage for this block if not default
- * IN - rotate: if true, allows rotation of block during fit
- * OUT - save_name: hostlist of midplanes used in block
- * IN/OUT - size: requested/returned count of midplanes in block
- * IN - start: geo location of where to start the allocation
- * IN - start_req: if set use the start variable to start at
- * return success of allocation/validation of params
- */
- extern int new_ba_request(select_ba_request_t* ba_request)
- {
- int i=0;
- float sz=1;
- int i2, picked, total_sz=1, size2=0;
- uint16_t *geo_ptr;
- int messed_with = 0;
- int checked[DIM_SIZE[X]];
- uint16_t geo[cluster_dims];
- memset(geo, 0, sizeof(geo));
- ba_request->save_name= NULL;
- ba_request->rotate_count= 0;
- ba_request->elongate_count = 0;
- ba_request->elongate_geos = list_create(_destroy_geo);
- memcpy(geo, ba_request->geometry, sizeof(geo));
- if (ba_request->deny_pass == (uint16_t)NO_VAL)
- ba_request->deny_pass = ba_deny_pass;
- if (!(cluster_flags & CLUSTER_FLAG_BG)) {
- if (geo[X] != (uint16_t)NO_VAL) {
- for (i=0; i<cluster_dims; i++) {
- if ((geo[i] < 1) || (geo[i] > DIM_SIZE[i])) {
- error("new_ba_request Error, "
- "request geometry is invalid %d",
- geo[i]);
- return 0;
- }
- }
- ba_request->size = ba_request->geometry[X];
- } else if (ba_request->size) {
- ba_request->geometry[X] = ba_request->size;
- } else
- return 0;
- return 1;
- }
- if (geo[X] != (uint16_t)NO_VAL) {
- for (i=0; i<cluster_dims; i++){
- if ((geo[i] < 1) || (geo[i] > DIM_SIZE[i])) {
- error("new_ba_request Error, "
- "request geometry is invalid dim %d "
- "can't be %c, largest is %c",
- i,
- alpha_num[geo[i]],
- alpha_num[DIM_SIZE[i]]);
- return 0;
- }
- }
- _append_geo(geo, ba_request->elongate_geos, 0);
- sz=1;
- for (i=0; i<cluster_dims; i++)
- sz *= ba_request->geometry[i];
- ba_request->size = sz;
- sz=0;
- }
- deny_pass = &ba_request->deny_pass;
- if (ba_request->elongate || sz) {
- sz=1;
- /* decompose the size into a cubic geometry */
- ba_request->rotate= 1;
- ba_request->elongate = 1;
- for (i=0; i<cluster_dims; i++) {
- total_sz *= DIM_SIZE[i];
- geo[i] = 1;
- }
- if (ba_request->size==1) {
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- goto endit;
- }
- if (ba_request->size<=DIM_SIZE[Y]) {
- geo[X] = 1;
- geo[Y] = ba_request->size;
- geo[Z] = 1;
- sz=ba_request->size;
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- i = ba_request->size/4;
- if (!(ba_request->size%2)
- && i <= DIM_SIZE[Y]
- && i <= DIM_SIZE[Z]
- && i*i == ba_request->size) {
- geo[X] = 1;
- geo[Y] = i;
- geo[Z] = i;
- sz=ba_request->size;
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- if (ba_request->size > total_sz || ba_request->size < 1) {
- return 0;
- }
- sz = ba_request->size % (DIM_SIZE[Y] * DIM_SIZE[Z]);
- if (!sz) {
- i = ba_request->size / (DIM_SIZE[Y] * DIM_SIZE[Z]);
- geo[X] = i;
- geo[Y] = DIM_SIZE[Y];
- geo[Z] = DIM_SIZE[Z];
- sz=ba_request->size;
- if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- else
- error("%d I was just trying to add a "
- "geo of %d%d%d "
- "while I am trying to request "
- "%d midplanes",
- __LINE__, geo[X], geo[Y], geo[Z],
- ba_request->size);
- }
- // startagain:
- picked=0;
- for(i=0; i<DIM_SIZE[X]; i++)
- checked[i]=0;
- for (i=0; i<cluster_dims; i++) {
- total_sz *= DIM_SIZE[i];
- geo[i] = 1;
- }
- sz = 1;
- picked=0;
- tryagain:
- size2 = ba_request->size;
- //messedup:
- for (i=picked; i<cluster_dims; i++) {
- if (size2 <= 1)
- break;
- sz = size2 % DIM_SIZE[i];
- if (!sz) {
- geo[i] = DIM_SIZE[i];
- size2 /= DIM_SIZE[i];
- } else if (size2 > DIM_SIZE[i]) {
- for(i2=(DIM_SIZE[i]-1); i2 > 1; i2--) {
- /* go through each number to see if
- the size is divisable by a smaller
- number that is
- good in the other dims. */
- if (!(size2%i2) && !checked[i2]) {
- size2 /= i2;
- if (i==0)
- checked[i2]=1;
- if (i2<DIM_SIZE[i]) {
- geo[i] = i2;
- } else {
- goto tryagain;
- }
- if ((i2-1)!=1 &&
- i!=(cluster_dims-1))
- break;
- }
- }
- /* This size can not be made into a
- block return. If you want to try
- until we find the next largest block
- uncomment the code below and the goto
- above. If a user specifies a max
- node count the job will never
- run.
- */
- if (i2==1) {
- if (!list_count(
- ba_request->elongate_geos))
- error("Can't make a block of "
- "%d into a cube.",
- ba_request->size);
- goto endit;
- /* ba_request->size +=1; */
- /* goto startagain; */
- }
- } else {
- geo[i] = sz;
- break;
- }
- }
- if ((geo[X]*geo[Y]) <= DIM_SIZE[Y]) {
- ba_request->geometry[X] = 1;
- ba_request->geometry[Y] = geo[X] * geo[Y];
- ba_request->geometry[Z] = geo[Z];
- _append_geo(ba_request->geometry,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- if ((geo[X]*geo[Z]) <= DIM_SIZE[Y]) {
- ba_request->geometry[X] = 1;
- ba_request->geometry[Y] = geo[Y];
- ba_request->geometry[Z] = geo[X] * geo[Z];
- _append_geo(ba_request->geometry,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- /* Make sure geo[X] is even and then see if we can get
- it into the Y or Z dim. */
- if (!(geo[X]%2) && ((geo[X]/2) <= DIM_SIZE[Y])) {
- if (geo[Y] == 1) {
- ba_request->geometry[Y] = geo[X]/2;
- messed_with = 1;
- } else
- ba_request->geometry[Y] = geo[Y];
- if (!messed_with && geo[Z] == 1) {
- messed_with = 1;
- ba_request->geometry[Z] = geo[X]/2;
- } else
- ba_request->geometry[Z] = geo[Z];
- if (messed_with) {
- messed_with = 0;
- ba_request->geometry[X] = 2;
- _append_geo(ba_request->geometry,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- }
- if (geo[X] == DIM_SIZE[X]
- && (geo[Y] < DIM_SIZE[Y]
- || geo[Z] < DIM_SIZE[Z])) {
- if (DIM_SIZE[Y]<DIM_SIZE[Z]) {
- i = DIM_SIZE[Y];
- DIM_SIZE[Y] = DIM_SIZE[Z];
- DIM_SIZE[Z] = i;
- }
- ba_request->geometry[X] = geo[X];
- ba_request->geometry[Y] = geo[Y];
- ba_request->geometry[Z] = geo[Z];
- if (ba_request->geometry[Y] < DIM_SIZE[Y]) {
- i = (DIM_SIZE[Y] - ba_request->geometry[Y]);
- ba_request->geometry[Y] +=i;
- }
- if (ba_request->geometry[Z] < DIM_SIZE[Z]) {
- i = (DIM_SIZE[Z] - ba_request->geometry[Z]);
- ba_request->geometry[Z] +=i;
- }
- for(i = DIM_SIZE[X]; i>0; i--) {
- ba_request->geometry[X]--;
- i2 = (ba_request->geometry[X]
- * ba_request->geometry[Y]
- * ba_request->geometry[Z]);
- if (i2 < ba_request->size) {
- ba_request->geometry[X]++;
- messed_with = 1;
- break;
- }
- }
- if (messed_with) {
- messed_with = 0;
- _append_geo(ba_request->geometry,
- ba_request->elongate_geos,
- ba_request->rotate);
- }
- }
- if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- else
- error("%d I was just trying to add a geo of %d%d%d "
- "while I am trying to request %d midplanes",
- __LINE__, geo[X], geo[Y], geo[Z],
- ba_request->size);
- /* Having the functions pow and powf on an aix system doesn't seem to
- * link well, so since this is only for aix and this doesn't really
- * need to be there just don't allow this extra calculation.
- */
- #ifndef HAVE_AIX
- /* see if We can find a cube or square root of the
- size to make an easy cube */
- for(i=0; i<cluster_dims-1; i++) {
- sz = powf((float)ba_request->size,
- (float)1/(cluster_dims-i));
- if (pow(sz,(cluster_dims-i)) == ba_request->size)
- break;
- }
- if (i < (cluster_dims-1)) {
- /* we found something that looks like a cube! */
- int i3 = i;
- for (i=0; i<i3; i++)
- geo[i] = 1;
- for (i=i3; i<cluster_dims; i++)
- if (sz<=DIM_SIZE[i])
- geo[i] = sz;
- else
- goto endit;
- if ((geo[X]*geo[Y]*geo[Z]) == ba_request->size)
- _append_geo(geo,
- ba_request->elongate_geos,
- ba_request->rotate);
- else
- error("%d I was just trying to add "
- "a geo of %d%d%d "
- "while I am trying to request "
- "%d midplanes",
- __LINE__, geo[X], geo[Y], geo[Z],
- ba_request->size);
- }
- #endif //HAVE_AIX
- }
- endit:
- if (!(geo_ptr = list_peek(ba_request->elongate_geos)))
- return 0;
- ba_request->elongate_count++;
- ba_request->geometry[X] = geo_ptr[X];
- ba_request->geometry[Y] = geo_ptr[Y];
- ba_request->geometry[Z] = geo_ptr[Z];
- sz=1;
- for (i=0; i<cluster_dims; i++)
- sz *= ba_request->geometry[i];
- ba_request->size = sz;
- return 1;
- }
- /**
- * print a block request
- */
- extern void print_ba_request(select_ba_request_t* ba_request)
- {
- int i;
- if (ba_request == NULL){
- error("print_ba_request Error, request is NULL");
- return;
- }
- debug(" ba_request:");
- debug(" geometry:\t");
- for (i=0; i<cluster_dims; i++){
- debug("%d", ba_request->geometry[i]);
- }
- debug(" size:\t%d", ba_request->size);
- debug(" conn_type:\t%d", ba_request->conn_type[X]);
- debug(" rotate:\t%d", ba_request->rotate);
- debug(" elongate:\t%d", ba_request->elongate);
- }
- /* If emulating a system set up a known configuration for wires in a
- * system of the size given.
- * If a real bluegene system, query the system and get all wiring
- * information of the system.
- */
- extern void init_wires(void)
- {
- int x, y, z, i;
- ba_mp_t *source = NULL;
- if (_wires_initialized)
- return;
- for(x=0;x<DIM_SIZE[X];x++) {
- for(y=0;y<DIM_SIZE[Y];y++) {
- for(z=0;z<DIM_SIZE[Z];z++) {
- source = &ba_main_grid[x][y][z];
- for(i=0; i<NUM_PORTS_PER_NODE; i++) {
- _switch_config(source, source,
- X, i, i);
- _switch_config(source, source,
- Y, i, i);
- _switch_config(source, source,
- Z, i, i);
- }
- }
- }
- }
- #ifdef HAVE_BG_FILES
- _set_external_wires(0,0,NULL,NULL);
- if (bridge_setup_system() == -1)
- return;
- #endif
- _wires_initialized = true;
- return;
- }
- /*
- * copy the path of the nodes given
- *
- * IN nodes List of ba_mp_t *'s: nodes to be copied
- * OUT dest_nodes List of ba_mp_t *'s: filled in list of nodes
- * wiring.
- * Return on success SLURM_SUCCESS, on error SLURM_ERROR
- */
- extern int copy_node_path(List nodes, List *dest_nodes)
- {
- int rc = SLURM_ERROR;
- #ifdef HAVE_BG_L_P
- ListIterator itr = NULL;
- ListIterator itr2 = NULL;
- ba_mp_t *ba_node = NULL, *new_ba_node = NULL;
- int dim;
- ba_switch_t *curr_switch = NULL, *new_switch = NULL;
- if (!nodes)
- return SLURM_ERROR;
- if (!*dest_nodes)
- *dest_nodes = list_create(destroy_ba_node);
- itr = list_iterator_create(nodes);
- while ((ba_node = list_next(itr))) {
- itr2 = list_iterator_create(*dest_nodes);
- while ((new_ba_node = list_next(itr2))) {
- if (ba_node->coord[X] == new_ba_node->coord[X] &&
- ba_node->coord[Y] == new_ba_node->coord[Y] &&
- ba_node->coord[Z] == new_ba_node->coord[Z])
- break; /* we found it */
- }
- list_iterator_destroy(itr2);
- if (!new_ba_node) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("adding %c%c%c as a new node",
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]]);
- new_ba_node = ba_copy_mp(ba_node);
- ba_setup_mp(new_ba_node, false, false);
- list_push(*dest_nodes, new_ba_node);
- }
- new_ba_node->used = true;
- for(dim=0;dim<cluster_dims;dim++) {
- curr_switch = &ba_node->axis_switch[dim];
- new_switch = &new_ba_node->axis_switch[dim];
- if (curr_switch->int_wire[0].used) {
- if (!_copy_the_path(*dest_nodes,
- curr_switch, new_switch,
- 0, dim)) {
- rc = SLURM_ERROR;
- break;
- }
- }
- }
- }
- list_iterator_destroy(itr);
- rc = SLURM_SUCCESS;
- #endif
- return rc;
- }
- extern ba_mp_t *coord2ba_mp(const uint16_t *coord)
- {
- if ((coord[X] >= DIM_SIZE[X]) || (coord[Y] >= DIM_SIZE[Y]) ||
- (coord[Z] >= DIM_SIZE[Z])) {
- error("Invalid coordinate %d:%d:%d",
- coord[X], coord[Y], coord[Z]);
- return NULL;
- }
- return &ba_main_grid[coord[X]][coord[Y]][coord[Z]];
- }
- /*
- * Try to allocate a block.
- *
- * IN - ba_request: allocation request
- * OUT - results: List of results of the allocation request. Each
- * list entry will be a coordinate. allocate_block will create the
- * list, but the caller must destroy it.
- *
- * return: success or error of request
- */
- extern int allocate_block(select_ba_request_t* ba_request, List results)
- {
- if (!ba_initialized){
- error("Error, configuration not initialized, "
- "calling ba_init(NULL, 1)");
- ba_init(NULL, 1);
- }
- if (!ba_request){
- error("allocate_block Error, request not initialized");
- return 0;
- }
- // _backup_ba_system();
- if (_find_match(ba_request, results)){
- return 1;
- } else {
- return 0;
- }
- }
- /*
- * Admin wants to remove a previous allocation.
- * will allow Admin to delete a previous allocation retrival by letter code.
- */
- extern int remove_block(List nodes, bool is_small)
- {
- int dim;
- ba_mp_t* curr_ba_node = NULL;
- ba_mp_t* ba_node = NULL;
- ba_switch_t *curr_switch = NULL;
- ListIterator itr;
- itr = list_iterator_create(nodes);
- while ((curr_ba_node = (ba_mp_t*) list_next(itr))) {
- /* since the list that comes in might not be pointers
- to the main list we need to point to that main list */
- ba_node = &ba_main_grid[curr_ba_node->coord[X]]
- [curr_ba_node->coord[Y]]
- [curr_ba_node->coord[Z]];
- if (curr_ba_node->used)
- ba_node->used &= (~BA_MP_USED_TRUE);
- /* Small blocks don't use wires, and only have 1 node,
- so just break. */
- if (is_small)
- break;
- for(dim=0;dim<cluster_dims;dim++) {
- curr_switch = &ba_node->axis_switch[dim];
- if (curr_switch->int_wire[0].used) {
- _reset_the_path(curr_switch, 0, 1, dim);
- }
- }
- }
- list_iterator_destroy(itr);
- return 1;
- }
- /*
- * Used to set a block into a virtual system. The system can be
- * cleared first and this function sets all the wires and midplanes
- * used in the nodelist given. The nodelist is a list of ba_mp_t's
- * that are already set up. This is very handly to test if there are
- * any passthroughs used by one block when adding another block that
- * also uses those wires, and neither use any overlapping
- * midplanes. Doing a simple bitmap & will not reveal this.
- *
- * Returns SLURM_SUCCESS if nodelist fits into system without
- * conflict, and SLURM_ERROR if nodelist conflicts with something
- * already in the system.
- */
- extern int check_and_set_mp_list(List nodes)
- {
- int rc = SLURM_ERROR;
- #ifdef HAVE_BG_L_P
- int i, j;
- ba_switch_t *ba_switch = NULL, *curr_ba_switch = NULL;
- ba_mp_t *ba_node = NULL, *curr_ba_node = NULL;
- ListIterator itr = NULL;
- if (!nodes)
- return rc;
- itr = list_iterator_create(nodes);
- while ((ba_node = list_next(itr))) {
- /* info("checking %c%c%c", */
- /* ba_node->coord[X], */
- /* ba_node->coord[Y], */
- /* ba_node->coord[Z]); */
- curr_ba_node = &ba_main_grid[ba_node->coord[X]]
- [ba_node->coord[Y]]
- [ba_node->coord[Z]];
- if (ba_node->used && curr_ba_node->used) {
- /* Only error if the midplane isn't already
- * marked down or in a error state outside of
- * the bluegene block.
- */
- uint16_t base_state, node_flags;
- base_state = curr_ba_node->state & NODE_STATE_BASE;
- node_flags = curr_ba_node->state & NODE_STATE_FLAGS;
- if (!(node_flags & (NODE_STATE_DRAIN | NODE_STATE_FAIL))
- && (base_state != NODE_STATE_DOWN)) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("I have already been to "
- "this node %c%c%c %s",
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]],
- node_state_string(
- curr_ba_node->state));
- rc = SLURM_ERROR;
- goto end_it;
- }
- }
- if (ba_node->used)
- curr_ba_node->used = ba_node->used;
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("check_and_set_mp_list: "
- "%s is used ?= %d %d",
- curr_ba_node->coord_str,
- curr_ba_node->used, ba_node->used);
- for(i=0; i<cluster_dims; i++) {
- ba_switch = &ba_node->axis_switch[i];
- curr_ba_switch = &curr_ba_node->axis_switch[i];
- //info("checking dim %d", i);
- for(j=0; j<NUM_PORTS_PER_NODE; j++) {
- //info("checking port %d", j);
- if (ba_switch->int_wire[j].used
- && curr_ba_switch->int_wire[j].used
- && j != curr_ba_switch->
- int_wire[j].port_tar) {
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("%c%c%c dim %d port %d "
- "is already in use to %d",
- alpha_num[ba_node->
- coord[X]],
- alpha_num[ba_node->
- coord[Y]],
- alpha_num[ba_node->
- coord[Z]],
- i,
- j,
- curr_ba_switch->
- int_wire[j].port_tar);
- rc = SLURM_ERROR;
- goto end_it;
- }
- if (!ba_switch->int_wire[j].used)
- continue;
- /* info("setting %c%c%c dim %d port %d -> %d",*/
- /* alpha_num[ba_node->coord[X]], */
- /* alpha_num[ba_node->coord[Y]], */
- /* alpha_num[ba_node->coord[Z]], */
- /* i, */
- /* j, */
- /* ba_switch->int_wire[j].port_tar); */
- curr_ba_switch->int_wire[j].used = 1;
- curr_ba_switch->int_wire[j].port_tar
- = ba_switch->int_wire[j].port_tar;
- }
- }
- }
- rc = SLURM_SUCCESS;
- end_it:
- list_iterator_destroy(itr);
- #endif
- return rc;
- }
- /*
- * Used to find, and set up midplanes and the wires in the virtual
- * system and return them in List results
- *
- * IN/OUT results - a list with a NULL destroyer filled in with
- * midplanes and wires set to create the block with the api. If
- * only interested in the hostlist NULL can be excepted also.
- * IN ba_request - request for the block
- *
- * To be set in the ba_request
- * start - where to start the allocation. (optional)
- * geometry or size - the requested geometry of the block. (required)
- * conn_type - mesh, torus, or small. (required)
- *
- * RET char * - hostlist of midplanes results represent must be
- * xfreed. NULL on failure
- */
- extern char *set_bg_block(List results, select_ba_request_t* ba_request)
- {
- char *name = NULL;
- ba_mp_t* ba_node = NULL;
- int send_results = 0;
- int found = 0;
- xassert(ba_request);
- if (cluster_dims == 1) {
- if (ba_request->start[X]>=DIM_SIZE[X])
- return NULL;
- ba_request->size = ba_request->geometry[X];
- ba_node = &ba_main_grid[ba_request->start[X]][0][0];
- } else {
- int dim;
- ba_request->size = 1;
- for (dim=0; dim<cluster_dims; dim++) {
- if (ba_request->start[dim] >= DIM_SIZE[dim])
- return NULL;
- if ((int16_t)ba_request->geometry[dim] <= 0) {
- error("problem with geometry of %c in dim %d, "
- "needs to be at least 1",
- alpha_num[ba_request->geometry[dim]],
- dim);
- return NULL;
- }
- ba_request->size *= ba_request->geometry[dim];
- }
- ba_node = coord2ba_mp(ba_request->start);
- }
- if (!ba_node)
- return NULL;
- if (!results)
- results = list_create(NULL);
- else
- send_results = 1;
- /* This midplane should have already been checked if it was in
- use or not */
- list_append(results, ba_node);
- if (ba_request->conn_type[0] >= SELECT_SMALL) {
- /* adding the ba_node and ending */
- ba_node->used |= BA_MP_USED_TRUE;
- name = xstrdup_printf("%s", ba_node->coord_str);
- goto end_it;
- } else if (ba_request->conn_type[0] == SELECT_NAV)
- ba_request->conn_type[0] = bg_conf->default_conn_type[0];
- found = _find_x_path(results, ba_node,
- ba_node->coord,
- ba_request->geometry[X],
- 1,
- ba_request->conn_type[0], BLOCK_ALGO_FIRST);
- if (!found) {
- bool is_small = 0;
- if (ba_request->conn_type[0] == SELECT_SMALL)
- is_small = 1;
- debug2("trying less efficient code");
- remove_block(results, is_small);
- list_flush(results);
- list_append(results, ba_node);
- found = _find_x_path(results, ba_node,
- ba_node->coord,
- ba_request->geometry[X],
- 1,
- ba_request->conn_type[0],
- BLOCK_ALGO_SECOND);
- }
- if (found) {
- if (cluster_flags & CLUSTER_FLAG_BG) {
- List start_list = NULL;
- ListIterator itr;
- start_list = list_create(NULL);
- itr = list_iterator_create(results);
- while ((ba_node = (ba_mp_t*) list_next(itr))) {
- list_append(start_list, ba_node);
- }
- list_iterator_destroy(itr);
- if (!_fill_in_coords(results,
- start_list,
- ba_request->geometry,
- ba_request->conn_type[0])) {
- list_destroy(start_list);
- goto end_it;
- }
- list_destroy(start_list);
- }
- } else {
- goto end_it;
- }
- name = _set_internal_wires(results,
- ba_request->size,
- ba_request->conn_type[0]);
- end_it:
- if (!send_results && results) {
- list_destroy(results);
- results = NULL;
- }
- if (name!=NULL) {
- debug2("name = %s", name);
- } else {
- debug2("can't allocate");
- xfree(name);
- }
- return name;
- }
- /* Rotate a 3-D geometry array through its six permutations */
- extern void ba_rotate_geo(uint16_t *req_geometry, int rot_cnt)
- {
- uint16_t tmp;
- switch (rot_cnt) {
- case 0: /* ABC -> ACB */
- case 2: /* CAB -> CBA */
- case 4: /* BCA -> BAC */
- SWAP(req_geometry[Y], req_geometry[Z], tmp);
- break;
- case 1: /* ACB -> CAB */
- case 3: /* CBA -> BCA */
- case 5: /* BAC -> ABC */
- SWAP(req_geometry[X], req_geometry[Y], tmp);
- break;
- }
- }
- /********************* Local Functions *********************/
- /*
- * This function is here to check options for rotating and elongating
- * and set up the request based on the count of each option
- */
- static int _check_for_options(select_ba_request_t* ba_request)
- {
- int temp;
- int set=0;
- uint16_t *geo = NULL;
- ListIterator itr;
- if (ba_request->rotate) {
- rotate_again:
- debug2("Rotating! %d",ba_request->rotate_count);
- if (ba_request->rotate_count==(cluster_dims-1)) {
- temp=ba_request->geometry[X];
- ba_request->geometry[X]=ba_request->geometry[Z];
- ba_request->geometry[Z]=temp;
- ba_request->rotate_count++;
- set=1;
- } else if (ba_request->rotate_count<(cluster_dims*2)) {
- temp=ba_request->geometry[X];
- ba_request->geometry[X]=ba_request->geometry[Y];
- ba_request->geometry[Y]=ba_request->geometry[Z];
- ba_request->geometry[Z]=temp;
- ba_request->rotate_count++;
- set=1;
- } else
- ba_request->rotate = false;
- if (set) {
- if (ba_request->geometry[X]<=DIM_SIZE[X]
- && ba_request->geometry[Y]<=DIM_SIZE[Y]
- && ba_request->geometry[Z]<=DIM_SIZE[Z])
- return 1;
- else {
- set = 0;
- goto rotate_again;
- }
- }
- }
- if (ba_request->elongate) {
- elongate_again:
- debug2("Elongating! %d",ba_request->elongate_count);
- ba_request->rotate_count=0;
- ba_request->rotate = true;
- set = 0;
- itr = list_iterator_create(ba_request->elongate_geos);
- for(set=0; set<=ba_request->elongate_count; set++)
- geo = list_next(itr);
- list_iterator_destroy(itr);
- if (geo == NULL)
- return 0;
- ba_request->elongate_count++;
- ba_request->geometry[X] = geo[X];
- ba_request->geometry[Y] = geo[Y];
- ba_request->geometry[Z] = geo[Z];
- if (ba_request->geometry[X]<=DIM_SIZE[X]
- && ba_request->geometry[Y]<=DIM_SIZE[Y]
- && ba_request->geometry[Z]<=DIM_SIZE[Z]) {
- return 1;
- } else
- goto elongate_again;
- }
- return 0;
- }
- /*
- * grab all the geometries that we can get and append them to the list geos
- */
- static int _append_geo(uint16_t *geometry, List geos, int rotate)
- {
- ListIterator itr;
- uint16_t *geo_ptr = NULL;
- uint16_t *geo = NULL;
- int temp_geo;
- int i, j;
- if (rotate) {
- for (i = (cluster_dims - 1); i >= 0; i--) {
- for (j = 1; j <= i; j++) {
- if ((geometry[j-1] > geometry[j])
- && (geometry[j] <= DIM_SIZE[j-i])
- && (geometry[j-1] <= DIM_SIZE[j])) {
- temp_geo = geometry[j-1];
- geometry[j-1] = geometry[j];
- geometry[j] = temp_geo;
- }
- }
- }
- }
- itr = list_iterator_create(geos);
- while ((geo_ptr = list_next(itr)) != NULL) {
- if (geometry[X] == geo_ptr[X]
- && geometry[Y] == geo_ptr[Y]
- && geometry[Z] == geo_ptr[Z])
- break;
- }
- list_iterator_destroy(itr);
- if (geo_ptr == NULL) {
- geo = xmalloc(sizeof(int)*cluster_dims);
- geo[X] = geometry[X];
- geo[Y] = geometry[Y];
- geo[Z] = geometry[Z];
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("adding geo %c%c%c",
- alpha_num[geo[X]], alpha_num[geo[Y]],
- alpha_num[geo[Z]]);
- list_append(geos, geo);
- }
- return 1;
- }
- /*
- * Fill in the paths and extra midplanes we need for the block.
- * Basically copy the x path sent in with the start_list in each Y and
- * Z dimension filling in every midplane for the block and then
- * completing the Y and Z wiring, tying the whole block together.
- *
- * IN/OUT results - total list of midplanes after this function
- * returns successfully. Should be
- * an exact copy of the start_list at first.
- * IN start_list - exact copy of results at first, This should only be
- * a list of midplanes on the X dim. We will work off this and
- * the geometry to fill in this wiring for the X dim in all the
- * Y and Z coords.
- * IN geometry - What the block looks like
- * IN conn_type - Mesh or Torus
- *
- * RET: 0 on failure 1 on success
- */
- static int _fill_in_coords(List results, List start_list,
- uint16_t *geometry, int conn_type)
- {
- ba_mp_t *ba_node = NULL;
- ba_mp_t *check_node = NULL;
- int rc = 1;
- ListIterator itr = NULL;
- int y=0, z=0;
- ba_switch_t *curr_switch = NULL;
- ba_switch_t *next_switch = NULL;
- if (!start_list || !results)
- return 0;
- /* go through the start_list and add all the midplanes */
- itr = list_iterator_create(start_list);
- while ((check_node = (ba_mp_t*) list_next(itr))) {
- curr_switch = &check_node->axis_switch[X];
- for(y=0; y<geometry[Y]; y++) {
- if ((check_node->coord[Y]+y) >= DIM_SIZE[Y]) {
- rc = 0;
- goto failed;
- }
- for(z=0; z<geometry[Z]; z++) {
- if ((check_node->coord[Z]+z) >= DIM_SIZE[Z]) {
- rc = 0;
- goto failed;
- }
- ba_node = &ba_main_grid
- [check_node->coord[X]]
- [check_node->coord[Y]+y]
- [check_node->coord[Z]+z];
- if ((ba_node->coord[Y] == check_node->coord[Y])
- && (ba_node->coord[Z]
- == check_node->coord[Z]))
- continue;
- if (!_node_used(ba_node, geometry[X])) {
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("here Adding %c%c%c",
- alpha_num[ba_node->
- coord[X]],
- alpha_num[ba_node->
- coord[Y]],
- alpha_num[ba_node->
- coord[Z]]);
- list_append(results, ba_node);
- next_switch = &ba_node->axis_switch[X];
- /* since we are going off the
- * main system we can send NULL
- * here
- */
- _copy_the_path(NULL, curr_switch,
- next_switch,
- 0, X);
- } else {
- rc = 0;
- goto failed;
- }
- }
- }
- }
- list_iterator_destroy(itr);
- itr = list_iterator_create(start_list);
- check_node = (ba_mp_t*) list_next(itr);
- list_iterator_destroy(itr);
- itr = list_iterator_create(results);
- while ((ba_node = (ba_mp_t*) list_next(itr))) {
- if (!_find_yz_path(ba_node,
- check_node->coord,
- geometry,
- conn_type)){
- rc = 0;
- goto failed;
- }
- }
- if (deny_pass) {
- if ((*deny_pass & PASS_DENY_Y)
- && (*deny_pass & PASS_FOUND_Y)) {
- debug("We don't allow Y passthoughs");
- rc = 0;
- } else if ((*deny_pass & PASS_DENY_Z)
- && (*deny_pass & PASS_FOUND_Z)) {
- debug("We don't allow Z passthoughs");
- rc = 0;
- }
- }
- failed:
- list_iterator_destroy(itr);
- return rc;
- }
- /*
- * Copy a path through the wiring of a switch to another switch on a
- * starting port on a dimension.
- *
- * IN/OUT: nodes - Local list of midplanes you are keeping track of. If
- * you visit any new midplanes a copy from ba_main_grid
- * will be added to the list. If NULL the path will be
- * set in mark_switch of the main virtual system (ba_main_grid).
- * IN: curr_switch - The switch you want to copy the path of
- * IN/OUT: mark_switch - The switch you want to fill in. On success
- * this switch will contain a complete path from the curr_switch
- * starting from the source port.
- * IN: source - source port number (If calling for the first time
- * should be 0 since we are looking for 1 at the end)
- * IN: dim - Dimension XYZ
- *
- * RET: on success 1, on error 0
- */
- static int _copy_the_path(List nodes, ba_switch_t *curr_switch,
- ba_switch_t *mark_switch,
- int source, int dim)
- {
- uint16_t *mp_tar;
- uint16_t *mark_mp_tar;
- uint16_t *node_curr;
- uint16_t port_tar, port_tar1;
- ba_switch_t *next_switch = NULL;
- ba_switch_t *next_mark_switch = NULL;
- /* Copy the source used and port_tar */
- mark_switch->int_wire[source].used =
- curr_switch->int_wire[source].used;
- mark_switch->int_wire[source].port_tar =
- curr_switch->int_wire[source].port_tar;
- port_tar = curr_switch->int_wire[source].port_tar;
- /* Now to the same thing from the other end */
- mark_switch->int_wire[port_tar].used =
- curr_switch->int_wire[port_tar].used;
- mark_switch->int_wire[port_tar].port_tar =
- curr_switch->int_wire[port_tar].port_tar;
- port_tar1 = port_tar;
- /* follow the path */
- node_curr = curr_switch->ext_wire[0].mp_tar;
- mp_tar = curr_switch->ext_wire[port_tar].mp_tar;
- if (mark_switch->int_wire[source].used)
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("setting dim %d %c%c%c %d-> %c%c%c %d",
- dim,
- alpha_num[node_curr[X]],
- alpha_num[node_curr[Y]],
- alpha_num[node_curr[Z]],
- source,
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar);
- if (port_tar == 1) {
- /* found the end of the line */
- mark_switch->int_wire[1].used =
- curr_switch->int_wire[1].used;
- mark_switch->int_wire[1].port_tar =
- curr_switch->int_wire[1].port_tar;
- return 1;
- }
- mark_mp_tar = mark_switch->ext_wire[port_tar].mp_tar;
- port_tar = curr_switch->ext_wire[port_tar].port_tar;
- if (node_curr[X] == mp_tar[X]
- && node_curr[Y] == mp_tar[Y]
- && node_curr[Z] == mp_tar[Z]) {
- /* We are going to the same node! this should never
- happen */
- debug5("something bad happened!! "
- "we are on %c%c%c and are going to it "
- "from port %d - > %d",
- alpha_num[node_curr[X]],
- alpha_num[node_curr[Y]],
- alpha_num[node_curr[Z]],
- port_tar1, port_tar);
- return 0;
- }
- /* see what the next switch is going to be */
- next_switch = &ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]].
- axis_switch[dim];
- if (!nodes) {
- /* If no nodes then just get the next switch to fill
- in from the main system */
- next_mark_switch = &ba_main_grid[mark_mp_tar[X]]
- [mark_mp_tar[Y]]
- [mark_mp_tar[Z]]
- .axis_switch[dim];
- } else {
- ba_mp_t *ba_node = NULL;
- ListIterator itr = list_iterator_create(nodes);
- /* see if we have already been to this node */
- while ((ba_node = list_next(itr))) {
- if (ba_node->coord[X] == mark_mp_tar[X] &&
- ba_node->coord[Y] == mark_mp_tar[Y] &&
- ba_node->coord[Z] == mark_mp_tar[Z])
- break; /* we found it */
- }
- list_iterator_destroy(itr);
- if (!ba_node) {
- /* If node grab a copy and add it to the list */
- ba_node = ba_copy_mp(&ba_main_grid[mark_mp_tar[X]]
- [mark_mp_tar[Y]]
- [mark_mp_tar[Z]]);
- ba_setup_mp(ba_node, false, false);
- list_push(nodes, ba_node);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("haven't seen %c%c%c adding it",
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]]);
- }
- next_mark_switch = &ba_node->axis_switch[dim];
- }
- /* Keep going until we reach the end of the line */
- return _copy_the_path(nodes, next_switch, next_mark_switch,
- port_tar, dim);
- }
- static int _find_yz_path(ba_mp_t *ba_node, uint16_t *first,
- uint16_t *geometry, int conn_type)
- {
- ba_mp_t *next_node = NULL;
- uint16_t *mp_tar = NULL;
- ba_switch_t *dim_curr_switch = NULL;
- ba_switch_t *dim_next_switch = NULL;
- int i2;
- int count = 0;
- for(i2=1;i2<=2;i2++) {
- if (geometry[i2] > 1) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("%d node %c%c%c port 2 -> ",
- i2,
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]]);
- dim_curr_switch = &ba_node->axis_switch[i2];
- if (dim_curr_switch->int_wire[2].used) {
- debug5("returning here");
- return 0;
- }
- mp_tar = dim_curr_switch->ext_wire[2].mp_tar;
- next_node =
- &ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]];
- dim_next_switch = &next_node->axis_switch[i2];
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("%c%c%c port 5",
- alpha_num[next_node->coord[X]],
- alpha_num[next_node->coord[Y]],
- alpha_num[next_node->coord[Z]]);
- if (dim_next_switch->int_wire[5].used) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("returning here 2");
- return 0;
- }
- debug5("%d %d %d %d",i2, mp_tar[i2],
- first[i2], geometry[i2]);
- /* Here we need to see where we are in
- * reference to the geo of this dimension. If
- * we have not gotten the number we need in
- * the direction we just go to the next node
- * with 5 -> 1. If we have all the midplanes
- * we need then we go through and finish the
- * torus if needed
- */
- if (mp_tar[i2] < first[i2])
- count = mp_tar[i2]+(DIM_SIZE[i2]-first[i2]);
- else
- count = (mp_tar[i2]-first[i2]);
- if (count == geometry[i2]) {
- debug5("found end of me %c%c%c",
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- if (conn_type == SELECT_TORUS) {
- dim_curr_switch->int_wire[0].used = 1;
- dim_curr_switch->int_wire[0].port_tar
- = 2;
- dim_curr_switch->int_wire[2].used = 1;
- dim_curr_switch->int_wire[2].port_tar
- = 0;
- dim_curr_switch = dim_next_switch;
- if (deny_pass
- && (mp_tar[i2] != first[i2])) {
- if (i2 == 1)
- *deny_pass |=
- PASS_FOUND_Y;
- else
- *deny_pass |=
- PASS_FOUND_Z;
- }
- while (mp_tar[i2] != first[i2]) {
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("on dim %d at %d "
- "looking for %d",
- i2,
- mp_tar[i2],
- first[i2]);
- if (dim_curr_switch->
- int_wire[2].used) {
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("returning"
- " here 3");
- return 0;
- }
- dim_curr_switch->
- int_wire[2].used = 1;
- dim_curr_switch->
- int_wire[2].port_tar
- = 5;
- dim_curr_switch->
- int_wire[5].used
- = 1;
- dim_curr_switch->
- int_wire[5].
- port_tar = 2;
- mp_tar = dim_curr_switch->
- ext_wire[2].mp_tar;
- next_node = &ba_main_grid
- [mp_tar[X]]
- [mp_tar[Y]]
- [mp_tar[Z]];
- dim_curr_switch =
- &next_node->
- axis_switch[i2];
- }
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("back to first on dim %d "
- "at %d looking for %d",
- i2,
- mp_tar[i2],
- first[i2]);
- dim_curr_switch->
- int_wire[5].used = 1;
- dim_curr_switch->
- int_wire[5].port_tar
- = 1;
- dim_curr_switch->
- int_wire[1].used
- = 1;
- dim_curr_switch->
- int_wire[1].
- port_tar = 5;
- }
- } else if (count < geometry[i2]) {
- if (conn_type == SELECT_TORUS ||
- (conn_type == SELECT_MESH &&
- (mp_tar[i2] != first[i2]))) {
- dim_curr_switch->
- int_wire[0].used = 1;
- dim_curr_switch->
- int_wire[0].port_tar
- = 2;
- dim_curr_switch->
- int_wire[2].used
- = 1;
- dim_curr_switch->
- int_wire[2].
- port_tar = 0;
- dim_next_switch->int_wire[5].used
- = 1;
- dim_next_switch->
- int_wire[5].port_tar
- = 1;
- dim_next_switch->
- int_wire[1].used = 1;
- dim_next_switch->
- int_wire[1].port_tar
- = 5;
- }
- } else {
- error("We were only looking for %d "
- "in the %d dim, but now we have %d",
- geometry[i2], i2, count);
- return 0;
- }
- } else if ((geometry[i2] == 1) && (conn_type == SELECT_TORUS)) {
- /* FIX ME: This is put here because we got
- into a state where the Y dim was not being
- processed correctly. This will set up the
- 0 -> 1 port correctly. We should probably
- find out why this was happening in the
- first place though. A reproducer was to
- have
- MPs=[310x323] Type=TORUS
- MPs=[200x233] Type=TORUS
- MPs=[300x303] Type=TORUS
- MPs=[100x133] Type=TORUS
- MPs=[000x033] Type=TORUS
- MPs=[400x433] Type=TORUS
- and then add
- MPs=[330x333] Type=TORUS
- */
- dim_curr_switch = &ba_node->axis_switch[i2];
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("%d node %c%c%c port 0 -> 1",
- i2,
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]]);
- dim_curr_switch->int_wire[0].used = 1;
- dim_curr_switch->int_wire[0].port_tar = 1;
- dim_curr_switch->int_wire[1].used = 1;
- dim_curr_switch->int_wire[1].port_tar = 0;
- }
- }
- return 1;
- }
- #ifndef HAVE_BG_FILES
- /** */
- static int _emulate_ext_wiring(ba_mp_t ***grid)
- {
- int x;
- ba_mp_t *source = NULL, *target = NULL;
- if (cluster_dims == 1) {
- for(x=0;x<DIM_SIZE[X];x++) {
- source = &grid[x][0][0];
- if (x<(DIM_SIZE[X]-1))
- target = &grid[x+1][0][0];
- else
- target = &grid[0][0][0];
- _set_external_wires(X, x, source, target);
- }
- } else {
- int y,z;
- for(x=0;x<DIM_SIZE[X];x++) {
- for(y=0;y<DIM_SIZE[Y];y++) {
- for(z=0;z<DIM_SIZE[Z];z++) {
- source = &grid[x][y][z];
- if (x<(DIM_SIZE[X]-1)) {
- target = &grid[x+1][y][z];
- } else
- target = &grid[0][y][z];
- _set_external_wires(X, x, source,
- target);
- if (y<(DIM_SIZE[Y]-1))
- target = &grid[x][y+1][z];
- else
- target = &grid[x][0][z];
- _set_external_wires(Y, y, source,
- target);
- if (z<(DIM_SIZE[Z]-1))
- target = &grid[x][y][z+1];
- else
- target = &grid[x][y][0];
- _set_external_wires(Z, z, source,
- target);
- }
- }
- }
- }
- return 1;
- }
- #endif
- static int _reset_the_path(ba_switch_t *curr_switch, int source,
- int target, int dim)
- {
- uint16_t *mp_tar;
- uint16_t *node_curr;
- int port_tar, port_tar1;
- ba_switch_t *next_switch = NULL;
- if (source < 0 || source >= NUM_PORTS_PER_NODE) {
- fatal("source port was %d can only be 0->%d",
- source, NUM_PORTS_PER_NODE);
- }
- if (target < 0 || target >= NUM_PORTS_PER_NODE) {
- fatal("target port was %d can only be 0->%d",
- target, NUM_PORTS_PER_NODE);
- }
- /*set the switch to not be used */
- if (!curr_switch->int_wire[source].used) {
- /* This means something overlapping the removing block
- already cleared this, or the path just never was
- complete in the first place. */
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("I reached the end, the source isn't used");
- return 1;
- }
- curr_switch->int_wire[source].used = 0;
- port_tar = curr_switch->int_wire[source].port_tar;
- if (port_tar < 0 || port_tar >= NUM_PORTS_PER_NODE) {
- fatal("port_tar port was %d can only be 0->%d",
- source, NUM_PORTS_PER_NODE);
- return 1;
- }
- port_tar1 = port_tar;
- curr_switch->int_wire[source].port_tar = source;
- curr_switch->int_wire[port_tar].used = 0;
- curr_switch->int_wire[port_tar].port_tar = port_tar;
- if (port_tar == target) {
- return 1;
- }
- /* follow the path */
- node_curr = curr_switch->ext_wire[0].mp_tar;
- mp_tar = curr_switch->ext_wire[port_tar].mp_tar;
- port_tar = curr_switch->ext_wire[port_tar].port_tar;
- if (source == port_tar1) {
- debug("got this bad one %c%c%c %d %d -> %c%c%c %d",
- alpha_num[node_curr[X]],
- alpha_num[node_curr[Y]],
- alpha_num[node_curr[Z]],
- source,
- port_tar1,
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar);
- return 0;
- }
- debug5("from %c%c%c %d %d -> %c%c%c %d",
- alpha_num[node_curr[X]],
- alpha_num[node_curr[Y]],
- alpha_num[node_curr[Z]],
- source,
- port_tar1,
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar);
- if (node_curr[X] == mp_tar[X]
- && node_curr[Y] == mp_tar[Y]
- && node_curr[Z] == mp_tar[Z]) {
- debug5("%d something bad happened!!", dim);
- return 0;
- }
- next_switch =
- &ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]].axis_switch[dim];
- return _reset_the_path(next_switch, port_tar, target, dim);
- // return 1;
- }
- extern void ba_create_system()
- {
- int x,y,z, i = 0;
- if (ba_main_grid)
- ba_destroy_system();
- best_count=BEST_COUNT_INIT;
- ba_main_grid = (ba_mp_t***)
- xmalloc(sizeof(ba_mp_t**) * DIM_SIZE[X]);
- for (x=0; x<DIM_SIZE[X]; x++) {
- ba_main_grid[x] = (ba_mp_t**)
- xmalloc(sizeof(ba_mp_t*) * DIM_SIZE[Y]);
- for (y=0; y<DIM_SIZE[Y]; y++) {
- ba_main_grid[x][y] = (ba_mp_t*)
- xmalloc(sizeof(ba_mp_t)
- * DIM_SIZE[Z]);
- for (z=0; z<DIM_SIZE[Z]; z++){
- ba_mp_t *ba_mp = &ba_main_grid[x][y][z];
- ba_mp->coord[X] = x;
- ba_mp->coord[Y] = y;
- ba_mp->coord[Z] = z;
- snprintf(ba_mp->coord_str,
- sizeof(ba_mp->coord_str),
- "%c%c%c",
- alpha_num[ba_mp->coord[X]],
- alpha_num[ba_mp->coord[Y]],
- alpha_num[ba_mp->coord[Z]]);
- ba_setup_mp(ba_mp, true, false);
- ba_mp->state = NODE_STATE_IDLE;
- /* This might get changed
- later, but just incase set
- it up here.
- */
- ba_mp->index = i++;
- }
- }
- }
- if ((cluster_flags & CLUSTER_FLAG_BGL) ||
- (cluster_flags & CLUSTER_FLAG_BGP)) {
- init_wires();
- #ifndef HAVE_BG_FILES
- _emulate_ext_wiring(ba_main_grid);
- #endif
- }
- path = list_create(_delete_path_list);
- best_path = list_create(_delete_path_list);
- }
- /** */
- extern void ba_destroy_system(void)
- {
- int x, y;
- if (path) {
- list_destroy(path);
- path = NULL;
- }
- if (best_path) {
- list_destroy(best_path);
- best_path = NULL;
- }
- #ifdef HAVE_BG_FILES
- if (bg)
- bridge_free_bg(bg);
- #endif
- _mp_map_initialized = false;
- _wires_initialized = true;
- if (ba_main_grid) {
- for (x=0; x<DIM_SIZE[X]; x++) {
- for (y=0; y<DIM_SIZE[Y]; y++)
- xfree(ba_main_grid[x][y]);
- xfree(ba_main_grid[x]);
- }
- xfree(ba_main_grid);
- ba_main_grid = NULL;
- }
- }
- extern bool ba_sub_block_in_bitmap(select_jobinfo_t *jobinfo,
- bitstr_t *usable_bitmap, bool step)
- {
- /* This shouldn't be called. */
- xassert(0);
- return false;
- }
- extern int ba_sub_block_in_bitmap_clear(select_jobinfo_t *jobinfo,
- bitstr_t *usable_bitmap)
- {
- /* this doesn't do anything since above doesn't. */
- return SLURM_SUCCESS;
- }
- extern ba_mp_t *ba_sub_block_in_record(
- bg_record_t *bg_record, uint32_t *node_count,
- select_jobinfo_t *jobinfo)
- {
- /* This shouldn't be called. */
- xassert(0);
- return false;
- }
- extern int ba_sub_block_in_record_clear(
- bg_record_t *bg_record, struct step_record *step_ptr)
- {
- /* this doesn't do anything since above doesn't. */
- return SLURM_SUCCESS;
- }
- extern void ba_sync_job_to_block(bg_record_t *bg_record,
- struct job_record *job_ptr)
- {
- xassert(bg_record);
- xassert(job_ptr);
- bg_record->job_running = job_ptr->job_id;
- bg_record->job_ptr = job_ptr;
- }
- extern bitstr_t *ba_create_ba_mp_cnode_bitmap(bg_record_t *bg_record)
- {
- return NULL;
- }
- extern void ba_set_ionode_str(bg_record_t *bg_record)
- {
- char bitstring[BITSIZE];
- if (!bg_record->ionode_bitmap)
- return;
- bit_fmt(bitstring, BITSIZE, bg_record->ionode_bitmap);
- bg_record->ionode_str = xstrdup(bitstring);
- }
- extern struct job_record *ba_remove_job_in_block_job_list(
- bg_record_t *bg_record, struct job_record *in_job_ptr)
- {
- return NULL;
- }
- static void _delete_path_list(void *object)
- {
- ba_path_switch_t *path_switch = (ba_path_switch_t *)object;
- if (path_switch) {
- xfree(path_switch);
- }
- return;
- }
- /**
- * algorithm for finding match
- */
- static int _find_match(select_ba_request_t *ba_request, List results)
- {
- int x=0;
- ba_mp_t *ba_node = NULL;
- char *name=NULL;
- int startx = DIM_SIZE[X]-1;
- uint16_t *geo_ptr;
- if (!(cluster_flags & CLUSTER_FLAG_BG))
- return 0;
- /* set up the geo here */
- if (!(geo_ptr = list_peek(ba_request->elongate_geos)))
- return 0;
- ba_request->rotate_count=0;
- ba_request->elongate_count=1;
- ba_request->geometry[X] = geo_ptr[X];
- ba_request->geometry[Y] = geo_ptr[Y];
- ba_request->geometry[Z] = geo_ptr[Z];
- if (ba_request->geometry[X]>DIM_SIZE[X]
- || ba_request->geometry[Y]>DIM_SIZE[Y]
- || ba_request->geometry[Z]>DIM_SIZE[Z])
- if (!_check_for_options(ba_request))
- return 0;
- start_again:
- x=0;
- if (x == startx)
- x = startx-1;
- while (x!=startx) {
- x++;
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("finding %c%c%c try %d",
- alpha_num[ba_request->geometry[X]],
- alpha_num[ba_request->geometry[Y]],
- alpha_num[ba_request->geometry[Z]],
- x);
- new_node:
- ba_node = coord2ba_mp(ba_request->start);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("starting at %s", ba_node->coord_str);
- if (!_node_used(ba_node, ba_request->geometry[X])) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("trying this node %s %c%c%c %d",
- ba_node->coord_str,
- alpha_num[ba_request->geometry[X]],
- alpha_num[ba_request->geometry[Y]],
- alpha_num[ba_request->geometry[Z]],
- ba_request->conn_type[X]);
- name = set_bg_block(results, ba_request);
- if (name) {
- ba_request->save_name = xstrdup(name);
- xfree(name);
- return 1;
- }
- if (results) {
- bool is_small = 0;
- if (ba_request->conn_type[0] == SELECT_SMALL)
- is_small = 1;
- remove_block(results, is_small);
- list_flush(results);
- }
- if (ba_request->start_req)
- goto requested_end;
- //exit(0);
- debug2("trying something else");
- }
- if ((DIM_SIZE[Z] - ba_request->start[Z]-1)
- >= ba_request->geometry[Z])
- ba_request->start[Z]++;
- else {
- ba_request->start[Z] = 0;
- if ((DIM_SIZE[Y] - ba_request->start[Y]-1)
- >= ba_request->geometry[Y])
- ba_request->start[Y]++;
- else {
- ba_request->start[Y] = 0;
- if ((DIM_SIZE[X] - ba_request->start[X]-1)
- >= ba_request->geometry[X])
- ba_request->start[X]++;
- else {
- if (ba_request->size == 1)
- goto requested_end;
- if (!_check_for_options(ba_request))
- return 0;
- else {
- memset(ba_request->start, 0,
- sizeof(ba_request->
- start));
- goto start_again;
- }
- }
- }
- }
- goto new_node;
- }
- requested_end:
- debug2("1 can't allocate");
- return 0;
- }
- /*
- * Used to check if midplane is usable in the block we are creating
- *
- * IN: ba_node - node to check if is used
- * IN: x_size - How big is the block in the X dim used to see if the
- * wires are full hence making this midplane unusable.
- */
- static bool _node_used(ba_mp_t* ba_node, int x_size)
- {
- ba_switch_t* ba_switch = NULL;
- /* if we've used this node in another block already */
- if (!ba_node || ba_node->used) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("node %s used", ba_node->coord_str);
- return true;
- }
- /* Check If we've used this node's switches completely in another
- block already. Right now we are only needing to look at
- the X dim since it is the only one with extra wires. This
- can be set up to do all the dim's if in the future if it is
- needed. We only need to check this if we are planning on
- using more than 1 midplane in the block creation */
- if (x_size > 1) {
- /* get the switch of the X Dimension */
- ba_switch = &ba_node->axis_switch[X];
- /* If both of these ports are used then the node
- is in use since there are no more wires we
- can use since these can not connect to each
- other they must be connected to the other ports.
- */
- if (ba_switch->int_wire[3].used
- && ba_switch->int_wire[5].used) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("switch full in the X dim on node %s!",
- ba_node->coord_str);
- return true;
- }
- }
- return false;
- }
- static void _switch_config(ba_mp_t* source, ba_mp_t* target, int dim,
- int port_src, int port_tar)
- {
- ba_switch_t* config = NULL, *config_tar = NULL;
- int i;
- if (!source || !target)
- return;
- config = &source->axis_switch[dim];
- config_tar = &target->axis_switch[dim];
- for(i=0;i<cluster_dims;i++) {
- /* Set the coord of the source target node to the target */
- config->ext_wire[port_src].mp_tar[i] = target->coord[i];
- /* Set the coord of the target back to the source */
- config_tar->ext_wire[port_tar].mp_tar[i] = source->coord[i];
- }
- /* Set the port of the source target node to the target */
- config->ext_wire[port_src].port_tar = port_tar;
- /* Set the port of the target back to the source */
- config_tar->ext_wire[port_tar].port_tar = port_src;
- }
- static int _set_external_wires(int dim, int count, ba_mp_t* source,
- ba_mp_t* target)
- {
- #ifdef HAVE_BG_FILES
- #ifdef HAVE_BGL
- #define UNDER_POS 7
- #define NODE_LEN 5
- #define VAL_NAME_LEN 12
- #else
- #define UNDER_POS 9
- #define NODE_LEN 7
- #define VAL_NAME_LEN 16
- #endif
- int rc;
- int i;
- rm_wire_t *my_wire = NULL;
- rm_port_t *my_port = NULL;
- char *wire_id = NULL;
- int from_port, to_port;
- int wire_num;
- char from_node[NODE_LEN];
- char to_node[NODE_LEN];
- if (working_cluster_rec) {
- error("Can't do this cross-cluster");
- return -1;
- }
- if (!have_db2) {
- error("Can't access DB2 library, run from service node");
- return -1;
- }
- if (!bg) {
- if ((rc = bridge_get_bg(&bg)) != SLURM_SUCCESS) {
- error("bridge_get_BG(): %d", rc);
- return -1;
- }
- }
- if (bg == NULL)
- return -1;
- if ((rc = bridge_get_data(bg, RM_WireNum, &wire_num))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_BPNum): %d", rc);
- wire_num = 0;
- }
- /* find out system wires on each mp */
- for (i=0; i<wire_num; i++) {
- if (i) {
- if ((rc = bridge_get_data(bg, RM_NextWire, &my_wire))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_NextWire): %d", rc);
- break;
- }
- } else {
- if ((rc = bridge_get_data(bg, RM_FirstWire, &my_wire))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_FirstWire): %d", rc);
- break;
- }
- }
- if ((rc = bridge_get_data(my_wire, RM_WireID, &wire_id))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_FirstWire): %d", rc);
- break;
- }
- if (!wire_id) {
- error("No Wire ID was returned from database");
- continue;
- }
- if (wire_id[UNDER_POS] != '_')
- continue;
- switch(wire_id[0]) {
- case 'X':
- dim = X;
- break;
- case 'Y':
- dim = Y;
- break;
- case 'Z':
- dim = Z;
- break;
- }
- if (strlen(wire_id) < VAL_NAME_LEN) {
- error("Wire_id isn't correct %s",wire_id);
- continue;
- }
- memset(&from_node, 0, sizeof(from_node));
- memset(&to_node, 0, sizeof(to_node));
- strncpy(from_node, wire_id+2, NODE_LEN-1);
- strncpy(to_node, wire_id+UNDER_POS+1, NODE_LEN-1);
- free(wire_id);
- if ((rc = bridge_get_data(my_wire, RM_WireFromPort, &my_port))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_FirstWire): %d", rc);
- break;
- }
- if ((rc = bridge_get_data(my_port, RM_PortID, &from_port))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_PortID): %d", rc);
- break;
- }
- if ((rc = bridge_get_data(my_wire, RM_WireToPort, &my_port))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_WireToPort): %d", rc);
- break;
- }
- if ((rc = bridge_get_data(my_port, RM_PortID, &to_port))
- != SLURM_SUCCESS) {
- error("bridge_get_data(RM_PortID): %d", rc);
- break;
- }
- source = loc2ba_mp(from_node);
- if (!source) {
- error("1 loc2ba_mp: mpid %s not known", from_node);
- continue;
- }
- if (!validate_coord(source->coord))
- continue;
- target = loc2ba_mp(to_node);
- if (!target) {
- error("2 loc2ba_mp: mpid %s not known", to_node);
- continue;
- }
- if (!validate_coord(target->coord))
- continue;
- _switch_config(source, target, dim, from_port, to_port);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("dim %d from %s %d -> %s %d",
- dim,
- source->coord_str,
- from_port,
- target->coord_str,
- to_port);
- }
- #else
- _switch_config(source, source, dim, 0, 0);
- _switch_config(source, source, dim, 1, 1);
- if (dim!=X) {
- _switch_config(source, target, dim, 2, 5);
- _switch_config(source, source, dim, 3, 3);
- _switch_config(source, source, dim, 4, 4);
- return 1;
- }
- if (cluster_dims == 1) {
- if (count == 0)
- _switch_config(source, source, dim, 5, 5);
- else if (count < DIM_SIZE[X]-1)
- _switch_config(source, target, dim, 2, 5);
- else
- _switch_config(source, source, dim, 2, 2);
- _switch_config(source, source, dim, 3, 3);
- _switch_config(source, source, dim, 4, 4);
- return 1;
- }
- /* set up x */
- /* always 2->5 of next. If it is the last it will go to the first.*/
- _switch_config(source, target, dim, 2, 5);
- /* set up split x */
- if (DIM_SIZE[X] == 1) {
- } else if (DIM_SIZE[X] == 4) {
- switch(count) {
- case 0:
- case 3:
- /* 0 and 3rd Node */
- /* nothing */
- break;
- case 1:
- /* 1st Node */
- target = &ba_main_grid[0]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 0th */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 2:
- /* 2nd Node */
- target = &ba_main_grid[3]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 3rd and back */
- _switch_config(source, target, dim, 4, 3);
- _switch_config(source, target, dim, 3, 4);
- break;
- default:
- fatal("got %d for a count on a %d X-dim system",
- count, DIM_SIZE[X]);
- break;
- }
- } else if (DIM_SIZE[X] == 5) {
- /* 4 X dim fixes for wires */
- switch(count) {
- case 0:
- case 2:
- /* 0th and 2nd node */
- /* Only the 2-5 is used here
- so nothing else */
- break;
- case 1:
- /* 1st node */
- /* change target to 4th node */
- target = &ba_main_grid[4]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 4th */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 3:
- /* 3rd node */
- /* change target to 2th node */
- target = &ba_main_grid[2]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 2nd */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 4:
- /* 4th node */
- /* change target to 1st node */
- target = &ba_main_grid[1]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 1st */
- _switch_config(source, target, dim, 4, 3);
- break;
- default:
- fatal("got %d for a count on a %d X-dim system",
- count, DIM_SIZE[X]);
- break;
- }
- } else if (DIM_SIZE[X] == 8) {
- switch(count) {
- case 0:
- case 4:
- /* 0 and 4th Node */
- /* nothing */
- break;
- case 1:
- case 5:
- /* 1st Node */
- target = &ba_main_grid[count-1]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of previous */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 2:
- /* 2nd Node */
- target = &ba_main_grid[7]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of last */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 3:
- /* 3rd Node */
- target = &ba_main_grid[6]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 6th */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 6:
- /* 6th Node */
- target = &ba_main_grid[3]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 3rd */
- _switch_config(source, target, dim, 4, 3);
- break;
- case 7:
- /* 7th Node */
- target = &ba_main_grid[2]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 2nd */
- _switch_config(source, target, dim, 4, 3);
- break;
- default:
- fatal("got %d for a count on a %d X-dim system",
- count, DIM_SIZE[X]);
- break;
- }
- } else if (DIM_SIZE[X] == 9) {
- switch(count) {
- case 0:
- case 4:
- /* 0 and 4th Node */
- /* nothing */
- case 5:
- case 6:
- case 7:
- /*already handled below */
- break;
- case 1:
- /* 1st Node */
- target = &ba_main_grid[7]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 7th and back */
- _switch_config(source, target, dim, 4, 3);
- _switch_config(target, source, dim, 4, 3);
- break;
- case 2:
- /* 2nd Node */
- target = &ba_main_grid[6]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 6th and back */
- _switch_config(source, target, dim, 4, 3);
- _switch_config(target, source, dim, 4, 3);
- break;
- case 3:
- /* 3rd Node */
- target = &ba_main_grid[5]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 5th and back */
- _switch_config(source, target, dim, 4, 3);
- _switch_config(target, source, dim, 4, 3);
- break;
- case 8:
- /* 8th Node */
- target = &ba_main_grid[0]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of 0th */
- _switch_config(source, target, dim, 4, 3);
- break;
- default:
- fatal("got %d for a count on a %d X-dim system",
- count, DIM_SIZE[X]);
- break;
- }
- } else if (DIM_SIZE[X] == 13) {
- int temp_num = 0;
- switch(count) {
- case 0:
- case 6:
- /* 0 and 6th Node no split */
- case 8:
- case 9:
- case 10:
- case 11:
- case 12:
- /* already taken care of in the next case so
- * do nothing
- */
- break;
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- /* get the node count - 1 then subtract it
- * from 12 to get the new target and then go
- * from 4->3 and back again
- */
- temp_num = 12 - (count - 1);
- if (temp_num < 5)
- fatal("node %d shouldn't go to %d",
- count, temp_num);
- target = &ba_main_grid[temp_num]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 */
- _switch_config(source, target, dim, 4, 3);
- /* and back 4->3 */
- _switch_config(target, source, dim, 4, 3);
- break;
- case 7:
- /* 7th Node */
- target = &ba_main_grid[count-1]
- [source->coord[Y]]
- [source->coord[Z]];
- /* 4->3 of previous */
- _switch_config(source, target, dim, 4, 3);
- break;
- default:
- fatal("got %d for a count on a %d X-dim system",
- count, DIM_SIZE[X]);
- break;
- }
- } else {
- fatal("We don't have a config to do a BG system with %d "
- "in the X-dim.", DIM_SIZE[X]);
- }
- #endif
- return 1;
- }
- static char *_set_internal_wires(List nodes, int size, int conn_type)
- {
- ba_mp_t* ba_node[size+1];
- int count=0, i;
- uint16_t *start = NULL;
- uint16_t *end = NULL;
- char *name = NULL;
- ListIterator itr;
- hostlist_t hostlist;
- if (!nodes)
- return NULL;
- hostlist = hostlist_create(NULL);
- itr = list_iterator_create(nodes);
- while ((ba_node[count] = list_next(itr))) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("name = %s", ba_node[count]->coord_str);
- hostlist_push(hostlist, ba_node[count]->coord_str);
- count++;
- }
- list_iterator_destroy(itr);
- start = ba_node[0]->coord;
- end = ba_node[count-1]->coord;
- name = hostlist_ranged_string_xmalloc(hostlist);
- hostlist_destroy(hostlist);
- for (i=0;i<count;i++) {
- if (!ba_node[i]->used) {
- ba_node[i]->used |= BA_MP_USED_TRUE;
- } else {
- debug("No network connection to create "
- "bgblock containing %s", name);
- debug("Use smap to define bgblocks in "
- "bluegene.conf");
- xfree(name);
- return NULL;
- }
- }
- if (conn_type == SELECT_TORUS)
- for (i=0;i<count;i++) {
- _set_one_dim(start, end, ba_node[i]->coord);
- }
- return name;
- }
- /*
- * Used to find a complete path based on the conn_type for an x dim.
- * When starting to wire a block together this should be called first.
- *
- * IN/OUT: results - contains the number of midplanes we are
- * potentially going to use in the X dim.
- * IN: ba_node - current node we are looking at and have already added
- * to results.
- * IN: start - coordinates of the first midplane (so we know when when
- * to end with a torus)
- * IN: x_size - How many midplanes are we looking for in the X dim
- * IN: found - count of how many midplanes we have found in the x dim
- * IN: conn_type - MESH or TORUS
- * IN: algo - algorythm to try an allocation by
- *
- * RET: 0 on failure, 1 on success
- */
- static int _find_x_path(List results, ba_mp_t *ba_node,
- uint16_t *start, int x_size,
- int found, int conn_type, block_algo_t algo)
- {
- ba_switch_t *curr_switch = NULL;
- ba_switch_t *next_switch = NULL;
- int port_tar = 0;
- int source_port=0;
- int target_port=1;
- int broke = 0, not_first = 0;
- int ports_to_try[2] = {4, 2};
- uint16_t *mp_tar = NULL;
- int i = 0;
- ba_mp_t *next_node = NULL;
- ba_mp_t *check_node = NULL;
- /* int highest_phys_x = x_size - start[X]; */
- /* info("highest_phys_x is %d", highest_phys_x); */
- ListIterator itr = NULL;
- if (!ba_node || !results || !start)
- return 0;
- curr_switch = &ba_node->axis_switch[X];
- /* we don't need to go any further */
- if (x_size == 1) {
- /* Only set this if Torus since mesh doesn't have any
- * connections in this path */
- if (conn_type == SELECT_TORUS) {
- curr_switch->int_wire[source_port].used = 1;
- curr_switch->int_wire[source_port].port_tar =
- target_port;
- curr_switch->int_wire[target_port].used = 1;
- curr_switch->int_wire[target_port].port_tar =
- source_port;
- }
- return 1;
- }
- if (algo == BLOCK_ALGO_FIRST) {
- ports_to_try[0] = 4;
- ports_to_try[1] = 2;
- } else if (algo == BLOCK_ALGO_SECOND) {
- ports_to_try[0] = 2;
- ports_to_try[1] = 4;
- } else {
- error("Unknown algo %d", algo);
- return 0;
- }
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("Algo(%d) found - %d", algo, found);
- /* Check the 2 ports we can leave though in ports_to_try */
- for(i=0;i<2;i++) {
- /* info("trying port %d", ports_to_try[i]); */
- /* check to make sure it isn't used */
- if (!curr_switch->int_wire[ports_to_try[i]].used) {
- /* looking at the next node on the switch
- and it's port we are going to */
- mp_tar = curr_switch->
- ext_wire[ports_to_try[i]].mp_tar;
- port_tar = curr_switch->
- ext_wire[ports_to_try[i]].port_tar;
- /* info("%c%c%c port %d goes to %c%c%c port %d", */
- /* alpha_num[ba_node->coord[X]], */
- /* alpha_num[ba_node->coord[Y]], */
- /* alpha_num[ba_node->coord[Z]], */
- /* ports_to_try[i], */
- /* alpha_num[mp_tar[X]], */
- /* alpha_num[mp_tar[Y]], */
- /* alpha_num[mp_tar[Z]], */
- /* port_tar); */
- /* check to see if we are back at the start of the
- block */
- if ((mp_tar[X] == start[X]
- && mp_tar[Y] == start[Y]
- && mp_tar[Z] == start[Z])) {
- broke = 1;
- goto broke_it;
- }
- /* check to see if the port points to itself */
- if ((mp_tar[X] == ba_node->coord[X]
- && mp_tar[Y] == ba_node->coord[Y]
- && mp_tar[Z] == ba_node->coord[Z])) {
- continue;
- }
- /* check to see if I am going to a place I have
- already been before */
- itr = list_iterator_create(results);
- while ((next_node = list_next(itr))) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("Algo(%d) looking at %c%c%c "
- "and %c%c%c",
- algo,
- alpha_num[next_node->coord[X]],
- alpha_num[next_node->coord[Y]],
- alpha_num[next_node->coord[Z]],
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- if ((mp_tar[X] == next_node->coord[X]
- && mp_tar[Y] == next_node->coord[Y]
- && mp_tar[Z] == next_node->coord[Z])) {
- not_first = 1;
- break;
- }
- }
- list_iterator_destroy(itr);
- if (not_first && found < DIM_SIZE[X]) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) already been there "
- "before", algo);
- not_first = 0;
- continue;
- }
- not_first = 0;
- broke_it:
- next_node = &ba_main_grid[mp_tar[X]]
- [mp_tar[Y]]
- [mp_tar[Z]];
- next_switch = &next_node->axis_switch[X];
- if ((conn_type == SELECT_MESH) && (found == (x_size))) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) we found the end of "
- "the mesh", algo);
- return 1;
- }
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("Algo(%d) Broke = %d Found = %d "
- "x_size = %d",
- algo, broke, found, x_size);
- if (broke && (found == x_size)) {
- goto found_path;
- } else if (found == x_size) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) finishing the torus!",
- algo);
- if (deny_pass && (*deny_pass & PASS_DENY_X)) {
- info("we don't allow passthroughs 1");
- return 0;
- }
- if (best_path)
- list_flush(best_path);
- else
- best_path =
- list_create(_delete_path_list);
- if (path)
- list_flush(path);
- else
- path = list_create(_delete_path_list);
- _finish_torus(results,
- curr_switch, 0, X, 0, start);
- if (best_count < BEST_COUNT_INIT) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) Found a best "
- "path with %d steps.",
- algo, best_count);
- _set_best_path();
- return 1;
- } else {
- return 0;
- }
- } else if (broke) {
- broke = 0;
- continue;
- }
- if (!_node_used(next_node, x_size)) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) found %d looking at "
- "%c%c%c %d going to %c%c%c %d",
- algo,
- found,
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]],
- ports_to_try[i],
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar);
- itr = list_iterator_create(results);
- while ((check_node = list_next(itr))) {
- if ((mp_tar[X] == check_node->coord[X]
- && mp_tar[Y] ==
- check_node->coord[Y]
- && mp_tar[Z] ==
- check_node->coord[Z])) {
- break;
- }
- }
- list_iterator_destroy(itr);
- if (!check_node) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) add %c%c%c",
- algo,
- alpha_num[next_node->
- coord[X]],
- alpha_num[next_node->
- coord[Y]],
- alpha_num[next_node->
- coord[Z]]);
- list_append(results, next_node);
- } else {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) Hey this is "
- "already added %c%c%c",
- algo,
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- continue;
- }
- found++;
- /* look for the next closest midplane */
- if (!_find_x_path(results, next_node,
- start, x_size,
- found, conn_type, algo)) {
- _remove_node(results, next_node->coord);
- found--;
- continue;
- } else {
- found_path:
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) added node "
- "%c%c%c %d %d -> "
- "%c%c%c %d %d",
- algo,
- alpha_num[ba_node->
- coord[X]],
- alpha_num[ba_node->
- coord[Y]],
- alpha_num[ba_node->
- coord[Z]],
- source_port,
- ports_to_try[i],
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar,
- target_port);
- curr_switch->int_wire[source_port].used
- = 1;
- curr_switch->int_wire
- [source_port].port_tar
- = ports_to_try[i];
- curr_switch->int_wire
- [ports_to_try[i]].used = 1;
- curr_switch->int_wire
- [ports_to_try[i]].port_tar
- = source_port;
- next_switch->int_wire[port_tar].used
- = 1;
- next_switch->int_wire[port_tar].port_tar
- = target_port;
- next_switch->int_wire[target_port].used
- = 1;
- next_switch->int_wire
- [target_port].port_tar
- = port_tar;
- return 1;
- }
- }
- }
- }
- if (algo == BLOCK_ALGO_FIRST) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) couldn't find path", algo);
- return 0;
- } else if (algo == BLOCK_ALGO_SECOND) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) looking for the next free node "
- "starting at %c%c%c",
- algo,
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]]);
- if (best_path)
- list_flush(best_path);
- else
- best_path = list_create(_delete_path_list);
- if (path)
- list_flush(path);
- else
- path = list_create(_delete_path_list);
- _find_next_free_using_port_2(curr_switch, 0, results, X, 0);
- if (best_count < BEST_COUNT_INIT) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) yes found next free %d", algo,
- best_count);
- mp_tar = _set_best_path();
- if (deny_pass && (*deny_pass & PASS_DENY_X)
- && (*deny_pass & PASS_FOUND_X)) {
- debug("We don't allow X passthoughs.");
- return 0;
- }
- /* info("got here with %c%c%c", */
- /* alpha_num[mp_tar[X]], */
- /* alpha_num[mp_tar[Y]], */
- /* alpha_num[mp_tar[Z]]); */
- next_node = &ba_main_grid[mp_tar[X]]
- [mp_tar[Y]]
- [mp_tar[Z]];
- next_switch = &next_node->axis_switch[X];
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) found %d looking at %c%c%c "
- "going to %c%c%c %d",
- algo, found,
- alpha_num[ba_node->coord[X]],
- alpha_num[ba_node->coord[Y]],
- alpha_num[ba_node->coord[Z]],
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]],
- port_tar);
- list_append(results, next_node);
- found++;
- if (_find_x_path(results, next_node,
- start, x_size, found,
- conn_type, algo)) {
- return 1;
- } else {
- found--;
- _reset_the_path(curr_switch, 0, 1, X);
- _remove_node(results, next_node->coord);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) couldn't finish "
- "the path off this one", algo);
- }
- }
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("Algo(%d) couldn't find path", algo);
- return 0;
- }
- error("We got here meaning there is a bad algo, "
- "but this should never happen algo(%d)", algo);
- return 0;
- }
- static int _remove_node(List results, uint16_t *mp_tar)
- {
- ListIterator itr;
- ba_mp_t *ba_node = NULL;
- itr = list_iterator_create(results);
- while ((ba_node = (ba_mp_t*) list_next(itr))) {
- #ifdef HAVE_BG_L_P
- if (mp_tar[X] == ba_node->coord[X]
- && mp_tar[Y] == ba_node->coord[Y]
- && mp_tar[Z] == ba_node->coord[Z]) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("removing %c%c%c from list",
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- list_remove (itr);
- break;
- }
- #else
- if (mp_tar[X] == ba_node->coord[X]) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("removing %d from list",
- mp_tar[X]);
- list_remove (itr);
- break;
- }
- #endif
- }
- list_iterator_destroy(itr);
- return 1;
- }
- static int _find_next_free_using_port_2(ba_switch_t *curr_switch,
- int source_port,
- List nodes,
- int dim,
- int count)
- {
- ba_switch_t *next_switch = NULL;
- ba_path_switch_t *path_add =
- (ba_path_switch_t *) xmalloc(sizeof(ba_path_switch_t));
- ba_path_switch_t *path_switch = NULL;
- ba_path_switch_t *temp_switch = NULL;
- uint16_t port_tar;
- int target_port = 0;
- int port_to_try = 2;
- uint16_t *mp_tar= curr_switch->ext_wire[0].mp_tar;
- uint16_t *node_src = curr_switch->ext_wire[0].mp_tar;
- int used = 0;
- int broke = 0;
- ba_mp_t *ba_node = NULL;
- ListIterator itr;
- path_add->geometry[X] = node_src[X];
- #ifdef HAVE_3D
- path_add->geometry[Y] = node_src[Y];
- path_add->geometry[Z] = node_src[Z];
- #endif
- path_add->dim = dim;
- path_add->in = source_port;
- if (count>=best_count)
- goto return_0;
- itr = list_iterator_create(nodes);
- while ((ba_node = (ba_mp_t*) list_next(itr))) {
- if (mp_tar[X] == ba_node->coord[X]
- && mp_tar[Y] == ba_node->coord[Y]
- && mp_tar[Z] == ba_node->coord[Z]) {
- broke = 1;
- break;
- }
- }
- list_iterator_destroy(itr);
- if (!broke && count>0 &&
- !ba_main_grid[mp_tar[X]][mp_tar[Y]][mp_tar[Z]].used) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("this one not found %c%c%c",
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- broke = 0;
- if ((source_port%2))
- target_port=1;
- list_flush(best_path);
- path_add->out = target_port;
- list_push(path, path_add);
- itr = list_iterator_create(path);
- while ((path_switch = (ba_path_switch_t*) list_next(itr))){
- temp_switch = (ba_path_switch_t *)
- xmalloc(sizeof(ba_path_switch_t));
- temp_switch->geometry[X] = path_switch->geometry[X];
- temp_switch->geometry[Y] = path_switch->geometry[Y];
- temp_switch->geometry[Z] = path_switch->geometry[Z];
- temp_switch->dim = path_switch->dim;
- temp_switch->in = path_switch->in;
- temp_switch->out = path_switch->out;
- list_append(best_path, temp_switch);
- }
- list_iterator_destroy(itr);
- best_count = count;
- return 1;
- }
- used=0;
- if (!curr_switch->int_wire[port_to_try].used) {
- itr = list_iterator_create(path);
- while ((path_switch =
- (ba_path_switch_t*) list_next(itr))){
- if (((path_switch->geometry[X] == node_src[X])
- && (path_switch->geometry[Y] == node_src[Y])
- && (path_switch->geometry[Z] == mp_tar[Z]))) {
- if ( path_switch->out
- == port_to_try) {
- used = 1;
- break;
- }
- }
- }
- list_iterator_destroy(itr);
- /* check to see if wire 0 is used with this port */
- if (curr_switch->
- ext_wire[port_to_try].mp_tar[X]
- == curr_switch->ext_wire[0].mp_tar[X]
- && curr_switch->ext_wire[port_to_try].mp_tar[Y]
- == curr_switch->ext_wire[0].mp_tar[Y]
- && curr_switch->ext_wire[port_to_try].mp_tar[Z]
- == curr_switch->ext_wire[0].mp_tar[Z]) {
- used = 1;
- }
- if (!used) {
- port_tar = curr_switch->
- ext_wire[port_to_try].port_tar;
- mp_tar = curr_switch->
- ext_wire[port_to_try].mp_tar;
- next_switch = &ba_main_grid
- [mp_tar[X]][mp_tar[Y]][mp_tar[Z]]
- .axis_switch[X];
- count++;
- path_add->out = port_to_try;
- list_push(path, path_add);
- _find_next_free_using_port_2(next_switch,
- port_tar, nodes,
- dim, count);
- while ((temp_switch = list_pop(path)) != path_add){
- xfree(temp_switch);
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("something here 1");
- }
- }
- }
- return_0:
- xfree(path_add);
- return 0;
- }
- /*
- * Used to tie the end of the block to the start. best_path and path
- * should both be set up before calling this function.
- *
- * IN: curr_switch -
- * IN: source_port -
- * IN: dim -
- * IN: count -
- * IN: start -
- *
- * RET: 0 on failure, 1 on success
- *
- * Sets up global variable best_path, and best_count. On success
- * best_count will be >= BEST_COUNT_INIT you can call _set_best_path
- * to apply this path to the main system (ba_main_grid)
- */
- static int _finish_torus(List results,
- ba_switch_t *curr_switch, int source_port,
- int dim, int count, uint16_t *start)
- {
- ba_switch_t *next_switch = NULL;
- ba_path_switch_t *path_add = xmalloc(sizeof(ba_path_switch_t));
- ba_path_switch_t *path_switch = NULL;
- ba_path_switch_t *temp_switch = NULL;
- uint16_t port_tar;
- int target_port=0;
- int ports_to_try[2] = {3,5};
- uint16_t *mp_tar= curr_switch->ext_wire[0].mp_tar;
- uint16_t *node_src = curr_switch->ext_wire[0].mp_tar;
- int i;
- int used=0;
- ListIterator itr;
- path_add->geometry[X] = node_src[X];
- path_add->geometry[Y] = node_src[Y];
- path_add->geometry[Z] = node_src[Z];
- path_add->dim = dim;
- path_add->in = source_port;
- if (count>=best_count) {
- xfree(path_add);
- return 0;
- }
- if (mp_tar[X] == start[X]
- && mp_tar[Y] == start[Y]
- && mp_tar[Z] == start[Z]) {
- if ((source_port%2))
- target_port=1;
- if (!curr_switch->int_wire[target_port].used) {
- list_flush(best_path);
- path_add->out = target_port;
- list_push(path, path_add);
- itr = list_iterator_create(path);
- while ((path_switch = list_next(itr))) {
- temp_switch = xmalloc(sizeof(ba_path_switch_t));
- temp_switch->geometry[X] =
- path_switch->geometry[X];
- temp_switch->geometry[Y] =
- path_switch->geometry[Y];
- temp_switch->geometry[Z] =
- path_switch->geometry[Z];
- temp_switch->dim = path_switch->dim;
- temp_switch->in = path_switch->in;
- temp_switch->out = path_switch->out;
- list_append(best_path,temp_switch);
- }
- list_iterator_destroy(itr);
- best_count = count;
- return 1;
- }
- }
- if (source_port==0 || source_port==3 || source_port==5) {
- ports_to_try[0] = 4;
- ports_to_try[1] = 2;
- }
- for(i=0;i<2;i++) {
- used=0;
- if (!curr_switch->int_wire[ports_to_try[i]].used) {
- itr = list_iterator_create(path);
- while ((path_switch = list_next(itr))){
- if (((path_switch->geometry[X] == node_src[X])
- && (path_switch->geometry[Y]
- == node_src[Y])
- && (path_switch->geometry[Z]
- == mp_tar[Z]))) {
- if ( path_switch->out
- == ports_to_try[i]) {
- used = 1;
- break;
- }
- }
- }
- list_iterator_destroy(itr);
- /* check to see if wire 0 is used with this port */
- if ((curr_switch->
- ext_wire[ports_to_try[i]].mp_tar[X] ==
- curr_switch->ext_wire[0].mp_tar[X] &&
- curr_switch->
- ext_wire[ports_to_try[i]].mp_tar[Y] ==
- curr_switch->ext_wire[0].mp_tar[Y] &&
- curr_switch->
- ext_wire[ports_to_try[i]].mp_tar[Z] ==
- curr_switch->ext_wire[0].mp_tar[Z])) {
- continue;
- }
- if (!used) {
- ba_mp_t *next_node = NULL;
- port_tar = curr_switch->
- ext_wire[ports_to_try[i]].port_tar;
- mp_tar = curr_switch->
- ext_wire[ports_to_try[i]].mp_tar;
- /* Check to see if I am going to a place I have
- already been before, because even
- though we may be able to do this
- electrically this doesn't mean the
- under lying infrastructure will
- allow it. */
- itr = list_iterator_create(results);
- while ((next_node = list_next(itr))) {
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("finishing_torus: "
- "looking at %c%c%c "
- "and %c%c%c",
- alpha_num[next_node->
- coord[X]],
- alpha_num[next_node->
- coord[Y]],
- alpha_num[next_node->
- coord[Z]],
- alpha_num[mp_tar[X]],
- alpha_num[mp_tar[Y]],
- alpha_num[mp_tar[Z]]);
- if ((mp_tar[X] == next_node->coord[X])
- && (mp_tar[Y]
- == next_node->coord[Y])
- && (mp_tar[Z]
- == next_node->coord[Z])) {
- break;
- }
- }
- list_iterator_destroy(itr);
- if (next_node) {
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO)
- info("finishing_torus: "
- "Can't finish torus with "
- "%c%c%c we already were "
- "there.",
- alpha_num[next_node->
- coord[X]],
- alpha_num[next_node->
- coord[Y]],
- alpha_num[next_node->
- coord[Z]]);
- continue;
- }
- next_switch = &ba_main_grid
- [mp_tar[X]][mp_tar[Y]][mp_tar[Z]]
- .axis_switch[dim];
- count++;
- path_add->out = ports_to_try[i];
- list_push(path, path_add);
- _finish_torus(results, next_switch, port_tar,
- dim, count, start);
- while ((temp_switch = list_pop(path))
- != path_add){
- xfree(temp_switch);
- if (ba_debug_flags
- & DEBUG_FLAG_BG_ALGO_DEEP)
- info("something here 3");
- }
- }
- }
- }
- xfree(path_add);
- return 0;
- }
- /*
- * using best_path set up previously from _finish_torus or
- * _find_next_free_using_port_2. Will set up the path contained there
- * into the main virtual system. With will also set the passthrough
- * flag if there was a passthrough used.
- */
- static uint16_t *_set_best_path()
- {
- ListIterator itr;
- ba_path_switch_t *path_switch = NULL;
- ba_switch_t *curr_switch = NULL;
- uint16_t *geo = NULL;
- if (!best_path)
- return NULL;
- itr = list_iterator_create(best_path);
- while ((path_switch = (ba_path_switch_t*) list_next(itr))) {
- if (deny_pass && path_switch->in > 1 && path_switch->out > 1) {
- *deny_pass |= PASS_FOUND_X;
- debug2("got a passthrough in X");
- }
- if (ba_debug_flags & DEBUG_FLAG_BG_ALGO_DEEP)
- info("mapping %c%c%c %d->%d",
- alpha_num[path_switch->geometry[X]],
- alpha_num[path_switch->geometry[Y]],
- alpha_num[path_switch->geometry[Z]],
- path_switch->in, path_switch->out);
- if (!geo)
- geo = path_switch->geometry;
- curr_switch = &ba_main_grid
- [path_switch->geometry[X]]
- [path_switch->geometry[Y]]
- [path_switch->geometry[Z]].
- axis_switch[path_switch->dim];
- curr_switch->int_wire[path_switch->in].used = 1;
- curr_switch->int_wire[path_switch->in].port_tar =
- path_switch->out;
- curr_switch->int_wire[path_switch->out].used = 1;
- curr_switch->int_wire[path_switch->out].port_tar =
- path_switch->in;
- }
- list_iterator_destroy(itr);
- best_count=BEST_COUNT_INIT;
- return geo;
- }
- static int _set_one_dim(uint16_t *start, uint16_t *end, uint16_t *coord)
- {
- int dim;
- ba_switch_t *curr_switch = NULL;
- for(dim=0;dim<cluster_dims;dim++) {
- if (start[dim]==end[dim]) {
- curr_switch = &ba_main_grid
- [coord[X]][coord[Y]][coord[Z]].axis_switch[dim];
- if (!curr_switch->int_wire[0].used
- && !curr_switch->int_wire[1].used) {
- curr_switch->int_wire[0].used = 1;
- curr_switch->int_wire[0].port_tar = 1;
- curr_switch->int_wire[1].used = 1;
- curr_switch->int_wire[1].port_tar = 0;
- }
- }
- }
- return 1;
- }
- static void _destroy_geo(void *object)
- {
- uint16_t *geo_ptr = (uint16_t *)object;
- xfree(geo_ptr);
- }