escript  Revision_
blocktools.h
Go to the documentation of this file.
00001 /*****************************************************************************
00002 *
00003 * Copyright (c) 2014 by University of Queensland
00004 * http://www.uq.edu.au
00005 *
00006 * Primary Business: Queensland, Australia
00007 * Licensed under the Open Software License version 3.0
00008 * http://www.opensource.org/licenses/osl-3.0.php
00009 *
00010 * Development until 2012 by Earth Systems Science Computational Center (ESSCC)
00011 * Development 2012-2013 by School of Earth Sciences
00012 * Development from 2014 by Centre for Geoscience Computing (GeoComp)
00013 *
00014 *****************************************************************************/
00015 
00016 
00017 #include <vector>
00018 
00019 
00020 /* This file contains two main classes for dealing with a large 3D region which has been divided 
00021  into a 3D Grid of Blocks (usually to be distributed). Each block is divided into 27 subblocks.
00022  The first and last subblocks in each dimension are cubes.
00023  
00024   class Block mangages a single block. It has methods for copying between a flat array (storing
00025  all the values in the 3D block) and buffers storing individual subblocks.
00026  These buffers can be used to transfer individual subblocks to other blocks (using some external 
00027  means).
00028    
00029   class BlockGrid deals with the position of a given block in relation to the rest of the blocks
00030  in the grid. (In an MPI setting, there would be one block per rank.) 
00031  It also required transfers of subblocks into and out of the block in order to make the whole
00032  (global) array consistant.
00033    
00034   Each block has a region "inset" wide in from each edge which is shared with nieghbouring blocks.
00035  Where these regions overlap with another block, the block closest to block 0,0,0 are used.
00036  Or more precisely, values move left->right then lowy->highy and finally lowz -> highz.
00037 
00038  Please don't mix external calls into this file, it may be useful to separate it for 
00039  debugging purposes.
00040 
00041  Types required:
00042       neighbourID_t - Stores the label of a neighbouring block.
00043               In an MPI setting, this will be the type used refer to ranks
00044       coord_t       - Stores a position of a block in the grid of blocks (it could be
00045               within one dimension or overall in a flat structure.
00046               It is not (necessarily) the same as neighbourID_t because
00047               coord_t should be _unsigned_ and there is no guarantee
00048               that neighbourID_t will even be an integral type.
00049 */
00050 
00051 #ifdef ESYS_MPI
00052 #include <esysUtils/Esys_MPI.h>
00053 typedef Esys_MPI_rank neighbourID_t;    // This should be the MPI_rank type
00054 typedef unsigned coord_t;       // if we ever get more than 2^32 ranks, we have other problems
00055 
00056 
00057 #else
00058 
00059 typedef int neighbourID_t;  // This should be the MPI_rank type
00060 typedef unsigned coord_t;       // if we ever get more than 2^32 ranks, we have other problems
00061 
00062 #endif
00063 
00064 
00065 typedef std::pair<neighbourID_t, int> neighpair;
00066 typedef std::vector<neighpair> neighbourvector;
00067 
00068 
00069 
00070 typedef struct 
00071 {
00072 public:
00073   neighbourID_t sourceID;   // ranks involved in communication
00074   neighbourID_t destID;
00075   int tag;
00076   unsigned char srcbuffid;      // number of buffer to use for coms
00077   unsigned char destbuffid;
00078 } message;
00079 
00080 typedef std::vector<message> messvec;
00081 
00082 
00083 
00084 
00085 class BlockGrid
00086 {
00087 public:  
00088     BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz);
00089 
00090     neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const;
00091 
00092 
00093 // generate all incoming com messages for this block.
00094 // for each subblock (27 of them), there may be an x, y, z direction to search in  
00095 void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
00096 
00097 
00098 // generate all outgoing com messages for this block
00099 void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v);
00100 private:
00101    coord_t xmax;
00102    coord_t ymax;
00103    coord_t zmax;  
00104 };
00105 
00106 
00107 
00108 
00109 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13)
00110  They do not exist, such buffers woudl be:
00111    1) big
00112    2) unnecessary since the centre sub-block is not sent anywhere
00113 
00114 Note that this class does not deal with data transfer between blocks
00115 Sub-blocks are copied to and from buffers. Other code is required to
00116 actually move the data.
00117 
00118 "dpp" == doubles per point and gives the number of doubles that make up each "point"
00119 This is required to calculate offsets and buffer sizes.
00120 
00121 */
00122 class Block
00123 {
00124 public: 
00125   
00126     // s? specifiy the [local] size (in points) of each dimension
00127     Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, 
00128       size_t ymidlen, size_t zmidlen, unsigned int dpp=1);
00129     
00130     ~Block();    
00131     
00132     // Out buffers are loaded with the contents of the flat array and are
00133     // to be sent to other blocks
00134     double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
00135     double* getOutBuffer(unsigned char bid);
00136 
00137     // In buffers are populated from external communications
00138     // and copied back to the flat array
00139     double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz);
00140     double* getInBuffer(unsigned char bid);
00141     
00142     // return number of doubles in the given block
00143     size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz);
00144     size_t getBuffSize(unsigned char bid);
00145       
00146     // where does the subblock specified start in a source array
00147     size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz);
00148 
00149     // debug only
00150     void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out);
00151     
00152     // Copy a 3d region from a flat array into a buffer
00153     void copyToBuffer(unsigned char buffid, double* src);
00154 
00155     // Copy a 3d region from a buffer into a flat array
00156     void copyFromBuffer(unsigned char buffid, double* dest);    
00157     
00158     
00159     void copyAllToBuffer(double* src);
00160     
00161     void copyUsedFromBuffer(double* dest);
00162 
00163     void setUsed(unsigned char buffid);    
00164   
00165 private:    
00166   
00167     // determines the dimensions of each subblock
00168     void populateDimsTable(); 
00169     void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);
00170     void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen);  
00171   
00172   
00173   
00174     double* inbuff;
00175     double* outbuff;
00176     size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays
00177     size_t flatoffsets[27]; // starting point of each block within a flat array
00178     bool used[27];
00179     size_t dims[27][3]; // dimension of each subblock 
00180     size_t sx;
00181     size_t sy;
00182     size_t sz;
00183     size_t inset;
00184     size_t xmidlen;
00185     size_t ymidlen;
00186     size_t zmidlen;    
00187     double* inbuffptr[27];
00188     double* outbuffptr[27];
00189     const unsigned int dpsize;  // number of doubles which make up a point
00190 
00191 
00192     
00193 };
00194 
00195 // Returns the MPI message tag to use for a transfer between the two subblocks
00196 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz);
00197 
00198 // computes the tag based on the destination and the direction it comes from
00199 // the booleans indicate whether a negative shift in that direction is required
00200 int getTag(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz);
00201 
00202 
00203 // the booleans indicate whether a negative shift in that direction is required
00204 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz);
00205 
00206 
00207 /* Now the 2D versions */
00208 
00209 // 2D version
00210 class BlockGrid2
00211 {
00212 public:  
00213     BlockGrid2(coord_t maxx, coord_t maxy);
00214 
00215     neighbourID_t getNID(coord_t x, coord_t y) const;
00216 
00217 
00218 // generate all incoming com messages for this block.
00219 // for each subblock (9 of them), there may be an x, y direction to search in  
00220 void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v);
00221 
00222 
00223 // generate all outgoing com messages for this block
00224 void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v);
00225 private:
00226    coord_t xmax;
00227    coord_t ymax;
00228 };
00229 
00230 // The 2D version - there is no block 4
00231 class Block2
00232 {
00233 public: 
00234   
00235     // s? specifiy the [local] size (in points) of each dimension
00236     Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, 
00237       size_t ymidlen, unsigned int dpp=1);
00238     
00239     ~Block2();    
00240     
00241     // Out buffers are loaded with the contents of the flat array and are
00242     // to be sent to other blocks
00243     double* getOutBuffer(unsigned char subx, unsigned char suby);
00244     double* getOutBuffer(unsigned char bid);
00245 
00246     // In buffers are populated from external communications
00247     // and copied back to the flat array
00248     double* getInBuffer(unsigned char subx, unsigned char suby);
00249     double* getInBuffer(unsigned char bid);
00250     
00251     // return number of doubles in the given block
00252     size_t getBuffSize(unsigned char subx, unsigned char suby);
00253     size_t getBuffSize(unsigned char bid);
00254       
00255     // where does the subblock specified start in a source array
00256     size_t startOffset(unsigned char subx, unsigned char suby);
00257 
00258     // debug only
00259     void displayBlock(unsigned char subx, unsigned char suby, bool out);
00260     
00261     // Copy a 3d region from a flat array into a buffer
00262     void copyToBuffer(unsigned char buffid, double* src);
00263 
00264     // Copy a 3d region from a buffer into a flat array
00265     void copyFromBuffer(unsigned char buffid, double* dest);    
00266     
00267     
00268     void copyAllToBuffer(double* src);
00269     
00270     void copyUsedFromBuffer(double* dest);
00271 
00272     void setUsed(unsigned char buffid);    
00273   
00274 private:    
00275   
00276     // determines the dimensions of each subblock
00277     void populateDimsTable(); 
00278     void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen);
00279     void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen);  
00280   
00281   
00282   
00283     double* inbuff;
00284     double* outbuff;
00285     size_t buffoffsets[9];  // offsets of the various blocks within the buffer arrays
00286     size_t flatoffsets[9];  // starting point of each block within a flat array
00287     bool used[9];
00288     size_t dims[9][2];  // dimension of each subblock 
00289     size_t sx;
00290     size_t sy;
00291     size_t inset;
00292     size_t xmidlen;
00293     size_t ymidlen;
00294     double* inbuffptr[9];
00295     double* outbuffptr[9];
00296     const unsigned int dpsize;  // number of doubles which make up a point
00297 };
00298 
00299 
00300 // Returns the MPI message tag to use for a transfer between the two subblocks
00301 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety);
00302 
00303 // computes the tag based on the destination and the direction it comes from
00304 // the booleans indicate whether a negative shift in that direction is required
00305 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
00306 
00307 
00308 // the booleans indicate whether a negative shift in that direction is required
00309 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay);
00310