escript
Revision_
|
00001 /***************************************************************************** 00002 * 00003 * Copyright (c) 2014 by University of Queensland 00004 * http://www.uq.edu.au 00005 * 00006 * Primary Business: Queensland, Australia 00007 * Licensed under the Open Software License version 3.0 00008 * http://www.opensource.org/licenses/osl-3.0.php 00009 * 00010 * Development until 2012 by Earth Systems Science Computational Center (ESSCC) 00011 * Development 2012-2013 by School of Earth Sciences 00012 * Development from 2014 by Centre for Geoscience Computing (GeoComp) 00013 * 00014 *****************************************************************************/ 00015 00016 00017 #include <vector> 00018 00019 00020 /* This file contains two main classes for dealing with a large 3D region which has been divided 00021 into a 3D Grid of Blocks (usually to be distributed). Each block is divided into 27 subblocks. 00022 The first and last subblocks in each dimension are cubes. 00023 00024 class Block mangages a single block. It has methods for copying between a flat array (storing 00025 all the values in the 3D block) and buffers storing individual subblocks. 00026 These buffers can be used to transfer individual subblocks to other blocks (using some external 00027 means). 00028 00029 class BlockGrid deals with the position of a given block in relation to the rest of the blocks 00030 in the grid. (In an MPI setting, there would be one block per rank.) 00031 It also required transfers of subblocks into and out of the block in order to make the whole 00032 (global) array consistant. 00033 00034 Each block has a region "inset" wide in from each edge which is shared with nieghbouring blocks. 00035 Where these regions overlap with another block, the block closest to block 0,0,0 are used. 00036 Or more precisely, values move left->right then lowy->highy and finally lowz -> highz. 00037 00038 Please don't mix external calls into this file, it may be useful to separate it for 00039 debugging purposes. 00040 00041 Types required: 00042 neighbourID_t - Stores the label of a neighbouring block. 00043 In an MPI setting, this will be the type used refer to ranks 00044 coord_t - Stores a position of a block in the grid of blocks (it could be 00045 within one dimension or overall in a flat structure. 00046 It is not (necessarily) the same as neighbourID_t because 00047 coord_t should be _unsigned_ and there is no guarantee 00048 that neighbourID_t will even be an integral type. 00049 */ 00050 00051 #ifdef ESYS_MPI 00052 #include <esysUtils/Esys_MPI.h> 00053 typedef Esys_MPI_rank neighbourID_t; // This should be the MPI_rank type 00054 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems 00055 00056 00057 #else 00058 00059 typedef int neighbourID_t; // This should be the MPI_rank type 00060 typedef unsigned coord_t; // if we ever get more than 2^32 ranks, we have other problems 00061 00062 #endif 00063 00064 00065 typedef std::pair<neighbourID_t, int> neighpair; 00066 typedef std::vector<neighpair> neighbourvector; 00067 00068 00069 00070 typedef struct 00071 { 00072 public: 00073 neighbourID_t sourceID; // ranks involved in communication 00074 neighbourID_t destID; 00075 int tag; 00076 unsigned char srcbuffid; // number of buffer to use for coms 00077 unsigned char destbuffid; 00078 } message; 00079 00080 typedef std::vector<message> messvec; 00081 00082 00083 00084 00085 class BlockGrid 00086 { 00087 public: 00088 BlockGrid(coord_t maxx, coord_t maxy, coord_t maxz); 00089 00090 neighbourID_t getNID(coord_t x, coord_t y, coord_t z) const; 00091 00092 00093 // generate all incoming com messages for this block. 00094 // for each subblock (27 of them), there may be an x, y, z direction to search in 00095 void generateInNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v); 00096 00097 00098 // generate all outgoing com messages for this block 00099 void generateOutNeighbours(coord_t blockx, coord_t blocky, coord_t blockz, messvec& v); 00100 private: 00101 coord_t xmax; 00102 coord_t ymax; 00103 coord_t zmax; 00104 }; 00105 00106 00107 00108 00109 /* Do not ask about buffers for sub-block 1,1,1 (also known as #13) 00110 They do not exist, such buffers woudl be: 00111 1) big 00112 2) unnecessary since the centre sub-block is not sent anywhere 00113 00114 Note that this class does not deal with data transfer between blocks 00115 Sub-blocks are copied to and from buffers. Other code is required to 00116 actually move the data. 00117 00118 "dpp" == doubles per point and gives the number of doubles that make up each "point" 00119 This is required to calculate offsets and buffer sizes. 00120 00121 */ 00122 class Block 00123 { 00124 public: 00125 00126 // s? specifiy the [local] size (in points) of each dimension 00127 Block(size_t sx, size_t sy, size_t sz, size_t inset, size_t xmidlen, 00128 size_t ymidlen, size_t zmidlen, unsigned int dpp=1); 00129 00130 ~Block(); 00131 00132 // Out buffers are loaded with the contents of the flat array and are 00133 // to be sent to other blocks 00134 double* getOutBuffer(unsigned char subx, unsigned char suby, unsigned char subz); 00135 double* getOutBuffer(unsigned char bid); 00136 00137 // In buffers are populated from external communications 00138 // and copied back to the flat array 00139 double* getInBuffer(unsigned char subx, unsigned char suby, unsigned char subz); 00140 double* getInBuffer(unsigned char bid); 00141 00142 // return number of doubles in the given block 00143 size_t getBuffSize(unsigned char subx, unsigned char suby, unsigned char subz); 00144 size_t getBuffSize(unsigned char bid); 00145 00146 // where does the subblock specified start in a source array 00147 size_t startOffset(unsigned char subx, unsigned char suby, unsigned char subz); 00148 00149 // debug only 00150 void displayBlock(unsigned char subx, unsigned char suby, unsigned char subz, bool out); 00151 00152 // Copy a 3d region from a flat array into a buffer 00153 void copyToBuffer(unsigned char buffid, double* src); 00154 00155 // Copy a 3d region from a buffer into a flat array 00156 void copyFromBuffer(unsigned char buffid, double* dest); 00157 00158 00159 void copyAllToBuffer(double* src); 00160 00161 void copyUsedFromBuffer(double* dest); 00162 00163 void setUsed(unsigned char buffid); 00164 00165 private: 00166 00167 // determines the dimensions of each subblock 00168 void populateDimsTable(); 00169 void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen); 00170 void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen, size_t zmidlen); 00171 00172 00173 00174 double* inbuff; 00175 double* outbuff; 00176 size_t buffoffsets[27]; // offsets of the various blocks within the buffer arrays 00177 size_t flatoffsets[27]; // starting point of each block within a flat array 00178 bool used[27]; 00179 size_t dims[27][3]; // dimension of each subblock 00180 size_t sx; 00181 size_t sy; 00182 size_t sz; 00183 size_t inset; 00184 size_t xmidlen; 00185 size_t ymidlen; 00186 size_t zmidlen; 00187 double* inbuffptr[27]; 00188 double* outbuffptr[27]; 00189 const unsigned int dpsize; // number of doubles which make up a point 00190 00191 00192 00193 }; 00194 00195 // Returns the MPI message tag to use for a transfer between the two subblocks 00196 int getTag(unsigned char sourcex, unsigned char sourcey, unsigned char sourcez, unsigned char targetx, unsigned char targety, unsigned char targetz); 00197 00198 // computes the tag based on the destination and the direction it comes from 00199 // the booleans indicate whether a negative shift in that direction is required 00200 int getTag(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz); 00201 00202 00203 // the booleans indicate whether a negative shift in that direction is required 00204 unsigned char getSrcBuffID(unsigned char destx, unsigned char desty, unsigned char destz, bool deltax, bool deltay, bool deltaz); 00205 00206 00207 /* Now the 2D versions */ 00208 00209 // 2D version 00210 class BlockGrid2 00211 { 00212 public: 00213 BlockGrid2(coord_t maxx, coord_t maxy); 00214 00215 neighbourID_t getNID(coord_t x, coord_t y) const; 00216 00217 00218 // generate all incoming com messages for this block. 00219 // for each subblock (9 of them), there may be an x, y direction to search in 00220 void generateInNeighbours(coord_t blockx, coord_t blocky, messvec& v); 00221 00222 00223 // generate all outgoing com messages for this block 00224 void generateOutNeighbours(coord_t blockx, coord_t blocky, messvec& v); 00225 private: 00226 coord_t xmax; 00227 coord_t ymax; 00228 }; 00229 00230 // The 2D version - there is no block 4 00231 class Block2 00232 { 00233 public: 00234 00235 // s? specifiy the [local] size (in points) of each dimension 00236 Block2(size_t sx, size_t sy, size_t inset, size_t xmidlen, 00237 size_t ymidlen, unsigned int dpp=1); 00238 00239 ~Block2(); 00240 00241 // Out buffers are loaded with the contents of the flat array and are 00242 // to be sent to other blocks 00243 double* getOutBuffer(unsigned char subx, unsigned char suby); 00244 double* getOutBuffer(unsigned char bid); 00245 00246 // In buffers are populated from external communications 00247 // and copied back to the flat array 00248 double* getInBuffer(unsigned char subx, unsigned char suby); 00249 double* getInBuffer(unsigned char bid); 00250 00251 // return number of doubles in the given block 00252 size_t getBuffSize(unsigned char subx, unsigned char suby); 00253 size_t getBuffSize(unsigned char bid); 00254 00255 // where does the subblock specified start in a source array 00256 size_t startOffset(unsigned char subx, unsigned char suby); 00257 00258 // debug only 00259 void displayBlock(unsigned char subx, unsigned char suby, bool out); 00260 00261 // Copy a 3d region from a flat array into a buffer 00262 void copyToBuffer(unsigned char buffid, double* src); 00263 00264 // Copy a 3d region from a buffer into a flat array 00265 void copyFromBuffer(unsigned char buffid, double* dest); 00266 00267 00268 void copyAllToBuffer(double* src); 00269 00270 void copyUsedFromBuffer(double* dest); 00271 00272 void setUsed(unsigned char buffid); 00273 00274 private: 00275 00276 // determines the dimensions of each subblock 00277 void populateDimsTable(); 00278 void populateOffsetTable(size_t inset, size_t xmidlen, size_t ymidlen); 00279 void createBuffArrays(double* startaddress, double* buffptr[27], size_t inset, size_t xmidlen, size_t ymidlen); 00280 00281 00282 00283 double* inbuff; 00284 double* outbuff; 00285 size_t buffoffsets[9]; // offsets of the various blocks within the buffer arrays 00286 size_t flatoffsets[9]; // starting point of each block within a flat array 00287 bool used[9]; 00288 size_t dims[9][2]; // dimension of each subblock 00289 size_t sx; 00290 size_t sy; 00291 size_t inset; 00292 size_t xmidlen; 00293 size_t ymidlen; 00294 double* inbuffptr[9]; 00295 double* outbuffptr[9]; 00296 const unsigned int dpsize; // number of doubles which make up a point 00297 }; 00298 00299 00300 // Returns the MPI message tag to use for a transfer between the two subblocks 00301 int getTag2(unsigned char sourcex, unsigned char sourcey, unsigned char targetx, unsigned char targety); 00302 00303 // computes the tag based on the destination and the direction it comes from 00304 // the booleans indicate whether a negative shift in that direction is required 00305 int getTag2(unsigned char destx, unsigned char desty, bool deltax, bool deltay); 00306 00307 00308 // the booleans indicate whether a negative shift in that direction is required 00309 unsigned char getSrcBuffID2(unsigned char destx, unsigned char desty, bool deltax, bool deltay); 00310