libdap
Updated for version 3.17.0
|
00001 // D4StreamMarshaller.cc 00002 00003 // -*- mode: c++; c-basic-offset:4 -*- 00004 00005 // This file is part of libdap, A C++ implementation of the OPeNDAP Data 00006 // Access Protocol. 00007 00008 // Copyright (c) 2012 OPeNDAP, Inc. 00009 // Author: James Gallagher <jgallagher@opendap.org> 00010 // 00011 // This library is free software; you can redistribute it and/or 00012 // modify it under the terms of the GNU Lesser General Public 00013 // License as published by the Free Software Foundation; either 00014 // version 2.1 of the License, or (at your option) any later version. 00015 // 00016 // This library is distributed in the hope that it will be useful, 00017 // but WITHOUT ANY WARRANTY; without even the implied warranty of 00018 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 00019 // Lesser General Public License for more details. 00020 // 00021 // You should have received a copy of the GNU Lesser General Public 00022 // License along with this library; if not, write to the Free Software 00023 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 00024 // 00025 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112. 00026 00027 #include "config.h" 00028 00029 #include <byteswap.h> 00030 #include <cassert> 00031 00032 #include <iostream> 00033 #include <sstream> 00034 #include <iomanip> 00035 #include <limits> 00036 00037 //#define DODS_DEBUG 1 00038 00039 #include "D4StreamMarshaller.h" 00040 00041 #if USE_XDR_FOR_IEEE754_ENCODING 00042 #include "XDRUtils.h" 00043 #include "util.h" 00044 #endif 00045 00046 #include "debug.h" 00047 00048 using namespace std; 00049 00050 namespace libdap { 00051 00052 #if 0 00053 // We decided to use int64_t to represent sizes of both arrays and strings, 00054 // So this code is not used. jhrg 10/4/13 00055 00056 // From the Google protobuf library 00057 inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) { 00058 // Splitting into 32-bit pieces gives better performance on 32-bit 00059 // processors. 00060 uint32_t part0 = static_cast<uint32_t>(value ); 00061 uint32_t part1 = static_cast<uint32_t>(value >> 28); 00062 uint32_t part2 = static_cast<uint32_t>(value >> 56); 00063 00064 int size; 00065 00066 // Here we can't really optimize for small numbers, since the value is 00067 // split into three parts. Checking for numbers < 128, for instance, 00068 // would require three comparisons, since you'd have to make sure part1 00069 // and part2 are zero. However, if the caller is using 64-bit integers, 00070 // it is likely that they expect the numbers to often be very large, so 00071 // we probably don't want to optimize for small numbers anyway. Thus, 00072 // we end up with a hard coded binary search tree... 00073 if (part2 == 0) { 00074 if (part1 == 0) { 00075 if (part0 < (1 << 14)) { 00076 if (part0 < (1 << 7)) { 00077 size = 1; goto size1; 00078 } else { 00079 size = 2; goto size2; 00080 } 00081 } else { 00082 if (part0 < (1 << 21)) { 00083 size = 3; goto size3; 00084 } else { 00085 size = 4; goto size4; 00086 } 00087 } 00088 } else { 00089 if (part1 < (1 << 14)) { 00090 if (part1 < (1 << 7)) { 00091 size = 5; goto size5; 00092 } else { 00093 size = 6; goto size6; 00094 } 00095 } else { 00096 if (part1 < (1 << 21)) { 00097 size = 7; goto size7; 00098 } else { 00099 size = 8; goto size8; 00100 } 00101 } 00102 } 00103 } else { 00104 if (part2 < (1 << 7)) { 00105 size = 9; goto size9; 00106 } else { 00107 size = 10; goto size10; 00108 } 00109 } 00110 00111 // GOOGLE_LOG(FATAL) << "Can't get here."; 00112 00113 size10: target[9] = static_cast<uint8_t>((part2 >> 7) | 0x80); 00114 size9 : target[8] = static_cast<uint8_t>((part2 ) | 0x80); 00115 size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80); 00116 size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80); 00117 size6 : target[5] = static_cast<uint8_t>((part1 >> 7) | 0x80); 00118 size5 : target[4] = static_cast<uint8_t>((part1 ) | 0x80); 00119 size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80); 00120 size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80); 00121 size2 : target[1] = static_cast<uint8_t>((part0 >> 7) | 0x80); 00122 size1 : target[0] = static_cast<uint8_t>((part0 ) | 0x80); 00123 00124 target[size-1] &= 0x7F; 00125 return target + size; 00126 } 00127 #endif 00128 00129 #if USE_XDR_FOR_IEEE754_ENCODING 00130 00135 void D4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type) 00136 { 00137 dods_uint64 size = num * width; 00138 // char *buf = (char*)malloc(size); jhrg 7/23/13 00139 vector<char> buf(size); 00140 XDR xdr; 00141 xdrmem_create(&xdr, &buf[0], size, XDR_ENCODE); 00142 try { 00143 if(!xdr_array(&xdr, &val, (unsigned int *)&num, size, width, XDRUtils::xdr_coder(type))) 00144 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array"); 00145 00146 if (xdr_getpos(&xdr) != size) 00147 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array"); 00148 00149 // If this is a little-endian host, twiddle the bytes 00150 static bool twiddle_bytes = !is_host_big_endian(); 00151 if (twiddle_bytes) { 00152 if (width == 4) { 00153 dods_float32 *lbuf = reinterpret_cast<dods_float32*>(&buf[0]); 00154 while (num--) { 00155 dods_int32 *i = reinterpret_cast<dods_int32*>(lbuf++); 00156 *i = bswap_32(*i); 00157 } 00158 } 00159 else { // width == 8 00160 dods_float64 *lbuf = reinterpret_cast<dods_float64*>(&buf[0]); 00161 while (num--) { 00162 dods_int64 *i = reinterpret_cast<dods_int64*>(lbuf++); 00163 *i = bswap_64(*i); 00164 } 00165 } 00166 } 00167 00168 d_out.write(&buf[0], size); 00169 } 00170 catch (...) { 00171 xdr_destroy(&xdr); 00172 throw; 00173 } 00174 xdr_destroy(&xdr); 00175 } 00176 #endif 00177 00185 D4StreamMarshaller::D4StreamMarshaller(ostream &out, bool write_data) : 00186 d_out(out), d_write_data(write_data) 00187 { 00188 assert(sizeof(std::streamsize) >= sizeof(int64_t)); 00189 00190 #if USE_XDR_FOR_IEEE754_ENCODING 00191 // XDR is used if the call std::numeric_limits<double>::is_iec559() 00192 // returns false indicating that the compiler is not using IEEE 754. 00193 // If it is, we just write out the bytes. 00194 xdrmem_create(&d_scalar_sink, d_ieee754_buf, sizeof(dods_float64), XDR_ENCODE); 00195 #endif 00196 00197 // This will cause exceptions to be thrown on i/o errors. The exception 00198 // will be ostream::failure 00199 out.exceptions(ostream::failbit | ostream::badbit); 00200 } 00201 00202 D4StreamMarshaller::~D4StreamMarshaller() 00203 { 00204 #if USE_XDR_FOR_IEEE754_ENCODING 00205 xdr_destroy(&d_scalar_sink); 00206 #endif 00207 } 00208 00211 void D4StreamMarshaller::reset_checksum() 00212 { 00213 d_checksum.Reset(); 00214 } 00215 00226 string D4StreamMarshaller::get_checksum() 00227 { 00228 ostringstream oss; 00229 oss.setf(ios::hex, ios::basefield); 00230 oss << setfill('0') << setw(8) << d_checksum.GetCrc32(); 00231 00232 return oss.str(); 00233 } 00234 00241 void D4StreamMarshaller::put_checksum() 00242 { 00243 Crc32::checksum chk = d_checksum.GetCrc32(); 00244 d_out.write(reinterpret_cast<char*>(&chk), sizeof(Crc32::checksum)); 00245 } 00246 00251 void D4StreamMarshaller::checksum_update(const void *data, unsigned long len) 00252 { 00253 d_checksum.AddData(reinterpret_cast<const uint8_t*>(data), len); 00254 } 00255 00256 void D4StreamMarshaller::put_byte(dods_byte val) 00257 { 00258 checksum_update(&val, sizeof(dods_byte)); 00259 00260 if (d_write_data) { 00261 DBG( std::cerr << "put_byte: " << val << std::endl ); 00262 00263 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_byte)); 00264 } 00265 } 00266 00267 void D4StreamMarshaller::put_int8(dods_int8 val) 00268 { 00269 checksum_update(&val, sizeof(dods_int8)); 00270 00271 if (d_write_data) { 00272 DBG( std::cerr << "put_int8: " << val << std::endl ); 00273 00274 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int8)); 00275 } 00276 } 00277 00278 void D4StreamMarshaller::put_int16(dods_int16 val) 00279 { 00280 checksum_update(&val, sizeof(dods_int16)); 00281 00282 if (d_write_data) 00283 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int16)); 00284 } 00285 00286 void D4StreamMarshaller::put_int32(dods_int32 val) 00287 { 00288 checksum_update(&val, sizeof(dods_int32)); 00289 00290 if (d_write_data) 00291 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int32)); 00292 } 00293 00294 void D4StreamMarshaller::put_int64(dods_int64 val) 00295 { 00296 checksum_update(&val, sizeof(dods_int64)); 00297 00298 if (d_write_data) 00299 d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_int64)); 00300 } 00301 00302 void D4StreamMarshaller::put_float32(dods_float32 val) 00303 { 00304 #if !USE_XDR_FOR_IEEE754_ENCODING 00305 assert(std::numeric_limits<float>::is_iec559); 00306 00307 checksum_update(&val, sizeof(dods_float32)); 00308 00309 if (d_write_data) 00310 d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float32)); 00311 00312 #else 00313 // This code uses XDR to convert from a local representation to IEEE754; 00314 // The extra 'twiddle' operation makes the byte-order correct for this 00315 // host should it not be big-endian. Also note the assert() at the 00316 // start of the method. 00317 00318 if (d_write_data) { 00319 if (std::numeric_limits<float>::is_iec559 ) { 00320 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float32)); 00321 } 00322 else { 00323 if (!xdr_setpos(&d_scalar_sink, 0)) 00324 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable"); 00325 00326 if (!xdr_float(&d_scalar_sink, &val)) 00327 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable"); 00328 00329 if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float32)) 00330 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable"); 00331 00332 // If this is a little-endian host, twiddle the bytes 00333 static bool twiddle_bytes = !is_host_big_endian(); 00334 if (twiddle_bytes) { 00335 dods_int32 *i = reinterpret_cast<dods_int32*>(&d_ieee754_buf); 00336 *i = bswap_32(*i); 00337 } 00338 00339 d_out.write(d_ieee754_buf, sizeof(dods_float32)); 00340 } 00341 } 00342 #endif 00343 } 00344 00345 void D4StreamMarshaller::put_float64(dods_float64 val) 00346 { 00347 #if !USE_XDR_FOR_IEEE754_ENCODING 00348 assert(std::numeric_limits<double>::is_iec559); 00349 00350 checksum_update(&val, sizeof(dods_float64)); 00351 00352 if (d_write_data) 00353 d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float64)); 00354 00355 #else 00356 // See the comment above in put_float32() 00357 if (d_write_data) { 00358 if (std::numeric_limits<double>::is_iec559) 00359 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float64)); 00360 else { 00361 if (!xdr_setpos(&d_scalar_sink, 0)) 00362 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable"); 00363 00364 if (!xdr_double(&d_scalar_sink, &val)) 00365 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable"); 00366 00367 if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float64)) 00368 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable"); 00369 00370 // If this is a little-endian host, twiddle the bytes 00371 static bool twiddle_bytes = !is_host_big_endian(); 00372 if (twiddle_bytes) { 00373 dods_int64 *i = reinterpret_cast<dods_int64*>(&d_ieee754_buf); 00374 *i = bswap_64(*i); 00375 } 00376 00377 d_out.write(d_ieee754_buf, sizeof(dods_float64)); 00378 } 00379 } 00380 #endif 00381 } 00382 00383 void D4StreamMarshaller::put_uint16(dods_uint16 val) 00384 { 00385 checksum_update(&val, sizeof(dods_uint16)); 00386 00387 if (d_write_data) 00388 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint16)); 00389 } 00390 00391 void D4StreamMarshaller::put_uint32(dods_uint32 val) 00392 { 00393 checksum_update(&val, sizeof(dods_uint32)); 00394 00395 if (d_write_data) 00396 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint32)); 00397 } 00398 00399 void D4StreamMarshaller::put_uint64(dods_uint64 val) 00400 { 00401 checksum_update(&val, sizeof(dods_uint64)); 00402 00403 if (d_write_data) 00404 d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint64)); 00405 } 00406 00415 void D4StreamMarshaller::put_count(int64_t count) 00416 { 00417 d_out.write(reinterpret_cast<const char*>(&count), sizeof(int64_t)); 00418 } 00419 00420 void D4StreamMarshaller::put_str(const string &val) 00421 { 00422 checksum_update(val.c_str(), val.length()); 00423 00424 if (d_write_data) { 00425 int64_t len = val.length(); 00426 00427 d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t)); 00428 d_out.write(val.data(), val.length()); 00429 } 00430 } 00431 00432 void D4StreamMarshaller::put_url(const string &val) 00433 { 00434 put_str(val); 00435 } 00436 00437 void D4StreamMarshaller::put_opaque_dap4(const char *val, int64_t len) 00438 { 00439 assert(val); 00440 assert(len >= 0); 00441 00442 checksum_update(val, len); 00443 00444 if (d_write_data) { 00445 d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t)); 00446 d_out.write(val, len); 00447 } 00448 } 00449 00455 void D4StreamMarshaller::put_vector(char *val, int64_t num_bytes) 00456 { 00457 assert(val); 00458 assert(num_bytes >= 0); 00459 00460 checksum_update(val, num_bytes); 00461 00462 if (d_write_data) 00463 d_out.write(val, num_bytes); 00464 } 00465 00466 void D4StreamMarshaller::put_vector(char *val, int64_t num_elem, int elem_size) 00467 { 00468 assert(val); 00469 assert(num_elem >= 0); 00470 assert(elem_size > 0); 00471 00472 int64_t bytes; 00473 00474 switch (elem_size) { 00475 case 1: 00476 assert(!"Don't call this method for bytes, use put_vector(val, bytes) instead"); 00477 bytes = num_elem; 00478 break; 00479 case 2: 00480 // Don't bother testing the sign bit 00481 assert(!(num_elem & 0x4000000000000000)); // 0x 40 00 --> 0100 0000 00482 bytes = num_elem << 1; 00483 break; 00484 case 4: 00485 assert(!(num_elem & 0x6000000000000000)); // 0x 60 00 --> 0110 0000 00486 bytes = num_elem << 2; 00487 break; 00488 case 8: 00489 assert(!(num_elem & 0x7000000000000000)); // 0111 0000 00490 bytes = num_elem << 3; 00491 break; 00492 default: 00493 bytes = num_elem * elem_size; 00494 break; 00495 } 00496 00497 checksum_update(val, bytes); 00498 00499 if (d_write_data) 00500 d_out.write(val, bytes); 00501 } 00502 00512 void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem) 00513 { 00514 #if !USE_XDR_FOR_IEEE754_ENCODING 00515 00516 assert(std::numeric_limits<float>::is_iec559); 00517 assert(val); 00518 assert(num_elem >= 0); 00519 // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so 00520 // make sure that doesn't overflow a 63-bit integer (the max positive value in 00521 // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits 00522 // to test that num can be multiplied by 4. A 00523 assert(!(num_elem & 0xe000000000000000)); 00524 00525 num_elem = num_elem << 2; // num_elem is now the number of bytes 00526 00527 checksum_update(val, num_elem); 00528 00529 if (d_write_data) 00530 d_out.write(val, num_elem); 00531 00532 #else 00533 assert(val); 00534 assert(num_elem >= 0); 00535 // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so 00536 // make sure that doesn't overflow a 63-bit integer (the max positive value in 00537 // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits 00538 // to test that num can be multiplied by 4. A 00539 assert(!(num_elem & 0xe000000000000000)); 00540 00541 int64_t bytes = num_elem << 2; // num_elem is now the number of bytes 00542 00543 checksum_update(val, bytes); 00544 00545 if (d_write_data) { 00546 if (!std::numeric_limits<float>::is_iec559) { 00547 // If not using IEEE 754, use XDR to get it that way. 00548 m_serialize_reals(val, num_elem, 4, type); 00549 } 00550 else { 00551 d_out.write(val, bytes); 00552 } 00553 } 00554 #endif 00555 } 00556 00565 void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem) 00566 { 00567 #if !USE_XDR_FOR_IEEE754_ENCODING 00568 00569 assert(std::numeric_limits<double>::is_iec559); 00570 assert(val); 00571 assert(num_elem >= 0); 00572 // See comment above 00573 assert(!(num_elem & 0xf000000000000000)); 00574 00575 num_elem = num_elem << 3; // num_elem is now the number of bytes 00576 00577 checksum_update(val, num_elem); 00578 00579 if (d_write_data) 00580 d_out.write(val, num_elem); 00581 #else 00582 assert(val); 00583 assert(num_elem >= 0); 00584 // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so 00585 // make sure that doesn't overflow a 63-bit integer (the max positive value in 00586 // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits 00587 // to test that num can be multiplied by 4. A 00588 assert(!(num_elem & 0xe000000000000000)); 00589 00590 int64_t bytes = num_elem << 3; // num_elem is now the number of bytes 00591 00592 checksum_update(val, bytes); 00593 00594 if (d_write_data) { 00595 if (!std::numeric_limits<double>::is_iec559) { 00596 // If not using IEEE 754, use XDR to get it that way. 00597 m_serialize_reals(val, num_elem, 8, type); 00598 } 00599 else { 00600 d_out.write(val, bytes); 00601 } 00602 } 00603 #endif 00604 00605 } 00606 00607 void D4StreamMarshaller::put_vector_part(char *val, unsigned int num, int width, Type type) 00608 { 00609 assert(val); 00610 assert(num >= 0); 00611 assert(width > 0); 00612 00613 switch(type) { 00614 case dods_byte_c: 00615 case dods_char_c: 00616 case dods_int8_c: 00617 case dods_uint8_c: 00618 put_vector(val, num); 00619 break; 00620 00621 case dods_int16_c: 00622 case dods_uint16_c: 00623 case dods_int32_c: 00624 case dods_uint32_c: 00625 case dods_int64_c: 00626 case dods_uint64_c: 00627 put_vector(val, num, width); 00628 break; 00629 00630 case dods_enum_c: 00631 if (width == 1) 00632 put_vector(val, num); 00633 else 00634 put_vector(val, num, width); 00635 break; 00636 00637 case dods_float32_c: 00638 put_vector_float32(val, num); 00639 break; 00640 00641 case dods_float64_c: 00642 put_vector_float32(val, num); 00643 break; 00644 00645 case dods_str_c: 00646 case dods_url_c: 00647 throw InternalErr(__FILE__, __LINE__, "Array of String should not be passed to put_vector."); 00648 00649 case dods_array_c: 00650 throw InternalErr(__FILE__, __LINE__, "Array of Array not allowed."); 00651 00652 case dods_opaque_c: 00653 case dods_structure_c: 00654 case dods_sequence_c: 00655 throw InternalErr(__FILE__, __LINE__, "Array of String should not be passed to put_vector."); 00656 00657 case dods_grid_c: 00658 throw InternalErr(__FILE__, __LINE__, "Grid is not part of DAP4."); 00659 00660 default: 00661 throw InternalErr(__FILE__, __LINE__, "Unknown datatype."); 00662 break; 00663 } 00664 } 00665 00666 void D4StreamMarshaller::dump(ostream &strm) const 00667 { 00668 strm << DapIndent::LMarg << "D4StreamMarshaller::dump - (" << (void *) this << ")" << endl; 00669 } 00670 00671 } // namespace libdap 00672