libdap  Updated for version 3.17.0
D4StreamMarshaller.cc
00001 // D4StreamMarshaller.cc
00002 
00003 // -*- mode: c++; c-basic-offset:4 -*-
00004 
00005 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
00006 // Access Protocol.
00007 
00008 // Copyright (c) 2012 OPeNDAP, Inc.
00009 // Author: James Gallagher <jgallagher@opendap.org>
00010 //
00011 // This library is free software; you can redistribute it and/or
00012 // modify it under the terms of the GNU Lesser General Public
00013 // License as published by the Free Software Foundation; either
00014 // version 2.1 of the License, or (at your option) any later version.
00015 //
00016 // This library is distributed in the hope that it will be useful,
00017 // but WITHOUT ANY WARRANTY; without even the implied warranty of
00018 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
00019 // Lesser General Public License for more details.
00020 //
00021 // You should have received a copy of the GNU Lesser General Public
00022 // License along with this library; if not, write to the Free Software
00023 // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
00024 //
00025 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
00026 
00027 #include "config.h"
00028 
00029 #include <byteswap.h>
00030 #include <cassert>
00031 
00032 #include <iostream>
00033 #include <sstream>
00034 #include <iomanip>
00035 #include <limits>
00036 
00037 //#define DODS_DEBUG 1
00038 
00039 #include "D4StreamMarshaller.h"
00040 
00041 #if USE_XDR_FOR_IEEE754_ENCODING
00042 #include "XDRUtils.h"
00043 #include "util.h"
00044 #endif
00045 
00046 #include "debug.h"
00047 
00048 using namespace std;
00049 
00050 namespace libdap {
00051 
00052 #if 0
00053 // We decided to use int64_t to represent sizes of both arrays and strings,
00054 // So this code is not used. jhrg 10/4/13
00055 
00056 // From the Google protobuf library
00057 inline uint8_t* WriteVarint64ToArrayInline(uint64_t value, uint8_t* target) {
00058   // Splitting into 32-bit pieces gives better performance on 32-bit
00059   // processors.
00060   uint32_t part0 = static_cast<uint32_t>(value      );
00061   uint32_t part1 = static_cast<uint32_t>(value >> 28);
00062   uint32_t part2 = static_cast<uint32_t>(value >> 56);
00063 
00064   int size;
00065 
00066   // Here we can't really optimize for small numbers, since the value is
00067   // split into three parts.  Checking for numbers < 128, for instance,
00068   // would require three comparisons, since you'd have to make sure part1
00069   // and part2 are zero.  However, if the caller is using 64-bit integers,
00070   // it is likely that they expect the numbers to often be very large, so
00071   // we probably don't want to optimize for small numbers anyway.  Thus,
00072   // we end up with a hard coded binary search tree...
00073   if (part2 == 0) {
00074     if (part1 == 0) {
00075       if (part0 < (1 << 14)) {
00076         if (part0 < (1 << 7)) {
00077           size = 1; goto size1;
00078         } else {
00079           size = 2; goto size2;
00080         }
00081       } else {
00082         if (part0 < (1 << 21)) {
00083           size = 3; goto size3;
00084         } else {
00085           size = 4; goto size4;
00086         }
00087       }
00088     } else {
00089       if (part1 < (1 << 14)) {
00090         if (part1 < (1 << 7)) {
00091           size = 5; goto size5;
00092         } else {
00093           size = 6; goto size6;
00094         }
00095       } else {
00096         if (part1 < (1 << 21)) {
00097           size = 7; goto size7;
00098         } else {
00099           size = 8; goto size8;
00100         }
00101       }
00102     }
00103   } else {
00104     if (part2 < (1 << 7)) {
00105       size = 9; goto size9;
00106     } else {
00107       size = 10; goto size10;
00108     }
00109   }
00110 
00111   // GOOGLE_LOG(FATAL) << "Can't get here.";
00112 
00113   size10: target[9] = static_cast<uint8_t>((part2 >>  7) | 0x80);
00114   size9 : target[8] = static_cast<uint8_t>((part2      ) | 0x80);
00115   size8 : target[7] = static_cast<uint8_t>((part1 >> 21) | 0x80);
00116   size7 : target[6] = static_cast<uint8_t>((part1 >> 14) | 0x80);
00117   size6 : target[5] = static_cast<uint8_t>((part1 >>  7) | 0x80);
00118   size5 : target[4] = static_cast<uint8_t>((part1      ) | 0x80);
00119   size4 : target[3] = static_cast<uint8_t>((part0 >> 21) | 0x80);
00120   size3 : target[2] = static_cast<uint8_t>((part0 >> 14) | 0x80);
00121   size2 : target[1] = static_cast<uint8_t>((part0 >>  7) | 0x80);
00122   size1 : target[0] = static_cast<uint8_t>((part0      ) | 0x80);
00123 
00124   target[size-1] &= 0x7F;
00125   return target + size;
00126 }
00127 #endif
00128 
00129 #if USE_XDR_FOR_IEEE754_ENCODING
00130 
00135 void D4StreamMarshaller::m_serialize_reals(char *val, unsigned int num, int width, Type type)
00136 {
00137     dods_uint64 size = num * width;
00138     // char *buf = (char*)malloc(size); jhrg 7/23/13
00139     vector<char> buf(size);
00140     XDR xdr;
00141     xdrmem_create(&xdr, &buf[0], size, XDR_ENCODE);
00142     try {
00143         if(!xdr_array(&xdr, &val, (unsigned int *)&num, size, width, XDRUtils::xdr_coder(type)))
00144             throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
00145 
00146         if (xdr_getpos(&xdr) != size)
00147             throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 array");
00148 
00149         // If this is a little-endian host, twiddle the bytes
00150         static bool twiddle_bytes = !is_host_big_endian();
00151         if (twiddle_bytes) {
00152             if (width == 4) {
00153                 dods_float32 *lbuf = reinterpret_cast<dods_float32*>(&buf[0]);
00154                 while (num--) {
00155                     dods_int32 *i = reinterpret_cast<dods_int32*>(lbuf++);
00156                     *i = bswap_32(*i);
00157                 }
00158             }
00159             else { // width == 8
00160                 dods_float64 *lbuf = reinterpret_cast<dods_float64*>(&buf[0]);
00161                 while (num--) {
00162                     dods_int64 *i = reinterpret_cast<dods_int64*>(lbuf++);
00163                     *i = bswap_64(*i);
00164                 }
00165             }
00166         }
00167 
00168         d_out.write(&buf[0], size);
00169     }
00170     catch (...) {
00171         xdr_destroy(&xdr);
00172         throw;
00173     }
00174     xdr_destroy(&xdr);
00175 }
00176 #endif
00177 
00185 D4StreamMarshaller::D4StreamMarshaller(ostream &out, bool write_data) :
00186         d_out(out), d_write_data(write_data)
00187 {
00188         assert(sizeof(std::streamsize) >= sizeof(int64_t));
00189 
00190 #if USE_XDR_FOR_IEEE754_ENCODING
00191     // XDR is used if the call std::numeric_limits<double>::is_iec559()
00192     // returns false indicating that the compiler is not using IEEE 754.
00193     // If it is, we just write out the bytes.
00194     xdrmem_create(&d_scalar_sink, d_ieee754_buf, sizeof(dods_float64), XDR_ENCODE);
00195 #endif
00196 
00197     // This will cause exceptions to be thrown on i/o errors. The exception
00198     // will be ostream::failure
00199     out.exceptions(ostream::failbit | ostream::badbit);
00200 }
00201 
00202 D4StreamMarshaller::~D4StreamMarshaller()
00203 {
00204 #if USE_XDR_FOR_IEEE754_ENCODING
00205     xdr_destroy(&d_scalar_sink);
00206 #endif
00207 }
00208 
00211 void D4StreamMarshaller::reset_checksum()
00212 {
00213     d_checksum.Reset();
00214 }
00215 
00226 string D4StreamMarshaller::get_checksum()
00227 {
00228     ostringstream oss;
00229     oss.setf(ios::hex, ios::basefield);
00230     oss << setfill('0') << setw(8) << d_checksum.GetCrc32();
00231 
00232     return oss.str();
00233 }
00234 
00241 void D4StreamMarshaller::put_checksum()
00242 {
00243     Crc32::checksum chk = d_checksum.GetCrc32();
00244     d_out.write(reinterpret_cast<char*>(&chk), sizeof(Crc32::checksum));
00245 }
00246 
00251 void D4StreamMarshaller::checksum_update(const void *data, unsigned long len)
00252 {
00253     d_checksum.AddData(reinterpret_cast<const uint8_t*>(data), len);
00254 }
00255 
00256 void D4StreamMarshaller::put_byte(dods_byte val)
00257 {
00258     checksum_update(&val, sizeof(dods_byte));
00259 
00260     if (d_write_data) {
00261         DBG( std::cerr << "put_byte: " << val << std::endl );
00262 
00263         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_byte));
00264     }
00265 }
00266 
00267 void D4StreamMarshaller::put_int8(dods_int8 val)
00268 {
00269     checksum_update(&val, sizeof(dods_int8));
00270 
00271     if (d_write_data) {
00272         DBG( std::cerr << "put_int8: " << val << std::endl );
00273 
00274         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int8));
00275     }
00276 }
00277 
00278 void D4StreamMarshaller::put_int16(dods_int16 val)
00279 {
00280     checksum_update(&val, sizeof(dods_int16));
00281 
00282     if (d_write_data)
00283         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int16));
00284 }
00285 
00286 void D4StreamMarshaller::put_int32(dods_int32 val)
00287 {
00288     checksum_update(&val, sizeof(dods_int32));
00289 
00290     if (d_write_data)
00291         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_int32));
00292 }
00293 
00294 void D4StreamMarshaller::put_int64(dods_int64 val)
00295 {
00296     checksum_update(&val, sizeof(dods_int64));
00297 
00298     if (d_write_data)
00299         d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_int64));
00300 }
00301 
00302 void D4StreamMarshaller::put_float32(dods_float32 val)
00303 {
00304 #if !USE_XDR_FOR_IEEE754_ENCODING
00305         assert(std::numeric_limits<float>::is_iec559);
00306 
00307     checksum_update(&val, sizeof(dods_float32));
00308 
00309     if (d_write_data)
00310         d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float32));
00311 
00312 #else
00313     // This code uses XDR to convert from a local representation to IEEE754;
00314     // The extra 'twiddle' operation makes the byte-order correct for this
00315     // host should it not be big-endian. Also note the assert() at the
00316     // start of the method.
00317 
00318     if (d_write_data) {
00319         if (std::numeric_limits<float>::is_iec559 ) {
00320             d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float32));
00321         }
00322         else {
00323             if (!xdr_setpos(&d_scalar_sink, 0))
00324                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
00325 
00326             if (!xdr_float(&d_scalar_sink, &val))
00327                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
00328 
00329             if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float32))
00330                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float32 variable");
00331 
00332             // If this is a little-endian host, twiddle the bytes
00333             static bool twiddle_bytes = !is_host_big_endian();
00334             if (twiddle_bytes) {
00335                 dods_int32 *i = reinterpret_cast<dods_int32*>(&d_ieee754_buf);
00336                 *i = bswap_32(*i);
00337             }
00338 
00339             d_out.write(d_ieee754_buf, sizeof(dods_float32));
00340         }
00341     }
00342 #endif
00343 }
00344 
00345 void D4StreamMarshaller::put_float64(dods_float64 val)
00346 {
00347 #if !USE_XDR_FOR_IEEE754_ENCODING
00348         assert(std::numeric_limits<double>::is_iec559);
00349 
00350     checksum_update(&val, sizeof(dods_float64));
00351 
00352     if (d_write_data)
00353         d_out.write(reinterpret_cast<const char*>(&val), sizeof(dods_float64));
00354 
00355 #else
00356     // See the comment above in put_float32()
00357     if (d_write_data) {
00358         if (std::numeric_limits<double>::is_iec559)
00359             d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_float64));
00360         else {
00361             if (!xdr_setpos(&d_scalar_sink, 0))
00362                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
00363 
00364             if (!xdr_double(&d_scalar_sink, &val))
00365                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
00366 
00367             if (xdr_getpos(&d_scalar_sink) != sizeof(dods_float64))
00368                 throw InternalErr(__FILE__, __LINE__, "Error serializing a Float64 variable");
00369 
00370             // If this is a little-endian host, twiddle the bytes
00371             static bool twiddle_bytes = !is_host_big_endian();
00372             if (twiddle_bytes) {
00373                 dods_int64 *i = reinterpret_cast<dods_int64*>(&d_ieee754_buf);
00374                 *i = bswap_64(*i);
00375             }
00376 
00377             d_out.write(d_ieee754_buf, sizeof(dods_float64));
00378         }
00379     }
00380 #endif
00381 }
00382 
00383 void D4StreamMarshaller::put_uint16(dods_uint16 val)
00384 {
00385     checksum_update(&val, sizeof(dods_uint16));
00386 
00387     if (d_write_data)
00388         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint16));
00389 }
00390 
00391 void D4StreamMarshaller::put_uint32(dods_uint32 val)
00392 {
00393     checksum_update(&val, sizeof(dods_uint32));
00394 
00395     if (d_write_data)
00396         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint32));
00397 }
00398 
00399 void D4StreamMarshaller::put_uint64(dods_uint64 val)
00400 {
00401     checksum_update(&val, sizeof(dods_uint64));
00402 
00403     if (d_write_data)
00404         d_out.write(reinterpret_cast<char*>(&val), sizeof(dods_uint64));
00405 }
00406 
00415 void D4StreamMarshaller::put_count(int64_t count)
00416 {
00417         d_out.write(reinterpret_cast<const char*>(&count), sizeof(int64_t));
00418 }
00419 
00420 void D4StreamMarshaller::put_str(const string &val)
00421 {
00422     checksum_update(val.c_str(), val.length());
00423 
00424     if (d_write_data) {
00425         int64_t len = val.length();
00426 
00427         d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t));
00428         d_out.write(val.data(), val.length());
00429     }
00430 }
00431 
00432 void D4StreamMarshaller::put_url(const string &val)
00433 {
00434     put_str(val);
00435 }
00436 
00437 void D4StreamMarshaller::put_opaque_dap4(const char *val, int64_t len)
00438 {
00439     assert(val);
00440     assert(len >= 0);
00441 
00442     checksum_update(val, len);
00443 
00444     if (d_write_data) {
00445         d_out.write(reinterpret_cast<const char*>(&len), sizeof(int64_t));
00446         d_out.write(val, len);
00447     }
00448 }
00449 
00455 void D4StreamMarshaller::put_vector(char *val, int64_t num_bytes)
00456 {
00457     assert(val);
00458     assert(num_bytes >= 0);
00459 
00460     checksum_update(val, num_bytes);
00461 
00462     if (d_write_data)
00463         d_out.write(val, num_bytes);
00464 }
00465 
00466 void D4StreamMarshaller::put_vector(char *val, int64_t num_elem, int elem_size)
00467 {
00468         assert(val);
00469         assert(num_elem >= 0);
00470         assert(elem_size > 0);
00471 
00472         int64_t bytes;
00473 
00474         switch (elem_size) {
00475         case 1:
00476                 assert(!"Don't call this method for bytes, use put_vector(val, bytes) instead");
00477                 bytes = num_elem;
00478                 break;
00479         case 2:
00480                 // Don't bother testing the sign bit
00481                 assert(!(num_elem & 0x4000000000000000)); // 0x 40 00 --> 0100 0000
00482                 bytes = num_elem << 1;
00483                 break;
00484         case 4:
00485                 assert(!(num_elem & 0x6000000000000000)); // 0x 60 00 --> 0110 0000
00486                 bytes = num_elem << 2;
00487                 break;
00488         case 8:
00489                 assert(!(num_elem & 0x7000000000000000)); // 0111 0000
00490                 bytes = num_elem << 3;
00491                 break;
00492         default:
00493                 bytes = num_elem * elem_size;
00494                 break;
00495         }
00496 
00497     checksum_update(val, bytes);
00498 
00499     if (d_write_data)
00500         d_out.write(val, bytes);
00501 }
00502 
00512 void D4StreamMarshaller::put_vector_float32(char *val, int64_t num_elem)
00513 {
00514 #if !USE_XDR_FOR_IEEE754_ENCODING
00515 
00516         assert(std::numeric_limits<float>::is_iec559);
00517         assert(val);
00518         assert(num_elem >= 0);
00519         // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
00520         // make sure that doesn't overflow a 63-bit integer (the max positive value in
00521         // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
00522         // to test that num can be multiplied by 4. A
00523         assert(!(num_elem & 0xe000000000000000));
00524 
00525         num_elem = num_elem << 2;       // num_elem is now the number of bytes
00526 
00527     checksum_update(val, num_elem);
00528 
00529     if (d_write_data)
00530         d_out.write(val, num_elem);
00531 
00532 #else
00533         assert(val);
00534         assert(num_elem >= 0);
00535         // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
00536         // make sure that doesn't overflow a 63-bit integer (the max positive value in
00537         // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
00538         // to test that num can be multiplied by 4. A
00539         assert(!(num_elem & 0xe000000000000000));
00540 
00541         int64_t bytes = num_elem << 2;  // num_elem is now the number of bytes
00542 
00543     checksum_update(val, bytes);
00544 
00545     if (d_write_data) {
00546         if (!std::numeric_limits<float>::is_iec559) {
00547             // If not using IEEE 754, use XDR to get it that way.
00548             m_serialize_reals(val, num_elem, 4, type);
00549         }
00550         else {
00551             d_out.write(val, bytes);
00552         }
00553     }
00554 #endif
00555 }
00556 
00565 void D4StreamMarshaller::put_vector_float64(char *val, int64_t num_elem)
00566 {
00567 #if !USE_XDR_FOR_IEEE754_ENCODING
00568 
00569         assert(std::numeric_limits<double>::is_iec559);
00570         assert(val);
00571         assert(num_elem >= 0);
00572         // See comment above
00573         assert(!(num_elem & 0xf000000000000000));
00574 
00575         num_elem = num_elem << 3;       // num_elem is now the number of bytes
00576 
00577     checksum_update(val, num_elem);
00578 
00579     if (d_write_data)
00580         d_out.write(val, num_elem);
00581 #else
00582         assert(val);
00583         assert(num_elem >= 0);
00584         // sizeof() a 32-bit float is 4, so we're going to send 4 * num_elem bytes, so
00585         // make sure that doesn't overflow a 63-bit integer (the max positive value in
00586         // a signed int64; use 1110 0000 0.. (0xe000 ...) to mask for non-zero bits
00587         // to test that num can be multiplied by 4. A
00588         assert(!(num_elem & 0xe000000000000000));
00589 
00590         int64_t bytes = num_elem << 3;  // num_elem is now the number of bytes
00591 
00592     checksum_update(val, bytes);
00593 
00594     if (d_write_data) {
00595         if (!std::numeric_limits<double>::is_iec559) {
00596             // If not using IEEE 754, use XDR to get it that way.
00597             m_serialize_reals(val, num_elem, 8, type);
00598         }
00599         else {
00600             d_out.write(val, bytes);
00601         }
00602     }
00603 #endif
00604 
00605 }
00606 
00607 void D4StreamMarshaller::put_vector_part(char *val, unsigned int num, int width, Type type)
00608 {
00609     assert(val);
00610     assert(num >= 0);
00611     assert(width > 0);
00612 
00613     switch(type) {
00614     case dods_byte_c:
00615     case dods_char_c:
00616     case dods_int8_c:
00617     case dods_uint8_c:
00618         put_vector(val, num);
00619         break;
00620 
00621     case dods_int16_c:
00622     case dods_uint16_c:
00623     case dods_int32_c:
00624     case dods_uint32_c:
00625     case dods_int64_c:
00626     case dods_uint64_c:
00627         put_vector(val, num, width);
00628         break;
00629 
00630     case dods_enum_c:
00631         if (width == 1)
00632             put_vector(val, num);
00633         else
00634             put_vector(val, num, width);
00635         break;
00636 
00637     case dods_float32_c:
00638         put_vector_float32(val, num);
00639         break;
00640 
00641     case dods_float64_c:
00642         put_vector_float32(val, num);
00643         break;
00644 
00645     case dods_str_c:
00646     case dods_url_c:
00647         throw InternalErr(__FILE__, __LINE__, "Array of String should not be passed to put_vector.");
00648 
00649     case dods_array_c:
00650         throw InternalErr(__FILE__, __LINE__, "Array of Array not allowed.");
00651 
00652     case dods_opaque_c:
00653     case dods_structure_c:
00654     case dods_sequence_c:
00655         throw InternalErr(__FILE__, __LINE__, "Array of String should not be passed to put_vector.");
00656 
00657     case dods_grid_c:
00658         throw InternalErr(__FILE__, __LINE__, "Grid is not part of DAP4.");
00659 
00660     default:
00661         throw InternalErr(__FILE__, __LINE__, "Unknown datatype.");
00662         break;
00663     }
00664 }
00665 
00666 void D4StreamMarshaller::dump(ostream &strm) const
00667 {
00668     strm << DapIndent::LMarg << "D4StreamMarshaller::dump - (" << (void *) this << ")" << endl;
00669 }
00670 
00671 } // namespace libdap
00672