SHOGUN
v3.2.0
|
00001 /* 00002 * This program is free software; you can redistribute it and/or modify 00003 * it under the terms of the GNU General Public License as published by 00004 * the Free Software Foundation; either version 3 of the License, or 00005 * (at your option) any later version. 00006 * 00007 * Written (W) 2011 Shashwat Lal Das 00008 * Copyright (C) 2011 Berlin Institute of Technology and Max-Planck-Society 00009 */ 00010 00011 #include <shogun/lib/memory.h> 00012 #include <shogun/io/streaming/StreamingFile.h> 00013 00014 #include <ctype.h> 00015 00016 namespace shogun 00017 { 00029 /* For dense vectors */ 00030 #define GET_VECTOR(fname, conv, sg_type) \ 00031 void CStreamingFile::get_vector \ 00032 (sg_type*& vector, int32_t& num_feat) \ 00033 { \ 00034 vector=NULL; \ 00035 num_feat=-1; \ 00036 SG_ERROR("Read function not supported by the feature type!") \ 00037 } 00038 00039 GET_VECTOR(get_bool_vector, atoi, bool) 00040 GET_VECTOR(get_byte_vector, atoi, uint8_t) 00041 GET_VECTOR(get_char_vector, atoi, char) 00042 GET_VECTOR(get_int_vector, atoi, int32_t) 00043 GET_VECTOR(get_shortreal_vector, atof, float32_t) 00044 GET_VECTOR(get_real_vector, atof, float64_t) 00045 GET_VECTOR(get_short_vector, atoi, int16_t) 00046 GET_VECTOR(get_word_vector, atoi, uint16_t) 00047 GET_VECTOR(get_int8_vector, atoi, int8_t) 00048 GET_VECTOR(get_uint_vector, atoi, uint32_t) 00049 GET_VECTOR(get_long_vector, atoi, int64_t) 00050 GET_VECTOR(get_ulong_vector, atoi, uint64_t) 00051 GET_VECTOR(get_longreal_vector, atoi, floatmax_t) 00052 #undef GET_VECTOR 00053 00054 /* For dense vectors with labels */ 00055 #define GET_VECTOR_AND_LABEL(fname, conv, sg_type) \ 00056 void CStreamingFile::get_vector_and_label \ 00057 (sg_type*& vector, int32_t& num_feat, float64_t& label) \ 00058 { \ 00059 vector=NULL; \ 00060 num_feat=-1; \ 00061 SG_ERROR("Read function not supported by the feature type!") \ 00062 } 00063 00064 GET_VECTOR_AND_LABEL(get_bool_vector_and_label, str_to_bool, bool) 00065 GET_VECTOR_AND_LABEL(get_byte_vector_and_label, atoi, uint8_t) 00066 GET_VECTOR_AND_LABEL(get_char_vector_and_label, atoi, char) 00067 GET_VECTOR_AND_LABEL(get_int_vector_and_label, atoi, int32_t) 00068 GET_VECTOR_AND_LABEL(get_shortreal_vector_and_label, atof, float32_t) 00069 GET_VECTOR_AND_LABEL(get_real_vector_and_label, atof, float64_t) 00070 GET_VECTOR_AND_LABEL(get_short_vector_and_label, atoi, int16_t) 00071 GET_VECTOR_AND_LABEL(get_word_vector_and_label, atoi, uint16_t) 00072 GET_VECTOR_AND_LABEL(get_int8_vector_and_label, atoi, int8_t) 00073 GET_VECTOR_AND_LABEL(get_uint_vector_and_label, atoi, uint32_t) 00074 GET_VECTOR_AND_LABEL(get_long_vector_and_label, atoi, int64_t) 00075 GET_VECTOR_AND_LABEL(get_ulong_vector_and_label, atoi, uint64_t) 00076 GET_VECTOR_AND_LABEL(get_longreal_vector_and_label, atoi, floatmax_t) 00077 #undef GET_VECTOR_AND_LABEL 00078 00079 /* For string vectors */ 00080 #define GET_STRING(fname, conv, sg_type) \ 00081 void CStreamingFile::get_string \ 00082 (sg_type*& vector, int32_t& num_feat) \ 00083 { \ 00084 vector=NULL; \ 00085 num_feat=-1; \ 00086 SG_ERROR("Read function not supported by the feature type!") \ 00087 } 00088 00089 GET_STRING(get_bool_string, str_to_bool, bool) 00090 GET_STRING(get_byte_string, atoi, uint8_t) 00091 GET_STRING(get_char_string, atoi, char) 00092 GET_STRING(get_int_string, atoi, int32_t) 00093 GET_STRING(get_shortreal_string, atof, float32_t) 00094 GET_STRING(get_real_string, atof, float64_t) 00095 GET_STRING(get_short_string, atoi, int16_t) 00096 GET_STRING(get_word_string, atoi, uint16_t) 00097 GET_STRING(get_int8_string, atoi, int8_t) 00098 GET_STRING(get_uint_string, atoi, uint32_t) 00099 GET_STRING(get_long_string, atoi, int64_t) 00100 GET_STRING(get_ulong_string, atoi, uint64_t) 00101 GET_STRING(get_longreal_string, atoi, floatmax_t) 00102 #undef GET_STRING 00103 00104 /* For string vectors with labels */ 00105 #define GET_STRING_AND_LABEL(fname, conv, sg_type) \ 00106 void CStreamingFile::get_string_and_label \ 00107 (sg_type*& vector, int32_t& num_feat, float64_t& label) \ 00108 { \ 00109 vector=NULL; \ 00110 num_feat=-1; \ 00111 SG_ERROR("Read function not supported by the feature type!") \ 00112 } 00113 00114 GET_STRING_AND_LABEL(get_bool_string_and_label, str_to_bool, bool) 00115 GET_STRING_AND_LABEL(get_byte_string_and_label, atoi, uint8_t) 00116 GET_STRING_AND_LABEL(get_char_string_and_label, atoi, char) 00117 GET_STRING_AND_LABEL(get_int_string_and_label, atoi, int32_t) 00118 GET_STRING_AND_LABEL(get_shortreal_string_and_label, atof, float32_t) 00119 GET_STRING_AND_LABEL(get_real_string_and_label, atof, float64_t) 00120 GET_STRING_AND_LABEL(get_short_string_and_label, atoi, int16_t) 00121 GET_STRING_AND_LABEL(get_word_string_and_label, atoi, uint16_t) 00122 GET_STRING_AND_LABEL(get_int8_string_and_label, atoi, int8_t) 00123 GET_STRING_AND_LABEL(get_uint_string_and_label, atoi, uint32_t) 00124 GET_STRING_AND_LABEL(get_long_string_and_label, atoi, int64_t) 00125 GET_STRING_AND_LABEL(get_ulong_string_and_label, atoi, uint64_t) 00126 GET_STRING_AND_LABEL(get_longreal_string_and_label, atoi, floatmax_t) 00127 #undef GET_STRING_AND_LABEL 00128 00129 /* For sparse vectors */ 00130 #define GET_SPARSE_VECTOR(fname, conv, sg_type) \ 00131 \ 00132 void CStreamingFile::get_sparse_vector \ 00133 (SGSparseVectorEntry<sg_type>*& vector, int32_t& num_feat) \ 00134 { \ 00135 vector=NULL; \ 00136 num_feat=-1; \ 00137 SG_ERROR("Read function not supported by the feature type!") \ 00138 } 00139 00140 GET_SPARSE_VECTOR(get_bool_sparse_vector, str_to_bool, bool) 00141 GET_SPARSE_VECTOR(get_byte_sparse_vector, atoi, uint8_t) 00142 GET_SPARSE_VECTOR(get_char_sparse_vector, atoi, char) 00143 GET_SPARSE_VECTOR(get_int_sparse_vector, atoi, int32_t) 00144 GET_SPARSE_VECTOR(get_shortreal_sparse_vector, atof, float32_t) 00145 GET_SPARSE_VECTOR(get_real_sparse_vector, atof, float64_t) 00146 GET_SPARSE_VECTOR(get_short_sparse_vector, atoi, int16_t) 00147 GET_SPARSE_VECTOR(get_word_sparse_vector, atoi, uint16_t) 00148 GET_SPARSE_VECTOR(get_int8_sparse_vector, atoi, int8_t) 00149 GET_SPARSE_VECTOR(get_uint_sparse_vector, atoi, uint32_t) 00150 GET_SPARSE_VECTOR(get_long_sparse_vector, atoi, int64_t) 00151 GET_SPARSE_VECTOR(get_ulong_sparse_vector, atoi, uint64_t) 00152 GET_SPARSE_VECTOR(get_longreal_sparse_vector, atoi, floatmax_t) 00153 #undef GET_SPARSE_VECTOR 00154 00155 /* For sparse vectors with labels */ 00156 #define GET_SPARSE_VECTOR_AND_LABEL(fname, conv, sg_type) \ 00157 \ 00158 void CStreamingFile::get_sparse_vector_and_label \ 00159 (SGSparseVectorEntry<sg_type>*& vector, \ 00160 int32_t& num_feat, \ 00161 float64_t& label) \ 00162 { \ 00163 vector=NULL; \ 00164 num_feat=-1; \ 00165 SG_ERROR("Read function not supported by the feature type!") \ 00166 } 00167 00168 GET_SPARSE_VECTOR_AND_LABEL(get_bool_sparse_vector_and_label, str_to_bool, bool) 00169 GET_SPARSE_VECTOR_AND_LABEL(get_byte_sparse_vector_and_label, atoi, uint8_t) 00170 GET_SPARSE_VECTOR_AND_LABEL(get_char_sparse_vector_and_label, atoi, char) 00171 GET_SPARSE_VECTOR_AND_LABEL(get_int_sparse_vector_and_label, atoi, int32_t) 00172 GET_SPARSE_VECTOR_AND_LABEL(get_shortreal_sparse_vector_and_label, atof, float32_t) 00173 GET_SPARSE_VECTOR_AND_LABEL(get_real_sparse_vector_and_label, atof, float64_t) 00174 GET_SPARSE_VECTOR_AND_LABEL(get_short_sparse_vector_and_label, atoi, int16_t) 00175 GET_SPARSE_VECTOR_AND_LABEL(get_word_sparse_vector_and_label, atoi, uint16_t) 00176 GET_SPARSE_VECTOR_AND_LABEL(get_int8_sparse_vector_and_label, atoi, int8_t) 00177 GET_SPARSE_VECTOR_AND_LABEL(get_uint_sparse_vector_and_label, atoi, uint32_t) 00178 GET_SPARSE_VECTOR_AND_LABEL(get_long_sparse_vector_and_label, atoi, int64_t) 00179 GET_SPARSE_VECTOR_AND_LABEL(get_ulong_sparse_vector_and_label, atoi, uint64_t) 00180 GET_SPARSE_VECTOR_AND_LABEL(get_longreal_sparse_vector_and_label, atoi, floatmax_t) 00181 #undef GET_SPARSE_VECTOR_AND_LABEL 00182 00183 void CStreamingFile::get_vector(VwExample*& ex, int32_t &len) 00184 { 00185 SG_ERROR("Read function not supported by the feature type!\n") 00186 } 00187 00188 void CStreamingFile::get_vector_and_label(VwExample*& ex, int32_t& len, float64_t& label) 00189 { 00190 SG_ERROR("Read function not supported by the feature type!\n") 00191 } 00192 00193 } 00194 using namespace shogun; 00195 00196 CStreamingFile::CStreamingFile() : CSGObject() 00197 { 00198 buf=NULL; 00199 filename=NULL; 00200 } 00201 00202 CStreamingFile::CStreamingFile(const char* fname, char rw) : CSGObject() 00203 { 00204 task=rw; 00205 filename=get_strdup(fname); 00206 int mode = O_LARGEFILE; 00207 00208 switch (rw) 00209 { 00210 case 'r': 00211 mode |= O_RDONLY; 00212 break; 00213 case 'w': 00214 mode |= O_WRONLY; 00215 break; 00216 default: 00217 SG_ERROR("Unknown mode '%c'\n", task) 00218 } 00219 00220 if (filename) 00221 { 00222 int file = open((const char*) filename, mode); 00223 if (file < 0) 00224 SG_ERROR("Error opening file '%s'\n", filename) 00225 00226 buf = new CIOBuffer(file); 00227 SG_REF(buf); 00228 } 00229 else 00230 SG_ERROR("Error getting the file name!\n") 00231 } 00232 00233 CStreamingFile::~CStreamingFile() 00234 { 00235 SG_FREE(filename); 00236 SG_UNREF(buf); 00237 }