SHOGUN
v3.2.0
|
Go to the source code of this file.
Defines | |
#define | GET_VECTOR(sg_type) |
#define | GET_MATRIX(read_func, sg_type) |
#define | GET_NDARRAY(read_func, sg_type) |
#define | GET_SPARSE_MATRIX(sg_type) |
#define | SET_VECTOR(sg_type) |
#define | SET_MATRIX(sg_type) |
#define | SET_SPARSE_MATRIX(sg_type) |
#define | GET_STRING_LIST(sg_type) |
#define | SET_STRING_LIST(sg_type) |
#define | WRITE_SPARSE_MATRIX_HEADER(sg_type) |
#define | WRITE_STRING_LIST_HEADER(sg_type) |
#define | READ_MEMORY_BLOCK(chunk_type, sg_type) |
#define | WRITE_MEMORY_BLOCK(chunk_type, sg_type) |
#define | READ_SPARSE_MATRIX(chunk_type, sg_type) |
#define | WRITE_SPARSE_MATRIX(chunk_type, sg_type) |
#define | READ_STRING_LIST(chunk_type, sg_type) |
#define | WRITE_STRING_LIST(chunk_type, sg_type) |
#define GET_MATRIX | ( | read_func, | |
sg_type | |||
) |
void CProtobufFile::get_matrix(sg_type*& matrix, int32_t& num_feat, int32_t& num_vec) \
{ \
read_and_validate_global_header(ShogunVersion::MATRIX); \
MatrixHeader data_header=read_matrix_header(); \
num_feat=data_header.num_cols(); \
num_vec=data_header.num_rows(); \
read_memory_block(matrix, num_feat*num_vec, data_header.num_messages()); \
}
Definition at line 73 of file ProtobufFile.cpp.
#define GET_NDARRAY | ( | read_func, | |
sg_type | |||
) |
void CProtobufFile::get_ndarray(sg_type*& array, int32_t*& dims, int32_t& num_dims) \
{ \
SG_NOTIMPLEMENTED \
}
Definition at line 97 of file ProtobufFile.cpp.
#define GET_SPARSE_MATRIX | ( | sg_type | ) |
void CProtobufFile::get_sparse_matrix( \ SGSparseVector<sg_type>*& matrix, int32_t& num_feat, int32_t& num_vec) \ { \ read_and_validate_global_header(ShogunVersion::SPARSE_MATRIX); \ SparseMatrixHeader data_header=read_sparse_matrix_header(); \ num_feat=data_header.num_features(); \ num_vec=data_header.num_vectors(); \ read_sparse_matrix(matrix, data_header); \ }
Definition at line 112 of file ProtobufFile.cpp.
#define GET_STRING_LIST | ( | sg_type | ) |
void CProtobufFile::get_string_list( \ SGString<sg_type>*& strings, int32_t& num_str, \ int32_t& max_string_len) \ { \ read_and_validate_global_header(ShogunVersion::STRING_LIST); \ StringListHeader data_header=read_string_list_header(); \ num_str=data_header.num_str(); \ max_string_len=data_header.max_string_len(); \ read_string_list(strings, data_header); \ }
Definition at line 208 of file ProtobufFile.cpp.
#define GET_VECTOR | ( | sg_type | ) |
void CProtobufFile::get_vector(sg_type*& vector, int32_t& len) \
{ \
read_and_validate_global_header(ShogunVersion::VECTOR); \
VectorHeader data_header=read_vector_header(); \
len=data_header.len(); \
read_memory_block(vector, len, data_header.num_messages()); \
}
Definition at line 50 of file ProtobufFile.cpp.
#define READ_MEMORY_BLOCK | ( | chunk_type, | |
sg_type | |||
) |
void CProtobufFile::read_memory_block(sg_type*& vector, uint64_t len, int32_t num_messages) \ { \ vector=SG_MALLOC(sg_type, len); \ \ chunk_type chunk; \ int32_t elements_in_message=message_size/sizeof(sg_type); \ for (int32_t i=0; i<num_messages; i++) \ { \ read_message(chunk); \ \ int32_t num_elements_to_read=0; \ if ((len-(i+1)*elements_in_message)<=0) \ num_elements_to_read=len-i*elements_in_message; \ else \ num_elements_to_read=elements_in_message; \ \ for (int32_t j=0; j<num_elements_to_read; j++) \ vector[j+i*elements_in_message]=chunk.data(j); \ } \ }
Definition at line 446 of file ProtobufFile.cpp.
#define READ_SPARSE_MATRIX | ( | chunk_type, | |
sg_type | |||
) |
void CProtobufFile::read_sparse_matrix( \ SGSparseVector<sg_type>*& matrix, const SparseMatrixHeader& data_header) \ { \ matrix=SG_MALLOC(SGSparseVector<sg_type>, data_header.num_vectors()); \ \ UInt64Chunk feat_index_chunk; \ chunk_type entry_chunk; \ read_message(feat_index_chunk); \ read_message(entry_chunk); \ \ int32_t elements_in_message=message_size/sizeof(sg_type); \ int32_t buffer_counter=0; \ for (uint32_t i=0; i<data_header.num_vectors(); i++) \ { \ matrix[i]=SGSparseVector<sg_type>(data_header.num_feat_entries(i)); \ for (int32_t j=0; j<matrix[i].num_feat_entries; j++) \ { \ matrix[i].features[j].feat_index=feat_index_chunk.data(buffer_counter); \ matrix[i].features[j].entry=entry_chunk.data(buffer_counter); \ buffer_counter++; \ \ if (buffer_counter==elements_in_message) \ { \ read_message(feat_index_chunk); \ read_message(entry_chunk); \ buffer_counter=0; \ } \ } \ } \ }
Definition at line 518 of file ProtobufFile.cpp.
#define READ_STRING_LIST | ( | chunk_type, | |
sg_type | |||
) |
void CProtobufFile::read_string_list( \ SGString<sg_type>*& strings, const StringListHeader& data_header) \ { \ strings=SG_MALLOC(SGString<sg_type>, data_header.num_str()); \ \ chunk_type chunk; \ read_message(chunk); \ int32_t elements_in_message=message_size/sizeof(sg_type); \ int32_t buffer_counter=0; \ for (uint32_t i=0; i<data_header.num_str(); i++) \ { \ strings[i]=SGString<sg_type>(data_header.str_len(i)); \ for (int32_t j=0; j<strings[i].slen; j++) \ { \ strings[i].string[j]=chunk.data(buffer_counter); \ buffer_counter++; \ \ if (buffer_counter==elements_in_message) \ { \ read_message(chunk); \ buffer_counter=0; \ } \ } \ } \ }
Definition at line 614 of file ProtobufFile.cpp.
#define SET_MATRIX | ( | sg_type | ) |
void CProtobufFile::set_matrix(const sg_type* matrix, int32_t num_feat, int32_t num_vec) \ { \ int32_t num_messages=compute_num_messages(num_feat*num_vec, sizeof(sg_type)); \ write_global_header(ShogunVersion::MATRIX); \ write_matrix_header(num_feat, num_vec, num_messages); \ write_memory_block(matrix, num_feat*num_vec, num_messages); \ }
Definition at line 161 of file ProtobufFile.cpp.
#define SET_SPARSE_MATRIX | ( | sg_type | ) |
void CProtobufFile::set_sparse_matrix( \ const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \ { \ write_global_header(ShogunVersion::SPARSE_MATRIX); \ write_sparse_matrix_header(matrix, num_feat, num_vec); \ write_sparse_matrix(matrix, num_vec); \ }
Definition at line 184 of file ProtobufFile.cpp.
#define SET_STRING_LIST | ( | sg_type | ) |
void CProtobufFile::set_string_list( \ const SGString<sg_type>* strings, int32_t num_str) \ { \ write_global_header(ShogunVersion::STRING_LIST); \ write_string_list_header(strings, num_str); \ write_string_list(strings, num_str); \ }
Definition at line 234 of file ProtobufFile.cpp.
#define SET_VECTOR | ( | sg_type | ) |
void CProtobufFile::set_vector(const sg_type* vector, int32_t len) \ { \ int32_t num_messages=compute_num_messages(len, sizeof(sg_type)); \ write_global_header(ShogunVersion::VECTOR); \ write_vector_header(len, num_messages); \ write_memory_block(vector, len, num_messages); \ }
Definition at line 138 of file ProtobufFile.cpp.
#define WRITE_MEMORY_BLOCK | ( | chunk_type, | |
sg_type | |||
) |
void CProtobufFile::write_memory_block(const sg_type* vector, uint64_t len, int32_t num_messages) \ { \ chunk_type chunk; \ int32_t elements_in_message=message_size/sizeof(sg_type); \ for (int32_t i=0; i<num_messages; i++) \ { \ \ int32_t num_elements_to_write=0; \ if ((len-(i+1)*elements_in_message)<=0) \ num_elements_to_write=len-i*elements_in_message; \ else \ num_elements_to_write=elements_in_message; \ \ for (int32_t j=0; j<num_elements_to_write; j++) \ chunk.add_data(vector[j+i*elements_in_message]); \ \ write_message(chunk); \ chunk.Clear(); \ } \ }
Definition at line 482 of file ProtobufFile.cpp.
#define WRITE_SPARSE_MATRIX | ( | chunk_type, | |
sg_type | |||
) |
Definition at line 565 of file ProtobufFile.cpp.
#define WRITE_SPARSE_MATRIX_HEADER | ( | sg_type | ) |
void CProtobufFile::write_sparse_matrix_header( \ const SGSparseVector<sg_type>* matrix, int32_t num_feat, int32_t num_vec) \ { \ SparseMatrixHeader data_header; \ data_header.set_num_features(num_feat); \ data_header.set_num_vectors(num_vec); \ for (int32_t i=0; i<num_vec; i++) \ { \ data_header.add_num_feat_entries(matrix[i].num_feat_entries); \ } \ \ write_message(data_header); \ }
Definition at line 351 of file ProtobufFile.cpp.
#define WRITE_STRING_LIST | ( | chunk_type, | |
sg_type | |||
) |
void CProtobufFile::write_string_list( \ const SGString<sg_type>* strings, int32_t num_str) \ { \ chunk_type chunk; \ int32_t elements_in_message=message_size/sizeof(sg_type); \ int32_t buffer_counter=0; \ for (int32_t i=0; i<num_str; i++) \ { \ for (int32_t j=0; j<strings[i].slen; j++) \ { \ chunk.add_data(strings[i].string[j]); \ buffer_counter++; \ \ if (buffer_counter==elements_in_message) \ { \ write_message(chunk); \ chunk.Clear(); \ buffer_counter=0; \ } \ } \ } \ \ if (buffer_counter!=0) \ write_message(chunk); \ }
Definition at line 655 of file ProtobufFile.cpp.
#define WRITE_STRING_LIST_HEADER | ( | sg_type | ) |
void CProtobufFile::write_string_list_header(const SGString<sg_type>* strings, int32_t num_str) \ { \ int32_t max_string_len=0; \ StringListHeader data_header; \ data_header.set_num_str(num_str); \ for (int32_t i=0; i<num_str; i++) \ { \ data_header.add_str_len(strings[i].slen); \ if (strings[i].slen>max_string_len) \ max_string_len=strings[i].slen; \ } \ data_header.set_max_string_len(max_string_len); \ write_message(data_header); \ }
Definition at line 381 of file ProtobufFile.cpp.