// ----------------------------------------------------------------------
// File: EosChecksumBenchmark.cc
// Author: Andreas-Joachim Peters - CERN
// ----------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2011 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see .*
************************************************************************/
/*-----------------------------------------------------------------------------*/
#include
#include
/*-----------------------------------------------------------------------------*/
#include "common/LayoutId.hh"
#include "common/Logging.hh"
#include "common/Timing.hh"
#include "common/StringConversion.hh"
#include "fst/checksum/ChecksumPlugins.hh"
/*-----------------------------------------------------------------------------*/
#include
#include
/*-----------------------------------------------------------------------------*/
XrdPosixXrootd posixXrootd;
// 1GB mem buffer
#define MEMORYBUFFERSIZE 256ll*1024ll*1024ll
int main(int argc, char* argv[])
{
eos::common::VirtualIdentity vid = eos::common::VirtualIdentity::Root();
eos::common::Logging& g_logging = eos::common::Logging::GetInstance();
g_logging.SetUnit("eoschecksumbenchmark@localhost");
g_logging.gShortFormat = true;
g_logging.SetLogPriority(LOG_DEBUG);
std::vector checksumnames;
std::vector checksumids;
checksumnames.push_back("adler32");
checksumnames.push_back("crc32");
checksumnames.push_back("md5");
checksumnames.push_back("crc32c");
checksumnames.push_back("sha1");
checksumnames.push_back("xxhash64");
checksumnames.push_back("blake3");
checksumnames.push_back("hwh64");
checksumnames.push_back("crc64");
checksumids.push_back(eos::common::LayoutId::kAdler);
checksumids.push_back(eos::common::LayoutId::kCRC32);
checksumids.push_back(eos::common::LayoutId::kMD5);
checksumids.push_back(eos::common::LayoutId::kCRC32C);
checksumids.push_back(eos::common::LayoutId::kSHA1);
checksumids.push_back(eos::common::LayoutId::kXXHASH64);
checksumids.push_back(eos::common::LayoutId::kBLAKE3);
checksumids.push_back(eos::common::LayoutId::kHWH64);
checksumids.push_back(eos::common::LayoutId::kCRC64);
size_t nforks = 1;
if (argc == 2) {
nforks = atoi(argv[1]);
if (nforks <= 0) {
fprintf(stdout, "info: forcing the nforks=1");
nforks = 1;
}
}
for (size_t foker = 0; foker < nforks; foker ++) {
if (!fork()) {
srandom(foker);
XrdOucString size;
eos::common::StringConversion::GetReadableSizeString(size, MEMORYBUFFERSIZE,
"B");
// allocate a block
eos_static_info("allocating %s", size.c_str());
char* buffer = (char*) malloc(MEMORYBUFFERSIZE);
char* xsbuffer = (char*) malloc(MEMORYBUFFERSIZE / 100);
if ((!buffer) || (!xsbuffer)) {
fprintf(stderr, "error: failed to allocate reference buffers!\n");
exit(-1);
}
eos_static_info("write randomized contents into %s", size.c_str());
for (off_t i = 0; i < MEMORYBUFFERSIZE; i++) {
buffer[i] = (rand()) % 256;
}
eos_static_info("write zeros into xs buffers");
for (off_t i = 0; i < MEMORYBUFFERSIZE / 100; i++) {
xsbuffer[i] = 0;
}
eos_static_info("allocated %s", size.c_str());
std::vector blocksize;
blocksize.push_back(4096);
blocksize.push_back(128 * 1024);
blocksize.push_back(1024 * 1024);
blocksize.push_back(4 * 1024 * 1024);
blocksize.push_back(128 * 1024 * 1024);
for (size_t bs = 0; bs < blocksize.size(); bs++) {
for (size_t i = 0; i < checksumnames.size(); i++) {
eos_static_info("benchmarking checksum algorithm %s [%d]",
checksumnames[i].c_str(), checksumids[i]);
std::unique_ptr checksum =
eos::fst::ChecksumPlugins::GetChecksumObject(checksumids[i]);
if (!checksum) {
eos_static_err("failed to get checksum algorithm %s", checksumnames[i].c_str());
} else {
eos::common::Timing tm("Checksumming");
COMMONTIMING("START", &tm);
char* ptr = buffer;
off_t offset = 0;
for (size_t j = 0; j < MEMORYBUFFERSIZE / blocksize[bs]; j++) {
checksum->Add(ptr, blocksize[bs], offset);
offset += blocksize[bs];
ptr += blocksize[bs];
}
checksum->Finalize();
COMMONTIMING("STOP", &tm);
XrdOucString sizestring;
eos::common::StringConversion::GetReadableSizeString(sizestring, blocksize[bs],
"B");
eos_static_info("checksum( %-10s ) = %s realtime=%.02f [ms] blocksize=%s rate=%.02f",
checksumnames[i].c_str(), checksum->GetHexChecksum(), tm.RealTime(),
sizestring.c_str(), MEMORYBUFFERSIZE / tm.RealTime() / 1000.0);
}
}
}
exit(0);
}
}
for (size_t foker = 0; foker < nforks; foker ++) {
wait(0);
}
}