//------------------------------------------------------------------------------
//! @file LayoutId.hh
//! @author Andreas-Joachim Peters - CERN
//! @brief Class with static members helping to deal with layout types
//------------------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2011 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see .*
************************************************************************/
#ifndef __EOSCOMMON_LAYOUTID__HH__
#define __EOSCOMMON_LAYOUTID__HH__
#include "common/Namespace.hh"
#include "XrdOuc/XrdOucEnv.hh"
#include "XrdOuc/XrdOucString.hh"
#include "XrdSfs/XrdSfsInterface.hh"
#include "XrdCl/XrdClFileSystem.hh"
#include "StringConversion.hh"
#include
#include
EOSCOMMONNAMESPACE_BEGIN
//------------------------------------------------------------------------------
//! Class with static members helping to deal with layout types
//------------------------------------------------------------------------------
class LayoutId
{
public:
typedef unsigned long layoutid_t;
// A layout id is constructed as an xor as defined in GetId()
static const uint64_t OssXsBlockSize = 4 * 1024; ///< block xs size 4KB
//--------------------------------------------------------------------------
//! Definition of layout errors
//--------------------------------------------------------------------------
enum eLayoutError {
// this is used on FSTs in the Fmd Synchronization
kOrphan = 0x1, ///< layout produces an orphan
kUnregistered = 0x2, ///< layout has an unregistered stripe
kReplicaWrong = 0x4, ///< layout has the wrong number of replicas
kMissing = 0x8 ///< layout has an entry which is missing on disk
};
//--------------------------------------------------------------------------
//! Definition of checksum types
//--------------------------------------------------------------------------
enum eChecksum {
kNone = 0x1,
kAdler = 0x2,
kCRC32 = 0x3,
kMD5 = 0x4,
kSHA1 = 0x5,
kCRC32C = 0x6,
kCRC64 = 0x7,
kSHA256 = 0x8,
kXXHASH64 = 0x9,
kBLAKE3 = 0xa,
kHWH64 = 0xb,
kXSmax = kHWH64
};
//--------------------------------------------------------------------------
//! Definition of file layout types
//--------------------------------------------------------------------------
enum eLayoutType {
kPlain = 0x0,
kReplica = 0x1,
kArchive = 0x2,
kRaidDP = 0x3,
kRaid6 = 0x4,
kQrain = 0x5,
kRaid5 = 0x6,
};
//--------------------------------------------------------------------------
//! Definition of IO types
//--------------------------------------------------------------------------
enum eIoType {
kLocal = 0x0,
kXrdCl = 0x1,
kDavix = 0x4
};
//--------------------------------------------------------------------------
//! Get type of io access based on the given URL (path)
//--------------------------------------------------------------------------
static eIoType
GetIoType(const char* path)
{
XrdOucString spath = path;
if (spath.beginswith("root:")) {
return kXrdCl;
}
// Definition of predefined block sizes
if (spath.beginswith("http:")) {
return kDavix;
}
if (spath.beginswith("https:")) {
return kDavix;
}
if (spath.beginswith("s3:")) {
return kDavix;
}
if (spath.beginswith("s3s:")) {
return kDavix;
}
return kLocal;
}
//--------------------------------------------------------------------------
//! Definition of predefined block sizes
//--------------------------------------------------------------------------
enum eBlockSize {
k4k = 0x0,
k64k = 0x1,
k128k = 0x2,
k512k = 0x3,
k1M = 0x4,
k4M = 0x5,
k16M = 0x6,
k64M = 0x7
};
//--------------------------------------------------------------------------
//! Get a reed solomon layout by number of redundancy stripes
//--------------------------------------------------------------------------
static int
GetReedSLayoutByParity(int redundancystripes)
{
switch (redundancystripes) {
case 1:
return kRaid5;
case 2:
return kRaid6;
case 3:
return kArchive;
case 4:
return kQrain;
default:
return 0;
}
}
//--------------------------------------------------------------------------
//! Build a layout id from given parameters
//--------------------------------------------------------------------------
static unsigned long
GetId(int layout,
int checksum = 1,
int stripesize = 1,
int stripewidth = 0,
int blockchecksum = 1,
int excessreplicas = 0,
int redundancystripes = 0)
{
unsigned long id = (checksum |
((layout & 0xf) << 4) |
(((stripesize - 1) & 0xff) << 8) |
((stripewidth & 0xf) << 16) |
((blockchecksum & 0xf) << 20) |
((excessreplicas & 0xf) << 24));
// Set the number of parity stripes depending on the layout type if not
// already set explicitly
if (redundancystripes == 0) {
redundancystripes = GetRedundancyFromLayoutType(layout);
}
id |= ((redundancystripes & 0x7) << 28);
return id;
}
//--------------------------------------------------------------------------
//! Convert the blocksize enum to bytes
//--------------------------------------------------------------------------
static unsigned long
BlockSize(int blocksize)
{
switch (blocksize) {
case k4k:
return (4 * 1024);
case k64k:
return (64 * 1024);
case k128k:
return (128 * 1024);
case k512k:
return (512 * 1024);
case k1M:
return (1024 * 1024);
case k4M:
return (4 * 1024 * 1024);
case k16M:
return (16 * 1024 * 1024);
case k64M:
return (64 * 1024 * 1024);
default:
return 0;
}
}
//--------------------------------------------------------------------------
//! Convert bytes to blocksize enum
//--------------------------------------------------------------------------
static int
BlockSizeEnum(unsigned long blocksize)
{
switch (blocksize) {
case (4*1024):
return k4k;
case (64*1024):
return k64k;
case (128*1024):
return k128k;
case (512*1024):
return k512k;
case (1024 * 1024):
return k1M;
case (4 * 1024 * 1024):
return k4M;
case (16 * 1024 * 1024):
return k16M;
case (64 * 1024 * 1024):
return k64M;
default:
return 0;
}
}
//--------------------------------------------------------------------------
//! Get Checksum enum from given layout
//--------------------------------------------------------------------------
static unsigned long
GetChecksum(unsigned long layout)
{
return (layout & 0xf);
}
//--------------------------------------------------------------------------
//! Set checksum in the layout encoding
//!
//! @param layout input layout encoding
//! @param xs_type checksum type
//!
//! @return new layout encoding
//--------------------------------------------------------------------------
static unsigned long
SetChecksum(unsigned long layout, unsigned long xs_type)
{
xs_type &= 0x0f;
if ((xs_type < kNone) || (xs_type > kXSmax)) {
xs_type = kNone;
}
// Wipe out the old checksum type
unsigned long tmp = layout & 0xfffffff0;
// Set the new blockxs value
tmp |= xs_type;
return tmp;
}
//--------------------------------------------------------------------------
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(unsigned long layout)
{
if ((layout & 0xf) == kAdler) {
return 4;
}
if ((layout & 0xf) == kCRC32) {
return 4;
}
if ((layout & 0xf) == kCRC32C) {
return 4;
}
if ((layout & 0xf) == kMD5) {
return 16;
}
if ((layout & 0xf) == kSHA1) {
return 20;
}
if ((layout & 0xf) == kCRC64) {
return 8;
}
if ((layout & 0xf) == kHWH64) {
return 8;
}
if ((layout & 0xf) == kSHA256) {
return 32;
}
if ((layout & 0xf) == kXXHASH64) {
return 8;
}
if ((layout & 0xf) == kBLAKE3) {
return 32;
}
return 0;
}
//--------------------------------------------------------------------------
//! Get length of Layout checksum in bytes
//--------------------------------------------------------------------------
static unsigned long
GetChecksumLen(const std::string& xs_type)
{
if (xs_type == "adler") {
return 4;
} else if (xs_type == "blake3") {
return 32;
} else if (xs_type == "crc32") {
return 4;
} else if (xs_type == "crc32c") {
return 4;
} else if (xs_type == "xxhash64") {
return 8;
} else if (xs_type == "crc64") {
return 8;
} else if (xs_type == "md5") {
return 16;
} else if (xs_type == "sha") {
return 20;
} else if (xs_type == "sha256") {
return 32;
} else if (xs_type == "hwh64") {
return 8;
} else {
return 0;
}
}
static std::string
GetEmptyFileChecksum(unsigned long layout)
{
std::string hexchecksum;
std::string binchecksum;
binchecksum.resize(40);
switch ((layout & 0xf)) {
case kAdler:
hexchecksum = "00000001";
break;
case kCRC32:
hexchecksum = "00000000";
break;
case kCRC32C:
hexchecksum = "00000000";
break;
case kMD5:
hexchecksum = "d41d8cd98f00b204e9800998ecf8427e";
break;
case kSHA1:
hexchecksum = "da39a3ee5e6b4b0d3255bfef95601890afd80709";
break;
case kBLAKE3:
hexchecksum =
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262";
break;
case kHWH64:
case kCRC64:
case kXXHASH64:
hexchecksum =
"0000000000000000";
break;
}
for (unsigned int i = 0; i < hexchecksum.length(); i += 2) {
// hex2binary conversion
char hex[3];
hex[0] = hexchecksum[i];
hex[1] = hexchecksum[i + 1];
hex[2] = 0;
binchecksum[i / 2] = strtol(hex, 0, 16);
}
binchecksum.erase(hexchecksum.length() / 2);
binchecksum.resize(hexchecksum.length() / 2);
return binchecksum;
}
//--------------------------------------------------------------------------
//! Return layout type enum
//--------------------------------------------------------------------------
static unsigned long
GetLayoutType(unsigned long layout)
{
return ((layout >> 4) & 0xf);
}
//--------------------------------------------------------------------------
//! Test for RAIN layout e.g. raid6, archive, qrain
//--------------------------------------------------------------------------
static bool
IsRain(unsigned long layout)
{
// everything but plain and replica
return (GetLayoutType(layout) > kReplica);
}
//--------------------------------------------------------------------------
//! Set file layout type in the layout encoding
//!
//! @param layout input layout encoding
//! @param ftype new file layout type
//!
//! @return new layout encoding
//--------------------------------------------------------------------------
static unsigned long
SetLayoutType(unsigned long layout, unsigned long ftype)
{
ftype &= 0xf;
if ((ftype < kPlain) || (ftype > kRaid6)) {
ftype = kPlain;
}
// Wipe out the old file layout type
unsigned long tmp = layout & 0xfffff0f;
// Shift to the right position
ftype <<= 4;
// Set the new file layout type
tmp |= ftype;
return tmp;
}
//--------------------------------------------------------------------------
//! Return layout stripe enum
//--------------------------------------------------------------------------
static unsigned long
GetStripeNumber(unsigned long layout)
{
return ((layout >> 8) & 0xff);
}
//--------------------------------------------------------------------------
//! Modify layout stripe number
//--------------------------------------------------------------------------
static void
SetStripeNumber(unsigned long& layout, int stripes)
{
unsigned long tmp = stripes & 0xff;
tmp <<= 8;
tmp &= 0xff00;
layout &= 0xffff00ff;
layout |= tmp;
}
//--------------------------------------------------------------------------
//! Return layout blocksize in bytes
//--------------------------------------------------------------------------
static unsigned long
GetBlocksize(unsigned long layout)
{
return BlockSize(((layout >> 16) & 0xf));
}
//--------------------------------------------------------------------------
//! Return layout blocksize enum
//--------------------------------------------------------------------------
static unsigned long
GetBlocksizeType(unsigned long layout)
{
return ((layout >> 16) & 0xf);
}
//--------------------------------------------------------------------------
//! Return layout checksum enum
//--------------------------------------------------------------------------
static unsigned long
GetBlockChecksum(unsigned long layout)
{
// disable block checksum in replica layouts
if (GetLayoutType(layout) == kReplica) {
return kNone;
}
return ((layout >> 20) & 0xf);
}
//--------------------------------------------------------------------------
//! Set blockchecksum in the layout encoding
//!
//! @param layout input layout encoding
//! @param blockxs block checksum type
//!
//! @return new layout encoding
//--------------------------------------------------------------------------
static unsigned long
SetBlockChecksum(unsigned long layout, unsigned long blockxs)
{
blockxs &= 0xf;
if ((blockxs < kNone) || (blockxs > kXSmax)) {
blockxs = kNone;
}
// Wipe out the old blockxs type
unsigned long tmp = layout & 0xff0fffff;
// Shift the blockxs in the right position
blockxs <<= 20;
// Set the new blockxs value
tmp |= blockxs;
return tmp;
}
//--------------------------------------------------------------------------
//! Return excess replicas
//--------------------------------------------------------------------------
static unsigned long
GetExcessStripeNumber(unsigned long layout)
{
return ((layout >> 24) & 0xf);
}
//--------------------------------------------------------------------------
//! Return redundancy stripes
//--------------------------------------------------------------------------
static unsigned long
GetRedundancyStripeNumber(unsigned long layout)
{
return ((layout >> 28) & 0x7);
}
//--------------------------------------------------------------------------
//! Return redundancy stripes
//--------------------------------------------------------------------------
static unsigned long
GetRedundancy(unsigned long layout, unsigned long locations)
{
if ((GetLayoutType(layout) == kPlain) ||
(GetLayoutType(layout) == kReplica)) {
return locations;
}
int num_parity_stripes = GetRedundancyStripeNumber(layout);
int num_all_stripes = GetStripeNumber(layout) + 1;
int redundancy = num_parity_stripes + 1 + (locations - num_all_stripes);
if (redundancy < 0) {
redundancy = 0;
}
// return the remaining redundancy value for a file
return redundancy;
}
static std::string
GetRedundancySymbol(bool has_tape, int redundancy)
{
char sbst[256];
snprintf(sbst, sizeof(sbst), "d%lu::t%i ", has_tape ?
((redundancy > 0) ? (redundancy - 1) : 0) : redundancy, (has_tape ? 1 : 0));
return std::string(sbst);
}
//--------------------------------------------------------------------------
//! Build block checksum layout from block checksum enum
//--------------------------------------------------------------------------
static unsigned long
MakeBlockChecksum(unsigned long xs)
{
return (xs << 20);
}
//--------------------------------------------------------------------------
//! Return length of checksum
//--------------------------------------------------------------------------
static unsigned long
GetBlockChecksumLen(unsigned long layout)
{
return GetChecksumLen((layout >> 20) & 0xf);
}
//--------------------------------------------------------------------------
//! Get stripe(replica) file size based on the full size of the file and
//! the given layout
//!
//! @param lid layout id
//! @param fsize file size
//!
//! @return stripe(replica) file size
//--------------------------------------------------------------------------
static uint64_t
GetStripeFileSize(const unsigned long lid, const uint64_t fsize)
{
if (!IsRain(lid)) {
return fsize;
}
int num_data_stripes = 0;
int num_all_stripes = GetStripeNumber(lid) + 1;
int num_parity_stripes = GetRedundancyStripeNumber(lid);
// In case values don't make sense just return the full size of the file
if (num_parity_stripes >= num_all_stripes) {
return fsize;
}
uint64_t block_sz = GetBlocksize(lid);
num_data_stripes = num_all_stripes - num_parity_stripes;
uint64_t group_sz = block_sz * std::pow(num_data_stripes, 2);
return static_cast(std::ceil((float)fsize / group_sz)) *
num_data_stripes * block_sz;
}
//--------------------------------------------------------------------------
//! Return multiplication factor for a given layout
//! E.g. the physical space factor for a given layout
//--------------------------------------------------------------------------
static double
GetSizeFactor(unsigned long layout)
{
if (GetLayoutType(layout) == kPlain) {
return 1.0;
}
if (GetLayoutType(layout) == kReplica) {
return 1.0 * (GetStripeNumber(layout) + 1 + GetExcessStripeNumber(layout));
}
if (GetLayoutType(layout) == kRaidDP)
return 1.0 * (((1.0 * (GetStripeNumber(layout) + 1)) /
(GetStripeNumber(layout) + 1 - GetRedundancyStripeNumber(
layout))) + GetExcessStripeNumber(layout));
if (GetLayoutType(layout) == kRaid6)
return 1.0 * (((1.0 * (GetStripeNumber(layout) + 1)) /
(GetStripeNumber(layout) + 1 - GetRedundancyStripeNumber(
layout))) + GetExcessStripeNumber(layout));
if (GetLayoutType(layout) == kArchive)
return 1.0 * (((1.0 * (GetStripeNumber(layout) + 1)) /
(GetStripeNumber(layout) + 1 - GetRedundancyStripeNumber(
layout))) + GetExcessStripeNumber(layout));
return 1.0;
}
//--------------------------------------------------------------------------
//! Return minimum number of replicas which have to be online for a layout
//! to be readable
//--------------------------------------------------------------------------
static size_t
GetMinOnlineReplica(unsigned long layout)
{
if (GetLayoutType(layout) == kPlain) {
return 1;
}
if (GetLayoutType(layout) == kReplica) {
return 1;
}
return (1 + GetStripeNumber(layout) - GetRedundancyStripeNumber(layout));
}
//--------------------------------------------------------------------------
//! Return number of replicas which have to be online for a layout to be
//! immediately writable
//--------------------------------------------------------------------------
static unsigned long
GetOnlineStripeNumber(unsigned long layout)
{
return (GetStripeNumber(layout) + 1);
}
//--------------------------------------------------------------------------
//! Return checksum type as string
//--------------------------------------------------------------------------
static const char*
GetChecksumString(unsigned long layout)
{
if (GetChecksum(layout) == kNone) {
return "none";
}
if (GetChecksum(layout) == kAdler) {
return "adler";
}
if (GetChecksum(layout) == kBLAKE3) {
return "blake3";
}
if (GetChecksum(layout) == kCRC32) {
return "crc32";
}
if (GetChecksum(layout) == kCRC32C) {
return "crc32c";
}
if (GetChecksum(layout) == kMD5) {
return "md5";
}
if (GetChecksum(layout) == kSHA1) {
return "sha";
}
if (GetChecksum(layout) == kSHA256) {
return "sha256";
}
if (GetChecksum(layout) == kCRC64) {
return "crc64";
}
if (GetChecksum(layout) == kXXHASH64) {
return "xxhash64";
}
if (GetChecksum(layout) == kHWH64) {
return "hwh64";
}
return "none";
}
//--------------------------------------------------------------------------
//! Get checksum type from string representation
//--------------------------------------------------------------------------
static int
GetChecksumFromString(const std::string& checksum)
{
if ((checksum == "adler") || (checksum == "adler32")) {
return kAdler;
} else if (checksum == "blake3") {
return kBLAKE3;
} else if (checksum == "crc32") {
return kCRC32;
} else if (checksum == "crc32c") {
return kCRC32C;
} else if (checksum == "md5") {
return kMD5;
} else if ((checksum == "sha") || (checksum == "sha1")) {
return kSHA1;
} else if (checksum == "crc64") {
return kCRC64;
} else if (checksum == "sha256") {
return kSHA256;
} else if (checksum == "xxhash64") {
return kXXHASH64;
} else if (checksum == "hwh64") {
return kHWH64;
} else if (checksum == "none") {
return kNone;
}
return -1;
}
//--------------------------------------------------------------------------
//! Return checksum type but masking adler as adler32
//--------------------------------------------------------------------------
static const char*
GetChecksumStringReal(unsigned long layout)
{
if (GetChecksum(layout) == kNone) {
return "none";
}
if (GetChecksum(layout) == kAdler) {
return "adler32";
}
if (GetChecksum(layout) == kBLAKE3) {
return "blake3";
}
if (GetChecksum(layout) == kCRC32) {
return "crc32";
}
if (GetChecksum(layout) == kCRC32C) {
return "crc32c";
}
if (GetChecksum(layout) == kMD5) {
return "md5";
}
if (GetChecksum(layout) == kSHA1) {
return "sha1";
}
if (GetChecksum(layout) == kSHA256) {
return "sha256";
}
if (GetChecksum(layout) == kCRC64) {
return "crc64";
}
if (GetChecksum(layout) == kXXHASH64) {
return "xxhash64";
}
if (GetChecksum(layout) == kHWH64) {
return "hwh64";
}
return "none";
}
//--------------------------------------------------------------------------
//! Return block checksum type as string
//--------------------------------------------------------------------------
static const char*
GetBlockChecksumString(unsigned long layout)
{
if (GetBlockChecksum(layout) == kNone) {
return "none";
}
if (GetBlockChecksum(layout) == kAdler) {
return "adler";
}
if (GetBlockChecksum(layout) == kBLAKE3) {
return "blake3";
}
if (GetBlockChecksum(layout) == kCRC32) {
return "crc32";
}
if (GetBlockChecksum(layout) == kCRC32C) {
return "crc32c";
}
if (GetBlockChecksum(layout) == kMD5) {
return "md5";
}
if (GetBlockChecksum(layout) == kSHA1) {
return "sha";
}
if (GetBlockChecksum(layout) == kSHA256) {
return "sha256";
}
if (GetBlockChecksum(layout) == kCRC64) {
return "crc64";
}
if (GetBlockChecksum(layout) == kXXHASH64) {
return "xxhash64";
}
if (GetBlockChecksum(layout) == kHWH64) {
return "hwh64";
}
return "none";
}
//--------------------------------------------------------------------------
//! Return blocksize as string
//--------------------------------------------------------------------------
static const char*
GetBlockSizeString(unsigned long layout)
{
if (GetBlocksizeType(layout) == k4k) {
return "4k";
}
if (GetBlocksizeType(layout) == k64k) {
return "64k";
}
if (GetBlocksizeType(layout) == k128k) {
return "128k";
}
if (GetBlocksizeType(layout) == k512k) {
return "512k";
}
if (GetBlocksizeType(layout) == k1M) {
return "1M";
}
if (GetBlocksizeType(layout) == k4M) {
return "4M";
}
if (GetBlocksizeType(layout) == k16M) {
return "16M";
}
if (GetBlocksizeType(layout) == k64M) {
return "64M";
}
return "illegal";
}
//--------------------------------------------------------------------------
//! Return layout type as string
//--------------------------------------------------------------------------
static const char*
GetLayoutTypeString(unsigned long layout)
{
if (GetLayoutType(layout) == kPlain) {
return "plain";
}
if (GetLayoutType(layout) == kReplica) {
return "replica";
}
if (GetLayoutType(layout) == kRaidDP) {
return "raiddp";
}
if (GetLayoutType(layout) == kRaid5) {
return "raid5";
}
if (GetLayoutType(layout) == kRaid6) {
return "raid6";
}
if (GetLayoutType(layout) == kArchive) {
return "archive";
}
if (GetLayoutType(layout) == kQrain) {
return "qrain";
}
return "none";
}
//--------------------------------------------------------------------------
//! Return layout type from string representation
//--------------------------------------------------------------------------
static int
GetLayoutFromString(const string& layout)
{
if (layout == "plain") {
return kPlain;
} else if (layout == "replica") {
return kReplica;
} else if (layout == "raiddp") {
return kRaidDP;
} else if (layout == "raid5") {
return kRaid5;
} else if (layout == "raid6") {
return kRaid6;
} else if (layout == "qrain") {
return kQrain;
} else if (layout == "archive") {
return kArchive;
}
return -1;
}
//--------------------------------------------------------------------------
//! Return layout type enum from env definition
//--------------------------------------------------------------------------
static unsigned long
GetLayoutFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.layout.type"))) {
int layout = GetLayoutFromString(val);
return (layout != -1) ? layout : kPlain;
}
return kPlain;
}
//--------------------------------------------------------------------------
//! Return layout redundancy stripe number from layout type
//--------------------------------------------------------------------------
static int
GetRedundancyFromLayoutType(unsigned long layout_type)
{
int redundancystripes = 0;
if (layout_type == kRaid5) {
redundancystripes = 1;
} else if (layout_type == kRaidDP) {
redundancystripes = 2;
} else if (layout_type == kRaid6) {
redundancystripes = 2;
} else if (layout_type == kArchive) {
redundancystripes = 3;
} else if (layout_type == kQrain) {
redundancystripes = 4;
}
return redundancystripes;
}
//--------------------------------------------------------------------------
//! Return layout redundancy stripe number from layout string
//--------------------------------------------------------------------------
static int
GetRedundancyFromLayoutString(const std::string& layout)
{
int layout_type = GetLayoutFromString(layout);
return GetRedundancyFromLayoutType(layout_type);
}
//--------------------------------------------------------------------------
//! Return layout stripe number as string
//--------------------------------------------------------------------------
static std::string
GetStripeNumberString(unsigned long layout)
{
int n = GetStripeNumber(layout) + 1;
if (n < 256) {
return std::to_string(n);
} else {
return "none";
}
}
//--------------------------------------------------------------------------
//! Return checksum enum from env definition
//--------------------------------------------------------------------------
static unsigned long
GetChecksumFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.layout.checksum"))) {
XrdOucString xsum = val;
if (xsum == "adler") {
return kAdler;
}
if (xsum == "blake3") {
return kBLAKE3;
}
if (xsum == "crc32") {
return kCRC32;
}
if (xsum == "crc32c") {
return kCRC32C;
}
if (xsum == "md5") {
return kMD5;
}
if (xsum == "sha") {
return kSHA1;
}
if (xsum == "sha256") {
return kSHA256;
}
if (xsum == "crc64") {
return kCRC64;
}
if (xsum == "xxhash64") {
return kXXHASH64;
}
if (xsum == "hwh64") {
return kHWH64;
}
}
return kNone;
}
//--------------------------------------------------------------------------
//! Return block checksum enum from env definition
//--------------------------------------------------------------------------
static unsigned long
GetBlockChecksumFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.layout.blockchecksum"))) {
return GetBlockChecksumFromString(val);
}
return kNone;
}
//--------------------------------------------------------------------------
//! Return bandwidth string from env
//--------------------------------------------------------------------------
static std::string
GetBandwidthFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.iobw"))) {
return std::string(env.Get("eos.iobw"));
}
return "";
}
//--------------------------------------------------------------------------
//! Return iotype string from env
//--------------------------------------------------------------------------
static std::string
GetIotypeFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.iotype"))) {
return std::string(env.Get("eos.iotype"));
}
return "";
}
//--------------------------------------------------------------------------
//! Return block checksum enum from string representation
//--------------------------------------------------------------------------
static unsigned long
GetBlockChecksumFromString(const std::string& bxs_type)
{
if (bxs_type == "adler") {
return kAdler;
}
if (bxs_type == "crc32") {
return kCRC32;
}
if (bxs_type == "crc32c") {
return kCRC32C;
}
if (bxs_type == "md5") {
return kMD5;
}
if (bxs_type == "sha") {
return kSHA1;
}
return kNone;
}
//--------------------------------------------------------------------------
//! Return blocksize enum from env definition
//--------------------------------------------------------------------------
static unsigned long
GetBlocksizeFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.layout.blocksize"))) {
XrdOucString bs = val;
if (bs == "4k") {
return k4k;
}
if (bs == "64k") {
return k64k;
}
if (bs == "128k") {
return k128k;
}
if (bs == "512k") {
return k512k;
}
if (bs == "1M") {
return k1M;
}
if (bs == "4M") {
return k4M;
}
if (bs == "16M") {
return k16M;
}
if (bs == "64M") {
return k64M;
}
}
return 0;
}
//----------------------------------------------------------------------------
//! Check if this is a valid blocksize specification
//!
//! @param bs block size in human readable format
//!
//! @return true if valid, otherwise false
//----------------------------------------------------------------------------
static bool IsValidBlocksize(const std::string& bs)
{
std::set valid {"4k", "64k", "128k", "512k", "1M", "4M",
"16M", "64M"};
return (valid.find(bs) != valid.end());
}
//----------------------------------------------------------------------------
//! Return number of stripes enum from env definition
//----------------------------------------------------------------------------
static unsigned long
GetStripeNumberFromEnv(XrdOucEnv& env)
{
const char* val = 0;
if ((val = env.Get("eos.layout.nstripes"))) {
int n = atoi(val);
if (((n - 1) >= 0) && ((n - 1) <= 255)) {
return n;
}
}
return (1);
}
//----------------------------------------------------------------------------
//! Print a layout human readable
//----------------------------------------------------------------------------
static std::string PrintLayoutString(int layout)
{
std::string dump;
dump = GetLayoutTypeString(layout);
dump += ":";
dump += GetStripeNumberString(layout).c_str();
dump += "[";
dump += std::to_string(GetStripeNumber(layout) + 1 - GetRedundancyStripeNumber(
layout));
dump += "]";
dump += ":";
dump += GetChecksumString(layout);
dump += ":";
dump += GetBlockChecksumString(layout);
dump += "[";
dump += GetBlockSizeString(layout);
dump += "]";
return dump;
}
//----------------------------------------------------------------------------
//! Convert a conversion id string to layout id value to be stored directly
//! in the IFileMD object
//!
//! @param conv_id conversion id in the form of #~
//----------------------------------------------------------------------------
static unsigned long
GetLidFromConversionId(const std::string& conv_id)
{
using eos::common::StringConversion;
std::string space;
std::string layout;
std::string plctplcy;
if (!StringConversion::SplitKeyValue(conv_id, space, layout, "#")) {
return 0u;
}
if (layout.find('~') != std::string::npos) {
StringConversion::SplitKeyValue(layout, layout, plctplcy, "~");
}
unsigned long lid {0ul};
try {
lid = std::stoul(layout, 0, 16);
} catch (...) {}
return lid;
}
//----------------------------------------------------------------------------
//! Convert a = string to an env representation
//----------------------------------------------------------------------------
static const char*
GetEnvFromConversionIdString(XrdOucString& out,
const char* conversionlayoutidstring)
{
if (!conversionlayoutidstring) {
return NULL;
}
std::string keyval = conversionlayoutidstring;
std::string plctplcy;
// check if this is already a complete env representation
if ((keyval.find("eos.layout.type") != std::string::npos) &&
(keyval.find("eos.layout.nstripes") != std::string::npos) &&
(keyval.find("eos.layout.blockchecksum") != std::string::npos) &&
(keyval.find("eos.layout.checksum") != std::string::npos) &&
(keyval.find("eos.layout.blocksize") != std::string::npos) &&
(keyval.find("eos.space") != std::string::npos)) {
out = conversionlayoutidstring;
return out.c_str();
}
std::string space;
std::string layout;
if (!eos::common::StringConversion::SplitKeyValue(keyval, space, layout, "#")) {
return NULL;
}
if (((int)layout.find("~")) != STR_NPOS) {
eos::common::StringConversion::SplitKeyValue(layout, layout, plctplcy, "~");
}
errno = 0;
unsigned long long lid = strtoll(layout.c_str(), 0, 16);
if (errno) {
return NULL;
}
std::string group("");
std::string spaceStripped("");
if (eos::common::StringConversion::SplitKeyValue(space, spaceStripped,
group, ".")) {
space = spaceStripped;
}
out = "eos.layout.type=";
out += GetLayoutTypeString(lid);
out += "&eos.layout.nstripes=";
out += GetStripeNumberString(lid).c_str();
out += "&eos.layout.blockchecksum=";
out += GetBlockChecksumString(lid);
out += "&eos.layout.checksum=";
out += GetChecksumString(lid);
out += "&eos.layout.blocksize=";
out += GetBlockSizeString(lid);
out += "&eos.space=";
out += space.c_str();
if (plctplcy.length()) {
out += "&eos.placementpolicy=";
out += plctplcy.c_str();
}
if (group != "") {
out += "&eos.group=";
out += group.c_str();
}
return out.c_str();
}
//----------------------------------------------------------------------------
//! Map POSIX-like open flags to SFS open flags - used on the FUSE mount
//!
//! @param flags_sfs SFS open flags
//!
//! @return SFS-like open flags
//!
//----------------------------------------------------------------------------
static XrdSfsFileOpenMode
MapFlagsPosix2Sfs(int oflags)
{
XrdSfsFileOpenMode sfs_flags = SFS_O_RDONLY; // 0x0000
if (oflags & O_CREAT) {
sfs_flags |= SFS_O_CREAT;
}
if (oflags & O_RDWR) {
sfs_flags |= SFS_O_RDWR;
}
if (oflags & O_TRUNC) {
sfs_flags |= SFS_O_TRUNC;
}
if (oflags & O_WRONLY) {
sfs_flags |= SFS_O_WRONLY;
}
if (oflags & O_APPEND) {
sfs_flags |= SFS_O_RDWR;
}
// !!!
// Could also forward O_EXLC as XrdCl::OpenFlags::Flags::New but there is
// no corresponding flag in SFS
// !!!
return sfs_flags;
}
//----------------------------------------------------------------------------
//! Map SFS-like open flags to XrdCl open flags
//!
//! @param flags_sfs SFS open flags
//!
//! @return XrdCl-like open flags
//----------------------------------------------------------------------------
static XrdCl::OpenFlags::Flags
MapFlagsSfs2XrdCl(XrdSfsFileOpenMode flags_sfs)
{
XrdCl::OpenFlags::Flags xflags = XrdCl::OpenFlags::None;
if (flags_sfs & SFS_O_CREAT) {
xflags |= XrdCl::OpenFlags::Delete;
}
if (flags_sfs & SFS_O_WRONLY) {
xflags |= XrdCl::OpenFlags::Update;
}
if (flags_sfs & SFS_O_RDWR) {
xflags |= XrdCl::OpenFlags::Update;
}
if (flags_sfs & SFS_O_TRUNC) {
xflags |= XrdCl::OpenFlags::Delete;
}
if ((!(flags_sfs & SFS_O_TRUNC)) &&
(!(flags_sfs & SFS_O_WRONLY)) &&
(!(flags_sfs & SFS_O_CREAT)) &&
(!(flags_sfs & SFS_O_RDWR))) {
xflags |= XrdCl::OpenFlags::Read;
}
if (flags_sfs & SFS_O_POSC) {
xflags |= XrdCl::OpenFlags::POSC;
}
if (flags_sfs & SFS_O_NOWAIT) {
xflags |= XrdCl::OpenFlags::NoWait;
}
if (flags_sfs & SFS_O_RAWIO) {
// no idea what to do
}
if (flags_sfs & SFS_O_RESET) {
xflags |= XrdCl::OpenFlags::Refresh;
}
if (flags_sfs & SFS_O_REPLICA) {
// emtpy
}
if (flags_sfs & SFS_O_MKPTH) {
xflags |= XrdCl::OpenFlags::MakePath;
}
return xflags;
}
//----------------------------------------------------------------------------
//! Map SFS-like open mode to XrdCl open mode
//!
//! @param mode_sfs SFS open mode
//!
//! @return XrdCl-like open mode
//----------------------------------------------------------------------------
static XrdCl::Access::Mode
MapModeSfs2XrdCl(mode_t mode_sfs)
{
XrdCl::Access::Mode mode_xrdcl = XrdCl::Access::Mode::None;
if (mode_sfs & S_IRUSR) {
mode_xrdcl |= XrdCl::Access::Mode::UR;
}
if (mode_sfs & S_IWUSR) {
mode_xrdcl |= XrdCl::Access::Mode::UW;
}
if (mode_sfs & S_IXUSR) {
mode_xrdcl |= XrdCl::Access::Mode::UX;
}
if (mode_sfs & S_IRGRP) {
mode_xrdcl |= XrdCl::Access::Mode::GR;
}
if (mode_sfs & S_IWGRP) {
mode_xrdcl |= XrdCl::Access::Mode::GW;
}
if (mode_sfs & S_IXGRP) {
mode_xrdcl |= XrdCl::Access::Mode::GX;
}
if (mode_sfs & S_IROTH) {
mode_xrdcl |= XrdCl::Access::Mode::OR;
}
if (mode_sfs & S_IXOTH) {
mode_xrdcl |= XrdCl::Access::Mode::OX;
}
return mode_xrdcl;
}
//----------------------------------------------------------------------------
//! Get expected stripe size for RAIN layouts given the logical size of the
//! file
//!
//! @param lid layout id
//! @param file size logical file size
//!
//! @return expected physical files of the stripes
//----------------------------------------------------------------------------
static uint64_t ExpectedStripeSize(const uint32_t lid,
const uint64_t file_size)
{
if (!IsRain(lid)) {
return file_size;
}
uint64_t stripe_size = 0ull;
unsigned long layout_type = GetLayoutType(lid);
unsigned long block_size = GetBlocksize(lid);
unsigned long num_all_stripes = GetStripeNumber(lid) + 1;
unsigned long num_parity_stripes = GetRedundancyStripeNumber(lid);
unsigned long num_data_stripes = num_all_stripes - num_parity_stripes;
unsigned long rain_group_sz = 0ull;
unsigned long stripe_group_sz = 0ull;
if (layout_type == kRaidDP) {
rain_group_sz = num_data_stripes * num_data_stripes * block_size;
stripe_group_sz = num_data_stripes * block_size;
} else {
rain_group_sz = num_data_stripes * block_size;
stripe_group_sz = block_size;
}
stripe_size = std::ceil(1.0 * file_size / rain_group_sz) * stripe_group_sz;
stripe_size += OssXsBlockSize;
return stripe_size;
}
//--------------------------------------------------------------------------
//! Constructor
//--------------------------------------------------------------------------
LayoutId() = default;
//--------------------------------------------------------------------------
//! Destructor
//--------------------------------------------------------------------------
~LayoutId() = default;
};
EOSCOMMONNAMESPACE_END
#endif