// ----------------------------------------------------------------------
// File: QoS.cc
// Author: Mihai Patrascoiu - CERN
// ----------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2019 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see .*
************************************************************************/
//------------------------------------------------------------------------------
// This file is included in the source code of XrdMgmOfs.cc
//------------------------------------------------------------------------------
namespace
{
//----------------------------------------------------------------------------
//! Helper class for retrieving QoS properties.
//!
//! The class takes as input an entry metadata pointer,
//! which it will use to query for properties.
//!
//! The "qos_class" property retrieval mechanism:
//!
//! Initially, an attempt is made to retrieve it from extended attributes.
//! If that fails, an attempt is made to match the list of attributes
//! against a defined QoS class. If no match is found, "null" is returned.
//!
//! The class should be called under lock to ensure thread safety.
//----------------------------------------------------------------------------
template
class QoSGetter
{
public:
//--------------------------------------------------------------------------
//! Constructor
//--------------------------------------------------------------------------
QoSGetter(T _md) : md(_md) {}
//--------------------------------------------------------------------------
//! Destructor
//--------------------------------------------------------------------------
virtual ~QoSGetter() = default;
//--------------------------------------------------------------------------
//! Retrieve all QoS properties
//--------------------------------------------------------------------------
eos::IFileMD::QoSAttrMap All();
//--------------------------------------------------------------------------
//! Retrieve CDMI-specific QoS properties
//--------------------------------------------------------------------------
eos::IFileMD::QoSAttrMap CDMI();
//--------------------------------------------------------------------------
//! Retrieve QoS property by key
//--------------------------------------------------------------------------
std::string Get(const std::string& key) const;
private:
//--------------------------------------------------------------------------
// Methods retrieving a particular QoS property
//--------------------------------------------------------------------------
std::string Attr(const char* key) const;
std::string ChecksumType() const;
std::string Class() const;
std::string DiskSize() const;
std::string LayoutType() const;
std::string Id() const;
std::string Path() const;
std::string Placement() const;
std::string Replica() const;
std::string Size() const;
T md; ///< pointer to entry metadata
///< dispatch table based on QoS key word
std::map> dispatch {
{
"checksum", [this]()
{
return QoSGetter::ChecksumType();
}
},
{
"current_qos", [this]()
{
return QoSGetter::Class();
}
},
{
"disksize", [this]()
{
return QoSGetter::DiskSize();
}
},
{
"layout", [this]()
{
return QoSGetter::LayoutType();
}
},
{
"id", [this]()
{
return QoSGetter::Id();
}
},
{
"path", [this]()
{
return QoSGetter::Path();
}
},
{
"placement", [this]()
{
return QoSGetter::Placement();
}
},
{
"replica", [this]()
{
return QoSGetter::Replica();
}
},
{
"size", [this]()
{
return QoSGetter::Size();
}
},
{
"target_qos", [this]()
{
return QoSGetter::Attr("user.eos.qos.target");
}
},
};
};
//----------------------------------------------------------------------------
// Retrieve all QoS properties
//----------------------------------------------------------------------------
template
eos::IFileMD::QoSAttrMap QoSGetter::All()
{
eos::IFileMD::QoSAttrMap qosMap = CDMI();
for (auto& it : dispatch) {
qosMap.emplace(it.first, it.second());
}
return qosMap;
}
//----------------------------------------------------------------------------
// Retrieve CDMI-specific QoS properties
//----------------------------------------------------------------------------
template
eos::IFileMD::QoSAttrMap QoSGetter::CDMI()
{
eos::IFileMD::QoSAttrMap cdmiMap;
std::string qos_name = QoSGetter::Get("current_qos");
if (gOFS->mQoSClassMap.count(qos_name)) {
const QoSClass qos_class = gOFS->mQoSClassMap.at(qos_name);
std::ostringstream splacement;
size_t count = 0;
splacement << "[";
for (auto& location : qos_class.locations) {
splacement << " " << location;
if (++count < qos_class.locations.size()) {
splacement << ",";
}
}
splacement << " ]";
cdmiMap[CDMI_REDUNDANCY_TAG] = std::to_string(qos_class.cdmi_redundancy);
cdmiMap[CDMI_LATENCY_TAG] = std::to_string(qos_class.cdmi_latency);
cdmiMap[CDMI_PLACEMENT_TAG] = splacement.str();
}
return cdmiMap;
}
//----------------------------------------------------------------------------
// Retrieve QoS property by key
//----------------------------------------------------------------------------
template
std::string QoSGetter::Get(const std::string& key) const
{
std::string value = "";
auto it = dispatch.find(key);
if (it != dispatch.end()) {
value = it->second();
}
return value;
}
//----------------------------------------------------------------------------
// Methods retrieving a particular QoS property
//
// Depending on the property, they are implemened via templated
// or specialized templated functions.
//----------------------------------------------------------------------------
//----------------------------------------------------------------------------
// Retrieve extended attributes
//----------------------------------------------------------------------------
template
std::string QoSGetter::Attr(const char* key) const
{
std::string value = "null";
if (md->hasAttribute(key)) {
value = md->getAttribute(key);
}
return value;
}
//----------------------------------------------------------------------------
// Retrieve checksum type
//----------------------------------------------------------------------------
template<>
std::string QoSGetter::ChecksumType() const
{
return eos::common::LayoutId::GetChecksumStringReal(md->getLayoutId());
}
template<>
std::string QoSGetter::ChecksumType() const
{
std::string value = QoSGetter::Attr("sys.forced.checksum");
int checksum_id = eos::common::LayoutId::GetChecksumFromString(value);
return eos::common::LayoutId::GetChecksumStringReal(checksum_id);
}
//----------------------------------------------------------------------------
// Retrieve size
//----------------------------------------------------------------------------
template<>
std::string QoSGetter::Size() const
{
return std::to_string(md->getSize());
}
template<>
std::string QoSGetter::Size() const
{
return std::to_string(md->getTreeSize());
}
//----------------------------------------------------------------------------
// Retrieve disk size
//----------------------------------------------------------------------------
template<>
std::string QoSGetter::DiskSize() const
{
uint64_t physicalSize = md->getSize() *
eos::common::LayoutId::GetSizeFactor(md->getLayoutId());
return std::to_string(physicalSize);
}
template<>
std::string QoSGetter::DiskSize() const
{
return QoSGetter::Size();
}
//----------------------------------------------------------------------------
// Retrieve layout type
//----------------------------------------------------------------------------
template<>
std::string QoSGetter::LayoutType() const
{
return eos::common::LayoutId::GetLayoutTypeString(md->getLayoutId());
}
template<>
std::string QoSGetter::LayoutType() const
{
return QoSGetter::Attr("sys.forced.layout");
}
//----------------------------------------------------------------------------
// Retrieve id
//----------------------------------------------------------------------------
template
std::string QoSGetter::Id() const
{
return std::to_string(md->getId());
}
//----------------------------------------------------------------------------
// Retrieve path
//----------------------------------------------------------------------------
template
std::string QoSGetter::Path() const
{
std::string path = "null";
try {
path = gOFS->eosView->getUri(md.get());
} catch (eos::MDException& e) {
eos_static_debug("msg=\"exception retrieving path\" fxid=%08llx "
"ec=%d emsg=\"%s\"", md->getId(),
e.getErrno(), e.getMessage().str().c_str());
}
return path;
}
//----------------------------------------------------------------------------
// Retrieve placement
//----------------------------------------------------------------------------
template
std::string QoSGetter::Placement() const
{
std::string placement = "null";
try {
std::string path = gOFS->eosView->getUri(md.get());
// compute parent path only for filemd types
// (avoids specialized template helper function for path deduction)
if constexpr(std::is_same::value) {
path = eos::common::Path(path).GetParentPath();
}
eos::mgm::Scheduler::tPlctPolicy plctplcy;
eos::common::VirtualIdentity vid;
std::string targetgeotag;
XrdOucErrInfo error;
XrdOucEnv env;
eos::IContainerMD::XAttrMap attrmap;
gOFS->_attr_ls(path.c_str(), error, vid, 0, attrmap);
Policy::GetPlctPolicy(path.c_str(), attrmap, vid, env, plctplcy,
targetgeotag);
placement = Scheduler::PlctPolicyString(plctplcy);
} catch (eos::MDException& e) {
eos_static_debug("msg=\"exception retrieving path\" fxid=%08llx "
"ec=%d emsg=\"%s\"", md->getId(),
e.getErrno(), e.getMessage().str().c_str());
}
return placement;
}
//----------------------------------------------------------------------------
// Retrieve replica
//----------------------------------------------------------------------------
template<>
std::string QoSGetter::Replica() const
{
return std::to_string(md->getNumLocation());
}
template<>
std::string QoSGetter::Replica() const
{
return QoSGetter::Attr("sys.forced.nstripes");
}
//----------------------------------------------------------------------------
// Retrieve QoS class
//----------------------------------------------------------------------------
template
std::string QoSGetter::Class() const
{
std::string qos_class = QoSGetter::Attr("user.eos.qos.class");
if (qos_class == "null") {
eos::IFileMD::QoSAttrMap attributes;
attributes["checksum"] = QoSGetter::ChecksumType();
attributes["layout"] = QoSGetter::LayoutType();
attributes["placement"] = QoSGetter::Placement();
attributes["replica"] = QoSGetter::Replica();
for (const auto& it_class : gOFS->mQoSClassMap) {
const auto& class_attributes = it_class.second.attributes;
bool match = true;
for (auto it = attributes.begin();
it != attributes.end() && match; it++) {
if ((!class_attributes.count(it->first)) ||
(class_attributes.at(it->first) != it->second)) {
match = false;
}
}
if (match) {
qos_class = it_class.second.name;
break;
}
}
}
return qos_class;
}
//----------------------------------------------------------------------------
//! Helper function to check the given = is a valid QoS property
//!
//! @param key QoS key
//! @param value QoS value
//!
//! @return true if pair is valid, false otherwise
//----------------------------------------------------------------------------
bool IsValidQoSProperty(const std::string& key, const std::string& value)
{
if (key == "placement") {
return Scheduler::PlctPolicyFromString(value) != -1;
} else if (key == "layout") {
return eos::common::LayoutId::GetLayoutFromString(value) != -1;
} else if (key == "checksum") {
return eos::common::LayoutId::GetChecksumFromString(value) != -1;
} else if (key == "replica") {
int number = std::stoi(value);
return (number >= 1 && number <= 16);
}
return false;
}
//----------------------------------------------------------------------------
//! Helper function to extract a QoS property,
//! given the entry metadata object and the key
//!
//! @param md the ile or container metadata object
//! @param key the QoS key
//!
//! @return the request QoS property
//----------------------------------------------------------------------------
std::string QoSValueFromMd(const eos::FileOrContainerMD& md,
const char* key)
{
if (md.file) {
return QoSGetter {md.file} .Get(key);
}
return QoSGetter {md.container} .Get(key);
}
}
//------------------------------------------------------------------------------
// List QoS properties for a given entry - low-level API
//------------------------------------------------------------------------------
int
XrdMgmOfs::_qos_ls(const char* path, XrdOucErrInfo& error,
eos::common::VirtualIdentity& vid,
eos::IFileMD::QoSAttrMap& map,
bool only_cdmi)
{
static const char* epname = "qos_ls";
EXEC_TIMING_BEGIN("QoSLs");
gOFS->MgmStats.Add("QoSLs", vid.uid, vid.gid, 1);
errno = 0;
eos_info("msg=\"list QoS values\" path=%s only_cdmi=%d", path, only_cdmi);
eos::Prefetcher::prefetchItemAndWait(gOFS->eosView, path);
std::string cmd_retrieved_qos;
try {
eos::common::RWMutexReadLock vlock(gOFS->eosViewRWMutex);
eos::FileOrContainerMD md = gOFS->eosView->getItem(path).get();
if (md.file) {
map = (only_cdmi) ? QoSGetter {md.file} .CDMI()
: QoSGetter {md.file} .All();
} else {
map = (only_cdmi) ? QoSGetter {md.container} .CDMI()
: QoSGetter {md.container} .All();
if (map.count("current_qos") && map["current_qos"] != "null") {
cmd_retrieved_qos = map["current_qos"];
}
}
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception retrieving item metadata\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
// Check if identified QoS class needs to be updated
// Note: applies only to containers
if (!errno && cmd_retrieved_qos.length()) {
eos::common::RWMutexWriteLock wlock(gOFS->eosViewRWMutex);
try {
eos::IContainerMDPtr cmd = gOFS->eosView->getContainer(path);
std::string attr_qos;
if (cmd->hasAttribute("user.eos.qos.class")) {
attr_qos = cmd->getAttribute("user.eos.qos.class");
}
if (cmd_retrieved_qos != attr_qos) {
eos_info("msg=\"setting QoS class match in extended attributes\" "
"path=%s qos_class=%s", path, cmd_retrieved_qos.c_str());
cmd->setAttribute("user.eos.qos.class", cmd_retrieved_qos);
eosView->updateContainerStore(cmd.get());
}
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception setting extended attributes\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
}
EXEC_TIMING_END("QoSLs");
if (errno) {
std::string keys = (only_cdmi) ? "cdmi" : "all";
return Emsg(epname, error, errno, "list QoS values",
SSTR("keys=" << keys << " path=" << path).c_str());
}
return SFS_OK;
}
//------------------------------------------------------------------------------
// Get QoS property for a given entry by key - low-level API
//------------------------------------------------------------------------------
int
XrdMgmOfs::_qos_get(const char* path, XrdOucErrInfo& error,
eos::common::VirtualIdentity& vid,
const char* key,
XrdOucString& value)
{
static const char* epname = "qos_get";
EXEC_TIMING_BEGIN("QoSGet");
gOFS->MgmStats.Add("QoSGet", vid.uid, vid.gid, 1);
errno = 0;
eos_info("msg=\"get QoS value\" path=%s key=%s",
path, (key ? key : "(null)"));
if (!key) {
return Emsg(epname, error, EINVAL, "get QoS value - empty key");
}
eos::Prefetcher::prefetchItemAndWait(gOFS->eosView, path);
std::string cmd_retrieved_qos;
try {
eos::common::RWMutexReadLock vlock(gOFS->eosViewRWMutex);
eos::FileOrContainerMD md = gOFS->eosView->getItem(path).get();
value = QoSValueFromMd(md, key).c_str();
if (md.container && !strcmp(key, "current_qos") && value != "null") {
cmd_retrieved_qos = value.c_str();
}
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception retrieving item metadata\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
// Check if identified QoS class needs to be updated
// Note: applies only to containers
if (!errno && cmd_retrieved_qos.length()) {
eos::common::RWMutexWriteLock wlock(gOFS->eosViewRWMutex);
try {
eos::IContainerMDPtr cmd = gOFS->eosView->getContainer(path);
std::string attr_qos;
if (cmd->hasAttribute("user.eos.qos.class")) {
attr_qos = cmd->getAttribute("user.eos.qos.class").c_str();
}
if (cmd_retrieved_qos != attr_qos) {
eos_info("msg=\"setting QoS class match in extended attributes\" "
"path=%s qos_class=%s", path, value.c_str());
cmd->setAttribute("user.eos.qos.class", value.c_str());
eosView->updateContainerStore(cmd.get());
}
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception setting extended attributes\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
}
EXEC_TIMING_END("QoSGet");
if (!value.length()) {
return Emsg(epname, error, EINVAL, "get QoS value - invalid key",
SSTR(key << " path=" << path).c_str());
}
if (errno) {
return Emsg(epname, error, errno, "get QoS value",
SSTR(key << " path=" << path).c_str());
}
return SFS_OK;
}
//----------------------------------------------------------------------------
// Schedule QoS properties for a given entry - low-level API
// If no value is provided for a QoS property, it will be left unchanged.
//----------------------------------------------------------------------------
int
XrdMgmOfs::_qos_set(const char* path, XrdOucErrInfo& error,
eos::common::VirtualIdentity& vid,
const eos::mgm::QoSClass& qos,
std::string& conversion_id)
{
using eos::common::LayoutId;
static const char* epname = "qos_set";
EXEC_TIMING_BEGIN("QoSSet");
gOFS->MgmStats.Add("QoSSet", vid.uid, vid.gid, 1);
errno = 0;
eos_info("msg=\"set QoS class\" path=%s qos_class=%s",
path, qos.name.c_str());
// Validate QoS class properties
for (const auto& it : qos.attributes) {
if (!IsValidQoSProperty(it.first, it.second)) {
eos_static_err("msg=\"invalid QoS property %s=%s\"",
it.first.c_str(), it.second.c_str());
return Emsg(epname, error, EINVAL,
"set QoS class due to invalid fields", it.first.c_str());
}
}
eos::FileOrContainerMD md;
std::string current_qos;
eos::Prefetcher::prefetchItemAndWait(gOFS->eosView, path);
try {
eos::common::RWMutexReadLock vlock(gOFS->eosViewRWMutex);
md = gOFS->eosView->getItem(path).get();
current_qos = QoSValueFromMd(md, "current_qos");
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception retrieving file metadata\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
if (!md.file && !md.container) {
return Emsg(epname, error, errno, "retrieve item metadata", path);
}
// Abort if current QoS is same as target QoS
if (current_qos == qos.name) {
return Emsg(epname, error, EINVAL,
"set QoS class identical with current class", path);
}
if (md.file) {
// For files:
// - create a new conversion job,
// - store the QoS target extended attributes
eos::IFileMDPtr fmd = 0;
eos::common::FileId::fileid_t fileid = 0;
eos::IFileMD::location_t fsid = 0;
LayoutId::layoutid_t layoutid = 0;
unsigned long current_layout = 0;
unsigned long current_checksumid = 0;
unsigned long current_nstripes = 0;
try {
eos::common::RWMutexReadLock vlock(gOFS->eosViewRWMutex);
fmd = gOFS->eosView->getFile(path);
fileid = fmd->getId();
layoutid = fmd->getLayoutId();
// Extract current QoS properties
current_layout = LayoutId::GetLayoutType(layoutid);
current_checksumid = LayoutId::GetChecksum(layoutid);
current_nstripes = LayoutId::GetStripeNumber(layoutid) + 1;
if (fmd->getNumLocation()) {
const auto& locations_vect = fmd->getLocations();
fsid = *(locations_vect.begin());
}
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception retrieving file metadata\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
return Emsg(epname, error, e.getErrno(), "retrieve file metadata", path);
}
// Extract current scheduling space
std::string space = "";
{
eos::common::RWMutexReadLock vlock(FsView::gFsView.ViewMutex);
auto filesystem = FsView::gFsView.mIdView.lookupByID(fsid);
if (!filesystem) {
return Emsg(epname, error, errno, "retrieve filesystem location", path);
}
space = filesystem->GetString("schedgroup");
}
// Extract new layout components from QoS class
int layout = -1;
int checksumid = -1;
int nstripes = -1;
std::string policy = "";
for (const auto& it : qos.attributes) {
if (it.first == "layout") {
layout = LayoutId::GetLayoutFromString(it.second);
} else if (it.first == "replica") {
nstripes = std::stoi(it.second);
} else if (it.first == "checksum") {
checksumid = LayoutId::GetChecksumFromString(it.second);
} else if (it.first == "placement") {
policy = it.second;
}
}
// Generate layout id
layout = (layout != -1) ? layout : current_layout;
nstripes = (nstripes != -1) ? nstripes : current_nstripes;
checksumid = (checksumid != -1) ? checksumid : current_checksumid;
layoutid = LayoutId::GetId(layout, checksumid, nstripes,
LayoutId::k4M, LayoutId::kCRC32C,
LayoutId::GetRedundancyStripeNumber(layoutid));
// Generate conversion id
conversion_id = SSTR(
std::hex << std::setw(16) << std::setfill('0') << fileid
<< ":" << space
<< "#" << std::setw(8) << std::setfill('0') << layoutid
<< (policy.length() ? ("~" + policy) : ""));
eos_info("msg=\"set QoS class - scheduling conversion job\" path=%s "
"layout=%d nstripes=%d checksum=%d policy=%s space=%s "
"conversion_file=%s", path, layout, nstripes, checksumid,
policy.c_str(), space.c_str(), conversion_id.c_str());
// Create conversion job
std::string conversion_file = SSTR(
gOFS->MgmProcConversionPath.c_str() << "/" << conversion_id);
eos::common::VirtualIdentity rootvid = eos::common::VirtualIdentity::Root();
// Push conversion job to QuarkDB
if (!gOFS->mConverterDriver->ScheduleJob(fileid, conversion_id)) {
return Emsg(epname, error, errno, "schedule QoS conversion job to QuarkDB",
conversion_id.c_str());
}
// Add the target QoS attribute
try {
eos::common::RWMutexWriteLock wlock(gOFS->eosViewRWMutex);
fmd = gOFS->eosView->getFile(path);
fmd->setAttribute("user.eos.qos.target", qos.name);
eosView->updateFileStore(fmd.get());
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception setting extended attributes\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
}
} else {
// For containers, only set the QoS target extended attribute
try {
eos::common::RWMutexWriteLock wlock(gOFS->eosViewRWMutex);
eos::IContainerMDPtr cmd = gOFS->eosView->getContainer(path);
cmd->setAttribute("user.eos.qos.target", qos.name);
eosView->updateContainerStore(cmd.get());
} catch (eos::MDException& e) {
errno = e.getErrno();
eos_debug("msg=\"exception setting extended attributes\" path=%s "
"ec=%d emsg=\"%s\"", path, e.getErrno(),
e.getMessage().str().c_str());
return Emsg(epname, error, e.getErrno(), "set extended attributes", path);
}
conversion_id = SSTR(path << "|" << qos.name);
}
EXEC_TIMING_END("QoSSet");
if (errno) {
return Emsg(epname, error, errno, "set QoS properties", path);
}
return SFS_OK;
}