//------------------------------------------------------------------------------ // File: Locators.cc // Author: Georgios Bitzes - CERN //------------------------------------------------------------------------------ /************************************************************************ * EOS - the CERN Disk Storage System * * Copyright (C) 2019 CERN/Switzerland * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see .* ************************************************************************/ #include "common/Namespace.hh" #include "common/Locators.hh" #include "common/Logging.hh" #include "common/StringConversion.hh" #include "common/StringTokenizer.hh" #include "common/InstanceName.hh" #include "common/Assert.hh" #include "common/FileSystem.hh" #include "common/ParseUtils.hh" #include "common/StringUtils.hh" EOSCOMMONNAMESPACE_BEGIN //------------------------------------------------------------------------------ // Constructor //------------------------------------------------------------------------------ FileSystemLocator::FileSystemLocator(const std::string& _host, int _port, const std::string& _storagepath) : host(_host), port(_port), storagepath(_storagepath) { storageType = FileSystemLocator::parseStorageType(_storagepath); } //------------------------------------------------------------------------------ // Try to parse a "queuepath" //------------------------------------------------------------------------------ bool FileSystemLocator::fromQueuePath(const std::string& queuepath, FileSystemLocator& out) { std::string queue = queuepath; if (!startsWith(queue, "/eos/")) { return false; } queue.erase(0, 5); //---------------------------------------------------------------------------- // Chop /eos/, extract host+port //---------------------------------------------------------------------------- size_t slashLocation = queue.find("/"); if (slashLocation == std::string::npos) { return false; } std::string hostPort = std::string(queue.begin(), queue.begin() + slashLocation); queue.erase(0, slashLocation); //---------------------------------------------------------------------------- // Separate host from port //---------------------------------------------------------------------------- size_t separator = hostPort.find(":"); if (separator == std::string::npos) { return false; } out.host = std::string(hostPort.begin(), hostPort.begin() + separator); hostPort.erase(0, separator + 1); int64_t port; if (!ParseInt64(hostPort, port)) { return false; } out.port = port; //---------------------------------------------------------------------------- // Chop "/fst/", extract local path //---------------------------------------------------------------------------- if (!startsWith(queue, "/fst")) { return false; } queue.erase(0, 4); out.storagepath = queue; if (out.storagepath.size() < 2) { // Empty, or "/"? Reject return false; } out.storageType = FileSystemLocator::parseStorageType(out.storagepath); if (out.storageType == StorageType::Unknown) { return false; } return true; } //---------------------------------------------------------------------------- //! Parse storage type from storage path string //---------------------------------------------------------------------------- FileSystemLocator::StorageType FileSystemLocator::parseStorageType( const std::string& storagepath) { if (storagepath.find("/") == 0) { return StorageType::Local; } else if (storagepath.find("root://") == 0) { return StorageType ::Xrd; } else if (storagepath.find("s3://") == 0) { return StorageType::S3; } else if (storagepath.find("dav://") == 0) { return StorageType::WebDav; } else if (storagepath.find("http://") == 0) { return StorageType::HTTP; } else if (storagepath.find("https://") == 0) { return StorageType::HTTPS; } else { return StorageType::Unknown; } } //------------------------------------------------------------------------------ // Get host //------------------------------------------------------------------------------ std::string FileSystemLocator::getHost() const { return host; } //------------------------------------------------------------------------------ // Get hostport, concatenated together as "host:port" //------------------------------------------------------------------------------ std::string FileSystemLocator::getHostPort() const { return SSTR(host << ":" << port); } //------------------------------------------------------------------------------ // Get queuepath //------------------------------------------------------------------------------ std::string FileSystemLocator::getQueuePath() const { return SSTR("/eos/" << host << ":" << port << "/fst" << storagepath); } //------------------------------------------------------------------------------ // Get "FST queue", ie /eos/example.com:3002/fst //------------------------------------------------------------------------------ std::string FileSystemLocator::getFSTQueue() const { return SSTR("/eos/" << host << ":" << port << "/fst"); } //------------------------------------------------------------------------------ // Get port //------------------------------------------------------------------------------ int FileSystemLocator::getPort() const { return port; } //------------------------------------------------------------------------------ // Get storage path //------------------------------------------------------------------------------ std::string FileSystemLocator::getStoragePath() const { return storagepath; } //---------------------------------------------------------------------------- // Get storage type //---------------------------------------------------------------------------- FileSystemLocator::StorageType FileSystemLocator::getStorageType() const { return storageType; } //---------------------------------------------------------------------------- // Check whether filesystem is local or remote //---------------------------------------------------------------------------- bool FileSystemLocator::isLocal() const { return storageType == StorageType::Local; } //------------------------------------------------------------------------------ // Empty constructor //------------------------------------------------------------------------------ SharedHashLocator::SharedHashLocator() { mInitialized = false; } //------------------------------------------------------------------------------ // Constructor: Pass the EOS instance name, BaseView type, and name. // // Once we drop the MQ entirely, the instance name can be removed. //------------------------------------------------------------------------------ SharedHashLocator::SharedHashLocator(const std::string& instanceName, Type type, const std::string& name) : mInitialized(true), mInstanceName(instanceName), mType(type), mName(name) { switch (mType) { case Type::kSpace: { mMqSharedHashPath = SSTR("/config/" << instanceName << "/space/" << name); mBroadcastQueue = "/eos/*/mgm"; break; } case Type::kGroup: { mMqSharedHashPath = SSTR("/config/" << instanceName << "/group/" << name); mBroadcastQueue = "/eos/*/mgm"; break; } case Type::kNode: { std::string hostPort = eos::common::StringConversion::GetHostPortFromQueue( name.c_str()).c_str(); mMqSharedHashPath = SSTR("/config/" << instanceName << "/node/" << hostPort); mBroadcastQueue = SSTR("/eos/" << hostPort << "/fst"); mName = hostPort; break; } case Type::kGlobalConfigHash: { mMqSharedHashPath = SSTR("/config/" << instanceName << "/mgm/"); mBroadcastQueue = "/eos/*/mgm"; break; } default: { eos_assert("should never reach here"); } } } //------------------------------------------------------------------------------ //! Constructor: Same as above, but auto-discover instance name. //------------------------------------------------------------------------------ SharedHashLocator::SharedHashLocator(Type type, const std::string& name) : SharedHashLocator(InstanceName::get(), type, name) {} //------------------------------------------------------------------------------ // Constructor: Special case for FileSystems, as they work a bit differently // than the rest. //------------------------------------------------------------------------------ SharedHashLocator::SharedHashLocator(const FileSystemLocator& fsLocator, bool bc2mgm): mInitialized(true), mType(Type::kFilesystem) { mMqSharedHashPath = fsLocator.getQueuePath(); mBroadcastQueue = fsLocator.getFSTQueue(); mFilesystemChannel = SSTR(fsLocator.getHostPort() << "||" << fsLocator.getStoragePath()); if (bc2mgm) { mBroadcastQueue = "/eos/*/mgm"; } } //------------------------------------------------------------------------------ //! Convenience "Constructors": Make locator for space, group, node //------------------------------------------------------------------------------ SharedHashLocator SharedHashLocator::makeForSpace(const std::string& name) { return SharedHashLocator(Type::kSpace, name); } SharedHashLocator SharedHashLocator::makeForGroup(const std::string& name) { return SharedHashLocator(Type::kGroup, name); } SharedHashLocator SharedHashLocator::makeForNode(const std::string& name) { return SharedHashLocator(Type::kNode, name); } SharedHashLocator SharedHashLocator::makeForGlobalHash() { return SharedHashLocator(Type::kGlobalConfigHash, ""); } //------------------------------------------------------------------------------ // Get "config queue" for shared hash //------------------------------------------------------------------------------ std::string SharedHashLocator::getConfigQueue() const { return mMqSharedHashPath; } //------------------------------------------------------------------------------ // Get "broadcast queue" for shared hash //------------------------------------------------------------------------------ std::string SharedHashLocator::getBroadcastQueue() const { return mBroadcastQueue; } //---------------------------------------------------------------------------- // Check if this object is actually pointing to something //---------------------------------------------------------------------------- bool SharedHashLocator::empty() const { return !mInitialized; } //------------------------------------------------------------------------------ // Produce SharedHashLocator by parsing config queue //------------------------------------------------------------------------------ bool SharedHashLocator::fromConfigQueue(const std::string& configQueue, SharedHashLocator& out) { std::vector parts = common::StringTokenizer::split>(configQueue, '/'); std::reverse(parts.begin(), parts.end()); if (parts.empty() || parts.back() != "config") { return false; } parts.pop_back(); if (parts.empty()) { return false; } SharedHashLocator::Type type; std::string instanceName = parts.back(); parts.pop_back(); if (parts.empty()) { return false; } else if (parts.back() == "node") { type = Type::kNode; } else if (parts.back() == "space") { type = Type::kSpace; } else if (parts.back() == "group") { type = Type::kGroup; } else if (parts.back() == "mgm" && parts.size() == 1u) { type = Type::kGlobalConfigHash; out = SharedHashLocator(instanceName, type, ""); return true; } else { return false; } parts.pop_back(); if (parts.empty()) { return false; } std::string name = parts.back(); parts.pop_back(); if (!parts.empty()) { return false; } if (instanceName.empty() || name.empty()) { return false; } out = SharedHashLocator(instanceName, type, name); return true; } //------------------------------------------------------------------------------ // Get QDB key for this queue //------------------------------------------------------------------------------ std::string SharedHashLocator::getQDBKey() const { switch (mType) { case Type::kSpace: { return SSTR("eos-hash||space||" << mName); } case Type::kGroup: { return SSTR("eos-hash||group||" << mName); } case Type::kNode: { return SSTR("eos-hash||node||" << mName); } case Type::kGlobalConfigHash: { return SSTR("eos-global-config-hash"); } case Type::kFilesystem: { return SSTR("eos-hash||fs||" << mFilesystemChannel); } default: { eos_assert("should never reach here"); return ""; } } } EOSCOMMONNAMESPACE_END