// ---------------------------------------------------------------------- // File: ClusterDataTypes // Author: Abhishek Lekshmanan - CERN // ---------------------------------------------------------------------- /************************************************************************ * EOS - the CERN Disk Storage System * * Copyright (C) 2023 CERN/Switzerland * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see .* ************************************************************************/ #ifndef EOS_CLUSTERDATATYPES_HH #define EOS_CLUSTERDATATYPES_HH #include "common/FileSystem.hh" #include namespace eos::mgm::placement { using fsid_t = eos::common::FileSystem::fsid_t; // We use a item_id to represent a storage element, negative numbers represent // storage elements in the hierarchy, ie. groups/racks/room/site etc. using item_id_t = int32_t; using epoch_id_t = uint64_t; using ConfigStatus = eos::common::ConfigStatus; using ActiveStatus = eos::common::ActiveStatus; // A struct representing a disk, this is the lowest level of the hierarchy, // disk ids map 1:1 to fsids, however it is necessary that the last bit of fsid_t // is not used, as we use a int32_t for the rest of the placement hierarchy. // the struct is packed to 8 bytes, so upto 8192 disks // can fit in a single 64kB cache, it is recommended to keep this struct aligned struct Disk { fsid_t id; mutable std::atomic config_status {ConfigStatus::kUnknown}; mutable std::atomic active_status {ActiveStatus::kUndefined}; mutable std::atomic weight{0}; // we really don't need floating point precision mutable std::atomic percent_used{0}; Disk () : id(0) {} explicit Disk(fsid_t _id) : id(_id) {} Disk(fsid_t _id, ConfigStatus _config_status, ActiveStatus _active_status, uint8_t _weight, uint8_t _percent_used = 0) : id(_id), config_status(_config_status), active_status(_active_status), weight(_weight), percent_used(_percent_used) {} // explicit copy constructor as atomic types are not copyable Disk(const Disk& other) : Disk(other.id, other.config_status.load(std::memory_order_relaxed), other.active_status.load(std::memory_order_relaxed), other.weight.load(std::memory_order_relaxed), other.percent_used.load(std::memory_order_relaxed)) { } Disk& operator=(const Disk& other) { id = other.id; config_status.store(other.config_status.load(std::memory_order_relaxed), std::memory_order_relaxed); weight.store(other.weight.load(std::memory_order_relaxed), std::memory_order_relaxed); percent_used.store(other.percent_used.load(std::memory_order_relaxed), std::memory_order_relaxed); return *this; } friend bool operator<(const Disk& l, const Disk& r) { return l.id < r.id; } std::string to_string() const { std::stringstream ss; ss << "id: " << id << "\n" << "ConfigStatus: " << common::FileSystem::GetConfigStatusAsString(config_status.load(std::memory_order_relaxed)) <<"\n" << "ActiveStatus: " << common::FileSystem::GetActiveStatusAsString(active_status.load(std::memory_order_relaxed)) << "\n" << "Weight: " << weight.load(std::memory_order_relaxed) << "\n" << "UsedPercent: " << percent_used.load(std::memory_order_relaxed); return ss.str(); } }; static_assert(sizeof(Disk) == 8, "Disk data type not aligned to 8 bytes!"); // some common storage elements, these could be user defined in the future enum class StdBucketType : uint8_t { GROUP=0, RACK, ROOM, SITE, ROOT, COUNT}; constexpr uint8_t get_bucket_type(StdBucketType t) { return static_cast(t); } inline std::string BucketTypeToStr(StdBucketType t) { switch (t) { case StdBucketType::GROUP: return "group"; case StdBucketType::RACK: return "rack"; case StdBucketType::ROOM: return "room"; case StdBucketType::SITE: return "site"; case StdBucketType::ROOT: return "root"; default: return "unknown"; } } // Determining placement of replicas for a file // We need to understand how many storage elements we select at each level // of the hierarchy, for example for a 2 replica file, with 2 sites, // we'd select 1 per site, and then going further down the hierarchy, we'd have // to select 1 per room etc. until we reach our last abstraction at the group // where we'd need to select as many replicas as we have left, in this case 2. // we really don't want a tree that's more than 16 levels deep? constexpr uint8_t MAX_PLACEMENT_HEIGHT = 16; using selection_rules_t = std::array; static selection_rules_t kDefault2Replica = {-1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; struct Bucket { item_id_t id; uint32_t total_weight; uint8_t bucket_type; std::vector items; Bucket() = default; Bucket(item_id_t _id, uint8_t type) : id(_id), total_weight(0), bucket_type(type) { } friend bool operator<(const Bucket& l, const Bucket& r) { return l.id < r.id; } std::string to_string() const { std::stringstream ss; ss << "id: " << id <<"\n" << "Total Weight: " << total_weight << "\n" << "Bucket Type: " << BucketTypeToStr(static_cast(bucket_type)) << "\nItem List: "; for (const auto& it : items) { ss << it << ", "; } return ss.str(); } }; // Constant to offset the group id, so group ids would be starting from this offset // in memory they'd be stored at -group_id constexpr int kBaseGroupOffset = -10; struct ClusterData { std::vector disks; std::vector buckets; bool setDiskStatus(fsid_t id, ConfigStatus status) { if (id > disks.size()) { return false; } disks[id - 1].config_status.store(status, std::memory_order_release); return true; } bool setDiskWeight(fsid_t id, uint8_t weight) { if (id > disks.size()) { return false; } disks[id - 1].weight.store(weight, std::memory_order_release); return true; } std::string getDisksAsString() const { std::string result_str; for (const auto& d: disks) { result_str.append(d.to_string()); result_str.append("\n"); } return result_str; } std::string getBucketsAsString() const { std::string result_str; for (const auto& b: buckets) { if (b.id==0 && b.bucket_type == 0) { continue; } result_str.append(b.to_string()); result_str.append("\n"); } return result_str; } }; inline bool isValidBucketId(item_id_t id, const ClusterData& data) { return id < 0 && (-id < data.buckets.size()); } } // eos::mgm::placement #endif // EOS_CLUSTERDATATYPES_HH