//------------------------------------------------------------------------------ // File: FileSystem.hh // Author: Andreas-Joachim Peters - CERN //------------------------------------------------------------------------------ /************************************************************************ * EOS - the CERN Disk Storage System * * Copyright (C) 2011 CERN/Switzerland * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU General Public License as published by * * the Free Software Foundation, either version 3 of the License, or * * (at your option) any later version. * * * * This program is distributed in the hope that it will be useful, * * but WITHOUT ANY WARRANTY; without even the implied warranty of * * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * * GNU General Public License for more details. * * * * You should have received a copy of the GNU General Public License * * along with this program. If not, see .* ************************************************************************/ #ifndef __EOSCOMMON_FILESYSTEM_HH__ #define __EOSCOMMON_FILESYSTEM_HH__ #include "common/Namespace.hh" #include "common/Locators.hh" #include "common/table_formatter/TableCell.hh" #include "mq/SharedHashWrapper.hh" #include #include #ifndef __APPLE__ #include #else #include #include #endif #include #include //! Forward declarations namespace eos { namespace mq { class MessagingRealm; } } //namespace qclient //{ //class SharedManager; //} EOSCOMMONNAMESPACE_BEGIN //! Values for a boot status enum class BootStatus : int8_t { kOpsError = -2, kBootFailure = -1, kDown = 0, kBootSent = 1, kBooting = 2, kBooted = 3 }; //! Values for a drain status enum class DrainStatus : int8_t { kNoDrain = 0, kDrainPrepare = 1, kDrainWait = 2, kDraining = 3, kDrained = 4, kDrainStalling = 5, kDrainExpired = 6, kDrainFailed = 7 }; //! Values describing if a filesystem is online or offline //! (combination of multiple conditions) enum class ActiveStatus : int8_t { kUndefined = -1, kOffline = 0, kOnline = 1, kOverload = 2 }; //! Values for a configuration status - stored persistently in the //! filesystem configuration //! forcing an int8_t as we wouldn't need a larger range of integers for these statuses enum class ConfigStatus : int8_t { kUnknown = -1, kOff = 0, kEmpty, kDrainDead, kGroupDrain, kDrain, kRO, kWO, kRW }; inline bool operator<(ConfigStatus one, ConfigStatus two) { return static_cast(one) < static_cast(two); } inline bool operator<=(ConfigStatus one, ConfigStatus two) { return static_cast(one) <= static_cast(two); } #define EOS_TAPE_FSID 65535 #define EOS_TAPE_MODE_T (0x10000000ll) //------------------------------------------------------------------------------ //! Contains a batch of key-value updates on the attributes of a filesystem. //! Build up the batch, then submit it through applyBatch. //! //! Note: If the batch mixes durable, transient, and/or local values, three //! "transactions" will take place: //! //! - All durable values will be updated atomically. //! - All transient values will be updated atomically. //! - All local values will be updated atomically. //! //! However, the entire operation as a whole will NOT be atomic. //------------------------------------------------------------------------------ class FileSystemUpdateBatch { public: //---------------------------------------------------------------------------- //! Type definitions //---------------------------------------------------------------------------- using fsid_t = uint32_t; //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- FileSystemUpdateBatch(); //---------------------------------------------------------------------------- //! Set filesystem ID - durable. //---------------------------------------------------------------------------- void setId(fsid_t fsid); //---------------------------------------------------------------------------- //! Set the draining status - local. //---------------------------------------------------------------------------- void setDrainStatusLocal(DrainStatus status); //---------------------------------------------------------------------------- //! Set durable string. //! //! All observers of this filesystem are guaranteed to receive the update //! eventually, and all updates are guaranteed to be applied in the same //! order for all observers. //! //! If two racing setStringDurable are attempted on the same key, only one //! will survive, and all observers will agree as to which one survives. //---------------------------------------------------------------------------- void setStringDurable(const std::string& key, const std::string& value); //---------------------------------------------------------------------------- //! Set transient string. Depending on network instabilities, //! process restarts, or the phase of the moon, some or all observers of this //! filesystem may or may not receive the update. //! //! Transient updates may be applied out of order, and if multiple subscribers //! try to modify the same value, it's possible that observers will not all //! converge on a single consistent value. //---------------------------------------------------------------------------- void setStringTransient(const std::string& key, const std::string& value); //---------------------------------------------------------------------------- //! Set local string. This node, and only this node will store the value, //! and only until process restart. //---------------------------------------------------------------------------- void setStringLocal(const std::string& key, const std::string& value); //---------------------------------------------------------------------------- //! Set durable int64_t - serialize as string automatically. //---------------------------------------------------------------------------- void setLongLongDurable(const std::string& key, int64_t value); //---------------------------------------------------------------------------- //! Set transient int64_t - serialize as string automatically. //---------------------------------------------------------------------------- void setLongLongTransient(const std::string& key, int64_t value); //---------------------------------------------------------------------------- //! Set local int64_t - serialize as string automatically. //---------------------------------------------------------------------------- void setLongLongLocal(const std::string& key, int64_t value); //---------------------------------------------------------------------------- //! Get MQ update batch //---------------------------------------------------------------------------- const mq::SharedHashWrapper::Batch& getBatch() const; private: mq::SharedHashWrapper::Batch mBatch; }; //------------------------------------------------------------------------------ //! Describes how to locate an FST: host + port. //------------------------------------------------------------------------------ class FstLocator { public: //---------------------------------------------------------------------------- //! Empty constructor //---------------------------------------------------------------------------- FstLocator(); //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- FstLocator(const std::string& host, int port); //---------------------------------------------------------------------------- //! Get host //---------------------------------------------------------------------------- std::string getHost() const; //---------------------------------------------------------------------------- //! Get port //---------------------------------------------------------------------------- int getPort() const; //---------------------------------------------------------------------------- //! Get queuepath //---------------------------------------------------------------------------- std::string getQueuePath() const; //---------------------------------------------------------------------------- //! Get host:port //---------------------------------------------------------------------------- std::string getHostPort() const; //---------------------------------------------------------------------------- //! Try to parse from queuepath //---------------------------------------------------------------------------- static bool fromQueuePath(const std::string& queuepath, FstLocator& out); private: std::string mHost; int mPort; }; //------------------------------------------------------------------------------ //! Describes a FileSystem Group: //! - Space //! - Index //------------------------------------------------------------------------------ class GroupLocator { public: //---------------------------------------------------------------------------- //! Empty constructor //---------------------------------------------------------------------------- GroupLocator(); //---------------------------------------------------------------------------- //! Get group (space.index) //---------------------------------------------------------------------------- std::string getGroup() const; //---------------------------------------------------------------------------- //! Get space //---------------------------------------------------------------------------- std::string getSpace() const; //---------------------------------------------------------------------------- //! Get index //---------------------------------------------------------------------------- int getIndex() const; //---------------------------------------------------------------------------- //! Parse full group (space.index) //! //! NOTE: In case parsing fails, "out" will still be filled //! with "description.0" to match legacy behaviour. //---------------------------------------------------------------------------- static bool parseGroup(const std::string& description, GroupLocator& out); private: std::string mGroup; std::string mSpace; int mIndex = 0; }; //------------------------------------------------------------------------------ //! Describes critical parameters of a FileSystem, which are necessary to have //! when registering a filesystem on the MGM. //! //! A FileSystemLocator can physically locate a FileSystem, but we still can't //! operate it on the MGM without knowing more information. (id, group, uuid) //------------------------------------------------------------------------------ class FileSystemCoreParams { public: //---------------------------------------------------------------------------- //! Constructor //---------------------------------------------------------------------------- FileSystemCoreParams(uint32_t id, const FileSystemLocator& fsLocator, const GroupLocator& grpLocator, const std::string& uuid, ConfigStatus cfg, const std::string& sharedfs); //---------------------------------------------------------------------------- //! Get locator //---------------------------------------------------------------------------- const FileSystemLocator& getLocator() const; //---------------------------------------------------------------------------- //! Get group locator //---------------------------------------------------------------------------- const GroupLocator& getGroupLocator() const; //---------------------------------------------------------------------------- //! Get id //---------------------------------------------------------------------------- uint32_t getId() const; //---------------------------------------------------------------------------- //! Get uuid //---------------------------------------------------------------------------- std::string getUuid() const; //---------------------------------------------------------------------------- //! Get current ConfigStatus //---------------------------------------------------------------------------- ConfigStatus getConfigStatus() const; //---------------------------------------------------------------------------- //! Get queuepath //---------------------------------------------------------------------------- std::string getQueuePath() const; //---------------------------------------------------------------------------- //! Get host //---------------------------------------------------------------------------- std::string getHost() const; //---------------------------------------------------------------------------- //! Get hostport //---------------------------------------------------------------------------- std::string getHostPort() const; //---------------------------------------------------------------------------- //! Get FST queue //---------------------------------------------------------------------------- std::string getFSTQueue() const; //---------------------------------------------------------------------------- //! Get group (space.index) //---------------------------------------------------------------------------- std::string getGroup() const; //---------------------------------------------------------------------------- //! Get space //---------------------------------------------------------------------------- std::string getSpace() const; //---------------------------------------------------------------------------- //! Get shared file system name //---------------------------------------------------------------------------- std::string getSharedFs() const; private: uint32_t mFsId; FileSystemLocator mLocator; GroupLocator mGroup; std::string mUuid; ConfigStatus mConfigStatus; std::string mSharedFs; }; //------------------------------------------------------------------------------ //! Base Class abstracting the internal representation of a filesystem inside //! the MGM and FST //------------------------------------------------------------------------------ class FileSystem { protected: //! This filesystem's locator object FileSystemLocator mLocator; //! Locator for shared hash SharedHashLocator mHashLocator; //! Messaging realm mq::MessagingRealm* mRealm; //! boot status stored inside the object not the hash BootStatus mInternalBootStatus; public: //! Struct & Type definitions typedef uint32_t fsid_t; ///< File system ID type //! Snapshot Structure of a filesystem struct fs_snapshot_t { fsid_t mId; std::string mQueue; std::string mQueuePath; std::string mPath; std::string mErrMsg; std::string mGroup; std::string mUuid; std::string mHost; std::string mHostPort; std::string mProxyGroup; std::string mSharedFs; std::string mS3Credentials; int8_t mFileStickyProxyDepth; int32_t mPort; std::string mGeoTag; std::string mForceGeoTag; size_t mPublishTimestamp; int mGroupIndex; std::string mSpace; BootStatus mStatus; ConfigStatus mConfigStatus; DrainStatus mDrainStatus; ActiveStatus mActiveStatus; long long mHeadRoom; unsigned int mErrCode; time_t mBootSentTime; time_t mBootDoneTime; double mDiskUtilization; double mDiskWriteRateMb; double mDiskReadRateMb; double mNetEthRateMiB; double mNetInRateMiB; double mNetOutRateMiB; double mNominalFilled; double mDiskFilled; long long mDiskCapacity; long long mDiskFreeBytes; long mDiskType; long mDiskBsize; long mDiskBlocks; // @todo (esindril) to be dropped long mDiskBused; long mDiskBfree; long mDiskBavail; long mDiskFiles; long mDiskFused; long mDiskFfree; long mFiles; long mDiskNameLen; long mDiskRopen; long mDiskWopen; long mMaxDiskRopen; long mMaxDiskWopen; long mScanIoRate; ///< Maximum scan rate in MB/s long mScanEntryInterval; ///< Time after which a scanned file is rescanned long mScanRainEntryInterval; ///< Time after which a scanned rain file is rescanned long mScanDiskInterval; ///< Time after which the disk scanner runs again long mScanNsInterval; ///< Time after which the disk scanner runs again long mScanNsRate; ///< Max ns scan rate in entries/s ///< Time in seconds after which fsck inconsistencies are refreshed long mFsckRefreshInterval; time_t mGracePeriod; time_t mDrainPeriod; //-------------------------------------------------------------------------- //! Empty constructor //-------------------------------------------------------------------------- fs_snapshot_t(); //-------------------------------------------------------------------------- //! "Absorb" all information contained within coreParams into this object. //! Fields which are not present in coreParams (ie mNetInRateMiB) remain //! unchanged. //-------------------------------------------------------------------------- void fillFromCoreParams(const FileSystemCoreParams& coreParams); }; //---------------------------------------------------------------------------- //! Constructor //! @param queuepath Named Queue to specify the receiver filesystem of //! modifications e.g. /eos//fst/ //! @param queue Named Queue to specify the receiver of modifications //! e.g. /eos//fst //! @param som Handle to the shared object manager to store filesystem //! key-value pairs //! @param bc2mgm If true we broad cast to the management server //---------------------------------------------------------------------------- FileSystem(const FileSystemLocator& locator, mq::MessagingRealm* realm, bool b2mgm = false); //---------------------------------------------------------------------------- //! Destructor //---------------------------------------------------------------------------- virtual ~FileSystem() = default; //---------------------------------------------------------------------------- // Get underlying hash locator //---------------------------------------------------------------------------- SharedHashLocator getHashLocator() const; //---------------------------------------------------------------------------- // Enums //---------------------------------------------------------------------------- //! Value indication the way a boot message should be executed on an FST node enum eBootConfig { kBootOptional = 0, kBootForced = 1, kBootResync = 2 }; //---------------------------------------------------------------------------- // Get file system status as a string //---------------------------------------------------------------------------- static const char* GetStatusAsString(BootStatus status); static const char* GetDrainStatusAsString(DrainStatus status); static const char* GetConfigStatusAsString(ConfigStatus status); static const char* GetActiveStatusAsString(ActiveStatus status); //---------------------------------------------------------------------------- //! Parse a string status into the enum value //---------------------------------------------------------------------------- static BootStatus GetStatusFromString(const char* ss); //---------------------------------------------------------------------------- //! Parse a drain status into the enum value //---------------------------------------------------------------------------- static DrainStatus GetDrainStatusFromString(const char* ss); //---------------------------------------------------------------------------- //! Parse a configuration status into the enum value //---------------------------------------------------------------------------- static ConfigStatus GetConfigStatusFromString(const char* ss); //---------------------------------------------------------------------------- //! Convert input to file system id //! //! @param value input string //! //! @return file system id if conversion successful, otherwise 0 //---------------------------------------------------------------------------- static eos::common::FileSystem::fsid_t ConvertToFsid(const std::string& value); //---------------------------------------------------------------------------- //! Serializes hash contents as follows 'key1=val1 key2=val2 ... keyn=valn' //! but return only keys that don't start with filter_prefix. If specified, //! the string values will be curl encoded //! //! @param contents key value pairs //! @param filter_prefix prefix used for filtering keys //! //! @return string representation of the content for the hash //---------------------------------------------------------------------------- static std::string SerializeWithFilter(const std::map& contents, std::list filter_prefixes); //---------------------------------------------------------------------------- //! Cache Members //---------------------------------------------------------------------------- BootStatus cStatus; ///< cache value of the status time_t cStatusTime; ///< unix time stamp of last update of the cached status XrdSysMutex cStatusLock; ///< lock protecting the cached status std::atomic cConfigStatus; ///< cached value of the config status XrdSysMutex cConfigLock; ///< lock protecting the cached config status time_t cConfigTime; ///< unix time stamp of last update of the cached config status //---------------------------------------------------------------------------- //! Apply the given batch of updates //---------------------------------------------------------------------------- bool applyBatch(const FileSystemUpdateBatch& batch); //---------------------------------------------------------------------------- //! Apply the given core parameters //---------------------------------------------------------------------------- bool applyCoreParams(const FileSystemCoreParams& params); //---------------------------------------------------------------------------- //! Set a single local long long //---------------------------------------------------------------------------- bool SetLongLongLocal(const std::string& key, int64_t value); //---------------------------------------------------------------------------- //! Set a key-value pair in a filesystem and evt. broadcast it. //---------------------------------------------------------------------------- bool SetString(const char* key, const char* str, bool broadcast = true); //---------------------------------------------------------------------------- //! Set a filesystem ID. //---------------------------------------------------------------------------- bool SetId(fsid_t fsid) { return SetString("id", std::to_string(fsid).c_str()); } //---------------------------------------------------------------------------- //! Set a double value by name and evt. broadcast it. //---------------------------------------------------------------------------- bool SetDouble(const char* key, double f) { return SetString(key, std::to_string(f).c_str()); } //---------------------------------------------------------------------------- //! Set a long long value and evt. broadcast it. //---------------------------------------------------------------------------- bool SetLongLong(const char* key, long long l, bool broadcast = true) { return SetString(key, std::to_string(l).c_str(), broadcast); } //---------------------------------------------------------------------------- //! Set the filesystem status. //---------------------------------------------------------------------------- bool SetStatus(BootStatus status, bool broadcast = true) { mInternalBootStatus = status; return SetString("stat.boot", GetStatusAsString(status), broadcast); } //---------------------------------------------------------------------------- //! Set the draining status //---------------------------------------------------------------------------- bool SetDrainStatus(DrainStatus status) { return SetString("local.drain", GetDrainStatusAsString(status), false); } //---------------------------------------------------------------------------- //! Remove a key from a filesystem and evt. broadcast it. //---------------------------------------------------------------------------- bool RemoveKey(const char* key, bool broadcast = true); //---------------------------------------------------------------------------- //! Set the activation status //---------------------------------------------------------------------------- void SetActiveStatus(ActiveStatus active); //---------------------------------------------------------------------------- //! Get the activation status //---------------------------------------------------------------------------- inline ActiveStatus GetActiveStatus() const { return mActStatus; } //---------------------------------------------------------------------------- //! Get all keys in a vector of strings. //---------------------------------------------------------------------------- bool GetKeys(std::vector& keys); //---------------------------------------------------------------------------- //! Get the string value by key //---------------------------------------------------------------------------- std::string GetString(const char* key); //---------------------------------------------------------------------------- //! Get a long long value by key //---------------------------------------------------------------------------- long long GetLongLong(const char* key); //---------------------------------------------------------------------------- //! Get a double value by key //---------------------------------------------------------------------------- double GetDouble(const char* key); //---------------------------------------------------------------------------- //! Return the filesystem id. //---------------------------------------------------------------------------- fsid_t GetId() { return (fsid_t) GetLongLong("id"); } //---------------------------------------------------------------------------- //! Return the filesystem queue path. //---------------------------------------------------------------------------- std::string GetQueuePath() { return mLocator.getQueuePath(); } //---------------------------------------------------------------------------- //! Return the filesystem host //---------------------------------------------------------------------------- std::string GetHost() { return mLocator.getHost(); } //---------------------------------------------------------------------------- //! Return the filesystem queue name //---------------------------------------------------------------------------- std::string GetQueue() { return mLocator.getFSTQueue(); } //---------------------------------------------------------------------------- //! Return the filesystem path //---------------------------------------------------------------------------- std::string GetPath() { return mLocator.getStoragePath(); } //---------------------------------------------------------------------------- //! Return the filesystem status (via a cache) //---------------------------------------------------------------------------- BootStatus GetStatus(bool cached = false); //---------------------------------------------------------------------------- //! Get internal boot status //---------------------------------------------------------------------------- BootStatus GetInternalBootStatus() { return mInternalBootStatus; } //---------------------------------------------------------------------------- //! Return the drain status //---------------------------------------------------------------------------- DrainStatus GetDrainStatus() { return GetDrainStatusFromString(GetString("local.drain").c_str()); } //---------------------------------------------------------------------------- //! Return the configuration status (via cache) //---------------------------------------------------------------------------- ConfigStatus GetConfigStatus(bool cached = false); //---------------------------------------------------------------------------- //! Retrieve FileSystem's core parameters //---------------------------------------------------------------------------- FileSystemCoreParams getCoreParams(); //---------------------------------------------------------------------------- //! Snapshots all variables of a filesystem into a snapshot struct //! //! @param fs snapshot struct to be filled //! @param dolock indicates if the shared hash representing the filesystem has //! to be locked or not //! //! @return true if successful, otherwise false //---------------------------------------------------------------------------- bool SnapShotFileSystem(FileSystem::fs_snapshot_t& fs, bool dolock = true); //---------------------------------------------------------------------------- //! Function printing the file system info to the table //---------------------------------------------------------------------------- void Print(TableHeader& table_mq_header, TableData& table_mq_data, std::string listformat, const std::string& filter = ""); //---------------------------------------------------------------------------- //! Store a configuration key-val pair. //! Internally, these keys are not prefixed with 'stat.' //! //! @param key key string //! @param val value string //---------------------------------------------------------------------------- void CreateConfig(std::string& key, std::string& val); //---------------------------------------------------------------------------- //! Get heartbeatTime //---------------------------------------------------------------------------- time_t getLocalHeartbeatTime() const; //---------------------------------------------------------------------------- //! Get local heartbeat delta //---------------------------------------------------------------------------- int getLocalHeartbeatDelta() const; //---------------------------------------------------------------------------- //! Set heartbeatTime //---------------------------------------------------------------------------- void setLocalHeartbeatTime(time_t t); //---------------------------------------------------------------------------- //! Check if local heartbeat is recent enough //---------------------------------------------------------------------------- bool hasHeartbeat() const; //---------------------------------------------------------------------------- //! Delete shared hash object corresponding to this file system and also //! broadcast the message. This should be called only when an explicit removal //! of the file system is request though "fs rm" //---------------------------------------------------------------------------- void DeleteSharedHash(); //-------------------------------------------------------------------------- //! Get used bytes //-------------------------------------------------------------------------- uint64_t GetUsedbytes(); //-------------------------------------------------------------------------- //! Get space name //-------------------------------------------------------------------------- std::string GetSpace(); //---------------------------------------------------------------------------- //! Get SharedFs name //---------------------------------------------------------------------------- std::string getSharedFs(); private: std::atomic mActStatus; ///< File system active status }; EOSCOMMONNAMESPACE_END; #endif