// @project      The CERN Tape Archive (CTA)
// @copyright    Copyright © 2015-2022 CERN
// @license      This program is free software, distributed under the terms of the GNU General Public
//               Licence version 3 (GPL Version 3), copied verbatim in the file "COPYING". You can
//               redistribute it and/or modify it under the terms of the GPL Version 3, or (at your
//               option) any later version.
//
//               This program is distributed in the hope that it will be useful, but WITHOUT ANY
//               WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
//               PARTICULAR PURPOSE. See the GNU General Public License for more details.
//
//               In applying this licence, CERN does not waive the privileges and immunities
//               granted to it by virtue of its status as an Intergovernmental Organization or
//               submit itself to any jurisdiction.
syntax = "proto2";
package cta.objectstore.serializers;

// The types of the objects. It will be used to allow introspection
// for the contents.
enum ObjectType {
  RootEntry_t = 0;
  AgentRegister_t = 1;
  Agent_t = 2;
  DriveRegister_t = 3;
  DriveState_t = 4;
  SchedulerGlobalLock_t = 6;
  ArchiveRequest_t = 7;
  RetrieveRequest_t = 8;
  ArchiveQueue_t = 9;
  ArchiveQueueShard_t = 90;
  RetrieveQueue_t = 10;
  RetrieveQueueShard_t = 100;
  RepackRequest_t = 11;
  RepackIndex_t = 12;
  RepackQueue_t = 13;
  GenericObject_t = 1000;
}

// The base object header. This will allow introspection and automatic
// "garbage collection", i.e. returning an unprocessed object belonging
// to a dead agent to the right queue or container.
// - The type allows automatic management of the content without knowing its
// payload.
// - The version allows multi schema to be used at the object level (smooth schema
// evolution, on an object by object basis).
// - The owner is the authoritative source of ownership for the object.
// It allows arbitration of the actual owner in case the object is pointed
// to by several containers (during a transition, or after a failure).
// - The backup owner allows the object to be returned to a previous container
// in case of failure of a owner (when it is an agent).
message ObjectHeader {
  required ObjectType type = 1;
  required uint64 version = 2;
  required string owner = 3;
  required string backupowner = 4;
  required bytes payload = 5;
}

// A placeholder object for the implementation of neutral object handlers
message GenericObject {
}

// ===========================  Root Entry =====================================
// The objects making up the root entry.

// User information record for requesters
message RequesterIdentity {
  required string name = 10;
  required string group = 11;
}

// Pointer to the drive register (register defined further)
message DriveRegisterPointer {
  required string address = 90;
  required EntryLog log = 91;
}

// Pointer to the agent register (register defined further)
message AgentRegisterPointer {
  required string address = 100;
  required EntryLog log = 101;
}

message RepackIndexPointer {
  required string address = 105;
}

message RepackQueuePointer {
  required string address = 107;
}


// Pointer to the scheduler global lock
message SchedulerGlobalLockPointer {
  required string address = 110;
  required EntryLog log = 111;
}

// Pointer to the archive queue
message ArchiveQueuePointer {
  required string address = 120;
  required string name = 121;
}

// Pointer to the tape queue
message RetrieveQueuePointer {
  required string address = 130;
  required string vid = 131;
}

// The root entry. This entry contains all the most static information, i.e.
// the admin handled configuration information
message RootEntry {
  repeated ArchiveQueuePointer archive_queue_to_transfer_for_user_pointers = 1050;
  repeated ArchiveQueuePointer archive_queue_failed_pointers = 1062;
  repeated ArchiveQueuePointer archive_queue_to_report_for_user_pointers = 1068;
  repeated ArchiveQueuePointer archive_queue_to_transfer_for_repack_pointers = 1069;
  repeated ArchiveQueuePointer archive_queue_to_report_to_repack_for_success_pointers = 1072;
  repeated ArchiveQueuePointer archive_queue_to_report_to_repack_for_failure_pointers = 1073;
  repeated RetrieveQueuePointer retrieve_queue_to_transfer_for_user_pointers = 1060;
  repeated RetrieveQueuePointer retrieve_queue_to_report_for_user_pointers = 1063;
  repeated RetrieveQueuePointer retrieve_queue_failed_pointers = 1065;
  repeated RetrieveQueuePointer retrieve_queue_to_report_to_repack_for_success_pointers = 1066;
  repeated RetrieveQueuePointer retrieve_queue_to_report_to_repack_for_failure_pointers = 1067;
  repeated RetrieveQueuePointer retrieve_queue_to_transfer_for_repack_pointers = 1071;
  optional DriveRegisterPointer driveregisterpointer = 1070;
  optional AgentRegisterPointer agentregisterpointer = 1080;
  optional RepackIndexPointer repackindexpointer = 1085;
  optional RepackQueuePointer repackrequestspendingqueuepointer = 1086;
  optional RepackQueuePointer repackrequeststoexpandqueuepointer = 1088;
  optional string agentregisterintent = 1090;
  optional SchedulerGlobalLockPointer schedulerlockpointer = 1100;
}

//=========== Sub-objects ======================================================

// ------------- Agent handling ------------------------------------------------

// The agent object represents a running process. It is a payload to an object
// itself, and it can be owned by a watchdog or a global agent registry, which
// lists the agents not yet watched by a watchdog.
// - The description is a freetext string used for logging.
// - The watchdogs themselves own the agents they are watching, so the agents can
// be returned to the agent registry when they fail.
// - The heartbeat is incremented from time to time by the agent.
// - The timeout_us is the amount of microseconds after which the absence of
// change to the heartbeat can be interpreted as agent failure.
// - The ownedobject list is an ownership intent log that points to the objects
// that the agent is about to create, intends to own, or fully owns.
// The objects in this list can be considered for being returned to a backup
// owner.
message Agent {
  required string description = 2000;
  required uint64 heartbeat = 2001;
  required uint64 timeout_us = 2002;
  repeated string ownedobjects = 2003;
  optional bool being_garbage_collected = 2004 [default = false];
  optional bool gc_needed = 2005 [default = false];
}

message AgentRegister {
  repeated string agents = 2100;
  repeated string untrackedagents = 2101;
}

// ------------- Mount criteria and quota  -------------------------------------

message MountCriteria {
  required uint64 maxFilesBeforeMount = 3200;
  required uint64 maxBytesBeforeMount = 3201;
  required uint64 maxSecondsBeforeMount = 3202;
  required uint32 quota = 3203;
}

message TapeFile {
  required string vid = 9120;
  required uint64 fseq = 9121;
  required uint64 blockid = 9122;
  required uint64 filesize = 9123;
  required uint32 copynb = 9124;
  required uint64 creationtime = 9125;
  //required string checksumtype = 9126;     DEPRECATED
  //required string checksumvalue = 9127;    DEPRECATED
  required bytes checksumblob = 9128;
}

message DiskFileInfo {
  //required bytes recoveryblob = 8900;      DEPRECATED
  //required string group = 8910;            DEPRECATED
  //required string owner = 8930;            DEPRECATED
  required uint32 owner_uid = 8920;
  required uint32 gid = 8925;
  required string path = 8940;
}

message ArchiveFile {
  required uint64 archivefileid = 4351;
  required uint64 filesize = 4352;
  required string diskfileid = 4353;
  required string diskinstance= 4354;
  required DiskFileInfo diskfileinfo= 4355;
  //required string checksumtype = 4356;     DEPRECATED
  //required string checksumvalue = 4357;    DEPRECATED
  required bytes checksumblob = 4362;
  required uint64 creationtime = 4358;
  repeated TapeFile tapefiles = 4359;
  required uint64 reconciliationtime = 4360;
  required string storageclass = 4361;
}

// ------------- Drives handling  ----------------------------------------------

message DiskSpaceReservation {
  // Each drive keeps tabs of its intended
  required string disk_system_name = 5100;
  required uint64 reserved_bytes = 5110;
}

message DriveConfig {
  required string category = 13000;
  required string key = 13001;
  required string value = 13002;
  required string source = 13003;
}

message DriveState {
  required string drivename = 5000;
  required string host = 5001;
  required string logicallibrary = 5002;
  optional uint64 sessionid = 5003;
  optional uint64 bytestransferedinsession = 5004;
  optional uint64 filestransferedinsession = 5005;
  optional double latestbandwidth = 5006;
  optional uint64 sessionstarttime = 5007;
  optional uint64 mountstarttime = 5008;
  optional uint64 transferstarttime = 5009;
  optional uint64 unloadstarttime = 5010;
  optional uint64 unmountstarttime = 5011;
  optional uint64 drainingstarttime = 5012;
  optional uint64 downorupstarttime = 5013;
  optional uint64 probestarttime = 5026;
  optional uint64 cleanupstarttime = 5014;
  optional uint64 lastupdatetime = 5015;
  optional uint64 startstarttime = 5016;
  optional uint64 shutdowntime = 5027;
  required uint32 mounttype = 5017;
  required uint32 drivestatus = 5018;
  required bool desiredUp = 5019;
  required bool desiredForceDown = 5020;
  optional string currentvid = 5021;
  optional string cta_version = 5035;
  optional uint64 current_priority = 5028;
  optional string current_activity = 5029;
  optional double current_activity_weight = 5030;
  optional string currenttapepool = 5022;
  optional uint32 nextmounttype = 5023;
  optional string nextvid = 5024;
  optional string nexttapepool = 5025;
  optional uint64 next_priority = 5031;
  optional string next_activity = 5032;
  optional double next_activity_weight = 5033;
  repeated DiskSpaceReservation disk_space_reservations = 5034;
  optional string dev_file_name = 5036;
  optional string raw_library_slot = 5037;
  repeated DriveConfig drive_config = 5038;
  optional string reason = 5039;
  optional string comment = 5040;
  optional string current_vo = 5041;
  optional string next_vo = 5042;
// TODO: implement or remove  required EntryLog creationlog = 5023;
}

message DriveStatePointer {
  required string drivename = 7010;
  required string drivestateaddress = 7011;
}

message DriveRegister {
  repeated DriveStatePointer drives = 7001;
}

// ------------- Scheduler global lock handling  -------------------------------

message SchedulerGlobalLock {
  required uint64 nextmountid = 8000;
}

message EntryLog {
  required string username = 8950;
  required string host = 8960;
  required uint64 time = 8970;
}

message MountPolicy {
  required string name = 8980;
  required uint64 archivepriority = 8981;
  required uint64 archiveminrequestage = 8982;
  required uint64 retrievepriority = 8983;
  required uint64 retieveminrequestage = 8984;
  required EntryLog creationlog = 8986;
  required EntryLog lastmodificationlog = 8987;
  required string comment = 8988;
}

// ------------- Archive Jobs --------------------------------------------------

// The status of the individual archive jobs. The jobs are materialised
// by table entries in the ArchiveRequest.
// This life cycle represented by the following enum

enum ArchiveJobStatus {
  AJS_ToTransferForUser = 1;
  AJS_ToReportToUserForTransfer = 2;
  AJS_Complete = 3;
  AJS_ToReportToUserForFailure = 997;
  AJS_Failed = 998;
  AJS_Abandoned = 999;
  AJS_ToTransferForRepack = 1001;
  AJS_ToReportToRepackForSuccess = 1005;
  AJS_ToReportToRepackForFailure = 1006;
}

message ArchiveJob {
  required uint32 copynb = 4400;
  required string tapepool = 4401;
  required string archivequeueaddress = 4402;
  required string owner = 4403;
  required ArchiveJobStatus status = 4404;
  required uint32 totalretries = 4405;
  required uint32 retrieswithinmount = 4406;
  required uint64 lastmountwithfailure = 4407;
  required uint32 maxtotalretries = 4408;
  required uint32 maxretrieswithinmount = 4409;
  repeated string failurelogs = 4410;
  required uint32 maxreportretries = 4411;
  required uint32 totalreportretries = 4412;
  repeated string reportfailurelogs = 4413;
}

message JobDestinationVid {
  optional string destination_vid = 4414;
  optional uint32 copy_nb = 4415;
}

message ArchiveRequestRepackInfo {
  required string repack_request_address = 4450;
  required string file_buffer_url = 4453;
  required uint64 fseq = 4455;
  repeated JobDestinationVid jobs_destination = 4456;
}

message ArchiveRequest {
  required uint64 archivefileid = 8990;
  required MountPolicy mountpolicy = 8995; //to be removed for cta/CTA#1107
  optional string mountpolicyname = 9100;  //TODO: Once it has been deployed, make it required and remove mountpolicy
  //required string checksumtype = 9000;     DEPRECATED
  //required string checksumvalue = 9010;    DEPRECATED
  required bytes checksumblob = 9011;
  required uint64 creationtime = 9015;
  required uint64 reconcilationtime = 9017;
  required DiskFileInfo diskfileinfo = 9040;
  required string diskfileid = 9050;
  required string diskinstance = 9055;
  required string archivereporturl = 9057;
  required string archiveerrorreporturl = 9058;
  required uint64 filesize = 9060;
  required RequesterIdentity requester = 9070;
  required string srcurl = 9080;
  required string storageclass = 9090;
  required EntryLog creationlog = 9091;
  repeated ArchiveJob jobs = 9092;
  required bool reportdecided = 9093;
  required bool isrepack = 9095;
  optional ArchiveRequestRepackInfo repack_info = 9097;
  optional bool isfailed = 9099 [default = false];
}

// ------------- Retrieve Jobs -------------------------------------------------

// The status of the individual retrieve jobs. The jobs are materialised
// by table entries in the RetrieveRequest.
// This life cycle represented by the following enum
// There is no complete state as the completion of one jobs implies the
// completion of the whole requests, and leads to the immediate deletion
// of the request.

enum RetrieveJobStatus {
  RJS_ToTransfer = 1;
  RJS_ToReportToUserForFailure = 997;
  RJS_Failed = 998;
  RJS_ToReportToRepackForSuccess = 1002; //For Retrieve request created by a Repack request
  RJS_ToReportToRepackForFailure = 1003;
}

message SchedulerRetrieveRequest {
  required RequesterIdentity requester = 9100;
  required uint64 ArchiveFileId = 9101;
  required string dstURL = 9102;
  required DiskFileInfo diskfileinfo = 9103;
  required EntryLog entrylog = 9106;
  required string retrieveerrorreporturl = 9110;
  required bool isverifyonly = 9111;
}

message RetrieveJob {
  required uint32 copynb = 9200;
  required uint32 maxtotalretries = 9201;
  required uint32 maxretrieswithinmount = 9202;
  required uint32 retrieswithinmount = 9203;
  required uint32 totalretries = 9204;
  required RetrieveJobStatus status = 9205;
  required uint64 lastmountwithfailure = 9206;
  repeated string failurelogs = 9207;
  required uint32 maxreportretries = 9208;
  required uint32 totalreportretries = 9209;
  repeated string reportfailurelogs = 9210;
  optional bool isfailed = 9099 [default = false];
}

message RetrieveRequestArchiveRoute {
  required uint32 copynb = 9400;
  required string tapepool = 9410;
}

message RetrieveRequestRepackInfo {
  repeated RetrieveRequestArchiveRoute archive_routes = 9500;
  repeated uint32 copy_nbs_to_rearchive = 9510;
  required string repack_request_address = 9520;
  required string file_buffer_url = 9530;
  required uint64 fseq = 9540;
  optional bool force_disabled_tape = 9560 [deprecated=true];
  optional bool has_user_provided_file = 9163 [default = false];
}

// The different timings of the lifecycle of a RetrieveRequest (creation time, first select time, request complete)
message LifecycleTimings {
  optional uint64 creation_time = 9160 [default = 0];
  optional uint64 first_selected_time = 9161 [default = 0];
  optional uint64 completed_time = 9162 [default = 0];
}

message RetrieveActivityWeight {
  required uint64 priority = 9170;
  required string disk_instance_name = 9171;
  required string activity = 9172;
  required double weight = 9173;
  required int64 creation_time = 9174;
}

message RetrieveRequest {
  required SchedulerRetrieveRequest schedulerrequest = 9150;
  required MountPolicy mountpolicy = 9151;
  optional string mountpolicyname = 9165;
  optional RetrieveActivityWeight activity_weight = 9160 [deprecated=true]; // Deprecated for cta/CTA#1077
  optional string activity = 9164;
  required ArchiveFile archivefile = 9152;
  required uint32 activecopynb = 9153;
  repeated RetrieveJob jobs = 9154;
  required string failurereporturl = 9155;
  required string failurereportlog = 9156;
  required bool isrepack = 9157;
  optional RetrieveRequestRepackInfo repack_info = 9158;
  optional LifecycleTimings lifecycle_timings = 9159;
  optional string disk_system_name = 9161;
  required bool isverifyonly = 9162;
  optional bool isfailed = 9166 [default = false];
}

message ValueCountPair {
  required uint64 value = 9301;
  required uint64 count = 9302;
}

message StringCountPair {
  required string value = 9303;
  required uint64 count = 9304;
}

message ArchiveJobPointer {
  required uint64 fileid = 3000;
  required uint64 size = 3001;
  required string address = 3002;
  required uint32 copynb = 3003;
  required uint64 priority = 3004;
  required uint64 minarchiverequestage = 3005;
  required uint64 starttime = 3007;
  required string mountpolicyname = 3008;
}

message ArchiveQueueShardPointer {
  required string address = 10200;
  required uint64 shardjobscount = 10201;
  required uint64 shardbytescount = 10202;
}

message ArchiveQueueShard {
  repeated ArchiveJobPointer archivejobs = 10300;
  required uint64 archivejobstotalsize = 10301;
}

message ArchiveQueue {
  required string tapepool = 10000;
  repeated ArchiveQueueShardPointer archivequeueshards = 10010;
  repeated ValueCountPair prioritymap = 10031;
  repeated ValueCountPair minarchiverequestagemap = 10032;
  repeated StringCountPair mountpolicynamemap = 10034;
  required uint64 archivejobstotalsize = 10040;
  required uint64 archivejobscount = 10045;
  required uint64 oldestjobcreationtime = 10050;
  required uint64 mapsrebuildcount = 10060;
  optional uint64 youngestjobcreationtime = 10070 [default = 0];
}

message RetrieveJobPointer {
  // The retrieve job pointer needs to hold the sufficient information for all the running counters of the queue (priority, activity...)
  required uint64 size = 3101;
  required string address = 3102;
  required uint32 copynb = 3103;
  required uint64 fseq = 3107;
  required uint64 priority = 3104;
  required uint64 minretrieverequestage = 3105;
  required uint64 starttime = 3108;
  required string mountpolicyname = 3112;
  // For activity (if present), we need disk instance and activity name (priority is always provided)
  optional string disk_instance_name = 3109;
  optional string activity = 3110;
  optional string destination_disk_system_name = 3111;
}

message RetrieveQueueShardPointer {
  required string address = 10400;
  required uint64 shardjobscount = 10401;
  required uint64 shardbytescount = 10402;
  required uint64 minfseq = 10403;
  required uint64 maxfseq = 10404;
}

message RetrieveQueueShard {
  repeated RetrieveJobPointer retrievejobs = 10500;
  required uint64 retrievejobstotalsize = 10501;
}

message RetrieveActivityCountPair {
  optional RetrieveActivityWeight retrieve_activity_weight = 10600 [deprecated=true]; // Marked as deprecated and optional for cta/CTA#1077
  required string activity = 10602;
  required uint64 count = 10601;
}

message RetrieveQueueCleanupInfo {
  optional bool doCleanup = 14100;
  optional string assignedAgent = 14110;
  optional uint64 heartbeat = 14120;
}

message RetrieveQueue {
  required string vid = 10100;
  repeated RetrieveQueueShardPointer retrievequeueshards = 10111;
  repeated ValueCountPair prioritymap = 10131;
  repeated ValueCountPair minretrieverequestagemap = 10132;
  repeated StringCountPair mountpolicynamemap = 10134;
  repeated RetrieveActivityCountPair activity_map = 10136;
  required uint64 retrievejobstotalsize = 10140;
  required uint64 retrievejobscount = 10145;
  required uint64 oldestjobcreationtime = 10150;
  required uint64 mapsrebuildcount = 10160;
  required uint64 maxshardsize = 10170;
  optional uint64 sleep_for_free_space_since = 10180;
  optional string disk_system_slept_for = 10190;
  optional uint64 sleep_time = 10200;
  optional uint64 youngestjobcreationtime = 10210 [default = 0];
  optional RetrieveQueueCleanupInfo cleanupInfo = 10220;
}

// ------------- Repack data strcutures ----------------------------------------

enum RepackRequestStatus {
  // those value are matching the dataStructures/RepackInfo.hpp: RepackInfo::Status values.
  RRS_Pending = 1;
  RRS_ToExpand = 2;
  RRS_Starting = 3;
  RRS_Running = 4;
  RRS_Complete = 5;
  RRS_Failed = 6;
}

// In order to properly handle retries in case of failure during reporting, we hold
// a series of booleans tracking the status of the statistics recording and deletion of the
// sub requests. Subrequest deleted records whether a sub request is expected to be present or not.
// The process deleting the sub request will
// first mark it as delete with this status and then actually delete it. In case of crash in this
// process, the switch of this status will be re-done and the sub request deleted on retry.
// On the opposite side, this information will allow the creation process to know if a reference to the
// sub request should be interpreted as an unfulfilled creation intent (deleted=false) and create the
// missing sub request or the completion of the request (which can happen anytime after sub request
// creation).
// Likewise, the "accounted" booleans or copyNbs will prevent double counting in case a report (for success or failure)
// need to be retried after a process failure. The same flag is used for both success and failure. Archive requires a
// set of copyNbs as a single repack can lead to several archives (in case we create new copies).
message RepackSubRequestPointer {
  required string address = 10500;
  required uint64 fseq = 10510;
  required bool retrieve_accounted = 10530;
  repeated uint32 archive_copynb_accounted = 10534;
  required bool subrequest_deleted = 10540;
}

message RepackDestinationInfo {
  required string vid = 20000;
  required uint64 files = 20001 [default = 0];
  required uint64 bytes = 20002 [default = 0];
}

message RepackRequest {
  required string vid = 11000;
  required string buffer_url = 11005;
  required RepackRequestStatus status = 11010;
  required bool add_copies_mode = 11400;
  required bool move_mode = 11410;
  optional uint64 totalfilesontapeatstart = 11415;
  optional uint64 totalbytesontapeatstart = 11416;
  optional bool allfilesselectedatstart = 11417;
  required uint64 totalfilestoretrieve = 11420;
  required uint64 totalbytestoretrieve = 11430;
  // We can have more files to archive than we have to retrieve if we plan to also archive a missing
  // copy of a multi copy file.
  required uint64 totalfilestoarchive = 11440;
  required uint64 totalbytestoarchive = 11450;
  required uint64 userprovidedfiles = 11460;
  required uint64 userprovidedbytes = 11470;
  required uint64 retrievedfiles = 11480;
  required uint64 retrievedbytes = 11490;
  required uint64 archivedfiles = 11500;
  required uint64 archivedbytes = 11510;
  required uint64 failedtoretrievefiles = 11520;
  required uint64 failedtoretrievebytes = 11530;
  required uint64 failedtocreatearchivereq = 11535;
  required uint64 failedtoarchivefiles = 11540;
  required uint64 failedtoarchivebytes = 11550;
  required uint64 lastexpandedfseq = 11560;
  //A RepackRequest can be in status Running even if the expansion
  //is not finished. This boolean is used to indicate wether
  //the expansion of the RepackRequest is done or not
  required bool is_expand_finished = 11561;
  required bool is_expand_started = 11562;
  required MountPolicy mount_policy = 11563;
  optional string mountpolicyname = 11575;  //TODO: Once it has been deployed, make it required and remove mountpolicy
  optional bool force_disabled_tape = 11564 [deprecated=true];
  required bool is_complete = 11565;
  required bool no_recall = 11566;
  repeated RepackSubRequestPointer subrequests = 11570;
  repeated RepackDestinationInfo destination_infos = 11571;
  required EntryLog creation_log = 11572;
  required uint64 repack_finished_time = 11573;
  optional uint64 maxfilestoselect = 11574;
}

message RepackRequestIndexPointer {
  required string vid = 12000;
  required string address = 12010;
}

message RepackIndex {
  repeated RepackRequestIndexPointer repackrequestpointers = 12100;
}

message RepackRequestQueuePointer {
  required string address = 12101;
}

message RepackQueue {
  repeated RepackRequestQueuePointer repackrequestpointers = 12200;
}