// ----------------------------------------------------------------------
// File: com_find.cc
// Author: Andreas-Joachim Peters - CERN
// ----------------------------------------------------------------------
/************************************************************************
* EOS - the CERN Disk Storage System *
* Copyright (C) 2011 CERN/Switzerland *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see .*
************************************************************************/
/*----------------------------------------------------------------------------*/
#include "console/ConsoleMain.hh"
#include "console/commands/ICmdHelper.hh"
#include "common/StringTokenizer.hh"
#include "common/StringConversion.hh"
#include "XrdPosix/XrdPosixXrootd.hh"
#include "XrdOuc/XrdOucEnv.hh"
/*----------------------------------------------------------------------------*/
extern int com_file(char*);
/* Find files/directories */
int
com_old_find(char* arg1)
{
XrdPosixXrootd Xroot;
// split subcommands
XrdOucString oarg = arg1;
eos::common::StringTokenizer subtokenizer(arg1);
subtokenizer.GetLine();
XrdOucString s1;
XrdOucString path;
XrdOucString option = "";
XrdOucString attribute = "";
XrdOucString maxdepth = "";
XrdOucString olderthan = "";
XrdOucString youngerthan = "";
XrdOucString printkey = "";
XrdOucString filter = "";
XrdOucString stripes = "";
XrdOucString versions = "";
XrdOucString filematch = "";
XrdOucString in = "mgm.cmd=find&";
bool valid = false;
if (wants_help(arg1)) {
goto com_find_usage;
}
while ((s1 = subtokenizer.GetToken()).length() && (s1.beginswith("-"))) {
valid = false;
if (s1 == "-j") {
option += "j";
continue;
}
if (s1 == "-s") {
option += "s";
valid = true;
}
if (s1 == "-d") {
option += "d";
valid = true;
}
if (s1 == "-f") {
option += "f";
valid = true;
}
if (s1 == "-0") {
option += "f0";
valid = true;
}
if (s1 == "-m") {
option += "fG";
valid = true;
}
if (s1 == "--size") {
option += "S";
valid = true;
}
if (s1 == "--fs") {
option += "L";
valid = true;
}
if (s1 == "--checksum") {
option += "X";
valid = true;
}
if (s1 == "--ctime") {
option += "C";
valid = true;
}
if (s1 == "--mtime") {
option += "M";
valid = true;
}
if (s1 == "--fid") {
option += "F";
valid = true;
}
if (s1 == "--nrep") {
option += "R";
valid = true;
}
if (s1 == "--online") {
option += "O";
valid = true;
}
if (s1 == "--fileinfo") {
option += "I";
valid = true;
}
if (s1 == "--nunlink") {
option += "U";
valid = true;
}
if (s1 == "--uid") {
option += "u";
valid = true;
}
if (s1 == "--gid") {
option += "g";
valid = true;
}
if (s1 == "--stripediff") {
option += "D";
valid = true;
}
if (s1 == "--faultyacl") {
option += "A";
valid = true;
}
if (s1 == "--count") {
option += "Z";
valid = true;
}
if (s1 == "--hosts") {
option += "H";
valid = true;
}
if (s1 == "--partition") {
option += "P";
valid = true;
}
if (s1 == "--childcount") {
option += "l";
valid = true;
}
if (s1 == "--xurl") {
option += "x";
valid = true;
}
if (s1 == "-1") {
option += "1";
valid = true;
}
if (s1.beginswith("-h") || (s1.beginswith("--help"))) {
goto com_find_usage;
}
if (s1 == "-x") {
valid = true;
attribute = subtokenizer.GetToken();
if (!attribute.length()) {
goto com_find_usage;
}
if ((attribute.find("&")) != STR_NPOS) {
goto com_find_usage;
}
}
if (s1 == "--maxdepth") {
valid = true;
maxdepth = subtokenizer.GetToken();
if (!maxdepth.length()) {
goto com_find_usage;
}
}
if ((s1 == "-ctime") || (s1 == "-mtime")) {
valid = true;
XrdOucString period = "";
period = subtokenizer.GetToken();
if (!period.length()) {
goto com_find_usage;
}
bool do_olderthan;
do_olderthan = false;
bool do_youngerthan;
do_youngerthan = false;
if (period.beginswith("+")) {
do_olderthan = true;
}
if (period.beginswith("-")) {
do_youngerthan = true;
}
if ((!do_olderthan) && (!do_youngerthan)) {
goto com_find_usage;
}
period.erase(0, 1);
time_t now = time(NULL);
now -= (86400 * strtoul(period.c_str(), 0, 10));
char snow[1024];
snprintf(snow, sizeof(snow) - 1, "%lu", now);
if (do_olderthan) {
olderthan = snow;
}
if (do_youngerthan) {
youngerthan = snow;
}
if (s1 == "-ctime") {
option += "C";
}
if (s1 == "-mtime") {
option += "M";
}
}
if (s1 == "-c") {
valid = true;
option += "c";
filter = subtokenizer.GetToken();
if (!filter.length()) {
goto com_find_usage;
}
if ((filter.find("%%")) != STR_NPOS) {
goto com_find_usage;
}
}
if (s1 == "--purge") {
valid = true;
versions = subtokenizer.GetToken();
if (!versions.length()) {
goto com_find_usage;
}
}
if (s1 == "-name") {
valid = true;
filematch = subtokenizer.GetToken();
option += "f";
if (!filematch.length()) {
goto com_find_usage;
}
}
if (s1 == "-layoutstripes") {
valid = true;
stripes = subtokenizer.GetToken();
if (!stripes.length()) {
goto com_find_usage;
}
}
if (s1 == "-p") {
valid = true;
option += "p";
printkey = subtokenizer.GetToken();
if (!printkey.length()) {
goto com_find_usage;
}
}
if (s1 == "-b") {
valid = true;
option += "b";
}
if (!valid) {
goto com_find_usage;
}
}
if (s1.length()) {
path = s1;
}
if (path == "help") {
goto com_find_usage;
}
if (!path.endswith("/")) {
if (!path.endswith(":")) {
// if the user gave file: as a search path we shouldn't add '/'=root
path += "/";
}
}
if (path.beginswith("root://") || path.beginswith("file:")) {
// -------------------------------------------------------------
// do a find with XRootd or local file system
// -------------------------------------------------------------
bool XRootD = path.beginswith("root:");
std::vector< std::vector > found_dirs;
std::map > found;
XrdOucString protocol;
XrdOucString hostport;
XrdOucString sPath;
if (path == "/") {
fprintf(stderr, "error: I won't do a find on '/'\n");
global_retc = EINVAL;
return (0);
}
const char* v = 0;
if (!(v = eos::common::StringConversion::ParseUrl(path.c_str(), protocol,
hostport))) {
global_retc = EINVAL;
return (0);
}
sPath = v;
std::string Path = v;
if (sPath == "" && (protocol == "file")) {
sPath = getenv("PWD");
Path = getenv("PWD");
if (!sPath.endswith("/")) {
sPath += "/";
Path += "/";
}
}
found_dirs.resize(1);
found_dirs[0].resize(1);
found_dirs[0][0] = Path.c_str();
int deepness = 0;
do {
struct stat buf;
found_dirs.resize(deepness + 2);
// loop over all directories in that deepness
for (unsigned int i = 0; i < found_dirs[deepness].size(); i++) {
Path = found_dirs[deepness][i].c_str();
XrdOucString url = "";
eos::common::StringConversion::CreateUrl(protocol.c_str(), hostport.c_str(),
Path.c_str(), url);
int rstat = 0;
rstat = (XRootD) ? XrdPosixXrootd::Stat(url.c_str(), &buf) : stat(url.c_str(),
&buf);
if (!rstat) {
//
if (S_ISDIR(buf.st_mode)) {
// add all children
DIR* dir = (XRootD) ? XrdPosixXrootd::Opendir(url.c_str()) : opendir(
url.c_str());
if (dir) {
struct dirent* entry;
while ((entry = (XRootD) ? XrdPosixXrootd::Readdir(dir) : readdir(dir))) {
XrdOucString curl = "";
XrdOucString cpath = Path.c_str();
cpath += entry->d_name;
if ((!strcmp(entry->d_name, ".")) || (!strcmp(entry->d_name, ".."))) {
continue; // skip . and .. directories
}
eos::common::StringConversion::CreateUrl(protocol.c_str(), hostport.c_str(),
cpath.c_str(), curl);
if (!((XRootD) ? XrdPosixXrootd::Stat(curl.c_str(), &buf) : stat(curl.c_str(),
&buf))) {
if (S_ISDIR(buf.st_mode)) {
curl += "/";
cpath += "/";
found_dirs[deepness + 1].push_back(cpath.c_str());
(void) found[curl.c_str()].size();
} else {
found[url.c_str()].insert(entry->d_name);
}
}
}
(XRootD) ? XrdPosixXrootd::Closedir(dir) : closedir(dir);
}
}
}
}
deepness++;
} while (found_dirs[deepness].size());
bool show_files = false;
bool show_dirs = false;
if ((option.find("f") == STR_NPOS) && (option.find("d") == STR_NPOS)) {
show_files = show_dirs = true;
} else {
if (option.find("f") != STR_NPOS) {
show_files = true;
}
if (option.find("d") != STR_NPOS) {
show_dirs = true;
}
}
std::map >::const_iterator it;
for (it = found.begin(); it != found.end(); it++) {
std::set::const_iterator sit;
if (show_dirs) {
fprintf(stdout, "%s\n", it->first.c_str());
}
for (sit = it->second.begin(); sit != it->second.end(); sit++) {
if (show_files) {
fprintf(stdout, "%s%s\n", it->first.c_str(), sit->c_str());
}
}
}
return 0;
}
if (path.beginswith("as3:")) {
// ----------------------------------------------------------------
// this is nightmare code because of a missing proper CLI for S3
// ----------------------------------------------------------------
XrdOucString hostport;
XrdOucString protocol;
int rc = system("which s3 >&/dev/null");
if (WEXITSTATUS(rc)) {
fprintf(stderr,
"error: you miss the executable provided by libs3 in your PATH\n");
exit(-1);
}
if (path.endswith("/")) {
path.erase(path.length() - 1);
}
XrdOucString sPath = path.c_str();
XrdOucString sOpaque;
int qpos = 0;
if ((qpos = sPath.find("?")) != STR_NPOS) {
sOpaque.assign(sPath, qpos + 1);
sPath.erase(qpos);
}
XrdOucString fPath = eos::common::StringConversion::ParseUrl(sPath.c_str(),
protocol, hostport);
XrdOucEnv env(sOpaque.c_str());
if (env.Get("s3.key")) {
setenv("S3_SECRET_ACCESS_KEY", env.Get("s3.key"), 1);
}
if (env.Get("s3.id")) {
setenv("S3_ACCESS_KEY_ID", env.Get("s3.id"), 1);
}
// Apply the ROOT compatability environment variables
const char* cstr = getenv("S3_ACCESS_KEY");
if (cstr) {
setenv("S3_SECRET_ACCESS_KEY", cstr, 1);
}
cstr = getenv("S3_ACESSS_ID");
if (cstr) {
setenv("S3_ACCESS_KEY_ID", cstr, 1);
}
// check that the environment is set
if (!getenv("S3_ACCESS_KEY_ID") ||
!getenv("S3_HOSTNAME") ||
!getenv("S3_SECRET_ACCESS_KEY")) {
fprintf(stderr,
"error: you have to set the S3 environment variables S3_ACCESS_KEY_ID | S3_ACCESS_ID, S3_HOSTNAME (or use a URI), S3_SECRET_ACCESS_KEY | S3_ACCESS_KEY\n");
global_retc = EINVAL;
return (0);
}
XrdOucString s3env;
s3env = "env S3_ACCESS_KEY_ID=";
s3env += getenv("S3_ACCESS_KEY_ID");
s3env += " S3_HOSTNAME=";
s3env += getenv("S3_HOSTNAME");
s3env += " S3_SECRET_ACCESS_KEY=";
s3env += getenv("S3_SECRET_ACCESS_KEY");
XrdOucString cmd = "bash -c \"";
cmd += s3env;
cmd += " s3 list ";
// extract bucket from path
int bpos = fPath.find("/");
XrdOucString bucket;
if (bpos != STR_NPOS) {
bucket.assign(fPath, 0, bpos - 1);
} else {
bucket = fPath.c_str();
}
XrdOucString match;
if (bpos != STR_NPOS) {
match.assign(fPath, bpos + 1);
} else {
match = "";
}
if ((!bucket.length()) || (bucket.find("*") != STR_NPOS)) {
fprintf(stderr, "error: no bucket specified or wildcard in bucket name!\n");
global_retc = EINVAL;
return (0);
}
cmd += bucket.c_str();
cmd += " | awk '{print \\$1}' ";
if (match.length()) {
if (match.endswith("*")) {
match.erase(match.length() - 1);
match.insert("^", 0);
}
if (match.beginswith("*")) {
match.erase(0, 1);
match += "$";
}
cmd += " | egrep '";
cmd += match.c_str();
cmd += "'";
}
cmd += " | grep -v 'Bucket' | grep -v '\\-\\-\\-\\-\\-\\-\\-\\-\\-\\-' | grep -v 'Key' | awk -v prefix=";
cmd += "'";
cmd += bucket.c_str();
cmd += "' ";
cmd += "'{print \\\"as3:\\\"prefix\\\"/\\\"\\$1}'";
cmd += "\"";
rc = system(cmd.c_str());
if (WEXITSTATUS(rc)) {
fprintf(stderr, "error: failed to run %s\n", cmd.c_str());
}
}
// the find to change a layout
if ((stripes.length())) {
XrdOucString subfind = oarg;
XrdOucString repstripes = " ";
repstripes += stripes;
repstripes += " ";
subfind.replace("-layoutstripes", "");
subfind.replace(repstripes, " -f -s ");
int rc = com_old_find((char*) subfind.c_str());
std::vector files_found;
files_found.clear();
command_result_stdout_to_vector(files_found);
unsigned long long cnt = 0;
unsigned long long goodentries = 0;
unsigned long long badentries = 0;
for (unsigned int i = 0; i < files_found.size(); i++) {
if (!files_found[i].length()) {
continue;
}
XrdOucString cline = "layout ";
cline += files_found[i].c_str();
cline += " -stripes ";
cline += stripes;
rc = com_file((char*) cline.c_str());
if (rc) {
badentries++;
} else {
goodentries++;
}
cnt++;
}
rc = 0;
if (!silent) {
fprintf(stderr, "nentries=%llu good=%llu bad=%llu\n", cnt, goodentries,
badentries);
}
return 0;
}
// the find with consistency check
if ((option.find("c")) != STR_NPOS) {
XrdOucString subfind = oarg;
subfind.replace("-c", "-s -f");
subfind.replace(filter, "");
int rc = com_old_find((char*) subfind.c_str());
std::vector files_found;
files_found.clear();
command_result_stdout_to_vector(files_found);
unsigned long long cnt = 0;
unsigned long long goodentries = 0;
unsigned long long badentries = 0;
for (unsigned int i = 0; i < files_found.size(); i++) {
if (!files_found[i].length()) {
continue;
}
XrdOucString cline = "check ";
cline += files_found[i].c_str();
cline += " ";
cline += filter;
rc = com_file((char*) cline.c_str());
if (rc) {
badentries++;
} else {
goodentries++;
}
cnt++;
}
rc = 0;
if (!silent) {
fprintf(stderr, "nentries=%llu good=%llu bad=%llu\n", cnt, goodentries,
badentries);
}
return 0;
}
path = abspath(path.c_str());
if (!s1.length() && (path == "/")) {
fprintf(stderr,
"error: you didnt' provide any path and would query '/' - will not do that!\n");
return EINVAL;
}
in += "mgm.path=";
in += path;
in += "&mgm.option=";
in += option;
if (attribute.length()) {
in += "&mgm.find.attribute=";
in += attribute;
}
if (maxdepth.length()) {
in += "&mgm.find.maxdepth=";
in += maxdepth;
}
if (olderthan.length()) {
in += "&mgm.find.olderthan=";
in += olderthan;
}
if (youngerthan.length()) {
in += "&mgm.find.youngerthan=";
in += youngerthan;
}
if (versions.length()) {
in += "&mgm.find.purge.versions=";
in += versions;
}
if (filematch.length()) {
in += "&mgm.find.match=";
in += filematch;
}
if (printkey.length()) {
in += "&mgm.find.printkey=";
in += printkey;
}
XrdOucEnv* result;
result = client_command(in);
if ((option.find("s")) == STR_NPOS) {
global_retc = output_result(result);
} else {
if (result) {
global_retc = 0;
} else {
global_retc = EINVAL;
}
}
return (0);
com_find_usage:
fprintf(stdout,
"usage: find [-name ] [--xurl] [--childcount] [--purge ] [--count] [-s] [-d] [-f] [-0] [-1] [-ctime +|-] [-m] [-x =] [-p ] [-b] [-c %%tags] [-layoutstripes ] \n");
fprintf(stdout,
" -f -d : find files(-f) or directories (-d) in \n");
fprintf(stdout,
" -name : find by name or wildcard match\n");
fprintf(stdout,
" -x = : find entries with =\n");
fprintf(stdout,
" -0 : find 0-size files \n");
fprintf(stdout,
" -g : find files with mixed scheduling groups\n");
fprintf(stdout,
" -p : additionally print the value of for each entry\n");
fprintf(stdout,
" -b : query the server balance of the files found\n");
fprintf(stdout,
" -c %%tags : find all files with inconsistencies defined by %%tags [ see help of 'file check' command]\n");
fprintf(stdout,
" -s : run as a subcommand (in silent mode)\n");
fprintf(stdout,
" -ctime + : find files older than days\n");
fprintf(stdout,
" -ctime - : find files younger than days\n");
fprintf(stdout,
" -layoutstripes : apply new layout with stripes to all files found\n");
fprintf(stdout,
" --maxdepth : descend only levels\n");
fprintf(stdout,
" -1 : find files which are atleast 1 hour old\n");
fprintf(stdout,
" --stripediff : find files which have not the nominal number of stripes(replicas)\n");
fprintf(stdout,
" --faultyacl : find directories with illegal ACLs\n");
fprintf(stdout,
" --count : just print global counters for files/dirs found\n");
fprintf(stdout,
" --xurl : print the XRootD URL instead of the path name\n");
fprintf(stdout,
" --childcount : print the number of children in each directory\n");
fprintf(stdout,
" --purge | atomic\n");
fprintf(stdout,
" : remove versioned files keeping versions - to remove all old versions use --purge 0 ! To \n"
" apply the settings of the extended attribute definition use =-1! To remove all atomic upload\n"
" left-overs older than a day user --purge atomic\n");
fprintf(stdout,
" default : find files and directories\n");
fprintf(stdout,
" find [--nrep] [--nunlink] [--size] [--fileinfo] [--online] [--hosts] [--partition] [--fid] [--fs] [--checksum] [--ctime] [--mtime] [--uid] [--gid] \n"
" : find files and print out the requested meta data as key value pairs\n");
fprintf(stdout,
" path=file:... : do a find in the local file system (options ignored) - 'file:' is the current working directory \n");
fprintf(stdout,
" path=root:... : do a find on a plain XRootD server (options ignored) - does not work on native XRootD clusters\n");
fprintf(stdout,
" path=as3:... : do a find on an S3 bucket\n");
fprintf(stdout,
" path=... : all other paths are considered to be EOS paths!\n");
global_retc = EINVAL;
return (0);
}
/*
void com_oldfind_help()
{
std::ostringstream oss;
oss << "Usage: oldfind [-name ] [--xurl] [--childcount] [--purge ] [--count] [-s] [-d] [-f] [-0] [-1] [-g] [-uid ] [-nuid ] [-gid ] [-ngid ] [-flag ] [-nflag ] [-ctime +|-] [-m] [-x =] [-p ] [-b] [--layoutstripes ] "
<< std::endl;
oss << " -f -d : find files(-f) or directories (-d) in "
<< std::endl;
oss << " --name : find by name or wildcard match" << std::endl;
oss << " -x = : find entries with =" << std::endl;
oss << " -0 : find 0-size files only" << std::endl;
oss << " -g : find files with mixed scheduling groups" <<
std::endl;
oss << " -p : additionally print the value of for each entry"
<< std::endl;
oss << " -b : query the server balance of the files found" <<
std::endl;
oss << " -s : run as a subcommand (in silent mode)" <<
std::endl;
oss << " -uid : entries owned by given user id number" <<
std::endl;
oss << " -nuid : entries not owned by given user id number" <<
std::endl;
oss << " -gid : entries owned by given group id number" <<
std::endl;
oss << " -ngid : entries not owned by given group id number" <<
std::endl;
oss << " -flag : directories with specified UNIX access flag, e.g. 755"
<< std::endl;
oss << " -nflag : directories not with specified UNIX access flag, e.g. 755"
<< std::endl;
oss << " -ctime + : find files older than days" << std::endl;
oss << " -ctime - : find files younger than days" << std::endl;
oss << " --layoutstripes : apply new layout with stripes to all files found"
<< std::endl;
oss << " --maxdepth : descend only levels" << std::endl;
oss << " -1 : find files which are at least 1 hour old" <<
std::endl;
oss << " --stripediff : find files which have not the nominal number of stripes(replicas)"
<< std::endl;
oss << " --faultyacl : find directories with illegal ACLs" <<
std::endl;
oss << " --count : just print global counters for files/dirs found"
<< std::endl;
oss << " --xurl : print the XRootD URL instead of the path name"
<< std::endl;
oss << " --childcount : print the number of children in each directory"
<< std::endl;
oss << " --purge | atomic" << std::endl;
oss << " : remove versioned files keeping versions - to remove all old versions use --purge 0"
<< std::endl;
oss << " To apply the settings of the extended attribute definition use =-1"
<< std::endl;
oss << " To remove all atomic upload left-overs older than a day user --purge atomic"
<< std::endl;
oss << " default : find files and directories" << std::endl;
oss << " find [--nrep] [--nunlink] [--size] [--fileinfo] [--online] [--hosts] [--partition] [--fid] [--fs] [--checksum] [--ctime] [--mtime] [--uid] [--gid] "
<< std::endl;
oss << " : find files and print out the requested meta data as key value pairs"
<< std::endl;
oss << " path=file:... : do a find in the local file system (options ignored) - 'file:' is the current working directory"
<< std::endl;
oss << " path=root:... : do a find on a plain XRootD server (options ignored) - does not work on native XRootD clusters"
<< std::endl;
oss << " path=as3:... : do a find on an S3 bucket" << std::endl;
oss << " path=... : all other paths are considered to be EOS paths!"
<< std::endl;
std::cerr << oss.str() << std::endl;
}
*/