PluginManager/bin/refresh.cc

0001 #include "FWCore/PluginManager/interface/CacheParser.h"
0002 #include "FWCore/PluginManager/interface/PluginCapabilities.h"
0003 #include "FWCore/PluginManager/interface/PluginFactoryBase.h"
0004 #include "FWCore/PluginManager/interface/PluginFactoryManager.h"
0005 #include "FWCore/PluginManager/interface/SharedLibrary.h"
0006 #include "FWCore/PluginManager/interface/standard.h"
0007
0008 #include "FWCore/Utilities/interface/Exception.h"
0009 #include "FWCore/Utilities/interface/Algorithms.h"
0010
0011 #include <boost/program_options.hpp>
0012
0013 #include <algorithm>
0014 #include <cstdlib>
0015 #include <filesystem>
0016 #include <fstream>
0017 #include <functional>
0018 #include <iostream>
0019 #include <set>
0020 #include <string>
0021 #include <utility>
0022
0023 #include <sys/wait.h>
0024
0025 using namespace edmplugin;
0026
0027 // We process DSOs by forking and processing them in the child, in bunches of
0028 // PER_PROCESS_DSO. Once a bunch has been processed we exit the child and
0029 // spawn a new one.
0030 // Due to the different behavior of the linker on different platforms, we process
0031 // a different number of DSOs per process.
0032 // For macosx the cost is to actually resolve weak symbols and it grows with
0033 // the number of libraries as well, so we try to process not too many libraries.
0034 // In linux the main cost is to load the library itself, so we try to load as
0035 // many as reasonable in a single process to avoid having to reload them in
0036 // the subsequent process. Obviuously keeping the PER_PROCESS_DSO low keeps the
0037 // memory usage low as well.
0038 #ifdef __APPLE__
0039 #define PER_PROCESS_DSO 20
0040 #elif defined(__aarch64__)
0041 #define PER_PROCESS_DSO 10
0042 #else
0043 #define PER_PROCESS_DSO 2000
0044 #endif
0045
0046 namespace std {
0047   ostream& operator<<(std::ostream& o, vector<std::string> const& iValue) {
0048     std::string sep("");
0049     std::string commaSep(",");
0050     for (std::vector<std::string>::const_iterator it = iValue.begin(), itEnd = iValue.end(); it != itEnd; ++it) {
0051       o << sep << *it;
0052       sep = commaSep;
0053     }
0054     return o;
0055   }
0056 }  // namespace std
0057 namespace {
0058   struct Listener {
0059     typedef edmplugin::CacheParser::NameAndType NameAndType;
0060     typedef edmplugin::CacheParser::NameAndTypes NameAndTypes;
0061
0062     void newFactory(edmplugin::PluginFactoryBase const* iBase) {
0063       using std::placeholders::_1;
0064       using std::placeholders::_2;
0065       iBase->newPluginAdded_.connect(std::bind(std::mem_fn(&Listener::newPlugin), this, _1, _2));
0066     }
0067     void newPlugin(std::string const& iCategory, edmplugin::PluginInfo const& iInfo) {
0068       nameAndTypes_.push_back(NameAndType(iInfo.name_, iCategory));
0069     }
0070
0071     NameAndTypes nameAndTypes_;
0072   };
0073 }  // namespace
0074 int main(int argc, char** argv) try {
0075   using namespace boost::program_options;
0076   using std::placeholders::_1;
0077
0078   static char const* const kPathsOpt = "paths";
0079   static char const* const kPathsCommandOpt = "paths,p";
0080   //static char const* const kAllOpt = "all";
0081   //static char const* const kAllCommandOpt = "all,a";
0082   static char const* const kHelpOpt = "help";
0083   static char const* const kHelpCommandOpt = "help,h";
0084
0085   std::string descString(argv[0]);
0086   descString += " [options] [[--";
0087   descString += kPathsOpt;
0088   descString += "] path [path]] \nAllowed options";
0089   options_description desc(descString);
0090   std::string defaultDir(".");
0091   std::vector<std::string> defaultDirList = edmplugin::standard::config().searchPath();
0092   if (!defaultDirList.empty()) {
0093     defaultDir = defaultDirList[0];
0094   }
0095   desc.add_options()(kHelpCommandOpt, "produce help message")(
0096       kPathsCommandOpt,
0097       value<std::vector<std::string> >()->default_value(std::vector<std::string>(1, defaultDir)),
0098       "a directory or a list of files to scan")
0099       //(kAllCommandOpt, "when no paths given, try to update caches for all known directories [default is to only scan the first directory]")
0100       ;
0101
0102   positional_options_description p;
0103   p.add(kPathsOpt, -1);
0104
0105   variables_map vm;
0106   try {
0107     store(command_line_parser(argc, argv).options(desc).positional(p).run(), vm);
0108     notify(vm);
0109   } catch (error const& iException) {
0110     std::cerr << iException.what();
0111     return 1;
0112   }
0113
0114   if (vm.count(kHelpOpt)) {
0115     std::cout << desc << std::endl;
0116     return 0;
0117   }
0118
0119   using std::filesystem::path;
0120
0121   /*if(argc == 1) {
0122     std::cerr << "Requires at least one argument.  Please pass either one directory or a list of files (all in the same directory)." << std::endl;
0123     return 1;
0124   } */
0125
0126   int returnValue = EXIT_SUCCESS;
0127
0128   try {
0129     std::vector<std::string> requestedPaths(vm[kPathsOpt].as<std::vector<std::string> >());
0130
0131     //first find the directory and create a list of files to look at in that directory
0132     path directory(requestedPaths[0]);
0133     std::vector<std::string> files;
0134     bool removeMissingFiles = false;
0135     if (std::filesystem::is_directory(directory)) {
0136       if (requestedPaths.size() > 1) {
0137         std::cerr << "if a directory is given then only one argument is allowed" << std::endl;
0138         return 1;
0139       }
0140
0141       //if asked to look at whole directory, then we can also remove missing files
0142       removeMissingFiles = true;
0143
0144       std::filesystem::directory_iterator file(directory);
0145       std::filesystem::directory_iterator end;
0146
0147       path cacheFile(directory);
0148       cacheFile /= standard::cachefileName();
0149
0150       std::filesystem::file_time_type cacheLastChange = std::filesystem::file_time_type::min();
0151       if (exists(cacheFile)) {
0152         cacheLastChange = last_write_time(cacheFile);
0153       }
0154       for (; file != end; ++file) {
0155         path filename(*file);
0156         path shortName(file->path().filename());
0157         const std::string& stringName = shortName.string();
0158
0159         static std::string const kPluginPrefix(standard::pluginPrefix());
0160         if (stringName.size() < kPluginPrefix.size()) {
0161           continue;
0162         }
0163         if (stringName.substr(0, kPluginPrefix.size()) != kPluginPrefix) {
0164           continue;
0165         }
0166
0167         if (last_write_time(filename) > cacheLastChange) {
0168           files.push_back(stringName);
0169         }
0170       }
0171     } else {
0172       //we have files
0173       directory = directory.parent_path();
0174       for (std::vector<std::string>::iterator it = requestedPaths.begin(), itEnd = requestedPaths.end(); it != itEnd;
0175            ++it) {
0176         std::filesystem::path f(*it);
0177         if (!exists(f)) {
0178           std::cerr << "the file '" << f.string() << "' does not exist" << std::endl;
0179           return 1;
0180         }
0181         if (is_directory(f)) {
0182           std::cerr << "either one directory or a list of files are allowed as arguments" << std::endl;
0183           return 1;
0184         }
0185         if (directory != f.parent_path()) {
0186           std::cerr << "all files must have be in the same directory (" << directory.string()
0187                     << ")\n"
0188                        " the file "
0189                     << f.string() << " does not." << std::endl;
0190         }
0191 #if (BOOST_VERSION / 100000) >= 1 && ((BOOST_VERSION / 100) % 1000) >= 47
0192         files.push_back(f.filename().string());
0193 #else
0194         files.push_back(f.leaf());
0195 #endif
0196       }
0197     }
0198
0199     path cacheFile(directory);
0200     cacheFile /= edmplugin::standard::cachefileName();  //path(s_cacheFile);
0201
0202     CacheParser::LoadableToPlugins old;
0203     if (exists(cacheFile)) {
0204       std::ifstream cf(cacheFile.string().c_str());
0205       if (!cf) {
0206         cms::Exception("FailedToOpen") << "unable to open file '" << cacheFile.string()
0207                                        << "' for reading even though it is present.\n"
0208                                           "Please check permissions on the file.";
0209       }
0210       CacheParser::read(cf, old);
0211     }
0212
0213     //load each file and 'listen' to which plugins are loaded
0214     Listener listener;
0215     edmplugin::PluginFactoryManager* pfm = edmplugin::PluginFactoryManager::get();
0216     pfm->newFactory_.connect(std::bind(std::mem_fn(&Listener::newFactory), &listener, _1));
0217     edm::for_all(*pfm, std::bind(std::mem_fn(&Listener::newFactory), &listener, _1));
0218
0219     // We open the cache file before forking so that all the children will
0220     // use it.
0221     std::string temporaryFilename = (cacheFile.string() + ".tmp");
0222     std::ofstream cf(temporaryFilename.c_str());
0223     if (!cf) {
0224       cms::Exception("FailedToOpen") << "unable to open file '" << temporaryFilename
0225                                      << "' for writing.\n"
0226                                         "Please check permissions on the file.";
0227     }
0228     // Sort the files so that they are loaded "by subsystem", hopefully meaning
0229     // they share more dependencies.
0230     std::sort(files.begin(), files.end());
0231
0232     for (size_t fi = 0, fe = files.size(); fi < fe; fi += PER_PROCESS_DSO) {
0233       CacheParser::LoadableToPlugins ltp;
0234       pid_t worker = fork();
0235       if (worker == 0) {
0236         // This the child process.
0237         // We load the DSO and find out its plugins, write to the cache
0238         // stream and exit, leaving the parent to spawn a new proces.
0239         size_t ci = PER_PROCESS_DSO;
0240         while (ci && fi != fe) {
0241           path loadableFile(directory);
0242           loadableFile /= (files[fi]);
0243           listener.nameAndTypes_.clear();
0244
0245           try {
0246             try {
0247               edmplugin::SharedLibrary lib(loadableFile);
0248               //PluginCapabilities is special, the plugins do not call it.  Instead, for each shared library load
0249               // we need to ask it to try to find plugins
0250               PluginCapabilities::get()->tryToFind(lib);
0251               ltp[files[fi]] = listener.nameAndTypes_;
0252
0253             } catch (cms::Exception const& iException) {
0254               if (iException.category() == "PluginLibraryLoadError") {
0255                 std::cerr << "Caught exception " << iException.what() << " will ignore " << files[fi]
0256                           << " and continue." << std::endl;
0257               } else {
0258                 throw;
0259               }
0260             }
0261           } catch (std::exception& iException) {
0262             std::cerr << "Caught exception " << iException.what() << std::endl;
0263             exit(1);
0264           }
0265           ++fi;
0266           --ci;
0267         }
0268         CacheParser::write(ltp, cf);
0269         cf << std::flush;
0270         _exit(0);
0271       } else {
0272         // Throw if any of the child died with non 0 status.
0273         int status = 0;
0274         waitpid(worker, &status, 0);
0275         if (WIFEXITED(status) != 0 and WEXITSTATUS(status) != 0) {
0276           std::cerr << "Error in child process while processing: " << WEXITSTATUS(status) << std::endl;
0277           exit(WEXITSTATUS(status));
0278         }
0279         if (WIFSIGNALED(status) != 0) {
0280           std::cerr << "Child process got signal while processing: " << WTERMSIG(status) << std::endl;
0281           exit(128 + WTERMSIG(status));
0282         }
0283       }
0284     }
0285
0286     cf << std::flush;
0287
0288     // We read the new cache and we merge it with the old one.
0289     CacheParser::LoadableToPlugins ltp;
0290     std::ifstream icf(temporaryFilename.c_str());
0291     if (!icf) {
0292       cms::Exception("FailedToOpen") << "unable to open file '" << temporaryFilename.c_str()
0293                                      << "' for reading even though it is present.\n"
0294                                         "Please check permissions on the file.";
0295     }
0296     CacheParser::read(icf, ltp);
0297
0298     for (CacheParser::LoadableToPlugins::iterator itFile = ltp.begin(); itFile != ltp.end(); ++itFile) {
0299       old[itFile->first] = itFile->second;
0300     }
0301
0302     // If required, we remove the plugins which are missing. Notice that old is
0303     // now the most updated copy of the cache.
0304     if (removeMissingFiles) {
0305       for (CacheParser::LoadableToPlugins::iterator itFile = old.begin(); itFile != old.end();
0306            /*don't advance the iterator here because it may have become invalid */) {
0307         path loadableFile(directory);
0308         loadableFile /= (itFile->first);
0309         if (not exists(loadableFile)) {
0310           std::cout << "removing file '" << temporaryFilename.c_str() << "'" << std::endl;
0311           CacheParser::LoadableToPlugins::iterator itToItemBeingRemoved = itFile;
0312           //advance the iterator while it is still valid
0313           ++itFile;
0314           old.erase(itToItemBeingRemoved);
0315         } else {
0316           //since we are not advancing the iterator in the for loop, do it here
0317           ++itFile;
0318         }
0319       }
0320     }
0321
0322     // We finally write the final cache.
0323     std::ofstream fcf(temporaryFilename.c_str());
0324     if (!fcf) {
0325       cms::Exception("FailedToOpen") << "unable to open file '" << temporaryFilename.c_str()
0326                                      << "' for writing.\n"
0327                                         "Please check permissions on the file.";
0328     }
0329     CacheParser::write(old, fcf);
0330     rename(temporaryFilename.c_str(), cacheFile.string().c_str());
0331   } catch (std::exception& iException) {
0332     std::cerr << "Caught exception " << iException.what() << std::endl;
0333     returnValue = EXIT_FAILURE;
0334   }
0335
0336   return returnValue;
0337 } catch (std::exception const& iException) {
0338   std::cerr << iException.what() << std::endl;
0339   return 1;
0340 }