DemonstratorTools/src/utilities.cc

0001 #include "L1Trigger/DemonstratorTools/interface/utilities.h"
0002
0003 #include <algorithm>
0004 #include <fstream>
0005 #include <regex>
0006 #include <unordered_map>
0007 #include <boost/iostreams/filter/gzip.hpp>
0008 #include <boost/iostreams/filter/lzma.hpp>
0009 #include <boost/iostreams/filtering_stream.hpp>
0010
0011 #ifdef CMSSW_GIT_HASH
0012 #include "FWCore/MessageLogger/interface/MessageLogger.h"
0013 #endif
0014 #include "L1Trigger/DemonstratorTools/interface/BoardData.h"
0015
0016 namespace {
0017
0018   l1t::demo::BoardData createBoardDataFromRows(const std::string& id,
0019                                                const std::vector<size_t>& channels,
0020                                                const std::vector<std::vector<l1t::demo::Frame>>& dataRows) {
0021     l1t::demo::BoardData boardData(id);
0022
0023     for (size_t i = 0; i < channels.size(); i++) {
0024       std::vector<l1t::demo::Frame> channelData(dataRows.size());
0025       for (size_t j = 0; j < dataRows.size(); j++)
0026         channelData.at(j) = dataRows.at(j).at(i);
0027       boardData.add(channels.at(i), channelData);
0028     }
0029
0030     return boardData;
0031   }
0032
0033   std::vector<std::string> searchAndTokenize(std::istream& file, const std::string& linePrefix) {
0034     std::string line;
0035
0036     while (getline(file, line)) {
0037       // Strip leading spaces
0038       size_t startIndex = line.find_first_not_of(" \t");
0039       if (startIndex != std::string::npos)
0040         line = line.substr(startIndex);
0041
0042       if (line.empty())
0043         continue;
0044       if (line[0] == '#')
0045         continue;
0046
0047       if (line.rfind(linePrefix, 0) != std::string::npos) {
0048         std::vector<std::string> tokens;
0049
0050         // Split the line into tokens
0051         const std::regex delimiterRegex("\\s+");
0052         std::sregex_token_iterator it(line.begin() + linePrefix.size(), line.end(), delimiterRegex, -1);
0053
0054         for (; it != std::sregex_token_iterator(); it++) {
0055           const std::string token(it->str());
0056           if (token.empty())
0057             continue;
0058           tokens.push_back(token);
0059         }
0060
0061         return tokens;
0062       } else
0063         throw std::logic_error("Found unexpected line found when searching for \"" + linePrefix + "\": \"" + line +
0064                                "\"");
0065     }
0066     throw std::logic_error("Couldn't find any line starting with \"" + linePrefix + "\"");
0067   }
0068
0069 }  // namespace
0070
0071 namespace l1t::demo {
0072
0073   FileFormat parseFileFormat(const std::string& s) {
0074     static const std::unordered_map<std::string, FileFormat> kFormatStringMap({{"EMP", FileFormat::EMPv1},
0075                                                                                {"emp", FileFormat::EMPv1},
0076                                                                                {"EMPv1", FileFormat::EMPv1},
0077                                                                                {"EMPv2", FileFormat::EMPv2},
0078                                                                                {"APx", FileFormat::APx},
0079                                                                                {"apx", FileFormat::APx},
0080                                                                                {"X2O", FileFormat::X2O},
0081                                                                                {"x2O", FileFormat::X2O}});
0082
0083     const auto it = kFormatStringMap.find(s);
0084     if (it == kFormatStringMap.end())
0085       throw std::runtime_error("Could not convert '" + s + "' to FileFormat enum value");
0086
0087     return it->second;
0088   }
0089
0090   BoardData readAPxFile(std::istream&);
0091
0092   BoardData readEMPFileV1(std::istream&);
0093
0094   BoardData readEMPFileV2(std::istream&);
0095
0096   BoardData readX2OFile(std::istream&);
0097
0098   BoardData read(const std::string& filePath, const FileFormat format) {
0099     std::ifstream file(filePath, std::ios_base::in | std::ios_base::binary);
0100
0101     if (not file.is_open())
0102       throw std::runtime_error("Could not open file '" + filePath + "'");
0103
0104     boost::iostreams::filtering_istream stream;
0105     if (filePath.rfind(".gz") == filePath.length() - 3)
0106       stream.push(boost::iostreams::gzip_decompressor());
0107     else if (filePath.rfind(".xz") == filePath.length() - 3)
0108       stream.push(boost::iostreams::lzma_decompressor());
0109
0110     stream.push(file);
0111
0112     return read(stream, format);
0113   }
0114
0115   BoardData read(std::istream& file, const FileFormat format) {
0116     switch (format) {
0117       case FileFormat::APx:
0118         return readAPxFile(file);
0119       case FileFormat::EMPv1:
0120         return readEMPFileV1(file);
0121       case FileFormat::EMPv2:
0122         return readEMPFileV2(file);
0123       case FileFormat::X2O:
0124         return readX2OFile(file);
0125     }
0126
0127     std::ostringstream messageStream;
0128     messageStream << "No read function registered for format " << format;
0129     throw std::runtime_error(messageStream.str());
0130   }
0131
0132   BoardData readAPxFile(std::istream& file) {
0133     std::string line;
0134
0135     // Complain if file is empty
0136     if (not std::getline(file, line))
0137       throw std::runtime_error("Specified file is empty!");
0138
0139     // If first line is sideband, skip it and move onto 2nd line
0140     if (line.find("#Sideband") == 0) {
0141       if (not std::getline(file, line))
0142         throw std::runtime_error("APx file has incorrect format: Link labels and data missing!");
0143     }
0144
0145     // Parse link labels
0146     if (line.find("#LinkLabel") != 0)
0147       throw std::runtime_error(
0148           "APx file has incorrect format: Link header does not start with '#LinkLabel' (line is '" + line + "')");
0149
0150     std::vector<size_t> indices;
0151     const std::regex delimiterRegex("\\s+");
0152     std::sregex_token_iterator it(line.begin() + 10, line.end(), delimiterRegex, -1);
0153     for (; it != std::sregex_token_iterator(); it++) {
0154       const std::string token(it->str());
0155       if (token.empty())
0156         continue;
0157
0158       if (token.find("LINK_") != 0)
0159         throw std::runtime_error("Link column name '" + token + "' (does not start with 'LINK_')");
0160       if (token.size() == 5)
0161         throw std::runtime_error("Link column name '" + token + "' is too short");
0162       if (not std::all_of(token.begin() + 5, token.end(), ::isdigit))
0163         throw std::runtime_error("Link column name '" + token + "' does not end with a number");
0164
0165       indices.push_back(std::stoul(token.substr(5)));
0166     }
0167
0168     // Check for '#BeginData' line
0169     if (not std::getline(file, line))
0170       throw std::runtime_error("APx file has incorrect format: Data missing!");
0171     if (line != "#BeginData")
0172       throw std::runtime_error("APx file has incorrect format: '#BeginData' line missing (found '" + line + "')");
0173
0174     // Parse link data
0175     std::vector<std::vector<l1t::demo::Frame>> dataRows;
0176     while (std::getline(file, line)) {
0177       it = std::sregex_token_iterator(line.begin(), line.end(), delimiterRegex, -1);
0178       size_t count = 0;
0179       for (; it != std::sregex_token_iterator(); it++, count++) {
0180         const std::string token(it->str());
0181
0182         if ((token.find("0x") != 0) or (not std::all_of(token.begin() + 2, token.end(), ::isxdigit)))
0183           throw std::runtime_error("APx file has incorrect format: Data token '" + token +
0184                                    "' is not hexadecimal number");
0185
0186         if (count == 0) {
0187           size_t rowIndex = std::stoul(token, nullptr, 16);
0188           if (rowIndex != dataRows.size())
0189             throw std::runtime_error("APx file has incorrect format: Expected data row " +
0190                                      std::to_string(dataRows.size()) + ", but found row " + std::to_string(rowIndex));
0191           dataRows.push_back(std::vector<l1t::demo::Frame>(indices.size()));
0192         }
0193         // Sideband info
0194         else if ((count % 2) == 1) {
0195           uint16_t sbValue = std::stoul(token, nullptr, 16);
0196           dataRows.back().at((count - 1) / 2).valid = (sbValue & 0x1);
0197           dataRows.back().at((count - 1) / 2).startOfPacket = ((sbValue >> 1) & 0x1);
0198           dataRows.back().at((count - 1) / 2).endOfPacket = ((sbValue >> 3) & 0x1);
0199         }
0200         // Data word
0201         else
0202           dataRows.back().at((count - 1) / 2).data = ap_uint<64>(std::stoull(token, nullptr, 16));
0203       }
0204
0205       if (count != (2 * indices.size() + 1))
0206         throw std::runtime_error("APx file has incorrect format: Line has incorrect number of tokens (expected " +
0207                                  std::to_string(2 * indices.size() + 1) + ", found " + std::to_string(count) + "!");
0208     }
0209
0210     return createBoardDataFromRows("", indices, dataRows);
0211   }
0212
0213   BoardData readEMPFileV1(std::istream& file) {
0214     // 1) Search for ID string
0215     std::string id, line;
0216     while (getline(file, line)) {
0217       if (line.empty())
0218         continue;
0219       if (line[0] == '#')
0220         continue;
0221
0222       if (line.rfind("Board ", 0) != std::string::npos) {
0223         id = line.substr(6);
0224         break;
0225       } else
0226         throw std::logic_error("Found unexpected line found when searching for board ID: \"" + line + "\"");
0227     }
0228
0229     // 2) Search for column labels (i.e. list of channels/links)
0230     searchAndTokenize(file, "Quad/Chan :");
0231     const auto tokens = searchAndTokenize(file, "Link :");
0232     std::vector<size_t> channels;
0233     std::transform(tokens.begin(), tokens.end(), std::back_inserter(channels), [](const std::string& s) {
0234       return std::stoull(s);
0235     });
0236
0237     // 3) Read the main data rows
0238     const std::regex delimiterRegex("\\s+");
0239     static const std::regex frameRegex("([01]s)?([01]v)([0-9a-fA-F]{16})");
0240     std::vector<std::vector<Frame>> dataRows;
0241     while (file.good() and getline(file, line)) {
0242       if (line.empty() or line[0] == '#')
0243         continue;
0244
0245       std::ostringstream prefixStream;
0246       prefixStream << "Frame ";
0247       prefixStream << std::setw(4) << std::setfill('0') << dataRows.size();
0248       prefixStream << " :";
0249
0250       const std::string prefix(prefixStream.str());
0251       if (line.rfind(prefix, 0) == std::string::npos)
0252         throw std::logic_error("Found unexpected line found when searching for \"" + prefix + "\": \"" + line + "\"");
0253
0254       std::vector<l1t::demo::Frame> row;
0255       std::sregex_token_iterator it(line.begin() + prefix.size(), line.end(), delimiterRegex, -1);
0256       for (; it != std::sregex_token_iterator(); it++) {
0257         const std::string token(it->str());
0258         if (token.empty())
0259           continue;
0260
0261         std::smatch what;
0262         if (not std::regex_match(token, what, frameRegex))
0263           throw std::logic_error("Token '" + token + "' doesn't match the valid format");
0264
0265         l1t::demo::Frame value;
0266         // Import strobe if the strobe group is matched
0267         if (what[1].matched) {
0268           value.strobe = (what[1] == "1s");
0269         }
0270
0271         value.valid = (what[2] == "1v");
0272         value.data = ap_uint<64>(std::stoull(what[3].str(), nullptr, 16));
0273
0274         row.push_back(value);
0275       }
0276
0277       dataRows.push_back(row);
0278     }
0279
0280     return createBoardDataFromRows(id, channels, dataRows);
0281   }
0282
0283   BoardData readEMPFileV2(std::istream& file) {
0284     // 1) Search for ID string
0285     std::string id, line;
0286     while (getline(file, line)) {
0287       if (line.empty())
0288         continue;
0289       if (line[0] == '#')
0290         continue;
0291
0292       if (line.rfind("ID: ", 0) != std::string::npos) {
0293         id = line.substr(4);
0294         break;
0295       } else
0296         throw std::logic_error("Found unexpected line found when searching for board ID: \"" + line + "\"");
0297     }
0298
0299     // 2) Check that next line states metadata formatting
0300     getline(file, line);
0301     if (line.find("Metadata: (strobe,) start of orbit, start of packet, end of packet, valid") != 0)
0302       throw std::logic_error("Expected metadata line following 'ID' line. Instead found:" + line);
0303
0304     // 3) Search for column labels (i.e. list of channels/links)
0305     const auto tokens = searchAndTokenize(file, "Link  ");
0306     std::vector<size_t> channels;
0307     std::transform(tokens.begin(), tokens.end(), std::back_inserter(channels), [](const std::string& s) {
0308       return std::stoull(s);
0309     });
0310
0311     // 4) Read the main data rows
0312     const std::regex delimiterRegex("\\s\\s+");
0313     static const std::regex frameRegex("([01])?([01])([01])([01])([01]) ([0-9a-fA-F]{16})");
0314     std::vector<std::vector<Frame>> dataRows;
0315     while (file.good() and getline(file, line)) {
0316       if (line.empty() or line[0] == '#')
0317         continue;
0318
0319       std::ostringstream prefixStream;
0320       prefixStream << "Frame ";
0321       prefixStream << std::setw(4) << std::setfill('0') << dataRows.size();
0322       prefixStream << "  ";
0323
0324       const std::string prefix(prefixStream.str());
0325       if (line.rfind(prefix, 0) == std::string::npos)
0326         throw std::logic_error("Found unexpected line found when searching for \"" + prefix + "\": \"" + line + "\"");
0327
0328       std::vector<l1t::demo::Frame> row;
0329       std::sregex_token_iterator it(line.begin() + prefix.size(), line.end(), delimiterRegex, -1);
0330       for (; it != std::sregex_token_iterator(); it++) {
0331         const std::string token(it->str());
0332         if (token.empty())
0333           continue;
0334
0335         std::smatch what;
0336         if (not std::regex_match(token, what, frameRegex))
0337           throw std::logic_error("Token '" + token + "' doesn't match the valid format");
0338
0339         l1t::demo::Frame value;
0340         // Import strobe if the strobe group is matched
0341         if (what[1].matched) {
0342           value.strobe = (what[1] == "1");
0343         }
0344
0345         value.startOfOrbit = (what[2] == "1");
0346         value.startOfPacket = (what[3] == "1");
0347         value.endOfPacket = (what[4] == "1");
0348         value.valid = (what[5] == "1");
0349         value.data = ap_uint<64>(std::stoull(what[6].str(), nullptr, 16));
0350
0351         row.push_back(value);
0352       }
0353
0354       dataRows.push_back(row);
0355     }
0356
0357     return createBoardDataFromRows(id, channels, dataRows);
0358   }
0359
0360   BoardData readX2OFile(std::istream& file) {
0361     throw std::runtime_error("Reading X2O file format not yet implemented. Will be done ASAP.");
0362   }
0363
0364   void writeAPxFile(const BoardData&, std::ostream&);
0365
0366   void writeEMPFileV1(const BoardData&, std::ostream&);
0367
0368   void writeEMPFileV2(const BoardData&, std::ostream&);
0369
0370   void writeX2OFile(const BoardData&, std::ostream&);
0371
0372   void write(const BoardData& data, const std::string& filePath, const FileFormat format) {
0373     // Open file
0374 #ifdef CMSSW_GIT_HASH
0375     edm::LogInfo("L1TDemonstratorTools")
0376 #else
0377     std::cout
0378 #endif
0379         << "Writing board data (" << std::distance(data.begin(), data.end()) << " channels, "
0380         << data.begin()->second.size() << " frames) to file '" << filePath << "' (format: " << format << ")"
0381         << std::endl;
0382
0383     std::ofstream file(filePath, std::ios_base::out | std::ios_base::binary);
0384
0385     if (not file.is_open())
0386       throw std::runtime_error("Could not open file '" + filePath + "'");
0387
0388     boost::iostreams::filtering_ostream stream;
0389     if (filePath.rfind(".gz") == filePath.length() - 3)
0390       stream.push(boost::iostreams::gzip_compressor());
0391     else if (filePath.rfind(".xz") == filePath.length() - 3)
0392       stream.push(boost::iostreams::lzma_compressor());
0393     stream.push(file);
0394     write(data, stream, format);
0395   }
0396
0397   void write(const BoardData& data, std::ostream& file, const FileFormat format) {
0398     // Check that number of frames is same for every channel
0399     const auto firstChannel = data.begin();
0400
0401     for (const auto& channel : data) {
0402       const auto i = channel.first;
0403       const auto channelData = channel.second;
0404       if (channelData.size() != firstChannel->second.size())
0405         throw std::runtime_error("Cannot write board data to file - channels do not all have the same length (" +
0406                                  std::to_string(channelData.size()) + " words on channel " + std::to_string(i) +
0407                                  ", but " + std::to_string(firstChannel->second.size()) + " words on channel " +
0408                                  std::to_string(firstChannel->first) + ")");
0409     }
0410
0411     // Call relevant write function
0412     switch (format) {
0413       case FileFormat::APx:
0414         writeAPxFile(data, file);
0415         return;
0416       case FileFormat::EMPv1:
0417         writeEMPFileV1(data, file);
0418         return;
0419       case FileFormat::EMPv2:
0420         writeEMPFileV2(data, file);
0421         return;
0422       case FileFormat::X2O:
0423         writeX2OFile(data, file);
0424         return;
0425     }
0426   }
0427
0428   void writeAPxFile(const BoardData& data, std::ostream& file) {
0429     // Note: APx sideband encoding
0430     //   Short-term, simulation only:
0431     //     0 -> Valid
0432     //     1 -> EOF
0433     //   Planned (from ~ May 2021)
0434     //     0 -> Valid
0435     //     1 -> SOF (Start Of Frame)
0436     //     2 -> FFO (First Frame of Orbit)
0437     //     3 -> EOF (End Of Frame)
0438     //     4 -> FERR (Frame Error)
0439     //     5 -> RSV1
0440     //     6 -> RSV2
0441     //     7 -> RSV3
0442
0443     file << std::setfill('0');
0444     file << "#Sideband ON" << std::endl;
0445
0446     // Channel header
0447     file << "#LinkLabel";
0448     for (const auto& channel : data) {
0449       const auto i = channel.first;
0450       file << "                LINK_" << std::setw(2) << i << "    ";
0451     }
0452     file << std::endl;
0453
0454     file << "#BeginData" << std::endl;
0455
0456     // Frames
0457     file << std::hex;
0458     const auto firstChannel = data.begin();
0459     for (size_t i = 0; i < firstChannel->second.size(); i++) {
0460       file << "0x" << std::setw(4) << i;
0461       for (const auto& channel : data) {
0462         //const auto j = channel.first;
0463         const auto channelData = channel.second;
0464         uint16_t sideband = channelData.at(i).valid;
0465         sideband |= channelData.at(i).startOfPacket << 1;
0466         sideband |= channelData.at(i).endOfPacket << 3;
0467         file << "    0x" << std::setw(2) << sideband;
0468         file << " 0x" << std::setw(16) << uint64_t(channelData.at(i).data);
0469       }
0470       file << std::endl;
0471     }
0472   }
0473
0474   void writeEMPFileV1(const BoardData& data, std::ostream& file) {
0475     file << std::setfill('0');
0476
0477     // Board name/id
0478     file << "Board CMSSW" << std::endl;
0479
0480     // Quad/chan header
0481     file << " Quad/Chan :";
0482     for (const auto& channel : data) {
0483       const auto i = channel.first;
0484       file << "         q" << std::setw(2) << i / 4 << 'c' << std::setw(1) << i % 4 << "       ";
0485     }
0486     file << std::endl;
0487
0488     // Link header
0489     file << "      Link :";
0490     for (const auto& channel : data) {
0491       const auto i = channel.first;
0492       file << "          " << std::setw(3) << i << "        ";
0493     }
0494     file << std::endl;
0495
0496     // Frames
0497     const auto firstChannel = data.begin();
0498     for (size_t i = 0; i < firstChannel->second.size(); i++) {
0499       file << "Frame " << std::setw(4) << i << " :";
0500       for (const auto& channel : data) {
0501         //const auto j = channel.first;
0502         const auto channelData = channel.second;
0503         file << " ";
0504         //TODO: Add strobe if zero anywhere on channel
0505         file << "  ";
0506         file << std::setw(1) << channelData.at(i).valid << "v" << std::setw(16) << std::hex
0507              << uint64_t(channelData.at(i).data);
0508       }
0509       file << std::endl << std::dec;
0510     }
0511   }
0512
0513   void writeEMPFileV2(const BoardData& data, std::ostream& file) {
0514     file << std::setfill('0');
0515
0516     // Board name/id
0517     file << "ID: " << data.name() << std::endl;
0518     file << "Metadata: (strobe,) start of orbit, start of packet, end of packet, valid" << std::endl;
0519     file << std::endl;
0520
0521     // Link header
0522     file << "      Link  ";
0523     std::map<size_t, bool> strobedLinkMap;
0524     for (const auto& channel : data) {
0525       const auto i = channel.first;
0526       strobedLinkMap[i] =
0527           std::any_of(channel.second.begin(), channel.second.end(), [](const Frame& x) { return not x.strobe; });
0528       if (strobedLinkMap.at(i))
0529         file << " ";
0530       file << "            " << std::setw(3) << i << "        ";
0531     }
0532     file << std::endl;
0533
0534     // Frames
0535     const auto firstChannel = data.begin();
0536     for (size_t i = 0; i < firstChannel->second.size(); i++) {
0537       file << "Frame " << std::setw(4) << i << "  ";
0538       for (const auto& channel : data) {
0539         //const auto j = channel.first;
0540         const auto channelData = channel.second;
0541         file << "  ";
0542         if (strobedLinkMap.at(channel.first))
0543           file << std::setw(1) << channelData.at(i).strobe;
0544         file << std::setw(1) << channelData.at(i).startOfOrbit << channelData.at(i).startOfPacket
0545              << channelData.at(i).endOfPacket << channelData.at(i).valid;
0546         file << " " << std::setw(16) << std::hex << uint64_t(channelData.at(i).data);
0547       }
0548       file << std::endl << std::dec;
0549     }
0550   }
0551
0552   void writeX2OFile(const BoardData& data, std::ostream& file) {
0553     throw std::runtime_error("Writing X2O file format not yet implemented. Will be done ASAP.");
0554   }
0555
0556 }  // namespace l1t::demo