MPICore/test/testMPI.cc

0001 #include <iostream>
0002 #include <iomanip>
0003 #include <cstdlib>
0004 #include <string>
0005 #include <algorithm>
0006 #include <vector>
0007 #include <random>
0008 #include <utility>
0009 #include <mpi.h>
0010 #include <unistd.h>
0011
0012 // std::cout << "\n\t++++++++++++++++++++++++++++++++++++++++++++++++++++++++++";
0013 // std::cout << "\n\t+ (1)  Non Blocking Scatter.                             +";
0014 // std::cout << "\n\t+ (2)  Blocking Scatter.                                 +";
0015 // std::cout << "\n\t+ (3)  Non Blocking Send and Receive.                    +";
0016 // std::cout << "\n\t+ (4)  Blocking Send and Receive.                        +";
0017 // std::cout << "\n\t+ (5)  Non Blocking Send and Receive with Multiple Tasks +";
0018 // std::cout << "\n\t++++++++++++++++++++++++++++++++++++++++++++++++++++++++++";
0019
0020 struct MPIData {
0021   int num_procs{0};
0022   int rank{0};
0023
0024   std::pair<int, int> workSplit;
0025   std::vector<float> input1;          //declare vector 1.
0026   std::vector<float> input2;          //declare vector 2.
0027   std::vector<float> output;          //declare vector fulled only by root to get result from workers.
0028   std::vector<float> reference;       //declare vector to verify the ruslt form each process.
0029   std::vector<float> vectorWorkers1;  //declare vector 1 for workers only.
0030   std::vector<float> vectorWorkers2;  //declare vector 2 for workers only.
0031   std::vector<int> displacement;      //declare vector for selecting location of each element to be sent.
0032   std::vector<int> numberToSend;
0033 };
0034
0035 int choices = 5;             //number of functions of MPI.
0036 int size = 21;               //default size of vectors.
0037 unsigned int runNumber = 5;  //default number of time to run each function in order to take the average.
0038 int precision = 4;           //default digits after decimal point.
0039 int function = 5;            //Total number of functions in the program.
0040 int root = 0;
0041 int choice = 0;  //user Choice to select function to run.
0042
0043 std::vector<int> userChoices(1, 1);  //save convertion integer to vector.
0044
0045 const std::vector<int> chooseFunction(int toInteger);  //Convert integers to a vector.
0046 std::vector<std::pair<float, float>> timing(
0047     choices, std::make_pair(0, 0));  //to save time of scatter/send and gather/receive for each function.
0048
0049 void randomGenerator(std::vector<float>& vect);                    //generate uniform random numbers.
0050 std::pair<int, int> splitProcess(int works, int numberOfProcess);  //calcualte for each process number of works.
0051 const std::vector<int> numberDataSend(
0052     int numberOfProcess, std::pair<int, int> splitWorks);  //findout number of data to be sent for each process.
0053 const std::vector<int> displacmentData(
0054     int numberOfProcess,
0055     std::pair<int, int> splitWorks,
0056     const std::vector<int>& numberDataSend);  //findout the index of data to be sent for each process
0057 void checkingResultsPrintout(std::vector<float>& reference,
0058                              std::vector<float>& output,
0059                              std::pair<int, int> workSplit,
0060                              const std::vector<int>& displacement,
0061                              const std::vector<int>& numberDataSend);
0062
0063 const std::pair<float, float> nonBlockScatter(MPIData& mpiInput);
0064 const std::pair<float, float> blockScatter(MPIData& mpiInput);
0065 const std::pair<float, float> nonBlockSend(MPIData& mpiInput);
0066 const std::pair<float, float> blockSend(MPIData& mpiInput);
0067 const std::pair<float, float> multiNonBlockSend(MPIData& mpiInput);
0068
0069 void compare(const std::vector<std::pair<float, float>>& timing,
0070              int choices,
0071              const std::vector<int>& digits,
0072              int runNumber);  //to printout the time for each function that user chose.
0073
0074 const std::pair<float, float> returnAverage(const std::pair<float, float> (*mpiFunctions)(MPIData&),
0075                                             MPIData& mpiInput,
0076                                             unsigned int runNumber);  //to get the average of time for each function.
0077
0078 int main(int argc, char* argv[]) {
0079   int c;
0080
0081   while ((c = getopt(argc, argv, "s:r:n:")) != -1) {
0082     switch (c) {
0083       case 's':
0084         try {
0085           size = std::stoll(optarg, nullptr, 0);
0086         } catch (std::exception& err) {
0087           std::cout << "\n\tError Must be integer Argument!";
0088           std::cout << "\n\t" << err.what() << std::endl;
0089           return 0;
0090         }
0091         break;
0092       case 'r':
0093         try {
0094           //size = std::stoll(optarg, nullptr, 0);
0095           choice = std::stoll(optarg, nullptr, 0);
0096           userChoices = chooseFunction(choice);
0097           //runNumber = std::stoll(argv[3], nullptr, 0);
0098         } catch (std::exception& err) {
0099           std::cout << "\n\tError Must be integer Argument!";
0100           std::cout << "\n\t" << err.what() << std::endl;
0101           return 0;
0102         }
0103         break;
0104       case 'n':
0105         try {
0106           //size = std::stoll(argv[1], nullptr, 0);
0107           //choice = std::stoll(argv[2], nullptr, 0);
0108           //userChoices = chooseFunction(choice);
0109           runNumber = std::stoll(optarg, nullptr, 0);
0110         } catch (std::exception& err) {
0111           std::cout << "\n\tError Must be integer Argument!";
0112           std::cout << "\n\t" << err.what() << std::endl;
0113           return 0;
0114         }
0115         break;
0116       default:
0117         abort();
0118     }
0119   }
0120
0121   //   if (argc == 2) {
0122   //     try {
0123   //       size = std::stoll(argv[1], nullptr, 0);
0124   //     } catch (std::exception& err) {
0125   //       std::cout << "\n\tError Must be integer Argument!";
0126   //       std::cout << "\n\t" << err.what() << std::endl;
0127   //       return 0;
0128   //     }
0129   //   } else if (argc == 3) {
0130   //     try {
0131   //       size = std::stoll(argv[1], nullptr, 0);
0132   //       choice = std::stoll(argv[2], nullptr, 0);
0133   //       userChoices = chooseFunction(choice);
0134   //     } catch (std::exception& err) {
0135   //       std::cout << "\n\tError Must be integer Argument!";
0136   //       std::cout << "\n\t" << err.what() << std::endl;
0137   //       return 0;
0138   //     }
0139   //   }else if (argc > 3) {
0140   //     try {
0141   //       size = std::stoll(argv[1], nullptr, 0);
0142   //       choice = std::stoll(argv[2], nullptr, 0);
0143   //       userChoices = chooseFunction(choice);
0144   //       runNumber = std::stoll(argv[3], nullptr, 0);
0145   //     } catch (std::exception& err) {
0146   //       std::cout << "\n\tError Must be integer Argument!";
0147   //       std::cout << "\n\t" << err.what() << std::endl;
0148   //       return 0;
0149   //     }
0150   //   }
0151
0152   MPIData mpiInputs;  //greate object from structur to pass into MPI functios.
0153
0154   MPI_Init(&argc, &argv);                            //initialize communicator environment.
0155   mpiInputs.num_procs = MPI::COMM_WORLD.Get_size();  //get total size of processes.
0156   mpiInputs.rank = MPI::COMM_WORLD.Get_rank();       //get each process number.
0157
0158   mpiInputs.input1.resize(size);  //initialize size.
0159   mpiInputs.input2.resize(size);
0160   mpiInputs.output.resize(size);
0161   mpiInputs.reference.resize(size);
0162
0163   mpiInputs.workSplit = splitProcess(size, mpiInputs.num_procs);
0164
0165   if (!mpiInputs.workSplit.first) {
0166     MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
0167     return 0;
0168   }
0169
0170   mpiInputs.numberToSend = numberDataSend(mpiInputs.num_procs, mpiInputs.workSplit);
0171   mpiInputs.displacement = displacmentData(mpiInputs.num_procs, mpiInputs.workSplit, mpiInputs.numberToSend);
0172
0173   mpiInputs.vectorWorkers1.resize(
0174       mpiInputs.numberToSend[mpiInputs.rank]);  //Resizing each process with appropriate Receiving Data.
0175   mpiInputs.vectorWorkers2.resize(mpiInputs.numberToSend[mpiInputs.rank]);
0176
0177   if (!mpiInputs.rank)  //Only for root
0178   {
0179     randomGenerator(mpiInputs.input1);  //generate random floating numbers from(0,1) Only in the root.
0180     randomGenerator(mpiInputs.input2);
0181     std::cout << "\n\tNumber of Processes " << mpiInputs.num_procs << std::endl;
0182     std::cout << "\tNumber of workSplit First " << mpiInputs.workSplit.first << std::endl;
0183     std::cout << "\tNumber of workSplit Second " << mpiInputs.workSplit.second << std::endl;
0184     for (int j = 0; j < size; j++) {
0185       mpiInputs.reference[j] = mpiInputs.input1[j] + mpiInputs.input2[j];  //Summing for verification.
0186     }
0187   }
0188
0189   for (long unsigned int i = 0; i < userChoices.size(); ++i) {
0190     if (userChoices[i] == 1) {
0191       timing[0] = returnAverage(nonBlockScatter, mpiInputs, runNumber);
0192
0193     } else if (userChoices[i] == 2) {
0194       timing[1] = returnAverage(blockScatter, mpiInputs, runNumber);
0195
0196     } else if (userChoices[i] == 3) {
0197       timing[2] = returnAverage(nonBlockSend, mpiInputs, runNumber);
0198
0199     } else if (userChoices[i] == 4) {
0200       timing[3] = returnAverage(blockSend, mpiInputs, runNumber);
0201
0202     } else if (userChoices[i] == 5) {
0203       timing[4] = returnAverage(multiNonBlockSend, mpiInputs, runNumber);
0204
0205     } else {
0206       std::cout << "\n\n\tError the User has not chose any number of Function!\n";
0207       break;
0208     }
0209   }
0210
0211   if (!mpiInputs.rank) {
0212     compare(timing, choices, userChoices, runNumber);
0213   }
0214
0215   MPI::Finalize();
0216
0217   return 0;
0218 }
0219
0220 void randomGenerator(std::vector<float>& vect) {
0221   std::random_device rand;
0222   std::default_random_engine gener(rand());
0223   std::uniform_real_distribution<> dis(0., 1.);
0224   int size = vect.size();
0225   for (int i = 0; i < size; i++) {
0226     vect.at(i) = dis(gener);
0227   }
0228 }
0229
0230 std::pair<int, int> splitProcess(int works, int numberOfProcess) {
0231   std::pair<int, int> Return{0, 0};
0232   if (numberOfProcess > 1 && numberOfProcess <= works) {
0233     Return.first = works / (numberOfProcess - 1);   //number of cycle for each process.
0234     Return.second = works % (numberOfProcess - 1);  //extra cycle for process.
0235   } else {
0236     std::cout << "\tError Either No worker are found OR Number Processes Larger than Length!!!\n";
0237   }
0238
0239   return Return;
0240 }
0241 const std::vector<int> numberDataSend(int numberOfProcess, std::pair<int, int> splitWorks) {
0242   std::vector<int> dataSend(numberOfProcess, splitWorks.first);
0243   dataSend[0] = 0;
0244   for (int i = 1; i < splitWorks.second + 1; i++)  //neglect root
0245   {
0246     dataSend[i] += 1;  //extra work for each first processes.
0247   }
0248   return dataSend;
0249 }
0250 const std::vector<int> displacmentData(int numberOfProcess,
0251                                        std::pair<int, int> splitWorks,
0252                                        const std::vector<int>& numberDataSend) {
0253   std::vector<int> displacment(numberOfProcess, splitWorks.first);
0254
0255   displacment[0] = 0;
0256   displacment[1] = 0;  //start Here.
0257
0258   for (int i = 2; i < numberOfProcess; i++)  //neglect root
0259   {
0260     displacment[i] = numberDataSend[i - 1] + displacment[i - 1];  //extra work for each first processes.
0261   }
0262   return displacment;
0263 }
0264
0265 void checkingResultsPrintout(std::vector<float>& reference,
0266                              std::vector<float>& output,
0267                              std::pair<int, int> workSplit,
0268                              const std::vector<int>& displacement,
0269                              const std::vector<int>& numberDataSend) {
0270   float percent{0.0};
0271   float totalError{0.0};
0272   int p{1};
0273   for (int j = 0; j < size; j++) {
0274     percent = ((reference[j] - output[j]) / reference[j]) * 100;
0275     totalError += percent;
0276   }
0277   if (totalError) {
0278     std::cout << "\n-------------------------------------------------------\n";
0279     std::cout << "| RootSum | WorksSum | Error   | Error %  | Process # |";
0280     std::cout << "\n-------------------------------------------------------\n";
0281     std::cout.precision(precision);
0282     for (int j = 0; j < size; j++) {
0283       std::cout << "| " << reference[j] << "  | " << output[j] << "  |" << std::setw(9) << reference[j] - output[j]
0284                 << " |" << std::setw(9) << percent << " |" << std::setw(9) << p << " |\n";
0285
0286       if (j + 1 == displacement[p + 1]) {
0287         ++p;
0288       }
0289     }
0290     std::cout << "-------------------------------------------------------\n";
0291     std::cout << "-Total Error is " << totalError << std::endl;
0292     for (long unsigned int j = 1; j < displacement.size(); j++) {
0293       std::cout << "Process [" << j << "]"
0294                 << " Worked On " << numberDataSend[j] << " Data\n";
0295     }
0296   }
0297 }
0298
0299 const std::pair<float, float> nonBlockScatter(MPIData& mpiInput) {
0300   std::pair<float, float> returnValue;
0301   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0302   double startTimeScatter = 0;
0303   double endTimeScatter = 0;
0304   double startTimeGather = 0;
0305   double endTimeGather = 0;
0306
0307   MPI_Request requestRootScatter[2];
0308   MPI_Request requestRootGather;
0309
0310   startTimeScatter = MPI_Wtime();  //get time before scattering.
0311
0312   //Non-Blocking Scatter.
0313   MPI_Iscatterv(&mpiInput.input1[0],
0314                 &mpiInput.numberToSend[0],
0315                 &mpiInput.displacement[0],
0316                 MPI_FLOAT,
0317                 &mpiInput.vectorWorkers1[0],
0318                 mpiInput.numberToSend[mpiInput.rank],
0319                 MPI_FLOAT,
0320                 0,
0321                 MPI_COMM_WORLD,
0322                 &requestRootScatter[0]);
0323   MPI_Iscatterv(&mpiInput.input2[0],
0324                 &mpiInput.numberToSend[0],
0325                 &mpiInput.displacement[0],
0326                 MPI_FLOAT,
0327                 &mpiInput.vectorWorkers2[0],
0328                 mpiInput.numberToSend[mpiInput.rank],
0329                 MPI_FLOAT,
0330                 0,
0331                 MPI_COMM_WORLD,
0332                 &requestRootScatter[1]);
0333   MPI_Waitall(2, requestRootScatter, MPI_STATUS_IGNORE);
0334
0335   endTimeScatter = MPI_Wtime();  //get time after scattering.
0336
0337   if (mpiInput.rank)  //Only for Workers
0338   {
0339     for (long unsigned int i = 0; i < mpiInput.vectorWorkers1.size(); i++) {
0340       mpiInput.vectorWorkers1[i] += mpiInput.vectorWorkers2[i];
0341     }
0342   }
0343
0344   startTimeGather = MPI_Wtime();  //get time before Gathering.
0345
0346   //Non Blocking Gathering.
0347   MPI_Igatherv(&mpiInput.vectorWorkers1[0],
0348                mpiInput.numberToSend[mpiInput.rank],
0349                MPI_FLOAT,
0350                &mpiInput.output[0],
0351                &mpiInput.numberToSend[0],
0352                &mpiInput.displacement[0],
0353                MPI_FLOAT,
0354                0,
0355                MPI_COMM_WORLD,
0356                &requestRootGather);
0357
0358   MPI_Wait(&requestRootGather, MPI_STATUS_IGNORE);
0359   endTimeGather = MPI_Wtime();  //get time after Gathering.
0360
0361   if (!mpiInput.rank)  //Only root print out the results.
0362   {
0363     checkingResultsPrintout(
0364         mpiInput.reference, mpiInput.output, mpiInput.workSplit, mpiInput.displacement, mpiInput.numberToSend);
0365     returnValue.first = (endTimeScatter - startTimeScatter) * 1000;
0366     returnValue.second = (endTimeGather - startTimeGather) * 1000;
0367   }
0368   return returnValue;
0369 }
0370 const std::pair<float, float> blockScatter(MPIData& mpiInput) {
0371   std::pair<float, float> returnValue;
0372   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0373
0374   double startTimeScatter = 0;
0375   double endTimeScatter = 0;
0376   double startTimeGather = 0;
0377   double endTimeGather = 0;
0378
0379   //Blocking Scattering.
0380   mpiInput.vectorWorkers1.resize(
0381       mpiInput.numberToSend[mpiInput.rank]);  //Resizing each process with appropriate Receiving Data.
0382   mpiInput.vectorWorkers2.resize(mpiInput.numberToSend[mpiInput.rank]);
0383
0384   startTimeScatter = MPI_Wtime();
0385   MPI_Scatterv(&mpiInput.input1[0],
0386                &mpiInput.numberToSend[0],
0387                &mpiInput.displacement[0],
0388                MPI_FLOAT,
0389                &mpiInput.vectorWorkers1[0],
0390                mpiInput.numberToSend[mpiInput.rank],
0391                MPI_FLOAT,
0392                0,
0393                MPI_COMM_WORLD);
0394   MPI_Scatterv(&mpiInput.input2[0],
0395                &mpiInput.numberToSend[0],
0396                &mpiInput.displacement[0],
0397                MPI_FLOAT,
0398                &mpiInput.vectorWorkers2[0],
0399                mpiInput.numberToSend[mpiInput.rank],
0400                MPI_FLOAT,
0401                0,
0402                MPI_COMM_WORLD);
0403   endTimeScatter = MPI_Wtime();
0404
0405   if (mpiInput.rank)  //Only for Workers
0406   {
0407     for (long unsigned int i = 0; i < mpiInput.vectorWorkers1.size(); i++) {
0408       mpiInput.vectorWorkers1[i] += mpiInput.vectorWorkers2[i];
0409     }
0410   }
0411
0412   startTimeGather = MPI_Wtime();
0413   //Blocking Gathering.
0414   MPI_Gatherv(&mpiInput.vectorWorkers1[0],
0415               mpiInput.numberToSend[mpiInput.rank],
0416               MPI_FLOAT,
0417               &mpiInput.output[0],
0418               &mpiInput.numberToSend[0],
0419               &mpiInput.displacement[0],
0420               MPI_FLOAT,
0421               0,
0422               MPI_COMM_WORLD);
0423
0424   endTimeGather = MPI_Wtime();
0425
0426   if (!mpiInput.rank)  //Only root print out the results.
0427   {
0428     checkingResultsPrintout(
0429         mpiInput.reference, mpiInput.output, mpiInput.workSplit, mpiInput.displacement, mpiInput.numberToSend);
0430     returnValue.first = (endTimeScatter - startTimeScatter) * 1000;
0431     returnValue.second = (endTimeGather - startTimeGather) * 1000;
0432   }
0433   return returnValue;
0434 }
0435 const std::pair<float, float> nonBlockSend(MPIData& mpiInput) {
0436   std::pair<float, float> returnValue;
0437   double startTimeRootSend = 0;
0438   double endTimeRootSend = 0;
0439   double startTimeRootRecv = 0;
0440   double endTimeRootRecv = 0;
0441
0442   MPI_Request requestRootSend[2];
0443   MPI_Request requestRootRecv;
0444   MPI_Request requestWorkerSend;
0445   MPI_Request requestWorkerRecv[1];
0446
0447   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0448
0449   if (!mpiInput.rank)  //Only for root
0450   {
0451     // std::cout << "\n\t\tNon-Blocking Send and Receive " << std::endl;
0452     startTimeRootSend = MPI_Wtime();
0453     for (int i = 1; i < mpiInput.num_procs; i++) {
0454       MPI_Issend(&mpiInput.input1[mpiInput.displacement[i]],
0455                  mpiInput.numberToSend[i],
0456                  MPI_FLOAT,
0457                  i,
0458                  0,
0459                  MPI_COMM_WORLD,
0460                  &requestRootSend[0]);  //Tag is 0
0461       MPI_Issend(&mpiInput.input2[mpiInput.displacement[i]],
0462                  mpiInput.numberToSend[i],
0463                  MPI_FLOAT,
0464                  i,
0465                  0,
0466                  MPI_COMM_WORLD,
0467                  &requestRootSend[1]);
0468       MPI_Waitall(2, requestRootSend, MPI_STATUS_IGNORE);
0469     }
0470     endTimeRootSend = MPI_Wtime();
0471   }
0472
0473   if (mpiInput.rank)  //Only for Workers
0474   {
0475     MPI_Irecv(&mpiInput.vectorWorkers1[0],
0476               mpiInput.numberToSend[mpiInput.rank],
0477               MPI_FLOAT,
0478               root,
0479               0,
0480               MPI_COMM_WORLD,
0481               &requestWorkerRecv[0]);
0482     MPI_Irecv(&mpiInput.vectorWorkers2[0],
0483               mpiInput.numberToSend[mpiInput.rank],
0484               MPI_FLOAT,
0485               root,
0486               0,
0487               MPI_COMM_WORLD,
0488               &requestWorkerRecv[1]);
0489
0490     MPI_Waitall(2, requestWorkerRecv, MPI_STATUS_IGNORE);
0491     for (long unsigned int i = 0; i < mpiInput.vectorWorkers1.size(); i++) {
0492       mpiInput.vectorWorkers1[i] += mpiInput.vectorWorkers2[i];
0493     }
0494     MPI_Issend(&mpiInput.vectorWorkers1[0],
0495                mpiInput.numberToSend[mpiInput.rank],
0496                MPI_FLOAT,
0497                root,
0498                0,
0499                MPI_COMM_WORLD,
0500                &requestWorkerSend);  //Tag is 0
0501     MPI_Wait(&requestWorkerSend, MPI_STATUS_IGNORE);
0502   }
0503
0504   if (!mpiInput.rank)  //Only for root
0505   {
0506     startTimeRootRecv = MPI_Wtime();
0507     for (int i = 1; i < mpiInput.num_procs; i++) {
0508       MPI_Irecv(&mpiInput.output[mpiInput.displacement[i]],
0509                 mpiInput.numberToSend[i],
0510                 MPI_FLOAT,
0511                 i,
0512                 0,
0513                 MPI_COMM_WORLD,
0514                 &requestRootRecv);
0515       MPI_Wait(&requestRootRecv, MPI_STATUS_IGNORE);
0516     }
0517     endTimeRootRecv = MPI_Wtime();
0518
0519     checkingResultsPrintout(mpiInput.reference,
0520                             mpiInput.output,
0521                             mpiInput.workSplit,
0522                             mpiInput.displacement,
0523                             mpiInput.numberToSend);  //Only root print out the results.
0524     returnValue.first = (endTimeRootSend - startTimeRootSend) * 1000;
0525     returnValue.second = (endTimeRootRecv - startTimeRootRecv) * 1000;
0526   }
0527   return returnValue;
0528 }
0529 const std::pair<float, float> blockSend(MPIData& mpiInput) {
0530   std::pair<float, float> returnValue;
0531   double startTimeRootSend = 0;
0532   double endTimeRootSend = 0;
0533   double startTimeRootRecv = 0;
0534   double endTimeRootRecv = 0;
0535
0536   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0537
0538   if (!mpiInput.rank)  //Only for root
0539   {
0540     // std::cout << "\n\t\tBlocking Send and Receive " << std::endl;
0541     startTimeRootSend = MPI_Wtime();
0542     for (int i = 1; i < mpiInput.num_procs; i++) {
0543       MPI_Send(&mpiInput.input1[mpiInput.displacement[i]],
0544                mpiInput.numberToSend[i],
0545                MPI_FLOAT,
0546                i,
0547                0,
0548                MPI_COMM_WORLD);  //Tag is 0
0549       MPI_Send(&mpiInput.input2[mpiInput.displacement[i]], mpiInput.numberToSend[i], MPI_FLOAT, i, 0, MPI_COMM_WORLD);
0550     }
0551     endTimeRootSend = MPI_Wtime();
0552   }
0553
0554   if (mpiInput.rank)  //Only for Workers
0555   {
0556     MPI_Recv(&mpiInput.vectorWorkers1[0],
0557              mpiInput.numberToSend[mpiInput.rank],
0558              MPI_FLOAT,
0559              root,
0560              0,
0561              MPI_COMM_WORLD,
0562              MPI_STATUS_IGNORE);
0563     MPI_Recv(&mpiInput.vectorWorkers2[0],
0564              mpiInput.numberToSend[mpiInput.rank],
0565              MPI_FLOAT,
0566              root,
0567              0,
0568              MPI_COMM_WORLD,
0569              MPI_STATUS_IGNORE);
0570
0571     for (long unsigned int i = 0; i < mpiInput.vectorWorkers1.size(); i++) {
0572       mpiInput.vectorWorkers1[i] += mpiInput.vectorWorkers2[i];
0573     }
0574     MPI_Send(&mpiInput.vectorWorkers1[0],
0575              mpiInput.numberToSend[mpiInput.rank],
0576              MPI_FLOAT,
0577              root,
0578              0,
0579              MPI_COMM_WORLD);  //Tag is 0
0580   }
0581
0582   if (!mpiInput.rank)  //Only for root
0583   {
0584     startTimeRootRecv = MPI_Wtime();
0585     for (int i = 1; i < mpiInput.num_procs; i++) {
0586       MPI_Recv(&mpiInput.output[mpiInput.displacement[i]],
0587                mpiInput.numberToSend[i],
0588                MPI_FLOAT,
0589                i,
0590                0,
0591                MPI_COMM_WORLD,
0592                MPI_STATUS_IGNORE);
0593     }
0594     endTimeRootRecv = MPI_Wtime();
0595
0596     checkingResultsPrintout(mpiInput.reference,
0597                             mpiInput.output,
0598                             mpiInput.workSplit,
0599                             mpiInput.displacement,
0600                             mpiInput.numberToSend);  //Only root print out the results.
0601     returnValue.first = (endTimeRootSend - startTimeRootSend) * 1000;
0602     returnValue.second = (endTimeRootRecv - startTimeRootRecv) * 1000;
0603   }
0604   return returnValue;
0605 }
0606 const std::pair<float, float> multiNonBlockSend(MPIData& mpiInput) {
0607   std::pair<float, float> returnValue;
0608   int lastPointCount = 0;
0609   double startTimeRootSend = 0;
0610   double endTimeRootSend = 0;
0611   double startTimeRootRecv = 0;
0612   double endTimeRootRecv = 0;
0613
0614   MPI_Request requestRootSend[2];
0615   MPI_Request requestRootRecv;
0616   MPI_Request requestWorkerSend;
0617   MPI_Request requestWorkerRecv[2];
0618
0619   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0620
0621   if (!mpiInput.rank)  //Only for root
0622   {
0623     // std::cout << "\n\t\tNon-Blocking Send and Receive with Multiple Tasks" << std::endl;
0624     int flage = 0;  //set operation to processed.
0625     startTimeRootSend = MPI_Wtime();
0626     for (int i = 1; i < mpiInput.num_procs; i++) {
0627       MPI_Issend(&mpiInput.input1[mpiInput.displacement[i]],
0628                  mpiInput.numberToSend[i],
0629                  MPI_FLOAT,
0630                  i,
0631                  0,
0632                  MPI_COMM_WORLD,
0633                  &requestRootSend[0]);  //Tag is 0
0634       MPI_Issend(&mpiInput.input2[mpiInput.displacement[i]],
0635                  mpiInput.numberToSend[i],
0636                  MPI_FLOAT,
0637                  i,
0638                  0,
0639                  MPI_COMM_WORLD,
0640                  &requestRootSend[1]);
0641       do {
0642         MPI_Testall(2, requestRootSend, &flage, MPI_STATUS_IGNORE);  //2 for two requests above. Check on flage.
0643         for (; lastPointCount < size && !flage;
0644              lastPointCount++)  //do the summing while waiting for the sending request is done.
0645         {
0646           mpiInput.reference[lastPointCount] = mpiInput.input1[lastPointCount] + mpiInput.input2[lastPointCount];
0647
0648           MPI_Testall(2, requestRootSend, &flage, MPI_STATUS_IGNORE);  //2 for two requests above. Check on flage.
0649         }
0650       } while (!flage);
0651     }
0652     endTimeRootSend = MPI_Wtime();
0653   }
0654
0655   if (mpiInput.rank)  //Only for Workers
0656   {
0657     MPI_Irecv(&mpiInput.vectorWorkers1[0],
0658               mpiInput.numberToSend[mpiInput.rank],
0659               MPI_FLOAT,
0660               root,
0661               0,
0662               MPI_COMM_WORLD,
0663               &requestWorkerRecv[0]);
0664     MPI_Irecv(&mpiInput.vectorWorkers2[0],
0665               mpiInput.numberToSend[mpiInput.rank],
0666               MPI_FLOAT,
0667               root,
0668               0,
0669               MPI_COMM_WORLD,
0670               &requestWorkerRecv[1]);
0671     MPI_Waitall(2, requestWorkerRecv, MPI_STATUS_IGNORE);  //2 for two requests above.
0672
0673     for (long unsigned int i = 0; i < mpiInput.vectorWorkers1.size(); i++) {
0674       mpiInput.vectorWorkers1[i] += mpiInput.vectorWorkers2[i];
0675     }
0676     MPI_Issend(&mpiInput.vectorWorkers1[0],
0677                mpiInput.numberToSend[mpiInput.rank],
0678                MPI_FLOAT,
0679                root,
0680                0,
0681                MPI_COMM_WORLD,
0682                &requestWorkerSend);  //Tag is 0
0683     MPI_Wait(&requestWorkerSend, MPI_STATUS_IGNORE);
0684   }
0685
0686   if (!mpiInput.rank)  //Only for root
0687   {
0688     int flage2 = 0;  //set operation to processed.
0689     startTimeRootRecv = MPI_Wtime();
0690     for (int i = 1; i < mpiInput.num_procs; i++) {
0691       MPI_Irecv(&mpiInput.output[mpiInput.displacement[i]],
0692                 mpiInput.numberToSend[i],
0693                 MPI_FLOAT,
0694                 i,
0695                 0,
0696                 MPI_COMM_WORLD,
0697                 &requestRootRecv);
0698       do {
0699         MPI_Test(&requestRootRecv, &flage2, MPI_STATUS_IGNORE);  //Check on flage2.
0700         for (; lastPointCount < size && !flage2;
0701              lastPointCount++)  //do the summing while waiting for the sending request is done.
0702         {
0703           mpiInput.reference[lastPointCount] = mpiInput.input1[lastPointCount] + mpiInput.input2[lastPointCount];
0704
0705           MPI_Test(&requestRootRecv, &flage2, MPI_STATUS_IGNORE);  //Check on flage.
0706         }
0707       } while (!flage2);
0708     }
0709     endTimeRootRecv = MPI_Wtime();
0710     for (; lastPointCount < size; lastPointCount++) {
0711       mpiInput.reference[lastPointCount] = mpiInput.input1[lastPointCount] + mpiInput.input2[lastPointCount];
0712     }
0713     checkingResultsPrintout(mpiInput.reference,
0714                             mpiInput.output,
0715                             mpiInput.workSplit,
0716                             mpiInput.displacement,
0717                             mpiInput.numberToSend);  //Only root print out the results.
0718     returnValue.first = (endTimeRootSend - startTimeRootSend) * 1000;
0719     returnValue.second = (endTimeRootRecv - startTimeRootRecv) * 1000;
0720   }
0721   return returnValue;
0722 }
0723
0724 const std::vector<int> chooseFunction(int toInteger) {
0725   std::vector<int> digits(0, 0);
0726   std::vector<int> ERROR(0, 0);
0727
0728   int digit{1};
0729
0730   while (toInteger > 0) {
0731     digit = toInteger % 10;
0732     if (digit > choices) {
0733       std::cout << "\n\tError Must be integer Argument <= " << choices << std::endl;
0734       return ERROR;
0735     }
0736     digits.push_back(digit);
0737     toInteger /= 10;
0738   }
0739   std::reverse(digits.begin(), digits.end());
0740   return digits;
0741 }
0742
0743 void compare(const std::vector<std::pair<float, float>>& timing,
0744              int choices,
0745              const std::vector<int>& digits,
0746              int runNumber) {
0747   std::cout.setf(std::ios::fixed, std::ios::floatfield);
0748
0749   int j{0};
0750   int k{0};
0751   for (long unsigned int i = 0; i < timing.size(); ++i) {
0752     if (timing[i].first) {
0753       switch (i) {
0754         case 0:
0755           std::cout << "\n\t\t(1) Non-Blocking Scatter " << std::endl;
0756           break;
0757         case 1:
0758           std::cout << "\n\t\t(2) Blocking Scatter " << std::endl;
0759           break;
0760         case 2:
0761           std::cout << "\n\t\t(3) Non-Blocking Send and Receive " << std::endl;
0762           break;
0763         case 3:
0764           std::cout << "\n\t\t(4) Blocking Send and Receive " << std::endl;
0765           break;
0766         case 4:
0767           std::cout << "\n\t\t(5) Non-Blocking Send and Receive with Multiple Tasks" << std::endl;
0768           break;
0769         default:
0770           std::cout << "\nSomething went wrong!\n";
0771       }
0772     }
0773   }
0774   std::cout << "\n\n\t=============================================================";
0775   std::cout << "\n\t|| Func ||  Scatter/Send ||   Gather/Receive  || Number Run||";
0776   std::cout << "\n\t=============================================================";
0777   for (long unsigned int i = 0; i < timing.size(); ++i) {
0778     if (timing[i].first) {
0779       if (k < j) {
0780         std::cout << "\n\t------------------------------------------------------------";
0781       }
0782       std::cout.flags(std::ios::fixed | std::ios::showpoint);
0783       std::cout.precision(precision);
0784       std::cout << "\n\t||  " << std::setw(1) << digits[k] << "   ||     " << std::setw(5) << timing[i].first
0785                 << "    ||        " << std::setw(5) << timing[i].second << "     ||    " << std::setw(3) << runNumber
0786                 << "    ||";
0787       j += 2;
0788       ++k;
0789     }
0790   }
0791   std::cout << "\n\t=============================================================\n\n";
0792 }
0793
0794 const std::pair<float, float> returnAverage(const std::pair<float, float> (*mpiFunctions)(MPIData&),
0795                                             MPIData& mpiInput,
0796                                             unsigned int runNumber) {
0797   std::pair<float, float> output;
0798   for (long unsigned int i = 0; i < runNumber; ++i) {
0799     auto accum = mpiFunctions(mpiInput);
0800     output.first += accum.first;
0801     output.second += accum.second;
0802   }
0803   output.first /= runNumber;
0804   output.second /= runNumber;
0805   return output;
0806 }