interface/common/nnet_activation.h

0001 //
0002 //    rfnoc-hls-neuralnet: Vivado HLS code for neural-net building blocks
0003 //
0004 //    Copyright (C) 2017 EJ Kreinar
0005 //
0006 //    This program is free software: you can redistribute it and/or modify
0007 //    it under the terms of the GNU General Public License as published by
0008 //    the Free Software Foundation, either version 3 of the License, or
0009 //    (at your option) any later version.
0010 //
0011 //    This program is distributed in the hope that it will be useful,
0012 //    but WITHOUT ANY WARRANTY; without even the implied warranty of
0013 //    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
0014 //    GNU General Public License for more details.
0015 //
0016 //    You should have received a copy of the GNU General Public License
0017 //    along with this program.  If not, see <http://www.gnu.org/licenses/>.
0018 //
0019
0020 #ifndef NNET_ACTIVATION_H_
0021 #define NNET_ACTIVATION_H_
0022
0023 #include <cmath>
0024 #include "ap_fixed.h"
0025 #include "nnet_common.h"
0026
0027 namespace nnet {
0028
0029   struct activ_config {
0030     // IO size
0031     static const unsigned n_in = 10;
0032
0033     // Internal info
0034     static const unsigned table_size = 1024;
0035
0036     // Resource reuse info
0037     static const unsigned io_type = io_parallel;
0038     static const unsigned reuse_factor = 1;
0039
0040     // Internal data type definitions
0041     //typedef ap_fixed<18,8> table_t;
0042     typedef float table_t;
0043   };
0044
0045   // *************************************************
0046   //       LINEAR Activation -- See Issue 53
0047   // *************************************************
0048   template <class data_T, class res_T, typename CONFIG_T>
0049   void linear(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0050     for (unsigned ii = 0; ii < CONFIG_T::n_in; ii++) {
0051       res[ii] = data[ii];
0052     }
0053   }
0054
0055   // *************************************************
0056   //       RELU Activation
0057   // *************************************************
0058   template <class data_T, class res_T, typename CONFIG_T>
0059   void relu(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0060     data_T datareg;
0061     for (unsigned ii = 0; ii < CONFIG_T::n_in; ii++) {
0062       datareg = data[ii];
0063       if (datareg > 0)
0064         res[ii] = datareg;
0065       else
0066         res[ii] = 0;
0067     }
0068   }
0069
0070   template <class data_T, class res_T, int MAX_INT, typename CONFIG_T>
0071   void relu_max(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0072     data_T datareg;
0073     for (unsigned ii = 0; ii < CONFIG_T::n_in; ii++) {
0074       datareg = data[ii];
0075       if (datareg < 0)
0076         res[ii] = 0;
0077       else if (datareg > MAX_INT)
0078         res[ii] = MAX_INT;
0079       else
0080         res[ii] = datareg;
0081     }
0082   }
0083
0084   template <class data_T, class res_T, typename CONFIG_T>
0085   void relu6(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0086     relu_max<data_T, res_T, 6, CONFIG_T>(data, res);
0087   }
0088
0089   // *************************************************
0090   //       Sigmoid Activation
0091   // *************************************************
0092   template <class out_T>
0093   inline out_T sigmoid_fcn_float(float input) {
0094     return 1.0 / (1 + exp(-input));
0095   }
0096
0097   template <class res_T, typename CONFIG_T, int N_TABLE>
0098   void init_sigmoid_table(res_T table_out[N_TABLE]) {
0099     // Default logistic sigmoid function:
0100     //   result = 1/(1+e^(-x))
0101     for (unsigned ii = 0; ii < N_TABLE; ii++) {
0102       // First, convert from table index to X-value (signed 8-bit, range -8 to +8)
0103       float in_val = 2 * 8.0 * (ii - float(N_TABLE) / 2.0) / float(N_TABLE);
0104       // Next, compute lookup table function
0105       res_T real_val = sigmoid_fcn_float<res_T>(in_val);
0106       //std::cout << "Lookup table In Value: " << in_val << " Result: " << real_val << std::endl;
0107       table_out[ii] = (res_T)real_val;
0108     }
0109   }
0110
0111   template <class data_T, class res_T, typename CONFIG_T>
0112   void sigmoid(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0113     // Initialize the lookup table
0114     res_T sigmoid_table[CONFIG_T::table_size];
0115     init_sigmoid_table<res_T, CONFIG_T, CONFIG_T::table_size>(sigmoid_table);
0116
0117     // Index into the lookup table based on data
0118     int data_round;
0119     unsigned index;
0120     for (unsigned ii = 0; ii < CONFIG_T::n_in; ii++) {
0121       data_round = data[ii] * CONFIG_T::table_size / 16;
0122       index = data_round + 8 * CONFIG_T::table_size / 16;
0123       /*if (index < 0)
0124         index = 0;*/
0125       if (index > CONFIG_T::table_size - 1)
0126         index = CONFIG_T::table_size - 1;
0127       res[ii] = (res_T)sigmoid_table[index];
0128     }
0129   }
0130
0131   // *************************************************
0132   //       Softmax Activation
0133   // *************************************************
0134   inline float exp_fcn_float(float input) { return exp(input); }
0135
0136   template <typename CONFIG_T, int N_TABLE>
0137   void init_exp_table(typename CONFIG_T::table_t table_out[N_TABLE]) {
0138     for (unsigned ii = 0; ii < N_TABLE; ii++) {
0139       // First, convert from table index to X-value (signed 8-bit, range -8 to +8)
0140       float in_val = 2 * 8.0 * (ii - float(N_TABLE) / 2.0) / float(N_TABLE);
0141       // Next, compute lookup table function
0142       typename CONFIG_T::table_t real_val = exp_fcn_float(in_val);
0143       //std::cout << "Lookup table In Value: " << in_val << " Result: " << real_val << std::endl;
0144       table_out[ii] = real_val;
0145     }
0146   }
0147
0148   template <typename CONFIG_T, int N_TABLE>
0149   void init_invert_table(typename CONFIG_T::table_t table_out[N_TABLE]) {
0150     // Inversion function:
0151     //   result = 1/x
0152     for (unsigned ii = 0; ii < N_TABLE; ii++) {
0153       // First, convert from table index to X-value (signed 8-bit, range 0 to +64)
0154       float in_val = 64.0 * ii / float(N_TABLE);
0155       // Next, compute lookup table function
0156       if (in_val > 0.0)
0157         table_out[ii] = 1.0 / in_val;
0158       else
0159         table_out[ii] = 0.0;
0160     }
0161   }
0162
0163   template <class data_T, class res_T, typename CONFIG_T>
0164   void softmax(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0165     // Initialize the lookup table
0166     typename CONFIG_T::table_t exp_table[CONFIG_T::table_size];
0167     init_exp_table<CONFIG_T, CONFIG_T::table_size>(exp_table);
0168
0169     typename CONFIG_T::table_t invert_table[CONFIG_T::table_size];
0170     init_invert_table<CONFIG_T, CONFIG_T::table_size>(invert_table);
0171
0172     // Index into the lookup table based on data for exponentials
0173     typename CONFIG_T::table_t exp_res[CONFIG_T::n_in];  // different, independent, fixed point precision
0174     typename CONFIG_T::table_t exp_diff_res[CONFIG_T::n_in]
0175                                            [CONFIG_T::n_in];  // different, independent, fixed point precision
0176     int data_round;
0177     int index;
0178     for (int ii = 0; ii < CONFIG_T::n_in; ii++) {
0179       exp_res[ii] = 0;
0180     }
0181     for (int ii = 0; ii < CONFIG_T::n_in; ii++) {
0182       for (int jj = 0; jj < CONFIG_T::n_in; jj++) {
0183         if (ii == jj)
0184           exp_diff_res[ii][jj] = 1;
0185         else {
0186           data_round = (data[jj] - data[ii]) * CONFIG_T::table_size / 16;
0187           index = data_round + 8 * CONFIG_T::table_size / 16;
0188           if (index < 0)
0189             index = 0;
0190           if (index > CONFIG_T::table_size - 1)
0191             index = CONFIG_T::table_size - 1;
0192           exp_diff_res[ii][jj] = exp_table[index];
0193         }
0194         exp_res[ii] += exp_diff_res[ii][jj];
0195       }
0196     }
0197
0198     //Second loop to invert
0199     for (unsigned ii = 0; ii < CONFIG_T::n_in; ii++) {
0200       int exp_res_index = exp_res[ii] * CONFIG_T::table_size / 64;
0201       if (exp_res_index < 0)
0202         exp_res_index = 0;
0203       if (exp_res_index > CONFIG_T::table_size - 1)
0204         exp_res_index = CONFIG_T::table_size - 1;
0205       //typename CONFIG_T::table_t exp_res_invert = invert_table[exp_res_index];
0206       res[ii] = (res_T)invert_table[exp_res_index];
0207     }
0208   }
0209
0210   // *************************************************
0211   //       TanH Activation
0212   // *************************************************
0213   template <typename CONFIG_T, int N_TABLE>
0214   void init_tanh_table(typename CONFIG_T::table_t table_out[N_TABLE]) {
0215     // Implement tanh lookup
0216     for (unsigned ii = 0; ii < N_TABLE; ii++) {
0217       // First, convert from table index to X-value (signed 8-bit, range -4 to +4)
0218       float in_val = 2 * 4.0 * (ii - float(N_TABLE) / 2.0) / float(N_TABLE);
0219       // Next, compute lookup table function
0220       typename CONFIG_T::table_t real_val = tanh(in_val);
0221       //std::cout << "Tanh:  Lookup table Index: " <<  ii<< " In Value: " << in_val << " Result: " << real_val << std::endl;
0222       table_out[ii] = real_val;
0223     }
0224   }
0225
0226   template <class data_T, class res_T, typename CONFIG_T>
0227   void tanh(data_T data[CONFIG_T::n_in], res_T res[CONFIG_T::n_in]) {
0228     // Initialize the lookup table
0229     typename CONFIG_T::table_t tanh_table[CONFIG_T::table_size];
0230     init_tanh_table<CONFIG_T, CONFIG_T::table_size>(tanh_table);
0231
0232     // Index into the lookup table based on data
0233     int data_round;
0234     int index;
0235     for (int ii = 0; ii < CONFIG_T::n_in; ii++) {
0236       data_round = data[ii] * CONFIG_T::table_size / 8;
0237       index = data_round + 4 * CONFIG_T::table_size / 8;
0238       //std::cout << "Input: "  << data[ii] << " Round: " << data_round << " Index: " << index << std::endl;
0239       if (index < 0)
0240         index = 0;
0241       if (index > CONFIG_T::table_size - 1)
0242         index = CONFIG_T::table_size - 1;
0243       res[ii] = (res_T)tanh_table[index];
0244     }
0245   }
0246
0247 }  // namespace nnet
0248
0249 #endif