void forward_activation_layer(layer l, network net) { copy_cpu(l.outputs*l.batch, net.input, 1, l.output, 1); activate_array(l.output, l.outputs*l.batch, l.activation); } void activate_array(float *x, const int n, const ACTIVATION a) { int i; for(i = 0; i < n; ++i){ x[i] = activate(x[i], a); } } float activate(float x, ACTIVATION a) { switch(a){ case LINEAR: return linear_activate(x); case LOGISTIC: return logistic_activate(x); case LOGGY: return loggy_activate(x); case RELU: return relu_activate(x); case ELU: return elu_activate(x); case SELU: return selu_activate(x); case RELIE: return relie_activate(x); case RAMP: return ramp_activate(x); case LEAKY: return leaky_activate(x); case TANH: return tanh_activate(x); case PLSE: return plse_activate(x); case STAIR: return stair_activate(x); case HARDTAN: return hardtan_activate(x); case LHTAN: return lhtan_activate(x); } return 0; } static inline float stair_activate(float x) { int n = floor(x); if (n%2 == 0) return floor(x/2.); else return (x - n) + floor(x/2.); } static inline float hardtan_activate(float x) { if (x < -1) return -1; if (x > 1) return 1; return x; } static inline float linear_activate(float x){return x;} static inline float logistic_activate(float x){return 1./(1. + exp(-x));} static inline float loggy_activate(float x){return 2./(1. + exp(-x)) - 1;} static inline float relu_activate(float x){return x*(x>0);} static inline float elu_activate(float x){return (x >= 0)*x + (x < 0)*(exp(x)-1);} static inline float selu_activate(float x){return (x >= 0)*1.0507*x + (x < 0)*1.0507*1.6732*(exp(x)-1);} static inline float relie_activate(float x){return (x>0) ? x : .01*x;} static inline float ramp_activate(float x){return x*(x>0)+.1*x;} static inline float leaky_activate(float x){return (x>0) ? x : .1*x;} static inline float tanh_activate(float x){return (exp(2*x)-1)/(exp(2*x)+1);} static inline float plse_activate(float x) { if(x < -4) return .01 * (x + 4); if(x > 4) return .01 * (x - 4) + 1; return .125*x + .5; } static inline float lhtan_activate(float x) { if(x < 0) return .001*x; if(x > 1) return .001*(x-1) + 1; return x; } static inline float lhtan_gradient(float x) { if(x > 0 && x < 1) return 1; return .001; } static inline float hardtan_gradient(float x) { if (x > -1 && x < 1) return 1; return 0; } static inline float linear_gradient(float x){return 1;} static inline float logistic_gradient(float x){return (1-x)*x;} static inline float loggy_gradient(float x) { float y = (x+1.)/2.; return 2*(1-y)*y; } static inline float stair_gradient(float x) { if (floor(x) == x) return 0; return 1; } static inline float relu_gradient(float x){return (x>0);} static inline float elu_gradient(float x){return (x >= 0) + (x < 0)*(x + 1);} static inline float selu_gradient(float x){return (x >= 0)*1.0507 + (x < 0)*(x + 1.0507*1.6732);} static inline float relie_gradient(float x){return (x>0) ? 1 : .01;} static inline float ramp_gradient(float x){return (x>0)+.1;} static inline float leaky_gradient(float x){return (x>0) ? 1 : .1;} static inline float tanh_gradient(float x){return 1-x*x;} static inline float plse_gradient(float x){return (x < 0 || x > 1) ? .01 : .125;} |
说点什么...