Refine the keras.Activation and add tf.keras.activations.

Author: AsakusaRinne

src/TensorFlowNET.Core/Keras/Activations/Activations.cs

@@ -1,4 +1,44 @@
-namespace Tensorflow.Keras
+using Newtonsoft.Json;
+using System.Reflection;
+using System.Runtime.Versioning;
+using Tensorflow.Keras.Common;
+
+namespace Tensorflow.Keras
 {
-    public delegate Tensor Activation(Tensor features, string name = null);
+    [JsonConverter(typeof(CustomizedActivationJsonConverter))]
+    public class Activation
+    {
+        public string Name { get; set; }
+        /// <summary>
+        /// The parameters are `features` and `name`.
+        /// </summary>
+        public Func<Tensor, string, Tensor> ActivationFunction { get; set; }
+
+        public Tensor Apply(Tensor input, string name = null) => ActivationFunction(input, name);
+
+        public static implicit operator Activation(Func<Tensor, string, Tensor> func)
+        {
+            return new Activation()
+            {
+                Name = func.GetMethodInfo().Name,
+                ActivationFunction = func
+            };
+        }
+    }
+
+    public interface IActivationsApi
+    {
+        Activation GetActivationFromName(string name);
+        Activation Linear { get; }
+
+        Activation Relu { get; }
+
+        Activation Sigmoid { get; }
+
+        Activation Softmax { get; }
+
+        Activation Tanh { get; }
+
+        Activation Mish { get; }
+    }
 }

src/TensorFlowNET.Core/Keras/ArgsDefinition/Convolution/ConvolutionalArgs.cs

@@ -26,27 +26,8 @@ public class ConvolutionalArgs : AutoSerializeLayerArgs
         public Shape DilationRate { get; set; } = (1, 1);
         [JsonProperty("groups")]
         public int Groups { get; set; } = 1;
-        public Activation Activation { get; set; }
-        private string _activationName;
         [JsonProperty("activation")]
-        public string ActivationName
-        {
-            get
-            {
-                if (string.IsNullOrEmpty(_activationName))
-                {
-                    return Activation.Method.Name;
-                }
-                else
-                {
-                    return _activationName;
-                }
-            }
-            set
-            {
-                _activationName = value;
-            }
-        }
+        public Activation Activation { get; set; }
         [JsonProperty("use_bias")]
         public bool UseBias { get; set; }
         [JsonProperty("kernel_initializer")]

src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/DenseArgs.cs

@@ -18,28 +18,8 @@ public class DenseArgs : LayerArgs
         /// <summary>
         /// Activation function to use.
         /// </summary>
-        public Activation Activation { get; set; }
-
-        private string _activationName;
         [JsonProperty("activation")]
-        public string ActivationName
-        {
-            get
-            {
-                if (string.IsNullOrEmpty(_activationName))
-                {
-                    return Activation.Method.Name;
-                }
-                else
-                {
-                    return _activationName;
-                }
-            }
-            set
-            {
-                _activationName = value;
-            }
-        }
+        public Activation Activation { get; set; }
 
         /// <summary>
         /// Whether the layer uses a bias vector.

src/TensorFlowNET.Core/Keras/ArgsDefinition/Core/EinsumDenseArgs.cs

@@ -35,27 +35,8 @@ public class EinsumDenseArgs : AutoSerializeLayerArgs
         /// <summary>
         /// Activation function to use.
         /// </summary>
-        public Activation Activation { get; set; }
-        private string _activationName;
         [JsonProperty("activation")]
-        public string ActivationName
-        {
-            get
-            {
-                if (string.IsNullOrEmpty(_activationName))
-                {
-                    return Activation.Method.Name;
-                }
-                else
-                {
-                    return _activationName;
-                }
-            }
-            set
-            {
-                _activationName = value;
-            }
-        }
+        public Activation Activation { get; set; }
 
         /// <summary>
         /// Initializer for the `kernel` weights matrix.

src/TensorFlowNET.Core/Keras/Common/CustomizedActivationJsonConverter.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Text;
+using static Tensorflow.Binding;
 
 namespace Tensorflow.Keras.Common
 {
@@ -31,20 +32,19 @@ public override void WriteJson(JsonWriter writer, object? value, JsonSerializer
             }
             else
             {
-                var token = JToken.FromObject((value as Activation)!.GetType().Name);
+                var token = JToken.FromObject(((Activation)value).Name);
                 token.WriteTo(writer);
             }
         }
 
         public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer)
         {
-            throw new NotImplementedException();
-            //var dims = serializer.Deserialize(reader, typeof(string));
-            //if (dims is null)
-            //{
-            //    throw new ValueError("Cannot deserialize 'null' to `Activation`.");
-            //}
-            //return new Shape((long[])(dims!));
+            var activationName = serializer.Deserialize<string>(reader);
+            if (tf.keras is null)
+            {
+                throw new RuntimeError("Tensorflow.Keras is not loaded, please install it first.");
+            }
+            return tf.keras.activations.GetActivationFromName(string.IsNullOrEmpty(activationName) ? "linear" : activationName);
         }
     }
 }

src/TensorFlowNET.Core/Keras/IKerasApi.cs

@@ -16,6 +16,7 @@ public interface IKerasApi
         IInitializersApi initializers { get; }
         ILayersApi layers { get; }
         ILossesApi losses { get; }
+        IActivationsApi activations { get; }
         IOptimizerApi optimizers { get; }
         IMetricsApi metrics { get; }
         IModelsApi models { get; }

src/TensorFlowNET.Keras/Activations.cs

@@ -6,45 +6,61 @@
 
 namespace Tensorflow.Keras
 {
-    public class Activations
+    public class Activations: IActivationsApi
     {
         private static Dictionary<string, Activation> _nameActivationMap;
-        private static Dictionary<Activation, string> _activationNameMap;
 
-        private static Activation _linear = (features, name) => features;
-        private static Activation _relu = (features, name)
-            => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features));
-        private static Activation _sigmoid = (features, name)
-            => tf.Context.ExecuteOp("Sigmoid", name, new ExecuteOpArgs(features));
-        private static Activation _softmax = (features, name)
-                => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features));
-        private static Activation _tanh = (features, name)
-            => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features));
-        private static Activation _mish = (features, name)
-            => features * tf.math.tanh(tf.math.softplus(features));
+        private static Activation _linear = new Activation()
+        {
+            Name = "linear",
+            ActivationFunction = (features, name) => features
+        };
+        private static Activation _relu = new Activation()
+        {
+            Name = "relu",
+            ActivationFunction = (features, name) => tf.Context.ExecuteOp("Relu", name, new ExecuteOpArgs(features))
+        };
+        private static Activation _sigmoid = new Activation()
+        {
+            Name = "sigmoid",
+            ActivationFunction = (features, name) => tf.Context.ExecuteOp("Sigmoid", name, new ExecuteOpArgs(features))
+        };
+        private static Activation _softmax = new Activation()
+        {
+            Name = "softmax",
+            ActivationFunction = (features, name) => tf.Context.ExecuteOp("Softmax", name, new ExecuteOpArgs(features))
+        };
+        private static Activation _tanh = new Activation()
+        {
+            Name = "tanh",
+            ActivationFunction = (features, name) => tf.Context.ExecuteOp("Tanh", name, new ExecuteOpArgs(features))
+        };
+        private static Activation _mish = new Activation()
+        {
+            Name = "mish",
+            ActivationFunction = (features, name) => features * tf.math.tanh(tf.math.softplus(features))
+        };
 
         /// <summary>
         /// Register the name-activation mapping in this static class.
         /// </summary>
         /// <param name="name"></param>
         /// <param name="activation"></param>
-        private static void RegisterActivation(string name, Activation activation)
+        private static void RegisterActivation(Activation activation)
         {
-            _nameActivationMap[name] = activation;
-            _activationNameMap[activation] = name;
+            _nameActivationMap[activation.Name] = activation;
         }
 
         static Activations()
         {
             _nameActivationMap = new Dictionary<string, Activation>();
-            _activationNameMap= new Dictionary<Activation, string>();
 
-            RegisterActivation("relu", _relu);
-            RegisterActivation("linear", _linear);
-            RegisterActivation("sigmoid", _sigmoid);
-            RegisterActivation("softmax", _softmax);
-            RegisterActivation("tanh", _tanh);
-            RegisterActivation("mish", _mish);
+            RegisterActivation(_relu);
+            RegisterActivation(_linear);
+            RegisterActivation(_sigmoid);
+            RegisterActivation(_softmax);
+            RegisterActivation(_tanh);
+            RegisterActivation(_mish);
         }
 
         public Activation Linear => _linear;
@@ -59,7 +75,7 @@ static Activations()
 
         public Activation Mish => _mish;
 
-        public static Activation GetActivationByName(string name)
+        public Activation GetActivationFromName(string name)
         {
             if (!_nameActivationMap.TryGetValue(name, out var res))
             {
@@ -70,17 +86,5 @@ public static Activation GetActivationByName(string name)
                 return res;
             }
         }
-
-        public static string GetNameByActivation(Activation activation)
-        {
-            if(!_activationNameMap.TryGetValue(activation, out var name))
-            {
-                throw new Exception($"Activation {activation} not found");
-            }
-            else
-            {
-                return name;
-            }
-        }
     }
 }

src/TensorFlowNET.Keras/KerasInterface.cs

@@ -45,7 +45,7 @@ public static KerasInterface Instance
         public Regularizers regularizers { get; } = new Regularizers();
         public ILayersApi layers { get; } = new LayersApi();
         public ILossesApi losses { get; } = new LossesApi();
-        public Activations activations { get; } = new Activations();
+        public IActivationsApi activations { get; } = new Activations();
         public Preprocessing preprocessing { get; } = new Preprocessing();
         ThreadLocal<BackendImpl> _backend = new ThreadLocal<BackendImpl>(() => new BackendImpl());
         public BackendImpl backend => _backend.Value;

src/TensorFlowNET.Keras/Layers/Convolution/Conv2DTranspose.cs

@@ -110,7 +110,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train
                 throw new NotImplementedException("");
 
             if (activation != null)
-                return activation(outputs);
+                return activation.Apply(outputs);
 
             return outputs;
         }

src/TensorFlowNET.Keras/Layers/Convolution/Convolutional.cs

@@ -117,7 +117,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train
             }
 
             if (activation != null)
-                outputs = activation(outputs);
+                outputs = activation.Apply(outputs);
 
             return outputs;
         }

src/TensorFlowNET.Keras/Layers/Core/Dense.cs

@@ -81,7 +81,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train
             if (args.UseBias)
                 outputs = tf.nn.bias_add(outputs, bias);
             if (args.Activation != null)
-                outputs = activation(outputs);
+                outputs = activation.Apply(outputs);
 
             return outputs;
         }

src/TensorFlowNET.Keras/Layers/Core/EinsumDense.cs

@@ -193,7 +193,7 @@ protected override Tensors Call(Tensors inputs, Tensor state = null, bool? train
             if (this.bias != null)
                 ret += this.bias.AsTensor();
             if (this.activation != null)
-                ret = this.activation(ret);
+                ret = this.activation.Apply(ret);
             return ret;
         }
         /// <summary>