Fix (#1036) and adjust the keras unittest

src/TensorFlowNET.Core/Operations/math_ops.cs

@@ -905,13 +905,29 @@ public static Tensor tensordot(Tensor a, Tensor b, NDArray axes, string name = n
                 var (a_reshape, a_free_dims, a_free_dims_static) = _tensordot_reshape(a, a_axes);
                 var (b_reshape, b_free_dims, b_free_dims_static) = _tensordot_reshape(b, b_axes, true);
                 var ab_matmul = matmul(a_reshape, b_reshape);
-                var dims = new List<int>();
-                dims.AddRange(a_free_dims);
-                dims.AddRange(b_free_dims);
-                if (ab_matmul.shape.Equals(dims))
-                    return ab_matmul;
+                if(a_free_dims is int[] a_free_dims_list && b_free_dims is int[] b_free_dims_list)
+                {
+                    var total_free_dims = a_free_dims_list.Concat(b_free_dims_list).ToArray();
+                    if (ab_matmul.shape.IsFullyDefined && ab_matmul.shape.as_int_list().SequenceEqual(total_free_dims))
+                    {
+                        return ab_matmul;
+                    }
+                    else
+                    {
+                        return array_ops.reshape(ab_matmul, ops.convert_to_tensor(total_free_dims), name);
+                    }
+                }
                 else
-                    return array_ops.reshape(ab_matmul, tf.constant(dims.ToArray()), name: name);
+                {
+                    var a_free_dims_tensor = ops.convert_to_tensor(a_free_dims, dtype: dtypes.int32);
+                    var b_free_dims_tensor = ops.convert_to_tensor(b_free_dims, dtype: dtypes.int32);
+                    var product = array_ops.reshape(ab_matmul, array_ops.concat(new[] { a_free_dims_tensor, b_free_dims_tensor }, 0), name);
+                    if(a_free_dims_static is not null && b_free_dims_static is not null)
+                    {
+                        product.shape = new Shape(a_free_dims_static.Concat(b_free_dims_static).ToArray());
+                    }
+                    return product;
+                }
             });
         }
 
@@ -927,14 +943,42 @@ public static Tensor tensordot(Tensor a, Tensor b, NDArray axes, string name = n
                 return (Binding.range(a.shape.ndim - axe, a.shape.ndim).ToArray(),
                     Binding.range(0, axe).ToArray());
             }
-            else
+            else if(axes.rank == 1)
             {
+                if (axes.shape[0] != 2)
+                {
+                    throw new ValueError($"`axes` must be an integer or have length 2. Received {axes}.");
+                }
                 (int a_axe, int b_axe) = (axes[0], axes[1]);
                 return (new[] { a_axe }, new[] { b_axe });
             }
+            else if(axes.rank == 2)
+            {
+                if (axes.shape[0] != 2)
+                {
+                    throw new ValueError($"`axes` must be an integer or have length 2. Received {axes}.");
+                }
+                int[] a_axes = new int[axes.shape[1]];
+                int[] b_axes = new int[axes.shape[1]];
+                for(int i = 0; i < a_axes.Length; i++)
+                {
+                    a_axes[i] = axes[0, i];
+                    b_axes[i] = axes[1, i];
+                    if (a_axes[i] == -1 || b_axes[i] == -1)
+                    {
+                        throw new ValueError($"Different number of contraction axes `a` and `b`," +
+                            $"{len(a_axes)} != {len(b_axes)}.");
+                    }
+                }
+                return (a_axes, b_axes);
+            }
+            else
+            {
+                throw new ValueError($"Invalid rank {axes.rank} to make tensor dot.");
+            }
         }
 
-        static (Tensor, int[], int[]) _tensordot_reshape(Tensor a, int[] axes, bool flipped = false)
+        static (Tensor, object, int[]) _tensordot_reshape(Tensor a, int[] axes, bool flipped = false)
         {
             if (a.shape.IsFullyDefined && isinstance(axes, (typeof(int[]), typeof(Tuple))))
             {
@@ -977,6 +1021,58 @@ public static Tensor tensordot(Tensor a, Tensor b, NDArray axes, string name = n
                 var reshaped_a = array_ops.reshape(a_trans, new_shape);
                 return (reshaped_a, free_dims, free_dims);
             }
+            else
+            {
+                int[] free_dims_static;
+                Tensor converted_shape_a, converted_axes, converted_free;
+                if (a.shape.ndim != -1)
+                {
+                    var shape_a = a.shape.as_int_list();
+                    for(int i = 0; i < axes.Length; i++)
+                    {
+                        if (axes[i] < 0)
+                        {
+                            axes[i] += shape_a.Length;
+                        }
+                    }
+                    var free = Enumerable.Range(0, shape_a.Length).Where(i => !axes.Contains(i)).ToArray();
+
+                    var axes_dims = axes.Select(i => shape_a[i]);
+                    var free_dims = free.Select(i => shape_a[i]).ToArray();
+                    free_dims_static = free_dims;
+                    converted_axes = ops.convert_to_tensor(axes, dtypes.int32, "axes");
+                    converted_free = ops.convert_to_tensor(free, dtypes.int32, "free");
+                    converted_shape_a = array_ops.shape(a);
+                }
+                else
+                {
+                    free_dims_static = null;
+                    converted_shape_a = array_ops.shape(a);
+                    var rank_a = array_ops.rank(a);
+                    converted_axes = ops.convert_to_tensor(axes, dtypes.int32, "axes");
+                    converted_axes = array_ops.where_v2(converted_axes >= 0, converted_axes, converted_axes + rank_a);
+                    (converted_free, var _) = gen_ops.list_diff(gen_math_ops.range(ops.convert_to_tensor(0), rank_a, ops.convert_to_tensor(1)), 
+                        converted_axes, dtypes.int32);
+                }
+                var converted_free_dims = array_ops.gather(converted_shape_a, converted_free);
+                var converted_axes_dims = array_ops.gather(converted_shape_a, converted_axes);
+                var prod_free_dims = reduce_prod(converted_free_dims);
+                var prod_axes_dims = reduce_prod(converted_axes_dims);
+                Tensor reshaped_a;
+                if (flipped)
+                {
+                    var perm = array_ops.concat(new[] { converted_axes, converted_free }, 0);
+                    var new_shape = array_ops.stack(new[] { prod_axes_dims, prod_free_dims });
+                    reshaped_a = array_ops.reshape(array_ops.transpose(a, perm), new_shape);
+                }
+                else
+                {
+                    var perm = array_ops.concat(new[] { converted_free, converted_axes }, 0);
+                    var new_shape = array_ops.stack(new[] { prod_free_dims, prod_axes_dims });
+                    reshaped_a = array_ops.reshape(array_ops.transpose(a, perm), new_shape);
+                }
+                return (reshaped_a, converted_free_dims, free_dims_static);
+            }
 
             throw new NotImplementedException("_tensordot_reshape");
         }

test/TensorFlowNET.Keras.UnitTest/Callbacks/EarlystoppingTest.cs

@@ -1,16 +1,11 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Tensorflow.Keras.UnitTest.Helpers;
-using static Tensorflow.Binding;
-using Tensorflow;
-using Tensorflow.Keras.Optimizers;
+using System.Collections.Generic;
 using Tensorflow.Keras.Callbacks;
 using Tensorflow.Keras.Engine;
-using System.Collections.Generic;
 using static Tensorflow.KerasApi;
-using Tensorflow.Keras;
 
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest.Callbacks
 {
     [TestClass]
     public class EarlystoppingTest
@@ -31,7 +26,7 @@ public void Earlystopping()
                 layers.Dense(10)
             });
 
-            
+
             model.summary();
 
             model.compile(optimizer: keras.optimizers.RMSprop(1e-3f),
@@ -55,7 +50,7 @@ public void Earlystopping()
             var callbacks = new List<ICallback>();
             callbacks.add(earlystop);
 
-            model.fit(x_train[new Slice(0, 2000)], y_train[new Slice(0, 2000)], batch_size, num_epochs,callbacks:callbacks);
+            model.fit(x_train[new Slice(0, 2000)], y_train[new Slice(0, 2000)], batch_size, num_epochs, callbacks: callbacks);
         }
 
     }

test/TensorFlowNET.Keras.UnitTest/EagerModeTestBase.cs

@@ -1,10 +1,8 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System;
-using Tensorflow;
-using Tensorflow.Keras;
 using static Tensorflow.Binding;
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest
 {
     public class EagerModeTestBase
     {

test/TensorFlowNET.Keras.UnitTest/GradientTest.cs

@@ -1,14 +1,11 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using System.Linq;
-using Tensorflow;
 using Tensorflow.Keras.Engine;
+using Tensorflow.NumPy;
 using static Tensorflow.Binding;
 using static Tensorflow.KerasApi;
-using Tensorflow.NumPy;
-using System;
-using Tensorflow.Keras.Optimizers;
 
-namespace TensorFlowNET.Keras.UnitTest;
+namespace Tensorflow.Keras.UnitTest;
 
 [TestClass]
 public class GradientTest : EagerModeTestBase

test/TensorFlowNET.Keras.UnitTest/InitializerTest.cs

@@ -1,12 +1,7 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using TensorFlowNET.Keras.UnitTest;
 using static Tensorflow.Binding;
 
-namespace TensorFlowNET.Keras.UnitTest;
+namespace Tensorflow.Keras.UnitTest;
 
 [TestClass]
 public class InitializerTest : EagerModeTestBase

test/TensorFlowNET.Keras.UnitTest/Layers/ActivationTest.cs

@@ -1,12 +1,10 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-using System.Collections.Generic;
-using static Tensorflow.Binding;
 using Tensorflow.NumPy;
+using static Tensorflow.Binding;
 using static Tensorflow.KerasApi;
-using Tensorflow;
 
-namespace TensorFlowNET.Keras.UnitTest {
+namespace Tensorflow.Keras.UnitTest.Layers
+{
     [TestClass]
     public class ActivationTest : EagerModeTestBase
     {

test/TensorFlowNET.Keras.UnitTest/Layers/AttentionTest.cs

@@ -1,15 +1,11 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using System;
-using System.Collections.Generic;
+using Tensorflow.Keras.Layers;
+using Tensorflow.Keras.Utils;
 using Tensorflow.NumPy;
 using static Tensorflow.Binding;
 using static Tensorflow.KerasApi;
-using Tensorflow.Keras.Layers;
-using Tensorflow;
-using Tensorflow.Keras.ArgsDefinition;
-using Tensorflow.Keras.Utils;
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest.Layers
 {
     [TestClass]
     public class AttentionTest : EagerModeTestBase
@@ -118,7 +114,8 @@ public void test_calculate_scores_multi_dim_concat()
             } }, dtype: np.float32);
             var attention_layer = (Attention)keras.layers.Attention(score_mode: "concat");
             //attention_layer.concat_score_weight = 1;
-            attention_layer.concat_score_weight = base_layer_utils.make_variable(new VariableArgs() {
+            attention_layer.concat_score_weight = base_layer_utils.make_variable(new VariableArgs()
+            {
                 Name = "concat_score_weight",
                 Shape = (1),
                 DType = TF_DataType.TF_FLOAT,
@@ -156,7 +153,7 @@ public void test_masked_attention()
 
             var query = keras.Input(shape: (4, 8));
             var value = keras.Input(shape: (2, 8));
-            var mask_tensor = keras.Input(shape:(4, 2));
+            var mask_tensor = keras.Input(shape: (4, 2));
             var attention_layer = keras.layers.MultiHeadAttention(num_heads: 2, key_dim: 2);
             attention_layer.Apply(new Tensor[] { query, value, mask_tensor });
 

test/TensorFlowNET.Keras.UnitTest/Layers/CosineSimilarity.Test.cs

@@ -1,11 +1,9 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Tensorflow.NumPy;
-using Tensorflow;
 using Tensorflow.Keras.Losses;
-using static Tensorflow.Binding;
+using Tensorflow.NumPy;
 using static Tensorflow.KerasApi;
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest.Layers
 {
     [TestClass]
     public class CosineSimilarity
@@ -16,7 +14,7 @@ public class CosineSimilarity
         NDArray y_pred_float = new float[,] { { 1.0f, 0.0f }, { 1.0f, 1.0f } };
 
         [TestMethod]
-      
+
         public void _Default()
         {
             //>>> # Using 'auto'/'sum_over_batch_size' reduction type.  
@@ -27,7 +25,7 @@ public void _Default()
             //>>> # loss = mean(sum(l2_norm(y_true) . l2_norm(y_pred), axis=1))  
             //>>> #       = -((0. + 0.) +  (0.5 + 0.5)) / 2  
             //-0.5
-            var loss = keras.losses.CosineSimilarity(axis : 1);
+            var loss = keras.losses.CosineSimilarity(axis: 1);
             var call = loss.Call(y_true_float, y_pred_float);
             Assert.AreEqual((NDArray)(-0.49999997f), call.numpy());
         }
@@ -41,7 +39,7 @@ public void _Sample_Weight()
             //- 0.0999
             var loss = keras.losses.CosineSimilarity();
             var call = loss.Call(y_true_float, y_pred_float, sample_weight: (NDArray)new float[] { 0.8f, 0.2f });
-            Assert.AreEqual((NDArray) (- 0.099999994f), call.numpy());
+            Assert.AreEqual((NDArray)(-0.099999994f), call.numpy());
         }
 
         [TestMethod]
@@ -53,7 +51,7 @@ public void _SUM()
             //...     reduction = tf.keras.losses.Reduction.SUM)
             //>>> cosine_loss(y_true, y_pred).numpy()
             //- 0.999
-            var loss = keras.losses.CosineSimilarity(axis: 1,reduction : ReductionV2.SUM);
+            var loss = keras.losses.CosineSimilarity(axis: 1, reduction: ReductionV2.SUM);
             var call = loss.Call(y_true_float, y_pred_float);
             Assert.AreEqual((NDArray)(-0.99999994f), call.numpy());
         }
@@ -67,7 +65,7 @@ public void _None()
             //...     reduction = tf.keras.losses.Reduction.NONE)
             //>>> cosine_loss(y_true, y_pred).numpy()
             //array([-0., -0.999], dtype = float32)
-            var loss = keras.losses.CosineSimilarity(axis :1, reduction: ReductionV2.NONE);
+            var loss = keras.losses.CosineSimilarity(axis: 1, reduction: ReductionV2.NONE);
             var call = loss.Call(y_true_float, y_pred_float);
             Assert.AreEqual((NDArray)new float[] { -0f, -0.99999994f }, call.numpy());
         }

test/TensorFlowNET.Keras.UnitTest/Layers/Huber.Test.cs

@@ -1,11 +1,9 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
-using Tensorflow.NumPy;
-using Tensorflow;
 using Tensorflow.Keras.Losses;
-using static Tensorflow.Binding;
+using Tensorflow.NumPy;
 using static Tensorflow.KerasApi;
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest.Layers
 {
     [TestClass]
     public class Huber
@@ -16,7 +14,7 @@ public class Huber
         NDArray y_pred_float = new float[,] { { 0.6f, 0.4f }, { 0.4f, 0.6f } };
 
         [TestMethod]
-      
+
         public void _Default()
         {
             //>>> # Using 'auto'/'sum_over_batch_size' reduction type.  
@@ -49,7 +47,7 @@ public void _SUM()
             //...     reduction = tf.keras.losses.Reduction.SUM)
             //>>> h(y_true, y_pred).numpy()
             //0.31
-            var loss = keras.losses.Huber(reduction : ReductionV2.SUM);
+            var loss = keras.losses.Huber(reduction: ReductionV2.SUM);
             var call = loss.Call(y_true_float, y_pred_float);
             Assert.AreEqual((NDArray)0.31f, call.numpy());
         }

test/TensorFlowNET.Keras.UnitTest/Layers/Layers.Convolution.Test.cs

@@ -1,10 +1,8 @@
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 using Tensorflow.NumPy;
-using Tensorflow;
-using Tensorflow.Operations;
 using static Tensorflow.KerasApi;
 
-namespace TensorFlowNET.Keras.UnitTest
+namespace Tensorflow.Keras.UnitTest.Layers
 {
     [TestClass]
     public class LayersConvolutionTest : EagerModeTestBase
@@ -14,7 +12,7 @@ public void BasicConv1D()
         {
             var filters = 8;
 
-            var conv = keras.layers.Conv1D(filters, kernel_size: 3,  activation: "linear");
+            var conv = keras.layers.Conv1D(filters, kernel_size: 3, activation: "linear");
 
             var x = np.arange(256.0f).reshape((8, 8, 4));
             var y = conv.Apply(x);