fix failing tests (#448)

* fix failing tests - Add ReadAfter helper for sequencing read ops - fix read dependency issues - fix momentum updates - update all expected.txt using python output * update tsh_version for travis
migueldeicaza · Oct 25, 2019 · b54bc4c · b54bc4c
1 parent fadde6c
commit b54bc4c
Show file tree

Hide file tree

Showing 15 changed files with 719 additions and 649 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,9 +1,15 @@
 language: csharp
 solution: TensorFlowSharp.sln
 script:
-  - tsh_version=1.4.0-pre1
+  - tsh_version=1.13.0
   - wget "https://www.nuget.org/api/v2/package/TensorFlowSharp/$tsh_version"
-  - tar xzvf $tsh_version native
+  - mkdir native
+  - tar xzvf $tsh_version runtimes/linux/native
+  - cp runtimes/linux/native/* native
+  - tar xzvf $tsh_version runtimes/win7-x64/native
+  - cp runtimes/win7-x64/native/* native
+  - tar xzvf $tsh_version runtimes/osx/native
+  - cp runtimes/osx/native/* native
   - msbuild /t:Restore $TRAVIS_BUILD_DIR/TensorFlowSharp.sln
   - cd $TRAVIS_BUILD_DIR/
   - msbuild /p:Configuration=Release TensorFlowSharp.sln

diff --git a/SampleTest/SampleTest.cs b/SampleTest/SampleTest.cs
@@ -341,7 +341,7 @@ void LinearRegression ()
                             var tensors = sesssion.GetRunner()
                                .AddInput(X, new TFTensor(train_x[j]))
                                .AddInput(Y, new TFTensor(train_y[j]))
-                               .AddTarget(updateOps).Fetch(sgd.Iterations.Read, cost, W.Read, b.Read, sgd.LearningRate.Read).Run();
+                               .AddTarget(updateOps).Fetch(sgd.Iterations.Read, cost, W.Read, b.Read, sgd.LearningRate).Run();
                             avgLoss += (double)tensors[1].GetValue();
                         }
                         var tensors2 = sesssion.GetRunner()

diff --git a/TensorFlowSharp/Optimizer.cs b/TensorFlowSharp/Optimizer.cs
@@ -22,7 +22,7 @@ public abstract class Optimizer
         /// <summary>
         /// Variable to keep track of the learning rate.
         /// </summary>
-        public Variable LearningRate { get; }
+        public TFOutput LearningRate { get; }
 
         /// <summary>
         /// The graph object. It is used for creating Ops through the construction of optimizer.
@@ -60,32 +60,39 @@ public Optimizer(TFGraph graph, string operName, float learningRate, float decay
             using (var scope = _graph.WithScope(_optimizerName))
             {
                 Iterations = _graph.Variable(_graph.Const(new TFTensor(0L)), trainable: false, operName: "iterations");
-                _updateOps.Add(_graph.AssignAddVariableOp(Iterations, _graph.Const(1L)));
                 var initialLearningRate = _graph.Const(learningRate);
-                LearningRate = _graph.Variable(initialLearningRate, trainable: false, operName: _lrName);
-                CreateDecayOps(decay, initialLearningRate);
+                var inc = _graph.AssignAddVariableOp(Iterations, _graph.Const(1L));
+                _updateOps.Add(inc);
+                using (_graph.WithDependencies(inc))
+                {
+                    LearningRate = CreateDecayOps(decay, initialLearningRate);
+                }
             }
         }
 
         /// <summary>
         /// Create learning rate time decay operation.
         /// </summary>
-        protected void CreateDecayOps(float decay, TFOutput initialLearningRate)
+        protected TFOutput CreateDecayOps(float decay, TFOutput initialLearningRate)
         {
             if (decay > 0)
             {
                 var _decay = _graph.Const(decay, "Decay");
                 var one = _graph.Const(1f);
-                _updateOps.Add(_graph.AssignVariableOp(LearningRate,
+                return
                     _graph.Mul(initialLearningRate,
                                 _graph.Div(one,
                                             _graph.Add(one,
                                                         _graph.Mul(_decay,
-                                                                    _graph.Cast(Iterations.Read, _decay.OutputType)
+                                                                    _graph.Cast(_graph.Sub(Iterations.ReadAfter(_graph.CurrentDependencies), _graph.Const(1L)), _decay.OutputType)
                                                                   )
                                                        )
-                                           )
-                               )));
+                                           ), operName:"learningrate"
+                               );
+            }
+            else
+            {
+                return initialLearningRate;
             }
         }
 
@@ -193,23 +200,23 @@ public override TFOperation[] ApplyGradient((TFOutput gradient, Variable variabl
             for (int i = 0; i < gradientsAndVariables.Length; i++)
             {
                 var gv = gradientsAndVariables[i];
-                var lr = _graph.Cast(LearningRate.Read, gv.gradient.OutputType);
+                var lr = _graph.Cast(LearningRate, gv.gradient.OutputType);
                 var m = _graph.Cast(_momentum, gv.gradient.OutputType);
                 // v = m * moment - lr * g
-                var velocity = _graph.Sub(_graph.Mul(m, moments[i]), _graph.Mul(lr, gv.gradient));
+                var velocity = _graph.Sub(_graph.Mul(m, moments[i]), gv.gradient);
                 // moment = v
                 _updateOps.Add(_graph.Assign(moments[i], velocity).Operation);
 
                 if (_nesterov)
                 {
                     // w = w + m * v - lr * g
-                    var op = _graph.AssignAddVariableOp(gv.variable, _graph.Sub(_graph.Mul(m, velocity), _graph.Mul(lr, gv.gradient)));
+                    var op = _graph.AssignAddVariableOp(gv.variable, _graph.Mul(lr, _graph.Sub(_graph.Mul(m, velocity), gv.gradient)));
                     _updateOps.Add(op);
                 }
                 else
                 {
                     // w = w + v
-                    _updateOps.Add(_graph.AssignAddVariableOp(gv.variable, velocity));
+                    _updateOps.Add(_graph.AssignAddVariableOp(gv.variable, _graph.Mul(lr, velocity)));
                 }
             }
             return _updateOps.ToArray();
@@ -266,7 +273,7 @@ public override TFOperation[] ApplyGradient((TFOutput gradient, Variable variabl
             for (int i = 0; i < gradientsAndVariables.Length; i++)
             {
                 var gv = gradientsAndVariables[i];
-                var lr = _graph.Cast(LearningRate.Read, gv.gradient.OutputType);
+                var lr = _graph.Cast(LearningRate, gv.gradient.OutputType);
 
                 // accum = g ** 2;
                 var accum = _graph.Add(accumulators[i], _graph.Square(gv.gradient));
@@ -311,7 +318,7 @@ public override TFOperation[] ApplyGradient((TFOutput gradient, Variable variabl
             for (int i = 0; i < gradientsAndVariables.Length; i++)
             {
                 var gv = gradientsAndVariables[i];
-                var lr = _graph.Cast(LearningRate.Read, gv.gradient.OutputType);
+                var lr = _graph.Cast(LearningRate, gv.gradient.OutputType);
 
                 // accum = beta * accum + (1 - beta) * g ** 2;
                 var first = _graph.Mul(_beta, accumulators[i]);

diff --git a/TensorFlowSharp/Variable.cs b/TensorFlowSharp/Variable.cs
@@ -53,11 +53,31 @@ public class Variable
 		/// <value>The read op.</value>
 		public TFOutput Read => readHandle;
 
-		/// <summary>
-		/// Returns the AssignVariableOp that is used to assign the initial value to the variable from the graph.
-		/// </summary>
-		/// <value>The assign op.</value>
-		public TFOperation Assign => assignOp;
+        /// <summary>
+        /// Returns the ReadVariableOp that is used to fetch the value of the variable from the graph.
+        /// </summary>
+        /// <value>The read op.</value>
+        public TFOutput ReadAfter(params TFOperation[] dependencies)
+        {
+            if(dependencies.Length > 0)
+            {
+                var graph = dependencies[0].graph;
+                using (graph.WithDependencies(dependencies))
+                {
+                    return graph.ReadVariableOp(variableHandle, readHandle.OutputType);
+                }
+            }
+            else
+            {
+                return readHandle;
+            }
+        }
+
+        /// <summary>
+        /// Returns the AssignVariableOp that is used to assign the initial value to the variable from the graph.
+        /// </summary>
+        /// <value>The assign op.</value>
+        public TFOperation Assign => assignOp;
 
 		/// <summary>
 		/// Returns the VarHandleOp that was created using the shape of the initial value.