From aa4c66ce6f0dbbd3fe952b18eac8564895495d57 Mon Sep 17 00:00:00 2001
From: Tero Keski-Valkama <tero.keski-valkama@cybercom.com>
Date: Fri, 7 Jul 2017 22:31:26 +0300
Subject: [PATCH] Removed stop_gradient operations from within the loop, as
 they create new operations into the model which are never cleaned up.

---
 model.py |  6 +++---
 train.py | 15 ++++++++++++---
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/model.py b/model.py
index 0dafa68..dcac353 100755
--- a/model.py
+++ b/model.py
@@ -187,12 +187,12 @@ def create(parameters):
     train_op = optimizer.apply_gradients(zip(grads, tvars))
 
     model = {
-        'output': output,
+        'output': tf.stop_gradient(output),
         'optimizer': train_op,
         'input': input,
         'target_output': target_output,
-        'cost': cost,
-        'reg_loss': reg_loss,
+        'cost': tf.stop_gradient(cost),
+        'reg_loss': tf.stop_gradient(reg_loss),
         'schedule_step': schedule_step,
         'input_noise': input_noise,
         'noise': noise
diff --git a/train.py b/train.py
index 58578a7..1cb1eb8 100755
--- a/train.py
+++ b/train.py
@@ -25,6 +25,14 @@
 
 import re
 
+def memory():
+    import os
+    import psutil
+    pid = os.getpid()
+    py = psutil.Process(pid)
+    memoryUse = py.memory_info()[0]/2.**30  # memory use in GB...I think
+    print('memory use:', memoryUse)
+
 def make_x_and_y(x, noise, amplitude_plusminus_factor):
     # Length of shifted x and y
     length = np.size(x, 0) - 1
@@ -92,7 +100,7 @@ def choose_value(sample):
                                               parameters['amplitude_plusminus_factor'])
 
             # Create first estimation and optimize for the first order.
-            [_, cost, reg_loss] = sess.run([model['optimizer'], tf.stop_gradient(model['cost']), tf.stop_gradient(model['reg_loss'])], feed_dict = {
+            [_, cost, reg_loss] = sess.run([model['optimizer'], model['cost'], model['reg_loss']], feed_dict = {
                 model['input']: x,
                 model['schedule_step']: iter,
                 model['noise']: parameters['noise'],
@@ -112,7 +120,7 @@ def choose_value(sample):
                         ", iters_since_loss_improved: ", iters_since_loss_improved
                 saver.save(sess, 'sound-model')
                 
-                [error, output] = sess.run([tf.stop_gradient(model['cost']), tf.stop_gradient(model['output'])], feed_dict = {
+                [error, output] = sess.run([model['cost'], model['output']], feed_dict = {
                     model['input']: x,
                     model['schedule_step']: iter,
                     model['noise']: parameters['noise'],
@@ -134,7 +142,7 @@ def choose_value(sample):
                 test_x = manage_data.getNextTrainingBatchSequence(testingData, training_length)
                 (original_test_x, test_x, test_y) = make_x_and_y(test_x, 0.0, 0.0)
 
-                [test_error, test_output] = sess.run([tf.stop_gradient(model['cost']), tf.stop_gradient(model['output'])],
+                [test_error, test_output] = sess.run([model['cost'], model['output']],
                     feed_dict={
                                model['input']: test_x,
                                model['schedule_step']: iter,
@@ -171,6 +179,7 @@ def choose_value(sample):
                 ##else:
                 ##    export_to_octave.save('train_error.mat', 'train_error', train_error_trend)
                 ##    export_to_octave.save('test_error.mat', 'test_error', test_error_trend)
+            memory()
             sys.stdout.flush()
             iter += 1
             now = time.time()