Incompatible shapes: [4,20,20,3] vs. [4,19,19,3]

[mfrohman123]

When training on custom dataset using .ipynb file. Custom dataset uses roughly 7,000 images of size 300*300:

InvalidArgumentError Traceback (most recent call last)
/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1326 try:
-> 1327 return fn(*args)
1328 except errors.OpError as e:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
1311 return self._call_tf_sessionrun(
-> 1312 options, feed_dict, fetch_list, target_list, run_metadata)
1313

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
1419 self._session, options, feed_dict, fetch_list, target_list,
-> 1420 status, run_metadata)
1421

/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in exit(self, type_arg, value_arg, traceback_arg)
515 compat.as_text(c_api.TF_Message(self.status.status)),
--> 516 c_api.TF_GetCode(self.status.status))
517 # Delete the underlying status object from memory otherwise it stays alive

InvalidArgumentError: Incompatible shapes: [4,20,20,3] vs. [4,19,19,3]
[[Node: optimizer/gradients/decoder/add_fc7_pool4_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](optimizer/gradients/decoder/add_fc7_pool4_grad/Shape, optimizer/gradients/decoder/add_fc7_pool4_grad/Shape_1)]]

During handling of the above exception, another exception occurred:

InvalidArgumentError Traceback (most recent call last)
in ()
31 summaries_dir='tensorboard_log/cityscapes',
32 summaries_name='configuration_01',
---> 33 training_loss_display_averaging=3)

~/Desktop/ai skin cancer/AI algorithms/fcn8s_tensorflow-master/fcn8s_tensorflow.py in train(self, train_generator, epochs, steps_per_epoch, learning_rate_schedule, keep_prob, l2_regularization, eval_dataset, eval_frequency, val_generator, val_steps, metrics, save_during_training, save_dir, save_best_only, save_tags, save_name, save_frequency, saver, monitor, record_summaries, summaries_frequency, summaries_dir, summaries_name, training_loss_display_averaging)
560 self.learning_rate: learning_rate,
561 self.keep_prob: keep_prob,
--> 562 self.l2_regularization_rate: l2_regularization})
563 training_writer.add_summary(summary=training_summary, global_step=self.g_step)
564 else:

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
903 try:
904 result = self._run(None, fetches, feed_dict, options_ptr,
--> 905 run_metadata_ptr)
906 if run_metadata:
907 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
1138 if final_fetches or final_targets or (handle and feed_dict_tensor):
1139 results = self._do_run(handle, final_targets, final_fetches,
-> 1140 feed_dict_tensor, options, run_metadata)
1141 else:
1142 results = []

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
1319 if handle is None:
1320 return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1321 run_metadata)
1322 else:
1323 return self._do_call(_prun_fn, handle, feeds, fetches)

/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
1338 except KeyError:
1339 pass
-> 1340 raise type(e)(node_def, op, message)
1341
1342 def _extend_graph(self):

InvalidArgumentError: Incompatible shapes: [4,20,20,3] vs. [4,19,19,3]
[[Node: optimizer/gradients/decoder/add_fc7_pool4_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](optimizer/gradients/decoder/add_fc7_pool4_grad/Shape, optimizer/gradients/decoder/add_fc7_pool4_grad/Shape_1)]]

Caused by op 'optimizer/gradients/decoder/add_fc7_pool4_grad/BroadcastGradientArgs', defined at:
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in
app.launch_new_instance()
File "/usr/local/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
app.start()
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 486, in start
self.io_loop.start()
File "/usr/local/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 127, in start
self.asyncio_loop.run_forever()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 422, in run_forever
self._run_once()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/base_events.py", line 1432, in _run_once
handle._run()
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/asyncio/events.py", line 145, in _run
self._callback(*self._args)
File "/usr/local/lib/python3.6/site-packages/tornado/platform/asyncio.py", line 117, in _handle_events
handler_func(fileobj, events)
File "/usr/local/lib/python3.6/site-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/tornado/stack_context.py", line 276, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2662, in run_cell
raw_cell, store_history, silent, shell_futures)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2785, in _run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2903, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2963, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "", line 5, in
variables_load_dir=None)
File "/Users/mattfrohman/Desktop/ai skin cancer/AI algorithms/fcn8s_tensorflow-master/fcn8s_tensorflow.py", line 111, in init
self.total_loss, self.train_op, self.learning_rate, self.global_step = self._build_optimizer()
File "/Users/mattfrohman/Desktop/ai skin cancer/AI algorithms/fcn8s_tensorflow-master/fcn8s_tensorflow.py", line 257, in _build_optimizer
train_op = optimizer.minimize(total_loss, global_step=global_step, name='train_op')
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/training/optimizer.py", line 399, in minimize
grad_loss=grad_loss)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/training/optimizer.py", line 492, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 488, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 625, in _GradientsHelper
lambda: grad_fn(op, *out_grads))
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 379, in _MaybeCompile
return grad_fn() # Exit early
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 625, in
lambda: grad_fn(op, *out_grads))
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py", line 845, in _AddGrad
rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py", line 671, in _broadcast_gradient_args
"BroadcastGradientArgs", s0=s0, s1=s1, name=name)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1654, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

...which was originally created as op 'decoder/add_fc7_pool4', defined at:
File "/usr/local/Cellar/python/3.6.5/Frameworks/Python.framework/Versions/3.6/lib/python3.6/runpy.py", line 193, in _run_module_as_main
"main", mod_spec)
[elided 23 identical lines from previous traceback]
File "", line 5, in
variables_load_dir=None)
File "/Users/mattfrohman/Desktop/ai skin cancer/AI algorithms/fcn8s_tensorflow-master/fcn8s_tensorflow.py", line 108, in init
self.fcn8s_output, self.l2_regularization_rate = self._build_decoder()
File "/Users/mattfrohman/Desktop/ai skin cancer/AI algorithms/fcn8s_tensorflow-master/fcn8s_tensorflow.py", line 213, in _build_decoder
add_fc7_pool4 = tf.add(fc7_conv2d_trans, pool4_1x1, name='add_fc7_pool4')
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_math_ops.py", line 296, in add
"Add", x=x, y=y, name=name)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 3290, in create_op
op_def=op_def)
File "/usr/local/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1654, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): Incompatible shapes: [4,20,20,3] vs. [4,19,19,3]
[[Node: optimizer/gradients/decoder/add_fc7_pool4_grad/BroadcastGradientArgs = BroadcastGradientArgs[T=DT_INT32, _device="/job:localhost/replica:0/task:0/device:CPU:0"](optimizer/gradients/decoder/add_fc7_pool4_grad/Shape, optimizer/gradients/decoder/add_fc7_pool4_grad/Shape_1)]]