Training problem

[peterlee909]

I set the training_flag = True
but I got error like this:

2019-11-27 16:12:15.632003: W tensorflow/core/common_runtime/bfc_allocator.cc:424] *****************************************************************************_______________________
2019-11-27 16:12:15.649697: W tensorflow/core/framework/op_kernel.cc:1599] OP_REQUIRES failed at constant_op.cc:77 : Resource exhausted: OOM when allocating tensor of shape [7,7,512,4096] and type float
2019-11-27 16:12:15.676957: E tensorflow/core/common_runtime/executor.cc:642] Executor failed to create kernel. Resource exhausted: OOM when allocating tensor of shape [7,7,512,4096] and type float
[[{{node fc6/weights/Adam/Initializer/zeros}}]]
Traceback (most recent call last):
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1365, in _do_call
return fn(*args)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1350, in _run_fn
target_list, run_metadata)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1443, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor of shape [7,7,512,4096] and type float
[[{{node fc6/weights/Adam/Initializer/zeros}}]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "main.py", line 272, in
run()
File "main.py", line 222, in run
correct_label, keep_prob, learning_rate)
File "main.py", line 158, in train_nn
sess.run(tf.global_variables_initializer())
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 956, in run
run_metadata_ptr)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1180, in _run
feed_dict_tensor, options, run_metadata)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1359, in _do_run
run_metadata)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\client\session.py", line 1384, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.ResourceExhaustedError: OOM when allocating tensor of shape [7,7,512,4096] and type float
[[node fc6/weights/Adam/Initializer/zeros (defined at C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\framework\ops.py:1751) ]]

Original stack trace for 'fc6/weights/Adam/Initializer/zeros':
File "main.py", line 272, in
run()
File "main.py", line 218, in run
logits, train_op, cross_entropy_loss = optimize(nn_last_layer, correct_label, learning_rate, num_classes)
File "main.py", line 129, in optimize
train_op = optimizer.minimize(cross_entropy_loss)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\optimizer.py", line 413, in minimize
name=name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\optimizer.py", line 597, in apply_gradients
self._create_slots(var_list)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\adam.py", line 131, in _create_slots
self._zeros_slot(v, "m", self._name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\optimizer.py", line 1156, in _zeros_slot
new_slot_variable = slot_creator.create_zeros_slot(var, op_name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\slot_creator.py", line 190, in create_zeros_slot
colocate_with_primary=colocate_with_primary)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\slot_creator.py", line 164, in create_slot_with_initializer
dtype)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\training\slot_creator.py", line 74, in _create_slot_var
validate_shape=validate_shape)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 1504, in get_variable
aggregation=aggregation)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 1247, in get_variable
aggregation=aggregation)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 567, in get_variable
aggregation=aggregation)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 519, in _true_getter
aggregation=aggregation)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 937, in _get_single_variable
aggregation=aggregation)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 258, in call
return cls._variable_v1_call(*args, **kwargs)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 219, in _variable_v1_call
shape=shape)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 197, in
previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 2523, in default_variable_creator
shape=shape)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 262, in call
return super(VariableMetaclass, cls).call(*args, **kwargs)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 1688, in init
shape=shape)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variables.py", line 1818, in _init_from_args
initial_value(), name="initial_value", dtype=dtype)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\variable_scope.py", line 906, in
partition_info=partition_info)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\init_ops.py", line 114, in call
return array_ops.zeros(shape, dtype)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\array_ops.py", line 2361, in zeros
output = fill(shape, constant(zero, dtype=dtype), name=name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\array_ops.py", line 171, in fill
result = gen_array_ops.fill(dims, value, name=name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\ops\gen_array_ops.py", line 3602, in fill
"Fill", dims=dims, value=value, name=name)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\framework\op_def_library.py", line 793, in _apply_op_helper
op_def=op_def)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\util\deprecation.py", line 507, in new_func
return func(*args, **kwargs)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3360, in create_op
attrs, op_def, compute_device)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\framework\ops.py", line 3429, in _create_op_internal
op_def=op_def)
File "C:\Users\10806337.conda\envs\RoadSegment\lib\site-packages\tensorflow_core\python\framework\ops.py", line 1751, in init
self._traceback = tf_stack.extract_stack()

Is there any problem with the code? Or just the problem of my machine?
Thanks.

[YBX420]

I have the same problem, and it might be the probelm of GPU.
I use 1050Ti and it always said that it is ran out of the memory