Conv2DCustomBackpropFilterOp only supports NHWC error

[merkoski]

Hi there! I'm trying to run this project (BTW, I'm looking forward to it!) and when it gets time to start creating a wavenet training model via "python train.py", I experience this error. Any workarounds? I'm using tensorflow-gpu on a Windows machine. Uninstalling tensorflow-gpu and installing tensorflow does not help.

Generated 328 test batches of size 1 in 0.297 sec
2018-06-18 13:30:33.533678: E T:\src\github\tensorflow\tensorflow\core\common_runtime\executor.cc:660] Executor failed to create kernel. Invalid argument: Conv2DCustomBackpropFilterOp only supports NHWC.
[[Node: model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_197"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose/BiasAdd_grad/tuple/control_dependency/_2947, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/ExpandDims_1/_2949)]]
Exiting due to Exception: Conv2DCustomBackpropFilterOp only supports NHWC.
[[Node: model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_197"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose/BiasAdd_grad/tuple/control_dependency/_2947, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/ExpandDims_1/_2949)]]

Caused by op 'model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Conv2DBackpropFilter', defined at:
File "train.py", line 128, in
main()
File "train.py", line 122, in main
train(args, log_dir, hparams)
File "train.py", line 76, in train
checkpoint = wavenet_train(args, log_dir, hparams, input_path)
File "D:\DL2\Tacotron-2\wavenet_vocoder\train.py", line 244, in wavenet_train
return train(log_dir, args, hparams, input_path)
File "D:\DL2\Tacotron-2\wavenet_vocoder\train.py", line 167, in train
model, stats = model_train_mode(args, feeder, hparams, global_step)
File "D:\DL2\Tacotron-2\wavenet_vocoder\train.py", line 119, in model_train_mode
model.add_optimizer(global_step)
File "D:\DL2\Tacotron-2\wavenet_vocoder\models\wavenet.py", line 365, in add_optimizer
gradients, variables = zip(*optimizer.compute_gradients(self.loss))
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\training\optimizer.py", line 526, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 494, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 636, in _GradientsHelper
lambda: grad_fn(op, *out_grads))
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 385, in _MaybeCompile
return grad_fn() # Exit early
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 636, in
lambda: grad_fn(op, *out_grads))
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\nn_grad.py", line 54, in _Conv2DBackpropInputGrad
data_format=op.get_attr("data_format")),
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1190, in conv2d_backprop_filter
dilations=dilations, name=name)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

...which was originally created as op 'model/inference/conv2d_transpose/conv2d_transpose', defined at:
File "train.py", line 128, in
main()
[elided 3 identical lines from previous traceback]
File "D:\DL2\Tacotron-2\wavenet_vocoder\train.py", line 167, in train
model, stats = model_train_mode(args, feeder, hparams, global_step)
File "D:\DL2\Tacotron-2\wavenet_vocoder\train.py", line 117, in model_train_mode
feeder.input_lengths, x=feeder.inputs)
File "D:\DL2\Tacotron-2\wavenet_vocoder\models\wavenet.py", line 169, in initialize
y_hat = self.step(x, c, g, softmax=False) #softmax is automatically computed inside softmax_cross_entropy if needed
File "D:\DL2\Tacotron-2\wavenet_vocoder\models\wavenet.py", line 435, in step
c = transposed_conv(c)
File "D:\DL2\Tacotron-2\wavenet_vocoder\models\modules.py", line 333, in call
return self.convt(inputs)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\layers\base.py", line 717, in call
outputs = self.call(inputs, *args, **kwargs)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\layers\convolutional.py", line 1667, in call
data_format=utils.convert_data_format(self.data_format, ndim=4))
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 1254, in conv2d_transpose
name=name)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1340, in conv2d_backprop_input
dilations=dilations, name=name)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\Users\merko\AppData\Local\conda\conda\envs\DL2\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InvalidArgumentError (see above for traceback): Conv2DCustomBackpropFilterOp only supports NHWC.
[[Node: model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_197"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose/BiasAdd_grad/tuple/control_dependency/_2947, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/ExpandDims_1/_2949)]]

2018-06-18 13:30:33.575389: W T:\src\github\tensorflow\tensorflow\core\kernels\queue_base.cc:277] _1_datafeeder/eval_queue: Skipping cancelled enqueue attempt with queue not closed
2018-06-18 13:30:33.581458: W T:\src\github\tensorflow\tensorflow\core\kernels\queue_base.cc:277] _0_datafeeder/intput_queue: Skipping cancelled enqueue attempt with queue not closed
Traceback (most recent call last):
File "train.py", line 128, in
main()
File "train.py", line 122, in main
train(args, log_dir, hparams)
File "train.py", line 78, in train
raise ('Error occured while training Wavenet, Exiting!')
TypeError: exceptions must derive from BaseException

[DanRuta]

I am also having this issue.

@ChenXi1992 Are you also using Windows? Also, (@merkoski too), what versions of python, tensorflow-gpu, etc, are you using, and did you tweak anything? Maybe we can find a common culprit.

[merkoski]

I'm afraid this doesn't help. In my case the GPU does work just fine, throughout the Tacotron modeling; it fails with the "Conv2DCustomBackpropFilterOp only supports NHWC" error during the Wavenet modeling.

I am on Windows10, Python 3.6.5, Tensorflow (GPU) 1.8.0.

[DanRuta]

GPU seems fine for me, also. Tacotron seems to be using it fine, but it fails at Wavenet. I haven't tried any other Wavenet implementations, yet, as I'm away. I had to set outputs_per_step to 5, in hparams.py (#4), but changed nothing in the code from the latest version. I'll dig around some more when I get some time.

I'm running Python 3.6.4, with Tensorflow gpu 1.8.0.

[merkoski]

Hey @DanRuta I had to do something similar, in my case specifying tacotron_cache_size of 3 in hparams (via command line in my case). I had to do this in order for the data to fit onto the GPU, but I wonder if a non-standard hparams is causing the issue somehow....

[harlyh]

Has anyone been able to find a working solution to this problem? For a while I've been stuck with the same error now :/

[DanRuta]

Not me, unfortunately. The existing suggestions were not applicable or didn't solve it.

[avivelor]

Does anyone have any new leads on getting Wavenet to work? I looked all over github and I can't seem to find a solution. I tried isolating my GPU for training, changing h-parameters, and changing data_formats to 'NCHW.' However, nothing seems to quite work :(.

I'm running Python 3.6.2 and Tensorflow GPU 1.8.0.
My GPU does fine when training Tacotron alone.

Here's my nvcc:
NVIDIA (R) Cuda compiler driver
Copyright (c) 2005-2017 NVIDIA Corporation
Built on Fri_Sep__1_21:08:32_Central_Daylight_Time_2017
Cuda compilation tools, release 9.0, V9.0.176

Here's my output from "tf.__version__sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))":

tf.__version__sess=tf.Session(config=tf.ConfigProto(log_device_placement=True))
2018-07-03 09:52:41.754046: I T:\src\github\tensorflow\tensorflow\core\platform\cpu_feature_guard.cc:140] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2
2018-07-03 09:52:42.004040: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:1356] Found device 0 with properties:
name: TITAN X (Pascal) major: 6 minor: 1 memoryClockRate(GHz): 1.531
pciBusID: 0000:01:00.0
totalMemory: 12.00GiB freeMemory: 9.93GiB
2018-07-03 09:52:42.011150: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:1435] Adding visible gpu devices: 0
2018-07-03 09:52:42.547125: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:923] Device interconnect StreamExecutor with strength 1 edge matrix:
2018-07-03 09:52:42.550875: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:929] 0
2018-07-03 09:52:42.553274: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:942] 0: N
2018-07-03 09:52:42.555655: I T:\src\github\tensorflow\tensorflow\core\common_runtime\gpu\gpu_device.cc:1053] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 9616 MB memory) -> physical GPU (device: 0, name: TITAN X (Pascal), pci bus id: 0000:01:00.0, compute capability: 6.1)
Device mapping:
/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: TITAN X (Pascal), pci bus id: 0000:01:00.0, compute capability: 6.1
2018-07-03 09:52:42.974984: I T:\src\github\tensorflow\tensorflow\core\common_runtime\direct_session.cc:284] Device mapping:
/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: TITAN X (Pascal), pci bus id: 0000:01:00.0, compute capability: 6.1

[ben-8878]

it seems that all have similar errors
so ask this question to the official tensorflow website and get some replies:

Have I written custom code
OS Platform and Distribution
TensorFlow installed from
TensorFlow version
Bazel version
CUDA/cuDNN version
GPU model and memory
Exact command to reproduce

[avivelor]

@v-yunbin
Posted on StackOverflow (official tensorflow website forms): https://stackoverflow.com/questions/51256459/conv2dcustombackpropfilterop-only-supports-nhwc-error-tacotron-2

Currently waiting for a reply. Perhaps we should upvote it to try to get a higher priority? Feel free to add to the post as well.

System information

• Have I written custom code (as opposed to using a stock example script provided in TensorFlow):
  Yes, I used the following: https://github.com/Rayhane-mamah/Tacotron-2
• OS Platform and Distribution (e.g., Linux Ubuntu 16.04):
  Windows 10 Home
  Alienware Desktop, Intel Core i7 CPU @ 3.60GHz
  64.0GB Ram, 64x processor
• TensorFlow installed from (source or binary):
  source, pip: https://www.tensorflow.org/install/install_windows
• TensorFlow version (use command below):
  Tensorflow GPU 1.8.0, command: pip install --upgrade tensorflow-gpu
• Python version:
  Python 3.6.5
• Bazel version (if compiling from source):
  NA
• GCC/Compiler version (if compiling from source):
  NA
• CUDA/cuDNN version:
  Cuda 9.0
• GPU model and memory:
  Nvidia Titan X
• Exact command to reproduce:
  "python train.py --model WaveNet" or "python train.py --model Both" (after 20000 tacotron steps).

[DaniKog]

Encountered the same issue. Tried running Wavenet training without changing anything in hparams.py, still is still occurred.

[ben-8878]

hi guys， my wavenet train has been training, replacing tensorflow cpu with tensorflow gpu(1.8 version)

[avivelor]

@v-yunbin, that's awesome to hear!
I'm still getting errors, so is there any chance you could share more details?
i.e:
*OS Platform and Distribution (e.g., Linux Ubuntu 16.04).
*TensorFlow installed from (source or binary).
*TensorFlow version (use command below).
*Python version (as well as from cmd or anaconda).
*CUDA/cuDNN version.
*GPU model and memory.

Thanks!

[ben-8878]

@avivelor
*OS Platform and Distribution: CentOS Linux release 7.3.1611 (Core)
*TensorFlow installed from: binary (use "pip" to install it )
*TensorFlow version tensorflow-gpu 1.8
*Python version: anaconda 3.5
*CUDA/cuDNN version: cuda-9.0
*GPU model and memory: 1T

[DanRuta]

Had another go at this today. I nuked my python installation, installed and used Python 3.6 with virtualenv, installing only the requirements.txt and tensorflow-gpu 1.9.0, but the issue persists.

I ran python train.py --model="Both" --hparams="outputs_per_step=11,tacotron_batch_size=16"

It trained the tacotron model for a bit over a couple of hours, and crashed with the NHWC error at the "Wavenet Train" stage.

*OS Platform and Distribution: Windows 10 64bit
*TensorFlow installed from: pip install tensorflow-gpu
*TensorFlow version 1.9.0 (or 1.8.0)
*Python version: 3.6
*CUDA/cuDNN version: cuda-9.0 (or 9.2)
*GPU model and memory: Nvidia gtx 1080 8GB

I preprocessed my data using the LJSpeech default option

I also tried v1.8.0 of tensorflow-gpu, running --model="Wavenet" (and then tacotron_batch_size 8 and 4), to no avail.

Finally, I reinstalled CUDA drivers, updating to 9.2 (with batch sizes of 16 or 8), also to no avail.

At this point, I'm no longer sure if this has anything to do with dependencies.

[Rayhane-mamah]

I was not able to reproduce the Error no matter how hard I tried. (I even changed the Conv2Dtranspose to channels_last and couldn't reproduce.

My setup:

• Ubuntu 16.04, 64-bit
• Tensorflow-gpu built from source (tried with 1.7 and 1.8)
• Python: 3.6 (anaconda)
• CUDA/CUDNN: 9.0/7.1
• GPU: Nvidia Titan Xp 12GB

I would ask from you guys to check that you don't have tensorflow (cpu) version installed in parallel on your machine, and try to build from source with exact same build I have.
Since this error doesn't seem to be coming from the repo, I am closing the issue. If you find any news, feel free to reopen.

[skeletonli]

It seems that the problem appear with Windows 10 64bit

[CherryCloris]

Hey, I got the same issue when I training the wavenet, Did you find the solution?Thanks very much……

[avivelor]

I fixed it by switching over to Ray's Setup. My setup: - Ubuntu 16.04, 64-bit (HWE installed from server) - Tensorflow-gpu built from source (tried with 1.7 and 1.8) - Python: 3.6 (anaconda) - CUDA/CUDNN: 9.0/7.1 - GPU: Nvidia GTX 1080

…

On Tue, Oct 9, 2018, 6:57 PM CherryCloris ***@***.***> wrote: Hey, I got the same issue when I training the wavenet, Did you find the solution?Thanks very much…… — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <#73 (comment)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AQPq-jgiAKn59icoi_OZje1P8PHDKfh6ks5ujVP2gaJpZM4UsWbX> .

[lucy3589]

I am meeting this error,because my tensorflow-cpu by deflault. so i use :conda install tensorflow-gpu

[PrasannaKumarARDS]

It's resolved ..
the issue is because of image_data_format the backend(Tensorflow/Theano/CNTK...) uses
if you are using Keras (backend as Tensorflow) the data_format='channels_last' with images reshaped
to (batch_size,row_size,column_size,channel_size) would solve the issue

[mrgloom]

@lucy3589 Why tensorflow-cpu cause this problem?

[Harish7nov]

It's resolved ..
the issue is because of image_data_format the backend(Tensorflow/Theano/CNTK...) uses
if you are using Keras (backend as Tensorflow) the data_format='channels_last' with images reshaped
to (batch_size,row_size,column_size,channel_size) would solve the issue

It worked !!!!

[MetalCase]

已解决..
问题是由于image_data_format后端（Tensorflow / Theano / CNTK ...）使用的原因，
如果您使用Keras（后端为Tensorflow），则data_format ='channels_last'的图像被重塑
为（batch_size，row_size，column_size， channel_size）将解决问题

有效 ！！！！

I'm a newbie. Can you be more specific?thank you

[jreus]

@PrasannaKumarARDS

It's resolved ..
the issue is because of image_data_format the backend(Tensorflow/Theano/CNTK...) uses
if you are using Keras (backend as Tensorflow) the data_format='channels_last' with images reshaped
to (batch_size,row_size,column_size,channel_size) would solve the issue

can you explain this a bit more?
Where do you find data_format='channels_last' ?
In what file does this change need to be made?

[rameshKrSah]

@PrasannaKumarARDS

It's resolved ..
the issue is because of image_data_format the backend(Tensorflow/Theano/CNTK...) uses
if you are using Keras (backend as Tensorflow) the data_format='channels_last' with images reshaped
to (batch_size,row_size,column_size,channel_size) would solve the issue

can you explain this a bit more?
Where do you find data_format='channels_last' ?
In what file does this change need to be made?

the data_format is an argument you can specify in the keras.layers.Conv1D()

[camjac251]

@PrasannaKumarARDS @rameshKrSah Do you know if how this could be implemented into a fork of this project? I'm using karamarieliu/gst_tacotron2_wavenet which is this project but with Tacotron 2 GST. Wavenet on the repo is similar to this repo's earlier 2018 modules.py.

I tried changing the ConvTranspose2d data_format to last but got this error.

Traceback (most recent call last):
  File "train.py", line 127, in <module>
    main()
  File "train.py", line 119, in main
    wavenet_train(args, log_dir, hparams, args.wavenet_input)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 244, in wavenet_train
    return train(log_dir, args, hparams, input_path)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 167, in train
    model, stats = model_train_mode(args, feeder, hparams, global_step)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 117, in model_train_mode
    feeder.input_lengths, x=feeder.inputs)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\wavenet.py", line 169, in initialize
    y_hat = self.step(x, c, g, softmax=False) #softmax is automatically computed inside softmax_cross_entropy if needed
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\wavenet.py", line 435, in step
    c = transposed_conv(c)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\modules.py", line 333, in __call__
    return self.convt(inputs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\layers\base.py", line 362, in __call__
    outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 728, in __call__
    self.build(input_shapes)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\keras\layers\convolutional.py", line 719, in build
    raise ValueError('The channel dimension of the inputs '
ValueError: The channel dimension of the inputs should be defined. Found `None`.

but with it as first, I get this

2020-03-18 13:38:25.399613: E T:\src\github\tensorflow\tensorflow\core\common_runtime\executor.cc:697] Executor failed to create kernel. Invalid argument: Conv2DCustomBackpropFilterOp only supports NHWC.
         [[Node: model/optimizer/gradients/model/inference/conv2d_transpose_1/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_199"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose_1/BiasAdd_grad/tuple/control_dependency/_2171, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/conv2d_transpose/BiasAdd/_2173)]]
Exiting due to Exception: Conv2DCustomBackpropFilterOp only supports NHWC.
         [[Node: model/optimizer/gradients/model/inference/conv2d_transpose_1/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_199"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose_1/BiasAdd_grad/tuple/control_dependency/_2171, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/conv2d_transpose/BiasAdd/_2173)]]

Caused by op 'model/optimizer/gradients/model/inference/conv2d_transpose_1/conv2d_transpose_grad/Conv2DBackpropFilter', defined at:
  File "train.py", line 127, in <module>
    main()
  File "train.py", line 119, in main
    wavenet_train(args, log_dir, hparams, args.wavenet_input)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 244, in wavenet_train
    return train(log_dir, args, hparams, input_path)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 167, in train
    model, stats = model_train_mode(args, feeder, hparams, global_step)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 119, in model_train_mode
    model.add_optimizer(global_step)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\wavenet.py", line 365, in add_optimizer
    gradients, variables = zip(*optimizer.compute_gradients(self.loss))
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\training\optimizer.py", line 514, in compute_gradients
    colocate_gradients_with_ops=colocate_gradients_with_ops)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 596, in gradients
    gate_gradients, aggregation_method, stop_gradients)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 779, in _GradientsHelper
    lambda: grad_fn(op, *out_grads))
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 398, in _MaybeCompile
    return grad_fn()  # Exit early
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 779, in <lambda>
    lambda: grad_fn(op, *out_grads))
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\nn_grad.py", line 54, in _Conv2DBackpropInputGrad
    data_format=op.get_attr("data_format")),
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1190, in conv2d_backprop_filter
    dilations=dilations, name=name)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\util\deprecation.py", line 454, in new_func
    return func(*args, **kwargs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\framework\ops.py", line 3155, in create_op
    op_def=op_def)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\framework\ops.py", line 1717, in __init__
    self._traceback = tf_stack.extract_stack()

...which was originally created as op 'model/inference/conv2d_transpose_1/conv2d_transpose', defined at:
  File "train.py", line 127, in <module>
    main()
[elided 2 identical lines from previous traceback]
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 167, in train
    model, stats = model_train_mode(args, feeder, hparams, global_step)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\train.py", line 117, in model_train_mode
    feeder.input_lengths, x=feeder.inputs)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\wavenet.py", line 169, in initialize
    y_hat = self.step(x, c, g, softmax=False) #softmax is automatically computed inside softmax_cross_entropy if needed
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\wavenet.py", line 435, in step
    c = transposed_conv(c)
  File "C:\Users\camja\Desktop\gst_tacotron2_wavenet\wavenet_vocoder\models\modules.py", line 333, in __call__
    return self.convt(inputs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\layers\base.py", line 362, in __call__
    outputs = super(Layer, self).__call__(inputs, *args, **kwargs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\keras\engine\base_layer.py", line 736, in __call__
    outputs = self.call(inputs, *args, **kwargs)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\keras\layers\convolutional.py", line 781, in call
    data_format=conv_utils.convert_data_format(self.data_format, ndim=4))
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\nn_ops.py", line 1254, in conv2d_transpose
    name=name)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1340, in conv2d_backprop_input
    dilations=dilations, name=name)
  File "C:\Users\camja\Anaconda3\envs\taco\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
    op_def=op_def)

InvalidArgumentError (see above for traceback): Conv2DCustomBackpropFilterOp only supports NHWC.
         [[Node: model/optimizer/gradients/model/inference/conv2d_transpose_1/conv2d_transpose_grad/Conv2DBackpropFilter = Conv2DBackpropFilter[T=DT_FLOAT, _class=["loc:@model/optimizer/clip_by_global_norm/mul_199"], data_format="NCHW", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 16], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](model/optimizer/gradients/model/inference/conv2d_transpose_1/BiasAdd_grad/tuple/control_dependency/_2171, model/optimizer/gradients/model/inference/conv2d_transpose/conv2d_transpose_grad/Shape, model/inference/conv2d_transpose/BiasAdd/_2173)]]

Am I missing something?

[HandsomeManKris]

It's resolved ..
the issue is because of image_data_format the backend(Tensorflow/Theano/CNTK...) uses
if you are using Keras (backend as Tensorflow) the data_format='channels_last' with images reshaped
to (batch_size,row_size,column_size,channel_size) would solve the issue

yes!! I find it useful

[KarolinaPondel]

Hi guys,
I'm trying to get WaveNet training working and I keep getting this problem. I can't find its location and don't know how to fix it. Or is there an update on this issue? I only have tansorflow-gpu installed. data_format='channels_last' doesn't work or I don't understand where to change it. Tacotron workout went through without any problems.

Exiting due to exception: Conv2DCustomBackpropInputOp only supports NHWC.
[[node WaveNet_model/optimizer_1/gradients/WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/Conv2D_grad/Conv2DBackpropInput (defined at /notebooks/Tacotron-2/wavenet_vocoder/models/wavenet.py:557) = Conv2DBackpropInput[T=DT_FLOAT, data_format="NCHW", dilations=[1, 1, 1, 1], padding="VALID", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:CPU:0"](WaveNet_model/optimizer_1/gradients/WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/Conv2D_grad/ShapeN, WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/ExpandDims_1, WaveNet_model/optimizer_1/gradients/WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/Squeeze_grad/Reshape)]]

Caused by op 'WaveNet_model/optimizer_1/gradients/WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/Conv2D_grad/Conv2DBackpropInput', defined at:
File "train.py", line 138, in
main()
File "train.py", line 130, in main
wavenet_train(args, log_dir, hparams, args.wavenet_input)
File "/notebooks/Tacotron-2/wavenet_vocoder/train.py", line 346, in wavenet_train
return train(log_dir, args, hparams, input_path)
File "/notebooks/Tacotron-2/wavenet_vocoder/train.py", line 230, in train
model, stats = model_train_mode(args, feeder, hparams, global_step)
File "/notebooks/Tacotron-2/wavenet_vocoder/train.py", line 178, in model_train_mode
model.add_optimizer(global_step)
File "/notebooks/Tacotron-2/wavenet_vocoder/models/wavenet.py", line 557, in add_optimizer
gradients = optimizer.compute_gradients(self.tower_loss[i])
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/training/optimizer.py", line 519, in compute_gradients
colocate_gradients_with_ops=colocate_gradients_with_ops)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py", line 630, in gradients
gate_gradients, aggregation_method, stop_gradients)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py", line 814, in _GradientsHelper
lambda: grad_fn(op, *out_grads))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py", line 408, in _MaybeCompile
return grad_fn() # Exit early
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gradients_impl.py", line 814, in
lambda: grad_fn(op, *out_grads))
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_grad.py", line 517, in _Conv2DGrad
data_format=data_format),
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/gen_nn_ops.py", line 1229, in conv2d_backprop_input
dilations=dilations, name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py", line 488, in new_func
return func(*args, **kwargs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 3274, in create_op
op_def=op_def)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/ops.py", line 1770, in init
self._traceback = tf_stack.extract_stack()

...which was originally created as op 'WaveNet_model/inference/final_convolution_2/final_convolution_2_1/final_convolution_2/conv1d/Conv2D', defined at:
File "train.py", line 138, in
main()
[elided 2 identical lines from previous traceback]
File "/notebooks/Tacotron-2/wavenet_vocoder/train.py", line 230, in train
model, stats = model_train_mode(args, feeder, hparams, global_step)
File "/notebooks/Tacotron-2/wavenet_vocoder/train.py", line 176, in model_train_mode
feeder.input_lengths, x=feeder.inputs)
File "/notebooks/Tacotron-2/wavenet_vocoder/models/wavenet.py", line 277, in initialize
y_hat_train = self.step(tower_x[i], tower_c[i], tower_g[i], softmax=False) #softmax is automatically computed inside softmax_cross_entropy if needed
File "/notebooks/Tacotron-2/wavenet_vocoder/models/wavenet.py", line 719, in step
x = conv(x)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/keras/engine/base_layer.py", line 757, in call
outputs = self.call(inputs, *args, **kwargs)
File "/notebooks/Tacotron-2/wavenet_vocoder/models/modules.py", line 382, in call
return super(Conv1D1x1, self).call(inputs, incremental=incremental, convolution_queue=convolution_queue)
File "/notebooks/Tacotron-2/wavenet_vocoder/models/modules.py", line 319, in call
outputs = self.layer.call(inputs_)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 384, in call
return super(Conv1D, self).call(inputs)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/keras/layers/convolutional.py", line 194, in call
outputs = self._convolution_op(inputs, self.kernel)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 868, in call
return self.conv_op(inp, filter)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 520, in call
return self.call(inp, filter)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 204, in call
name=self.name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/nn_ops.py", line 193, in _conv1d
name=name)
File "/usr/local/lib/python3.5/dist-packages/tensorflow/python/util/deprecation.py", line 553, in new_func
return func(*args, **kwargs)