Tutorial 3 - PyTorch Tutorial 3 Mixed Precision

[Adamage]

https://github.com/graphcore/tutorials/tree/sdk-release-2.2/tutorials/pytorch/tut3_mixed_precision

[Adamage]

Unfortunately application hangs for 50+ minutes after

[trace] Initializing session

Full log

[07:10:59.457] [poptorch:cpp] [debug] Graph lowered to Popart {
[07:10:59.482] [poptorch:cpp] [debug] graph(
[07:10:59.482] [poptorch:cpp] [debug]       input: %x.1 : Half(32, 1, 128, 128, strides=[16384, 16384, 128, 1], requires_grad=0, device=cpu) -> input [float16(32, 1, 128, 128)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %72 : Half(5, 1, 3, 3, strides=[9, 9, 3, 1], requires_grad=1, device=cpu) -> conv1.weight [float16(5, 1, 3, 3)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %73 : Half(5, strides=[1], requires_grad=1, device=cpu) -> conv1.bias [float16(5)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %74 : Half(12, 5, 5, 5, strides=[125, 25, 5, 1], requires_grad=1, device=cpu) -> conv2.weight [float16(12, 5, 5, 5)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %75 : Half(12, strides=[1], requires_grad=1, device=cpu) -> conv2.bias [float16(12)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %76 : Half(12, strides=[1], requires_grad=1, device=cpu) -> norm.weight [float16(12)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %77 : Half(12, strides=[1], requires_grad=1, device=cpu) -> norm.bias [float16(12)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %78 : Half(100, 41772, strides=[41772, 1], requires_grad=1, device=cpu) -> fc1.weight [float16(100, 41772)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %79 : Half(100, strides=[1], requires_grad=1, device=cpu) -> fc1.bias [float16(100)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %80 : Half(10, 100, strides=[100, 1], requires_grad=1, device=cpu) -> fc2.weight [float16(10, 100)]
[07:10:59.482] [poptorch:cpp] [debug]       param: %81 : Half(10, strides=[1], requires_grad=1, device=cpu) -> fc2.bias [float16(10)]
[07:10:59.482] [poptorch:cpp] [debug]   ):
[07:10:59.485] [poptorch:cpp] [debug]   %169 : Half(32, 5, 126, 126, strides=[79380, 15876, 126, 1], requires_grad=1, device=cpu) = popart::conv[dilations=[1, 1], group=1, kernel_shape=annotate(List[int], []), pads=[0, 0, 0, 0], strides=[1, 1]](%x.1, %72, %73) was lowered to Conv:0 [float16(32, 5, 126, 126), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.486] [poptorch:cpp] [debug]   %170 : Half(32, 5, 126, 126, strides=[79380, 15876, 126, 1], requires_grad=1, device=cpu) = popart::relu(%169) was lowered to Relu:0 [float16(32, 5, 126, 126), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.490] [poptorch:cpp] [debug]   %171 : Half(32, 5, 63, 63, strides=[19845, 3969, 63, 1], requires_grad=1, device=cpu) = popart::maxpool[num_outputs=1, kernel_shape=[2, 2], ceil_mode=0, dilations=[1, 1], pads=[0, 0, 0, 0], storage_order=0, strides=[2, 2]](%170) was lowered to MaxPool:0 [float16(32, 5, 63, 63), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.495] [poptorch:cpp] [debug]   %172 : Half(32, 12, 59, 59, strides=[41772, 3481, 59, 1], requires_grad=1, device=cpu) = popart::conv[dilations=[1, 1], group=1, kernel_shape=annotate(List[int], []), pads=[0, 0, 0, 0], strides=[1, 1]](%171, %74, %75) was lowered to Conv:0/1 [float16(32, 12, 59, 59), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.497] [poptorch:cpp] [debug]   %173 : Half(32, 12, 59, 59, strides=[41772, 3481, 59, 1], requires_grad=1, device=cpu) = popart::relu(%172) was lowered to Relu:0/1 [float16(32, 12, 59, 59), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.501] [poptorch:cpp] [debug]   %174 : Half(32, 12, 59, 59, strides=[41772, 3481, 59, 1], requires_grad=1, device=cpu) = popart::groupnormalization[num_groups=3, epsilon=9.9999997473787516e-06](%173, %76, %77) was lowered to GroupNormalization:0 [float16(32, 12, 59, 59), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.506] [poptorch:cpp] [debug]   %175 : Half(32, 41772, strides=[41772, 1], requires_grad=1, device=cpu) = popart::reshape_static_shape[shape=[32, 41772]](%174) was lowered to Reshape:0 [float16(32, 41772), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.509] [poptorch:cpp] [debug]   %176 : HalfTensor = popart::transpose[perm=[1, 0]](%78) was lowered to Transpose:0 [float16(41772, 100), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.511] [poptorch:cpp] [debug]   %177 : HalfTensor = popart::matmul(%175, %176) was lowered to MatMul:0 [float16(32, 100), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.514] [poptorch:cpp] [debug]   %178 : Half(32, 100, strides=[100, 1], requires_grad=1, device=cpu) = popart::add(%177, %79) was lowered to Add:0 [float16(32, 100), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.516] [poptorch:cpp] [debug]   %179 : Half(32, 100, strides=[100, 1], requires_grad=1, device=cpu) = popart::relu(%178) was lowered to Relu:0/2 [float16(32, 100), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.518] [poptorch:cpp] [debug]   %180 : HalfTensor = popart::transpose[perm=[1, 0]](%80) was lowered to Transpose:0/1 [float16(100, 10), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.520] [poptorch:cpp] [debug]   %181 : HalfTensor = popart::matmul(%179, %180) was lowered to MatMul:0/1 [float16(32, 10), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.522] [poptorch:cpp] [debug]   %182 : Half(32, 10, strides=[10, 1], requires_grad=1, device=cpu) = popart::add(%181, %81) was lowered to Add:0/1 [float16(32, 10), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.523] [poptorch:cpp] [debug]   %183 : HalfTensor = popart::transpose[perm=[1, 0]](%182) was lowered to Transpose:0/2 [float16(10, 32), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.525] [poptorch:cpp] [debug]   %184 : HalfTensor = popart::logsoftmax[axis=1](%183) was lowered to LogSoftmax:0 [float16(10, 32), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.527] [poptorch:cpp] [debug]   %185 : Half(32, 10, strides=[10, 1], requires_grad=1, device=cpu) = popart::transpose[perm=[1, 0]](%184) was lowered to Transpose:0/3 [float16(32, 10), mode(Pipelined), ipu(0), stage(0)]
[07:10:59.527] [poptorch:cpp] [debug]   return (
[07:10:59.527] [poptorch:cpp] [debug]     output: %185 : Half(32, 10, strides=[10, 1], requires_grad=1, device=cpu) -> Transpose:0/3 [float16(32, 10)]
[07:10:59.527] [poptorch:cpp] [debug]   )
[07:10:59.527] [poptorch:cpp] [debug] }
[07:10:59.527] [poptorch:cpp] [trace] Initializing session