Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
Add temp output to debug centos crash
Browse files Browse the repository at this point in the history
  • Loading branch information
DickJC123 committed Mar 1, 2022
1 parent c5198c2 commit e013a85
Showing 1 changed file with 49 additions and 46 deletions.
95 changes: 49 additions & 46 deletions tests/python/gpu/test_gluon_gpu.py
Original file line number Diff line number Diff line change
Expand Up @@ -597,7 +597,8 @@ def test_cudnn_dropout_reproducibility():
assert_almost_equal(a.grad, b.grad)

@mx.util.use_np
def test_cuda_graphs():
@pytest.mark.serial
def test_cuda_graphs(capsys):
class GraphTester(gluon.HybridBlock):
def __init__(self, function_to_test, **kwargs):
super(GraphTester, self).__init__(**kwargs)
Expand Down Expand Up @@ -653,54 +654,56 @@ def generate_inputs(self):

N = 10

with environment({'MXNET_ENABLE_CUDA_GRAPHS': '1',
'MXNET_USE_FUSION': '0'}):
device = mx.gpu(0)
for test_desc in tested_ops:
print("Testing ", test_desc.name)
inputs = test_desc.generate_inputs()
inputsg = [i.copy() for i in inputs]
for i in inputsg:
i.attach_grad()
seed = random.randint(0, 10000)
net = GraphTester(test_desc.f)
netg = GraphTester(test_desc.f)

# initialize parameters
net.initialize(device=device)
netg.initialize(device=device)

net(*inputs)
with capsys.disabled():
with environment({'MXNET_ENABLE_CUDA_GRAPHS': '1',
'MXNET_CUDA_GRAPHS_VERBOSE': '1',
'MXNET_USE_FUSION': '0'}):
device = mx.gpu(0)
for test_desc in tested_ops:
sys.stdout.write('Testing {}\n'.format( test_desc.name))
inputs = test_desc.generate_inputs()
inputsg = [i.copy() for i in inputs]
for i in inputsg:
i.attach_grad()
seed = random.randint(0, 10000)
net = GraphTester(test_desc.f)
netg = GraphTester(test_desc.f)

# initialize parameters
net.initialize(device=device)
netg.initialize(device=device)

net(*inputs)

for p1, p2 in zip(net.collect_params().values(), netg.collect_params().values()):
p2.set_data(p1.data())

netg.hybridize(static_alloc=True, static_shape=True)

print("Testing inference mode")
with random_seed(seed):
for _ in range(N):
assert_almost_equal(net(*inputs), netg(*inputsg))
for p1, p2 in zip(net.collect_params().values(), netg.collect_params().values()):
p2.set_data(p1.data())

mx.npx.waitall()
print("Testing training mode")
for _ in range(N):
with random_seed(seed):
with mx.autograd.record():
out = net(*inputs)
out.backward()
netg.hybridize(static_alloc=True, static_shape=True)

print(" Testing inference mode")
with random_seed(seed):
with mx.autograd.record():
outg = netg(*inputsg)
outg.backward()
for _ in range(N):
assert_almost_equal(net(*inputs), netg(*inputsg))

assert_almost_equal(out, outg)
for i, ig in zip(inputs, inputsg):
assert_almost_equal(i.grad, ig.grad)

for p1, p2 in zip(net.collect_params().values(), netg.collect_params().values()):
assert_almost_equal(p1.data(), p2.data())
if p1.grad_req != 'null':
assert_almost_equal(p1.grad(), p2.grad())
mx.npx.waitall()
print(" Testing training mode")
for _ in range(N):
with random_seed(seed):
with mx.autograd.record():
out = net(*inputs)
out.backward()

with random_seed(seed):
with mx.autograd.record():
outg = netg(*inputsg)
outg.backward()

assert_almost_equal(out, outg)
for i, ig in zip(inputs, inputsg):
assert_almost_equal(i.grad, ig.grad)

for p1, p2 in zip(net.collect_params().values(), netg.collect_params().values()):
assert_almost_equal(p1.data(), p2.data())
if p1.grad_req != 'null':
assert_almost_equal(p1.grad(), p2.grad())
mx.npx.waitall()

0 comments on commit e013a85

Please sign in to comment.