Error when saving multiple output model weights

[svolchkov]

I get the below exception (call stack from VS2015) when I do model.save_weights or a checkpoint. It has only happened with multiple-output models. I use Theano backend (Theano 0.8.2), CUDA 7.5. This is Windows 7. Please let me know if further information is required.

Exception thrown at 0x00000000188E2F12 (tk85.dll) in pythonw.exe: 0xC0000005: Access violation reading location 0x0000000000000368.

tk85.dll!00000000188e2f12() Unknown
tk85.dll!000000001895c5e1() Unknown
tk85.dll!000000001882159f() Unknown
[External Code] 
hdf5.dll!00000000268e1f49() Unknown
h5f.pyd!0000000027676aa2()  Unknown

python27.dll!_PyImport_LoadDynamicModule(char * name, char * pathname, _iobuf * fp) Line 54 C
python27.dll!import_submodule(_object * mod, char * subname, char * fullname) Line 2722 C
python27.dll!ensure_fromlist(_object * mod, _object * fromlist, char * buf, __int64 buflen, int recursive) Line 2629 C
python27.dll!import_module_level(char * name, _object * globals, _object * locals, _object * fromlist, int level) Line 2291 C
python27.dll!PyImport_ImportModuleLevel(char * name, _object * globals, object * locals, object * fromlist, int level) Line 2310 C
python27.dll!builtin___import(_object * self, _object * args, _object * kwds) Line 51 C
python27.dll!PyCFunction_Call(_object * func, _object * arg, _object * kw) Line 85 C
python27.dll!PyObject_Call(_object * func, _object * arg, _object * kw) Line 2547 C
python27.dll!PyEval_CallObjectWithKeywords(_object * func, _object * arg, _object * kw) Line 4220 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2623 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!PyEval_EvalCode(PyCodeObject * co, _object * globals, _object * locals) Line 675 C
python27.dll!PyImport_ExecCodeModuleEx(char * name, _object * co, char * pathname) Line 731 C
python27.dll!load_source_module(char * name, char * pathname, _iobuf * fp) Line 1122 C
python27.dll!load_package(char * name, char * pathname) Line 1188 C
python27.dll!import_submodule(_object * mod, char * subname, char * fullname) Line 2722 C
python27.dll!load_next(_object * mod, _object * altmod, char * * p_name, char * buf, __int64 * p_buflen) Line 2537 C
python27.dll!import_module_level(char * name, _object * globals, _object * locals, _object * fromlist, int level) Line 2247 C
python27.dll!PyImport_ImportModuleLevel(char * name, _object * globals, object * locals, object * fromlist, int level) Line 2310 C
python27.dll!builtin___import(_object * self, _object * args, _object * kwds) Line 51 C
python27.dll!PyCFunction_Call(_object * func, _object * arg, _object * kw) Line 85 C
python27.dll!PyObject_Call(_object * func, _object * arg, _object * kw) Line 2547 C
python27.dll!PyEval_CallObjectWithKeywords(_object * func, _object * arg, _object * kw) Line 4220 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2623 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!PyEval_EvalCode(PyCodeObject * co, _object * globals, _object * locals) Line 675 C
python27.dll!exec_statement(_frame * f, _object * prog, _object * globals, _object * locals) Line 5048 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2108 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!function_call(_object * func, _object * arg, _object * kw) Line 528 C
python27.dll!PyObject_Call(_object * func, _object * arg, _object * kw) Line 2547 C
python27.dll!ext_do_call(_object * func, _object * * * pp_stack, int flags, int na, int nk) Line 4664 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 3029 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!function_call(_object * func, _object * arg, _object * kw) Line 528 C
python27.dll!PyObject_Call(_object * func, _object * arg, _object * kw) Line 2547 C
python27.dll!ext_do_call(_object * func, _object * * * pp_stack, int flags, int na, int nk) Line 4664 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 3029 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!function_call(_object * func, _object * arg, _object * kw) Line 528 C
python27.dll!PyObject_Call(_object * func, _object * arg, _object * kw) Line 2547 C
python27.dll!ext_do_call(_object * func, _object * * * pp_stack, int flags, int na, int nk) Line 4664 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 3029 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4449 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!fast_function(_object * func, _object * * * pp_stack, int n, int na, int nk) Line 4436 C
python27.dll!call_function(_object * * * pp_stack, int oparg) Line 4370 C
python27.dll!PyEval_EvalFrameEx(_frame * f, int throwflag) Line 2991 C
python27.dll!PyEval_EvalCodeEx(PyCodeObject * co, _object * globals, _object * locals, _object * * args, int argcount, _object * * kws, int kwcount, _object * * defs, int defcount, _object * closure) Line 3582 C
python27.dll!PyEval_EvalCode(PyCodeObject * co, _object * globals, _object * locals) Line 675 C
python27.dll!run_mod(_mod * mod, const char * filename, _object * globals, _object * locals, PyCompilerFlags * flags, _arena * arena) Line 1377 C
python27.dll!PyRun_FileExFlags(_iobuf * fp, const char * filename, int start, _object * globals, _object * locals, int closeit, PyCompilerFlags * flags) Line 1363 C
python27.dll!PyRun_SimpleFileExFlags(_iobuf * fp, const char * filename, int closeit, PyCompilerFlags * flags) Line 949 C
python27.dll!PyRun_AnyFileExFlags(_iobuf * fp, const char * filename, int closeit, PyCompilerFlags * flags) Line 753 C
python27.dll!Py_Main(int argc, char * * argv) Line 648 C
pythonw.exe!__tmainCRTStartup() Line 578 C
[External Code]

[svolchkov]

Here is the code:

`model = Graph()
model.add_input("sequence_input", input_shape = (3,32,32))

model.add_node(Convolution2D(32,3,3, border_mode = 'same',
activation='relu'), name = "conv1", input="sequence_input",) #1
model.add_node(Dropout(0.2),name = "drop1", input="conv1") #2
model.add_node(Convolution2D(32,3,3, border_mode = 'same',
activation='relu'),name = "conv2", input="drop1") #3
model.add_node(MaxPooling2D(pool_size=(2,2)),name = "pool1", input="conv2") #4
model.add_node(Convolution2D(64,3,3, border_mode = 'same',
activation='relu'), name = "conv3", input="pool1") #5
model.add_node(Dropout(0.2),name = "drop2", input="conv3") #6
model.add_node(Convolution2D(64,3,3, border_mode = 'same',
activation='relu'), name = "conv4", input="drop2") #7
model.add_node(MaxPooling2D(pool_size=(2,2)), name = "pool2", input="conv4") #8
model.add_node(Convolution2D(128,3,3, border_mode = 'same',
activation='relu'), name = "conv5", input="pool2") #9
model.add_node(Dropout(0.2), name = "drop3", input="conv5") #10
model.add_node(Convolution2D(128,3,3, border_mode = 'same',
activation='relu'), name = "conv6", input="drop3") #11
model.add_node(MaxPooling2D(pool_size=(2,2)), name = "pool3", input="conv6") #12
model.add_node(Flatten(), name = "flat1", input="pool3") #13
model.add_node(Dropout(0.2), name = "drop4", input="flat1") #14
model.add_node(Dense(1024,activation='relu', W_constraint=maxnorm(3)),name = "dense1", input="drop4" ) #15
model.add_node(Dropout(0.2), name = "drop5", input="dense1") #16
model.add_node(Dense(512,activation='relu', W_constraint=maxnorm(3)),name = "dense2", input="drop5" ) #17
model.add_node(Dropout(0.2),name = "shared_layers", input="dense2") #18

model.add_node(Dense(num_classes, activation="softmax"), name="output1", input="shared_layers", create_output=True)
model.add_node(Dense(num_classes, activation="softmax"), name="output2", input="shared_layers", create_output=True)
model.add_node(Dense(num_classes, activation="softmax"), name="output3", input="shared_layers", create_output=True)
model.add_node(Dense(num_classes, activation="softmax"), name="output4", input="shared_layers", create_output=True)
model.add_node(Dense(num_classes, activation="softmax"), name="output5", input="shared_layers", create_output=True)

epochs = 30
lrate = 0.01
decay = lrate / epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)

model.compile(loss={"output1": "categorical_crossentropy",
"output2": "categorical_crossentropy",
"output3": "categorical_crossentropy",
"output4": "categorical_crossentropy",
"output5": "categorical_crossentropy"},
optimizer=sgd, metrics=['accuracy'])
print(model.summary())
filepath="svhn-weight-improvement-{epoch:02d}-{loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(filepath,monitor='loss', verbose=1,save_best_only=True,
mode='max')
callbacks_list=[checkpoint]

model.fit({"sequence_input": X_train, "output1": y_train1, "output2": y_train2,
"output3": y_train3,"output4": y_train4,"output5": y_train5},
validation_data=({"sequence_input": X_test, "output1": y_test1, "output2": y_test2,
"output3": y_test3,"output4": y_test4,"output5": y_test5}), nb_epoch=epochs,
batch_size=32, verbose=2, callbacks = callbacks_list)

scores = model.evaluate({"sequence_input": X_test, "output1": y_test1, "output2": y_test2,
"output3": y_test3,"output4": y_test4,"output5": y_test5},verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
model.save_weights("svhn_standard.h5",overwrite=True)`

[svolchkov]

Looks like that there was an issue with h5py module. I'm pickling model weights now and everything is fine.