redisvFormatCommand crash

[sungchune]

REDIS version : 3.0.0
#0 0x00007fa7256b45d7 in raise () from /lib64/libc.so.6

No symbol table info available.
#1 0x00007fa7256b5cc8 in abort () from /lib64/libc.so.6

No symbol table info available.
#2 0x00007fa7256f4e07 in __libc_message () from /lib64/libc.so.6

No symbol table info available.
#3 0x00007fa7256fac67 in malloc_printerr () from /lib64/libc.so.6

No symbol table info available.
#4 0x00007fa7256feb17 in _int_realloc () from /lib64/libc.so.6

No symbol table info available.
#5 0x00007fa7256ff702 in realloc () from /lib64/libc.so.6

No symbol table info available.
#6 0x000000000043121d in sdsMakeRoomFor (s=0x1777b <Address 0x1777b out of bounds>, addlen=97194) at sds.c:142

    newsh = 0x0
    sh = 0x1777b

#7 0x000000000043138a in sdscatlen (s=s@entry=0x7fa5c0000c48 "", t=t@entry=0x7fa5cb0fb5d0, len=len@entry=1) at sds.c:241

    sh = <optimized out>

#8 0x000000000042f3e4 in redisvFormatCommand (target=target@entry=0x7fa5cb0fb000, format=, ap=0x7fa5cb0fb048)

at hiredis.c:245
    c = 0x7fa5cb0fb5d0 "SET 450087140091610_LTE 0"
    cmd = 0x0
    pos = <optimized out>
    curarg = 0x7fa5c0000c48 ""
    newarg = <optimized out>
    touched = <optimized out>
    curargv = 0x0
    newargv = <optimized out>
    argc = 0
    totlen = 0
    error_type = 0
    j = <optimized out>
    __PRETTY_FUNCTION__ = "redisvFormatCommand"

#9 0x0000000000430de2 in redisvAppendCommand (c=0x7fa5c00008c0, format=, ap=) at hiredis.c:891

    cmd = 0x0
    len = <optimized out>

---Type to continue, or q to quit---
#10 0x0000000000430f8d in redisvCommand (c=0x7fa5c00008c0, format=, ap=ap@entry=0x7fa5cb0fb048) at hiredis.c:961

No locals.
#11 0x0000000000431057 in redisCommand (c=, format=) at hiredis.c:970

    ap = {{gp_offset = 16, fp_offset = 48, overflow_arg_area = 0x7fa5cb0fb120, reg_save_area = 0x7fa5cb0fb060}}
    reply = 0x0

#12 0x000000000040f3c2 in msg_hdlr (arg=0x0) at msg_thrd.c:2095

    ret = 0
    stat_idx = 5
    nbyte = 0
    recv_msg = "\250\025\000\000\000\000\000\000\001", '\000' <repeats 2038 times>
    redis_cmd = "SET 450087140091610_LTE 0", '\000' <repeats 230 times>
    send_msg = '\000' <repeats 1023 times>
    msg_hdr = {msg_type = 0 '\000', msg_stype = 0 '\000', length = 0, sess_id = 0}
    sync_body = {model = '\000' <repeats 23 times>, imsi = '\000' <repeats 15 times>, msisdn = '\000' <repeats 15 times>, lte_ip = 0, 
      cp_ip = 0, svc_pkg = '\000' <repeats 15 times>}
    mgpmd_res = {chn_id = 5544, result = 0, ma_code = 1}
    auth_res = {ver = 1 '\001', status = 0 '\000'}
    pchn = 0x7fa630ad3bd0
    session = 0x7fa618457b20
    redis = 0x7fa5c00008c0
    reply = 0x0

#13 0x00007fa725e2edf5 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.
#14 0x00007fa7257751ad in clone () from /lib64/libc.so.6

No symbol table info available.

[badboy]

Can you show the code?

[sungchune]

memset (redis_cmd, 0x00, sizeof(redis_cmd));
sprintf(redis_cmd, "RPUSH %s %s %s %s %s %s %s",
        session->ue_info.imsi,
        session->ue_info.ue_ver,
        session->ue_info.msisdn,
        session->ue_info.lte_ip,
        session->ue_info.cp_ip,
        session->ue_info.svc_pkg,
        session->ue_info.ue_model);
LOG(LOG_INF, "REDIS CMD = %s\n", redis_cmd);
if ( redis == NULL )
{
    redis = redis_connect(R_MASTER);
    if ( redis == NULL )
        add_to_list(redis_cmd);
    else
    {
        reply = redisCommand(redis, redis_cmd);
        if ( reply == NULL )
            add_to_list(redis_cmd);
        else
            freeReplyObject(reply);
    }
}
else
{
    reply = redisCommand(redis, redis_cmd);
    if ( reply == NULL )
        add_to_list(redis_cmd);
    else
        freeReplyObject(reply);
}
memset (redis_cmd, 0x00, sizeof(redis_cmd));
sprintf(redis_cmd, "SET %s_LTE 0", session->ue_info.imsi);
LOG(LOG_INF, "REDIS CMD = %s\n", redis_cmd);
if ( redis == NULL )
{
    redis = redis_connect(R_MASTER);
    if ( redis == NULL )
        add_to_list(redis_cmd);
    else
    {
        reply = redisCommand(redis, redis_cmd);
        if ( reply == NULL )
            add_to_list(redis_cmd);
        else
            freeReplyObject(reply);
    }
}
else
{
    reply = redisCommand(redis, redis_cmd); <<-- Crash
    if ( reply == NULL )
        add_to_list(redis_cmd);
    else
        freeReplyObject(reply);
}

[badboy]

Which version of hiredis did you use?

[sungchune]

hiredis version is "### 0.12.1 - January 26, 2015"

[badboy]

I can't reproduce it currently. It looks like some kind of memory corruption, as sdscatlen sees the correct sds string at 0x7fa5c0000c48, but one function call later sdsMakeRoomFor sees 0x1777b, though this should be the same location.

Is this reproducable on your side? Did you have any other memory-related issues so far?

[sungchune]

It has occured rarely so I can't reproduce it.
address of sds s in sdscatlen is 0x7fa5c0000c48, but it changes to 0x1777b in sdsMakeRoomFor funtion. is it same address?
The function is executing at thread. the stack size of thread is setting to 10MB. is it enough for hiredis or redis. How can I check memory curruption?

[badboy]

0x7fa5c0000c48 and 0x1777b clearly differ. :)

How large is your redis_cmd? It resides on the stack, right? this might cause the mentioned problems if it is to big.

[sungchune]

the size of redis_cmd is 256 (char redis_cmd[256];)
I think stack size is too small to execute redis operation. so I change the stack size to 20MB.

[badboy]

Please report back if this occurs again, currently I can't see another problem, but maybe there is.