Uploading error

[Xprogrammer777]

I ran with -d to know where the error was coming from but didn't understand anything:

1.0
Linux 264142e94d17 6.1.58+ #1 SMP PREEMPT_DYNAMIC Sat Nov 18 15:31:17 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux
Ubuntu 22.04.3 LTS \n \l

+ RESPONSE_FILE=/tmp/du_resp_debug
+ [[ '' == '' ]]
+ BIN_DEPS='sed basename date grep stat dd mkdir curl'
+ CURL_BIN=curl
+ which sed basename date grep stat dd mkdir curl
+ [[ 0 != 0 ]]
+ which readlink
+ [[ 0 == 0 ]]
++ readlink -m //test
+ [[ /test == \/\t\e\s\t ]]
+ HAVE_READLINK=1
+ builtin printf ''
+ [[ 0 == 0 ]]
+ PRINTF='builtin printf'
+ PRINTF_OPT='-v o'
+ [[ -e /root/.dropbox_uploader ]]
+ source /root/.dropbox_uploader
+ [[ '' != '' ]]
+ [[ '' != '' ]]
+ [[ sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP = '' ]]
+ COMMAND=upload
+ ARG1=model.pth
+ ARG2=model1.pth
+ let argnum=4-2
+ case $COMMAND in
+ [[ 2 -lt 2 ]]
+ FILE_DST=model1.pth
+ (( i=OPTIND+1 ))
+ (( i<4 ))
+ FILE_SRC=model.pth
+ db_upload model.pth /model1.pth
++ normalize_path model.pth
+++ builtin printf model.pth
++ path=model.pth
++ [[ 1 == 1 ]]
+++ readlink -m model.pth
++ new_path=/content/model.pth
++ [[ h == \/ ]]
++ echo /content/model.pth
+ local SRC=/content/model.pth
++ normalize_path /model1.pth
+++ builtin printf /model1.pth
++ path=/model1.pth
++ [[ 1 == 1 ]]
+++ readlink -m /model1.pth
++ new_path=/model1.pth
++ [[ h == \/ ]]
++ echo /model1.pth
+ local DST=/model1.pth
+ [[ ! -e /content/model.pth ]]
+ [[ ! -r /content/model.pth ]]
++ db_stat /model1.pth
+++ normalize_path /model1.pth
++++ builtin printf /model1.pth
+++ path=/model1.pth
+++ [[ 1 == 1 ]]
++++ readlink -m /model1.pth
+++ new_path=/model1.pth
+++ [[ h == \/ ]]
+++ echo /model1.pth
++ local FILE=/model1.pth
++ [[ /model1.pth == \/ ]]
++ curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Content-Type: application/json' --data '{"path": "/model1.pth"}' https://api.dropboxapi.com/2/files/get_metadata
++ check_http_response
++ CODE=0
++ case $CODE in
++ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
+++ sed -n 's/{".tag": *"*\([^"]*\)"*.*/\1/p' /tmp/du_resp_debug
++ local 'TYPE={"error":missing_scope'
++ case $TYPE in
++ echo ERR
+ TYPE=ERR
+ [[ ERR == \F\I\L\E ]]
+ [[ ERR == \E\R\R ]]
+ [[ h != \/ ]]
+ DST=/model1.pth
+ [[ -d /content/model.pth ]]
+ [[ -e /content/model.pth ]]
+ db_upload_file /content/model.pth /model1.pth
++ normalize_path /content/model.pth
+++ builtin printf /content/model.pth
++ path=/content/model.pth
++ [[ 1 == 1 ]]
+++ readlink -m /content/model.pth
++ new_path=/content/model.pth
++ [[ h == \/ ]]
++ echo /content/model.pth
+ local FILE_SRC=/content/model.pth
++ normalize_path /model1.pth
+++ builtin printf /model1.pth
++ path=/model1.pth
++ [[ 1 == 1 ]]
+++ readlink -m /model1.pth
++ new_path=/model1.pth
++ [[ h == \/ ]]
++ echo /model1.pth
+ local FILE_DST=/model1.pth
+ shopt -s nocasematch
++ basename /model1.pth
+ basefile_dst=model1.pth
+ [[ model1.pth == \t\h\u\m\b\s\.\d\b ]]
+ [[ model1.pth == \d\e\s\k\t\o\p\.\i\n\i ]]
+ [[ model1.pth == \.\d\s\_\s\t\o\r\e ]]
+ [[ model1.pth == \i\c\o\n\\\r ]]
+ [[ model1.pth == \.\d\r\o\p\b\o\x ]]
+ [[ model1.pth == \.\d\r\o\p\b\o\x\.\a\t\t\r ]]
+ shopt -u nocasematch
++ file_size /content/model.pth
+++ stat --format=%s /content/model.pth
++ SIZE=1868275926
++ '[' 0 -eq 0 ']'
++ echo 1868275926
++ return
+ FILE_SIZE=1868275926
++ db_stat /model1.pth
+++ normalize_path /model1.pth
++++ builtin printf /model1.pth
+++ path=/model1.pth
+++ [[ 1 == 1 ]]
++++ readlink -m /model1.pth
+++ new_path=/model1.pth
+++ [[ h == \/ ]]
+++ echo /model1.pth
++ local FILE=/model1.pth
++ [[ /model1.pth == \/ ]]
++ curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Content-Type: application/json' --data '{"path": "/model1.pth"}' https://api.dropboxapi.com/2/files/get_metadata
++ check_http_response
++ CODE=0
++ case $CODE in
++ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
+++ sed -n 's/{".tag": *"*\([^"]*\)"*.*/\1/p' /tmp/du_resp_debug
++ local 'TYPE={"error":missing_scope'
++ case $TYPE in
++ echo ERR
+ TYPE=ERR
+ [[ ERR != \E\R\R ]]
+ [[ ERR != \E\R\R ]]
+ [[ 1868275926 -gt 157286000 ]]
+ db_chunked_upload_file /content/model.pth /model1.pth
++ normalize_path /content/model.pth
+++ builtin printf /content/model.pth
++ path=/content/model.pth
++ [[ 1 == 1 ]]
+++ readlink -m /content/model.pth
++ new_path=/content/model.pth
++ [[ h == \/ ]]
++ echo /content/model.pth
+ local FILE_SRC=/content/model.pth
++ normalize_path /model1.pth
+++ builtin printf /model1.pth
++ path=/model1.pth
++ [[ 1 == 1 ]]
+++ readlink -m /model1.pth
++ new_path=/model1.pth
++ [[ h == \/ ]]
++ echo /model1.pth
+ local FILE_DST=/model1.pth
+ [[ 0 == 1 ]]
+ VERBOSE=0
+ CURL_PARAMETERS='-L -s'
++ file_size /content/model.pth
+++ stat --format=%s /content/model.pth
++ SIZE=1868275926
++ '[' 0 -eq 0 ']'
++ echo 1868275926
++ return
+ local FILE_SIZE=1868275926
+ local OFFSET=0
+ local UPLOAD_ID=
+ local UPLOAD_ERROR=0
+ local CHUNK_PARAMS=
+ let 'NUMBEROFCHUNK=(1868275926/1024/1024+50-1)/50'
+ [[ 0 == 1 ]]
+ print ' > Uploading "/content/model.pth" to "/model1.pth" by 36 chunks '
+ [[ 0 == 0 ]]
+ echo -ne ' > Uploading "/content/model.pth" to "/model1.pth" by 36 chunks '
 > Uploading "/content/model.pth" to "/model1.pth" by 36 chunks + curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Dropbox-API-Arg: {"close": false}' --header 'Content-Type: application/octet-stream' --data-binary @/dev/null https://content.dropboxapi.com/2/files/upload_session/start
+ check_http_response
+ CODE=0
+ case $CODE in
+ grep -q 'HTTP/1.1 400' /tmp/du_resp_debug
++ sed -n 's/{"session_id": *"*\([^"]*\)"*.*/\1/p' /tmp/du_resp_debug
+ SESSION_ID=
+ chunkNumber=1
+ [[ 0 != \1\8\6\8\2\7\5\9\2\6 ]]
+ let OFFSET_MB=0/1024/1024
+ dd if=/content/model.pth of=/tmp/du_chunk_29499 bs=1048576 skip=0 count=50
++ file_size /tmp/du_chunk_29499
+++ stat --format=%s /tmp/du_chunk_29499
++ SIZE=52428800
++ '[' 0 -eq 0 ']'
++ echo 52428800
++ return
+ local CHUNK_REAL_SIZE=52428800
+ [[ 0 == 1 ]]
+ echo
+ curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Dropbox-API-Arg: {"cursor": {"session_id": "","offset": 0},"close": false}' --header 'Content-Type: application/octet-stream' --data-binary @/tmp/du_chunk_29499 https://content.dropboxapi.com/2/files/upload_session/append_v2
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ [[ 0 != 1 ]]
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
*+ let UPLOAD_ERROR=0+1
+ [[ 1 -gt 2 ]]
+ [[ 0 != \1\8\6\8\2\7\5\9\2\6 ]]
+ let OFFSET_MB=0/1024/1024
+ dd if=/content/model.pth of=/tmp/du_chunk_29499 bs=1048576 skip=0 count=50
++ file_size /tmp/du_chunk_29499
+++ stat --format=%s /tmp/du_chunk_29499
++ SIZE=52428800
++ '[' 0 -eq 0 ']'
++ echo 52428800
++ return
+ local CHUNK_REAL_SIZE=52428800
+ [[ 0 == 1 ]]
+ echo
+ curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Dropbox-API-Arg: {"cursor": {"session_id": "","offset": 0},"close": false}' --header 'Content-Type: application/octet-stream' --data-binary @/tmp/du_chunk_29499 https://content.dropboxapi.com/2/files/upload_session/append_v2
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ [[ 0 != 1 ]]
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
*+ let UPLOAD_ERROR=1+1
+ [[ 2 -gt 2 ]]
+ [[ 0 != \1\8\6\8\2\7\5\9\2\6 ]]
+ let OFFSET_MB=0/1024/1024
+ dd if=/content/model.pth of=/tmp/du_chunk_29499 bs=1048576 skip=0 count=50
++ file_size /tmp/du_chunk_29499
+++ stat --format=%s /tmp/du_chunk_29499
++ SIZE=52428800
++ '[' 0 -eq 0 ']'
++ echo 52428800
++ return
+ local CHUNK_REAL_SIZE=52428800
+ [[ 0 == 1 ]]
+ echo
+ curl -X POST -L -s --show-error --globoff -i -o /tmp/du_resp_debug --header 'Authorization: Bearer sl.Bw5MpHQ7OkKdP5S3OsOj63TvaEMRrbumBpDdPqfCVycmGSkjYkg5ZghKYXtivuq-wCiKByUpbXVAbtFT62Ws_0RwsuTQuvEYi_9OK4ZB6j7tDVj9jFocLFdniUk15pnUJJe3As8-WgKP' --header 'Dropbox-API-Arg: {"cursor": {"session_id": "","offset": 0},"close": false}' --header 'Content-Type: application/octet-stream' --data-binary @/tmp/du_chunk_29499 https://content.dropboxapi.com/2/files/upload_session/append_v2
+ grep -q '^HTTP/1.1 200 OK' /tmp/du_resp_debug
+ [[ 0 != 1 ]]
+ print '*'
+ [[ 0 == 0 ]]
+ echo -ne '*'
*+ let UPLOAD_ERROR=2+1
+ [[ 3 -gt 2 ]]
+ print ' FAILED\n'
+ [[ 0 == 0 ]]
+ echo -ne ' FAILED\n'
 FAILED
+ print 'An error occurred requesting /chunked_upload\n'
+ [[ 0 == 0 ]]
+ echo -ne 'An error occurred requesting /chunked_upload\n'
An error occurred requesting /chunked_upload
+ ERROR_STATUS=1
+ return
+ (( i++  ))
+ (( i<4 ))
+ remove_temp_files
+ [[ 1 == 0 ]]
+ [[ 1 -ne 0 ]]
+ echo 'Some error occured. Please check the log.'
Some error occured. Please check the log.
+ exit 1

[itobey]

You should edit your logfile and remove the Authorization header.

I have the exact same issue with my upload script - however not on all machines. I use 3 machines, each on different architectures and different locations, resulting in different ISPs. 1 of the 3 still works. You can check the debug log of the request, which is written to /tmp/du_resp_debug. For me I get a HTTP 409 conflict with the error incorrect_offset. I was able to fix the issue in the shell script, however just using the most recent script worked as well. So if you are using an older version of the upload script, please update. If the issue still persists, you can do this in the shell script to retrieve the correct offset from the debug file and modify your request. There are just a few lines added, you will easily find out which lines by comparison. Just search for the correct while, there's only one with the error message An error occurred requesting /chunked_upload

while [[ $OFFSET != "$FILE_SIZE" ]]; do

    let OFFSET_MB=$OFFSET/1024/1024

    # Create the chunk
    dd if="$FILE_SRC" of="$CHUNK_FILE" bs=1048576 skip=$OFFSET_MB count=$CHUNK_SIZE 2> /dev/null
    local CHUNK_REAL_SIZE=$(file_size "$CHUNK_FILE")

    if [[ $VERBOSE == 1 ]]; then
        print " >> Uploading chunk $chunkNumber of $NUMBEROFCHUNK\n"
    fi

    # Uploading the chunk...
    echo > "$RESPONSE_FILE"
    $CURL_BIN $CURL_ACCEPT_CERTIFICATES -X POST $CURL_PARAMETERS --show-error --globoff -i -o "$RESPONSE_FILE" --header "Authorization: Bearer $OAUTH_ACCESS_TOKEN" --header "Dropbox-API-Arg: {\"cursor\": {\"session_id\": \"$SESSION_ID\",\"offset\": $OFFSET},\"close\": false}" --header "Content-Type: application/octet-stream" --data-binary @"$CHUNK_FILE" "$API_CHUNKED_UPLOAD_APPEND_URL"

    # Check HTTP response code
    HTTP_CODE=$(awk '/^HTTP\// {print $2}' "$RESPONSE_FILE")

    if [[ $HTTP_CODE == 200 ]]; then
        let OFFSET=$OFFSET+$CHUNK_REAL_SIZE
        UPLOAD_ERROR=0
        if [[ $VERBOSE != 1 ]]; then
            print "."
        fi
        ((chunkNumber=chunkNumber+1))
    elif [[ $HTTP_CODE == 409 ]]; then
        # Parsing response file for correct offset
        CORRECT_OFFSET=$(grep -o '"correct_offset": [0-9]*' "$RESPONSE_FILE" | awk '{print $2}')
        OFFSET=$CORRECT_OFFSET
    else
        if [[ $VERBOSE != 1 ]]; then
            print "*"
        fi
        let UPLOAD_ERROR=$UPLOAD_ERROR+1
        
        # On error, the upload is retried for max 3 times
        if [[ $UPLOAD_ERROR -gt 2 ]]; then
            print " FAILED\n"
            print "An error occurred requesting /chunked_upload\n"
            ERROR_STATUS=1
            return
        fi
    fi

done