Skip to content
This repository has been archived by the owner on Jul 24, 2024. It is now read-only.

storage/: handle special character in URL #614

Merged
merged 7 commits into from
Nov 25, 2020
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/storage/parse.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ func ParseBackend(rawURL string, options *BackendOptions) (*backup.StorageBacken
return nil, errors.Annotate(berrors.ErrStorageInvalidConfig, "empty store is not allowed")
}

rawURL = strings.ReplaceAll(rawURL, "+", "%2B")
kennytm marked this conversation as resolved.
Show resolved Hide resolved
u, err := url.Parse(rawURL)
if err != nil {
return nil, errors.Trace(err)
Expand Down
119 changes: 64 additions & 55 deletions tests/br_s3/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ TABLE="usertable"
DB_COUNT=3

# start the s3 server
export MINIO_ACCESS_KEY=brs3accesskey
export MINIO_SECRET_KEY=brs3secretkey
export MINIO_ACCESS_KEY='KEXI7MANNASOPDLAOIEF'
export MINIO_SECRET_KEY='MaKYxEGDInMPtEYECXRJLU+FPNKb/wAX/MElir7E'
export MINIO_BROWSER=off
export AWS_ACCESS_KEY_ID=$MINIO_ACCESS_KEY
export AWS_SECRET_ACCESS_KEY=$MINIO_SECRET_KEY
Expand All @@ -44,73 +44,82 @@ stop_minio() {
}
trap stop_minio EXIT

s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://mybucket

# Fill in the database
for i in $(seq $DB_COUNT); do
run_sql "CREATE DATABASE $DB${i};"
go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB${i}
done
S3_KEY=""
for p in $(seq 2); do
s3cmd --access_key=$MINIO_ACCESS_KEY --secret_key=$MINIO_SECRET_KEY --host=$S3_ENDPOINT --host-bucket=$S3_ENDPOINT --no-ssl mb s3://mybucket

for i in $(seq $DB_COUNT); do
row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
done
for i in $(seq $DB_COUNT); do
row_count_ori[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
done

# backup full
echo "backup start..."
BACKUP_LOG="backup.log"
rm -f $BACKUP_LOG
unset BR_LOG_TO_TERM
run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT" \
--log-file $BACKUP_LOG || \
( cat $BACKUP_LOG && BR_LOG_TO_TERM=1 && exit 1 )
cat $BACKUP_LOG
BR_LOG_TO_TERM=1
# backup full
echo "backup start..."
BACKUP_LOG="backup.log"
rm -f $BACKUP_LOG
unset BR_LOG_TO_TERM
run_br --pd $PD_ADDR backup full -s "s3://mybucket/$DB?endpoint=http://$S3_ENDPOINT$S3_KEY" \
--log-file $BACKUP_LOG || \
( cat $BACKUP_LOG && BR_LOG_TO_TERM=1 && exit 1 )
cat $BACKUP_LOG
BR_LOG_TO_TERM=1

if grep -i $MINIO_SECRET_KEY $BACKUP_LOG; then
echo "Secret key logged in log. Please remove them."
exit 1
fi
if grep -i $MINIO_SECRET_KEY $BACKUP_LOG; then
echo "Secret key logged in log. Please remove them."
exit 1
fi

for i in $(seq $DB_COUNT); do
run_sql "DROP DATABASE $DB${i};"
done
for i in $(seq $DB_COUNT); do
run_sql "DROP DATABASE $DB${i};"
done

# restore full
echo "restore start..."
RESTORE_LOG="restore.log"
rm -f $RESTORE_LOG
unset BR_LOG_TO_TERM
run_br restore full -s "s3://mybucket/$DB" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
--log-file $RESTORE_LOG || \
( cat $RESTORE_LOG && BR_LOG_TO_TERM=1 && exit 1 )
cat $RESTORE_LOG
BR_LOG_TO_TERM=1
# restore full
echo "restore start..."
RESTORE_LOG="restore.log"
rm -f $RESTORE_LOG
unset BR_LOG_TO_TERM
run_br restore full -s "s3://mybucket/$DB?$S3_KEY" --pd $PD_ADDR --s3.endpoint="http://$S3_ENDPOINT" \
--log-file $RESTORE_LOG || \
( cat $RESTORE_LOG && BR_LOG_TO_TERM=1 && exit 1 )
cat $RESTORE_LOG
BR_LOG_TO_TERM=1

if grep -i $MINIO_SECRET_KEY $RESTORE_LOG; then
echo "Secret key logged in log. Please remove them."
exit 1
fi
if grep -i $MINIO_SECRET_KEY $RESTORE_LOG; then
echo "Secret key logged in log. Please remove them."
exit 1
fi

for i in $(seq $DB_COUNT); do
row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
done
for i in $(seq $DB_COUNT); do
row_count_new[${i}]=$(run_sql "SELECT COUNT(*) FROM $DB${i}.$TABLE;" | awk '/COUNT/{print $2}')
done

fail=false
for i in $(seq $DB_COUNT); do
if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
fail=true
echo "TEST: [$TEST_NAME] fail on database $DB${i}"
fi
echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}"
done
fail=false
for i in $(seq $DB_COUNT); do
if [ "${row_count_ori[i]}" != "${row_count_new[i]}" ];then
fail=true
echo "TEST: [$TEST_NAME] fail on database $DB${i}"
fi
echo "database $DB${i} [original] row count: ${row_count_ori[i]}, [after br] row count: ${row_count_new[i]}"
done

if $fail; then
echo "TEST: [$TEST_NAME] failed!"
exit 1
else
echo "TEST: [$TEST_NAME] successed!"
fi

if $fail; then
echo "TEST: [$TEST_NAME] failed!"
exit 1
else
echo "TEST: [$TEST_NAME] successed!"
fi
# prepare for next test
S3_KEY="&access-key=$MINIO_ACCESS_KEY&secret-access-key=$MINIO_SECRET_KEY"
export AWS_ACCESS_KEY_ID=""
export AWS_SECRET_ACCESS_KEY=""
rm -rf "$TEST_DIR/$DB"
mkdir -p "$TEST_DIR/$DB"
done

for i in $(seq $DB_COUNT); do
run_sql "DROP DATABASE $DB${i};"
Expand Down