diff --git a/.gitignore b/.gitignore index 5d78da55e..44e26ca49 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ ebin/ deps/ tests/ -basho_bench +/basho_bench +/rel/basho_bench +package +.rebar *~ #*# diff --git a/Makefile b/Makefile index 1a3ccfa88..da2b87d9b 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,23 @@ -.PHONY: deps +REPO ?= basho_bench + +PKG_REVISION ?= $(shell git describe --tags) +PKG_VERSION ?= $(shell git describe --tags | tr - .) +PKG_ID = basho-bench-$(PKG_VERSION) +PKG_BUILD = 1 +BASE_DIR = $(shell pwd) +ERLANG_BIN = $(shell dirname $(shell which erl)) +REBAR ?= $(BASE_DIR)/rebar +OVERLAY_VARS ?= + all: deps compile ./rebar skip_deps=true escriptize +.PHONY: deps compile rel + +rel: deps compile + cd rel && ../rebar generate skip_deps=true $(OVERLAY_VARS) + deps: ./rebar get-deps @@ -33,3 +48,31 @@ JOBS := $(addprefix job,${TARGETS}) all_results: ${JOBS} ; echo "$@ successfully generated." ${JOBS}: job%: ; Rscript --vanilla priv/summary.r -i tests/$* + +## +## Packaging targets +## +.PHONY: package +export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE + +package.src: deps + mkdir -p package + rm -rf package/$(PKG_ID) + git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -) + ${MAKE} -C package/$(PKG_ID) deps + for dep in package/$(PKG_ID)/deps/*; do \ + echo "Processing dep: $${dep}"; \ + mkdir -p $${dep}/priv; \ + git --git-dir=$${dep}/.git describe --always --tags >$${dep}/priv/vsn.git; \ + done + find package/$(PKG_ID) -depth -name ".git" -exec rm -rf {} \; + tar -C package -czf package/$(PKG_ID).tar.gz $(PKG_ID) + +dist: package.src + cp package/$(PKG_ID).tar.gz . + +package: package.src + ${MAKE} -C package -f $(PKG_ID)/deps/node_package/Makefile + +pkgclean: distclean + rm -rf package diff --git a/pkg.vars.config b/pkg.vars.config new file mode 100644 index 000000000..0377c816e --- /dev/null +++ b/pkg.vars.config @@ -0,0 +1,24 @@ +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +%% +%% Packaging +%% +{package_name, "basho-bench"}. +{package_install_name, "basho_bench"}. +{package_install_user, "basho-bench"}. +{package_install_group, "basho-bench"}. +{package_install_user_desc, "Basho-bench user"}. +{package_shortdesc, "Basho benchmarking tool"}. +{package_desc, "Benchmarking tool"}. +{package_commands, {list, [[{name, "basho_bench"}]]}}. +{package_patch_dir, "basho-patches"}. +{bin_or_sbin, "bin"}. +{license_type, "OSS"}. +{copyright, "2014 Basho Technologies, Inc"}. +{vendor_name, "Basho Technologies, Inc"}. +{vendor_url, "http://basho.com"}. +{vendor_contact_name, "Basho Package Maintainer"}. +{vendor_contact_email, "packaging@basho.com"}. +{license_full_text, "This software is provided under license from Basho Technologies."}. +{solaris_pkgname, "BASHObasho-bench"}. diff --git a/rebar b/rebar index d8aef5789..89a27d415 100755 Binary files a/rebar and b/rebar differ diff --git a/rebar.config b/rebar.config index 8a0458214..aa2d6e181 100644 --- a/rebar.config +++ b/rebar.config @@ -1,3 +1,4 @@ +{require_otp_vsn, "R15|R16|17"}. {deps, [ @@ -8,32 +9,30 @@ %% increments. If someone wants to take advantage of a %% new folsom feature, that desire + float incr must be %% weighed. + {node_package, "2.0.*", {git, "git://github.com/basho/node_package", {tag, "2.0.0"}}}, {folsom, ".*", {git, "git://github.com/basho/folsom.git", {branch, "boundary-0.7.1+basho-bench-float"}}}, - - {lager, "2.*", {git, "git://github.com/basho/lager", {tag, "2.0.3"}}}, + {lager, "2.*", {git, "git://github.com/basho/lager", {tag, "2.1.0"}}}, {ibrowse, ".*", {git, "git://github.com/cmullaparthi/ibrowse.git", {tag, "v3.0.4"}}}, + {riakc, ".*", + {git, "git://github.com/basho/riak-erlang-client", {branch, "develop"}}}, + {mochiweb, "2.9.*", + {git, "git://github.com/basho/mochiweb", {tag, "v2.9.0"}}}, + {getopt, ".*", + {git, "git://github.com/jcomellas/getopt", {tag, "v0.8.2"}}}, + {casbench, "0.1", {git, "git://github.com/basho/casbench", "95ed55b494551577870984aeb1e0f683631a326f"}}, {erlcql, ".*", {git, "git://github.com/rpt/erlcql.git", - {branch, "master"}}}, - {riakc, ".*", - {git, "git://github.com/basho/riak-erlang-client", {branch, "develop"}}}, - {mochiweb, "1.5.1*", - {git, "git://github.com/basho/mochiweb", {tag, "1.5.1p6"}}}, - {velvet, "1.*", - {git, "git://github.com/basho/velvet", - "4bb0fd664ff065c4082ca8dd2e0683e920537d15"}}, - {getopt, ".*", - {git, "git://github.com/jcomellas/getopt", {tag, "v0.4"}}} + {branch, "master"}}} ]}. {erl_opts, [{src_dirs, [src]}, {parse_transform, lager_transform}]}. -{escript_incl_apps, [lager, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, velvet, goldrush]}. +{escript_incl_apps, [node_package, lager, getopt, bear, folsom, ibrowse, riakc, mochiweb, protobuffs, goldrush]}. {escript_emu_args, "%%! +K true -rsh ssh\n"}. %% Use this for the Java client bench driver diff --git a/rel/files/basho_bench b/rel/files/basho_bench new file mode 100755 index 000000000..c67d139b7 --- /dev/null +++ b/rel/files/basho_bench @@ -0,0 +1,9 @@ +#!/bin/sh + +# Pull environment for this install +. "{{runner_base_dir}}/lib/env.sh" + +# Make sure CWD is set to runner run dir +cd $RUNNER_BASE_DIR/lib/basho_bench*/ebin + +ERL_LIBS=$RUNNER_BASE_DIR $ERTS_PATH/escript basho_bench.beam "$@" diff --git a/rel/files/install_upgrade.escript b/rel/files/install_upgrade.escript new file mode 100644 index 000000000..56cea1963 --- /dev/null +++ b/rel/files/install_upgrade.escript @@ -0,0 +1,44 @@ +#!/usr/bin/env escript +%%! -noshell -noinput +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ft=erlang ts=4 sw=4 et + +-define(TIMEOUT, 60000). +-define(INFO(Fmt,Args), io:format(Fmt,Args)). + +main([NodeName, Cookie, ReleasePackage]) -> + TargetNode = start_distribution(NodeName, Cookie), + {ok, Vsn} = rpc:call(TargetNode, release_handler, unpack_release, + [ReleasePackage], ?TIMEOUT), + ?INFO("Unpacked Release ~p~n", [Vsn]), + {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler, + check_install_release, [Vsn], ?TIMEOUT), + {ok, OtherVsn, Desc} = rpc:call(TargetNode, release_handler, + install_release, [Vsn], ?TIMEOUT), + ?INFO("Installed Release ~p~n", [Vsn]), + ok = rpc:call(TargetNode, release_handler, make_permanent, [Vsn], ?TIMEOUT), + ?INFO("Made Release ~p Permanent~n", [Vsn]); +main(_) -> + init:stop(1). + +start_distribution(NodeName, Cookie) -> + MyNode = make_script_node(NodeName), + {ok, _Pid} = net_kernel:start([MyNode, shortnames]), + erlang:set_cookie(node(), list_to_atom(Cookie)), + TargetNode = make_target_node(NodeName), + case {net_kernel:hidden_connect_node(TargetNode), + net_adm:ping(TargetNode)} of + {true, pong} -> + ok; + {_, pang} -> + io:format("Node ~p not responding to pings.\n", [TargetNode]), + init:stop(1) + end, + TargetNode. + +make_target_node(Node) -> + [_, Host] = string:tokens(atom_to_list(node()), "@"), + list_to_atom(lists:concat([Node, "@", Host])). + +make_script_node(Node) -> + list_to_atom(lists:concat([Node, "_upgrader_", os:getpid()])). diff --git a/rel/files/vm.args b/rel/files/vm.args new file mode 100644 index 000000000..cce3b816f --- /dev/null +++ b/rel/files/vm.args @@ -0,0 +1,19 @@ +## Name of the node +-name healthb@127.0.0.1 + +## Cookie for distributed erlang +-setcookie healthb + +## Heartbeat management; auto-restarts VM if it dies or becomes unresponsive +## (Disabled by default..use with caution!) +##-heart + +## Enable kernel poll and a few async threads +##+K true +##+A 5 + +## Increase number of concurrent ports/sockets +##-env ERL_MAX_PORTS 4096 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/rel/reltool.config b/rel/reltool.config new file mode 100644 index 000000000..e60cfc75f --- /dev/null +++ b/rel/reltool.config @@ -0,0 +1,63 @@ +%% -*- mode: erlang -*- +%% ex: ft=erlang +{sys, [ + {lib_dirs, ["../deps"]}, + {erts, [{mod_cond, derived}, {app_file, strip}]}, + {app_file, strip}, + {rel, "basho_bench", "0.10.0", + [ + kernel, + stdlib, + bear, + lager, + folsom, + goldrush, + riakc, + ibrowse, + mochiweb + ]}, + {rel, "start_clean", "", + [ + kernel, + stdlib + ]}, + {boot_rel, "basho_bench"}, + {profile, embedded}, + {incl_cond, derived}, + {excl_archive_filters, [".*"]}, %% Do not archive built libs + {excl_sys_filters, ["^bin/(?!start_clean.boot)", + "^erts.*/bin/(dialyzer|typer)", + "^erts.*/(doc|info|include|lib|man|src)"]}, + {excl_app_filters, ["\.gitignore"]}, + {app, basho_bench, [{mod_cond, app}, {incl_cond, include}, {lib_dir, ".."}]}, + {app, hipe, [{incl_cond, exclude}]} + ]}. + +{target_dir, "basho_bench"}. +{overlay_vars, "vars.config"}. + +{overlay, [ + {template, "../deps/node_package/priv/base/env.sh", + "lib/env.sh"}, + {mkdir, "data/b_b"}, + + %% Copy base files for starting and interacting w/ node + {copy, "../deps/node_package/priv/base/erl", + "{{erts_vsn}}/bin/erl"}, + {copy, "../deps/node_package/priv/base/nodetool", + "{{erts_vsn}}/bin/nodetool"}, + {template, "../deps/node_package/priv/base/env.sh", + "lib/env.sh"}, + {copy, "files/vm.args", "etc/vm.args"}, + + {template, "files/basho_bench", "bin/basho_bench"}, + + {copy, "../examples/cs.config.sample", "etc/cs.config"}, + {copy, "../examples/riakc_pb.config", "etc/riakc_pb.config"}, + {copy, "../examples/httpraw.config", "etc/httpraw.config"}, + {copy, "../examples/http.config", "etc/http.config"}, + {copy, "../examples/null_test.config", "etc/null_test.config"} + + %%{copy, "files/install_upgrade.escript", "bin/install_upgrade.escript"}, + + ]}. diff --git a/rel/vars.config b/rel/vars.config new file mode 100644 index 000000000..00ff42b68 --- /dev/null +++ b/rel/vars.config @@ -0,0 +1,20 @@ +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +%% Platform-specific installation paths +{platform_bin_dir, "./bin"}. +{platform_data_dir, "./data"}. +{platform_etc_dir, "./etc"}. +{platform_lib_dir, "./lib"}. +{platform_log_dir, "./log"}. + +%% +{data_dir, "{{target_dir}}/data"}. +{runner_script_dir, "\`cd \\`dirname $0\\` && /bin/pwd\`"}. +{runner_base_dir, "{{runner_script_dir}}/.."}. +{runner_etc_dir, "$RUNNER_BASE_DIR/etc"}. +{runner_log_dir, "$RUNNER_BASE_DIR/log"}. +{runner_lib_dir, "$RUNNER_BASE_DIR/lib"}. +{runner_patch_dir, "$RUNNER_BASE_DIR/lib/basho-patches"}. +{pipe_dir, "/tmp/$RUNNER_BASE_DIR/"}. +{runner_user, ""}. diff --git a/src/basho_bench.app.src b/src/basho_bench.app.src index ee35ab0d5..1c3c74974 100644 --- a/src/basho_bench.app.src +++ b/src/basho_bench.app.src @@ -1,6 +1,6 @@ {application, basho_bench, [{description, "Riak Benchmarking Suite"}, - {vsn, "0.9"}, + {vsn, git}, {modules, []}, {registered, [ basho_bench_sup ]}, {applications, [kernel, diff --git a/src/basho_bench.erl b/src/basho_bench.erl index 38391bab9..0609418fc 100644 --- a/src/basho_bench.erl +++ b/src/basho_bench.erl @@ -307,7 +307,7 @@ setup_distributed_work() -> [pool:attach(SlaveName) || SlaveName <- SlaveNames], CodePaths = code:get_path(), rpc:multicall(SlaveNames, code, set_path, [CodePaths]), - Apps = [lager, basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, velvet, goldrush], + Apps = [lager, basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, goldrush], [distribute_app(App) || App <- Apps]. diff --git a/src/basho_bench_driver_cs.erl b/src/basho_bench_driver_cs.erl index b4c20f1a9..10a94927b 100644 --- a/src/basho_bench_driver_cs.erl +++ b/src/basho_bench_driver_cs.erl @@ -569,11 +569,15 @@ auth_sig(AccessKey, SecretKey, Method, ContentType, Date, Headers, Resource) -> CanonizedAmzHeaders, Resource ], - Signature = base64:encode(stanchion_utils:sha_mac(SecretKey, StringToSign)), + Signature = base64:encode(sha_mac(SecretKey, StringToSign)), ["AWS ", AccessKey, $:, Signature]. %% CS utilities +%% @doc copied from stanchion_utils +sha_mac(KeyData, STS) -> + crypto:hmac(sha, KeyData, STS). + setup_user_and_bucket(State) -> case basho_bench_config:get(cs_access_key, undefined) of undefined ->