diff --git a/.gitignore b/.gitignore index e505080ec..6df5e8066 100644 --- a/.gitignore +++ b/.gitignore @@ -11,7 +11,6 @@ log/ *~ current_counterexample.eqc .client_test/ -pkg.vars.config client_tests/python/ceph_tests/s3-tests riak-cs.png .local_dialyzer_plt diff --git a/Makefile b/Makefile index 74d95df8a..641dcabd7 100644 --- a/Makefile +++ b/Makefile @@ -166,20 +166,11 @@ PLT ?= $(HOME)/.riak-cs_dialyzer_plt .PHONY: package export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE -## Do not export RIAK_CS_EE_DEPS unless it is set, since even an empty -## variable will affect the build and 'export' by default makes it empty -## if it is unset -BUILD_EE = $(shell test -n "$${RIAK_CS_EE_DEPS+x}" && echo "true" || echo "false") -ifeq ($(BUILD_EE),true) -export RIAK_CS_EE_DEPS=true -endif - - package.src: deps mkdir -p package rm -rf package/$(PKG_ID) git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -) - cp rebar.config.script package/$(PKG_ID) + cp pkg.vars.config package/$(PKG_ID) make -C package/$(PKG_ID) deps mkdir -p package/$(PKG_ID)/priv git --git-dir=.git describe --tags >package/$(PKG_ID)/priv/vsn.git diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index bebe2422e..7ea8ff1a8 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,6 +1,6 @@ -#Riak CS 2.1.2 Release Notes +#Riak CS 2.1.2 Release Candidate Notes -Released March 31, 2016. +Released February 24, 2019. This is a backwards-compatible* release that updates node_package to address a recent [Product Advisory](http://docs.basho.com/riak/latest/community/product-advisories/codeinjectioninitfiles/), as well as fixes several bugs. @@ -15,7 +15,7 @@ Riak CS 2.1 is designed to work with Riak KV 2.1.1+. During the update to 2.1.2, a '==' omitted upload ID might be passed to a Riak CS node running an older version of CS. This may lead to process-crash by failing on decoding upload ID. ##Changes - +* For s3cmd users, experimental signature_v4 support has been made available through a simple on/off toggle in riak-cs.conf. With a default setting of "off", it allows in-situ upgrades without the need to change s3cfg files until after all nodes have been upgraded. Note: this function is currently unfinished and suffers from compatibility issues with some clients ([#1058](https://github.com/basho/riak_cs/issues/1058) / [#1060](https://github.com/basho/riak_cs/issues/1060)) and one potential security issue ([#1059](https://github.com/basho/riak_cs/issues/1059) * Due to a recent [Product Advisory](http://docs.basho.com/riak/latest/community/product-advisories/codeinjectioninitfiles/), node_package was bumped to version 3.0.0 to prevent a potential code injection on the riak init file. [[Issue 1297](https://github.com/basho/riak_cs/issues/1297), [PR 1306](https://github.com/basho/riak_cs/pull/1306), & [PR 109](https://github.com/basho/stanchion/pull/109)] * Multipart upload IDs no longer contain trailing '=' characters, which caused trouble for some clients. This change also makes upload IDs URL-safe. [[PR 1316](https://github.com/basho/riak_cs/pull/1316)] * When Riak is unavailable due to network partition or node being offline, a 500 error is returned. [[PR 1298](https://github.com/basho/riak_cs/pull/1298)] diff --git a/dialyzer.ignore-warnings.ee b/dialyzer.ignore-warnings similarity index 100% rename from dialyzer.ignore-warnings.ee rename to dialyzer.ignore-warnings diff --git a/dialyzer.ignore-warnings.oss b/dialyzer.ignore-warnings.oss deleted file mode 100644 index 6437a1c85..000000000 --- a/dialyzer.ignore-warnings.oss +++ /dev/null @@ -1,10 +0,0 @@ -# Warnings -Unknown functions: - riak_cs_multibag:choose_bag_id/1 - riak_cs_multibag:list_pool/0 - riak_cs_multibag:list_pool/1 - riak_cs_multibag:pool_name_for_bag/2 - riak_cs_multibag:pool_specs/1 - riak_cs_multibag:process_specs/0 - riak_repl_pb_api:get/5 - riak_repl_pb_api:get_clusterid/2 diff --git a/misc/mapdeps.erl b/misc/mapdeps.erl index d7ab9ee61..28381c9c5 100755 --- a/misc/mapdeps.erl +++ b/misc/mapdeps.erl @@ -74,8 +74,7 @@ map_dir(BaseDir) -> map_rebar(BaseDir, Path, Acc) -> case file:consult(Path) of {ok, Opts} -> - Deps = proplists:get_value(deps, Opts, []) ++ - proplists:get_value(deps_ee, Opts, []), + Deps = proplists:get_value(deps, Opts, []), lists:foldl( fun({DepName, _, _}, A) -> From = app_name(Path), diff --git a/pkg.vars.config b/pkg.vars.config new file mode 100644 index 000000000..9e922e8b8 --- /dev/null +++ b/pkg.vars.config @@ -0,0 +1,23 @@ +%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- + +%% +%% Packaging +%% +{package_name, "riak-cs"}. +{package_install_name, "riak-cs"}. +{package_install_user, "riakcs"}. +{package_install_group, "riak"}. +{package_install_user_desc, "Riak CS user"}. +{package_commands, {list, [[{name, "riak-cs"}], [{name, "riak-cs-access"}], [{name, "riak-cs-gc"}], [{name, "riak-cs-storage"}], [{name, "riak-cs-stanchion"}], [{name, "riak-cs-debug"}], [{name, "riak-cs-admin"}], [{name, "riak-cs-supercluster"}], [{name, "riak-cs-multibag"}]]}}. +{package_shortdesc, "Riak CS"}. +{package_patch_dir, "basho-patches"}. +{package_desc, "Riak CS"}. +{bin_or_sbin, "sbin"}. +{license_type, "Apache License, Version 2.0"}. +{copyright, "2013 Basho Technologies, Inc"}. +{vendor_name, "Basho Technologies, Inc"}. +{vendor_url, "http://basho.com"}. +{vendor_contact_name, "Basho Package Maintainer"}. +{vendor_contact_email, "packaging@basho.com"}. +{license_full_text, "This software is provided under license from Basho Technologies."}. +{solaris_pkgname, "BASHOriak-cs"}. \ No newline at end of file diff --git a/rebar.config b/rebar.config index 79f129fb1..32aab87f7 100644 --- a/rebar.config +++ b/rebar.config @@ -16,8 +16,6 @@ {xref_checks, []}. {xref_queries, - [{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$|^riak_repl_pb_api$|^riak_cs_multibag$)\" : Mod)", []}]}. -{xref_queries_ee, [{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$)\" : Mod)", []}]}. {reset_after_eunit, true}. @@ -51,10 +49,7 @@ {cluster_info, ".*", {git, "git://github.com/basho/cluster_info", {tag, "2.0.3"}}}, {xmerl, ".*", {git, "git://github.com/shino/xmerl", "1b016a05473e086abadbb3c12f63d167fe96c00f"}}, {erlcloud, ".*", {git, "git://github.com/basho/erlcloud.git", {tag, "0.4.6"}}}, - {rebar_lock_deps_plugin, ".*", {git, "git://github.com/seth/rebar_lock_deps_plugin.git", {tag, "3.1.0"}}} - ]}. - -{deps_ee, [ - {riak_repl_pb_api,".*",{git,"git@github.com:basho/riak_repl_pb_api.git", {tag, "2.1.1"}}}, - {riak_cs_multibag,".*",{git,"git@github.com:basho/riak_cs_multibag.git", {tag, "2.1.0p1"}}} + {rebar_lock_deps_plugin, ".*", {git, "git://github.com/seth/rebar_lock_deps_plugin.git", {tag, "3.1.0"}}}, + {riak_repl_pb_api,".*",{git,"git://github.com/basho/riak_repl_pb_api.git", {tag, "2.1.1"}}}, + {riak_cs_multibag,".*",{git,"git://github.com/basho/riak_cs_multibag.git", {tag, "2.1.0p1"}}} ]}. diff --git a/rebar.config.lock b/rebar.config.lock deleted file mode 100644 index ac76860b6..000000000 --- a/rebar.config.lock +++ /dev/null @@ -1,115 +0,0 @@ -%% THIS FILE IS GENERATED. DO NOT EDIT IT MANUALLY %% - -{sub_dirs,["rel"]}. -{require_otp_vsn,"R16|17"}. -{cover_enabled,false}. -{edoc_opts,[preprocess]}. -{lib_dirs,["deps","apps"]}. -{erl_opts,[debug_info,warnings_as_errors, - {parse_transform,lager_transform}, - {platform_define,"^[0-9]+",namespaced_types}]}. -{xref_checks,[]}. -{xref_queries,[{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$|^riak_repl_pb_api$|^riak_cs_multibag$)\" : Mod)", - []}]}. -{xref_queries_ee,[{"(XC - UC) || (XU - X - B - \"(^riak$|^riak_cs_dummy_reader$|^riak_core_bucket$|^app_helper$|^riakc_pb_socket_fake$|^riak_object$)\" : Mod)", - []}]}. -{reset_after_eunit,true}. -{plugin_dir,".plugins"}. -{plugins,[rebar_test_plugin,rebar_lock_deps_plugin]}. -{client_test,[{test_paths,["client_tests/erlang"]}, - {test_output,".client_test"}]}. -{riak_test,[{test_paths,["riak_test/tests","riak_test/src", - "deps/riak_cs_multibag/riak_test/tests", - "deps/riak_cs_multibag/riak_test/src"]}, - {test_output,"riak_test/ebin"}]}. -{deps,[{goldrush,".*", - {git,"git://github.com/DeadZen/goldrush.git", - "64864ba7fcf40988361340e48680b49a2c2938cf"}}, - {lager,".*", - {git,"git://github.com/basho/lager", - "f65dde85523b6aa6a259db1c93518187ae366515"}}, - {syslog,".*", - {git,"git://github.com/Vagabond/erlang-syslog", - "e24c9ee8f7bb3f066ec152c210af10c2c712759a"}}, - {lager_syslog,".*", - {git,"git://github.com/basho/lager_syslog", - "399a90d9c77afa66590b1a42fcf76035a304c229"}}, - {getopt,".*", - {git,"git://github.com/jcomellas/getopt.git", - "388dc95caa7fb97ec7db8cfc39246a36aba61bd8"}}, - {neotoma,".*", - {git,"git://github.com/seancribbs/neotoma.git", - "760928ec8870da02eb11bccb501e2700925d06c6"}}, - {cuttlefish,".*", - {git,"git://github.com/basho/cuttlefish.git", - "7ce217c3effb9990bb1aff34621013222c9977ec"}}, - {node_package,".*", - {git,"git://github.com/basho/node_package", - "c630bcb25de53d542b167a0891c32f9e47602d6a"}}, - {mochiweb,".*", - {git,"git://github.com/basho/mochiweb.git", - "ade2a9b29a11034eb550c1d79b4f991bf5ca05ba"}}, - {webmachine,".*", - {git,"git://github.com/basho/webmachine", - "7677c240f4a7ed020f4bab48278224966bb42311"}}, - {meck,".*", - {git,"git://github.com/basho/meck.git", - "dde759050eff19a1a80fd854d7375174b191665d"}}, - {protobuffs,".*", - {git,"git://github.com/basho/erlang_protobuffs.git", - "f88fc3c6881687432ddd5546b3c7b08009dfb26f"}}, - {riak_pb,".*", - {git,"git://github.com/basho/riak_pb", - "620bc7001dc788e5530078aa8be53c9d15d4fdb4"}}, - {riakc,".*", - {git,"git://github.com/basho/riak-erlang-client", - "1c75e31fd50e065aeebe787d608ce9b7ddeddb38"}}, - {eper,".*", - {git,"git://github.com/basho/eper.git", - "dd0a275b9e66d8e02e48bf6353b2f4881e55c1db"}}, - {druuid,".*", - {git,"git://github.com/kellymclaughlin/druuid.git", - "b3c5c2a52bb3f510d168b32e64f6fbc6a3c6a0e6"}}, - {poolboy,".*", - {git,"git://github.com/basho/poolboy", - "8bb45fbc715c5f493642a1cc572ec7017d0d5fa3"}}, - {edown,".*", - {git,"git://github.com/uwiger/edown.git", - "d62ec85281e451a46ba30045917c119d65b72a84"}}, - {parse_trans,".*", - {git,"git://github.com/uwiger/parse_trans.git", - "82cc00264aa1bad8fc5c0739b7541feb4a843432"}}, - {bear,".*", - {git,"git://github.com/boundary/bear.git", - "119234548783af19b8ec75c879c5062676b92571"}}, - {folsom,".*", - {git,"git://github.com/boundary/folsom", - "38e2cce7c64ce1f0a3a918d90394cfc0a940b1ba"}}, - {setup,".*", - {git,"git://github.com/uwiger/setup.git", - "51ee7c9f64d2bbe9dcbb58c278e8fbfd4d0ca5e2"}}, - {exometer_core,".*", - {git,"git://github.com/Feuerlabs/exometer_core", - "88588f26f226210a1fc9e70271d8a0611ba83d30"}}, - {cluster_info,".*", - {git,"git://github.com/basho/cluster_info", - "76c73fcac58c32f1603f8a1b98d7c441da92fa91"}}, - {xmerl,".*", - {git,"git://github.com/shino/xmerl", - "1b016a05473e086abadbb3c12f63d167fe96c00f"}}, - {ibrowse,".*", - {git,"git://github.com/cmullaparthi/ibrowse", - "e8ae353c16d4f0897abb9f80025b52925b974dd1"}}, - {erlcloud,".*", - {git,"git://github.com/basho/erlcloud.git", - "c2336913ff7525b58ce89a5c0d1fd0db171aa71d"}}, - {rebar_lock_deps_plugin,".*", - {git,"git://github.com/seth/rebar_lock_deps_plugin.git", - "9711549b8a84b065eb2edc22f8eb6ff85e3c94e8"}}]}. -{deps_ee,[{riak_repl_pb_api,".*", - {git,"git@github.com:basho/riak_repl_pb_api.git", - {tag,"2.1.1"}}}, - {riak_cs_multibag,".*", - {git,"git@github.com:basho/riak_cs_multibag.git", - {tag,"2.1.0p1"}}}]}. - diff --git a/rebar.config.script b/rebar.config.script deleted file mode 100644 index 74706a568..000000000 --- a/rebar.config.script +++ /dev/null @@ -1,96 +0,0 @@ -{IsEE, Package} = case os:getenv("RIAK_CS_EE_DEPS") of - false -> {false, "riak-cs"}; - _ -> {true, "riak-cs-ee"} - end, -io:format("Building ~s~n", [Package]), - -DepsUpdated = - case IsEE of - false -> - CONFIG; - true -> - {value, {deps, Deps}} = lists:keysearch(deps, 1, CONFIG), - {value, {deps_ee, DepsEE}} = lists:keysearch(deps_ee, 1, CONFIG), - lists:keyreplace(deps, 1, CONFIG, {deps, Deps ++ DepsEE}) - end, - -XrefQueryUpdated = - case IsEE of - false -> - DepsUpdated; - true -> - {value, {xref_queries_ee, QueriesEE}} = - lists:keysearch(xref_queries_ee, 1, DepsUpdated), - lists:keyreplace(xref_queries, 1, DepsUpdated, {xref_queries, QueriesEE}) - end, - -FinalConfig = XrefQueryUpdated, - -%% Write dialyzer.ignore_warnings - -{ok, _} = file:copy("dialyzer.ignore-warnings.ee", "dialyzer.ignore-warnings"), -case IsEE of - false -> - {ok, _} = file:copy("dialyzer.ignore-warnings.oss", - {"dialyzer.ignore-warnings", [append]}); - true -> ok -end, - -%% Write pkg.vars.config -case IsEE of - false -> - Bytes = "%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et - -%% -%% Packaging -%% -{package_name, \"riak-cs\"}. -{package_install_name, \"riak-cs\"}. -{package_install_user, \"riakcs\"}. -{package_install_group, \"riak\"}. -{package_install_user_desc, \"Riak CS user\"}. -{package_commands, {list, [[{name, \"riak-cs\"}], [{name, \"riak-cs-access\"}], [{name, \"riak-cs-gc\"}], [{name, \"riak-cs-storage\"}], [{name, \"riak-cs-stanchion\"}], [{name, \"riak-cs-debug\"}], [{name, \"riak-cs-admin\"}]]}}. -{package_shortdesc, \"Riak CS\"}. -{package_patch_dir, \"basho-patches\"}. -{package_desc, \"Riak CS\"}. -{bin_or_sbin, \"sbin\"}. -{license_type, \"Apache License, Version 2.0\"}. -{copyright, \"2013 Basho Technologies, Inc\"}. -{vendor_name, \"Basho Technologies, Inc\"}. -{vendor_url, \"http://basho.com\"}. -{vendor_contact_name, \"Basho Package Maintainer\"}. -{vendor_contact_email, \"packaging@basho.com\"}. -{license_full_text, \"This software is provided under license from Basho Technologies.\"}. -{solaris_pkgname, \"BASHOriak-cs\"}.", - file:write_file("pkg.vars.config", Bytes); - true -> - Bytes = "%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*- -%% ex: ts=4 sw=4 et - -%% -%% Packaging -%% -{package_name, \"riak-cs-ee\"}. -{package_install_name, \"riak-cs\"}. -{package_install_user, \"riakcs\"}. -{package_install_group, \"riak\"}. -{package_install_user_desc, \"Riak CS user\"}. -{package_commands, {list, [[{name, \"riak-cs\"}], [{name, \"riak-cs-access\"}], [{name, \"riak-cs-gc\"}], [{name, \"riak-cs-storage\"}], [{name, \"riak-cs-stanchion\"}], [{name, \"riak-cs-debug\"}], [{name, \"riak-cs-admin\"}], [{name, \"riak-cs-supercluster\"}], [{name, \"riak-cs-multibag\"}]]}}. -{package_shortdesc, \"Riak CS\"}. -{package_patch_dir, \"basho-patches\"}. -{package_desc, \"Riak CS\"}. -{bin_or_sbin, \"sbin\"}. -{license_type, \"Proprietary\"}. -{copyright, \"2013 Basho Technologies, Inc\"}. -{vendor_name, \"Basho Technologies, Inc\"}. -{vendor_url, \"http://basho.com\"}. -{vendor_contact_name, \"Basho Package Maintainer\"}. -{vendor_contact_email, \"packaging@basho.com\"}. -{license_full_text, \"This software is provided under license from Basho Technologies.\"}. -{solaris_pkgname, \"BASHOriak-cs-ee\"}. -{debuild_extra_options, \"-e RIAK_CS_EE_DEPS=true\"}.", - file:write_file("pkg.vars.config", Bytes) -end, - -FinalConfig. diff --git a/rel/files/riak_cs.schema b/rel/files/riak_cs.schema index 401da7003..3c3a28509 100644 --- a/rel/files/riak_cs.schema +++ b/rel/files/riak_cs.schema @@ -86,6 +86,17 @@ hidden ]}. +%% @doc Enable experimental signature_v4 compatibility. +%% Changing this setting to on will allow s3cmd to utilise +%% signature_v4 and thus function without the need to manually add +%% v2_signature support to your .s3cfg file, +%% Note: this function is unfinished and suffers from issues: +%% #1058, #1059, #1060. Use at your own risk. +{mapping, "auth_v4", "riak_cs.auth_v4_enabled", [ + {default, off}, + {datatype, flag} +]}. + %% @doc Root host name which Riak CS accepts. %% Your CS bucket at s3.example.com will be accessible %% via URL like http://bucket.s3.example.com/object/name diff --git a/riak_test/tests/repl_v3_test.erl b/riak_test/tests/repl_v3_test.erl index ec9906ce8..798b8abe6 100644 --- a/riak_test/tests/repl_v3_test.erl +++ b/riak_test/tests/repl_v3_test.erl @@ -26,16 +26,16 @@ -define(TEST_BUCKET, "riak-test-bucket"). -confirm() -> - case rt_config:get(build_type, oss) of - ee -> - confirm_ee(); - _ -> - lager:info("~s test is only valid on riak_ee, skipping", [?MODULE]), - pass - end. - -confirm_ee() -> +confirm() + ->case rt_config:get(build_type, oss) of + ee -> + confirm_ee(); + _ -> + lager:info("~s test is only valid on riak_ee, skipping", [?MODULE]), + pass + end. + +confirm_ee() -> {UserConfig, {RiakNodes, _CSNodes, _Stanchion}} = rtcs:setup2x2(), lager:info("UserConfig = ~p", [UserConfig]), [A,B,C,D] = RiakNodes,