From 5525ea801006a530bc7c4a900843249f0dc8f331 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 10:07:35 -0800 Subject: [PATCH 01/12] Getting some build info changed --- Makefile | 8 +- configure.ac | 1 + java/Makefile | 22 +- java/pom.xml | 2 +- java/src/main/bin/build.sh | 171 ------------ java/src/main/bin/release.sh | 60 ----- java/src/main/java/vowpalWabbit/VW.java | 4 + .../java/vowpalWabbit/jni/NativeUtils.java | 251 ------------------ .../java/vowpalWabbit/learner/VWLearners.java | 49 +--- vowpalwabbit/Makefile | 4 +- 10 files changed, 29 insertions(+), 543 deletions(-) delete mode 100755 java/src/main/bin/build.sh delete mode 100755 java/src/main/bin/release.sh delete mode 100644 java/src/main/java/vowpalWabbit/jni/NativeUtils.java diff --git a/Makefile b/Makefile index d3f0c0b9d1f..ad61c2c3fb4 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ MANPAGES = vw.1 default: vw -all: vw library_example java spanning_tree +all: vw library_example java spanning_tree python %.1: % help2man --no-info --name="Vowpal Wabbit -- fast online learning tool" ./$< > $@ @@ -117,10 +117,8 @@ library_example_gcov: vw_gcov python: vw cd python; $(MAKE) things -ifneq ($(JAVA_HOME),) java: vw cd java; $(MAKE) things -endif .FORCE: @@ -136,7 +134,7 @@ bigtests: .FORCE vw (cd big_tests && $(MAKE) $(MAKEFLAGS)) install: $(BINARIES) - cd vowpalwabbit; cp $(BINARIES) /usr/local/bin; cd ../cluster; $(MAKE) install + cd vowpalwabbit; cp $(BINARIES) /usr/local/bin; cd ../cluster; $(MAKE) install; cd ../java; $(MAKE) install; doc: (cd doc && doxygen Doxyfile) @@ -146,8 +144,6 @@ clean: cd cluster && $(MAKE) clean cd library && $(MAKE) clean cd python && $(MAKE) clean -ifneq ($(JAVA_HOME),) cd java && $(MAKE) clean -endif .PHONY: all clean install doc diff --git a/configure.ac b/configure.ac index 64e9042433a..ed047860e28 100644 --- a/configure.ac +++ b/configure.ac @@ -78,6 +78,7 @@ AC_CONFIG_FILES([ vowpalwabbit/Makefile cluster/Makefile library/Makefile + java/Makefile libvw.pc libvw_c_wrapper.pc ]) diff --git a/java/Makefile b/java/Makefile index f8545195eac..1603c59a05d 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,23 +1,26 @@ -ifeq ($(JAVA_HOME),) - $(warning No JAVA_HOME found, JNI building will fail. Please set JAVA_HOME when using JNI) - exit 1 -endif - VWLIBS := -L../vowpalwabbit -l vw STDLIBS = $(BOOST_LIBRARY) $(LIBS) JAVA_INCLUDE = -I $(JAVA_HOME)/include ifeq ($(UNAME), Linux) JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux + SHARED_LIBRARY = libvw_jni.so + INSTALL_PATH = /usr/lib endif ifeq ($(UNAME), FreeBSD) JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux + SHARED_LIBRARY = libvw_jni.so + INSTALL_PATH = /usr/lib endif ifeq "CYGWIN" "$(findstring CYGWIN,$(UNAME))" JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux + SHARED_LIBRARY = libvw_jni.so + INSTALL_PATH = /usr/lib endif ifeq ($(UNAME), Darwin) JAVA_INCLUDE += -I $(JAVA_HOME)/include/darwin + SHARED_LIBRARY = libvw_jni.dylib + INSTALL_PATH = /Library/Java/Extensions endif jni_SRCS = $(shell find src/main/c++ -name "*.cc") @@ -29,14 +32,14 @@ all: test: cd ..; $(MAKE) test -things: pom_version target/vw_jni.lib +things: pom_version target/$(SHARED_LIBRARY) pom_version: pom.xml ver=$$(grep AC_INIT ../configure.ac | cut -d '[' -f 3 | cut -d ']' -f 1) && \ newVer=$$(perl -e "@a=split('\.', '$$ver'); \$$a[2]++; print(join('.', @a))") && \ perl -pi -e "s/(\s*).*-SNAPSHOT/\1$$newVer-SNAPSHOT/" pom.xml -target/vw_jni.lib: $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a ../vowpalwabbit/liballreduce.a +target/$(SHARED_LIBRARY): $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a ../vowpalwabbit/liballreduce.a mkdir -p target; $(CXX) -shared $(FLAGS) -o $@ $^ $(VWLIBS) $(STDLIBS) $(JAVA_INCLUDE) @@ -48,7 +51,10 @@ target/vw_jni.lib: $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a .. -include $(jni_SRCS:.cc=.o) +install: target/$(SHARED_LIBRARY) + cp target/$(SHARED_LIBRARY) $(INSTALL_PATH); + .PHONY: clean clean: - rm -f target/vw_jni.lib + rm -f target/$(SHARED_LIBRARY) rm -f $(jni_SRCS:.cc=.o) diff --git a/java/pom.xml b/java/pom.xml index bebc8121417..a4ea1872837 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.github.johnlangford vw-jni - 8.2.2-SNAPSHOT + 8.3.3-SNAPSHOT jar Vowpal Wabbit JNI Layer diff --git a/java/src/main/bin/build.sh b/java/src/main/bin/build.sh deleted file mode 100755 index 835ac748010..00000000000 --- a/java/src/main/bin/build.sh +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env bash - -# ============================================================================= -# Constants: -# ============================================================================= -__not_darwin=1 -__brew_not_installed=2 - -make_base="cd /vowpal_wabbit; -make clean; -make vw java;" - -ubuntu_base="apt-get update -qq; -apt-get install -qq software-properties-common g++ make libboost-program-options-dev zlib1g-dev default-jdk;" - -ubuntu_12="$ubuntu_base -export JAVA_HOME=/usr/lib/jvm/java-6-openjdk-amd64; -$make_base -mv java/target/vw_jni.lib java/target/vw_jni.Ubuntu.12.amd64.lib" - -ubuntu_14_32="$ubuntu_base -export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-i386; -$make_base -mv java/target/vw_jni.lib java/target/vw_jni.Ubuntu.14.i386.lib" - -ubuntu_14="$ubuntu_base -export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64; -$make_base -mv java/target/vw_jni.lib java/target/vw_jni.Ubuntu.14.amd64.lib" - -early_red_hat="yum update -q -y; -yum install -q -y wget which zlib-devel java-1.7.0-openjdk-devel perl redhat-lsb-core; -cd /etc/yum.repos.d; -wget http://people.centos.org/tru/devtools-2/devtools-2.repo; -yum clean all; -yum install -q -y devtoolset-2-gcc devtoolset-2-gcc-c++ devtoolset-2-binutils; -ln -s /opt/rh/devtoolset-2/root/usr/bin/* /usr/local/bin/; -export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk.x86_64; -cd /vowpal_wabbit; -cat Makefile | sed 's/-fPIC/-fpermissive -fPIC/g' > Makefile.permissive;" - -red_hat_5="$early_red_hat -yum install -q -y make epel-release; -yum install -q -y boost141-devel; -make clean; -cat Makefile.permissive | sed 's/BOOST_LIBRARY\s*=\(.*\)/BOOST_LIBRARY =\1 -L \/usr\/lib64\/boost141/g' | sed 's/BOOST_INCLUDE\s*=\(.*\)/BOOST_INCLUDE =\1 -I \/usr\/include\/boost141/g' > Makefile.permissive.boost141; -make -f Makefile.permissive.boost141 vw java; -rm -f Makefile.permissive.boost141 Makefile.permissive; -mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.5.amd64.lib" - -red_hat_6="$early_red_hat -yum install -q -y boost-devel; -make clean; -make -f Makefile.permissive vw java; -rm -f Makefile.permissive; -mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.6.amd64.lib" - -red_hat_7="yum update -q -y; -yum install -q -y gcc-c++ make boost-devel zlib-devel java-1.7.0-openjdk-devel perl clang redhat-lsb-core; -export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk; -$make_base -mv java/target/vw_jni.lib java/target/vw_jni.Red_Hat.7.amd64.lib" - -# ============================================================================= -# Function Definitions: -# ============================================================================= - -# ----------------------------------------------------------------------------- -# Print red text to stderr. -# ----------------------------------------------------------------------------- -red() { - # https://linuxtidbits.wordpress.com/2008/08/11/output-color-on-bash-scripts/ - echo >&2 "$(tput setaf 1)${1}$(tput sgr0)" -} - -# ----------------------------------------------------------------------------- -# Print yellow text to stderr. -# ----------------------------------------------------------------------------- -yellow() { - echo >&2 "$(tput setaf 3)${1}$(tput sgr0)" -} - -die() { red $2; exit $1; } - -# ----------------------------------------------------------------------------- -# Check that the OS is OS X. If not, die. If so, check that brew is -# installed. If brew is not installed, ask the user if they want to install. -# If so, attempt to install. After attempting install, check for existence. -# If it still doesn't exist, fail. -# ----------------------------------------------------------------------------- -check_brew_installed() { - local os=$(uname) - if [[ "$os" != "Darwin" ]]; then - die $__not_darwin "Build script only supported on OS X. OS=${os}. Aborting ..." - else - if ! brew help 1>/dev/null 2>/dev/null; then - red "brew not installed. To install: Y or N?" - read should_install - if [[ "Y" == "${should_install^^}" ]]; then - ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - fi - if ! brew help 1>/dev/null 2>/dev/null; then - die $__brew_not_installed "brew not installed. Aborting ..." - fi - fi - fi -} - -install_brew_cask() { - if ! brew cask 1>/dev/null 2>/dev/null; then - yellow "Installing brew-cask..." - brew install caskroom/cask/brew-cask - fi -} - -install_brew_app() { - local app=$1 - if ! brew list | grep $app 1>/dev/null; then - yellow "installing brew app: $app" - brew install $app - fi -} - -install_cask_app() { - local app=$1 - if ! brew cask list | grep $app 1>/dev/null; then - yellow "installing brew cask app: $app" - brew cask install $app - fi -} - -run_docker() { - local machine=$1 - local script=$2 - docker run --rm -v $(pwd):/vowpal_wabbit $machine /bin/bash -c "$script" -} - -# ============================================================================= -# Main -# ============================================================================= - -check_brew_installed -install_brew_cask -install_cask_app "virtualbox" -install_brew_app "docker-machine" -install_brew_app "docker" - -docker-machine create --driver virtualbox default -docker-machine start default -eval "$(docker-machine env default)" - -set -e -set -u -set -x -run_docker "ubuntu:12.04" "$ubuntu_12" -run_docker "32bit/ubuntu:14.04" "$ubuntu_14_32" -run_docker "ubuntu:14.04" "$ubuntu_14" - -# This is VERY IMPORTANT for CentOS 5. CentOS 5 ships with a Boost version that is too old to be used with VW. As a result -# users who wish to run on CentOS 5 MUST upgrade their boost to version 1.41. That should ideally be done with the following -# two commands: -# sudo yum install epel-release -# sudo yum install boost141-devel -run_docker "centos:5" "$red_hat_5" - -run_docker "centos:6" "$red_hat_6" -run_docker "centos:7" "$red_hat_7" - -make clean -make vw java -mv java/target/vw_jni.lib java/target/vw_jni.$(uname -s).$(uname -m).lib diff --git a/java/src/main/bin/release.sh b/java/src/main/bin/release.sh deleted file mode 100755 index 1df95c89e87..00000000000 --- a/java/src/main/bin/release.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -set -e - -# The goal of this is to emulate the maven release plugin with the added -# steps necessary - -if (( $# < 1 )); then - echo "Usage: $0 ()" - echo "This program will do a maven release and requires the tag/ - The gpg key is optional and if not supplied a prompt will appear requesting it." - exit 1 -fi - -tag=$1 -if (( $# > 1)); then - gpg_key=$2 -fi - -url=$(cat java/pom.xml | pcregrep -M "(\n|.)*(.*)(\n|.)*" | grep "" | sed "s/<\/*url>//g") - -checkout_dir="java/target/checkout" -rm -rf $checkout_dir -checkout="git clone $url $checkout_dir" - -echo $checkout -$checkout -cd $checkout_dir - -branch="git checkout $tag" -echo $branch -$branch - -chmod 755 java/src/main/bin/build.sh -java/src/main/bin/build.sh - -# This will make sure that the cross platform build script made all the required libraries for the different supported OSs -if [ ! -f java/target/vw_jni.Ubuntu.12.amd64.lib ] || - [ ! -f java/target/vw_jni.Ubuntu.14.amd64.lib ] || - [ ! -f java/target/vw_jni.Ubuntu.14.i386.lib ] || - [ ! -f java/target/vw_jni.Red_Hat.5.amd64.lib ] || - [ ! -f java/target/vw_jni.Red_Hat.6.amd64.lib ] || - [ ! -f java/target/vw_jni.Red_Hat.7.amd64.lib ] || - [ ! -f java/target/vw_jni.Darwin.x86_64.lib ]; then - echo "Not all libraries built, failing!" - exit 1 -fi - -cd java - -version=$(grep AC_INIT ../configure.ac | cut -d '[' -f 3 | cut -d ']' -f 1) - -perl -pi -e "\$a=1 if (!\$a && s/.*/$version<\/version>/);" pom.xml - -mvn_cmd="mvn clean deploy -P release-sign-artifacts" -if [ ! -z $gpg_key ]; then - mvn_cmd="$mvn_cmd -Dgpg.passphrase=$gpg_key" -fi - -echo $mvn_cmd -$mvn_cmd diff --git a/java/src/main/java/vowpalWabbit/VW.java b/java/src/main/java/vowpalWabbit/VW.java index a082db86318..9cfa521020b 100644 --- a/java/src/main/java/vowpalWabbit/VW.java +++ b/java/src/main/java/vowpalWabbit/VW.java @@ -3,6 +3,10 @@ import vowpalWabbit.learner.VWLearners; public final class VW { + static { + System.loadLibrary("vw_jni"); + } + /** * Should not be directly instantiated. */ diff --git a/java/src/main/java/vowpalWabbit/jni/NativeUtils.java b/java/src/main/java/vowpalWabbit/jni/NativeUtils.java deleted file mode 100644 index 7c2ab1be12e..00000000000 --- a/java/src/main/java/vowpalWabbit/jni/NativeUtils.java +++ /dev/null @@ -1,251 +0,0 @@ -package vowpalWabbit.jni; - -import java.io.*; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.List; -import java.util.ArrayList; -import java.util.Map; -import java.util.HashMap; - -/** - * Simple library class for working with JNI (Java Native Interface), - * see here. - * - * @author Adam Heirnich <adam@adamh.cz>, http://www.adamh.cz - * @author Jon Morra - */ -public class NativeUtils { - - /** - * Private constructor - this class will never be instanced - */ - private NativeUtils() { - } - - // Identifies which OS distributions can share binaries - private static final Map> identicalOSs = new HashMap>(); - static { - // RedHat binaries - List redHatAlternatives = new ArrayList(); - redHatAlternatives.add("Red_Hat.6"); - identicalOSs.put("Scientific.6", redHatAlternatives); - } - - /** - * Shell call to lsb_release to get OS info - * @param lsb_args parameters to lsb_release. EX: "-i" or "-a" - * @param regexp a regular expression pattern used to parse the response from lsb_release - * For example: lsb_args = "-i", regexp = "Distributor ID: *(.*)$" - * @return A system dependent string identifying the OS. - * @throws IOException If an error occurs while running shell command - */ - public static String lsbRelease(String lsb_args, Pattern regexp) throws IOException { - BufferedReader reader = null; - try { - Process process = Runtime.getRuntime().exec("lsb_release " + lsb_args); - reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - Matcher matcher; - while ((line = reader.readLine()) != null) { - matcher = regexp.matcher(line); - if (matcher.matches()) { - return matcher.group(1); - } - } - } - finally { - reader.close(); - } - return null; - } - - /** - * Attempt to find the Linux distribution ID. - * @return The Linux distribution or null if the version cannot be found. - * @throws IOException If an I/O error occurs - */ - public static String getDistroName() throws IOException { - return lsbRelease("-i", Pattern.compile("Distributor ID:\\s*(.*)\\s*$")); - } - - /** - * This will attempt to find the Linux version by making use of {@code lsb_release -r} - * @return The Linux version or null if the version cannot be determined. - * @throws IOException If an I/O error occurs - */ - public static String getLinuxVersion() throws IOException { - return lsbRelease("-r", Pattern.compile("Release:\\s*(.*)\\s*$")); - } - - /** - * Returns the system dependent OS family. In the case of a Linux OS it will combine - * {@link NativeUtils#getDistroName()} and {@link NativeUtils#getLinuxVersion()}. - * @return A list system dependent string identifying the OS. This is a list because there are some OSs which are - * functionally identical to the one returned by lsbRelease. - * @throws UnsupportedEncodingException If an error occurs while determining the Linux specific - * information. - * @throws IOException If an I/O error occurs - * @throws IllegalStateException If the os.name property returns an unsupported OS. - */ - public static List getOsFamilies() throws IOException { - final String osName = System.getProperty("os.name"); - final String primaryName; - if (osName.toLowerCase().contains("mac")) { - primaryName = "Darwin"; - } - else if (osName.toLowerCase().contains("linux")) { - String distro = getDistroName(); - if (distro == null) { - throw new UnsupportedEncodingException("Cannot determine linux distribution"); - } - String version = getLinuxVersion(); - if (version == null) { - throw new UnsupportedOperationException("Cannot determine linux version"); - } - // get the major version. - // don't expect a period because linux version might not have one - primaryName = distro + "." + version.split("\\.")[0]; - } - else { - throw new IllegalStateException("Unsupported operating system " + osName); - } - - List viableFamilies = new ArrayList(); - viableFamilies.add(primaryName); - if (identicalOSs.containsKey(primaryName)) { - viableFamilies.addAll(identicalOSs.get(primaryName)); - } - return viableFamilies; - } - - /** - * Loads a library from current JAR archive by looking up platform dependent name. - * @param path The filename inside JAR as absolute path (beginning with '/'), e.g. /package/File.ext - * @param suffix The suffix to be appended to the name - * @throws UnsupportedEncodingException If an error occurs while determining the Linux specific - * information. - * @throws IOException If temporary file creation or read/write operation fails - */ - public static void loadOSDependentLibrary(String path, String suffix) throws IOException { - List osFamilies = getOsFamilies(); - String osDependentLib = null; - for (String osFamily : osFamilies) { - String currentOsDependentLib = path + "." + osFamily + "." + System.getProperty("os.arch") + suffix; - if (NativeUtils.class.getResource(currentOsDependentLib) != null) { - osDependentLib = currentOsDependentLib; - break; - } - } - if (osDependentLib != null) { - loadLibraryFromJar(osDependentLib); - } - else { - try { - loadLibraryFromJar(path + suffix); - } - catch (FileNotFoundException e) { - // If we cannot find an OS dependent library then try and load a library in a system independent fashion. - System.loadLibrary(path.replaceFirst("/", "")); - } - } - } - - /** - * Loads library from current JAR archive - * - * The file from JAR is copied into system temporary directory and then loaded. The temporary file is deleted after exiting. - * Method uses String as filename because the pathname is "abstract", not system-dependent. - * - * @param path The filename inside JAR as absolute path (beginning with '/'), e.g. /package/File.ext - * @throws IOException If temporary file creation or read/write operation fails - * @throws IllegalArgumentException If source file (param path) does not exist - * @throws IllegalArgumentException If the path is not absolute or if the filename is shorter than three characters (restriction of {@link File#createTempFile(java.lang.String, java.lang.String)}). - */ - public static void loadLibraryFromJar(String path) throws IOException { - if (!path.startsWith("/")) { - throw new IllegalArgumentException("The path has to be absolute (start with '/')."); - } - - // Obtain filename from path - String[] parts = path.split("/"); - String filename = (parts.length > 1) ? parts[parts.length - 1] : null; - - // Split filename to prefix and suffix (extension) - String prefix = ""; - String suffix = null; - if (filename != null) { - parts = filename.split("\\.", 2); - prefix = parts[0]; - suffix = (parts.length > 1) ? "." + parts[parts.length - 1] : null; // Thanks, davs! :-) - } - - // Check if the filename is okay - if (filename == null || prefix.length() < 3) { - throw new IllegalArgumentException("The filename has to be at least 3 characters long."); - } - - // Prepare temporary file - File temp = File.createTempFile(prefix, suffix); - temp.deleteOnExit(); - - if (!temp.exists()) { - throw new FileNotFoundException("File " + temp.getAbsolutePath() + " does not exist."); - } - - // Prepare buffer for data copying - byte[] buffer = new byte[1024]; - int readBytes; - - // Open and check input stream - InputStream is = NativeUtils.class.getResourceAsStream(path); - if (is == null) { - throw new FileNotFoundException("File " + path + " was not found inside JAR."); - } - - // Open output stream and copy data between source file in JAR and the temporary file - OutputStream os = new FileOutputStream(temp); - try { - while ((readBytes = is.read(buffer)) != -1) { - os.write(buffer, 0, readBytes); - } - } - finally { - // If read/write fails, close streams safely before throwing an exception - os.close(); - is.close(); - } - - // Finally, load the library - System.load(temp.getAbsolutePath()); - - final String libraryPrefix = prefix; - final String lockSuffix = ".lock"; - - // create lock file - final File lock = new File(temp.getAbsolutePath() + lockSuffix); - lock.createNewFile(); - lock.deleteOnExit(); - - // file filter for library file (without .lock files) - FileFilter tmpDirFilter = new FileFilter() { - public boolean accept(File pathname) { - return pathname.getName().startsWith(libraryPrefix) && !pathname.getName().endsWith(lockSuffix); - } - }; - - // get all library files from temp folder - String tmpDirName = System.getProperty("java.io.tmpdir"); - File tmpDir = new File(tmpDirName); - File[] tmpFiles = tmpDir.listFiles(tmpDirFilter); - - // delete all files which don't have n accompanying lock file - for (File tmpFile : tmpFiles) { - // Create a file to represent the lock and test. - File lockFile = new File(tmpFile.getAbsolutePath() + lockSuffix); - if (!lockFile.exists()) { - tmpFile.delete(); - } - } - } -} diff --git a/java/src/main/java/vowpalWabbit/learner/VWLearners.java b/java/src/main/java/vowpalWabbit/learner/VWLearners.java index d4f256e74c1..ed3b8a55015 100644 --- a/java/src/main/java/vowpalWabbit/learner/VWLearners.java +++ b/java/src/main/java/vowpalWabbit/learner/VWLearners.java @@ -1,8 +1,5 @@ package vowpalWabbit.learner; -import vowpalWabbit.jni.NativeUtils; - -import java.io.IOException; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -13,13 +10,16 @@ * @author jmorra */ final public class VWLearners { - private volatile static boolean loadedNativeLibrary = false; private static final Lock STATIC_LOCK = new ReentrantLock(); private enum VWReturnType { Unknown, ActionProbs, ActionScores, Multiclass, Multilabels, Prob, Scalar, Scalars } + static { + System.loadLibrary("vw_jni"); + } + private VWLearners() {} /** @@ -41,7 +41,7 @@ private VWLearners() {} */ @SuppressWarnings("unchecked") public static T create(final String command) { - long nativePointer = initializeVWJni(command); + long nativePointer = initialize(command); VWReturnType returnType = getReturnType(nativePointer); switch (returnType) { @@ -60,45 +60,6 @@ public static T create(final String command) { } } - /** - * @param command The same string that is passed to VW, see - * here - * for more information - * @return The pointer to the native object created on the C side - */ - private static long initializeVWJni(final String command) { - long nativePointer; - try { - nativePointer = initialize(command); - loadedNativeLibrary = true; - } - catch (UnsatisfiedLinkError e) { - loadNativeLibrary(); - nativePointer = initialize(command); - } - return nativePointer; - } - - private static void loadNativeLibrary() { - // By making use of a static lock here we make sure this code is only executed once globally. - if (!loadedNativeLibrary) { - STATIC_LOCK.lock(); - try { - if (!loadedNativeLibrary) { - NativeUtils.loadOSDependentLibrary("/vw_jni", ".lib"); - loadedNativeLibrary = true; - } - } - catch (IOException e) { - // Here I've chosen to rethrow the exception as an unchecked exception because if the native - // library cannot be loaded then the exception is not recoverable from. - throw new RuntimeException(e); - } - finally { - STATIC_LOCK.unlock(); - } - } - } private static native long initialize(String command); private static native VWReturnType getReturnType(long nativePointer); diff --git a/vowpalwabbit/Makefile b/vowpalwabbit/Makefile index c1caaf7865c..d1f0cf06f19 100644 --- a/vowpalwabbit/Makefile +++ b/vowpalwabbit/Makefile @@ -30,7 +30,7 @@ allreduce_OBJECTS = $(patsubst %.cc,%.o,$(allreduce_SOURCES)) $(CXX) -std=c++0x -I ../rapidjson/include -MM $< > $@ -include $(vw_DEPS) -%.o: %.cc %.h +%.o: %.cc %.h $(CXX) $(FLAGS) -c $< -o $@ %.o: %.cc @@ -49,7 +49,7 @@ active_interactor: active_interactor.cc $(CXX) $(FLAGS) -o $@ $+ install: $(BINARIES) - cp $(BINARIES) /usr/local/bin; cd cluster; $(MAKE) install + cp $(BINARIES) /usr/local/bin; cd cluster; $(MAKE) install; cd java; $(MAKE) install clean: rm -f *.o *.d $(BINARIES) *~ $(MANPAGES) libvw.a liballreduce.a From 523c69efdcdc8b7d9663b65d795528c679085e0e Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 10:18:52 -0800 Subject: [PATCH 02/12] Updating makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ad61c2c3fb4..126e277ab2f 100644 --- a/Makefile +++ b/Makefile @@ -83,7 +83,7 @@ MANPAGES = vw.1 default: vw -all: vw library_example java spanning_tree python +all: vw library_example java spanning_tree %.1: % help2man --no-info --name="Vowpal Wabbit -- fast online learning tool" ./$< > $@ From 5b33a2a4e505a138a3898e4aee5e0683881a9166 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 12:27:45 -0800 Subject: [PATCH 03/12] Trying to get Travis to pass --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ca8aa9980dc..79a836c4da3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ before_install: script: - make all - make python + - make install - mvn test -f java/pom.xml - make test - make test_gcov --always-make From f40d320930b3f0fe60a32c95adce2985cb518f9c Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 12:47:04 -0800 Subject: [PATCH 04/12] Trying to get Travis to pass --- .travis.yml | 1 - java/pom.xml | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 79a836c4da3..ca8aa9980dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,6 @@ before_install: script: - make all - make python - - make install - mvn test -f java/pom.xml - make test - make test_gcov --always-make diff --git a/java/pom.xml b/java/pom.xml index a4ea1872837..6055243d341 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -155,6 +155,8 @@ true + once + -Djava.library.path=${project.build.directory} From 48ee9aaa09ae639392ef8c0f639039705c852906 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 13:13:22 -0800 Subject: [PATCH 05/12] Making Java Makefile cleaner --- java/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/Makefile b/java/Makefile index 1603c59a05d..072624e83db 100644 --- a/java/Makefile +++ b/java/Makefile @@ -23,6 +23,7 @@ ifeq ($(UNAME), Darwin) INSTALL_PATH = /Library/Java/Extensions endif +LOCAL_LIBRARY = target/$(SHARED_LIBRARY) jni_SRCS = $(shell find src/main/c++ -name "*.cc") jni_OBJS = $(jni_SRCS:.cc=.o) @@ -32,14 +33,14 @@ all: test: cd ..; $(MAKE) test -things: pom_version target/$(SHARED_LIBRARY) +things: pom_version $(LOCAL_LIBRARY) pom_version: pom.xml ver=$$(grep AC_INIT ../configure.ac | cut -d '[' -f 3 | cut -d ']' -f 1) && \ newVer=$$(perl -e "@a=split('\.', '$$ver'); \$$a[2]++; print(join('.', @a))") && \ perl -pi -e "s/(\s*).*-SNAPSHOT/\1$$newVer-SNAPSHOT/" pom.xml -target/$(SHARED_LIBRARY): $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a ../vowpalwabbit/liballreduce.a +$(LOCAL_LIBRARY): $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a ../vowpalwabbit/liballreduce.a mkdir -p target; $(CXX) -shared $(FLAGS) -o $@ $^ $(VWLIBS) $(STDLIBS) $(JAVA_INCLUDE) @@ -52,9 +53,9 @@ target/$(SHARED_LIBRARY): $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/lib -include $(jni_SRCS:.cc=.o) install: target/$(SHARED_LIBRARY) - cp target/$(SHARED_LIBRARY) $(INSTALL_PATH); + cp $(LOCAL_LIBRARY) $(INSTALL_PATH); .PHONY: clean clean: - rm -f target/$(SHARED_LIBRARY) + rm -f $(LOCAL_LIBRARY) rm -f $(jni_SRCS:.cc=.o) From 95a3ef7e713d47710a2feabb6b6a02f3830d46c3 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Wed, 8 Feb 2017 14:15:47 -0800 Subject: [PATCH 06/12] Removing Java build from configure for now --- configure.ac | 1 - 1 file changed, 1 deletion(-) diff --git a/configure.ac b/configure.ac index ed047860e28..64e9042433a 100644 --- a/configure.ac +++ b/configure.ac @@ -78,7 +78,6 @@ AC_CONFIG_FILES([ vowpalwabbit/Makefile cluster/Makefile library/Makefile - java/Makefile libvw.pc libvw_c_wrapper.pc ]) From 8cfe2775a0b04dd2ea2ce227f0a8bb1469148f57 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Mon, 6 Mar 2017 08:06:23 -0800 Subject: [PATCH 07/12] Addressing code reviews --- java/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/Makefile b/java/Makefile index 072624e83db..5098c3289ee 100644 --- a/java/Makefile +++ b/java/Makefile @@ -1,25 +1,26 @@ VWLIBS := -L../vowpalwabbit -l vw STDLIBS = $(BOOST_LIBRARY) $(LIBS) JAVA_INCLUDE = -I $(JAVA_HOME)/include +LIBRARY_NAME = vw_jni ifeq ($(UNAME), Linux) JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux - SHARED_LIBRARY = libvw_jni.so + SHARED_LIBRARY = lib$(LIBRARY_NAME).so INSTALL_PATH = /usr/lib endif ifeq ($(UNAME), FreeBSD) JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux - SHARED_LIBRARY = libvw_jni.so + SHARED_LIBRARY = lib$(LIBRARY_NAME).so INSTALL_PATH = /usr/lib endif ifeq "CYGWIN" "$(findstring CYGWIN,$(UNAME))" JAVA_INCLUDE += -I $(JAVA_HOME)/include/linux - SHARED_LIBRARY = libvw_jni.so + SHARED_LIBRARY = lib$(LIBRARY_NAME).so INSTALL_PATH = /usr/lib endif ifeq ($(UNAME), Darwin) JAVA_INCLUDE += -I $(JAVA_HOME)/include/darwin - SHARED_LIBRARY = libvw_jni.dylib + SHARED_LIBRARY = lib$(LIBRARY_NAME).dylib INSTALL_PATH = /Library/Java/Extensions endif From 03a44ef81ee91d509996ec2b60ed5530fdcfffa6 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Mon, 6 Mar 2017 08:07:16 -0800 Subject: [PATCH 08/12] Addressing code reviews --- java/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/Makefile b/java/Makefile index 5098c3289ee..f8a630e0b52 100644 --- a/java/Makefile +++ b/java/Makefile @@ -53,7 +53,7 @@ $(LOCAL_LIBRARY): $(jni_OBJS) ../vowpalwabbit/main.o ../vowpalwabbit/libvw.a ../ -include $(jni_SRCS:.cc=.o) -install: target/$(SHARED_LIBRARY) +install: $(LOCAL_LIBRARY) cp $(LOCAL_LIBRARY) $(INSTALL_PATH); .PHONY: clean From 7bc889b96b3aa75464f7331895947134049a8fab Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Thu, 22 Jun 2017 13:56:51 -0700 Subject: [PATCH 09/12] Updating pom versions --- java/pom.xml | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/java/pom.xml b/java/pom.xml index 6055243d341..f3ea6a2a5e6 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -4,7 +4,7 @@ com.github.johnlangford vw-jni - 8.3.3-SNAPSHOT + 8.3.4-SNAPSHOT jar Vowpal Wabbit JNI Layer @@ -207,6 +207,19 @@ + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar + + + + @@ -221,22 +234,10 @@ - - org.apache.maven.plugins - maven-source-plugin - 2.1.2 - - - attach-sources - - jar - - - - org.apache.maven.plugins maven-gpg-plugin + 1.6 sign-artifacts From 042af670b1514eeb85f9fcca90b7d51a1bae7b96 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Thu, 22 Jun 2017 15:44:44 -0700 Subject: [PATCH 10/12] Adding fix to csoaa to allow predictions to come back --- .../learner/VWActionScoresLearnerTest.java | 47 +++++++++++++++++++ vowpalwabbit/csoaa.cc | 8 ++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/java/src/test/java/vowpalWabbit/learner/VWActionScoresLearnerTest.java b/java/src/test/java/vowpalWabbit/learner/VWActionScoresLearnerTest.java index b24f68b3aa2..37d86834879 100644 --- a/java/src/test/java/vowpalWabbit/learner/VWActionScoresLearnerTest.java +++ b/java/src/test/java/vowpalWabbit/learner/VWActionScoresLearnerTest.java @@ -17,6 +17,53 @@ public class VWActionScoresLearnerTest extends VWTestHelper { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testCSOAA() throws IOException { + String[][] data = new String[][]{ + new String[]{ + "1:1.0 | a_1 b_1 c_1", + "2:0.0 | a_2 b_2 c_2", + "3:2.0 | a_3 b_3 c_3" + }, + new String[]{ + "1:1.0 | b_1 c_1 d_1", + "2:0.0 | b_2 c_2 d_2" + }, + new String[]{ + "1:1.0 | a_1 b_1 c_1", + "3:2.0 | a_3 b_3 c_3" + } + }; + + VWActionScoresLearner vw = VWLearners.create("--csoaa_ldf mc --quiet --csoaa_rank"); + + ActionScores[] pred = new ActionScores[data.length]; + for (int j=0; j< 100; ++j) { + for (int i=0; ipred.multiclass = data.ec_seq[k]->l.cs.costs[0].class_index; else data.ec_seq[k]->pred.multiclass = 0; @@ -804,11 +804,11 @@ base_learner* csldf_setup(vw& all) prediction_type::prediction_type_t pred_type; if (ld.rank) - pred_type = prediction_type::multiclass; + pred_type = prediction_type::action_scores; else if (ld.is_probabilities) pred_type = prediction_type::prob; else - pred_type = prediction_type::action_scores; + pred_type = prediction_type::multiclass; ld.read_example_this_loop = 0; ld.need_to_clear = false; From 4599ef8b0c8e2ba503d038258934e21439fcc06d Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Thu, 22 Jun 2017 16:23:46 -0700 Subject: [PATCH 11/12] Getting python tests to pass --- python/tests/test_pyvw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_pyvw.py b/python/tests/test_pyvw.py index dd06c552f5f..c891a74a507 100644 --- a/python/tests/test_pyvw.py +++ b/python/tests/test_pyvw.py @@ -91,7 +91,7 @@ def test_prob_prediction_type(): def test_action_scores_prediction_type(): model = vw(loss_function='logistic', csoaa_ldf='m', quiet=True) model.learn('1 | a b c') - assert model.get_prediction_type() == model.pACTION_SCORES + assert model.get_prediction_type() == model.pMULTICLASS prediction = model.predict(' | a b c') assert isinstance(prediction, list) del model From 239013da87a16b4c452da891f12a9259a147b4e7 Mon Sep 17 00:00:00 2001 From: Jon Morra Date: Thu, 22 Jun 2017 16:53:57 -0700 Subject: [PATCH 12/12] Getting python tests passing --- python/tests/test_pyvw.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_pyvw.py b/python/tests/test_pyvw.py index c891a74a507..b9bf8d56f4a 100644 --- a/python/tests/test_pyvw.py +++ b/python/tests/test_pyvw.py @@ -93,7 +93,7 @@ def test_action_scores_prediction_type(): model.learn('1 | a b c') assert model.get_prediction_type() == model.pMULTICLASS prediction = model.predict(' | a b c') - assert isinstance(prediction, list) + assert isinstance(prediction, int) del model