From a4d66eba27487cc5d72f1ca94073ca6ca2340654 Mon Sep 17 00:00:00 2001 From: Andrew Jefferson Date: Sun, 25 Aug 2019 22:35:24 +0100 Subject: [PATCH 1/5] Switch to using the latest neo4j driver (beta) --- pom.xml | 2 +- src/test/java/com/neo4j/docker/TestPasswords.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 5f8076b5..3c7f1176 100644 --- a/pom.xml +++ b/pom.xml @@ -78,7 +78,7 @@ org.neo4j.driver neo4j-java-driver - 2.0.0-alpha02 + 4.0.0-beta01 test diff --git a/src/test/java/com/neo4j/docker/TestPasswords.java b/src/test/java/com/neo4j/docker/TestPasswords.java index af3e58a4..e7f1ef4d 100644 --- a/src/test/java/com/neo4j/docker/TestPasswords.java +++ b/src/test/java/com/neo4j/docker/TestPasswords.java @@ -103,7 +103,7 @@ private void verifyPasswordIsIncorrect( GenericContainer container, String passw { String boltUri = getBoltURIFromContainer(container); Assertions.assertThrows( org.neo4j.driver.exceptions.AuthenticationException.class, - () -> GraphDatabase.driver( boltUri, AuthTokens.basic( "neo4j", password ), TEST_DRIVER_CONFIG ) ); + () -> GraphDatabase.driver( boltUri, AuthTokens.basic( "neo4j", password ), TEST_DRIVER_CONFIG ).verifyConnectivity() ); } // when junit 5.5.0 is released, @ValueSource should support booleans. From 7d7f92eb134b24a3809d21213987755ec86d79a4 Mon Sep 17 00:00:00 2001 From: Andrew Jefferson Date: Thu, 8 Aug 2019 21:26:29 +0000 Subject: [PATCH 2/5] Revert "Revert "Merge pull request #181 from eastlondoner/andy/install-apoc-at-runtime"" This reverts commit 84425d99fd10ab50f05a59bfb9ca79ec0ec7ed05. Which re-enables plugin installation at runtime --- Makefile | 7 +- docker-image-src/3.3/Dockerfile | 3 +- docker-image-src/3.3/docker-entrypoint.sh | 42 ++++++ docker-image-src/3.4/Dockerfile | 3 +- docker-image-src/3.4/docker-entrypoint.sh | 42 ++++++ docker-image-src/3.5/Dockerfile | 3 +- docker-image-src/3.5/docker-entrypoint.sh | 42 ++++++ neo4jlabs-plugins.json | 7 + pom.xml | 20 +++ .../neo4j/docker/TestPluginInstallation.java | 141 ++++++++++++++++++ .../docker/plugins/ExampleNeo4jPlugin.java | 47 ++++++ .../com/neo4j/docker/plugins/JarBuilder.java | 45 ++++++ .../docker/utils/HostFileHttpHandler.java | 33 ++++ .../neo4j/docker/utils/HttpServerRule.java | 42 ++++++ src/test/resources/versions.json | 7 + 15 files changed, 480 insertions(+), 4 deletions(-) create mode 100644 neo4jlabs-plugins.json create mode 100644 src/test/java/com/neo4j/docker/TestPluginInstallation.java create mode 100644 src/test/java/com/neo4j/docker/plugins/ExampleNeo4jPlugin.java create mode 100644 src/test/java/com/neo4j/docker/plugins/JarBuilder.java create mode 100644 src/test/java/com/neo4j/docker/utils/HostFileHttpHandler.java create mode 100644 src/test/java/com/neo4j/docker/utils/HttpServerRule.java create mode 100644 src/test/resources/versions.json diff --git a/Makefile b/Makefile index 5f219d4d..e148b6cf 100644 --- a/Makefile +++ b/Makefile @@ -59,11 +59,16 @@ tmp/.image-id-%: tmp/local-context-%/.sentinel $( echo -n $$image >$@ -tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION)) +tmp/neo4jlabs-plugins.json: ./neo4jlabs-plugins.json +> mkdir -p $(@D) +> cp $< $@ + +tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION)) tmp/neo4jlabs-plugins.json > rm -rf $(@D) > mkdir -p $(@D) > cp -r $( cp $(filter %.tar.gz,$^) $(@D)/local-package +> cp $(filter %.json,$^) $(@D)/local-package > touch $@ tmp/image-%/.sentinel: docker-image-src/$(series)/Dockerfile docker-image-src/$(series)/docker-entrypoint.sh \ diff --git a/docker-image-src/3.3/Dockerfile b/docker-image-src/3.3/Dockerfile index 91547090..68922257 100644 --- a/docker-image-src/3.3/Dockerfile +++ b/docker-image-src/3.3/Dockerfile @@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J COPY ./local-package/* /tmp/ RUN apt update \ - && apt install -y curl gosu \ + && apt install -y curl gosu jq \ && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \ && echo "${TINI_SHA256} /sbin/tini" | sha256sum -c --strict --quiet \ && chmod +x /sbin/tini \ @@ -32,6 +32,7 @@ RUN apt update \ && chmod -R 777 "${NEO4J_HOME}" \ && ln -s /data "${NEO4J_HOME}"/data \ && ln -s /logs "${NEO4J_HOME}"/logs \ + && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \ && rm -rf /tmp/* \ && rm -rf /var/lib/apt/lists/* diff --git a/docker-image-src/3.3/docker-entrypoint.sh b/docker-image-src/3.3/docker-entrypoint.sh index 6d6113fa..6a9fc9e3 100755 --- a/docker-image-src/3.3/docker-entrypoint.sh +++ b/docker-image-src/3.3/docker-entrypoint.sh @@ -119,6 +119,38 @@ function check_mounted_folder_with_chown fi } +function load_plugin_from_github +{ + # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the + # correct format. + local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql + + local _plugins_dir="${NEO4J_HOME}/plugins" + if [ -d /plugins ]; then + local _plugins_dir="/plugins" + fi + local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )" + # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin + local _destination="${_plugins_dir}/${_plugin_name}.jar" + local _neo4j_version="$(neo4j --version | cut -d' ' -f2)" + + # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version + echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}" + local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")" + local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")" + if [[ -z "${_plugin_jar_url}" ]]; then + echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'" + echo >&2 "${_versions_json}" + fi + echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} " + curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}" + + if ! is_readable "${_destination}"; then + echo >&2 "Plugin at '${_destination}' is not readable" + exit 1 + fi +} + # If we're running as root, then run as the neo4j user. Otherwise # docker is running with --user and we simply use that user. Note # that su-exec, despite its name, does not replicate the functionality @@ -266,6 +298,10 @@ fi if [ -d /plugins ]; then if secure_mode_enabled; then + if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # We need write permissions + check_mounted_folder_with_chown "/plugins" + fi check_mounted_folder_readable "/plugins" fi NEO4J_dbms_directories_plugins="/plugins" @@ -341,6 +377,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do fi done +if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]' + for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do + load_plugin_from_github "${plugin_name}" + done +fi [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT} diff --git a/docker-image-src/3.4/Dockerfile b/docker-image-src/3.4/Dockerfile index 91547090..68922257 100644 --- a/docker-image-src/3.4/Dockerfile +++ b/docker-image-src/3.4/Dockerfile @@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J COPY ./local-package/* /tmp/ RUN apt update \ - && apt install -y curl gosu \ + && apt install -y curl gosu jq \ && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \ && echo "${TINI_SHA256} /sbin/tini" | sha256sum -c --strict --quiet \ && chmod +x /sbin/tini \ @@ -32,6 +32,7 @@ RUN apt update \ && chmod -R 777 "${NEO4J_HOME}" \ && ln -s /data "${NEO4J_HOME}"/data \ && ln -s /logs "${NEO4J_HOME}"/logs \ + && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \ && rm -rf /tmp/* \ && rm -rf /var/lib/apt/lists/* diff --git a/docker-image-src/3.4/docker-entrypoint.sh b/docker-image-src/3.4/docker-entrypoint.sh index 881424fc..e0b98da0 100755 --- a/docker-image-src/3.4/docker-entrypoint.sh +++ b/docker-image-src/3.4/docker-entrypoint.sh @@ -119,6 +119,38 @@ function check_mounted_folder_with_chown fi } +function load_plugin_from_github +{ + # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the + # correct format. + local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql + + local _plugins_dir="${NEO4J_HOME}/plugins" + if [ -d /plugins ]; then + local _plugins_dir="/plugins" + fi + local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )" + # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin + local _destination="${_plugins_dir}/${_plugin_name}.jar" + local _neo4j_version="$(neo4j --version | cut -d' ' -f2)" + + # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version + echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}" + local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")" + local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")" + if [[ -z "${_plugin_jar_url}" ]]; then + echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'" + echo >&2 "${_versions_json}" + fi + echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} " + curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}" + + if ! is_readable "${_destination}"; then + echo >&2 "Plugin at '${_destination}' is not readable" + exit 1 + fi +} + # If we're running as root, then run as the neo4j user. Otherwise # docker is running with --user and we simply use that user. Note # that su-exec, despite its name, does not replicate the functionality @@ -259,6 +291,10 @@ fi if [ -d /plugins ]; then if secure_mode_enabled; then + if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # We need write permissions + check_mounted_folder_with_chown "/plugins" + fi check_mounted_folder_readable "/plugins" fi NEO4J_dbms_directories_plugins="/plugins" @@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do fi done +if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]' + for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do + load_plugin_from_github "${plugin_name}" + done +fi [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT} diff --git a/docker-image-src/3.5/Dockerfile b/docker-image-src/3.5/Dockerfile index 91547090..68922257 100644 --- a/docker-image-src/3.5/Dockerfile +++ b/docker-image-src/3.5/Dockerfile @@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J COPY ./local-package/* /tmp/ RUN apt update \ - && apt install -y curl gosu \ + && apt install -y curl gosu jq \ && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \ && echo "${TINI_SHA256} /sbin/tini" | sha256sum -c --strict --quiet \ && chmod +x /sbin/tini \ @@ -32,6 +32,7 @@ RUN apt update \ && chmod -R 777 "${NEO4J_HOME}" \ && ln -s /data "${NEO4J_HOME}"/data \ && ln -s /logs "${NEO4J_HOME}"/logs \ + && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \ && rm -rf /tmp/* \ && rm -rf /var/lib/apt/lists/* diff --git a/docker-image-src/3.5/docker-entrypoint.sh b/docker-image-src/3.5/docker-entrypoint.sh index 881424fc..e0b98da0 100755 --- a/docker-image-src/3.5/docker-entrypoint.sh +++ b/docker-image-src/3.5/docker-entrypoint.sh @@ -119,6 +119,38 @@ function check_mounted_folder_with_chown fi } +function load_plugin_from_github +{ + # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the + # correct format. + local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql + + local _plugins_dir="${NEO4J_HOME}/plugins" + if [ -d /plugins ]; then + local _plugins_dir="/plugins" + fi + local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )" + # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin + local _destination="${_plugins_dir}/${_plugin_name}.jar" + local _neo4j_version="$(neo4j --version | cut -d' ' -f2)" + + # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version + echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}" + local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")" + local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")" + if [[ -z "${_plugin_jar_url}" ]]; then + echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'" + echo >&2 "${_versions_json}" + fi + echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} " + curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}" + + if ! is_readable "${_destination}"; then + echo >&2 "Plugin at '${_destination}' is not readable" + exit 1 + fi +} + # If we're running as root, then run as the neo4j user. Otherwise # docker is running with --user and we simply use that user. Note # that su-exec, despite its name, does not replicate the functionality @@ -259,6 +291,10 @@ fi if [ -d /plugins ]; then if secure_mode_enabled; then + if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # We need write permissions + check_mounted_folder_with_chown "/plugins" + fi check_mounted_folder_readable "/plugins" fi NEO4J_dbms_directories_plugins="/plugins" @@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do fi done +if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]' + for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do + load_plugin_from_github "${plugin_name}" + done +fi [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT} diff --git a/neo4jlabs-plugins.json b/neo4jlabs-plugins.json new file mode 100644 index 00000000..f4de86d2 --- /dev/null +++ b/neo4jlabs-plugins.json @@ -0,0 +1,7 @@ +{ + "apoc": "https://github.com/neo4j-contrib/neo4j-apoc-procedures/raw/master/versions.json", + "streams": "https://github.com/neo4j-contrib/neo4j-streams/raw/master/versions.json", + "graphql": "https://github.com/neo4j-contrib/neo4j-graphql/raw/master/versions.json", + "graph-algorithms": "https://github.com/neo4j-contrib/neo4j-graph-algorithms/raw/master/versions.json", + "_testing": "http://host.testcontainers.internal:3000/versions.json" +} diff --git a/pom.xml b/pom.xml index 3c7f1176..93c81e2d 100644 --- a/pom.xml +++ b/pom.xml @@ -9,6 +9,10 @@ 1.0-SNAPSHOT jar + + ${env.NEO4JVERSION} + + @@ -28,6 +32,16 @@ + + + org.neo4j + neo4j + ${neo4j.version} + provided + org.slf4j @@ -57,6 +71,12 @@ 5.4.2 test + + org.junit.jupiter + junit-jupiter-migrationsupport + 5.4.2 + test + org.testcontainers junit-jupiter diff --git a/src/test/java/com/neo4j/docker/TestPluginInstallation.java b/src/test/java/com/neo4j/docker/TestPluginInstallation.java new file mode 100644 index 00000000..91cd6d55 --- /dev/null +++ b/src/test/java/com/neo4j/docker/TestPluginInstallation.java @@ -0,0 +1,141 @@ +package com.neo4j.docker; + +import com.neo4j.docker.plugins.ExampleNeo4jPlugin; +import com.neo4j.docker.utils.HostFileHttpHandler; +import com.neo4j.docker.utils.HttpServerRule; +import com.neo4j.docker.plugins.JarBuilder; +import com.neo4j.docker.utils.SetContainerUser; +import com.neo4j.docker.utils.TestSettings; +import org.junit.Rule; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.migrationsupport.rules.EnableRuleMigrationSupport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.Testcontainers; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.shaded.com.google.common.io.Files; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.Driver; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Record; +import org.neo4j.driver.Session; +import org.neo4j.driver.StatementResult; + +import static com.neo4j.docker.utils.TestSettings.NEO4J_VERSION; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@EnableRuleMigrationSupport +public class TestPluginInstallation +{ + private static final int DEFAULT_BROWSER_PORT = 7474; + private static final int DEFAULT_BOLT_PORT = 7687; + + private static final String versions = "versions.json"; + private static final String myPlugin = "myPlugin.jar"; + + private static final Logger log = LoggerFactory.getLogger( TestPluginInstallation.class ); + + @Rule + public HttpServerRule httpServer = new HttpServerRule(); + + private GenericContainer container; + + private void createContainerWithTestingPlugin() + { + Testcontainers.exposeHostPorts( httpServer.PORT ); + container = new GenericContainer( TestSettings.IMAGE_ID ); + + container.withEnv( "NEO4J_AUTH", "neo4j/neo" ) + .withEnv( "NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes" ) + .withEnv( "NEO4JLABS_PLUGINS", "[\"_testing\"]" ) + .withExposedPorts( DEFAULT_BROWSER_PORT, DEFAULT_BOLT_PORT ) + .withLogConsumer( new Slf4jLogConsumer( log ) ); + + SetContainerUser.currentlyRunningUser( container ); + } + + @BeforeEach + public void setUp( @TempDir Path pluginsDir ) throws Exception + { + File versionsJson = pluginsDir.resolve( versions ).toFile(); + + Files.write( getResource( "versions.json" ).replace( "$NEO4J_VERSION", NEO4J_VERSION.toString() ), versionsJson, StandardCharsets.UTF_8 ); + + File myPluginJar = pluginsDir.resolve( myPlugin ).toFile(); + + new JarBuilder().createJarFor( myPluginJar, ExampleNeo4jPlugin.class, ExampleNeo4jPlugin.PrimitiveOutput.class ); + + httpServer.registerHandler( versions, new HostFileHttpHandler( versionsJson, "application/json" ) ); + httpServer.registerHandler( myPlugin, new HostFileHttpHandler( myPluginJar, "application/java-archive" ) ); + + createContainerWithTestingPlugin(); + container.setWaitStrategy( Wait.forHttp( "/" ).forPort( DEFAULT_BROWSER_PORT ).forStatusCode( 200 ) ); + } + + @Test + public void testPlugin() throws Exception + { + // When we start the neo4j docker container + container.start(); + + // Then the plugin is downloaded and placed in the plugins directory + String lsPluginsDir = container.execInContainer( "ls", "/var/lib/neo4j/plugins" ).getStdout(); + // Two options here because it varies depending on whether the plugins dir _only_ contains our file or if it contains multiple files + assertTrue( lsPluginsDir.contains( "\n_testing.jar\n" ) || lsPluginsDir.equals( "_testing.jar\n" ), "Plugin jar file not found in plugins directory" ); + + // When we connect to the database with the plugin + String boltAddress = "bolt://" + container.getContainerIpAddress() + ":" + container.getMappedPort( DEFAULT_BOLT_PORT ); + try ( Driver coreDriver = GraphDatabase.driver( boltAddress, AuthTokens.basic( "neo4j", "neo" ) ) ) + { + Session session = coreDriver.session(); + StatementResult res = session.run( "CALL dbms.procedures() YIELD name, signature RETURN name, signature" ); + + // Then the procedure from the plugin is listed + assertTrue( res.stream().anyMatch( x -> x.get( "name" ).asString().equals( "com.neo4j.docker.plugins.defaultValues" ) ), + "Missing procedure provided by our plugin" ); + + // When we call the procedure from the plugin + res = session.run( "CALL com.neo4j.docker.plugins.defaultValues" ); + + // Then we get the response we expect + Record record = res.single(); + String message = "Result from calling our procedure doesnt match our expectations"; + assertEquals( record.get( "string" ).asString(), "a string", message ); + assertEquals( record.get( "integer" ).asInt(), 42L, message ); + assertEquals( record.get( "aFloat" ).asDouble(), 3.14d, 0.000001, message ); + assertEquals( record.get( "aBoolean" ).asBoolean(), true, message ); + assertFalse( res.hasNext(), "Our procedure should only return a single result" ); + } + finally + { + container.stop(); + } + } + + private String getResource( String path ) throws IOException + { + InputStream inputStream = getClass().getClassLoader().getResourceAsStream( path ); + ByteArrayOutputStream result = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ( (length = inputStream.read( buffer )) != -1 ) + { + result.write( buffer, 0, length ); + } + return result.toString( "UTF-8" ); + } +} diff --git a/src/test/java/com/neo4j/docker/plugins/ExampleNeo4jPlugin.java b/src/test/java/com/neo4j/docker/plugins/ExampleNeo4jPlugin.java new file mode 100644 index 00000000..73ceb53d --- /dev/null +++ b/src/test/java/com/neo4j/docker/plugins/ExampleNeo4jPlugin.java @@ -0,0 +1,47 @@ +package com.neo4j.docker.plugins; + +import java.util.stream.Stream; + +import org.neo4j.graphdb.GraphDatabaseService; +import org.neo4j.logging.Log; +import org.neo4j.procedure.Context; +import org.neo4j.procedure.Name; +import org.neo4j.procedure.Procedure; + +/* +This class is a basic Neo4J plugin that defines a procedure which can be called via Cypher. + */ +public class ExampleNeo4jPlugin +{ + // Output data class containing primitive types + public static class PrimitiveOutput + { + public String string; + public long integer; + public double aFloat; + public boolean aBoolean; + + public PrimitiveOutput( String string, long integer, double aFloat, boolean aBoolean ) + { + this.string = string; + this.integer = integer; + this.aFloat = aFloat; + this.aBoolean = aBoolean; + } + } + + @Context + public GraphDatabaseService db; + + @Context + public Log log; + + // A Neo4j procedure that always returns fixed values + @Procedure + public Stream defaultValues( @Name( value = "string", defaultValue = "a string" ) String string, + @Name( value = "integer", defaultValue = "42" ) long integer, @Name( value = "float", defaultValue = "3.14" ) double aFloat, + @Name( value = "boolean", defaultValue = "true" ) boolean aBoolean ) + { + return Stream.of( new PrimitiveOutput( string, integer, aFloat, aBoolean ) ); + } +} \ No newline at end of file diff --git a/src/test/java/com/neo4j/docker/plugins/JarBuilder.java b/src/test/java/com/neo4j/docker/plugins/JarBuilder.java new file mode 100644 index 00000000..86d9938c --- /dev/null +++ b/src/test/java/com/neo4j/docker/plugins/JarBuilder.java @@ -0,0 +1,45 @@ +package com.neo4j.docker.plugins; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.util.jar.JarOutputStream; +import java.util.zip.ZipEntry; + +/** + * Utility to create jar files containing classes from the current classpath. + */ +public class JarBuilder +{ + public URL createJarFor( File f, Class... classesToInclude ) throws IOException + { + try ( FileOutputStream fout = new FileOutputStream( f ); JarOutputStream jarOut = new JarOutputStream( fout ) ) + { + for ( Class target : classesToInclude ) + { + String fileName = target.getName().replace( ".", "/" ) + ".class"; + jarOut.putNextEntry( new ZipEntry( fileName ) ); + jarOut.write( classCompiledBytes( fileName ) ); + jarOut.closeEntry(); + } + } + return f.toURI().toURL(); + } + + private byte[] classCompiledBytes( String fileName ) throws IOException + { + try ( InputStream in = getClass().getClassLoader().getResourceAsStream( fileName ) ) + { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + while ( in.available() > 0 ) + { + out.write( in.read() ); + } + + return out.toByteArray(); + } + } +} diff --git a/src/test/java/com/neo4j/docker/utils/HostFileHttpHandler.java b/src/test/java/com/neo4j/docker/utils/HostFileHttpHandler.java new file mode 100644 index 00000000..5a1b8586 --- /dev/null +++ b/src/test/java/com/neo4j/docker/utils/HostFileHttpHandler.java @@ -0,0 +1,33 @@ +package com.neo4j.docker.utils; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpHandler; + +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.file.Files; + +/** + * HttpHandler that responds to all hhtp requests with the given file from the file system + */ +public class HostFileHttpHandler implements HttpHandler +{ + private final File file; + private final String contentType; + + public HostFileHttpHandler( File fileToDownload, String contentType ) + { + this.file = fileToDownload; + this.contentType = contentType; + } + + @Override + public void handle( HttpExchange exchange ) throws IOException + { + exchange.getResponseHeaders().add( "Content-Type", contentType ); + exchange.sendResponseHeaders( HttpURLConnection.HTTP_OK, file.length() ); + Files.copy( this.file.toPath(), exchange.getResponseBody() ); + exchange.close(); + } +} \ No newline at end of file diff --git a/src/test/java/com/neo4j/docker/utils/HttpServerRule.java b/src/test/java/com/neo4j/docker/utils/HttpServerRule.java new file mode 100644 index 00000000..146fc850 --- /dev/null +++ b/src/test/java/com/neo4j/docker/utils/HttpServerRule.java @@ -0,0 +1,42 @@ +package com.neo4j.docker.utils; + +import com.sun.net.httpserver.HttpHandler; +import com.sun.net.httpserver.HttpServer; +import org.junit.rules.ExternalResource; + +import java.net.InetSocketAddress; + +/** + * Runs a HTTP Server with to allow integration testing + */ +public class HttpServerRule extends ExternalResource +{ + public final int PORT = 3000; + private HttpServer server; + + @Override + protected void before() throws Throwable + { + server = HttpServer.create( new InetSocketAddress( PORT ), 0 ); + server.setExecutor( null ); // creates a default executor + server.start(); + } + + @Override + protected void after() + { + if ( server != null ) + { + server.stop( 0 ); // doesn't wait all current exchange handlers complete + } + } + + // Register a handler to provide desired behaviour on a specific uri path + public void registerHandler( String uriToHandle, HttpHandler httpHandler ) + { + if (!uriToHandle.startsWith( "/" )){ + uriToHandle = '/' + uriToHandle; + } + server.createContext( uriToHandle, httpHandler ); + } +} \ No newline at end of file diff --git a/src/test/resources/versions.json b/src/test/resources/versions.json new file mode 100644 index 00000000..0d958750 --- /dev/null +++ b/src/test/resources/versions.json @@ -0,0 +1,7 @@ +[ + { + "neo4j": "$NEO4J_VERSION", + "_testing": "SNAPSHOT", + "jar": "http://host.testcontainers.internal:3000/myPlugin.jar" + } +] \ No newline at end of file From 8e0902a3a6c70d08b161d15cc7130ea48389e590 Mon Sep 17 00:00:00 2001 From: Andrew Jefferson Date: Fri, 9 Aug 2019 15:07:52 +0000 Subject: [PATCH 3/5] bring TestPluginInstallation.java up to date with changes to SetContainerUser --- src/test/java/com/neo4j/docker/TestPluginInstallation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/neo4j/docker/TestPluginInstallation.java b/src/test/java/com/neo4j/docker/TestPluginInstallation.java index 91cd6d55..c4e64bd8 100644 --- a/src/test/java/com/neo4j/docker/TestPluginInstallation.java +++ b/src/test/java/com/neo4j/docker/TestPluginInstallation.java @@ -65,7 +65,7 @@ private void createContainerWithTestingPlugin() .withExposedPorts( DEFAULT_BROWSER_PORT, DEFAULT_BOLT_PORT ) .withLogConsumer( new Slf4jLogConsumer( log ) ); - SetContainerUser.currentlyRunningUser( container ); + SetContainerUser.nonRootUser( container ); } @BeforeEach From 962557a918f79bc837678d6d5a51a88ea492946d Mon Sep 17 00:00:00 2001 From: Andrew Jefferson Date: Sun, 25 Aug 2019 22:53:20 +0100 Subject: [PATCH 4/5] adding install-plugins-at-runtime to 4.0 code --- docker-image-src/4.0/Dockerfile | 9 +++-- docker-image-src/4.0/docker-entrypoint.sh | 42 +++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/docker-image-src/4.0/Dockerfile b/docker-image-src/4.0/Dockerfile index 8d1199ab..1a509aaa 100644 --- a/docker-image-src/4.0/Dockerfile +++ b/docker-image-src/4.0/Dockerfile @@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J COPY ./local-package/* /tmp/ RUN apt update \ - && apt install -y curl gosu \ + && apt install -y curl gosu jq \ && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \ && echo "${TINI_SHA256} /sbin/tini" | sha256sum -c --strict --quiet \ && chmod +x /sbin/tini \ @@ -31,7 +31,10 @@ RUN apt update \ && chown -R neo4j:neo4j "${NEO4J_HOME}" \ && chmod -R 777 "${NEO4J_HOME}" \ && ln -s /data "${NEO4J_HOME}"/data \ - && ln -s /logs "${NEO4J_HOME}"/logs + && ln -s /logs "${NEO4J_HOME}"/logs \ + && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \ + && rm -rf /tmp/* \ + && rm -rf /var/lib/apt/lists/* ENV PATH "${NEO4J_HOME}"/bin:$PATH @@ -44,4 +47,4 @@ COPY docker-entrypoint.sh /docker-entrypoint.sh EXPOSE 7474 7473 7687 ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"] -CMD ["neo4j"] \ No newline at end of file +CMD ["neo4j"] diff --git a/docker-image-src/4.0/docker-entrypoint.sh b/docker-image-src/4.0/docker-entrypoint.sh index 841ed8ef..1d9a402d 100755 --- a/docker-image-src/4.0/docker-entrypoint.sh +++ b/docker-image-src/4.0/docker-entrypoint.sh @@ -119,6 +119,38 @@ function check_mounted_folder_with_chown fi } +function load_plugin_from_github +{ + # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the + # correct format. + local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql + + local _plugins_dir="${NEO4J_HOME}/plugins" + if [ -d /plugins ]; then + local _plugins_dir="/plugins" + fi + local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )" + # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin + local _destination="${_plugins_dir}/${_plugin_name}.jar" + local _neo4j_version="$(neo4j --version | cut -d' ' -f2)" + + # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version + echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}" + local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")" + local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")" + if [[ -z "${_plugin_jar_url}" ]]; then + echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'" + echo >&2 "${_versions_json}" + fi + echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} " + curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}" + + if ! is_readable "${_destination}"; then + echo >&2 "Plugin at '${_destination}' is not readable" + exit 1 + fi +} + # If we're running as root, then run as the neo4j user. Otherwise # docker is running with --user and we simply use that user. Note # that su-exec, despite its name, does not replicate the functionality @@ -255,6 +287,10 @@ fi if [ -d /plugins ]; then if secure_mode_enabled; then + if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # We need write permissions + check_mounted_folder_with_chown "/plugins" + fi check_mounted_folder_readable "/plugins" fi NEO4J_dbms_directories_plugins="/plugins" @@ -330,6 +366,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do fi done +if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then + # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]' + for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do + load_plugin_from_github "${plugin_name}" + done +fi [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT} From 9713b2acece9bdd928b99bf1270ef31b18ae0ffb Mon Sep 17 00:00:00 2001 From: Andrew Jefferson Date: Sun, 25 Aug 2019 23:36:36 +0100 Subject: [PATCH 5/5] Tweaks to pom and makefile so neo4j version env var name stuff works everywhere --- Makefile | 2 +- README.md | 2 +- devenv | 3 +++ devenv.local.template | 2 +- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e148b6cf..1f581723 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ ifndef NEO4J_VERSION endif tarball = neo4j-$(1)-$(2)-unix.tar.gz -dist_site := http://dist.neo4j.org +dist_site := https://dist.neo4j.org series := $(shell echo "$(NEO4J_VERSION)" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/') all: test diff --git a/README.md b/README.md index 2c8d37fc..44af3c20 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ docker run \ ## Neo4j 3.0 -Documentation for the Neo4j 3.0 image can be found [here](http://neo4j.com/docs/operations-manual/current/deployment/single-instance/docker/). +Documentation for the Neo4j 3.0 image can be found [here](https://neo4j.com/docs/operations-manual/current/deployment/single-instance/docker/). You can start a Neo4j 3.0 container like this: diff --git a/devenv b/devenv index 5b041e72..a85e3317 100644 --- a/devenv +++ b/devenv @@ -71,6 +71,9 @@ fi if [[ -f devenv.local ]]; then source devenv.local + # to be consistent with the rest of neo4j we should use NEO4JVERSION exclusively but unfortunately both with and without underscore are used in this repo + export NEO4JVERSION + NEO4J_VERSION="${NEO4JVERSION}" export NEO4J_VERSION else echo >&2 "Error: cannot find devenv.local" diff --git a/devenv.local.template b/devenv.local.template index 4ad8c6eb..9085a5d5 100644 --- a/devenv.local.template +++ b/devenv.local.template @@ -1,3 +1,3 @@ # -*- mode: shell-script -*- -NEO4J_VERSION= +NEO4JVERSION= \ No newline at end of file