Revert "Revert "Merge pull request #181 from eastlondoner/andy/instal…

…l-apoc-at-runtime"" This reverts commit 84425d9. Which re-enables plugin installation at runtime
neo4j · Aug 9, 2019 · ee950ed · ee950ed
1 parent fa72c38
commit ee950ed
Show file tree

Hide file tree

Showing 15 changed files with 480 additions and 4 deletions.
diff --git a/Makefile b/Makefile
@@ -59,11 +59,16 @@ tmp/.image-id-%: tmp/local-context-%/.sentinel
     $(<D)
 > echo -n $$image >$@
 
-tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION))
+tmp/neo4jlabs-plugins.json: ./neo4jlabs-plugins.json
+> mkdir -p $(@D)
+> cp $< $@
+
+tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION)) tmp/neo4jlabs-plugins.json
 > rm -rf $(@D)
 > mkdir -p $(@D)
 > cp -r $(<D)/* $(@D)
 > cp $(filter %.tar.gz,$^) $(@D)/local-package
+> cp $(filter %.json,$^) $(@D)/local-package
 > touch $@
 
 tmp/image-%/.sentinel: docker-image-src/$(series)/Dockerfile docker-image-src/$(series)/docker-entrypoint.sh \

diff --git a/docker-image-src/3.3/Dockerfile b/docker-image-src/3.3/Dockerfile
@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

diff --git a/docker-image-src/3.3/docker-entrypoint.sh b/docker-image-src/3.3/docker-entrypoint.sh
@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -266,6 +298,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -341,6 +377,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

diff --git a/docker-image-src/3.4/Dockerfile b/docker-image-src/3.4/Dockerfile
@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

diff --git a/docker-image-src/3.4/docker-entrypoint.sh b/docker-image-src/3.4/docker-entrypoint.sh
@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -259,6 +291,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

diff --git a/docker-image-src/3.5/Dockerfile b/docker-image-src/3.5/Dockerfile
@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

diff --git a/docker-image-src/3.5/docker-entrypoint.sh b/docker-image-src/3.5/docker-entrypoint.sh
@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -259,6 +291,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

diff --git a/neo4jlabs-plugins.json b/neo4jlabs-plugins.json
@@ -0,0 +1,7 @@
+{
+  "apoc": "https://github.com/neo4j-contrib/neo4j-apoc-procedures/raw/master/versions.json",
+  "streams": "https://github.com/neo4j-contrib/neo4j-streams/raw/master/versions.json",
+  "graphql": "https://github.com/neo4j-contrib/neo4j-graphql/raw/master/versions.json",
+  "graph-algorithms": "https://github.com/neo4j-contrib/neo4j-graph-algorithms/raw/master/versions.json",
+  "_testing": "http://host.testcontainers.internal:3000/versions.json"
+}
diff --git a/pom.xml b/pom.xml
@@ -9,6 +9,10 @@
 	<version>1.0-SNAPSHOT</version>
 	<packaging>jar</packaging>
 
+	<properties>
+		<neo4j.version>${env.NEO4JVERSION}</neo4j.version>
+	</properties>
+
 	<build>
 		<plugins>
 			<plugin>
@@ -28,6 +32,16 @@
 	</build>
 
 	<dependencies>
+		<dependency>
+			<!-- This gives us the Procedure API our runtime code uses.
+                 We have a `provided` scope on it, because when this is
+                 deployed in a Neo4j Instance, the API will be provided
+                 by Neo4j. -->
+			<groupId>org.neo4j</groupId>
+			<artifactId>neo4j</artifactId>
+			<version>${neo4j.version}</version>
+			<scope>provided</scope>
+		</dependency>
 		<dependency>
 			<!-- logging library -->
 			<groupId>org.slf4j</groupId>
@@ -57,6 +71,12 @@
 			<version>5.4.2</version>
 			<scope>test</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.junit.jupiter</groupId>
+			<artifactId>junit-jupiter-migrationsupport</artifactId>
+			<version>5.4.2</version>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.testcontainers</groupId>
 			<artifactId>junit-jupiter</artifactId>