Merge pull request #188 from eastlondoner/andy/install-apoc-at-runtime

(re)enables plugin installation at runtime

Makefile

@@ -15,7 +15,7 @@ ifndef NEO4J_VERSION
 endif
 
 tarball = neo4j-$(1)-$(2)-unix.tar.gz
-dist_site := http://dist.neo4j.org
+dist_site := https://dist.neo4j.org
 series := $(shell echo "$(NEO4J_VERSION)" | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1/')
 
 all: test
@@ -59,11 +59,16 @@ tmp/.image-id-%: tmp/local-context-%/.sentinel
     $(<D)
 > echo -n $$image >$@
 
-tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION))
+tmp/neo4jlabs-plugins.json: ./neo4jlabs-plugins.json
+> mkdir -p $(@D)
+> cp $< $@
+
+tmp/local-context-%/.sentinel: tmp/image-%/.sentinel in/$(call tarball,%,$(NEO4J_VERSION)) tmp/neo4jlabs-plugins.json
 > rm -rf $(@D)
 > mkdir -p $(@D)
 > cp -r $(<D)/* $(@D)
 > cp $(filter %.tar.gz,$^) $(@D)/local-package
+> cp $(filter %.json,$^) $(@D)/local-package
 > touch $@
 
 tmp/image-%/.sentinel: docker-image-src/$(series)/Dockerfile docker-image-src/$(series)/docker-entrypoint.sh \

README.md

@@ -18,7 +18,7 @@ docker run \
 
 ## Neo4j 3.0
 
-Documentation for the Neo4j 3.0 image can be found [here](http://neo4j.com/docs/operations-manual/current/deployment/single-instance/docker/).
+Documentation for the Neo4j 3.0 image can be found [here](https://neo4j.com/docs/operations-manual/current/deployment/single-instance/docker/).
 
 You can start a Neo4j 3.0 container like this:
 

devenv

@@ -71,6 +71,9 @@ fi
 
 if [[ -f devenv.local ]]; then
     source devenv.local
+    # to be consistent with the rest of neo4j we should use NEO4JVERSION exclusively but unfortunately both with and without underscore are used in this repo
+    export NEO4JVERSION
+    NEO4J_VERSION="${NEO4JVERSION}"
     export NEO4J_VERSION
 else
     echo >&2 "Error: cannot find devenv.local"

devenv.local.template

@@ -1,3 +1,3 @@
 # -*- mode: shell-script -*-
 
-NEO4J_VERSION=<the-version-you-want-to-test-against>
+NEO4JVERSION=<the-version-you-want-to-test-against>

docker-image-src/3.3/Dockerfile

@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

docker-image-src/3.3/docker-entrypoint.sh

@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -266,6 +298,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -341,6 +377,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

docker-image-src/3.4/Dockerfile

@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

docker-image-src/3.4/docker-entrypoint.sh

@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -259,6 +291,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

docker-image-src/3.5/Dockerfile

@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -32,6 +32,7 @@ RUN apt update \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
     && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
     && rm -rf /tmp/* \
     && rm -rf /var/lib/apt/lists/*
 

docker-image-src/3.5/docker-entrypoint.sh

@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -259,6 +291,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -334,6 +370,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

docker-image-src/4.0/Dockerfile

@@ -13,7 +13,7 @@ RUN addgroup --system neo4j && adduser --system --no-create-home --home "${NEO4J
 COPY ./local-package/* /tmp/
 
 RUN apt update \
-    && apt install -y curl gosu \
+    && apt install -y curl gosu jq \
     && curl -L --fail --silent --show-error "https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini" > /sbin/tini \
     && echo "${TINI_SHA256}  /sbin/tini" | sha256sum -c --strict --quiet \
     && chmod +x /sbin/tini \
@@ -31,7 +31,10 @@ RUN apt update \
     && chown -R neo4j:neo4j "${NEO4J_HOME}" \
     && chmod -R 777 "${NEO4J_HOME}" \
     && ln -s /data "${NEO4J_HOME}"/data \
-    && ln -s /logs "${NEO4J_HOME}"/logs
+    && ln -s /logs "${NEO4J_HOME}"/logs \
+    && mv /tmp/neo4jlabs-plugins.json /neo4jlabs-plugins.json \
+    && rm -rf /tmp/* \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV PATH "${NEO4J_HOME}"/bin:$PATH
 
@@ -44,4 +47,4 @@ COPY docker-entrypoint.sh /docker-entrypoint.sh
 EXPOSE 7474 7473 7687
 
 ENTRYPOINT ["/sbin/tini", "-g", "--", "/docker-entrypoint.sh"]
-CMD ["neo4j"]
+CMD ["neo4j"]

docker-image-src/4.0/docker-entrypoint.sh

@@ -119,6 +119,38 @@ function check_mounted_folder_with_chown
     fi
 }
 
+function load_plugin_from_github
+{
+  # Load a plugin at runtime. The provided github repository must have a versions.json on the master branch with the
+  # correct format.
+  local _plugin_name="${1}" #e.g. apoc, graph-algorithms, graph-ql
+
+  local _plugins_dir="${NEO4J_HOME}/plugins"
+  if [ -d /plugins ]; then
+    local _plugins_dir="/plugins"
+  fi
+  local _versions_json_url="$(jq --raw-output "with_entries( select(.key==\"${_plugin_name}\") ) | to_entries[] | .value" /neo4jlabs-plugins.json )"
+  # Using the same name for the plugin irrespective of version ensures we don't end up with different versions of the same plugin
+  local _destination="${_plugins_dir}/${_plugin_name}.jar"
+  local _neo4j_version="$(neo4j --version | cut -d' ' -f2)"
+
+  # Now we call out to github to get the versions.json for this plugin and we parse that to find the url for the correct plugin jar for our neo4j version
+  echo "Fetching versions.json for Plugin '${_plugin_name}' from ${_versions_json_url}"
+  local _versions_json="$(curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L "${_versions_json_url}")"
+  local _plugin_jar_url="$(echo "${_versions_json}" | jq --raw-output ".[] | select(.neo4j==\"${_neo4j_version}\") | .jar")"
+  if [[ -z "${_plugin_jar_url}" ]]; then
+    echo >&2 "No jar URL found for version '${_neo4j_version}' in versions.json from '${_versions_json_url}'"
+    echo >&2 "${_versions_json}"
+  fi
+  echo "Installing Plugin '${_plugin_name}' from ${_plugin_jar_url} to ${_destination} "
+  curl --silent --show-error --fail --retry 30 --retry-max-time 300 -L -o "${_destination}" "${_plugin_jar_url}"
+
+  if ! is_readable "${_destination}"; then
+    echo >&2 "Plugin at '${_destination}' is not readable"
+    exit 1
+  fi
+}
+
 # If we're running as root, then run as the neo4j user. Otherwise
 # docker is running with --user and we simply use that user.  Note
 # that su-exec, despite its name, does not replicate the functionality
@@ -255,6 +287,10 @@ fi
 
 if [ -d /plugins ]; then
     if secure_mode_enabled; then
+        if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+            # We need write permissions
+            check_mounted_folder_with_chown "/plugins"
+        fi
         check_mounted_folder_readable "/plugins"
     fi
     NEO4J_dbms_directories_plugins="/plugins"
@@ -330,6 +366,12 @@ for i in $( set | grep ^NEO4J_ | awk -F'=' '{print $1}' | sort -rn ); do
     fi
 done
 
+if [[ ! -z "${NEO4JLABS_PLUGINS:-}" ]]; then
+  # NEO4JLABS_PLUGINS should be a json array of plugins like '["graph-algorithms", "apoc-procedures", "streams", "graphql"]'
+  for plugin_name in $(echo "${NEO4JLABS_PLUGINS}" | jq --raw-output '.[]'); do
+    load_plugin_from_github "${plugin_name}"
+  done
+fi
 
 [ -f "${EXTENSION_SCRIPT:-}" ] && . ${EXTENSION_SCRIPT}
 

neo4jlabs-plugins.json

@@ -0,0 +1,7 @@
+{
+  "apoc": "https://github.com/neo4j-contrib/neo4j-apoc-procedures/raw/master/versions.json",
+  "streams": "https://github.com/neo4j-contrib/neo4j-streams/raw/master/versions.json",
+  "graphql": "https://github.com/neo4j-contrib/neo4j-graphql/raw/master/versions.json",
+  "graph-algorithms": "https://github.com/neo4j-contrib/neo4j-graph-algorithms/raw/master/versions.json",
+  "_testing": "http://host.testcontainers.internal:3000/versions.json"
+}