From f323e97151f449992e1c88914cabce6f7c8615d5 Mon Sep 17 00:00:00 2001 From: Antonio Piccolboni Date: Thu, 15 May 2014 19:05:57 -0700 Subject: [PATCH 1/2] tentative fix for amplab-extras/SparkR-pkg#53 --- pkg/inst/worker/worker.R | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/pkg/inst/worker/worker.R b/pkg/inst/worker/worker.R index 5a2f5a9112c7a..8046f4fd3b608 100644 --- a/pkg/inst/worker/worker.R +++ b/pkg/inst/worker/worker.R @@ -39,16 +39,22 @@ sink(stderr()) # read function dependencies depsLen <- readInt(inputCon) if (depsLen > 0) { - execFunctionDeps <- readRawLen(inputCon, depsLen) - - # load the dependencies into current environment - depsFileName <- tempfile(pattern="spark-exec", fileext=".deps") - depsFile <- file(depsFileName, open="wb") - writeBin(execFunctionDeps, depsFile, endian="big") - close(depsFile) - - load(depsFileName) - unlink(depsFileName) + execFunctionDeps <- readRawLen(inputCon, depsLen) +} +# Include packages as required +packageNames <- unserialize(readRaw(inputCon)) +for (pkg in packageNames) { + suppressPackageStartupMessages(require(as.character(pkg), character.only=TRUE)) +} +if (depsLen > 0) { + # load the dependencies into current environment + depsFileName <- tempfile(pattern="spark-exec", fileext=".deps") + depsFile <- file(depsFileName, open="wb") + writeBin(execFunctionDeps, depsFile, endian="big") + close(depsFile) + + load(depsFileName) + unlink(depsFileName) } # Include packages as required From b3c318d357b74cc51442b213f3604afb1e6a264e Mon Sep 17 00:00:00 2001 From: Antonio Piccolboni Date: Fri, 16 May 2014 11:53:13 -0700 Subject: [PATCH 2/2] delayed loading to have all namespaces available. --- pkg/inst/worker/worker.R | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pkg/inst/worker/worker.R b/pkg/inst/worker/worker.R index 5a2f5a9112c7a..0370f84c095bc 100644 --- a/pkg/inst/worker/worker.R +++ b/pkg/inst/worker/worker.R @@ -46,9 +46,6 @@ if (depsLen > 0) { depsFile <- file(depsFileName, open="wb") writeBin(execFunctionDeps, depsFile, endian="big") close(depsFile) - - load(depsFileName) - unlink(depsFileName) } # Include packages as required @@ -57,6 +54,11 @@ for (pkg in packageNames) { suppressPackageStartupMessages(require(as.character(pkg), character.only=TRUE)) } +if (depsLen > 0) { + load(depsFileName) + unlink(depsFileName) +} + # Read and set broadcast variables numBroadcastVars <- readInt(inputCon) if (numBroadcastVars > 0) {