tools/dev - updated intellij script to start controller and invoker l…

…ocally (#4142)

* tools/dev - updated intellij script for docker-compose
* Added support for docker for mac
* log-limit is 0 to avoid collection logs
* updates for wskdev with ansible

ansible/group_vars/all

@@ -172,7 +172,7 @@ invoker:
   port: 12001
   heap: "{{ invoker_heap | default('2g') }}"
   arguments: "{{ invoker_arguments | default('') }}"
-  userMemory: "{{ invoker_user_memory | default('2048 m') }}"
+  userMemory: "{{ invoker_user_memory | default('2048m') }}"
   instances: "{{ groups['invokers'] | length }}"
   # Specify if it is allowed to deploy more than 1 invoker on a single machine.
   allowMultipleInstances: "{{ invoker_allow_multiple_instances | default(false) }}"

core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerClient.scala

@@ -178,7 +178,7 @@ class DockerClient(dockerHost: Option[String] = None,
   def isOomKilled(id: ContainerId)(implicit transid: TransactionId): Future[Boolean] =
     runCmd(Seq("inspect", id.asString, "--format", "{{.State.OOMKilled}}"), config.timeouts.inspect).map(_.toBoolean)
 
-  private def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
+  protected def runCmd(args: Seq[String], timeout: Duration)(implicit transid: TransactionId): Future[String] = {
     val cmd = dockerCmd ++ args
     val start = transid.started(
       this,

core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerContainerFactory.scala

@@ -18,6 +18,7 @@
 package org.apache.openwhisk.core.containerpool.docker
 
 import akka.actor.ActorSystem
+
 import scala.concurrent.Await
 import scala.concurrent.ExecutionContext
 import scala.concurrent.Future
@@ -27,14 +28,16 @@ import org.apache.openwhisk.core.WhiskConfig
 import org.apache.openwhisk.core.containerpool.Container
 import org.apache.openwhisk.core.containerpool.ContainerFactory
 import org.apache.openwhisk.core.containerpool.ContainerFactoryProvider
+import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
 import org.apache.openwhisk.core.entity.ByteSize
 import org.apache.openwhisk.core.entity.ExecManifest
 import org.apache.openwhisk.core.entity.InvokerInstanceId
+
 import scala.concurrent.duration._
 import java.util.concurrent.TimeoutException
+
 import pureconfig._
 import org.apache.openwhisk.core.ConfigKeys
-import org.apache.openwhisk.core.containerpool.ContainerArgsConfig
 
 case class DockerContainerFactoryConfig(useRunc: Boolean)
 

core/invoker/src/main/scala/org/apache/openwhisk/core/containerpool/docker/DockerForMacContainerFactory.scala

@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.containerpool.docker
+
+import akka.actor.ActorSystem
+import org.apache.openwhisk.common.{Logging, TransactionId}
+import org.apache.openwhisk.core.WhiskConfig
+import org.apache.openwhisk.core.containerpool._
+import org.apache.openwhisk.core.entity.InvokerInstanceId
+
+import scala.concurrent.{ExecutionContext, Future}
+import scala.concurrent.duration._
+
+/**
+ * This factory provides a Docker for Mac client which exposes action container's ports on the host.
+ */
+object DockerForMacContainerFactoryProvider extends ContainerFactoryProvider {
+  override def instance(actorSystem: ActorSystem,
+                        logging: Logging,
+                        config: WhiskConfig,
+                        instanceId: InvokerInstanceId,
+                        parameters: Map[String, Set[String]]): ContainerFactory = {
+
+    new DockerContainerFactory(instanceId, parameters)(
+      actorSystem,
+      actorSystem.dispatcher,
+      logging,
+      new DockerForMacClient()(actorSystem.dispatcher)(logging, actorSystem),
+      new RuncClient()(actorSystem.dispatcher)(logging, actorSystem))
+  }
+
+}
+
+class DockerForMacClient(dockerHost: Option[String] = None)(executionContext: ExecutionContext)(implicit log: Logging,
+                                                                                                as: ActorSystem)
+    extends DockerClientWithFileAccess(dockerHost)(executionContext)
+    with DockerApiWithFileAccess {
+
+  implicit private val ec: ExecutionContext = executionContext
+
+  override def run(image: String, args: Seq[String] = Seq.empty[String])(
+    implicit transid: TransactionId): Future[ContainerId] = {
+    // b/c docker for mac doesn't have a routing to the action containers
+    // the port 8080 is exposed on the host on a random port number
+    val extraArgs: Seq[String] = Seq("-p", "0:8080") ++ args
+    super.run(image, extraArgs)
+  }
+  // See extended trait for description
+  override def inspectIPAddress(id: ContainerId, network: String)(
+    implicit transid: TransactionId): Future[ContainerAddress] = {
+    super
+      .runCmd(
+        Seq("inspect", "--format", """{{(index (index .NetworkSettings.Ports "8080/tcp") 0).HostPort}}""", id.asString),
+        10.seconds)
+      .flatMap {
+        case "<no value>" => Future.failed(new NoSuchElementException)
+        case stdout       => Future.successful(ContainerAddress("localhost", stdout.toInt))
+      }
+  }
+}

tools/dev/README.md

@@ -76,14 +76,28 @@ First setup OpenWhisk so that Controller and Invoker containers are up and runni
 It would inspect the running docker containers and then generate the launch configs with name 'controller0'
 and 'invoker0'.
 
-Key points to note:
+Now the docker container(s) (controller and/or invoker) can be stopped and they can be launched instead from within the IDE.
 
-1. Uses ~/tmp/openwhisk/controller (or invoker) as working directory.
-2. Changes the PORT to linked one. So controller gets started at 10001 only just like as its done in container.
+Key points to note:
 
-Now the docker container can be stopped and application can be launched from within the IDE.
+1. Controller uses port `10001` and Invoker uses port `12001`.
+2. Action activation logs are [disabled][2].
+3. SSL is disabled for Controller and Invoker.
+4. Make sure you have the loopback interface configured:
+   ```bash
+   sudo ifconfig lo0 alias 172.17.0.1/24
+   ```
+5. `~/.wskprops` must be updated with `APIHOST=http://localhost:10001` so that the `wsk` CLI communicates directly with the controller.
+6. On a MAC
+   * With Docker For Mac the invoker is configured to use a Container Factory that exposes ports for actions on the host,
+     as otherwise the invoker can't make HTTP requests to the actions.
+     You can read more at [docker/for-mac#171][7].
+
+   * When using [docker-compose][8] locally you have to update `/etc/hosts` with the line bellow:
+      ```
+      127.0.0.1       kafka zookeeper kafka.docker zookeeper.docker db.docker controller whisk.controller
+      ```
 
-**Note** - Currently only the controller can be run from IDE. Invoker posses some [problems][2].
 
 ### Configuration
 
@@ -168,3 +182,5 @@ $ ./gradlew :tools:dev:renderModuleDetails
 [4]: http://docs.groovy-lang.org/2.4.2/html/gapi/groovy/util/ConfigSlurper.html
 [5]: https://developer.github.com/v3/search/
 [6]: https://help.github.com/articles/creating-a-personal-access-token-for-the-command-line/
+[7]: https://github.com/docker/for-mac/issues/171
+[8]: https://github.com/apache/incubator-openwhisk-devtools/tree/master/docker-compose

tools/dev/build.gradle

@@ -26,6 +26,7 @@ def owHome = project.projectDir.parentFile.parentFile
 dependencies {
     compile "org.codehaus.groovy:groovy-all:2.4.14"
     compile 'org.apache.commons:commons-io:1.3.2'
+    compile 'org.apache.commons:commons-lang3:3.8.1'
 }
 
 task couchdbViews(type: JavaExec) {

tools/dev/src/main/groovy/intellijRunConfig.groovy

@@ -16,6 +16,7 @@
  */
 import groovy.json.JsonSlurper
 import groovy.text.SimpleTemplateEngine
+import org.apache.commons.lang3.SystemUtils
 
 assert args : "Expecting the OpenWhisk home directory to passed"
 owHome = args[0]
@@ -26,7 +27,7 @@ def configTemplate = '''<component name="ProjectRunConfigurationManager">
     <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
     <option name="MAIN_CLASS_NAME" value="$main" />
     <option name="VM_PARAMETERS" value="$sysProps" />
-    <option name="PROGRAM_PARAMETERS" value="0" />
+    <option name="PROGRAM_PARAMETERS" value="$programParams" />
     <option name="WORKING_DIRECTORY" value="$workingDir" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
     <option name="ALTERNATIVE_JRE_PATH" />
@@ -57,13 +58,15 @@ Map controllerEnv = null
 Map invokerEnv = null
 
 containerNames.each{cn ->
+
     //Inspect the specific container
     def inspectResult = "docker inspect $cn".execute().text
     def json = new JsonSlurper().parseText(inspectResult)
 
     def imageName = json[0].'Config'.'Image'
     if (imageName.contains("controller") || imageName.contains("invoker")){
-        def mappedPort = json.'NetworkSettings'.'Ports'.'8080/tcp'[0][0].'HostPort'
+        // pre-configure the local ports for controller and invoker
+        def mappedPort = imageName.contains("controller") ? '10001' : '12001'
 
         def envBaseMap = getEnvMap(json[0].'Config'.'Env')
         String type
@@ -84,6 +87,14 @@ containerNames.each{cn ->
 
         //Prepare system properties
         def sysProps = getSysProps(envMap,type)
+        // disable log collection. See more at: https://github.com/apache/incubator-openwhisk/issues/3195
+        sysProps += " -Dwhisk.log-limit.max=0 -Dwhisk.log-limit.std=0"
+        // disable https protocol for controller and invoker
+        sysProps = sysProps.replaceAll("protocol=https", "protocol=http")
+        if (SystemUtils.IS_OS_MAC){
+            sysProps = sysProps.replaceAll("use-runc=True", "use-runc=False")
+            sysProps += " -Dwhisk.spi.ContainerFactoryProvider=org.apache.openwhisk.core.containerpool.docker.DockerForMacContainerFactoryProvider"
+        }
 
         def templateBinding = [
                 main: meta[type].main,
@@ -92,7 +103,8 @@ containerNames.each{cn ->
                 env: encodeForXML(envMap),
                 sysProps : sysProps,
                 USER_HOME : '$USER_HOME$',
-                workingDir : getWorkDir(type)
+                workingDir : getWorkDir(type),
+                programParams: imageName.contains("controller") ? '0' : '--id  0'
         ]
 
         def engine = new SimpleTemplateEngine()
@@ -174,7 +186,7 @@ def getSysProps(def envMap, String type){
     def props = config[type].props
     def sysProps = transformEnv(envMap)
     sysProps.putAll(props)
-    sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('\'','')
+    sysProps.collect{k,v -> "-D$k='$v'"}.join(' ').replace('"','').replace('\'','')
 }
 
 //Implements the logic from transformEnvironment.sh
@@ -213,7 +225,7 @@ def transformEnv(Map<String, String> envMap){
  * This method converts it to map and add provided overrides with overrides from config
  */
 def getEnv(Map envMap, String type, Map overrides){
-    def ignoredKeys = ['PATH']
+    def ignoredKeys = ['PATH','JAVA_HOME','JAVA_VERSION','JAVA_TOOL_OPTIONS']
     def overridesFromConfig = config[type].env
     Map sortedMap = new TreeMap(envMap)
     sortedMap.putAll(overrides)