diff --git a/.gitignore b/.gitignore
index 06a1e6fedb671..21d2106f97ca6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@
 /target
 .DS_Store
 *.iml
+.local-execution-hints.log
diff --git a/pom.xml b/pom.xml
index 2c7aa5461f96e..b6d5fd71a3baa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -30,7 +30,13 @@
     </parent>
 
     <properties>
-        <elasticsearch.version>0.90.0.Beta1</elasticsearch.version>
+        <elasticsearch.version>0.90.10</elasticsearch.version>
+        <lucene.version>4.6.0</lucene.version>
+        <tests.jvms>1</tests.jvms>
+        <tests.shuffle>true</tests.shuffle>
+        <tests.output>onerror</tests.output>
+        <tests.client.ratio></tests.client.ratio>
+        <es.logger.level>INFO</es.logger.level>
     </properties>
 
     <repositories>
@@ -41,6 +47,13 @@
     </repositories>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.lucene</groupId>
+            <artifactId>lucene-test-framework</artifactId>
+            <version>${lucene.version}</version>
+            <scope>test</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.elasticsearch</groupId>
             <artifactId>elasticsearch</artifactId>
@@ -63,9 +76,10 @@
         </dependency>
 
         <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <version>6.3.1</version>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>${elasticsearch.version}</version>
+            <type>test-jar</type>
             <scope>test</scope>
         </dependency>
 
@@ -96,13 +110,95 @@
                 </configuration>
             </plugin>
             <plugin>
+                <groupId>com.carrotsearch.randomizedtesting</groupId>
+                <artifactId>junit4-maven-plugin</artifactId>
+                <version>2.0.12</version>
+                <executions>
+                    <execution>
+                        <id>tests</id>
+                        <phase>test</phase>
+                        <goals>
+                            <goal>junit4</goal>
+                        </goals>
+                        <configuration>
+                            <heartbeat>20</heartbeat>
+                            <jvmOutputAction>pipe,warn</jvmOutputAction>
+                            <leaveTemporary>true</leaveTemporary>
+                            <listeners>
+                                <report-ant-xml mavenExtensions="true"
+                                                dir="${project.build.directory}/surefire-reports"/>
+                                <report-text
+                                        showThrowable="true"
+                                        showStackTraces="true"
+                                        showOutput="${tests.output}"
+                                        showStatusOk="false"
+                                        showStatusError="true"
+                                        showStatusFailure="true"
+                                        showStatusIgnored="true"
+                                        showSuiteSummary="true"
+                                        timestamps="false"/>
+                                <report-execution-times file="${basedir}/.local-execution-hints.log"/>
+                            </listeners>
+                            <assertions>
+                                <enable/>
+                            </assertions>
+                            <parallelism>${tests.jvms}</parallelism>
+                            <balancers>
+                                <execution-times>
+                                    <fileset dir="${basedir}" includes=".local-execution-hints.log"/>
+                                </execution-times>
+                            </balancers>
+                            <includes>
+                                <include>**/*Tests.class</include>
+                                <include>**/*Test.class</include>
+                            </includes>
+                            <excludes>
+                                <exclude>**/Abstract*.class</exclude>
+                                <exclude>**/*StressTest.class</exclude>
+                            </excludes>
+                            <jvmArgs>
+                                <param>-Xmx512m</param>
+                                <param>-XX:MaxDirectMemorySize=512m</param>
+                                <param>-Des.logger.prefix=</param>
+                            </jvmArgs>
+                            <shuffleOnSlave>${tests.shuffle}</shuffleOnSlave>
+                            <sysouts>${tests.verbose}</sysouts>
+                            <seed>${tests.seed}</seed>
+                            <haltOnFailure>${tests.failfast}</haltOnFailure>
+                            <systemProperties>
+                                <!-- RandomizedTesting library system properties -->
+                                <tests.iters>${tests.iters}</tests.iters>
+                                <tests.maxfailures>${tests.maxfailures}</tests.maxfailures>
+                                <tests.failfast>${tests.failfast}</tests.failfast>
+                                <tests.class>${tests.class}</tests.class>
+                                <tests.method>${tests.method}</tests.method>
+                                <tests.nightly>${tests.nightly}</tests.nightly>
+                                <tests.badapples>${tests.badapples}</tests.badapples>
+                                <tests.weekly>${tests.weekly}</tests.weekly>
+                                <tests.slow>${tests.slow}</tests.slow>
+                                <tests.awaitsfix>${tests.awaitsfix}</tests.awaitsfix>
+                                <tests.slow>${tests.slow}</tests.slow>
+                                <tests.timeoutSuite>${tests.timeoutSuite}</tests.timeoutSuite>
+                                <tests.showSuccess>${tests.showSuccess}</tests.showSuccess>
+                                <tests.integration>${tests.integration}</tests.integration>
+                                <tests.cluster_seed>${tests.cluster_seed}</tests.cluster_seed>
+                                <tests.client.ratio>${tests.client.ratio}</tests.client.ratio>
+                                <es.node.local>${env.ES_TEST_LOCAL}</es.node.local>
+                                <es.node.mode>${es.node.mode}</es.node.mode>
+                                <es.logger.level>${es.logger.level}</es.logger.level>
+                                <java.awt.headless>true</java.awt.headless>
+                            </systemProperties>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <!-- we skip surefire to work with randomized testing above -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.11</version>
+                <version>2.15</version>
                 <configuration>
-                    <includes>
-                        <include>**/*Tests.java</include>
-                    </includes>
+                    <skipTests>true</skipTests>
                 </configuration>
             </plugin>
             <plugin>
diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
index cce5728b25b60..92435ea9be3d4 100644
--- a/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
+++ b/src/test/java/org/elasticsearch/script/python/PythonScriptEngineTests.java
@@ -23,32 +23,34 @@
 import org.elasticsearch.common.collect.MapBuilder;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.script.ExecutableScript;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
 /**
  *
  */
-@Test
-public class PythonScriptEngineTests {
+public class PythonScriptEngineTests extends ElasticsearchTestCase {
 
     private PythonScriptEngineService se;
 
-    @BeforeClass
+    @Before
     public void setup() {
         se = new PythonScriptEngineService(ImmutableSettings.Builder.EMPTY_SETTINGS);
     }
 
-    @AfterClass
+    @After
     public void close() {
+        // We need to clear some system properties
+        System.clearProperty("python.cachedir.skip");
+        System.clearProperty("python.console.encoding");
         se.close();
     }
 
@@ -95,6 +97,7 @@ public void testObjectMapInter() {
 
     @Test
     public void testAccessListInScript() {
+
         Map<String, Object> vars = new HashMap<String, Object>();
         Map<String, Object> obj2 = MapBuilder.<String, Object>newMapBuilder().put("prop2", "value2").map();
         Map<String, Object> obj1 = MapBuilder.<String, Object>newMapBuilder().put("prop1", "value1").put("obj2", obj2).map();
diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java
index 2caa518d4ea8e..0af0505eeca13 100644
--- a/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java
+++ b/src/test/java/org/elasticsearch/script/python/PythonScriptMultiThreadedTest.java
@@ -19,12 +19,12 @@
 
 package org.elasticsearch.script.python;
 
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.util.concurrent.jsr166y.ThreadLocalRandom;
 import org.elasticsearch.script.ExecutableScript;
-import org.testng.annotations.Test;
+import org.elasticsearch.test.ElasticsearchTestCase;
+import org.junit.After;
+import org.junit.Test;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -32,16 +32,19 @@
 import java.util.concurrent.CyclicBarrier;
 import java.util.concurrent.atomic.AtomicBoolean;
 
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
  *
  */
-@Test
-public class PythonScriptMultiThreadedTest {
+public class PythonScriptMultiThreadedTest extends ElasticsearchTestCase {
 
-    protected final ESLogger logger = Loggers.getLogger(getClass());
+    @After
+    public void close() {
+        // We need to clear some system properties
+        System.clearProperty("python.cachedir.skip");
+        System.clearProperty("python.console.encoding");
+    }
 
     @Test
     public void testExecutableNoRuntimeParams() throws Exception {
diff --git a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java
index 4b7cd28274841..642658d9b1c2e 100644
--- a/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java
+++ b/src/test/java/org/elasticsearch/script/python/PythonScriptSearchTests.java
@@ -21,17 +21,10 @@
 
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.client.Client;
-import org.elasticsearch.common.logging.ESLogger;
-import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.network.NetworkUtils;
-import org.elasticsearch.common.settings.ImmutableSettings;
-import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
 import org.elasticsearch.search.sort.SortOrder;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
+import org.elasticsearch.test.ElasticsearchIntegrationTest;
+import org.junit.After;
+import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.List;
@@ -42,55 +35,33 @@
 import static org.elasticsearch.index.query.FilterBuilders.scriptFilter;
 import static org.elasticsearch.index.query.QueryBuilders.*;
 import static org.elasticsearch.search.builder.SearchSourceBuilder.searchSource;
-import static org.hamcrest.MatcherAssert.assertThat;
 import static org.hamcrest.Matchers.equalTo;
 
 /**
  *
  */
-@Test
-public class PythonScriptSearchTests {
+public class PythonScriptSearchTests extends ElasticsearchIntegrationTest {
 
-    protected final ESLogger logger = Loggers.getLogger(getClass());
-
-    private Node node;
-
-    private Client client;
-
-    @BeforeMethod
-    public void createNodes() throws Exception {
-        node = NodeBuilder.nodeBuilder().settings(ImmutableSettings.settingsBuilder()
-                .put("path.data", "target/data")
-                .put("cluster.name", "test-cluster-" + NetworkUtils.getLocalAddress())
-                .put("gateway.type", "none")
-                .put("number_of_shards", 1)).node();
-        client = node.client();
-    }
-
-    @AfterMethod
-    public void closeNodes() {
-        client.close();
-        node.close();
+    @After
+    public void close() {
+        // We need to clear some system properties
+        System.clearProperty("python.cachedir.skip");
+        System.clearProperty("python.console.encoding");
     }
 
     @Test
     public void testPythonFilter() throws Exception {
-        client.admin().indices().prepareCreate("test").execute().actionGet();
-        client.prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())
-                .execute().actionGet();
-        client.admin().indices().prepareFlush().execute().actionGet();
-        client.prepareIndex("test", "type1", "2")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject())
-                .execute().actionGet();
-        client.admin().indices().prepareFlush().execute().actionGet();
-        client.prepareIndex("test", "type1", "3")
-                .setSource(jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject())
-                .execute().actionGet();
-        client.admin().indices().refresh(refreshRequest()).actionGet();
-
-        logger.info("running doc['num1'].value > 1");
-        SearchResponse response = client.prepareSearch()
+        wipeIndices("test");
+        createIndex("test");
+        index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject());
+        flush();
+        index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject());
+        flush();
+        index("test", "type1", "3", jsonBuilder().startObject().field("test", "value beck").field("num1", 3.0f).endObject());
+        refresh();
+
+        logger.info(" --> running doc['num1'].value > 1");
+        SearchResponse response = client().prepareSearch()
                 .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > 1").lang("python")))
                 .addSort("num1", SortOrder.ASC)
                 .addScriptField("sNum1", "python", "doc['num1'].value", null)
@@ -102,8 +73,8 @@ public void testPythonFilter() throws Exception {
         assertThat(response.getHits().getAt(1).id(), equalTo("3"));
         assertThat((Double) response.getHits().getAt(1).fields().get("sNum1").values().get(0), equalTo(3.0));
 
-        logger.info("running doc['num1'].value > param1");
-        response = client.prepareSearch()
+        logger.info(" --> running doc['num1'].value > param1");
+        response = client().prepareSearch()
                 .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", 2)))
                 .addSort("num1", SortOrder.ASC)
                 .addScriptField("sNum1", "python", "doc['num1'].value", null)
@@ -113,8 +84,8 @@ public void testPythonFilter() throws Exception {
         assertThat(response.getHits().getAt(0).id(), equalTo("3"));
         assertThat((Double) response.getHits().getAt(0).fields().get("sNum1").values().get(0), equalTo(3.0));
 
-        logger.info("running doc['num1'].value > param1");
-        response = client.prepareSearch()
+        logger.info(" --> running doc['num1'].value > param1");
+        response = client().prepareSearch()
                 .setQuery(filteredQuery(matchAllQuery(), scriptFilter("doc['num1'].value > param1").lang("python").addParam("param1", -1)))
                 .addSort("num1", SortOrder.ASC)
                 .addScriptField("sNum1", "python", "doc['num1'].value", null)
@@ -131,16 +102,16 @@ public void testPythonFilter() throws Exception {
 
     @Test
     public void testScriptFieldUsingSource() throws Exception {
-        client.admin().indices().prepareCreate("test").execute().actionGet();
-        client.prepareIndex("test", "type1", "1")
-                .setSource(jsonBuilder().startObject()
+        wipeIndices("test");
+        createIndex("test");
+        index("test", "type1", "1",
+                jsonBuilder().startObject()
                         .startObject("obj1").field("test", "something").endObject()
                         .startObject("obj2").startArray("arr2").value("arr_value1").value("arr_value2").endArray().endObject()
-                        .endObject())
-                .execute().actionGet();
-        client.admin().indices().refresh(refreshRequest()).actionGet();
+                        .endObject());
+        refresh();
 
-        SearchResponse response = client.prepareSearch()
+        SearchResponse response = client().prepareSearch()
                 .setQuery(matchAllQuery())
                 .addField("_source.obj1") // we also automatically detect _source in fields
                 .addScriptField("s_obj1", "python", "_source['obj1']", null)
@@ -171,32 +142,16 @@ public void testScriptFieldUsingSource() throws Exception {
 
     @Test
     public void testCustomScriptBoost() throws Exception {
-        // execute a search before we create an index
-        try {
-            client.prepareSearch().setQuery(termQuery("test", "value")).execute().actionGet();
-            assert false : "should fail";
-        } catch (Exception e) {
-            // ignore, no indices
-        }
-
-        try {
-            client.prepareSearch("test").setQuery(termQuery("test", "value")).execute().actionGet();
-            assert false : "should fail";
-        } catch (Exception e) {
-            // ignore, no indices
-        }
-
-        client.admin().indices().create(createIndexRequest("test")).actionGet();
-        client.index(indexRequest("test").type("type1").id("1")
-                .source(jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject())).actionGet();
-        client.index(indexRequest("test").type("type1").id("2")
-                .source(jsonBuilder().startObject().field("test", "value check").field("num1", 2.0f).endObject())).actionGet();
-        client.admin().indices().refresh(refreshRequest()).actionGet();
+        wipeIndices("test");
+        createIndex("test");
+        index("test", "type1", "1", jsonBuilder().startObject().field("test", "value beck").field("num1", 1.0f).endObject());
+        index("test", "type1", "2", jsonBuilder().startObject().field("test", "value beck").field("num1", 2.0f).endObject());
+        refresh();
 
         logger.info("--- QUERY_THEN_FETCH");
 
-        logger.info("running doc['num1'].value");
-        SearchResponse response = client.search(searchRequest()
+        logger.info(" --> running doc['num1'].value");
+        SearchResponse response = client().search(searchRequest()
                 .searchType(SearchType.QUERY_THEN_FETCH)
                 .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value").lang("python")))
         ).actionGet();
@@ -204,13 +159,13 @@ public void testCustomScriptBoost() throws Exception {
         assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
 
         assertThat(response.getHits().totalHits(), equalTo(2l));
-        logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
-        logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
+        logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
+        logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
         assertThat(response.getHits().getAt(0).id(), equalTo("2"));
         assertThat(response.getHits().getAt(1).id(), equalTo("1"));
 
-        logger.info("running -doc['num1'].value");
-        response = client.search(searchRequest()
+        logger.info(" --> running -doc['num1'].value");
+        response = client().search(searchRequest()
                 .searchType(SearchType.QUERY_THEN_FETCH)
                 .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("-doc['num1'].value").lang("python")))
         ).actionGet();
@@ -218,14 +173,14 @@ public void testCustomScriptBoost() throws Exception {
         assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
 
         assertThat(response.getHits().totalHits(), equalTo(2l));
-        logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
-        logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
+        logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
+        logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
         assertThat(response.getHits().getAt(0).id(), equalTo("1"));
         assertThat(response.getHits().getAt(1).id(), equalTo("2"));
 
 
-        logger.info("running doc['num1'].value * _score");
-        response = client.search(searchRequest()
+        logger.info(" --> running doc['num1'].value * _score");
+        response = client().search(searchRequest()
                 .searchType(SearchType.QUERY_THEN_FETCH)
                 .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("doc['num1'].value * _score").lang("python")))
         ).actionGet();
@@ -233,13 +188,13 @@ public void testCustomScriptBoost() throws Exception {
         assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
 
         assertThat(response.getHits().totalHits(), equalTo(2l));
-        logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
-        logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
+        logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
+        logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
         assertThat(response.getHits().getAt(0).id(), equalTo("2"));
         assertThat(response.getHits().getAt(1).id(), equalTo("1"));
 
-        logger.info("running param1 * param2 * _score");
-        response = client.search(searchRequest()
+        logger.info(" --> running param1 * param2 * _score");
+        response = client().search(searchRequest()
                 .searchType(SearchType.QUERY_THEN_FETCH)
                 .source(searchSource().explain(true).query(customScoreQuery(termQuery("test", "value")).script("param1 * param2 * _score").param("param1", 2).param("param2", 2).lang("python")))
         ).actionGet();
@@ -247,7 +202,7 @@ public void testCustomScriptBoost() throws Exception {
         assertThat("Failures " + Arrays.toString(response.getShardFailures()), response.getShardFailures().length, equalTo(0));
 
         assertThat(response.getHits().totalHits(), equalTo(2l));
-        logger.info("Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
-        logger.info("Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
+        logger.info(" --> Hit[0] {} Explanation {}", response.getHits().getAt(0).id(), response.getHits().getAt(0).explanation());
+        logger.info(" --> Hit[1] {} Explanation {}", response.getHits().getAt(1).id(), response.getHits().getAt(1).explanation());
     }
 }