diff --git a/test/RunTests b/test/RunTests
index f145ad15b4d..d4855d771c8 100755
--- a/test/RunTests
+++ b/test/RunTests
@@ -1577,8 +1577,9 @@ echo "1 | feature:1" | {VW} -a --initial_weight 0.1 --initial_t 0.3
     train-sets/ref/157.stderr
 
 # Test 158: test decision service json parsing
-{VW} -d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT
-    train-sets/ref/decisionservice.stderr
+{VW} -d train-sets/decisionservice.json --dsjson --cb_explore_adf --epsilon 0.2 --quadratic GT -P 1 -p cbe_adf_dsjson.predict
+    train-sets/ref/cbe_adf_dsjson.stderr
+    pred-sets/ref/cbe_adf_dsjson.predict
 
 # Test 159: test --bootstrap & --binary interaction
 {VW} -d train-sets/rcv1_mini.dat --bootstrap 5 --binary -c -k --passes 2
diff --git a/test/pred-sets/ref/cbe_adf_dsjson.predict b/test/pred-sets/ref/cbe_adf_dsjson.predict
new file mode 100644
index 00000000000..614cceb083c
--- /dev/null
+++ b/test/pred-sets/ref/cbe_adf_dsjson.predict
@@ -0,0 +1,6 @@
+0:0.0833333,1:0.0833333,2:0.0833333,3:0.0833333,4:0.0833333,5:0.0833333,6:0.0833333,7:0.0833333,8:0.0833333,9:0.0833333,10:0.0833333,11:0.0833333
+
+6:0.816667,5:0.0166667,9:0.0166667,2:0.0166667,10:0.0166667,1:0.0166667,3:0.0166667,7:0.0166667,4:0.0166667,0:0.0166667,8:0.0166667,11:0.0166667
+
+6:0.816667,5:0.0166667,9:0.0166667,2:0.0166667,1:0.0166667,3:0.0166667,10:0.0166667,4:0.0166667,0:0.0166667,7:0.0166667,8:0.0166667,11:0.0166667
+
diff --git a/test/train-sets/ref/decisionservice.stderr b/test/train-sets/ref/cbe_adf_dsjson.stderr
similarity index 75%
rename from test/train-sets/ref/decisionservice.stderr
rename to test/train-sets/ref/cbe_adf_dsjson.stderr
index 7eee9ec91ea..2f070887361 100644
--- a/test/train-sets/ref/decisionservice.stderr
+++ b/test/train-sets/ref/cbe_adf_dsjson.stderr
@@ -1,4 +1,5 @@
-creating quadratic features for pairs: GT 
+creating quadratic features for pairs: GT
+predictions = cbe_adf_dsjson.predict
 Num weight bits = 18
 learning rate = 0.5
 initial_t = 0
@@ -10,10 +11,11 @@ average  since         example        example  current  current  current
 loss     last          counter         weight    label  predict features
 -0.102041 -0.102041            1            1.0    known        0:0.0833333...      361
 -0.051020 0.000000            2            2.0    known        6:0.816667...      361
+-0.040816 -0.020408            3            3.0    known        6:0.816667...      361
 
 finished run
 number of examples = 3
 weighted example sum = 3.000000
 weighted label sum = 0.000000
-average loss = -0.367347
+average loss = -0.040816
 total feature number = 1083