New turnover, refactor fixed slippage adjustments

pyfolio/plotting.py

@@ -601,8 +601,8 @@ def plot_perf_stats(returns, factor_returns, ax=None):
 
 
 def show_perf_stats(returns, factor_returns, positions=None,
-                    transactions=None, live_start_date=None,
-                    bootstrap=False):
+                    transactions=None, turnover_denom='AGB',
+                    live_start_date=None, bootstrap=False):
     """
     Prints some performance metrics of the strategy.
 
@@ -623,6 +623,12 @@ def show_perf_stats(returns, factor_returns, positions=None,
     positions : pd.DataFrame
         Daily net position values.
          - See full explanation in create_full_tear_sheet.
+    transactions : pd.DataFrame
+        Prices and amounts of executed trades. One row per trade.
+        - See full explanation in tears.create_full_tear_sheet
+    turnover_denom : str
+        Either AGB or portfolio_value, default AGB.
+        - See full explanation in txn.get_turnover.
     live_start_date : datetime, optional
         The point in time when the strategy began live trading, after
         its backtest period.
@@ -641,7 +647,8 @@ def show_perf_stats(returns, factor_returns, positions=None,
         returns,
         factor_returns=factor_returns,
         positions=positions,
-        transactions=transactions)
+        transactions=transactions,
+        turnover_denom=turnover_denom)
 
     if live_start_date is not None:
         live_start_date = ep.utils.get_utc_timestamp(live_start_date)
@@ -666,13 +673,15 @@ def show_perf_stats(returns, factor_returns, positions=None,
             returns_is,
             factor_returns=factor_returns,
             positions=positions_is,
-            transactions=transactions_is)
+            transactions=transactions_is,
+            turnover_denom=turnover_denom)
 
         perf_stats_oos = perf_func(
             returns_oos,
             factor_returns=factor_returns,
             positions=positions_oos,
-            transactions=transactions_oos)
+            transactions=transactions_oos,
+            turnover_denom=turnover_denom)
 
         print('In-sample months: ' +
               str(int(len(returns_is) / APPROX_BDAYS_PER_MONTH)))
@@ -1407,7 +1416,7 @@ def plot_turnover(returns, transactions, positions,
     return ax
 
 
-def plot_slippage_sweep(returns, transactions, positions,
+def plot_slippage_sweep(returns, positions, transactions,
                         slippage_params=(3, 8, 10, 12, 15, 20, 50),
                         ax=None, **kwargs):
     """
@@ -1418,12 +1427,12 @@ def plot_slippage_sweep(returns, transactions, positions,
     returns : pd.Series
         Timeseries of portfolio returns to be adjusted for various
         degrees of slippage.
-    transactions : pd.DataFrame
-        Prices and amounts of executed trades. One row per trade.
-         - See full explanation in tears.create_full_tear_sheet.
     positions : pd.DataFrame
         Daily net position values.
          - See full explanation in tears.create_full_tear_sheet.
+    transactions : pd.DataFrame
+        Prices and amounts of executed trades. One row per trade.
+         - See full explanation in tears.create_full_tear_sheet.
     slippage_params: tuple
         Slippage pameters to apply to the return time series (in
         basis points).
@@ -1441,12 +1450,10 @@ def plot_slippage_sweep(returns, transactions, positions,
     if ax is None:
         ax = plt.gca()
 
-    turnover = txn.get_turnover(positions, transactions,
-                                period=None, average=False)
-
     slippage_sweep = pd.DataFrame()
     for bps in slippage_params:
-        adj_returns = txn.adjust_returns_for_slippage(returns, turnover, bps)
+        adj_returns = txn.adjust_returns_for_slippage(returns, positions,
+                                                      transactions, bps)
         label = str(bps) + " bps"
         slippage_sweep[label] = ep.cum_returns(adj_returns, 1)
 
@@ -1460,7 +1467,7 @@ def plot_slippage_sweep(returns, transactions, positions,
     return ax
 
 
-def plot_slippage_sensitivity(returns, transactions, positions,
+def plot_slippage_sensitivity(returns, positions, transactions,
                               ax=None, **kwargs):
     """
     Plots curve relating per-dollar slippage to average annual returns.
@@ -1470,12 +1477,12 @@ def plot_slippage_sensitivity(returns, transactions, positions,
     returns : pd.Series
         Timeseries of portfolio returns to be adjusted for various
         degrees of slippage.
-    transactions : pd.DataFrame
-        Prices and amounts of executed trades. One row per trade.
-         - See full explanation in tears.create_full_tear_sheet.
     positions : pd.DataFrame
         Daily net position values.
          - See full explanation in tears.create_full_tear_sheet.
+    transactions : pd.DataFrame
+        Prices and amounts of executed trades. One row per trade.
+         - See full explanation in tears.create_full_tear_sheet.
     ax : matplotlib.Axes, optional
         Axes upon which to plot.
     **kwargs, optional
@@ -1490,11 +1497,10 @@ def plot_slippage_sensitivity(returns, transactions, positions,
     if ax is None:
         ax = plt.gca()
 
-    turnover = txn.get_turnover(positions, transactions,
-                                period=None, average=False)
     avg_returns_given_slippage = pd.Series()
     for bps in range(1, 100):
-        adj_returns = txn.adjust_returns_for_slippage(returns, turnover, bps)
+        adj_returns = txn.adjust_returns_for_slippage(returns, positions,
+                                                      transactions, bps)
         avg_returns = ep.annual_return(adj_returns)
         avg_returns_given_slippage.loc[bps] = avg_returns
 
@@ -1566,7 +1572,7 @@ def plot_daily_turnover_hist(transactions, positions,
 
     if ax is None:
         ax = plt.gca()
-    turnover = txn.get_turnover(positions, transactions, period=None)
+    turnover = txn.get_turnover(positions, transactions)
     sns.distplot(turnover, ax=ax, **kwargs)
     ax.set_title('Distribution of daily turnover rates')
     ax.set_xlabel('Turnover rate')

pyfolio/tears.py

@@ -75,6 +75,7 @@ def create_full_tear_sheet(returns,
                            shares_held=None,
                            volumes=None,
                            percentile=None,
+                           turnover_denom='AGB',
                            set_context=True):
     """
     Generate a number of tear sheets that are useful
@@ -152,6 +153,9 @@ def create_full_tear_sheet(returns,
     bootstrap : boolean (optional)
         Whether to perform bootstrap analysis for the performance
         metrics. Takes a few minutes longer.
+    turnover_denom : str
+        Either AGB or portfolio_value, default AGB.
+        - See full explanation in txn.get_turnover.
     set_context : boolean, optional
         If True, set default plotting style context.
          - See plotting.context().
@@ -162,10 +166,9 @@ def create_full_tear_sheet(returns,
 
     if (unadjusted_returns is None) and (slippage is not None) and\
        (transactions is not None):
-        turnover = txn.get_turnover(positions, transactions,
-                                    period=None, average=False)
         unadjusted_returns = returns.copy()
-        returns = txn.adjust_returns_for_slippage(returns, turnover, slippage)
+        returns = txn.adjust_returns_for_slippage(returns, positions,
+                                                  transactions, slippage)
 
     positions = utils.check_intraday(estimate_intraday, returns,
                                      positions, transactions)
@@ -178,6 +181,7 @@ def create_full_tear_sheet(returns,
         cone_std=cone_std,
         benchmark_rets=benchmark_rets,
         bootstrap=bootstrap,
+        turnover_denom=turnover_denom,
         set_context=set_context)
 
     create_interesting_times_tear_sheet(returns,
@@ -227,7 +231,8 @@ def create_simple_tear_sheet(returns,
                              benchmark_rets=None,
                              slippage=None,
                              estimate_intraday='infer',
-                             live_start_date=None):
+                             live_start_date=None,
+                             turnover_denom='AGB'):
     """
     Simpler version of create_full_tear_sheet; generates summary performance
     statistics and important plots as a single image.
@@ -284,6 +289,9 @@ def create_simple_tear_sheet(returns,
     live_start_date : datetime, optional
         The point in time when the strategy began live trading,
         after its backtest period. This datetime should be normalized.
+    turnover_denom : str
+        Either AGB or portfolio_value, default AGB.
+        - See full explanation in txn.get_turnover.
     """
 
     positions = utils.check_intraday(estimate_intraday, returns,
@@ -293,9 +301,8 @@ def create_simple_tear_sheet(returns,
         benchmark_rets = utils.get_symbol_rets('SPY')
 
     if (slippage is not None) and (transactions is not None):
-        turnover = txn.get_turnover(positions, transactions,
-                                    period=None, average=False)
-        returns = txn.adjust_returns_for_slippage(returns, turnover, slippage)
+        returns = txn.adjust_returns_for_slippage(returns, positions,
+                                                  transactions, slippage)
 
     if (positions is not None) and (transactions is not None):
         vertical_sections = 11
@@ -313,6 +320,7 @@ def create_simple_tear_sheet(returns,
                              benchmark_rets,
                              positions=positions,
                              transactions=transactions,
+                             turnover_denom=turnover_denom,
                              live_start_date=live_start_date)
 
     if returns.index[0] < benchmark_rets.index[0]:
@@ -400,6 +408,7 @@ def create_returns_tear_sheet(returns, positions=None,
                               cone_std=(1.0, 1.5, 2.0),
                               benchmark_rets=None,
                               bootstrap=False,
+                              turnover_denom='AGB',
                               return_fig=False):
     """
     Generate a number of plots for analyzing a strategy's returns.
@@ -434,6 +443,9 @@ def create_returns_tear_sheet(returns, positions=None,
     bootstrap : boolean (optional)
         Whether to perform bootstrap analysis for the performance
         metrics. Takes a few minutes longer.
+    turnover_denom : str
+        Either AGB or portfolio_value, default AGB.
+        - See full explanation in txn.get_turnover.
     return_fig : boolean, optional
         If True, returns the figure that was plotted on.
     set_context : boolean, optional
@@ -451,6 +463,7 @@ def create_returns_tear_sheet(returns, positions=None,
     plotting.show_perf_stats(returns, benchmark_rets,
                              positions=positions,
                              transactions=transactions,
+                             turnover_denom=turnover_denom,
                              bootstrap=bootstrap,
                              live_start_date=live_start_date)
 
@@ -739,14 +752,14 @@ def create_txn_tear_sheet(returns, positions, transactions,
     if unadjusted_returns is not None:
         ax_slippage_sweep = plt.subplot(gs[4, :])
         plotting.plot_slippage_sweep(unadjusted_returns,
-                                     transactions,
                                      positions,
+                                     transactions,
                                      ax=ax_slippage_sweep
                                      )
         ax_slippage_sensitivity = plt.subplot(gs[5, :])
         plotting.plot_slippage_sensitivity(unadjusted_returns,
-                                           transactions,
                                            positions,
+                                           transactions,
                                            ax=ax_slippage_sensitivity
                                            )
     for ax in fig.axes:

pyfolio/tests/test_txn.py

@@ -17,21 +17,20 @@ def test_get_turnover(self):
         """
         Tests turnover using a 20 day period.
 
-        With no transactions the turnover should be 0.
+        With no transactions, the turnover should be 0.
 
-        with 100% of the porfolio value traded each day
-        the daily turnover rate should be 0.5.
-
-        For monthly turnover it should be the sum
-        of the daily turnovers because 20 days < 1 month.
-
-        e.g (20 days) * (0.5 daily turn) = 10x monthly turnover rate.
+        with 200% of the AGB traded each day, the daily
+        turnover rate should be 2.0.
         """
         dates = date_range(start='2015-01-01', freq='D', periods=20)
 
-        positions = DataFrame([[0.0, 10.0]]*len(dates),
+        # In this test, there is one sid (0) and a cash column
+        positions = DataFrame([[10.0, 10.0]]*len(dates),
                               columns=[0, 'cash'], index=dates)
 
+        # Set every other non-cash position to 40
+        positions[0][::2] = 40
+
         transactions = DataFrame(data=[],
                                  columns=['sid', 'amount', 'price', 'symbol'],
                                  index=dates)
@@ -41,23 +40,28 @@ def test_get_turnover(self):
         result = get_turnover(positions, transactions)
         assert_series_equal(result, expected)
 
-        # Monthly freq
-        index = date_range('01-01-2015', freq='M', periods=1)
-        expected = Series([0.0], index=index)
-        result = get_turnover(positions, transactions, period='M')
-        assert_series_equal(result, expected)
-
-        transactions = DataFrame(data=[[1, 1, 10, 'A']]*len(dates),
+        transactions = DataFrame(data=[[1, 1, 10, 0]]*len(dates) +
+                                 [[2, -1, 10, 0]]*len(dates),
                                  columns=['sid', 'amount', 'price', 'symbol'],
-                                 index=dates)
+                                 index=dates.append(dates)).sort_index()
 
-        expected = Series([0.5]*len(dates), index=dates)
+        # Turnover is more on day 1, because the day 0 AGB is set to zero
+        # in get_turnover. On most days, we get 0.8 because we have 20
+        # transacted and mean(10, 40) = 25, so 20/25.
+        expected = Series([1.0] + [0.8] * (len(dates) - 1), index=dates)
         result = get_turnover(positions, transactions)
+
         assert_series_equal(result, expected)
 
-        # Monthly freq: should be the sum of the daily freq
-        result = get_turnover(positions, transactions, period='M')
-        expected = Series([10.0], index=index)
+        # Test with denominator = 'portfolio_value'
+        result = get_turnover(positions, transactions,
+                              denominator='portfolio_value')
+
+        # Our portfolio value alternates between $20 and $50 so turnover
+        # should alternate between 20/20 = 1.0 and 20/50 = 0.4.
+        expected = Series([0.4, 1.0] * (int((len(dates) - 1) / 2) + 1),
+                          index=dates)
+
         assert_series_equal(result, expected)
 
     def test_adjust_returns_for_slippage(self):
@@ -76,7 +80,7 @@ def test_adjust_returns_for_slippage(self):
         slippage_bps = 10
         expected = Series([0.049]*len(dates), index=dates)
 
-        turnover = get_turnover(positions, transactions, average=False)
-        result = adjust_returns_for_slippage(returns, turnover, slippage_bps)
+        result = adjust_returns_for_slippage(returns, positions,
+                                             transactions, slippage_bps)
 
         assert_series_equal(result, expected)

pyfolio/timeseries.py

@@ -573,9 +573,12 @@ def rolling_fama_french(returns, factor_returns=None,
         DataFrame containing rolling beta coefficients to SMB, HML and UMD
     """
 
+    # We need to drop NaNs to regress
+    ret_no_na = returns.dropna()
+
     if factor_returns is None:
         factor_returns = ep.utils.load_portfolio_risk_factors(
-            start=returns.index[0], end=returns.index[-1])
+            start=ret_no_na.index[0], end=ret_no_na.index[-1])
         factor_returns = factor_returns.drop(['Mkt-RF', 'RF'],
                                              axis='columns')
 
@@ -590,7 +593,7 @@ def rolling_fama_french(returns, factor_returns=None,
     for beg, end in zip(factor_returns.index[:-rolling_window],
                         factor_returns.index[rolling_window:]):
         coeffs = linear_model.LinearRegression().fit(factor_returns[beg:end],
-                                                     returns[beg:end]).coef_
+                                                     ret_no_na[beg:end]).coef_
         regression_coeffs = np.append(regression_coeffs, [coeffs], axis=0)
 
     rolling_fama_french = pd.DataFrame(data=regression_coeffs[:, :3],
@@ -688,7 +691,7 @@ def value_at_risk(returns, period=None, sigma=2.0):
 
 
 def perf_stats(returns, factor_returns=None, positions=None,
-               transactions=None):
+               transactions=None, turnover_denom='AGB'):
     """
     Calculates various performance metrics of a strategy, for use in
     plotting.show_perf_stats.
@@ -707,7 +710,10 @@ def perf_stats(returns, factor_returns=None, positions=None,
          - See full explanation in tears.create_full_tear_sheet.
     transactions : pd.DataFrame
         Prices and amounts of executed trades. One row per trade.
-        - See full explanation in tears.create_full_tear_sheet
+        - See full explanation in tears.create_full_tear_sheet.
+    turnover_denom : str
+        Either AGB or portfolio_value, default AGB.
+        - See full explanation in txn.get_turnover.
 
     Returns
     -------
@@ -723,7 +729,8 @@ def perf_stats(returns, factor_returns=None, positions=None,
         stats['Gross leverage'] = gross_lev(positions).mean()
         if transactions is not None:
             stats['Daily turnover'] = get_turnover(positions,
-                                                   transactions).mean()
+                                                   transactions,
+                                                   turnover_denom).mean()
     if factor_returns is not None:
         for stat_func in FACTOR_STAT_FUNCS:
             res = stat_func(returns, factor_returns)