diff --git a/docs/api.rst b/docs/api.rst index abd4e60c..77c232e1 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -128,3 +128,27 @@ Diagnostic tests are useful for identifying model fit, sufficiency, and specific spreg.panel_rLMlag spreg.panel_rLMerror spreg.panel_Hausman + +DGP +----------- + +Tools for simulating synthetic data according to data-generating processes implied by different spatial model specifications + +.. autosummary:: + :toctree: generated/ + + spreg.dgp.make_error + spreg.dgp.make_x + spreg.dgp.make_wx + spreg.dgp.make_xb + spreg.dgp.make_wxg + spreg.dgp.dgp_errproc + spreg.dgp.dgp_ols + spreg.dgp.dgp_slx + spreg.dgp.dgp_sperror + spreg.dgp.dgp_slxerror + spreg.dgp.dgp_lag + spreg.dgp.dgp_spdurbin + spreg.dgp.dgp_lagerr + spreg.dgp.dgp_gns + spreg.dgp.dgp_mes \ No newline at end of file diff --git a/spreg/__init__.py b/spreg/__init__.py old mode 100644 new mode 100755 index 845e95e8..2de967b5 --- a/spreg/__init__.py +++ b/spreg/__init__.py @@ -2,6 +2,7 @@ import contextlib from importlib.metadata import PackageNotFoundError, version +from .dgp import * from .diagnostics import * from .diagnostics_panel import * from .diagnostics_sp import * diff --git a/spreg/dgp.py b/spreg/dgp.py new file mode 100644 index 00000000..7053e01f --- /dev/null +++ b/spreg/dgp.py @@ -0,0 +1,836 @@ +""" +Generating spatial models for specific DGP + +""" + +__author__ = "Luc Anselin lanselin@gmail.com,\ + Pedro Amaral pedrovma@gmail.com,\ + Renan Serenini renan.serenini@uniroma1.it" + +import numpy as np +import math +import libpysal +from scipy.linalg import expm +from .utils import inverse_prod + + +__all__ = [ + "make_error", + "make_x", + "make_wx", + "make_xb", + "make_wxg", + "dgp_errproc", + "dgp_ols", + "dgp_slx", + "dgp_sperror", + "dgp_slxerror", + "dgp_lag", + "dgp_spdurbin", + "dgp_lagerr", + "dgp_gns", + "dgp_mess" +] + + +def make_error(rng,n,mu=0,varu=1,method='normal'): + """ + make_error: generate error term for a given distribution + + Arguments: + ---------- + rng: random number object + n: number of observations + mu: mean (when needed) + varu: variance (when needed) + method: type of distribution, one of + normal, laplace, cauchy, lognormal + + Returns: + -------- + u: nx1 vector of random errors + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_error + >>> rng = np.random.default_rng(12345) + >>> make_error(rng,5) + array([[-1.42382504], + [ 1.26372846], + [-0.87066174], + [-0.25917323], + [-0.07534331]]) + + """ + # normal - standard normal is default + if method == 'normal': + sdu = math.sqrt(varu) + u = rng.normal(loc=mu,scale=sdu,size=n).reshape(n,1) + # laplace with thicker tails + elif method == 'laplace': + sdu = math.sqrt(varu/2.0) + u = rng.laplace(loc=mu,scale=sdu,size=n).reshape(n,1) + # cauchy, ill-behaved, no mean or variance defined + elif method == 'cauchy': + u = rng.standard_cauchy(size=n).reshape(n,1) + elif method == 'lognormal': + sdu = math.sqrt(varu) + u = rng.lognormal(mean=mu,sigma=sdu,size=n).reshape(n,1) + # all other yield warning + else: + print('Warning: Unsupported distribution') + u = None + return u + +def make_x(rng,n,mu=[0],varu=[1],cor=0,method='uniform'): + """ + make_x: generate a matrix of k columns of x for a given distribution + + Arguments: + ---------- + rng: random number object + n: number of observations + mu: mean as a list + varu: variance as a list + cor: correlation as a float (for bivariate normal only) + method: type of distribution, one of + uniform, normal, bivnormal (bivariate normal) + + Returns: + -------- + x: nxk matrix of x variables + + Note: + ----- + Uniform and normal generate separate draws, bivariate normal generates + correlated draws + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_x + >>> rng = np.random.default_rng(12345) + >>> make_x(rng,5,mu=[0,1],varu=[1,4]) + array([[0.78751508, 2.30580253], + [1.09728308, 4.14520464], + [2.76215497, 1.29373239], + [2.3426149 , 4.6609906 ], + [1.35484323, 6.52500165]]) + + """ + # check on k dimension + k = len(mu) + if k == len(varu): + # initialize + x = np.zeros((n,k)) + for i in range(k): + # uniform - range is derived from variance since var = (1/12)range^2 + # range is found as square root of 12 times variance + # for 0-1, varu should be 0.0833333 + # low is always 0 + if method == 'uniform': + sdu = math.sqrt(12.0*varu[i]) + x[:,i] = rng.uniform(low=0,high=sdu,size=n) + # normal - independent normal draws + elif method == 'normal': + sdu = math.sqrt(varu[i]) + x[:,i] = rng.normal(loc=mu[i],scale=sdu,size=n) + # bivariate normal - only for k=2 + elif method == 'bivnormal': + if k != 2: + print('Error: Wrong dimension for k') + x = None + return x + else: + ucov = cor* math.sqrt(varu[0]*varu[1]) + mcov = [[varu[0],ucov],[ucov,varu[1]]] + x = rng.multivariate_normal(mean=mu,cov=mcov,size=n) + return x + else: + print('Warning: Unsupported distribution') + x = None + else: + x = None + return x + +def make_wx(x,w,o=1): + """ + make_wx: generate a matrix spatially lagged x given matrix x + + x must be previously generated using make_x, no constant included + + Arguments: + ---------- + x: x matrix - no constant + w: row-standardized spatial weights in spreg format + o: order of contiguity, default o=1 + + Returns: + -------- + wx: nx(kxo) matrix of spatially lagged x variables + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_wx + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> make_wx(x,w)[0:5,:] + array([[1.12509217], + [1.87409079], + [1.36225472], + [2.1491645 ], + [2.80255786]]) + + """ + if w.n != x.shape[0]: + print("Error: incompatible weights dimensions") + return None + w1x = libpysal.weights.lag_spatial(w,x) + wx = w1x + if o > 1: + for i in range(1,o): + whx = libpysal.weights.lag_spatial(w,w1x) + w1x = whx + wx = np.hstack((wx,whx)) + return wx + + +def make_xb(x,beta): + """ + make_xb: generate a column xb as matrix x (constant added) + times list beta (includes coefficient for constant term) + + Arguments: + ---------- + x: n x (k-1) matrix for x variables + beta: k length list of regression coefficients + + Returns: + -------- + xb: nx1 vector of x times beta + + Examples + -------- + + >>> import numpy as np + >>> from spreg import make_x, make_xb + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,5,mu=[0,1],varu=[1,4]) + >>> make_xb(x,[1,2,3]) + array([[ 9.49243776], + [15.63018007], + [10.4055071 ], + [19.66820159], + [23.28469141]]) + """ + n = x.shape[0] + k = x.shape[1] + if k+1 != len(beta): + print("Error: Incompatible dimensions") + return None + else: + b = np.array(beta)[:,np.newaxis] + x1=np.hstack((np.ones((n,1)),x)) # include constant + xb = np.dot(x1,b) + return xb + +def make_wxg(wx,gamma): + """ + make_wxg: generate a column wxg as matrix wx (no constant) + times list gamma (coefficient of spatially lagged x) + + Arguments: + ---------- + wx: n x ((k-1)xo) matrix for spatially lagged x variables of all orders + gamma: (k-1)*o length list of regression coefficients for spatially lagged x + + Returns: + -------- + wxg: nx1 vector of wx times gamma + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_wx, make_wxg + >>> rng = np.random.default_rng(12345) + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> print(wx.shape) + (25, 1) + >>> make_wxg(wx,[2,4])[0:5,:] + array([[ 2.25018434, 4.50036868], + [ 3.74818158, 7.49636316], + [ 2.72450944, 5.44901889], + [ 4.298329 , 8.59665799], + [ 5.60511572, 11.21023145]]) + + """ + k = wx.shape[1] + if (k > 1): + if k != len(gamma): + print("Error: Incompatible dimensions") + return None + else: + g = np.array(gamma)[:,np.newaxis] + wxg = np.dot(wx,g) + else: # gamma is a scalar + wxg = wx * gamma + return wxg + +def dgp_errproc(u,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sar: generates pure spatial error process + + Arguments: + ---------- + u: random error vector + w: spatial weights object + lam: spatial autoregressive parameter + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + -------- + y : vector of observations following a pure SAR process + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, dgp_errproc + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_errproc(u,w)[0:5,:] + array([[-1.43760658], + [ 0.69778271], + [-0.7750646 ], + [-0.47750452], + [-0.72377417]]) + + """ + n0 = u.shape[0] + if w.n != n0: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + y = inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + y = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + return y + + +def dgp_ols(u,xb): + """ + dgp_ols: generates y for non-spatial process with given xb and error term u + + Arguments: + ---------- + u: random error vector + xb: vector of xb + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, dgp_ols + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> dgp_ols(u,xb)[0:5,:] + array([[5.22803968], + [3.60291127], + [1.02632633], + [1.37589879], + [5.07165754]]) + + """ + n1 = u.shape[0] + n2 = xb.shape[0] + if n1 != n2: + print("Error: dimension mismatch") + return None + y = xb + u + return y + +def dgp_slx(u,xb,wxg): + """ + dgp_slx: generates y for SLX with given xb, wxg, and error term u + + Arguments: + ---------- + u: random error vector + xb: vector of xb + wxg: vector of wxg + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_slx + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_slx(u, xb, wxg)[0:5,:] + array([[8.85854389], + [7.17524694], + [3.83674621], + [4.73103929], + [8.37023076]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + y = xb + wxg + u + return y + +def dgp_sperror(u,xb,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sperror: generates y for spatial error model with given xb, weights, + spatial parameter lam, error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + lam: spatial coefficient + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_sperror + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_sperror(u, xb, w)[0:5,:] + array([[5.21425813], + [3.03696553], + [1.12192347], + [1.15756751], + [4.42322667]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: incompatible weights dimensions") + return None + elif w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y = xb + u1 + return y + +def dgp_slxerror(u,xb,wxg,w,lam=0.5,model='sar',imethod='power_exp'): + """ + dgp_sperror: generates y for SLX spatial error model with xb, wxg, weights, + spatial parameter lam, model type (sar or ma), + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + lam: spatial coefficient + model: type of process ('sar' or 'ma') + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_slxerror + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_slxerror(u,xb,wxg,w)[0:5,:] + array([[8.84476235], + [6.6093012 ], + [3.93234334], + [4.51270801], + [7.7217999 ]]) + """ + + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y = xb + wxg + u1 + return y + +def dgp_lag(u,xb,w,rho=0.5,imethod='power_exp'): + """ + dgp_lag: generates y for spatial lag model with xb, weights, + spatial parameter rho, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_lag + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_lag(u, xb, w)[0:5,:] + array([[10.16582326], + [ 7.75359864], + [ 5.39821733], + [ 5.62244672], + [ 8.868168 ]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + y1 = xb + u + y = inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_spdurbin(u,xb,wxg,w,rho=0.5,imethod='power_exp'): + """ + dgp_spdurbin: generates y for spatial Durbin model with xb, wxg, weights, + spatial parameter rho, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + rho: spatial coefficient + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_spdurbin + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_spdurbin(u,xb,wxg,w)[0:5,:] + array([[18.06895353], + [15.18686487], + [11.95080505], + [12.55220513], + [15.75805066]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + y1 = xb + wxg + u + y = inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_lagerr(u,xb,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): + """ + dgp_lagerr: generates y for spatial lag model with sar or ma errors + with xb, weights, + spatial parameter rho, spatial parameter lambda, + model for spatial process, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient for lag + lam: spatial coefficient for error + model: spatial process for error + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_lagerr + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_lagerr(u, xb, w)[0:5,:] + array([[10.13845523], + [ 7.53009531], + [ 5.40644034], + [ 5.51132886], + [ 8.58872366]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + if model == 'sar': + u1 = inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y1 = xb + u1 + y = inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_gns(u,xb,wxg,w,rho=0.5,lam=0.2,model='sar',imethod='power_exp'): + """ + dgp_gns: generates y for general nested model with sar or ma errors + with xb, wxg, weights, + spatial parameter rho, spatial parameter lambda, + model for spatial process, + error term, method for inverse transform + + Arguments: + ---------- + u: random error + xb: vector of xb + wxg: vector of wxg + w: spatial weights + rho: spatial coefficient for lag + lam: spatial coefficient for error + model: spatial process for error + imethod: method for inverse transformation, default = 'power_exp' + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, make_wx, make_wxg, dgp_gns + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> wx = make_wx(x,w) + >>> wxg = make_wxg(wx,[2]) + >>> dgp_gns(u,xb,wxg,w)[0:5,:] + array([[18.04158549], + [14.96336153], + [11.95902806], + [12.44108728], + [15.47860632]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + n2 = wxg.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + elif n1 != n2: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + if model == 'sar': + u1 = inverse_prod(w,u,lam,inv_method=imethod) + elif model == 'ma': + u1 = u + lam * libpysal.weights.lag_spatial(w,u) + else: + print("Error: unsupported model type") + return None + y1 = xb + wxg + u1 + y = inverse_prod(w,y1,rho,inv_method=imethod) + return y + +def dgp_mess(u,xb,w,rho=0.5): + """ + dgp_mess: generates y for MESS spatial lag model with xb, weights, + spatial parameter rho (gets converted into alpha), + sigma/method for the error term + + Arguments: + ---------- + u: random error + xb: vector of xb + w: spatial weights + rho: spatial coefficient (converted into alpha) + + Returns: + ---------- + y: vector of observations on dependent variable + + Examples + -------- + + >>> import numpy as np + >>> import libpysal + >>> from spreg import make_x, make_xb, dgp_mess + >>> rng = np.random.default_rng(12345) + >>> u = make_x(rng,25,mu=[0],varu=[1], method='normal') + >>> x = make_x(rng,25,mu=[0],varu=[1]) + >>> xb = make_xb(x,[1,2]) + >>> w = libpysal.weights.lat2W(5, 5) + >>> w.transform = "r" + >>> dgp_mess(u, xb, w)[0:5,:] + array([[10.12104421], + [ 7.45561055], + [ 5.32807674], + [ 5.55549492], + [ 8.62685145]]) + """ + n0 = u.shape[0] + n1 = xb.shape[0] + if n0 != n1: + print("Error: dimension mismatch") + return None + if w.n != n1: + print("Error: incompatible weights dimensions") + return None + bigw = libpysal.weights.full(w)[0] + alpha=np.log(1-rho) #convert between rho and alpha + aw=-alpha*bigw # inverse exponential is -alpha + xbu = xb + u + y = np.dot(expm(aw),xbu) + return y + +def _test(): + import doctest + + start_suppress = np.get_printoptions()["suppress"] + np.set_printoptions(suppress=True) + doctest.testmod() + np.set_printoptions(suppress=start_suppress) + +if __name__ == "__main__": + _test() + \ No newline at end of file diff --git a/spreg/diagnostics_sp.py b/spreg/diagnostics_sp.py index bf0dbfe7..42dadc0d 100644 --- a/spreg/diagnostics_sp.py +++ b/spreg/diagnostics_sp.py @@ -1,7 +1,7 @@ """ Spatial diagnostics module """ -__author__ = "Luc Anselin luc.anselin@asu.edu, Daniel Arribas-Bel darribas@asu.edu" +__author__ = "Luc Anselin lanselin@gmail.com, Daniel Arribas-Bel darribas@asu.edu, Pedro Amaral pedrovma@gmail.com" from .utils import spdot @@ -19,7 +19,7 @@ class LMtests: """ - Lagrange Multiplier tests. Implemented as presented in :cite:`Anselin1996a` + Lagrange Multiplier tests. Implemented as presented in :cite:`Anselin1996a` and :cite:`KoleyBera2024` Attributes ---------- @@ -37,6 +37,11 @@ class LMtests: * 'rlme': Robust LM error test * 'lml' : LM lag test * 'rlml': Robust LM lag test + * 'sarma': LM SARMA test + * 'lmwx': LM test for WX + * 'rlmwx': Robust LM WX test + * 'lmspdurbin': Joint test for SDM + * 'rlmdurlag': Robust LM Lag - SDM Parameters ---------- @@ -54,9 +59,20 @@ class LMtests: (Only if 'rlml' or 'all' was in tests). Pair of statistic and p-value for the Robust LM lag test. sarma : tuple - (Only if 'rlml' or 'all' was in tests). Pair of statistic + (Only if 'sarma' or 'all' was in tests). Pair of statistic and p-value for the SARMA test. - + lmwx : tuple + (Only if 'lmwx' or 'all' was in tests). Pair of statistic + and p-value for the LM test for WX. + rlmwx : tuple + (Only if 'rlmwx' or 'all' was in tests). Pair of statistic + and p-value for the Robust LM WX test. + rlmdurlag : tuple + (Only if 'rlmdurlag' or 'all' was in tests). Pair of statistic + and p-value for the Robust LM Lag - SDM test. + lmspdurbin : tuple + (Only if 'lmspdurbin' or 'all' was in tests). Pair of statistic + and p-value for the Joint test for SDM. Examples -------- @@ -122,15 +138,36 @@ class LMtests: >>> print(round(lms.sarma[0],4), round(lms.sarma[1],4)) 4.1907 0.123 + + LM test for WX: + + >>> print(round(lms.lmwx[0],4), round(lms.lmwx[1],4)) + 1.3377 0.5123 + + Robust LM WX test: + + >>> print(round(lms.rlmwx[0],4), round(lms.rlmwx[1],4)) + 3.4532 0.1779 + + Robust LM Lag - SDM: + >>> print(round(lms.rlmdurlag[0],4), round(lms.rlmdurlag[1],4)) + 3.0971 0.0784 + + Joint test for SDM: + + >>> print(round(lms.lmspdurbin[0],4), round(lms.lmspdurbin[1],4)) + 4.4348 0.2182 """ def __init__(self, ols, w, tests=["all"]): cache = spDcache(ols, w) if tests == ["all"]: - tests = ["lme", "lml", "rlme", "rlml", "sarma"] - if "lme" in tests: + tests = ["lme", "lml", "rlme", "rlml", "sarma", "lmwx", "lmspdurbin", "rlmwx", + "rlmdurlag", "lmslxerr"] # added back in for access + if any(test in ["lme", "lmslxerr"] for test in tests): + #if "lme" in tests: self.lme = lmErr(ols, w, cache) - if "lml" in tests: + if any(test in ["lml", "rlmwx"] for test in tests): self.lml = lmLag(ols, w, cache) if "rlme" in tests: self.rlme = rlmErr(ols, w, cache) @@ -138,7 +175,17 @@ def __init__(self, ols, w, tests=["all"]): self.rlml = rlmLag(ols, w, cache) if "sarma" in tests: self.sarma = lmSarma(ols, w, cache) - + #if any(test in ["lmwx", "rlmdurlag", "lmslxerr"] for test in tests): + if any(test in ["lmwx", "rlmdurlag","lmslxerr"] for test in tests): + self.lmwx = lm_wx(ols, w) + if any(test in ["lmspdurbin", "rlmdurlag", "rlmwx"] for test in tests): + self.lmspdurbin = lm_spdurbin(ols, w) + if "rlmwx" in tests: + self.rlmwx = rlm_wx(ols, self.lmspdurbin, self.lml) + if "rlmdurlag" in tests: + self.rlmdurlag = rlm_durlag(self.lmspdurbin, self.lmwx) + if "lmslxerr" in tests: #currently removed - LA added back in for access + self.lmslxerr = lm_slxerr(ols, self.lme, self.lmwx) class MoranRes: """ @@ -630,6 +677,186 @@ def lmSarma(ols, w, spDcache): pval = chisqprob(lm, 2) return (lm[0][0], pval[0][0]) +def lm_wx(reg, w): + """ + LM test for WX. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + w : W + Spatial weights instance + + Returns + ------- + lmwx : tuple + Pair of statistic and p-value for the LM test for WX. + + """ + + # preliminaries + # set up X1 (constant) and X (no constant) as x1 and xx + x1 = reg.x + xx = x1[:,1:] + # WX + wx = w.sparse * xx + # end of preliminaries + # X'W'u + xtwtu = wx.T @ reg.u + # X'W'X1(X1'X1)-1X1WX + mx1 = wx.T @ x1 + mx = (mx1 @ reg.xtxi) @ mx1.T + xwwx = wx.T @ wx + xqx = xwwx - mx + xqxi = la.inv(xqx) + # RSgamma: (X'W'u)'(X'Q1X)-1(X'W'u) / sig2n + xpwpu = wx.T @ reg.u + rsg1 = (xpwpu.T @ xqxi) @ xpwpu + rsgam = rsg1[0][0] / reg.sig2n + pval = chisqprob(rsgam, (reg.k - 1)) + rsgamma = (rsgam,pval) + return(rsgamma) + +def lm_spdurbin(reg,w): + """ + Joint test for SDM. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + w : W + Spatial weights instance + + Returns + ------- + lmspdurbin : tuple + Pair of statistic and p-value for the Joint test for SDM. + + """ + + # preliminaries + # set up X1 (constant) and X (no constant) as x1 and xx + x1 = reg.x + xx = x1[:,1:] + k = x1.shape[1] + # WX + wx = w.sparse * xx + # X1b + xb = reg.predy + # WX1b + wxb = w.sparse * xb + # Wy + wy = w.sparse * reg.y + # y'W'e / sig2n + drho = (wy.T @ reg.u) / reg.sig2n + # X'W'e / sign2n + dgam = (wx.T @ reg.u) / reg.sig2n + # P = T = tr(W2 + W'W) + pp = w.trcWtW_WW + # end of preliminaries + # J_11: block matrix with X1'X1 and n/2sig2n + jj1a = np.hstack((reg.xtx,np.zeros((k,1)))) + jj1b = np.hstack((np.zeros((1,k)),np.array([reg.n/(2.0*reg.sig2n)]).reshape(1,1))) + jj11 = np.vstack((jj1a,jj1b)) + # J_12: matrix with k-1 rows X1'WX1b and X1'WX, and 1 row of zeros + jj12a = np.hstack((x1.T @ wxb, x1.T @ wx)) + jj12 = np.vstack((jj12a,np.zeros((1,k)))) + # J_22 matrix with diagonal elements b'X1'W'WX1b + T.sig2n and X'W'WX + # and off-diagonal element b'X1'W'WX + jj22a = wxb.T @ wxb + pp * reg.sig2n + jj22a = jj22a.reshape(1,1) + wxbtwx = (wxb.T @ wx).reshape(1,k-1) + jj22b = np.hstack((jj22a,wxbtwx)) + wxtwx = wx.T @ wx + jj22c = np.hstack((wxbtwx.T,wxtwx)) + jj22 = np.vstack((jj22b,jj22c)) + # J^22 (the inverse) from J^22 = (J_22 - J_21.J_11^-1.J_12)^-1 + jj11i = la.inv(jj11) + j121121 = (jj12.T @ jj11i) @ jj12 + jj22i1 = jj22 - j121121 + jj22i = la.inv(jj22i1) + # rescale by sig2n + jj22i = jj22i * reg.sig2n + # statistic + dd = np.vstack((drho,dgam)) + rsjoint = (dd.T @ jj22i) @ dd + rsjoint = rsjoint[0][0] + pval = chisqprob(rsjoint, k) + rsrhogam = (rsjoint, pval) + return(rsrhogam) + +def rlm_wx(reg,lmspdurbin,lmlag): + """ + Robust LM WX test. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + lmspdurbin : tuple + Joint test for SDM as in lm_spdurbin function + lmlag : tuple + LM Lag test as in lmLag function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Robust LM WX test. + + """ + # robust gamma = rsjoint - rsrho + rsgams = lmspdurbin[0] - lmlag[0] + pval = chisqprob(rsgams,(reg.k - 1)) + rsgamstar = (rsgams, pval) + return(rsgamstar) + +def rlm_durlag(lmspdurbin,lmwx): + """ + Robust LM Lag - SDM. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + lmspdurbin : tuple + Joint test for SDM as in lm_spdurbin function + lmwx : tuple + LM test for WX as in lm_wx function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Robust LM Lag - SDM test. + """ + + # robust rho = rsjoint - rsgam + rsrhos = lmspdurbin[0] - lmwx[0] + pval = chisqprob(rsrhos,1) + rsrhostar = (rsrhos, pval) + return(rsrhostar) + +def lm_slxerr(reg,lmerr,lmwx): + """ + Joint test for Error and WX. Implemented as presented in Koley & Bera (2024) :cite:`KoleyBera2024`. + + Attributes + ---------- + reg : OLS + Instance from an OLS regression + lmerr : tuple + LM Error test as in lmErr function + lmwx : tuple + LM test for WX as in lm_wx function + + Returns + ------- + rlmwx : tuple + Pair of statistic and p-value for the Joint test for Error and WX. + """ + rslamgam = lmerr[0] + lmwx[0] + pval = chisqprob(rslamgam,reg.k) + rslamgamma = (rslamgam,pval) + return(rslamgamma) def get_mI(reg, w, spDcache): """ @@ -725,15 +952,20 @@ def akTest(iv, w, spDcache): return (mi, ak[0][0], pval[0][0]) -def comfac_test(lambd, beta, gamma, vm): +def comfac_test(rho, beta, gamma, vm): """ - Computes the Spatial Common Factor Hypothesis test as shown in Anselin (1988, p. 226-229) + Computes the Spatial Common Factor Hypothesis test as shown in Anselin (1988, p. 226-229). + Note that for the Common Factor Hypothesis test to be valid, gamma has to equal + *negative* rho times beta for all beta parameters. + That is, when rho is positive, a positive beta means gamma must be negative and vice versa. + For a negative rho, beta, and gamma must have the same sign. + If those signs are not compatible, the test will not be meaningful. Parameters ---------- - lambd : float - Spatial autoregressive coefficient (as in lambd*Wy) + rho : float + Spatial autoregressive coefficient (as in rho*Wy) beta : array Coefficients of the exogenous (not spatially lagged) variables, without the constant (as in X*beta) gamma : array @@ -751,8 +983,8 @@ def comfac_test(lambd, beta, gamma, vm): with k-1 degrees of freedom """ - g = lambd * beta + gamma - G = np.vstack((lambd * np.eye(beta.shape[0]), np.eye(beta.shape[0]), beta.T)) + g = rho * beta + gamma + G = np.vstack((rho * np.eye(beta.shape[0]), np.eye(beta.shape[0]), beta.T)) GVGi = la.inv(np.dot(G.T, np.dot(vm, G))) W = np.dot(g.T, np.dot(GVGi, g))[0][0] diff --git a/spreg/error_sp.py b/spreg/error_sp.py index 765fddc8..cdbf6275 100644 --- a/spreg/error_sp.py +++ b/spreg/error_sp.py @@ -2,7 +2,7 @@ Spatial Error Models module """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ Daniel Arribas-Bel darribas@asu.edu, \ Pedro V. Amaral pedro.amaral@asu.edu" @@ -37,7 +37,9 @@ class BaseGM_Error(RegressionPropsY): independent (exogenous) variable, excluding the constant w : Sparse matrix Spatial weights sparse matrix - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -87,7 +89,7 @@ class BaseGM_Error(RegressionPropsY): [ 0.3257]]) """ - def __init__(self, y, x, w): + def __init__(self, y, x, w, hard_bound=False): # 1a. OLS --> \tilde{betas} ols = OLS.BaseOLS(y=y, x=x) @@ -97,7 +99,7 @@ def __init__(self, y, x, w): # 1b. GMM --> \tilde{\lambda1} moments = _momentsGM_Error(w, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) # 2a. OLS -->\hat{betas} xs = get_spFilter(w, lambda1, self.x) @@ -133,7 +135,7 @@ class GM_Error(BaseGM_Error): Spatial weights object (always needed) slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -147,7 +149,9 @@ class GM_Error(BaseGM_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside maximum/minimum bounds. Attributes ---------- output : dataframe @@ -287,26 +291,29 @@ class GM_Error(BaseGM_Error): """ def __init__( - self, y, x, w, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_ds=None, latex=False - ): + self, y, x, w, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_ds=None, latex=False, + hard_bound=False): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # intialize in case of None, contains constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant + self.title += " WITH SLX (SLX-Error)" - BaseGM_Error.__init__(self, y=y, x=x_constant, w=w.sparse) + BaseGM_Error.__init__(self, y=y, x=x_constant, w=w.sparse, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.output = pd.DataFrame(self.name_x, columns=['var_names']) @@ -338,7 +345,9 @@ class BaseGM_Endog_Error(RegressionPropsY): this should not contain any variables from x) w : Sparse matrix Spatial weights sparse matrix - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -396,7 +405,7 @@ class BaseGM_Endog_Error(RegressionPropsY): """ - def __init__(self, y, x, yend, q, w): + def __init__(self, y, x, yend, q, w, hard_bound=False): # 1a. TSLS --> \tilde{betas} tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q) @@ -407,7 +416,7 @@ def __init__(self, y, x, yend, q, w): # 1b. GMM --> \tilde{\lambda1} moments = _momentsGM_Error(w, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) # 2a. 2SLS -->\hat{betas} xs = get_spFilter(w, lambda1, self.x) @@ -450,7 +459,7 @@ class GM_Endog_Error(BaseGM_Endog_Error): Spatial weights object (always needed) slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -468,7 +477,9 @@ class GM_Endog_Error(BaseGM_Endog_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -645,23 +656,27 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize for None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" - BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend, q=q) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant + self.title += " WITH SLX (SLX-Error)" + BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend, q=q, hard_bound=hard_bound) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") @@ -726,7 +741,9 @@ class GM_Combo(BaseGM_Endog_Error): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -934,12 +951,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) set_warn(self, warn) if slx_lags == 0: yend2, q2 = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q) @@ -947,7 +966,7 @@ def __init__( yend2, q2, wx = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q, slx_lags) x_constant = np.hstack((x_constant, wx)) - BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend2, q=q2) + BaseGM_Endog_Error.__init__(self, y=y, x=x_constant, w=w.sparse, yend=yend2, q=q2, hard_bound=hard_bound) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = sp_att( @@ -956,11 +975,13 @@ def __init__( set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS - GM-COMBO MODEL" if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already in list self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend @@ -1008,7 +1029,7 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error If True, then a spatial lag of the dependent variable is included. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM or GNSM type. + If slx_lags>0, the specification becomes of the SLX-Error or GNSM type. vm : boolean If True, include variance-covariance matrix in summary results @@ -1026,6 +1047,9 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. **kwargs : keywords Additional arguments to pass on to the estimators. See the specific functions for details on what can be used. @@ -1209,54 +1233,54 @@ class GMM_Error(GM_Error, GM_Endog_Error, GM_Combo, GM_Error_Het, GM_Endog_Error def __init__( self, y, x, w, yend=None, q=None, estimator='het', add_wy=False, slx_lags=0, vm=False, name_y=None, name_x=None, name_w=None, name_yend=None, - name_q=None, name_ds=None, latex=False, **kwargs): + name_q=None, name_ds=None, latex=False, hard_bound=False, **kwargs): if estimator == 'het': if yend is None and not add_wy: GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error_Het.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo_Het.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) elif estimator == 'hom': if yend is None and not add_wy: GM_Error_Hom.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error_Hom.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo_Hom.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Hom.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) elif estimator == 'kp98': if yend is None and not add_wy: GM_Error.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif yend is not None and not add_wy: GM_Endog_Error.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) elif add_wy: GM_Combo.__init__(self, y=y, x=x, yend=yend, q=q, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, **kwargs) + name_yend=name_yend, name_q=name_q, name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound, **kwargs) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) else: set_warn(self, 'Combination of arguments passed to GMM_Error not allowed. Using default arguments instead.') GM_Error_Het.__init__(self, y=y, x=x, w=w, slx_lags=slx_lags, vm=vm, name_y=name_y, name_x=name_x, - name_w=name_w, name_ds=name_ds, latex=latex) + name_w=name_w, name_ds=name_ds, latex=latex, hard_bound=hard_bound) def _momentsGM_Error(w, u): diff --git a/spreg/error_sp_het.py b/spreg/error_sp_het.py index 50b838fb..24c9adcb 100755 --- a/spreg/error_sp_het.py +++ b/spreg/error_sp_het.py @@ -2,7 +2,7 @@ Spatial Error with Heteroskedasticity family of models """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ Pedro V. Amaral pedro.amaral@asu.edu, \ Daniel Arribas-Bel darribas@asu.edu, \ David C. Folch david.folch@asu.edu \ @@ -49,7 +49,9 @@ class BaseGM_Error_Het(RegressionPropsY): an additional stop condition. step1c : boolean If True, then include Step 1c from :cite:`Arraiz2010`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -107,7 +109,7 @@ class BaseGM_Error_Het(RegressionPropsY): [ 0.4118 0.168 ]] """ - def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): + def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False, hard_bound=False): self.step1c = step1c # 1a. OLS --> \tilde{betas} @@ -126,6 +128,7 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): lambda2 = UTILS.optim_moments(moments, vc1) else: lambda2 = lambda1 + lambda_old = lambda2 self.iteration, eps = 0, 1 @@ -147,6 +150,12 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, step1c=False): self.iteration += 1 self.iter_stop = UTILS.iter_msg(self.iteration, max_iter) + if hard_bound: + if abs(lambda3) >= 0.99: + raise Exception("Spatial error parameter was outside the bounds of -0.99 and 0.99") + else: + if abs(lambda3) >= 0.99: + set_warn(self, "Spatial error parameter was outside the bounds of -0.99 and 0.99") sigma = get_psi_sigma(w, self.u, lambda3) vc3 = get_vc_het(w, wA1, sigma) @@ -173,7 +182,7 @@ class GM_Error_Het(BaseGM_Error_Het): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -197,7 +206,9 @@ class GM_Error_Het(BaseGM_Error_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -354,19 +365,22 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES (HET)" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant + self.title += " WITH SLX (SLX-Error)" BaseGM_Error_Het.__init__( self, y, @@ -375,10 +389,12 @@ def __init__( max_iter=max_iter, step1c=step1c, epsilon=epsilon, + hard_bound = hard_bound ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.output = pd.DataFrame(self.name_x, columns=['var_names']) @@ -425,7 +441,9 @@ class BaseGM_Endog_Error_Het(RegressionPropsY): If "power_exp", then compute inverse using the power expansion. If "true_inv", then compute the true inverse. Note that true_inv will fail for large n. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -510,11 +528,22 @@ def __init__( epsilon=0.00001, step1c=False, inv_method="power_exp", + hard_bound=False, ): self.step1c = step1c # 1a. reg --> \tilde{betas} tsls = TSLS.BaseTSLS(y=y, x=x, yend=yend, q=q) + + if abs(tsls.betas[-1]) <= 0.9: + pass + else: + if tsls.betas[-1] < -0.9: + tsls.betas[-1] = -0.9 + else: + tsls.betas[-1] = 0.9 + tsls.u = tsls.y - spdot(tsls.z, tsls.betas) + self.x, self.z, self.h, self.y = tsls.x, tsls.z, tsls.h, tsls.y self.yend, self.q, self.n, self.k, self.hth = ( tsls.yend, @@ -539,6 +568,13 @@ def __init__( lambda2 = UTILS.optim_moments(moments, vc1) else: lambda2 = lambda1 + + # Forcing the 1st step lambda to be in the range [-0.9, 0.9] to avoid perfect collinearity in step 2 in case of SLX-Error or GNS models + #if lambda2 > 0.9: + # lambda_old = 0.9 + #elif lambda2 < -0.9: + # lambda_old = -0.9 + #else: lambda_old = lambda2 self.iteration, eps = 0, 1 @@ -548,6 +584,16 @@ def __init__( ys = UTILS.get_spFilter(w, lambda_old, self.y) yend_s = UTILS.get_spFilter(w, lambda_old, self.yend) tsls_s = TSLS.BaseTSLS(ys, xs, yend_s, h=self.h) + + if abs(tsls_s.betas[-1]) <= 0.9: + pass + else: + if tsls_s.betas[-1] < -0.9: + tsls_s.betas[-1] = -0.9 + else: + tsls_s.betas[-1] = 0.9 + tsls_s.u = tsls_s.y - spdot(tsls_s.z, tsls_s.betas) + self.predy = spdot(self.z, tsls_s.betas) self.u = self.y - self.predy @@ -563,11 +609,29 @@ def __init__( ) moments_i = UTILS._moments2eqs(wA1, w, self.u) lambda3 = UTILS.optim_moments(moments_i, vc2) + + #if abs(lambda3) <= 0.9: + # pass + #elif lambda3 > 0.9: + # lambda3 = 0.9 + #elif lambda3 < -0.9: + # lambda3 = -0.9 + eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 self.iter_stop = UTILS.iter_msg(self.iteration, max_iter) + if hard_bound: + if abs(lambda3) >= 0.99: + raise Exception("Spatial error parameter was outside the bounds of -0.99 and 0.99") + if abs(tsls_s.betas[-1]) >= 0.99: + raise Exception("Spatial lag parameter was outside the bounds of -0.99 and 0.99") + else: + if abs(lambda3) >= 0.99: + set_warn(self, "Spatial error parameter was outside the bounds of -0.99 and 0.99") + if abs(tsls_s.betas[-1]) >= 0.99: + set_warn(self, "Spatial lag parameter was outside the bounds of -0.99 and 0.99") zs = UTILS.get_spFilter(w, lambda3, self.z) P = get_P_hat(self, tsls.hthi, zs) @@ -603,7 +667,7 @@ class GM_Endog_Error_Het(BaseGM_Endog_Error_Het): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -635,7 +699,9 @@ class GM_Endog_Error_Het(BaseGM_Endog_Error_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -827,19 +893,21 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES (HET)" if slx_lags > 0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant + self.title += " WITH SLX (SLX-Error)" BaseGM_Endog_Error_Het.__init__( self, y=y, @@ -851,10 +919,12 @@ def __init__( step1c=step1c, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") # listing lambda last @@ -913,7 +983,9 @@ class BaseGM_Combo_Het(BaseGM_Endog_Error_Het): If "power_exp", then compute inverse using the power expansion. If "true_inv", then compute the true inverse. Note that true_inv will fail for large n. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -1021,6 +1093,7 @@ def __init__( epsilon=0.00001, step1c=False, inv_method="power_exp", + hard_bound=False, ): BaseGM_Endog_Error_Het.__init__( @@ -1034,6 +1107,7 @@ def __init__( step1c=step1c, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) @@ -1102,7 +1176,9 @@ class GM_Combo_Het(BaseGM_Combo_Het): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1315,12 +1391,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) if slx_lags == 0: @@ -1342,6 +1420,7 @@ def __init__( lag_q=lag_q, epsilon=epsilon, inv_method=inv_method, + hard_bound=hard_bound, ) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = UTILS.sp_att( @@ -1349,12 +1428,15 @@ def __init__( ) UTILS.set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS- GM-COMBO MODEL (HET)" + if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # no constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend diff --git a/spreg/error_sp_het_regimes.py b/spreg/error_sp_het_regimes.py index 2f812d74..9ba5b68e 100644 --- a/spreg/error_sp_het_regimes.py +++ b/spreg/error_sp_het_regimes.py @@ -67,7 +67,7 @@ class GM_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from Arraiz et al. Note: epsilon provides an additional stop condition. @@ -96,7 +96,9 @@ class GM_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -321,6 +323,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -369,6 +372,7 @@ def __init__( vm, name_x, latex, + hard_bound, ) else: raise Exception( @@ -392,13 +396,13 @@ def __init__( # 1b. GMM --> \tilde{\lambda1} moments = UTILS._moments2eqs(wA1, w.sparse, ols.u) - lambda1 = UTILS.optim_moments(moments) + lambda1 = UTILS.optim_moments(moments, hard_bound=hard_bound) if step1c: # 1c. GMM --> \tilde{\lambda2} sigma = get_psi_sigma(w.sparse, ols.u, lambda1) vc1 = get_vc_het(w.sparse, wA1, sigma) - lambda2 = UTILS.optim_moments(moments, vc1) + lambda2 = UTILS.optim_moments(moments, vc1, hard_bound=hard_bound) else: lambda2 = lambda1 lambda_old = lambda2 @@ -419,7 +423,7 @@ def __init__( sigma_i = get_psi_sigma(w.sparse, self.u, lambda_old) vc_i = get_vc_het(w.sparse, wA1, sigma_i) moments_i = UTILS._moments2eqs(wA1, w.sparse, self.u) - lambda3 = UTILS.optim_moments(moments_i, vc_i) + lambda3 = UTILS.optim_moments(moments_i, vc_i, hard_bound=hard_bound) eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 @@ -434,7 +438,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL (HET) - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM-HET) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (Error-HET) - REGIMES" self.name_x.append("lambda") self.kf += 1 @@ -449,7 +453,7 @@ def __init__( output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _error_het_regimes_multi( - self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, step1c, cols2regi, vm, name_x, latex + self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, step1c, cols2regi, vm, name_x, latex, hard_bound ): regi_ids = dict( @@ -488,6 +492,7 @@ def _error_het_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ), ) else: @@ -507,6 +512,7 @@ def _error_het_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ) ) @@ -620,7 +626,7 @@ class GM_Endog_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -657,7 +663,9 @@ class GM_Endog_Error_Het_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -927,6 +935,7 @@ def __init__( summ=True, add_lag=False, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) @@ -984,6 +993,7 @@ def __init__( name_q, add_lag, latex, + hard_bound, ) else: raise Exception( @@ -1024,7 +1034,7 @@ def __init__( # 1b. GMM --> \tilde{\lambda1} moments = UTILS._moments2eqs(wA1, w.sparse, tsls.u) - lambda1 = UTILS.optim_moments(moments) + lambda1 = UTILS.optim_moments(moments, hard_bound=hard_bound) if step1c: # 1c. GMM --> \tilde{\lambda2} @@ -1040,7 +1050,7 @@ def __init__( inv_method, filt=False, ) - lambda2 = UTILS.optim_moments(moments, vc1) + lambda2 = UTILS.optim_moments(moments, vc1, hard_bound=hard_bound) else: lambda2 = lambda1 lambda_old = lambda2 @@ -1077,7 +1087,7 @@ def __init__( inv_method, ) moments_i = UTILS._moments2eqs(wA1, w.sparse, self.u) - lambda3 = UTILS.optim_moments(moments_i, vc2) + lambda3 = UTILS.optim_moments(moments_i, vc2, hard_bound=hard_bound) eps = abs(lambda3 - lambda_old) lambda_old = lambda3 self.iteration += 1 @@ -1111,7 +1121,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS (HET) - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HET) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HET) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _endog_error_het_regimes_multi( @@ -1135,6 +1145,7 @@ def _endog_error_het_regimes_multi( name_q, add_lag, latex, + hard_bound, ): regi_ids = dict( @@ -1184,6 +1195,7 @@ def _endog_error_het_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ), ) else: @@ -1209,6 +1221,7 @@ def _endog_error_het_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ) ) @@ -1388,7 +1401,9 @@ class GM_Combo_Het_Regimes(GM_Endog_Error_Het_Regimes): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1691,6 +1706,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): if regime_lag_sep and not regime_err_sep: set_warn(self, "regime_err_sep set to True when regime_lag_sep=True.") @@ -1773,6 +1789,7 @@ def __init__( summ=False, add_lag=add_lag, latex=latex, + hard_bound=hard_bound, ) if regime_err_sep != True: @@ -1807,19 +1824,20 @@ def _work_error( name_w, name_regimes, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] x_r = x[regi_ids[r]] model = BaseGM_Error_Het( - y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, step1c=step1c + y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, step1c=step1c, hard_bound=hard_bound, ) set_warn(model, warn) model.w = w_r if slx_lags == 0: model.title = "GM SPATIALLY WEIGHTED LS (HET) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED LS + SLX (SDEM-HET) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED LS + SLX (Error-HET) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] @@ -1849,6 +1867,7 @@ def _work_endog_error( name_regimes, add_lag, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] @@ -1872,6 +1891,7 @@ def _work_endog_error( epsilon=epsilon, step1c=step1c, inv_method=inv_method, + hard_bound=hard_bound, ) set_warn(model, warn) if add_lag != False: @@ -1890,7 +1910,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM-HET) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HET) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HET) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/error_sp_hom.py b/spreg/error_sp_hom.py index 6208b8af..cfa78852 100644 --- a/spreg/error_sp_hom.py +++ b/spreg/error_sp_hom.py @@ -4,7 +4,7 @@ """ -__author__ = "Luc Anselin luc.anselin@asu.edu, Daniel Arribas-Bel darribas@asu.edu" +__author__ = "Luc Anselin lanselin@gmail.com, Daniel Arribas-Bel darribas@asu.edu" from scipy import sparse as SP import numpy as np @@ -50,7 +50,9 @@ class BaseGM_Error_Hom(RegressionPropsY): If A1='hom', then as in :cite:`Anselin2011` (default). If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -117,7 +119,7 @@ class BaseGM_Error_Hom(RegressionPropsY): [ -2.40000000e-03 3.00000000e-04 -1.00000000e-04 3.37000000e-02]] """ - def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): + def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc", hard_bound=False): if A1 == "hom": wA1 = get_A1_hom(w) elif A1 == "hom_sc": @@ -133,7 +135,7 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 1b. GM --> \tilde{\rho} moments = moments_hom(w, wA1, wA2, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -148,7 +150,7 @@ def __init__(self, y, x, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w, wA1, wA2, self.u) psi = get_vc_hom(w, wA1, wA2, self, lambda_old)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -180,7 +182,7 @@ class GM_Error_Hom(BaseGM_Error_Hom): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -206,7 +208,9 @@ class GM_Error_Hom(BaseGM_Error_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -357,19 +361,22 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED LEAST SQUARES (HOM)" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant + self.title += " WITH SLX (SLX-Error)" BaseGM_Error_Hom.__init__( self, y=y, @@ -378,10 +385,12 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.A1 = A1 @@ -429,7 +438,9 @@ class BaseGM_Endog_Error_Hom(RegressionPropsY): al. If A1='hom', then as in :cite:`Anselin2011`. If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- betas : array @@ -505,7 +516,7 @@ class BaseGM_Endog_Error_Hom(RegressionPropsY): """ - def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): + def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc", hard_bound=False): if A1 == "hom": wA1 = get_A1_hom(w) @@ -529,7 +540,7 @@ def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 1b. GM --> \tilde{\rho} moments = moments_hom(w, wA1, wA2, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -545,7 +556,7 @@ def __init__(self, y, x, yend, q, w, max_iter=1, epsilon=0.00001, A1="hom_sc"): # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w, wA1, wA2, self.u) psi = get_vc_hom(w, wA1, wA2, self, lambda_old, tsls_s.z)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -584,7 +595,7 @@ class GM_Endog_Error_Hom(BaseGM_Endog_Error_Hom): Spatial weights object slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -614,7 +625,9 @@ class GM_Endog_Error_Hom(BaseGM_Endog_Error_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -806,19 +819,22 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) self.title = "GM SPATIALLY WEIGHTED TWO STAGE LEAST SQUARES (HOM)" if slx_lags > 0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant + self.title += " WITH SLX (SLX-Error)" BaseGM_Endog_Error_Hom.__init__( self, y=y, @@ -829,10 +845,12 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # already includes constant self.name_yend = USER.set_name_yend(name_yend, yend) self.name_z = self.name_x + self.name_yend self.name_z.append("lambda") # listing lambda last @@ -891,7 +909,9 @@ class BaseGM_Combo_Hom(BaseGM_Endog_Error_Hom): al. If A1='hom', then as in :cite:`Anselin2011`. If A1='hom_sc' (default), then as in :cite:`Drukker2013` and :cite:`Drukker:2013aa`. - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- @@ -1003,6 +1023,7 @@ def __init__( max_iter=1, epsilon=0.00001, A1="hom_sc", + hard_bound=False, ): BaseGM_Endog_Error_Hom.__init__( @@ -1015,6 +1036,7 @@ def __init__( A1=A1, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) @@ -1082,7 +1104,9 @@ class GM_Combo_Hom(BaseGM_Combo_Hom): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1291,12 +1315,14 @@ def __init__( name_w=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None, includes constant set_warn(self, warn) if slx_lags == 0: yend2, q2 = set_endog(y, x_constant[:, 1:], w, yend, q, w_lags, lag_q) @@ -1315,6 +1341,7 @@ def __init__( lag_q=lag_q, max_iter=max_iter, epsilon=epsilon, + hard_bound=hard_bound, ) self.rho = self.betas[-2] self.predy_e, self.e_pred, warn = sp_att( @@ -1323,11 +1350,13 @@ def __init__( set_warn(self, warn) self.title = "SPATIALLY WEIGHTED 2SLS- GM-COMBO MODEL (HOM)" if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title += " WITH SLX (GNSM)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) +# self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x # constant already included self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend diff --git a/spreg/error_sp_hom_regimes.py b/spreg/error_sp_hom_regimes.py index f8d7dfae..0853e2db 100644 --- a/spreg/error_sp_hom_regimes.py +++ b/spreg/error_sp_hom_regimes.py @@ -69,7 +69,7 @@ class GM_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional @@ -102,7 +102,9 @@ class GM_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -332,6 +334,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -380,6 +383,7 @@ def __init__( vm, name_x, latex, + hard_bound, ) else: raise Exception( @@ -412,7 +416,7 @@ def __init__( # 1b. GM --> \tilde{\rho} moments = moments_hom(w.sparse, wA1, wA2, ols.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -430,7 +434,7 @@ def __init__( # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w.sparse, wA1, wA2, self.u) psi = get_vc_hom(w.sparse, wA1, wA2, self, lambda_old)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -446,7 +450,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL (HOM) - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM-HOM) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (Error-HOM) - REGIMES" self.name_x.append("lambda") self.kf += 1 self.chow = REGI.Chow(self) @@ -461,7 +465,7 @@ def __init__( output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _error_hom_regimes_multi( - self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, A1, cols2regi, vm, name_x, latex): + self, y, x, regimes, w, slx_lags, cores, max_iter, epsilon, A1, cols2regi, vm, name_x, latex, hard_bound): regi_ids = dict( (r, list(np.where(np.array(regimes) == r)[0])) for r in self.regimes_set @@ -499,6 +503,7 @@ def _error_hom_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ), ) else: @@ -518,6 +523,7 @@ def _error_hom_regimes_multi( self.name_w, self.name_regimes, slx_lags, + hard_bound, ) ) @@ -632,7 +638,7 @@ class GM_Endog_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. max_iter : int Maximum number of iterations of steps 2a and 2b from :cite:`Arraiz2010`. Note: epsilon provides an additional stop condition. @@ -665,7 +671,9 @@ class GM_Endog_Error_Hom_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -943,6 +951,7 @@ def __init__( summ=True, add_lag=False, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x, yend, q) @@ -999,6 +1008,7 @@ def __init__( name_q, add_lag, latex, + hard_bound, ) else: raise Exception( @@ -1046,7 +1056,7 @@ def __init__( # 1b. GM --> \tilde{\rho} moments = moments_hom(w.sparse, wA1, wA2, tsls.u) - lambda1 = optim_moments(moments) + lambda1 = optim_moments(moments, hard_bound=hard_bound) lambda_old = lambda1 self.iteration, eps = 0, 1 @@ -1073,7 +1083,7 @@ def __init__( # 2b. GM 2nd iteration --> \hat{\rho} moments = moments_hom(w.sparse, wA1, wA2, self.u) psi = get_vc_hom(w.sparse, wA1, wA2, self, lambda_old, tsls_s.z)[0] - lambda2 = optim_moments(moments, psi) + lambda2 = optim_moments(moments, psi, hard_bound=hard_bound) eps = abs(lambda2 - lambda_old) lambda_old = lambda2 self.iteration += 1 @@ -1106,7 +1116,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS (HOM) - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SDEM-HOM) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (Error-HOM) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) def _endog_error_hom_regimes_multi( @@ -1129,6 +1139,7 @@ def _endog_error_hom_regimes_multi( name_q, add_lag, latex, + hard_bound, ): regi_ids = dict( @@ -1177,6 +1188,7 @@ def _endog_error_hom_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ), ) else: @@ -1201,6 +1213,7 @@ def _endog_error_hom_regimes_multi( self.name_regimes, add_lag, slx_lags, + hard_bound, ) ) @@ -1381,7 +1394,9 @@ class GM_Combo_Hom_Regimes(GM_Endog_Error_Hom_Regimes): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1689,6 +1704,7 @@ def __init__( name_ds=None, name_regimes=None, latex=False, + hard_bound=False, ): if regime_lag_sep and not regime_err_sep: @@ -1771,6 +1787,7 @@ def __init__( summ=False, add_lag=add_lag, latex=latex, + hard_bound=hard_bound, ) if regime_err_sep != True: @@ -1805,19 +1822,20 @@ def _work_error( name_w, name_regimes, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] x_r = x[regi_ids[r]] model = BaseGM_Error_Hom( - y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1 + y_r, x_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1, hard_bound=hard_bound, ) set_warn(model, warn) model.w = w_r if slx_lags == 0: model.title = "GM SPATIALLY WEIGHTED 2SLS (HOM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HOM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HOM) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] @@ -1846,6 +1864,7 @@ def _work_endog_error( name_regimes, add_lag, slx_lags, + hard_bound, ): w_r, warn = REGI.w_regime(w, regi_ids[r], r, transform=True) y_r = y[regi_ids[r]] @@ -1861,7 +1880,7 @@ def _work_endog_error( ) x_constant = USER.check_constant(x_r) model = BaseGM_Endog_Error_Hom( - y_r, x_r, yend_r, q_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1 + y_r, x_r, yend_r, q_r, w_r.sparse, max_iter=max_iter, epsilon=epsilon, A1=A1, hard_bound=hard_bound, ) set_warn(model, warn) if add_lag != False: @@ -1879,7 +1898,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM-HOM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM-HOM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (Error-HOM) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/error_sp_regimes.py b/spreg/error_sp_regimes.py index 8c62c7d8..5c9ddd5d 100644 --- a/spreg/error_sp_regimes.py +++ b/spreg/error_sp_regimes.py @@ -60,7 +60,7 @@ class GM_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -80,7 +80,9 @@ class GM_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -364,7 +366,7 @@ def __init__( if slx_lags == 0: self.title = "GM SPATIALLY WEIGHTED MODEL - REGIMES" else: - self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SDEM) - REGIMES" + self.title = "GM SPATIALLY WEIGHTED MODEL + SLX (SLX-Error) - REGIMES" self.name_x.append("lambda") self.kf += 1 self.chow = REGI.Chow(self) @@ -538,7 +540,7 @@ class GM_Endog_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Always False, kept for consistency, ignored. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. vm : boolean If True, include variance-covariance matrix in summary results @@ -562,7 +564,9 @@ class GM_Endog_Error_Regimes(RegressionPropsY, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -945,7 +949,7 @@ def __init__( if slx_lags == 0: self.title = ("GM SPATIALLY WEIGHTED 2SLS - REGIMES") else: - self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SDEM) - REGIMES") + self.title = ("GM SPATIALLY WEIGHTED 2SLS WITH SLX (SLX-Error) - REGIMES") output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) @@ -1200,7 +1204,9 @@ class GM_Combo_Regimes(GM_Endog_Error_Regimes, REGI.Regimes_Frame): Name of regime variable for use in the output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. Attributes ---------- output : dataframe @@ -1634,7 +1640,7 @@ class GMM_Error_Regimes(GM_Error_Regimes, GM_Combo_Regimes, GM_Endog_Error_Regim If True, then a spatial lag of the dependent variable is included. slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM or GNSM type. + If slx_lags>0, the specification becomes of the SLX-Error or GNSM type. vm : boolean If True, include variance-covariance matrix in summary results @@ -1654,6 +1660,9 @@ class GMM_Error_Regimes(GM_Error_Regimes, GM_Combo_Regimes, GM_Endog_Error_Regim Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. **kwargs : keywords Additional arguments to pass on to the estimators. See the specific functions for details on what can be used. @@ -2020,7 +2029,7 @@ def _work_endog_error( if add_lag != False: model.title = "GM SPATIAL COMBO MODEL + SLX (GNSM) - REGIME %s" % r else: - model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SDEM) - REGIME %s" % r + model.title = "GM SPATIALLY WEIGHTED 2SLS + SLX (SLX-Error) - REGIME %s" % r model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) model.name_x = ["%s_%s" % (str(r), i) for i in name_x] diff --git a/spreg/ml_error.py b/spreg/ml_error.py index b6913d25..aad98fab 100644 --- a/spreg/ml_error.py +++ b/spreg/ml_error.py @@ -313,7 +313,7 @@ class ML_Error(BaseML_Error): Spatial weights sparse matrix slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. method : string if 'full', brute force calculation (full matrix expressions) if 'ord', Ord eigenvalue method @@ -485,14 +485,15 @@ def __init__( y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # initialize in case None includes constant set_warn(self, warn) - self.title = "ML SPATIAL ERROR" if slx_lags >0: lag_x = get_lags(w, x_constant[:, 1:], slx_lags) x_constant = np.hstack((x_constant, lag_x)) - name_x += USER.set_name_spatial_lags(name_x, slx_lags) - self.title += " WITH SLX (SDEM)" +# name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant from name_x + self.title += " WITH SLX (SLX-Error)" self.title += " (METHOD = " + method + ")" method = method.upper() @@ -501,7 +502,7 @@ def __init__( ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) + self.name_x = name_x self.name_x.append("lambda") self.name_w = USER.set_name_w(name_w, w) self.aic = DIAG.akaike(reg=self) diff --git a/spreg/ml_error_regimes.py b/spreg/ml_error_regimes.py index 721fd115..9793b9df 100644 --- a/spreg/ml_error_regimes.py +++ b/spreg/ml_error_regimes.py @@ -54,7 +54,7 @@ class ML_Error_Regimes(BaseML_Error, REGI.Regimes_Frame): Spatial weights sparse matrix slx_lags : integer Number of spatial lags of X to include in the model specification. - If slx_lags>0, the specification becomes of the SDEM type. + If slx_lags>0, the specification becomes of the SLX-Error type. method : string if 'full', brute force calculation (full matrix expressions) if 'ord', Ord eigenvalue computation @@ -83,7 +83,7 @@ class ML_Error_Regimes(BaseML_Error, REGI.Regimes_Frame): name_regimes : string Name of regimes variable for use in output latex : boolean - Specifies if summary is to be printed in latex format + Specifies if the table with the coefficients' results and their inference is to be printed in LaTeX format Attributes ---------- @@ -378,7 +378,7 @@ def __init__( self.title = "ML SPATIAL ERROR" if slx_lags >0: - self.title += " WITH SLX (SDEM)" + self.title += " WITH SLX (SLX-Error)" self.title += " - REGIMES (METHOD = " + method + ")" self.name_x = USER.set_name_x(name_x, x, constant=True) @@ -527,7 +527,7 @@ def _work_error( model.w = w_r model.title = "ML SPATIAL ERROR" if slx_lags >0: - model.title += " WITH SLX (SDEM)" + model.title += " WITH SLX (SLX-Error)" model.title += " - REGIME " + str(r) + " (METHOD = " + method + ")" model.name_ds = name_ds model.name_y = "%s_%s" % (str(r), name_y) diff --git a/spreg/ml_lag.py b/spreg/ml_lag.py index c3ec794c..5d350635 100755 --- a/spreg/ml_lag.py +++ b/spreg/ml_lag.py @@ -2,7 +2,8 @@ ML Estimation of Spatial Lag Model """ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ + Pedro V. Amaral pedrovma@gmail.com, \ Serge Rey srey@asu.edu, \ Levi Wolf levi.john.wolf@gmail.com" @@ -11,11 +12,11 @@ from scipy import sparse as sp from scipy.sparse.linalg import splu as SuperLU from .utils import RegressionPropsY, RegressionPropsVM, inverse_prod, set_warn, get_lags -from .sputils import spdot, spfill_diagonal, spinv, spbroadcast +from .sputils import spdot, spfill_diagonal, spinv, _spmultiplier from . import diagnostics as DIAG from . import user_output as USER import pandas as pd -from .output import output, _nonspat_top, _spat_pseudo_r2 +from .output import output, _nonspat_top, _spat_diag_out, _spat_pseudo_r2, _summary_impacts from .w_utils import symmetrize from libpysal import weights @@ -337,6 +338,11 @@ class ML_Lag(BaseML_Lag): if 'ord', Ord eigenvalue method epsilon : float tolerance criterion in mimimize_scalar function and inverse_product + spat_diag : boolean + If True, then compute Common Factor Hypothesis test when applicable + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results vm : boolean if True, include variance-covariance matrix in summary results @@ -394,6 +400,9 @@ class ML_Lag(BaseML_Lag): Akaike information criterion schwarz : float Schwarz criterion + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). predy_e : array predicted values from reduced form e_pred : array @@ -585,7 +594,9 @@ def __init__( slx_lags=0, method="full", epsilon=0.0000001, + spat_impacts=True, vm=False, + spat_diag=True, name_y=None, name_x=None, name_w=None, @@ -596,6 +607,7 @@ def __init__( y = USER.check_y(y, n) USER.check_weights(w, y, w_required=True) x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # needs to be initialized for none, now with constant set_warn(self, warn) method = method.upper() BaseML_Lag.__init__( @@ -605,24 +617,37 @@ def __init__( self.k += 1 if slx_lags>0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) + kx = len(name_x) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant self.title = "MAXIMUM LIKELIHOOD SPATIAL LAG WITH SLX - SPATIAL DURBIN MODEL" + " (METHOD = " + method + ")" + var_types = ['x'] * kx + ['wx'] * (kx-1) * slx_lags + ['rho'] else: self.title = "MAXIMUM LIKELIHOOD SPATIAL LAG" + " (METHOD = " + method + ")" + var_types = ['x'] * len(name_x) + ['rho'] + self.slx_lags = slx_lags self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x) + self.name_x = name_x # already has constant name_ylag = USER.set_name_yend_sp(self.name_y) self.name_x.append(name_ylag) # rho changed to last position self.name_w = USER.set_name_w(name_w, w) self.aic = DIAG.akaike(reg=self) self.schwarz = DIAG.schwarz(reg=self) self.output = pd.DataFrame(self.name_x, columns=['var_names']) - self.output['var_type'] = ['x'] * (len(self.name_x)-1) + ['rho'] + self.output['var_type'] = var_types self.output['regime'], self.output['equation'] = (0, 0) self.other_top = _spat_pseudo_r2(self) self.other_top += _nonspat_top(self, ml=True) - output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) + diag_out = None + if spat_diag and slx_lags==1: + diag_out = _spat_diag_out(self, w, 'yend', ml=True) + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts + output(reg=self, vm=vm, robust=False, other_end=diag_out, latex=latex) def lag_c_loglik(rho, n, e0, e1, W): # concentrated log-lik for lag model, no constants, brute force diff --git a/spreg/ml_lag_regimes.py b/spreg/ml_lag_regimes.py index a05c00b2..93da512f 100644 --- a/spreg/ml_lag_regimes.py +++ b/spreg/ml_lag_regimes.py @@ -11,9 +11,9 @@ import multiprocessing as mp from .ml_lag import BaseML_Lag from .utils import set_warn, get_lags -from platform import system +from .sputils import _spmultiplier import pandas as pd -from .output import output, _nonspat_top, _spat_pseudo_r2 +from .output import output, _nonspat_top, _spat_diag_out, _spat_pseudo_r2, _summary_impacts __all__ = ["ML_Lag_Regimes"] @@ -64,6 +64,11 @@ class ML_Lag_Regimes(BaseML_Lag, REGI.Regimes_Frame): If True, the spatial parameter for spatial lag is also computed according to different regimes. If False (default), the spatial parameter is fixed accross regimes. + spat_diag : boolean + If True, then compute Common Factor Hypothesis test when applicable + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results cores : boolean Specifies if multiprocessing is to be used Default: no multiprocessing, cores = False @@ -120,6 +125,9 @@ class ML_Lag_Regimes(BaseML_Lag, REGI.Regimes_Frame): if 'full': brute force (full matrix computations) if 'ord', Ord eigenvalue method if 'LU', LU sparse matrix decomposition + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). epsilon : float tolerance criterion used in minimize_scalar function and inverse_product mean_y : float @@ -309,6 +317,8 @@ def __init__( slx_lags=0, regime_lag_sep=False, cores=False, + spat_diag=True, + spat_impacts=True, vm=False, name_y=None, name_x=None, @@ -328,10 +338,11 @@ def __init__( set_warn(self, warn) name_x = USER.set_name_x(name_x, x_constant, constant=True) - if slx_lags >0: + if slx_lags > 0: lag_x = get_lags(w, x_constant, slx_lags) x_constant = np.hstack((x_constant, lag_x)) name_x += USER.set_name_spatial_lags(name_x, slx_lags) + kwx = lag_x.shape[1] self.name_x_r = USER.set_name_x(name_x, x_constant) + [USER.set_name_yend_sp(name_y)] self.method = method @@ -384,6 +395,8 @@ def __init__( cols2regi=cols2regi, method=method, epsilon=epsilon, + spat_diag=spat_diag, + spat_impacts=spat_impacts, vm=vm, name_y=name_y, name_x=name_x, @@ -407,23 +420,43 @@ def __init__( self.name_x.append("_Global_" + USER.set_name_yend_sp(name_y)) BaseML_Lag.__init__(self, y=y, x=x, w=w, method=method, epsilon=epsilon) self.kf += 1 # Adding a fixed k to account for spatial lag in Chow - # adding a fixed k to account for spatial lag in aic, sc + # adding a k to account for spatial lag in aic, sc self.k += 1 self.chow = REGI.Chow(self) self.aic = DIAG.akaike(reg=self) self.schwarz = DIAG.schwarz(reg=self) self.regime_lag_sep = regime_lag_sep self.output = pd.DataFrame(self.name_x, columns=['var_names']) + self.output['regime'] = x_rlist + ['_Global'] self.output['var_type'] = ['x'] * (len(self.name_x) - 1) + ['rho'] - self.output['regime'] = x_rlist+['_Global'] self.output['equation'] = 0 + self.slx_lags = slx_lags + diag_out = None + if slx_lags > 0: + self.title = ("MAXIMUM LIKELIHOOD SPATIAL DURBIN - REGIMES"+ " (METHOD = "+ method+ ")") + fixed_wx = cols2regi[-(kwx+1):-1].count(False) + kwx = kwx - fixed_wx + if kwx > 0: + for m in self.regimes_set: + r_output = self.output[(self.output['regime'] == str(m)) & (self.output['var_type'] == 'x')] + wx_index = r_output.index[-kwx:] + self.output.loc[wx_index, 'var_type'] = 'wx' + if fixed_wx > 0: + f_wx_index = self.output.index[-(fixed_wx+1):-1] + self.output.loc[f_wx_index, 'var_type'] = 'wx' + if spat_diag and slx_lags == 1: + diag_out = _spat_diag_out(self, w, 'yend', ml=True) + else: + self.title = ("MAXIMUM LIKELIHOOD SPATIAL LAG - REGIMES"+ " (METHOD = "+ method+ ")") + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts self.other_top = _spat_pseudo_r2(self) self.other_top += _nonspat_top(self, ml=True) - if slx_lags == 0: - self.title = ("MAXIMUM LIKELIHOOD SPATIAL LAG - REGIMES"+ " (METHOD = "+ method+ ")") - else: - self.title = ("MAXIMUM LIKELIHOOD SPATIAL DURBIN - REGIMES"+ " (METHOD = "+ method+ ")") - output(reg=self, vm=vm, robust=False, other_end=False, latex=latex) + output(reg=self, vm=vm, robust=False, other_end=diag_out, latex=latex) def ML_Lag_Regimes_Multi( self, @@ -437,6 +470,8 @@ def ML_Lag_Regimes_Multi( cols2regi, method, epsilon, + spat_diag, + spat_impacts, vm, name_y, name_x, @@ -558,9 +593,19 @@ def ML_Lag_Regimes_Multi( results[r].other_top = _spat_pseudo_r2(results[r]) results[r].other_top += _nonspat_top(results[r], ml=True) results[r].other_mid = "" - self.output = pd.concat([self.output, pd.DataFrame({'var_names': results[r].name_x, - 'var_type': ['x'] * (len(results[r].name_x) - 1) + ['rho'], - 'regime': r, 'equation': r})], ignore_index=True) + if slx_lags > 0: + kx = (len(results[r].name_x) - 1) // (slx_lags + 1) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['rho'] + else: + var_types = ['x'] * (len(results[r].name_x) - 1) + ['rho'] + results[r].output = pd.DataFrame({'var_names': results[r].name_x, + 'var_type': var_types, + 'regime': r, 'equation': r}) + self.output = pd.concat([self.output, results[r].output], ignore_index=True) + if spat_diag and slx_lags == 1: + results[r].other_mid += _spat_diag_out(results[r], None, 'yend', ml=True) + if spat_impacts and slx_lags == 0: + results[r].other_mid += _summary_impacts(results[r], _spmultiplier(results[r].w, results[r].rho)) counter += 1 self.multi = results self.chow = REGI.Chow(self) @@ -597,6 +642,9 @@ def _work( model.k += 1 # add 1 for proper df and aic, sc model.aic = DIAG.akaike(reg=model) model.schwarz = DIAG.schwarz(reg=model) + model.slx_lags = slx_lags + model.w = w_r + model.rho = model.betas[-1] return model diff --git a/spreg/ols.py b/spreg/ols.py index f22bac19..8efb9c70 100644 --- a/spreg/ols.py +++ b/spreg/ols.py @@ -175,7 +175,7 @@ class OLS(BaseOLS): name_ds : string Name of dataset for use in output latex : boolean - Specifies if summary is to be printed in latex format + Specifies if the table with the coefficients' results and their inference is to be printed in LaTeX format Attributes ---------- @@ -479,6 +479,7 @@ def __init__( ) self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) + self.slx_lags = slx_lags self.title = "ORDINARY LEAST SQUARES" if slx_lags > 0: self.title += " WITH SPATIALLY LAGGED X (SLX)" diff --git a/spreg/ols_regimes.py b/spreg/ols_regimes.py index 20b2acea..66ba3d8e 100755 --- a/spreg/ols_regimes.py +++ b/spreg/ols_regimes.py @@ -9,11 +9,11 @@ import pandas as pd from . import regimes as REGI from . import user_output as USER -from .utils import set_warn, RegressionProps_basic, spdot, RegressionPropsY, get_lags +from .utils import set_warn, RegressionProps_basic, spdot, RegressionPropsY, get_lags, optim_k from .ols import BaseOLS from .robust import hac_multi from .output import output, _spat_diag_out, _nonspat_mid, _nonspat_top - +from .skater_reg import Skater_reg class OLS_Regimes(BaseOLS, REGI.Regimes_Frame, RegressionPropsY): """ @@ -447,6 +447,7 @@ def __init__( else: USER.check_weights(w, y, w_required=False) set_warn(self, warn) + self.slx_lags = slx_lags self.name_x_r = USER.set_name_x(name_x, x_constant) self.constant_regi = constant_regi self.cols2regi = cols2regi @@ -529,7 +530,7 @@ def __init__( self.other_mid += _nonspat_mid(self, white_test=white_test) self.other_top += _nonspat_top(self) if spat_diag: - other_end += _spat_diag_out(self, w, 'ols', moran=moran) + other_end += _spat_diag_out(self, w, 'ols', moran=moran) #Must decide what to do with W. output(reg=self, vm=vm, robust=robust, other_end=other_end, latex=latex) def _ols_regimes_multi( @@ -666,7 +667,7 @@ def _ols_regimes_multi( other_end = "" if spat_diag: self._get_spat_diag_props(x_constant, sig2n_k) - other_end += _spat_diag_out(self, w, 'ols', moran=moran) + #other_end += _spat_diag_out(self, w, 'ols', moran=moran) Need to consider W before implementing output(reg=self, vm=vm, robust=robust, other_end=other_end, latex=latex) def _get_spat_diag_props(self, x, sig2n_k): @@ -706,6 +707,43 @@ def _work( return model +class OLS_Endog_Regimes(OLS_Regimes): + def __init__( + self, y, x, w, n_clusters=None, quorum=-np.inf, trace=True, **kwargs): + + n = USER.check_arrays(y, x) + y = USER.check_y(y, n) + USER.check_weights(w, y, w_required=True) + + # Standardize the variables + x_std = (x - np.mean(x, axis=0)) / np.std(x, axis=0) + + if not n_clusters: + if quorum < 0: + quorum = np.max([(x.shape[1]+1)*10, 30]) + n_clusters_opt = x.shape[0]*0.70//quorum + if n_clusters_opt < 2: + raise ValueError( + "The combination of the values of `N` and `quorum` is not compatible with regimes estimation.") + sk_reg_results = Skater_reg().fit(n_clusters_opt, w, x_std, {'reg':BaseOLS,'y':y,'x':x}, quorum=quorum, trace=True) + n_clusters = optim_k([sk_reg_results._trace[i][1][2] for i in range(1, len(sk_reg_results._trace))]) + self.clusters = sk_reg_results._trace[n_clusters-1][0] + else: + try: + # Call the Skater_reg method based on OLS + sk_reg_results = Skater_reg().fit(n_clusters, w, x_std, {'reg':BaseOLS,'y':y,'x':x}, quorum=quorum, trace=trace) + self.clusters = sk_reg_results.current_labels_ + except Exception as e: + if str(e) == "one or more input arrays have more columns than rows": + raise ValueError("One or more input ended up with more variables than observations. Please check your setting for `quorum`.") + else: + print("An error occurred:", e) + + self._trace = sk_reg_results._trace + self.SSR = [self._trace[i][1][2] for i in range(1, len(self._trace))] + + OLS_Regimes.__init__(self, y, x, regimes=self.clusters, w=w, name_regimes='Skater_reg', **kwargs) + def _test(): import doctest diff --git a/spreg/output.py b/spreg/output.py index 25f94e66..389fd12e 100644 --- a/spreg/output.py +++ b/spreg/output.py @@ -5,6 +5,7 @@ import textwrap as TW import numpy as np import pandas as pd +import math from . import diagnostics as diagnostics from . import diagnostics_tsls as diagnostics_tsls from . import diagnostics_sp as diagnostics_sp @@ -16,8 +17,6 @@ ############################################################################### def output(reg, vm, other_end=False, robust=False, latex=False): - if latex: - print("Warning: Latex output not implemented yet. Using standard output instead.") strSummary = output_start(reg) for eq in reg.output['equation'].unique(): try: @@ -26,7 +25,7 @@ def output(reg, vm, other_end=False, robust=False, latex=False): except: eq = None strSummary, reg = out_part_top(strSummary, reg, eq) - strSummary, reg = out_part_middle(strSummary, reg, robust, m=eq) + strSummary, reg = out_part_middle(strSummary, reg, robust, m=eq, latex=latex) strSummary, reg = out_part_end(strSummary, reg, vm, other_end, m=eq) reg.summary = strSummary reg.output.sort_values(by=['equation', 'regime'], inplace=True) @@ -102,7 +101,7 @@ def out_part_top(strSummary, reg, m): return (strSummary, _reg) -def out_part_middle(strSummary, reg, robust, m=None): +def out_part_middle(strSummary, reg, robust, m=None, latex=False): # Middle part of summary output. # m = None for single models, m = 1,2,3... for multiple equation models if m==None: @@ -127,27 +126,40 @@ def out_part_middle(strSummary, reg, robust, m=None): elif robust == "ogmm": strSummary += "Optimal GMM used to estimate the coefficients and the variance-covariance matrix\n" strSummary += "------------------------------------------------------------------------------------\n" - strSummary += ( - " Variable Coefficient Std.Error %1s-Statistic Probability\n" - % (_reg.__summary["summary_zt"]) - ) - strSummary += "------------------------------------------------------------------------------------\n" + m_output = reg.output[reg.output['equation'] == m] - for row in m_output.iloc[np.lexsort((m_output.index, m_output['regime']))].itertuples(): - try: - strSummary += "%20s %12.5f %12.5f %12.5f %12.5f\n" % ( - row.var_names, - row.coefficients, - row.std_err, - row.zt_stat, - row.prob - ) - except TypeError: # special case for models that do not have inference on the lambda term - strSummary += "%20s %12.5f \n" % ( - row.var_names, - row.coefficients - ) - strSummary += "------------------------------------------------------------------------------------\n" + if latex: + df_1 = m_output.iloc[np.lexsort((m_output.index, m_output['regime']))] + df_2 = df_1.loc[:, ['var_names', 'coefficients', 'std_err', 'zt_stat', 'prob']] + df_2 = df_2.set_axis(['Variable', 'Coefficient', 'Std.Error', _reg.__summary['summary_zt']+'-Statistic', 'Prob.'], axis='columns', copy=False) + cols = df_2.columns.difference(['Variable']) + df_2[cols] = df_2[cols].astype(float).map(lambda x: "%12.5f" % x) + df_2['Variable'] = df_2['Variable'].str.replace("_", "\_").str.replace("%", "\%") + df_inlatex = df_2.style.hide(axis='index').to_latex(hrules=True) + strSummary += df_inlatex + strSummary += "------------------------------------------------------------------------------------\n" + else: + strSummary += ( + " Variable Coefficient Std.Error %1s-Statistic Probability\n" + % (_reg.__summary["summary_zt"]) + ) + strSummary += "------------------------------------------------------------------------------------\n" + + for row in m_output.iloc[np.lexsort((m_output.index, m_output['regime']))].itertuples(): + try: + strSummary += "%20s %12.5f %12.5f %12.5f %12.5f\n" % ( + row.var_names, + row.coefficients, + row.std_err, + row.zt_stat, + row.prob + ) + except TypeError: # special case for models that do not have inference on the lambda term + strSummary += "%20s %12.5f \n" % ( + row.var_names, + row.coefficients + ) + strSummary += "------------------------------------------------------------------------------------\n" try: # Adding info on instruments if they are present name_q = _reg.name_q @@ -212,7 +224,7 @@ def _summary_chow(reg): name_x_r = reg.name_x_r joint, regi = reg.chow.joint, reg.chow.regi sum_text += "\n VARIABLE DF VALUE PROB\n" - if reg.cols2regi == "all": + if reg.cols2regi == "all" or set(reg.cols2regi) == {True}: names_chow = name_x_r[1:] else: names_chow = [name_x_r[1:][i] for i in np.where(reg.cols2regi)[0]] @@ -231,7 +243,7 @@ def _summary_chow(reg): 'df': reg.kr * (reg.nr - 1), 'value': joint[0], 'prob': joint[1]}])], ignore_index=True) for row in reg.output_chow.itertuples(): - sum_text += "%25s %2d %12.3f %9.4f\n" % ( + sum_text += "%20s %2d %12.3f %9.4f\n" % ( row.var_names, row.df, row.value, @@ -241,73 +253,152 @@ def _summary_chow(reg): return sum_text -def _spat_diag_out(reg, w, type, moran=False): +def _spat_diag_out(reg, w, type, moran=False, ml=False): strSummary = "\nDIAGNOSTICS FOR SPATIAL DEPENDENCE\n" - if not moran: - strSummary += ( - "TEST DF VALUE PROB\n" - ) - else: - strSummary += ( - "TEST MI/DF VALUE PROB\n" - ) - cache = diagnostics_sp.spDcache(reg, w) if type == "yend": - mi, ak, ak_p = diagnostics_sp.akTest(reg, w, cache) - reg.ak_test = ak, ak_p - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( - "Anselin-Kelejian Test", - 1, - reg.ak_test[0], - reg.ak_test[1], - ) + strSummary += ( + "TEST DF VALUE PROB\n") + if not ml: + mi, ak, ak_p = diagnostics_sp.akTest(reg, w, cache) + reg.ak_test = ak, ak_p + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Anselin-Kelejian Test", + 1, + reg.ak_test[0], + reg.ak_test[1], + ) + if any(reg.output['var_type'] == 'rho'): + if reg.slx_lags == 1 and not any(reg.output['var_type'] == 'yend'): + wx_indices = reg.output[(reg.output['var_type'] == 'wx') & (reg.output['regime'] != '_Global')].index + x_indices = [] + for m in reg.output['regime'].unique(): + x_indices.extend(reg.output[(reg.output['regime'] == m) & (reg.output['var_type'] == 'x')].index[1:]) + vm_indices = x_indices + wx_indices.tolist() + reg.output[reg.output['var_type'] == 'rho'].index.tolist() + cft, cft_p = diagnostics_sp.comfac_test(reg.rho, + reg.betas[x_indices], + reg.betas[wx_indices], + reg.vm[vm_indices, :][:, vm_indices]) + reg.cfh_test = cft, cft_p + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Common Factor Hypothesis Test", + len(wx_indices), + reg.cfh_test[0], + reg.cfh_test[1], + ) + elif type == "ols": - lm_tests = diagnostics_sp.LMtests(reg, w) + strSummary += "- SARMA -\n" + if not moran: + strSummary += ( + "TEST DF VALUE PROB\n" + ) + else: + strSummary += ( + "TEST MI/DF VALUE PROB\n" + ) + lm_tests = diagnostics_sp.LMtests(reg, w, tests=["lme", "lml", "rlme", "rlml", "sarma"]) + if reg.slx_lags == 0: + try: + lm_tests2 = diagnostics_sp.LMtests(reg, w, tests=["lmwx", "lmspdurbin", "rlmdurlag", "rlmwx","lmslxerr"]) + reg.lm_wx = lm_tests2.lmwx + reg.lm_spdurbin = lm_tests2.lmspdurbin + reg.rlm_wx = lm_tests2.rlmwx + reg.rlm_durlag = lm_tests2.rlmdurlag + reg.lm_slxerr = lm_tests2.lmslxerr #currently removed. - LA reinstated + koley_bera = True + except: + koley_bera = False reg.lm_error = lm_tests.lme reg.lm_lag = lm_tests.lml reg.rlm_error = lm_tests.rlme reg.rlm_lag = lm_tests.rlml reg.lm_sarma = lm_tests.sarma + + if moran: moran_res = diagnostics_sp.MoranRes(reg, w, z=True) reg.moran_res = moran_res.I, moran_res.zI, moran_res.p_norm - strSummary += "%-27s %8.4f %9.3f %9.4f\n" % ( + strSummary += "%-27s %8.4f %9.3f %9.4f\n" % ( "Moran's I (error)", reg.moran_res[0], reg.moran_res[1], reg.moran_res[2], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (lag)", 1, reg.lm_lag[0], reg.lm_lag[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Robust LM (lag)", 1, reg.rlm_lag[0], reg.rlm_lag[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (error)", 1, reg.lm_error[0], reg.lm_error[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Robust LM (error)", 1, reg.rlm_error[0], reg.rlm_error[1], ) - strSummary += "%-27s %2d %12.3f %9.4f\n\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Lagrange Multiplier (SARMA)", 2, reg.lm_sarma[0], reg.lm_sarma[1], ) + if reg.slx_lags == 0 and koley_bera: + strSummary += ( + "\n- Spatial Durbin -\nTEST DF VALUE PROB\n" + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "LM test for WX", + reg.k-1, + reg.lm_wx[0], + reg.lm_wx[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Robust LM WX test", + reg.k-1, + reg.rlm_wx[0], + reg.rlm_wx[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Lagrange Multiplier (lag)", + 1, + reg.lm_lag[0], + reg.lm_lag[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Robust LM Lag - SDM", + 1, + reg.rlm_durlag[0], + reg.rlm_durlag[1], + ) + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + "Joint test for SDM", + reg.k, + reg.lm_spdurbin[0], + reg.lm_spdurbin[1], + ) + #strSummary += ( + # "\n- Spatial Error and WX -\nTEST DF VALUE PROB\n" + #) + #strSummary += "%-27s %2d %12.3f %9.4f\n\n" % ( + # "Joint test for Error and WX", + # reg.k, + # reg.lm_slxerr[0], + # reg.lm_slxerr[1], + #) + return strSummary def _nonspat_top(reg, ml=False): @@ -376,10 +467,10 @@ def _nonspat_mid(reg, white_test=False): strSummary = "\nREGRESSION DIAGNOSTICS\n" if reg.mulColli: - strSummary += "MULTICOLLINEARITY CONDITION NUMBER %16.3f\n\n" % (reg.mulColli) + strSummary += "MULTICOLLINEARITY CONDITION NUMBER %15.3f\n\n" % (reg.mulColli) strSummary += "TEST ON NORMALITY OF ERRORS\n" strSummary += "TEST DF VALUE PROB\n" - strSummary += "%-27s %2d %14.3f %9.4f\n\n" % ( + strSummary += "%-27s %2d %14.3f %9.4f\n\n" % ( "Jarque-Bera", reg.jarque_bera["df"], reg.jarque_bera["jb"], @@ -388,13 +479,13 @@ def _nonspat_mid(reg, white_test=False): strSummary += "DIAGNOSTICS FOR HETEROSKEDASTICITY\n" strSummary += "RANDOM COEFFICIENTS\n" strSummary += "TEST DF VALUE PROB\n" - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Breusch-Pagan test", reg.breusch_pagan["df"], reg.breusch_pagan["bp"], reg.breusch_pagan["pvalue"], ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "Koenker-Bassett test", reg.koenker_bassett["df"], reg.koenker_bassett["kb"], @@ -409,7 +500,7 @@ def _nonspat_mid(reg, white_test=False): strSummary += ( "TEST DF VALUE PROB\n" ) - strSummary += "%-27s %2d %12.3f %9.4f\n" % ( + strSummary += "%-27s %2d %12.3f %9.4f\n" % ( "White", reg.white["df"], reg.white["wh"], @@ -476,3 +567,33 @@ def _summary_iteration(reg): pass return txt + +def _summary_impacts(reg, spmult): + """ + Spatial direct, indirect and total effects in spatial lag model. + Uses multipliers computed by sputils._spmultipliers. + + Attributes + ---------- + reg: spreg regression object + spmult: spatial multipliers as a dictionary + + Returns + ------- + strings with direct, indirect and total effects + + """ + variables = reg.output[reg.output['var_type'].isin(['x', 'wx', 'yend']) & (reg.output.index != 0)] + variables_index = variables.index + m1 = spmult['ati'] + btot = m1 * reg.betas[variables_index] + m2 = spmult['adi'] + bdir = m2 * reg.betas[variables_index] + m3 = spmult['aii'] + bind = m3 * reg.betas[variables_index] + strSummary = "\nSPATIAL LAG MODEL IMPACTS\n" + strSummary += " Variable Direct Indirect Total\n" + for i in range(len(variables)): + strSummary += "%20s %12.4f %12.4f %12.4f\n" % ( + variables['var_names'][variables_index[i]], bdir[i][0], bind[i][0], btot[i][0]) + return strSummary \ No newline at end of file diff --git a/spreg/robust.py b/spreg/robust.py index 7d20b103..caf58910 100644 --- a/spreg/robust.py +++ b/spreg/robust.py @@ -1,5 +1,5 @@ -__author__ = "Luc Anselin luc.anselin@asu.edu, \ - Pedro V. Amaral pedro.amaral@asu.edu, \ +__author__ = "Luc Anselin lanselin@gmail.com, \ + Pedro V. Amaral pedrovma@gmail.com, \ David C. Folch david.folch@asu.edu" import numpy as np diff --git a/spreg/skater_reg.py b/spreg/skater_reg.py index 9bfbd3fd..2b203fbf 100755 --- a/spreg/skater_reg.py +++ b/spreg/skater_reg.py @@ -4,11 +4,11 @@ from scipy.sparse import csgraph as cg from scipy.optimize import OptimizeWarning -from scipy.spatial.distance import cdist from collections import namedtuple from warnings import warn from libpysal.weights import w_subset from .utils import set_endog +from .twosls_regimes import TSLS_Regimes import time import numpy as np import copy @@ -68,7 +68,7 @@ class Skater_reg(object): y : array n*1, dependent variable X : array - n*k, independent variable, exlcuding the constant + n*k, independent variable, not including the constant bw : scalar bandwidth value consisting of either a distance or N nearest neighbors; user specified or obtained using @@ -308,7 +308,9 @@ def fit( print("cut made {}...".format(best_deletion)) if best_deletion.score > prev_score: raise ValueError( - "The score increased with the number of clusters. Please check your data." + ("The score increased with the number of clusters. " + "Please check your data.\nquorum: {}; n_clusters: {}" + ).format(quorum, n_clusters) ) prev_score = best_deletion.score MSF = new_MSF @@ -358,7 +360,7 @@ def score_spreg( Any region below quorum makes the score inf. current_labels: (N,) flat vector of labels expressing the classification of each observation into a region not considering the cut under evaluation. - current_tree: integer indicating the tree label is currently being considered for division + current_tree: integer indicating the tree's label currently being considered for division """ labels, subtree_quorums = self._prep_score( @@ -374,44 +376,62 @@ def score_spreg( if k not in ["reg", "y", "x", "w", "x_nd"] } trees_scores = {} - l_arrays = np.array(all_labels) - w_regi_i = None - for l in set_labels: - x = data_reg["x"][all_labels == l] - if np.linalg.matrix_rank(x) < x.shape[1]: - x = np.delete( - x, np.nonzero(np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10), 1 - ) - if "w" not in data_reg: - try: - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, **kargs - ) - except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, **kargs - ) - - else: - regi_ids = list(np.where(l_arrays == l)[0]) - w_ids = list(map(data_reg["w"].id_order.__getitem__, regi_ids)) - w_regi_i = w_subset(data_reg["w"], w_ids, silence_warnings=True) - try: - x = np.hstack((np.ones((x.shape[0], 1)), x)) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs - ) - except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - reg = data_reg["reg"]( - y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs - ) - trees_scores[l] = np.sum(reg.u ** 2) - score = sum(trees_scores.values()) + if data_reg["reg"].__name__ == "GM_Lag" or data_reg["reg"].__name__ == "BaseGM_Lag": + try: + x = np.hstack((np.ones((data_reg["x"].shape[0], 1)), data_reg["x"])) + reg = TSLS_Regimes( + y=data_reg["y"], + x=x, + yend=data_reg["yend"], + q=data_reg["q"], + regimes=all_labels,) + except: + x = _const_x(data_reg["x"]) + reg = TSLS_Regimes( + y=data_reg["y"], + x=x, + yend=data_reg["yend"], + q=data_reg["q"], + regimes=all_labels,) + score = np.dot(reg.u.T, reg.u)[0][0] + else: + + for l in set_labels: + x = data_reg["x"][all_labels == l] + if np.linalg.matrix_rank(x) < x.shape[1]: + small_diag_indices = np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10 + x = x[:, ~small_diag_indices] + + if "w" not in data_reg: + try: + x = np.hstack((np.ones((x.shape[0], 1)), x)) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, **kargs + ) + except np.linalg.LinAlgError: + x = _const_x(x) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, **kargs + ) + else: + l_arrays = np.array(all_labels) + + regi_ids = list(np.where(l_arrays == l)[0]) + w_ids = list(map(data_reg["w"].id_order.__getitem__, regi_ids)) + w_regi_i = w_subset(data_reg["w"], w_ids, silence_warnings=True) + try: + x = np.hstack((np.ones((x.shape[0], 1)), x)) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs + ) + except np.linalg.LinAlgError: + x = _const_x(x) + reg = data_reg["reg"]( + y=data_reg["y"][all_labels == l], x=x, w=w_regi_i, **kargs + ) + trees_scores[l] = np.dot(reg.u.T, reg.u)[0][0] + score = sum(trees_scores.values()) else: part_scores, score, trees_scores = self._data_reg_none( data, all_labels, l, set_labels @@ -465,14 +485,11 @@ def score_stats( if k not in ["reg", "y", "x", "w", "x_nd"] } trees_scores = {} - l_arrays = np.array(all_labels) - w_regi_i = None for l in set_labels: x = data_reg["x"][all_labels == l] if np.linalg.matrix_rank(x) < x.shape[1]: - x = np.delete( - x, np.nonzero(np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10), 1 - ) + small_diag_indices = np.abs(np.diag(np.linalg.qr(x)[1])) < 1e-10 + x = x[:, ~small_diag_indices] try: x = np.hstack((np.ones((x.shape[0], 1)), x)) @@ -480,8 +497,7 @@ def score_stats( data_reg["y"][all_labels == l], x, **kargs ).fit() except np.linalg.LinAlgError: - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) - x = np.hstack((np.ones((x.shape[0], 1)), x)) + x = _const_x(x) reg = data_reg["reg"]( data_reg["y"][all_labels == l], x, **kargs ).fit() @@ -527,6 +543,12 @@ def _data_reg_none(self, data, all_labels, l, set_labels): trees_scores = {l: part_scores[i] for i, l in enumerate(set_labels)} return part_scores, score, trees_scores + def _prep_lag(self, data_reg): + # if the model is a spatial lag, add the lagged dependent variable to the model + data_reg['yend'], data_reg['q'] = set_endog(data_reg["y"], data_reg["x"][:, 1:], data_reg["w"], yend=None, + q=None, w_lags=1, lag_q=True) + return data_reg + def find_cut( self, MSF, @@ -594,6 +616,13 @@ def tqdm(noop, desc=""): zero_in = (labels is not None) and (target_label is not None) best_deletion = deletion(np.nan, np.nan, np.inf) best_d_score = -np.inf + + try: + if data_reg["reg"].__name__ == "GM_Lag" or data_reg["reg"].__name__ == "BaseGM_Lag": + data_reg = self._prep_lag(data_reg) + except: + pass + try: old_score = sum(trees_scores.values()) except: @@ -662,6 +691,6 @@ def tqdm(noop, desc=""): def _const_x(x): - x = np.delete(x, np.nonzero(np.ptp(x, axis=0) == 0), 1) + x = x[:, np.ptp(x, axis=0) != 0] x = np.hstack((np.ones((x.shape[0], 1)), x)) - return x + return x \ No newline at end of file diff --git a/spreg/sputils.py b/spreg/sputils.py index 350033fe..65cae18c 100644 --- a/spreg/sputils.py +++ b/spreg/sputils.py @@ -272,6 +272,64 @@ def spisfinite(a): return np.isfinite(a.sum()) +def _spmultiplier(w, rho, method="simple", mtol=0.00000001): + """" + Spatial Lag Multiplier Calculation + Follows Kim, Phipps and Anselin (2003) (simple), and LeSage and Pace (2009) (full, power) + + Attributes + ---------- + w : PySAL format spatial weights matrix + rho : spatial autoregressive coefficient + method : one of "simple" (default), full" or "power" + mtol : tolerance for power iteration (default=0.00000001) + + Returns + ------- + multipliers : dictionary with + ati = average total impact multiplier + adi = average direct impact multiplier + aii = average indirect impact multiplier + pow = powers used in power approximation (otherwise 0) + + """ + multipliers = {"ati": 1.0, "adi": 1.0, "aii": 1.0} + multipliers["pow"] = 0 + multipliers["ati"] = 1.0 / (1.0 - rho) + n = w.n + if method == "simple": + pass + elif method == "full": + wf = w.full()[0] + id0 = np.identity(n) + irw0 = (id0 - rho * wf) + invirw0 = np.linalg.inv(irw0) + adii0 = np.sum(np.diag(invirw0)) + multipliers["adi"] = adii0 / n + elif method == "power": + wf = w.full()[0] + ws3 = SP.csr_array(wf) + rhop = rho + ww = ws3 + pow = 1 + adi = 1.0 + adidiff = 100.00 + + while adidiff > mtol: + pow = pow + 1 + ww = SP.csr_matrix.dot(ww, ws3) + trw = ww.diagonal().sum() + rhop = rhop * rho + adidiff = rhop * trw / n + adi = adi + adidiff + multipliers["adi"] = adi + multipliers["pow"] = pow + else: + print("Method not supported") + multipliers["aii"] = multipliers["ati"] - multipliers["adi"] + return (multipliers) + + def _test(): import doctest diff --git a/spreg/tests/test_error_sp_het.py b/spreg/tests/test_error_sp_het.py index 8b4e6e78..727c2951 100644 --- a/spreg/tests/test_error_sp_het.py +++ b/spreg/tests/test_error_sp_het.py @@ -5,219 +5,191 @@ from libpysal.common import RTOL import spreg - class TestBaseGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.X = np.hstack((np.ones(self.y.shape),self.X)) + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.BaseGM_Error_Het(self.y, self.X, self.w.sparse, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) + class TestGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.GM_Error_Het(self.y, self.X, self.w, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34951013222581306 - np.testing.assert_allclose(reg.pr2, pr2) - stde = np.array([11.47900385, 0.36812187, 0.16156816, 0.16804717]) - np.testing.assert_allclose(reg.std_err, stde, RTOL) - z_stat = np.array( - [ - [4.18122226e00, 2.89946274e-05], - [1.93003988e00, 5.36018970e-02], - [-3.45836247e00, 5.43469673e-04], - [2.45042960e00, 1.42685863e-02], - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - + np.testing.assert_allclose(reg.pr2,pr2) + stde = np.array([ 11.47900385, 0.36812187, 0.16156816, 0.16804717]) + np.testing.assert_allclose(reg.std_err,stde,RTOL) + z_stat = np.array([[ 4.18122226e+00, 2.89946274e-05], + [ 1.93003988e+00, 5.36018970e-02], + [ -3.45836247e+00, 5.43469673e-04], + [ 2.45042960e+00, 1.42685863e-02]]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) class TestBaseGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) yd = [] yd.append(db.by_col("CRIME")) self.yd = np.array(yd).T q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.BaseGM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.46604707]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.46604707]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -227,364 +199,211 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.GM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.GM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34648011338954804 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array([28.89009873, 0.77309965, 0.46798299, 0.17767558]) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (1.9175109006819244, 0.055173057472126787), - (0.60229035155742305, 0.54698088217644414), - (-1.4324949211864271, 0.15200223057569454), - (2.3151759776869496, 0.020603303355572443), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.pr2,pr2,RTOL) + std_err = np.array([ 28.89009873, 0.77309965, 0.46798299, + 0.17767558]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(1.9175109006819244, 0.055173057472126787), (0.60229035155742305, 0.54698088217644414), (-1.4324949211864271, 0.15200223057569454), (2.3151759776869496, 0.020603303355572443)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestBaseGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) - reg = HET.BaseGM_Combo_Het( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True - ) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + self.X = np.hstack((np.ones(self.y.shape),self.X)) + reg = HET.BaseGM_Combo_Het(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy, RTOL) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.std_y,stdy,RTOL) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) class TestGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag reg = HET.GM_Combo_Het(self.y, self.X, w=self.w, step1c=True) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - ep = np.array([28.30648145]) - np.testing.assert_allclose(reg.e_pred[0], ep, RTOL) - pe = np.array([52.16052155]) - np.testing.assert_allclose(reg.predy_e[0], pe, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + ep = np.array([ 28.30648145]) + np.testing.assert_allclose(reg.e_pred[0],ep,RTOL) + pe = np.array([ 52.16052155]) + np.testing.assert_allclose(reg.predy_e[0],pe,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) pr2 = 0.3001582877472412 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.35613102283621967 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (2.6202684885795335, 0.00878605635338265), - (2.2573385444145524, 0.023986928627746887), - (-4.0351698589183433, 5.456281036278686e-05), - (-0.42277935292121521, 0.67245625315942159), - (2.1225002455741895, 0.033795752094112265), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(2.6202684885795335, 0.00878605635338265), (2.2573385444145524, 0.023986928627746887), (-4.0351698589183433, 5.456281036278686e-05), (-0.42277935292121521, 0.67245625315942159), (2.1225002455741895, 0.033795752094112265)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_error_sp_het_regimes.py b/spreg/tests/test_error_sp_het_regimes.py index d9070a7e..8606fe8b 100644 --- a/spreg/tests/test_error_sp_het_regimes.py +++ b/spreg/tests/test_error_sp_het_regimes.py @@ -5,13 +5,12 @@ from spreg.error_sp_het import GM_Error_Het, GM_Endog_Error_Het, GM_Combo_Het from libpysal.common import RTOL - class TestGM_Error_Het_Regimes(unittest.TestCase): def setUp(self): - # Columbus: - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + #Columbus: + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("CRIME")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("HOVAL")) X.append(db.by_col("INC")) @@ -25,474 +24,284 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - self.r_var = "NSA" + self.w = libpysal.weights.Queen.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' + self.r_var = 'NSA' self.regimes = db.by_col(self.r_var) - # Artficial: + #Artficial: n = 256 - self.n2 = n / 2 - self.x_a1 = np.random.uniform(-10, 10, (n, 1)) - self.x_a2 = np.random.uniform(1, 5, (n, 1)) - self.q_a = self.x_a2 + np.random.normal(0, 1, (n, 1)) - self.x_a = np.hstack((self.x_a1, self.x_a2)) - self.y_a = np.dot( - np.hstack((np.ones((n, 1)), self.x_a)), np.array([[1], [0.5], [2]]) - ) + np.random.normal(0, 1, (n, 1)) + self.n2 = n/2 + self.x_a1 = np.random.uniform(-10,10,(n,1)) + self.x_a2 = np.random.uniform(1,5,(n,1)) + self.q_a = self.x_a2 + np.random.normal(0,1,(n,1)) + self.x_a = np.hstack((self.x_a1,self.x_a2)) + self.y_a = np.dot(np.hstack((np.ones((n,1)),self.x_a)),np.array([[1],[0.5],[2]])) + np.random.normal(0,1,(n,1)) latt = int(np.sqrt(n)) - self.w_a = libpysal.weights.util.lat2W(latt, latt) - self.w_a.transform = "r" - self.regi_a = [0] * (n // 2) + [1] * (n // 2) - self.w_a1 = libpysal.weights.util.lat2W(latt // 2, latt) - self.w_a1.transform = "r" - + self.w_a = libpysal.weights.util.lat2W(latt,latt) + self.w_a.transform='r' + self.regi_a = [0]*(n//2) + [1]*(n//2) + self.w_a1 = libpysal.weights.util.lat2W(latt//2,latt) + self.w_a1.transform='r' + def test_model(self): reg = SP.GM_Error_Het_Regimes(self.y, self.X, self.regimes, self.w) - betas = np.array( - [ - [62.95986466], - [-0.15660795], - [-1.49054832], - [60.98577615], - [-0.3358993], - [-0.82129289], - [0.54662719], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) + betas = np.array([[ 62.95986466], + [ -0.15660795], + [ -1.49054832], + [ 60.98577615], + [ -0.3358993 ], + [ -0.82129289], + [ 0.54662719]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) u = np.array([-2.19031456]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([17.91629456]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 17.91629456]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n,RTOL) k = 6 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 0.0, 1.0, 80.467003, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - e = np.array([2.77847355]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) + np.testing.assert_allclose(reg.k,k,RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 0. , 1. , 80.467003, 19.531 ]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + e = np.array([ 2.77847355]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 3.86154100e01, - -2.51553730e-01, - -8.20138673e-01, - 1.71714184e00, - -1.94929113e-02, - 1.23118051e-01, - 0.00000000e00, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 3.86154100e+01, -2.51553730e-01, -8.20138673e-01, + 1.71714184e+00, -1.94929113e-02, 1.23118051e-01, + 0.00000000e+00]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL*10) pr2 = 0.5515791216043385 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array( - [ - 6.21412987, - 0.15340022, - 0.44060473, - 7.6032169, - 0.19353719, - 0.73621596, - 0.13968272, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [ - [0.04190799, 0.83779526], - [0.5736724, 0.44880328], - [0.62498575, 0.42920056], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2,pr2, RTOL) + std_err = np.array([ 6.21412987, 0.15340022, 0.44060473, 7.6032169 , 0.19353719, + 0.73621596, 0.13968272]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.04190799, 0.83779526], + [ 0.5736724 , 0.44880328], + [ 0.62498575, 0.42920056]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 0.72341901308525713 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) def test_model_regi_error(self): - # Columbus: - reg = SP.GM_Error_Het_Regimes( - self.y, self.X, self.regimes, self.w, regime_err_sep=True - ) - betas = np.array( - [ - [60.74090229], - [-0.17492294], - [-1.33383387], - [0.68303064], - [66.30374279], - [-0.31841139], - [-1.27502813], - [0.11515312], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array([44.9411672, -0.34343354, -0.39946055, 0.0, 0.0, 0.0, 0.0, 0.0]) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + #Columbus: + reg = SP.GM_Error_Het_Regimes(self.y, self.X, self.regimes, self.w, regime_err_sep=True) + betas = np.array([[ 60.74090229], + [ -0.17492294], + [ -1.33383387], + [ 0.68303064], + [ 66.30374279], + [ -0.31841139], + [ -1.27502813], + [ 0.11515312]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 44.9411672 , -0.34343354, -0.39946055, 0. , + 0. , 0. , 0. , 0. ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) u = np.array([-0.05357818]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([15.77955818]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([0.70542044]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [3.11061225e-01, 5.77029704e-01], - [3.39747489e-01, 5.59975012e-01], - [3.86371771e-03, 9.50436364e-01], - [4.02884201e00, 4.47286322e-02], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 15.77955818]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([ 0.70542044]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[ 3.11061225e-01, 5.77029704e-01], + [ 3.39747489e-01, 5.59975012e-01], + [ 3.86371771e-03, 9.50436364e-01], + [ 4.02884201e+00, 4.47286322e-02]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 4.7467070503995412 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Error_Het_Regimes( - self.y_a, self.x_a, self.regi_a, w=self.w_a, regime_err_sep=True - ) - model1 = GM_Error_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Error_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a[int(self.n2) :], - w=self.w_a1, - ) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) + #Artficial: + model = SP.GM_Error_Het_Regimes(self.y_a, self.x_a, self.regi_a, w=self.w_a, regime_err_sep=True) + model1 = GM_Error_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Error_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) + np.testing.assert_allclose(model.betas,tbetas, RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) np.testing.assert_allclose(model.vm.diagonal(), vm, RTOL) def test_model_endog(self): - reg = SP.GM_Endog_Error_Het_Regimes( - self.y, self.X2, self.yd, self.q, self.regimes, self.w - ) - betas = np.array( - [ - [77.26679984], - [4.45992905], - [78.59534391], - [0.41432319], - [-3.20196286], - [-1.13672283], - [0.2174965], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([20.50716917]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - e = np.array([25.13517175]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) + reg = SP.GM_Endog_Error_Het_Regimes(self.y, self.X2, self.yd, self.q, self.regimes, self.w) + betas = np.array([[ 77.26679984], + [ 4.45992905], + [ 78.59534391], + [ 0.41432319], + [ -3.20196286], + [ -1.13672283], + [ 0.2174965 ]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 20.50716917]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + e = np.array([ 25.13517175]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) predy = np.array([-4.78118917]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n, RTOL) k = 6 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 1.0, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - yend = np.array([[0.0, 80.467003]]) - np.testing.assert_allclose(reg.yend[0].toarray(), yend, RTOL) - z = np.array([[0.0, 0.0, 1.0, 19.531, 0.0, 80.467003]]) - np.testing.assert_allclose(reg.z[0].toarray(), z, RTOL) + np.testing.assert_allclose(reg.k,k, RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 1. , 19.531]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + yend = np.array([[ 0. , 80.467003]]) + np.testing.assert_allclose(reg.yend[0].toarray(),yend,RTOL) + z = np.array([[ 0. , 0. , 1. , 19.531 , 0. , + 80.467003]]) + np.testing.assert_allclose(reg.z[0].toarray(),z,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 509.66122149, - 150.5845341, - 9.64413821, - 5.54782831, - -80.95846045, - -2.25308524, - -3.2045214, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 509.66122149, 150.5845341 , 9.64413821, 5.54782831, + -80.95846045, -2.25308524, -3.2045214 ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) pr2 = 0.19776512679331681 - np.testing.assert_allclose(reg.pr2, pr2) - std_err = np.array( - [ - 22.57567765, - 11.34616946, - 17.43881791, - 1.30953812, - 5.4830829, - 0.74634612, - 0.29973079, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [[0.0022216, 0.96240654], [0.13127347, 0.7171153], [0.14367307, 0.70465645]] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2,pr2) + std_err = np.array([ 22.57567765, 11.34616946, 17.43881791, 1.30953812, + 5.4830829 , 0.74634612, 0.29973079]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.0022216 , 0.96240654], + [ 0.13127347, 0.7171153 ], + [ 0.14367307, 0.70465645]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 1.2329971019087163 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) def test_model_endog_regi_error(self): - # Columbus: - reg = SP.GM_Endog_Error_Het_Regimes( - self.y, self.X2, self.yd, self.q, self.regimes, self.w, regime_err_sep=True - ) - betas = np.array( - [ - [7.92747424e01], - [5.78086230e00], - [-3.83173581e00], - [2.23210962e-01], - [8.26255251e01], - [5.48294187e-01], - [-1.28432891e00], - [3.57661629e-02], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array( - [ - 7.55988579e02, - 2.53659722e02, - -1.34288316e02, - -2.66141766e-01, - 0.00000000e00, - 0.00000000e00, - 0.00000000e00, - 0.00000000e00, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) - u = np.array([25.73781918]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([-10.01183918]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([26.5449135]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [0.00998573, 0.92040097], - [0.12660165, 0.72198192], - [0.12737281, 0.72117171], - [0.43507956, 0.50950696], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) - chow_j = 1.3756768204399892 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Endog_Error_Het_Regimes( - self.y_a, - self.x_a1, - yend=self.x_a2, - q=self.q_a, - regimes=self.regi_a, - w=self.w_a, - regime_err_sep=True, - ) - model1 = GM_Endog_Error_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a1[0 : int(self.n2)], - yend=self.x_a2[0 : int(self.n2)], - q=self.q_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Endog_Error_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a1[int(self.n2) :], - yend=self.x_a2[int(self.n2) :], - q=self.q_a[int(self.n2) :], - w=self.w_a1, - ) + #Columbus: + reg = SP.GM_Endog_Error_Het_Regimes(self.y, self.X2, self.yd, self.q, self.regimes, self.w, regime_err_sep=True) + betas = np.array([[70.45311952], + [ 4.3685181 ], + [-0.9 ], + [ 0.89261347], + [78.43187245], + [ 0.95437565], + [-0.9 ], + [ 0.73629337]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 1.210078e+03, 1.559293e+02, -7.540641e+01, 1.141481e+00, + 0.000000e+00, 0.000000e+00, 0.000000e+00, 0.000000e+00]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) + u = np.array([-8.925501]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([24.651481]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([20.195355]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[0.041109, 0.839328], + [0.102719, 0.748591], + [0. , 1. ], + [0.311214, 0.576936]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) + chow_j = 2.621305 + np.testing.assert_allclose(reg.chow.joint[0],chow_j, RTOL) + #Artficial: + model = SP.GM_Endog_Error_Het_Regimes(self.y_a, self.x_a1, yend=self.x_a2, q=self.q_a, regimes=self.regi_a, w=self.w_a, regime_err_sep=True) + model1 = GM_Endog_Error_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a1[0:int(self.n2)], yend=self.x_a2[0:int(self.n2)], q=self.q_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Endog_Error_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a1[int(self.n2):], yend=self.x_a2[int(self.n2):], q=self.q_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) + np.testing.assert_allclose(model.betas,tbetas, RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) np.testing.assert_allclose(model.vm.diagonal(), vm, RTOL) def test_model_combo(self): - reg = SP.GM_Combo_Het_Regimes( - self.y, self.X2, self.regimes, self.yd, self.q, w=self.w - ) - betas = np.array( - [ - [3.69372678e01], - [-8.29474998e-01], - [3.08667517e01], - [-7.23753444e-01], - [-3.01900940e-01], - [-2.21328949e-01], - [6.41902155e-01], - [-2.45714919e-02], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([0.94039246]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - e_filtered = np.array([0.8737864]) - np.testing.assert_allclose(reg.e_filtered[0], e_filtered, RTOL) - predy_e = np.array([18.68732105]) - np.testing.assert_allclose(reg.predy_e[0], predy_e, RTOL) - predy = np.array([14.78558754]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = SP.GM_Combo_Het_Regimes(self.y, self.X2, self.regimes, self.yd, self.q, w=self.w) + betas = np.array([[ 3.69372678e+01], + [ -8.29474998e-01], + [ 3.08667517e+01], + [ -7.23753444e-01], + [ -3.01900940e-01], + [ -2.21328949e-01], + [ 6.41902155e-01], + [ -2.45714919e-02]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 0.94039246]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + e_filtered = np.array([ 0.8737864]) + np.testing.assert_allclose(reg.e_filtered[0],e_filtered,RTOL) + predy_e = np.array([ 18.68732105]) + np.testing.assert_allclose(reg.predy_e[0],predy_e,RTOL) + predy = np.array([ 14.78558754]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n, RTOL) + np.testing.assert_allclose(reg.n,n, RTOL) k = 7 - np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([15.72598]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([[0.0, 0.0, 1.0, 19.531]]) - np.testing.assert_allclose(reg.x[0].toarray(), x, RTOL) - yend = np.array([[0.0, 80.467003, 24.7142675]]) - np.testing.assert_allclose(reg.yend[0].toarray(), yend, RTOL) - z = np.array([[0.0, 0.0, 1.0, 19.531, 0.0, 80.467003, 24.7142675]]) - np.testing.assert_allclose(reg.z[0].toarray(), z, RTOL) + np.testing.assert_allclose(reg.k,k, RTOL) + y = np.array([ 15.72598]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([[ 0. , 0. , 1. , 19.531]]) + np.testing.assert_allclose(reg.x[0].toarray(),x,RTOL) + yend = np.array([[ 0. , 80.467003 , 24.7142675]]) + np.testing.assert_allclose(reg.yend[0].toarray(),yend,RTOL) + z = np.array([[ 0. , 0. , 1. , 19.531 , 0. , + 80.467003 , 24.7142675]]) + np.testing.assert_allclose(reg.z[0].toarray(),z,RTOL) my = 35.128823897959187 - np.testing.assert_allclose(reg.mean_y, my, RTOL) + np.testing.assert_allclose(reg.mean_y,my, RTOL) sy = 16.732092091229699 - np.testing.assert_allclose(reg.std_y, sy, RTOL) - vm = np.array( - [ - 71.26851365, - -0.58278032, - 50.53169815, - -0.74561147, - -0.79510274, - -0.10823496, - -0.98141395, - 1.16575965, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) + np.testing.assert_allclose(reg.std_y,sy, RTOL) + vm = np.array([ 71.26851365, -0.58278032, 50.53169815, -0.74561147, + -0.79510274, -0.10823496, -0.98141395, 1.16575965]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) pr2 = 0.6504148883602958 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.527136896994038 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [ - 8.44206809, - 0.72363219, - 9.85790968, - 0.77218082, - 0.34084146, - 0.21752916, - 0.14371614, - 0.39226478, - ] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - chow_r = np.array( - [ - [0.54688708, 0.45959243], - [0.01035136, 0.91896175], - [0.03981108, 0.84185042], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 8.44206809, 0.72363219, 9.85790968, 0.77218082, 0.34084146, + 0.21752916, 0.14371614, 0.39226478]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + chow_r = np.array([[ 0.54688708, 0.45959243], + [ 0.01035136, 0.91896175], + [ 0.03981108, 0.84185042]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 0.78070369988354349 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) + np.testing.assert_allclose(reg.chow.joint[0],chow_j,RTOL) def test_model_combo_regi_error(self): - # Columbus: - reg = SP.GM_Combo_Het_Regimes( - self.y, - self.X2, - self.regimes, - self.yd, - self.q, - w=self.w, - regime_lag_sep=True, - regime_err_sep=True, - ) - betas = np.array( - [ - [42.01151458], - [-0.13917151], - [-0.65300184], - [0.54737064], - [0.2629229], - [34.21569751], - [-0.15236089], - [-0.49175217], - [0.65733173], - [-0.07713581], - ] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - vm = np.array( - [ - 77.49519689, - 0.57226879, - -1.18856422, - -1.28088712, - 0.866752, - 0.0, - 0.0, - 0.0, - 0.0, - 0.0, - ] - ) - np.testing.assert_allclose(reg.vm[0], vm, RTOL) - u = np.array([7.81039418]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([7.91558582]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) - e = np.array([7.22996911]) - np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - chow_r = np.array( - [ - [1.90869079e-01, 6.62194273e-01], - [4.56118982e-05, 9.94611401e-01], - [3.12104263e-02, 8.59771748e-01], - [1.56368204e-01, 6.92522476e-01], - [7.52928732e-01, 3.85550558e-01], - ] - ) - np.testing.assert_allclose(reg.chow.regi, chow_r, RTOL) + #Columbus: + reg = SP.GM_Combo_Het_Regimes(self.y, self.X2, self.regimes, self.yd, self.q, w=self.w, regime_lag_sep=True, regime_err_sep=True) + betas = np.array([[ 42.01151458], + [ -0.13917151], + [ -0.65300184], + [ 0.54737064], + [ 0.2629229 ], + [ 34.21569751], + [ -0.15236089], + [ -0.49175217], + [ 0.65733173], + [ -0.07713581]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + vm = np.array([ 77.49519689, 0.57226879, -1.18856422, -1.28088712, + 0.866752 , 0. , 0. , 0. , + 0. , 0. ]) + np.testing.assert_allclose(reg.vm[0],vm,RTOL) + u = np.array([ 7.81039418]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 7.91558582]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) + e = np.array([ 7.22996911]) + np.testing.assert_allclose(reg.e_filtered[0],e,RTOL) + chow_r = np.array([[ 1.90869079e-01, 6.62194273e-01], + [ 4.56118982e-05, 9.94611401e-01], + [ 3.12104263e-02, 8.59771748e-01], + [ 1.56368204e-01, 6.92522476e-01], + [ 7.52928732e-01, 3.85550558e-01]]) + np.testing.assert_allclose(reg.chow.regi,chow_r,RTOL) chow_j = 1.1316136604755913 - np.testing.assert_allclose(reg.chow.joint[0], chow_j, RTOL) - # Artficial: - model = SP.GM_Combo_Het_Regimes( - self.y_a, - self.x_a1, - yend=self.x_a2, - q=self.q_a, - regimes=self.regi_a, - w=self.w_a, - regime_err_sep=True, - regime_lag_sep=True, - ) - model1 = GM_Combo_Het( - self.y_a[0 : int(self.n2)].reshape(int(self.n2), 1), - self.x_a1[0 : int(self.n2)], - yend=self.x_a2[0 : int(self.n2)], - q=self.q_a[0 : int(self.n2)], - w=self.w_a1, - ) - model2 = GM_Combo_Het( - self.y_a[int(self.n2) :].reshape(int(self.n2), 1), - self.x_a1[int(self.n2) :], - yend=self.x_a2[int(self.n2) :], - q=self.q_a[int(self.n2) :], - w=self.w_a1, - ) + np.testing.assert_allclose(reg.chow.joint[0],chow_j,RTOL) + #Artficial: + model = SP.GM_Combo_Het_Regimes(self.y_a, self.x_a1, yend=self.x_a2, q=self.q_a, regimes=self.regi_a, w=self.w_a, regime_err_sep=True, regime_lag_sep=True) + model1 = GM_Combo_Het(self.y_a[0:int(self.n2)].reshape(int(self.n2),1), self.x_a1[0:int(self.n2)], yend=self.x_a2[0:int(self.n2)], q=self.q_a[0:int(self.n2)], w=self.w_a1) + model2 = GM_Combo_Het(self.y_a[int(self.n2):].reshape(int(self.n2),1), self.x_a1[int(self.n2):], yend=self.x_a2[int(self.n2):], q=self.q_a[int(self.n2):], w=self.w_a1) tbetas = np.vstack((model1.betas, model2.betas)) - np.testing.assert_allclose(model.betas, tbetas, RTOL) - vm = np.hstack((model1.vm.diagonal(), model2.vm.diagonal())) - # was this supposed to get tested? - + np.testing.assert_allclose(model.betas,tbetas,RTOL) + vm = np.hstack((model1.vm.diagonal(),model2.vm.diagonal())) + #was this supposed to get tested? -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_error_sp_het_sparse.py b/spreg/tests/test_error_sp_het_sparse.py index 16b30328..2890d454 100644 --- a/spreg/tests/test_error_sp_het_sparse.py +++ b/spreg/tests/test_error_sp_het_sparse.py @@ -6,150 +6,131 @@ from libpysal.common import RTOL import spreg - class TestBaseGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.BaseGM_Error_Het(self.y, self.X, self.w.sparse, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) + class TestGMErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): reg = HET.GM_Error_Het(self.y, self.X, self.w, step1c=True) - betas = np.array([[47.99626638], [0.71048989], [-0.55876126], [0.41178776]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([27.38122697]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([32.29765975]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.08577603]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 47.99626638], [ 0.71048989], [ -0.55876126], [ 0.41178776]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 27.38122697]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 32.29765975]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.08577603]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [1.31767529e02, -3.58368748e00, -1.65090647e00, 0.00000000e00], - [-3.58368748e00, 1.35513711e-01, 3.77539055e-02, 0.00000000e00], - [-1.65090647e00, 3.77539055e-02, 2.61042702e-02, 0.00000000e00], - [0.00000000e00, 0.00000000e00, 0.00000000e00, 2.82398517e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 1.31767529e+02, -3.58368748e+00, -1.65090647e+00, + 0.00000000e+00], + [ -3.58368748e+00, 1.35513711e-01, 3.77539055e-02, + 0.00000000e+00], + [ -1.65090647e+00, 3.77539055e-02, 2.61042702e-02, + 0.00000000e+00], + [ 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, + 2.82398517e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34951013222581306 - np.testing.assert_allclose(reg.pr2, pr2) - stde = np.array([11.47900385, 0.36812187, 0.16156816, 0.16804717]) - np.testing.assert_allclose(reg.std_err, stde, RTOL) - z_stat = np.array( - [ - [4.18122226e00, 2.89946274e-05], - [1.93003988e00, 5.36018970e-02], - [-3.45836247e00, 5.43469673e-04], - [2.45042960e00, 1.42685863e-02], - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_allclose(reg.xtx, xtx, RTOL) - + np.testing.assert_allclose(reg.pr2,pr2) + stde = np.array([ 11.47900385, 0.36812187, 0.16156816, 0.16804717]) + np.testing.assert_allclose(reg.std_err,stde,RTOL) + z_stat = np.array([[ 4.18122226e+00, 2.89946274e-05], + [ 1.93003988e+00, 5.36018970e-02], + [ -3.45836247e+00, 5.43469673e-04], + [ 2.45042960e+00, 1.42685863e-02]]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_allclose(reg.xtx,xtx,RTOL) class TestBaseGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) yd = [] yd.append(db.by_col("CRIME")) @@ -157,71 +138,62 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.BaseGM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.46604707]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w.sparse, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.46604707]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0].toarray()[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0].toarray()[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestGMEndogErrorHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -232,366 +204,213 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HET.GM_Endog_Error_Het( - self.y, self.X, self.yd, self.q, self.w, step1c=True - ) - betas = np.array([[55.39707924], [0.46563046], [-0.67038326], [0.41135023]]) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([26.51812895]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([53.94887405]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.GM_Endog_Error_Het(self.y, self.X, self.yd, self.q, self.w, step1c=True) + betas = np.array([[ 55.39707924], [ 0.46563046], [ -0.67038326], [ 0.41135023]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 26.51812895]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + predy = np.array([ 53.94887405]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 3 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([15.72598]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([5.03]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_allclose(reg.h[0].toarray()[0], h, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 15.72598]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 5.03]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_allclose(reg.h[0].toarray()[0],h,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [8.34637805e02, -2.16932259e01, -1.33327894e01, 1.65840848e00], - [-2.16932259e01, 5.97683070e-01, 3.39503523e-01, -3.90111107e-02], - [-1.33327894e01, 3.39503523e-01, 2.19008080e-01, -2.81929695e-02], - [1.65840848e00, -3.90111107e-02, -2.81929695e-02, 3.15686105e-02], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 8.34637805e+02, -2.16932259e+01, -1.33327894e+01, + 1.65840848e+00], + [ -2.16932259e+01, 5.97683070e-01, 3.39503523e-01, + -3.90111107e-02], + [ -1.33327894e+01, 3.39503523e-01, 2.19008080e-01, + -2.81929695e-02], + [ 1.65840848e+00, -3.90111107e-02, -2.81929695e-02, + 3.15686105e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL) pr2 = 0.34648011338954804 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) - std_err = np.array([28.89009873, 0.77309965, 0.46798299, 0.17767558]) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (1.9175109006819244, 0.055173057472126787), - (0.60229035155742305, 0.54698088217644414), - (-1.4324949211864271, 0.15200223057569454), - (2.3151759776869496, 0.020603303355572443), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - - + np.testing.assert_allclose(reg.pr2,pr2,RTOL) + std_err = np.array([ 28.89009873, 0.77309965, 0.46798299, + 0.17767558]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(1.9175109006819244, 0.055173057472126787), (0.60229035155742305, 0.54698088217644414), (-1.4324949211864271, 0.15200223057569454), (2.3151759776869496, 0.020603303355572443)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 49. , 704.371999 , 139.75 ], + [ 704.371999 , 11686.67338121, 2246.12800625], + [ 139.75 , 2246.12800625, 498.5851 ]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) + class TestBaseGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - reg = HET.BaseGM_Combo_Het( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True - ) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + reg = HET.BaseGM_Combo_Het(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, step1c=True) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy, RTOL) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.std_y,stdy,RTOL) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) class TestGMComboHet(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): # Only spatial lag reg = HET.GM_Combo_Het(self.y, self.X, w=self.w, step1c=True) - betas = np.array( - [[57.7778574], [0.73034922], [-0.59257362], [-0.2230231], [0.56636724]] - ) - np.testing.assert_allclose(reg.betas, betas, RTOL) - u = np.array([25.65156033]) - np.testing.assert_allclose(reg.u[0], u, RTOL) - ef = np.array([31.87664403]) - np.testing.assert_allclose(reg.e_filtered[0], ef, RTOL) - ep = np.array([28.30648145]) - np.testing.assert_allclose(reg.e_pred[0], ep, RTOL) - pe = np.array([52.16052155]) - np.testing.assert_allclose(reg.predy_e[0], pe, RTOL) - predy = np.array([54.81544267]) - np.testing.assert_allclose(reg.predy[0], predy, RTOL) + betas = np.array([[ 57.7778574 ], [ 0.73034922], [ -0.59257362], [ -0.2230231 ], [ 0.56636724]]) + np.testing.assert_allclose(reg.betas,betas,RTOL) + u = np.array([ 25.65156033]) + np.testing.assert_allclose(reg.u[0],u,RTOL) + ef = np.array([ 31.87664403]) + np.testing.assert_allclose(reg.e_filtered[0],ef,RTOL) + ep = np.array([ 28.30648145]) + np.testing.assert_allclose(reg.e_pred[0],ep,RTOL) + pe = np.array([ 52.16052155]) + np.testing.assert_allclose(reg.predy_e[0],pe,RTOL) + predy = np.array([ 54.81544267]) + np.testing.assert_allclose(reg.predy[0],predy,RTOL) n = 49 - np.testing.assert_allclose(reg.n, n) + np.testing.assert_allclose(reg.n,n) k = 4 - np.testing.assert_allclose(reg.k, k) - y = np.array([80.467003]) - np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_allclose(reg.x[0].toarray()[0], x, RTOL) - yend = np.array([35.4585005]) - np.testing.assert_allclose(reg.yend[0], yend, RTOL) - q = np.array([18.594, 24.7142675]) - np.testing.assert_allclose(reg.q[0].toarray()[0], q, RTOL) - z = np.array([1.0, 19.531, 15.72598, 35.4585005]) - np.testing.assert_allclose(reg.z[0].toarray()[0], z, RTOL) - i_s = "Maximum number of iterations reached." - np.testing.assert_string_equal(reg.iter_stop, i_s) + np.testing.assert_allclose(reg.k,k) + y = np.array([ 80.467003]) + np.testing.assert_allclose(reg.y[0],y,RTOL) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_allclose(reg.x[0].toarray()[0],x,RTOL) + yend = np.array([ 35.4585005]) + np.testing.assert_allclose(reg.yend[0],yend,RTOL) + q = np.array([ 18.594 , 24.7142675]) + np.testing.assert_allclose(reg.q[0].toarray()[0],q,RTOL) + z = np.array([ 1. , 19.531 , 15.72598 , 35.4585005]) + np.testing.assert_allclose(reg.z[0].toarray()[0],z,RTOL) + i_s = 'Maximum number of iterations reached.' + np.testing.assert_string_equal(reg.iter_stop,i_s) its = 1 - np.testing.assert_allclose(reg.iteration, its, RTOL) + np.testing.assert_allclose(reg.iteration,its,RTOL) my = 38.436224469387746 - np.testing.assert_allclose(reg.mean_y, my) + np.testing.assert_allclose(reg.mean_y,my) stdy = 18.466069465206047 - np.testing.assert_allclose(reg.std_y, stdy) - vm = np.array( - [ - [ - 4.86218274e02, - -2.77268729e00, - -1.59987770e00, - -1.01969471e01, - 2.74302006e00, - ], - [ - -2.77268729e00, - 1.04680972e-01, - 2.51172238e-02, - 1.95136385e-03, - 3.70052723e-03, - ], - [ - -1.59987770e00, - 2.51172238e-02, - 2.15655720e-02, - 7.65868344e-03, - -7.30173070e-03, - ], - [ - -1.01969471e01, - 1.95136385e-03, - 7.65868344e-03, - 2.78273684e-01, - -6.89402590e-02, - ], - [ - 2.74302006e00, - 3.70052723e-03, - -7.30173070e-03, - -6.89402590e-02, - 7.12034037e-02, - ], - ] - ) - np.testing.assert_allclose(reg.vm, vm, RTOL) + np.testing.assert_allclose(reg.std_y,stdy) + vm = np.array([[ 4.86218274e+02, -2.77268729e+00, -1.59987770e+00, + -1.01969471e+01, 2.74302006e+00], + [ -2.77268729e+00, 1.04680972e-01, 2.51172238e-02, + 1.95136385e-03, 3.70052723e-03], + [ -1.59987770e+00, 2.51172238e-02, 2.15655720e-02, + 7.65868344e-03, -7.30173070e-03], + [ -1.01969471e+01, 1.95136385e-03, 7.65868344e-03, + 2.78273684e-01, -6.89402590e-02], + [ 2.74302006e+00, 3.70052723e-03, -7.30173070e-03, + -6.89402590e-02, 7.12034037e-02]]) + np.testing.assert_allclose(reg.vm,vm,RTOL*10) pr2 = 0.3001582877472412 - np.testing.assert_allclose(reg.pr2, pr2, RTOL) + np.testing.assert_allclose(reg.pr2,pr2,RTOL) pr2_e = 0.35613102283621967 - np.testing.assert_allclose(reg.pr2_e, pr2_e, RTOL) - std_err = np.array( - [22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966] - ) - np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = np.array( - [ - (2.6202684885795335, 0.00878605635338265), - (2.2573385444145524, 0.023986928627746887), - (-4.0351698589183433, 5.456281036278686e-05), - (-0.42277935292121521, 0.67245625315942159), - (2.1225002455741895, 0.033795752094112265), - ] - ) - np.testing.assert_allclose(reg.z_stat, z_stat, RTOL) - hth = np.array( - [ - [ - 4.90000000e01, - 7.04371999e02, - 1.72131237e03, - 7.24743592e02, - 1.70735413e03, - ], - [ - 7.04371999e02, - 1.16866734e04, - 2.15575320e04, - 1.10925200e04, - 2.23848036e04, - ], - [ - 1.72131237e03, - 2.15575320e04, - 7.39058986e04, - 2.34796298e04, - 6.70145378e04, - ], - [ - 7.24743592e02, - 1.10925200e04, - 2.34796298e04, - 1.16146226e04, - 2.30304624e04, - ], - [ - 1.70735413e03, - 2.23848036e04, - 6.70145378e04, - 2.30304624e04, - 6.69879858e04, - ], - ] - ) - np.testing.assert_allclose(reg.hth, hth, RTOL) - + np.testing.assert_allclose(reg.pr2_e,pr2_e,RTOL) + std_err = np.array([ 22.05035768, 0.32354439, 0.14685221, 0.52751653, 0.26683966]) + np.testing.assert_allclose(reg.std_err,std_err,RTOL) + z_stat = np.array([(2.6202684885795335, 0.00878605635338265), (2.2573385444145524, 0.023986928627746887), (-4.0351698589183433, 5.456281036278686e-05), (-0.42277935292121521, 0.67245625315942159), (2.1225002455741895, 0.033795752094112265)]) + np.testing.assert_allclose(reg.z_stat,z_stat,RTOL) + hth = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03, + 7.24743592e+02, 1.70735413e+03], + [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04, + 1.10925200e+04, 2.23848036e+04], + [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04, + 2.34796298e+04, 6.70145378e+04], + [ 7.24743592e+02, 1.10925200e+04, 2.34796298e+04, + 1.16146226e+04, 2.30304624e+04], + [ 1.70735413e+03, 2.23848036e+04, 6.70145378e+04, + 2.30304624e+04, 6.69879858e+04]]) + np.testing.assert_allclose(reg.hth,hth,RTOL) -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/spreg/tests/test_error_sp_hom_sparse.py b/spreg/tests/test_error_sp_hom_sparse.py index 8c102661..09cb42c8 100644 --- a/spreg/tests/test_error_sp_hom_sparse.py +++ b/spreg/tests/test_error_sp_hom_sparse.py @@ -1,7 +1,7 @@ -""" +''' Unittests for spreg.error_sp_hom module -""" +''' import unittest import libpysal from spreg import error_sp_hom as HOM @@ -9,139 +9,94 @@ import numpy as np import spreg - class BaseGM_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.BaseGM_Error_Hom(self.y, self.X, self.w.sparse, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[47.9478524], [0.70633223], [-0.55595633], [0.41288558]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([27.466734]), 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([32.37298547]), 7 - ) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([53.000269]), 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) + reg = HOM.BaseGM_Error_Hom(self.y, self.X, self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 47.9478524 ], [ 0.70633223], [ -0.55595633], [ 0.41288558]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([27.466734]),6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 32.37298547]),5) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 53.000269]),6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) sig2 = 189.94459439729718 - self.assertAlmostEqual(reg.sig2, sig2) - vm = np.array( - [ - [1.51340717e02, -5.29057506e00, -1.85654540e00, -2.39139054e-03], - [-5.29057506e00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], - [-1.85654540e00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], - [-2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_array_almost_equal(reg.xtx, xtx, 4) - + self.assertAlmostEqual(reg.sig2,sig2,5) + vm = np.array([[ 1.51340717e+02, -5.29057506e+00, -1.85654540e+00, -2.39139054e-03], [ -5.29057506e+00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], [ -1.85654540e+00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], [ -2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_array_almost_equal(reg.xtx,xtx,4) class GM_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) X.append(db.by_col("CRIME")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Error_Hom(self.y, self.X, self.w, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[47.9478524], [0.70633223], [-0.55595633], [0.41288558]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([27.46673388]), 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([32.37298547]), 7 - ) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([53.00026912]), 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [1.51340717e02, -5.29057506e00, -1.85654540e00, -2.39139054e-03], - [-5.29057506e00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], - [-1.85654540e00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], - [-2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - self.assertAlmostEqual(reg.iteration, 1, 7) + reg = HOM.GM_Error_Hom(self.y, self.X, self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 47.9478524 ], [ 0.70633223], [ -0.55595633], [ 0.41288558]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([27.46673388]),6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 32.37298547]),5) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 53.00026912]),6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 1.51340717e+02, -5.29057506e+00, -1.85654540e+00, -2.39139054e-03], [ -5.29057506e+00, 2.46669610e-01, 5.14259101e-02, 3.19241302e-04], [ -1.85654540e+00, 5.14259101e-02, 3.20510550e-02, -5.95640240e-05], [ -2.39139054e-03, 3.19241302e-04, -5.95640240e-05, 3.36690159e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + self.assertAlmostEqual(reg.iteration,1,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.34950977055969729 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) sig2 = 189.94459439729718 - self.assertAlmostEqual(reg.sig2, sig2) - std_err = np.array([12.30206149, 0.49665844, 0.17902808, 0.18349119]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [3.89754616e00, 9.71723059e-05], - [1.42216900e00, 1.54977196e-01], - [-3.10541409e00, 1.90012806e-03], - [2.25016500e00, 2.44384731e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) - xtx = np.array( - [ - [4.90000000e01, 7.04371999e02, 1.72131237e03], - [7.04371999e02, 1.16866734e04, 2.15575320e04], - [1.72131237e03, 2.15575320e04, 7.39058986e04], - ] - ) - np.testing.assert_array_almost_equal(reg.xtx, xtx, 4) + self.assertAlmostEqual(reg.sig2,sig2,5) + std_err = np.array([ 12.30206149, 0.49665844, 0.17902808, 0.18349119]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,6) + z_stat = np.array([[ 3.89754616e+00, 9.71723059e-05], [ 1.42216900e+00, 1.54977196e-01], [ -3.10541409e+00, 1.90012806e-03], [ 2.25016500e+00, 2.44384731e-02]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) + xtx = np.array([[ 4.90000000e+01, 7.04371999e+02, 1.72131237e+03], [ 7.04371999e+02, 1.16866734e+04, 2.15575320e+04], [ 1.72131237e+03, 2.15575320e+04, 7.39058986e+04]]) + np.testing.assert_array_almost_equal(reg.xtx,xtx,4) class BaseGM_Endog_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) yd = [] yd.append(db.by_col("CRIME")) @@ -149,73 +104,52 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.BaseGM_Endog_Error_Hom( - self.y, self.X, self.yd, self.q, self.w.sparse, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([15.72598]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([5.03]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - betas = np.array([[55.36575166], [0.46432416], [-0.66904404], [0.43205526]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 6) - u = np.array([26.55390939]) - np.testing.assert_array_almost_equal(reg.u[0], u, 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([31.74114306]), 7 - ) - predy = np.array([53.91309361]) - np.testing.assert_array_almost_equal(reg.predy[0], predy, 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) + reg = HOM.BaseGM_Endog_Error_Hom(self.y, self.X, self.yd, self.q, self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([ 80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 15.72598]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 5.03]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + betas = np.array([[ 55.36575166], [ 0.46432416], [ -0.66904404], [ 0.43205526]]) + np.testing.assert_array_almost_equal(reg.betas,betas,6) + u = np.array([ 26.55390939]) + np.testing.assert_array_almost_equal(reg.u[0],u,6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 31.74114306]),5) + predy = np.array([ 53.91309361]) + np.testing.assert_array_almost_equal(reg.predy[0],predy,6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) sig2 = 190.59435238060928 - self.assertAlmostEqual(reg.sig2, sig2) - vm = np.array( - [ - [5.52064057e02, -1.61264555e01, -8.86360735e00, 1.04251912e00], - [-1.61264555e01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], - [-8.86360735e00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], - [1.04251912e00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + self.assertAlmostEqual(reg.sig2,sig2,4) + vm = np.array([[ 5.52064057e+02, -1.61264555e+01, -8.86360735e+00, 1.04251912e+00], [ -1.61264555e+01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], [ -8.86360735e+00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], [ 1.04251912e+00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,4) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) sig2 = 0 - # self.assertAlmostEqual(reg.sig2,sig2) - hth = np.array( - [ - [49.0, 704.371999, 139.75], - [704.371999, 11686.67338121, 2246.12800625], - [139.75, 2246.12800625, 498.5851], - ] - ) - np.testing.assert_array_almost_equal(reg.hth, hth, 4) - + #self.assertAlmostEqual(reg.sig2,sig2) + hth = np.array([[ 49. , 704.371999 , 139.75 ], [ 704.371999 , 11686.67338121, 2246.12800625], [ 139.75 , 2246.12800625, 498.5851]]) + np.testing.assert_array_almost_equal(reg.hth,hth,4) class GM_Endog_Error_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T @@ -226,227 +160,162 @@ def setUp(self): q = [] q.append(db.by_col("DISCBD")) self.q = np.array(q).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Endog_Error_Hom( - self.y, self.X, self.yd, self.q, self.w, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - z = np.array([1.0, 19.531, 15.72598]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 5.03]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([15.72598]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([5.03]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - betas = np.array([[55.36575166], [0.46432416], [-0.66904404], [0.43205526]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 6) - u = np.array([26.55390939]) - np.testing.assert_array_almost_equal(reg.u[0], u, 6) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([31.74114306]), 7 - ) - predy = np.array([53.91309361]) - np.testing.assert_array_almost_equal(reg.predy[0], predy, 6) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [5.52064057e02, -1.61264555e01, -8.86360735e00, 1.04251912e00], - [-1.61264555e01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], - [-8.86360735e00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], - [1.04251912e00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + reg = HOM.GM_Endog_Error_Hom(self.y, self.X, self.yd, self.q, self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([ 80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + z = np.array([ 1. , 19.531 , 15.72598]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 5.03 ]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 15.72598]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 5.03]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + betas = np.array([[ 55.36575166], [ 0.46432416], [ -0.66904404], [ 0.43205526]]) + np.testing.assert_array_almost_equal(reg.betas,betas,6) + u = np.array([ 26.55390939]) + np.testing.assert_array_almost_equal(reg.u[0],u,6) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 31.74114]),5) + predy = np.array([ 53.91309361]) + np.testing.assert_array_almost_equal(reg.predy[0],predy,6) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 5.52064057e+02, -1.61264555e+01, -8.86360735e+00, 1.04251912e+00], [ -1.61264555e+01, 5.44898242e-01, 2.39518645e-01, -1.88092950e-02], [ -8.86360735e+00, 2.39518645e-01, 1.55501840e-01, -2.18638648e-02], [ 1.04251912e+00, -1.88092950e-02, -2.18638648e-02, 3.71222222e-02]]) + np.testing.assert_array_almost_equal(reg.vm,vm,4) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.34647366525657419 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) sig2 = 190.59435238060928 - self.assertAlmostEqual(reg.sig2, sig2) - # std_err - std_err = np.array([23.49604343, 0.73817223, 0.39433722, 0.19267128]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [2.35638617, 0.01845372], - [0.62901874, 0.52933679], - [-1.69662923, 0.08976678], - [2.24244556, 0.02493259], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) + self.assertAlmostEqual(reg.sig2,sig2,4) + #std_err + std_err = np.array([ 23.49604343, 0.73817223, 0.39433722, 0.19267128]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,4) + z_stat = np.array([[ 2.35638617, 0.01845372], [ 0.62901874, 0.52933679], [-1.69662923, 0.08976678], [ 2.24244556, 0.02493259]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) class BaseGM_Combo_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): yd2, q2 = spreg.utils.set_endog(self.y, self.X, self.w, None, None, 1, True) - self.X = np.hstack((np.ones(self.y.shape), self.X)) + self.X = np.hstack((np.ones(self.y.shape),self.X)) self.X = sparse.csr_matrix(self.X) - reg = HOM.BaseGM_Combo_Hom( - self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, A1="hom_sc" - ) - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[10.12541428], [1.56832263], [0.15132076], [0.21033397]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([34.3450723]), 7) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([36.6149682]), 7 - ) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([46.1219307]), 7) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - vm = np.array( - [ - [2.33694742e02, -6.66856869e-01, -5.58304254e00, 4.85488380e00], - [-6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], - [-5.58304254e00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], - [4.85488380e00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - z = np.array([1.0, 19.531, 35.4585005]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 18.594]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([35.4585005]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([18.594]) - np.testing.assert_array_almost_equal(reg.q[0], q, 7) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) + reg = HOM.BaseGM_Combo_Hom(self.y, self.X, yend=yd2, q=q2, w=self.w.sparse, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 10.12541428], [ 1.56832263], [ 0.15132076], [ 0.21033397]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([34.3450723]),7) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 36.6149682]),7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 46.1219307]),7) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + vm = np.array([[ 2.33694742e+02, -6.66856869e-01, -5.58304254e+00, 4.85488380e+00], [ -6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], [ -5.58304254e+00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], [ 4.85488380e+00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) + z = np.array([ 1. , 19.531 , 35.4585005]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 18.594]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 35.4585005]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 18.594]) + np.testing.assert_array_almost_equal(reg.q[0],q,7) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) its = 1 - self.assertAlmostEqual(reg.iteration, its, 7) + self.assertAlmostEqual(reg.iteration,its,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) sig2 = 232.22680644168395 - self.assertAlmostEqual(reg.sig2, sig2, places=6) - hth = np.array( - [ - [49.0, 704.371999, 724.7435916], - [704.371999, 11686.67338121, 11092.519988], - [724.7435916, 11092.519988, 11614.62257048], - ] - ) - np.testing.assert_array_almost_equal(reg.hth, hth, 4) + self.assertAlmostEqual(reg.sig2,sig2, places=6) + hth = np.array([[ 49. , 704.371999 , 724.7435916 ], [ 704.371999 , 11686.67338121, 11092.519988 ], [ 724.7435916 , 11092.519988 , 11614.62257048]]) + np.testing.assert_array_almost_equal(reg.hth,hth,4) class GM_Combo_Hom_Tester(unittest.TestCase): def setUp(self): - db = libpysal.io.open(libpysal.examples.get_path("columbus.dbf"), "r") + db=libpysal.io.open(libpysal.examples.get_path("columbus.dbf"),"r") y = np.array(db.by_col("HOVAL")) - self.y = np.reshape(y, (49, 1)) + self.y = np.reshape(y, (49,1)) X = [] X.append(db.by_col("INC")) self.X = np.array(X).T self.X = sparse.csr_matrix(self.X) - self.w = libpysal.weights.Rook.from_shapefile( - libpysal.examples.get_path("columbus.shp") - ) - self.w.transform = "r" - + self.w = libpysal.weights.Rook.from_shapefile(libpysal.examples.get_path("columbus.shp")) + self.w.transform = 'r' def test_model(self): - reg = HOM.GM_Combo_Hom(self.y, self.X, w=self.w, A1="hom_sc") - np.testing.assert_array_almost_equal(reg.y[0], np.array([80.467003]), 7) - x = np.array([1.0, 19.531]) - np.testing.assert_array_almost_equal(reg.x[0].toarray()[0], x, 7) - betas = np.array([[10.12541428], [1.56832263], [0.15132076], [0.21033397]]) - np.testing.assert_array_almost_equal(reg.betas, betas, 7) - np.testing.assert_array_almost_equal(reg.u[0], np.array([34.3450723]), 7) - np.testing.assert_array_almost_equal( - reg.e_filtered[0], np.array([36.6149682]), 7 - ) - np.testing.assert_array_almost_equal(reg.e_pred[0], np.array([32.90372983]), 7) - np.testing.assert_array_almost_equal(reg.predy[0], np.array([46.1219307]), 7) - np.testing.assert_array_almost_equal(reg.predy_e[0], np.array([47.56327317]), 7) - self.assertAlmostEqual(reg.n, 49, 7) - self.assertAlmostEqual(reg.k, 3, 7) - z = np.array([1.0, 19.531, 35.4585005]) - np.testing.assert_array_almost_equal(reg.z[0].toarray()[0], z, 7) - h = np.array([1.0, 19.531, 18.594]) - np.testing.assert_array_almost_equal(reg.h[0].toarray()[0], h, 7) - yend = np.array([35.4585005]) - np.testing.assert_array_almost_equal(reg.yend[0], yend, 7) - q = np.array([18.594]) - np.testing.assert_array_almost_equal(reg.q[0].toarray()[0], q, 7) - i_s = "Maximum number of iterations reached." - self.assertAlmostEqual(reg.iter_stop, i_s, 7) - self.assertAlmostEqual(reg.iteration, 1, 7) + reg = HOM.GM_Combo_Hom(self.y, self.X, w=self.w, A1='hom_sc') + np.testing.assert_array_almost_equal(reg.y[0],np.array([80.467003]),7) + x = np.array([ 1. , 19.531]) + np.testing.assert_array_almost_equal(reg.x[0].toarray()[0],x,7) + betas = np.array([[ 10.12541428], [ 1.56832263], [ 0.15132076], [ 0.21033397]]) + np.testing.assert_array_almost_equal(reg.betas,betas,7) + np.testing.assert_array_almost_equal(reg.u[0],np.array([34.3450723]),7) + np.testing.assert_array_almost_equal(reg.e_filtered[0],np.array([ 36.6149682]),7) + np.testing.assert_array_almost_equal(reg.e_pred[0],np.array([ 32.90372983]),7) + np.testing.assert_array_almost_equal(reg.predy[0],np.array([ 46.1219307]),7) + np.testing.assert_array_almost_equal(reg.predy_e[0],np.array([47.56327317]),7) + self.assertAlmostEqual(reg.n,49,7) + self.assertAlmostEqual(reg.k,3,7) + z = np.array([ 1. , 19.531 , 35.4585005]) + np.testing.assert_array_almost_equal(reg.z[0].toarray()[0],z,7) + h = np.array([ 1. , 19.531, 18.594]) + np.testing.assert_array_almost_equal(reg.h[0].toarray()[0],h,7) + yend = np.array([ 35.4585005]) + np.testing.assert_array_almost_equal(reg.yend[0],yend,7) + q = np.array([ 18.594]) + np.testing.assert_array_almost_equal(reg.q[0].toarray()[0],q,7) + i_s = 'Maximum number of iterations reached.' + self.assertAlmostEqual(reg.iter_stop,i_s,7) + self.assertAlmostEqual(reg.iteration,1,7) my = 38.436224469387746 - self.assertAlmostEqual(reg.mean_y, my) + self.assertAlmostEqual(reg.mean_y,my) std_y = 18.466069465206047 - self.assertAlmostEqual(reg.std_y, std_y) + self.assertAlmostEqual(reg.std_y,std_y) pr2 = 0.28379825632694394 - self.assertAlmostEqual(reg.pr2, pr2) + self.assertAlmostEqual(reg.pr2,pr2) pr2_e = 0.25082892555141506 - self.assertAlmostEqual(reg.pr2_e, pr2_e) + self.assertAlmostEqual(reg.pr2_e,pr2_e) sig2 = 232.22680644168395 - self.assertAlmostEqual(reg.sig2, sig2, places=6) - std_err = np.array([15.28707761, 0.44072838, 0.40479714, 0.42263726]) - np.testing.assert_array_almost_equal(reg.std_err, std_err, 6) - z_stat = np.array( - [ - [6.62351206e-01, 5.07746167e-01], - [3.55847888e00, 3.73008780e-04], - [3.73818749e-01, 7.08539170e-01], - [4.97670189e-01, 6.18716523e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.z_stat, z_stat, 6) - vm = np.array( - [ - [2.33694742e02, -6.66856869e-01, -5.58304254e00, 4.85488380e00], - [-6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], - [-5.58304254e00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], - [4.85488380e00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01], - ] - ) - np.testing.assert_array_almost_equal(reg.vm, vm, 6) - + self.assertAlmostEqual(reg.sig2,sig2, places=6) + std_err = np.array([ 15.28707761, 0.44072838, 0.40479714, 0.42263726]) + np.testing.assert_array_almost_equal(reg.std_err,std_err,6) + z_stat = np.array([[ 6.62351206e-01, 5.07746167e-01], [ 3.55847888e+00, 3.73008780e-04], [ 3.73818749e-01, 7.08539170e-01], [ 4.97670189e-01, 6.18716523e-01]]) + np.testing.assert_array_almost_equal(reg.z_stat,z_stat,6) + vm = np.array([[ 2.33694742e+02, -6.66856869e-01, -5.58304254e+00, 4.85488380e+00], [ -6.66856869e-01, 1.94241504e-01, -5.42327138e-02, 5.37225570e-02], [ -5.58304254e+00, -5.42327138e-02, 1.63860721e-01, -1.44425498e-01], [ 4.85488380e+00, 5.37225570e-02, -1.44425498e-01, 1.78622255e-01]]) + np.testing.assert_array_almost_equal(reg.vm,vm,6) suite = unittest.TestSuite() -test_classes = [ - BaseGM_Error_Hom_Tester, - GM_Error_Hom_Tester, - BaseGM_Endog_Error_Hom_Tester, - GM_Endog_Error_Hom_Tester, - BaseGM_Combo_Hom_Tester, - GM_Combo_Hom_Tester, -] +test_classes = [BaseGM_Error_Hom_Tester, GM_Error_Hom_Tester,\ + BaseGM_Endog_Error_Hom_Tester, GM_Endog_Error_Hom_Tester, \ + BaseGM_Combo_Hom_Tester, GM_Combo_Hom_Tester] for i in test_classes: a = unittest.TestLoader().loadTestsFromTestCase(i) suite.addTest(a) -if __name__ == "__main__": +if __name__ == '__main__': runner = unittest.TextTestRunner() runner.run(suite) + diff --git a/spreg/tests/test_ml_error.py b/spreg/tests/test_ml_error.py index de9b051b..cf3a8605 100644 --- a/spreg/tests/test_ml_error.py +++ b/spreg/tests/test_ml_error.py @@ -1,7 +1,5 @@ import unittest import libpysal -from libpysal.examples import load_example -from libpysal.weights import Queen import numpy as np from scipy import sparse from spreg.ml_error import ML_Error @@ -14,14 +12,17 @@ class TestMLError(unittest.TestCase): def setUp(self): - south = load_example("South") - db = libpysal.io.open(south.get_path("south.dbf"), "r") - self.y_name = "HR90" - self.y = np.array(db.by_col(self.y_name)) + db = libpysal.io.open(libpysal.examples.get_path("baltim.dbf"), "r") + self.ds_name = "baltim.dbf" + self.y_name = "PRICE" + self.y = np.array(db.by_col(self.y_name)).T self.y.shape = (len(self.y), 1) - self.x_names = ["RD90", "PS90", "UE90", "DV90"] + self.x_names = ["NROOM", "AGE", "SQFT"] self.x = np.array([db.by_col(var) for var in self.x_names]).T - self.w = Queen.from_shapefile(south.get_path("south.shp")) + ww = libpysal.io.open(libpysal.examples.get_path("baltim_q.gal")) + self.w = ww.read() + ww.close() + self.w_name = "baltim_q.gal" self.w.transform = "r" def _estimate_and_compare(self, method="FULL", RTOL=RTOL): @@ -31,57 +32,58 @@ def _estimate_and_compare(self, method="FULL", RTOL=RTOL): w=self.w, name_y=self.y_name, name_x=self.x_names, - name_w="south_q.gal", + name_w=self.w_name, method=method, ) - betas = np.array([[6.1492], [4.4024], [1.7784], [-0.3781], [0.4858], [0.2991]]) + betas = np.array([[19.45930348], + [ 3.98928064], + [-0.16714232], + [ 0.57336871], + [ 0.71757002]]) np.testing.assert_allclose(reg.betas, betas, RTOL + 0.0001) - u = np.array([-5.97649777]) + u = np.array([29.870239]) np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([6.92258051]) + predy = np.array([17.129761]) np.testing.assert_allclose(reg.predy[0], predy, RTOL) - n = 1412 + n = 211 np.testing.assert_allclose(reg.n, n, RTOL) - k = 5 + k = 4 np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([0.94608274]) + y = np.array([47.]) np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, -0.39902838, 0.89645344, 6.85780705, 7.2636377]) + x = np.array([ 1. , 4. , 148. , 11.25]) np.testing.assert_allclose(reg.x[0], x, RTOL) - e = np.array([-4.92843327]) + e = np.array([44.392043]) np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - my = 9.5492931620846928 + my = 44.30718 np.testing.assert_allclose(reg.mean_y, my) - sy = 7.0388508798387219 + sy = 23.606077 np.testing.assert_allclose(reg.std_y, sy) vm = np.array( - [1.06476526, 0.05548248, 0.04544514, 0.00614425, 0.01481356, 0.00143001] + [3.775969e+01, 1.337534e+00, 4.440495e-03, 2.890193e-02, + 3.496050e-03] ) np.testing.assert_allclose(reg.vm.diagonal(), vm, RTOL) - sig2 = np.array([[32.40685441]]) + sig2 = np.array([[219.239799]]) np.testing.assert_allclose(reg.sig2, sig2, RTOL) - pr2 = 0.3057664820364818 - np.testing.assert_allclose(reg.pr2, pr2) + pr2 = 0.341471 + np.testing.assert_allclose(reg.pr2, pr2, RTOL) std_err = np.array( - [1.03187463, 0.23554719, 0.21317867, 0.07838525, 0.12171098, 0.03781546] + [6.144892, 1.156518, 0.066637, 0.170006, 0.059127] ) np.testing.assert_allclose(reg.std_err, std_err, RTOL) - z_stat = [ - (5.9592751097983534, 2.5335926307459251e-09), - (18.690182928021841, 5.9508619446611137e-78), - (8.3421632936950338, 7.2943630281051907e-17), - (-4.8232686291115678, 1.4122456582517099e-06), - (3.9913060809142995, 6.5710406838016854e-05), - (7.9088780724028922, 2.5971882547279339e-15), - ] + z_stat = [(3.166744811610107, 0.0015415552994677963), + (3.4493895324306485, 0.0005618555635414317), + (-2.5082495410045618, 0.012133094835810014), + (3.3726442232925864, 0.0007445008419860677), + (12.13599679437352, 6.807593113579489e-34)] np.testing.assert_allclose(reg.z_stat, z_stat, RTOL, atol=ATOL) - logll = -4471.407066887894 + logll = -881.269405 np.testing.assert_allclose(reg.logll, logll, RTOL) - aic = 8952.8141337757879 + aic = 1770.538809 np.testing.assert_allclose(reg.aic, aic, RTOL) - schwarz = 8979.0779458660545 + schwarz = 1783.946242 np.testing.assert_allclose(reg.schwarz, schwarz, RTOL) - def test_dense(self): self._estimate_and_compare(method="FULL") @@ -95,55 +97,57 @@ def test_ord(self): w=self.w, name_y=self.y_name, name_x=self.x_names, - name_w="south_q.gal", + name_w=self.w_name, method="ORD", ) - betas = np.array([[6.1492], [4.4024], [1.7784], [-0.3781], [0.4858], [0.2991]]) - np.testing.assert_allclose(reg.betas, betas, RTOL + 0.0001) - u = np.array([-5.97649777]) + betas = np.array([[19.45930348], + [ 3.98928064], + [-0.16714232], + [ 0.57336871], + [ 0.71757002]]) + np.testing.assert_allclose(reg.betas, betas, RTOL) + u = np.array([29.870239]) np.testing.assert_allclose(reg.u[0], u, RTOL) - predy = np.array([6.92258051]) + predy = np.array([17.129761]) np.testing.assert_allclose(reg.predy[0], predy, RTOL) - n = 1412 + n = 211 np.testing.assert_allclose(reg.n, n, RTOL) - k = 5 + k = 4 np.testing.assert_allclose(reg.k, k, RTOL) - y = np.array([0.94608274]) + y = np.array([47.]) np.testing.assert_allclose(reg.y[0], y, RTOL) - x = np.array([1.0, -0.39902838, 0.89645344, 6.85780705, 7.2636377]) + x = np.array([ 1. , 4. , 148. , 11.25]) np.testing.assert_allclose(reg.x[0], x, RTOL) - e = np.array([-4.92843327]) + e = np.array([44.392043]) np.testing.assert_allclose(reg.e_filtered[0], e, RTOL) - my = 9.5492931620846928 + my = 44.30718 np.testing.assert_allclose(reg.mean_y, my) - sy = 7.0388508798387219 + sy = 23.606077 np.testing.assert_allclose(reg.std_y, sy) vm = np.array( - [1.06476526, 0.05548248, 0.04544514, 0.00614425, 0.01481356, 0.001501] + [3.775969e+01, 1.337534e+00, 4.440495e-03, 2.890193e-02, + 3.586781e-03] ) np.testing.assert_allclose(reg.vm.diagonal(), vm, RTOL * 10) - sig2 = np.array([[32.40685441]]) + sig2 = np.array([[219.239799]]) np.testing.assert_allclose(reg.sig2, sig2, RTOL) - pr2 = 0.3057664820364818 + pr2 = 0.34147059826596426 np.testing.assert_allclose(reg.pr2, pr2) std_err = np.array( - [1.03187463, 0.23554719, 0.21317867, 0.07838525, 0.12171098, 0.038744] + [6.144892, 1.156518, 0.066637, 0.170006, 0.05989 ] ) np.testing.assert_allclose(reg.std_err, std_err, RTOL * 10) - z_stat = [ - (5.95927510, 2.5335927e-09), - (18.6901829, 5.9508630e-78), - (8.34216329, 7.2943634e-17), - (-4.8232686, 1.4122457e-06), - (3.99130608, 6.5710407e-05), - (7.71923784, 1.1702739e-14), - ] + z_stat = [(3.166744811610107, 0.0015415552994677963), + (3.4493895324306485, 0.0005618555635414317), + (-2.5082495410045618, 0.012133094835810014), + (3.3726442232925864, 0.0007445008419860677), + (11.981517603949666, 4.441183328428627e-33)] np.testing.assert_allclose(reg.z_stat, z_stat, rtol=RTOL, atol=ATOL) - logll = -4471.407066887894 + logll = -881.269405 np.testing.assert_allclose(reg.logll, logll, RTOL) - aic = 8952.8141337757879 + aic = 1770.538809 np.testing.assert_allclose(reg.aic, aic, RTOL) - schwarz = 8979.0779458660545 + schwarz = 1783.946242 np.testing.assert_allclose(reg.schwarz, schwarz, RTOL) diff --git a/spreg/tests/test_sp_panels.py b/spreg/tests/test_sp_panels.py index b940c5c1..4710d1e8 100755 --- a/spreg/tests/test_sp_panels.py +++ b/spreg/tests/test_sp_panels.py @@ -3,192 +3,87 @@ import libpysal from libpysal.common import RTOL from spreg.sp_panels import * - ATOL = 1e-12 class Test_GM_KKP(unittest.TestCase): def setUp(self): - nat = libpysal.examples.load_example("NCOVR") - self.db = libpysal.io.open(nat.get_path("NAT.dbf"), "r") - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("NAT.shp") - ) - self.w.transform = "r" - self.y_var0 = ["HR70", "HR80", "HR90"] - self.x_var0 = ["RD70", "RD80", "RD90", "PS70", "PS80", "PS90"] + nat = libpysal.examples.load_example('NCOVR') + self.db = libpysal.io.open(nat.get_path("NAT.dbf"),'r') + self.w = libpysal.weights.Queen.from_shapefile(libpysal.examples.get_path("NAT.shp")) + self.w.transform = 'r' + self.y_var0 = ['HR70','HR80','HR90'] + self.x_var0 = ['RD70','RD80','RD90','PS70','PS80','PS90'] self.y = np.array([self.db.by_col(name) for name in self.y_var0]).T self.x = np.array([self.db.by_col(name) for name in self.x_var0]).T - def test_wide_ident(self): - reg = GM_KKP( - self.y, - self.x, - self.w, - full_weights=False, - name_y=self.y_var0, - name_x=self.x_var0, - ) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49221562], - [3.62445753], - [1.31187779], - [0.41777589], - [22.81908224], - [39.90993228], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.26948117e-02, -1.98160325e-06, 7.38157674e-05], - [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], - [7.38157674e-05, 1.13099329e-03, 7.26783636e-03], - ] - ), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, ["CONSTANT", "RD", "PS", "lambda", " sigma2_v", "sigma2_1"] - ) - np.testing.assert_equal(reg.name_y, "HR") - def test_wide_full(self): - reg = GM_KKP(self.y, self.x, self.w, full_weights=True) + def test_wide_ident(self): + reg = GM_KKP(self.y,self.x,self.w,full_weights=False,name_y=self.y_var0, name_x=self.x_var0) + np.testing.assert_allclose(reg.betas,np.array([[ 6.49221562], + [ 3.62445753], + [ 1.31187779], + [ 0.41777589], + [22.81908224], + [39.90993228]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.26948117e-02, -1.98160325e-06, 7.38157674e-05], + [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], + [ 7.38157674e-05, 1.13099329e-03, 7.26783636e-03]]),RTOL*10) + np.testing.assert_equal(reg.name_x, ['CONSTANT', 'RD', 'PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') + + def test_wide_full(self): + reg = GM_KKP(self.y,self.x,self.w,full_weights=True) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49193589], - [3.55740165], - [1.29462748], - [0.4263399], - [22.47241979], - [45.82593532], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.45113773e-02, -2.14882672e-06, 8.54997693e-05], - [-2.14882672e-06, 8.41929187e-03, 1.24553497e-03], - [8.54997693e-05, 1.24553497e-03, 8.12448812e-03], - ] - ), - RTOL, - ) + np.testing.assert_allclose(reg.betas,np.array([[ 6.49193589], + [ 3.55740165], + [ 1.29462748], + [ 0.4263399 ], + [22.47241979], + [45.82593532]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.45113773e-02, -2.14882672e-06, 8.54997693e-05], + [-2.14882672e-06, 8.41929187e-03, 1.24553497e-03], + [ 8.54997693e-05, 1.24553497e-03, 8.12448812e-03]]),RTOL) - def test_long_ident(self): - bigy = self.y.reshape((self.y.size, 1), order="F") - bigx = self.x[:, 0:3].reshape((self.x.shape[0] * 3, 1), order="F") - bigx = np.hstack( - (bigx, self.x[:, 3:6].reshape((self.x.shape[0] * 3, 1), order="F")) - ) - reg = GM_KKP( - bigy, bigx, self.w, full_weights=False, name_y=["HR"], name_x=["RD", "PS"] - ) + def test_long_ident(self): + bigy = self.y.reshape((self.y.size,1),order="F") + bigx = self.x[:,0:3].reshape((self.x.shape[0]*3,1),order='F') + bigx = np.hstack((bigx,self.x[:,3:6].reshape((self.x.shape[0]*3,1),order='F'))) + reg = GM_KKP(bigy,bigx,self.w,full_weights=False,name_y=['HR'], name_x=['RD','PS']) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [6.49221562], - [3.62445753], - [1.31187779], - [0.41777589], - [22.81908224], - [39.90993228], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - reg.vm, - np.array( - [ - [1.26948117e-02, -1.98160325e-06, 7.38157674e-05], - [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], - [7.38157674e-05, 1.13099329e-03, 7.26783636e-03], - ] - ), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, ["CONSTANT", "RD", "PS", "lambda", " sigma2_v", "sigma2_1"] - ) - np.testing.assert_equal(reg.name_y, "HR") + np.testing.assert_allclose(reg.betas,np.array([[ 6.49221562], + [ 3.62445753], + [ 1.31187779], + [ 0.41777589], + [22.81908224], + [39.90993228]]),RTOL) + np.testing.assert_allclose(reg.vm,np.array([[ 1.26948117e-02, -1.98160325e-06, 7.38157674e-05], + [-1.98160325e-06, 7.69961725e-03, 1.13099329e-03], + [ 7.38157674e-05, 1.13099329e-03, 7.26783636e-03]]),RTOL*10) + np.testing.assert_equal(reg.name_x, ['CONSTANT', 'RD', 'PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') def test_regimes(self): regimes = self.db.by_col("SOUTH") - reg = GM_KKP( - self.y, - self.x, - self.w, - full_weights=False, - regimes=regimes, - name_y=self.y_var0, - name_x=self.x_var0, - ) - np.testing.assert_allclose( - reg.betas, - np.array( - [ - [5.25856482], - [3.19249165], - [1.0056967], - [7.94560642], - [3.13931041], - [1.53700634], - [0.35979407], - [22.5650005], - [39.71516708], - ] - ), - RTOL, - ) - np.testing.assert_allclose( - np.sqrt(reg.vm.diagonal()), - np.array([0.158986, 0.157543, 0.104128, 0.165254, 0.117737, 0.136666]), - RTOL, - ) - np.testing.assert_equal( - reg.name_x, - [ - "0_CONSTANT", - "0_RD", - "0_PS", - "1_CONSTANT", - "1_RD", - "1_PS", - "lambda", - " sigma2_v", - "sigma2_1", - ], - ) - np.testing.assert_equal(reg.name_y, "HR") - np.testing.assert_allclose( - reg.chow.regi, - np.array( - [ - [1.420430e02, 9.516507e-33], - [7.311490e-02, 7.868543e-01], - [9.652492e00, 1.890949e-03], - ] - ), - RTOL, - ) - np.testing.assert_allclose(reg.chow.joint[0], 158.7225, RTOL) + reg = GM_KKP(self.y,self.x,self.w,full_weights=False,regimes=regimes, + name_y=self.y_var0, name_x=self.x_var0) + np.testing.assert_allclose(reg.betas,np.array([[ 5.25856482], + [ 3.19249165], + [ 1.0056967 ], + [ 7.94560642], + [ 3.13931041], + [ 1.53700634], + [ 0.35979407], + [22.5650005 ], + [39.71516708]]),RTOL) + np.testing.assert_allclose(np.sqrt(reg.vm.diagonal()),np.array([0.158986, 0.157543, 0.104128, 0.165254, 0.117737, 0.136666]),RTOL) + np.testing.assert_equal(reg.name_x, ['0_CONSTANT', '0_RD', '0_PS', '1_CONSTANT', '1_RD', '1_PS', 'lambda', ' sigma2_v', 'sigma2_1']) + np.testing.assert_equal(reg.name_y, 'HR') + np.testing.assert_allclose(reg.chow.regi,np.array([[1.420430e+02, 9.516507e-33], + [7.311490e-02, 7.868543e-01], + [9.652492e+00, 1.890949e-03]]),RTOL*10) + np.testing.assert_allclose(reg.chow.joint[0],158.7225,RTOL) - -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() + diff --git a/spreg/tests/test_sur_error.py b/spreg/tests/test_sur_error.py index 9bcb2961..d592b262 100755 --- a/spreg/tests/test_sur_error.py +++ b/spreg/tests/test_sur_error.py @@ -2,13 +2,15 @@ import numpy as np import libpysal from libpysal.examples import load_example -from ..sur_utils import sur_dictxy -from ..sur_error import SURerrorML, SURerrorGM -from .test_sur import dict_compare +from spreg.sur_utils import sur_dictxy +from spreg.sur_error import SURerrorML, SURerrorGM from libpysal.common import RTOL ATOL = 0.0001 +def dict_compare(actual, desired, rtol, atol=1e-7): + for i in actual.keys(): + np.testing.assert_allclose(actual[i], desired[i], rtol, atol=atol) class Test_SUR_error(unittest.TestCase): def setUp(self): @@ -316,9 +318,7 @@ class Test_SUR_error_gm(unittest.TestCase): def setUp(self): nat = load_example("Natregimes") self.db = libpysal.io.open(nat.get_path("natregimes.dbf"), "r") - self.w = libpysal.weights.Queen.from_shapefile( - libpysal.examples.get_path("natregimes.shp") - ) + self.w = libpysal.weights.Queen.from_shapefile(nat.get_path("natregimes.shp")) self.w.transform = "r" def test_error_gm(self): # 2 equations diff --git a/spreg/tests/test_twosls_sp_regimes.py b/spreg/tests/test_twosls_sp_regimes.py index f763ac74..3d299381 100644 --- a/spreg/tests/test_twosls_sp_regimes.py +++ b/spreg/tests/test_twosls_sp_regimes.py @@ -22,7 +22,7 @@ def test___init__(self): X.append(self.db.by_col("INC")) X.append(self.db.by_col("HOVAL")) self.X = np.array(X).T - reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) betas = np.array([[ 45.14892906], [ -1.42593383], [ -0.11501037], @@ -121,7 +121,7 @@ def test_init_discbd(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, lag_q=False, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, lag_q=False, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 42.7266306 ], [ -0.15552345], [ 37.70545276], @@ -162,7 +162,7 @@ def test_lag_q(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, sig2n_k=True, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 37.87698329], [ -0.89426982], [ 31.4714777 ], @@ -187,7 +187,7 @@ def test_all_regi(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ 37.87698329, -0.89426982, 31.4714777 , -0.71640525, -0.28494432, -0.2294271 , 0.62996544]]) np.testing.assert_allclose(tbetas, reg.betas.T,RTOL) @@ -228,7 +228,7 @@ def test_all_regi_sig2(self): w = libpysal.weights.util.lat2W(latt,latt) w.transform='r' regi = [0]*(n//2) + [1]*(n//2) - model = GM_Lag_Regimes(y, x1, regi, q=q, yend=x2, w=w, regime_lag_sep=True, regime_err_sep=True) + model = GM_Lag_Regimes(y, x1, regi, q=q, yend=x2, w=w, regime_lag_sep=True, regime_err_sep=True, robust=None) w1 = libpysal.weights.util.lat2W(latt//2,latt) w1.transform='r' model1 = GM_Lag(y[0:(n//2)].reshape((n//2),1), x1[0:(n//2)],yend=x2[0:(n//2)], q=q[0:(n//2)], w=w1) @@ -244,7 +244,7 @@ def test_all_regi_sig2(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w,regime_lag_sep=True, regime_err_sep = True) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w,regime_lag_sep=True, regime_err_sep = True, robust=None) tbetas = np.array([[ 42.35827477], [ -0.09472413], [ -0.68794223], @@ -287,7 +287,7 @@ def test_fixed_const(self): yd = np.reshape(yd, (49,1)) q = np.array(self.db.by_col("DISCBD")) q = np.reshape(q, (49,1)) - reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, constant_regi='one', regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, X, self.regimes, yend=yd, q=q, w=self.w, constant_regi='one', regime_lag_sep=False, regime_err_sep=False, robust=None) tbetas = np.array([[ -0.37658823], [ -0.9666079 ], [ 35.5445944 ], @@ -328,7 +328,9 @@ def test_names(self): q_var = ['DISCBD'] q = np.array([self.db.by_col(name) for name in q_var]).T r_var = 'NSA' - reg = GM_Lag_Regimes(self.y, x, self.regimes, yend=yd, q=q, w=self.w, name_y=y_var, name_x=x_var, name_yend=yd_var, name_q=q_var, name_regimes=r_var, name_ds='columbus', name_w='columbus.gal', regime_lag_sep=False, regime_err_sep=False) + reg = GM_Lag_Regimes(self.y, x, self.regimes, yend=yd, q=q, w=self.w, name_y=y_var, name_x=x_var, + name_yend=yd_var, name_q=q_var, name_regimes=r_var, name_ds='columbus', + name_w='columbus.gal', regime_lag_sep=False, regime_err_sep=False, robust=None) betas = np.array([[ 37.87698329], [ -0.89426982], [ 31.4714777 ], @@ -350,5 +352,63 @@ def test_names(self): self.assertListEqual(reg.name_q, ['0_DISCBD', '0_W_INC', '0_W_DISCBD', '1_DISCBD', '1_W_INC', '1_W_DISCBD']) self.assertEqual(reg.name_y, y_var) + def test_slx(self): + X = [] + X.append(self.db.by_col("INC")) + X.append(self.db.by_col("HOVAL")) + self.X = np.array(X).T + reg = GM_Lag_Regimes(self.y, self.X, self.regimes, w=self.w, slx_lags=1, regime_lag_sep=False, + regime_err_sep=False, spat_diag=True) + betas = np.array([[ 8.18931398e+01], + [-1.53395435e+00], + [-1.69757478e-01], + [-2.27789212e+00], + [ 5.39079719e-01], + [ 7.51699174e+01], + [-1.08739064e+00], + [-2.83520279e-01], + [-1.68207712e+00], + [ 2.74687447e-01], + [-3.46101374e-02]]) + np.testing.assert_allclose(reg.betas, betas,RTOL) + vm = np.array( [[ 1.73599745e+03, -9.34710692e+00, -1.24643977e+00, -6.47037384e+01, + 7.12544407e+00, 1.67217894e+03, -3.05066669e+00, -2.35566285e+00, + -3.66089456e+01, -4.16736383e+00, -2.36649704e+01], + [-9.34710692e+00, 2.76627235e-01, -4.82833143e-02, 8.93797856e-02, + 3.01134955e-02, -9.78627573e+00, 2.41397355e-02, 1.18016819e-02, + 2.19536352e-01, 2.40036703e-02, 1.36868186e-01], + [-1.24643977e+00, -4.82833143e-02, 2.49271317e-02, 1.08816895e-01, + -3.05204771e-02, -8.06738318e-01, 6.26670530e-04, 1.40330458e-03, + 1.69512409e-02, 2.06235513e-03, 1.16361002e-02], + [-6.47037384e+01, 8.93797856e-02, 1.08816895e-01, 3.37057241e+00, + -6.18148061e-01, -6.02788288e+01, 1.07776025e-01, 8.56099990e-02, + 1.31783651e+00, 1.50360008e-01, 8.53645284e-01], + [ 7.12544407e+00, 3.01134955e-02, -3.05204771e-02, -6.18148061e-01, + 1.79091963e-01, 6.63593281e+00, -1.64138641e-02, -7.98832541e-03, + -1.48902330e-01, -1.62737808e-02, -9.27966724e-02], + [ 1.67217894e+03, -9.78627573e+00, -8.06738318e-01, -6.02788288e+01, + 6.63593281e+00, 1.77071537e+03, -7.45841412e+00, -1.64359023e+00, + -3.92564157e+01, -4.79661230e+00, -2.37950208e+01], + [-3.05066669e+00, 2.41397355e-02, 6.26670530e-04, 1.07776025e-01, + -1.64138641e-02, -7.45841412e+00, 4.80396971e-01, -9.88376849e-02, + 8.60302399e-02, 4.92161147e-02, 4.74983701e-02], + [-2.35566285e+00, 1.18016819e-02, 1.40330458e-03, 8.56099990e-02, + -7.98832541e-03, -1.64359023e+00, -9.88376849e-02, 4.80027099e-02, + -2.23775047e-02, 6.77837002e-03, 3.22304374e-02], + [-3.66089456e+01, 2.19536352e-01, 1.69512409e-02, 1.31783651e+00, + -1.48902330e-01, -3.92564157e+01, 8.60302399e-02, -2.23775047e-02, + 1.50194231e+00, -1.84011183e-02, 5.24380589e-01], + [-4.16736383e+00, 2.40036703e-02, 2.06235513e-03, 1.50360008e-01, + -1.62737808e-02, -4.79661230e+00, 4.92161147e-02, 6.77837002e-03, + -1.84011183e-02, 4.72613924e-02, 5.90507313e-02], + [-2.36649704e+01, 1.36868186e-01, 1.16361002e-02, 8.53645284e-01, + -9.27966724e-02, -2.37950208e+01, 4.74983701e-02, 3.22304374e-02, + 5.24380589e-01, 5.90507313e-02, 3.35692100e-01]]) + np.testing.assert_allclose(reg.vm, vm,RTOL) + ak_test = np.array([0.528338, 0.467306]) + np.testing.assert_allclose(reg.ak_test, ak_test,RTOL) + cfh_test = np.array([8.094023, 0.088194]) + np.testing.assert_allclose(reg.cfh_test, cfh_test,RTOL) + if __name__ == '__main__': unittest.main() diff --git a/spreg/twosls.py b/spreg/twosls.py index 0fe355b1..c0a5bc39 100644 --- a/spreg/twosls.py +++ b/spreg/twosls.py @@ -159,7 +159,6 @@ def __init__( hthi = la.inv(hth) zth = spdot(z.T, h) hty = spdot(h.T, y) - factor_1 = np.dot(zth, hthi) factor_2 = np.dot(factor_1, zth.T) # this one needs to be in cache to be used in AK diff --git a/spreg/twosls_regimes.py b/spreg/twosls_regimes.py index 50a36853..26458331 100644 --- a/spreg/twosls_regimes.py +++ b/spreg/twosls_regimes.py @@ -353,7 +353,7 @@ def __init__( vm=False, constant_regi="many", cols2regi="all", - regime_err_sep=True, + regime_err_sep=False, name_y=None, name_x=None, cores=False, diff --git a/spreg/twosls_sp.py b/spreg/twosls_sp.py index e5ff8552..a58139bc 100755 --- a/spreg/twosls_sp.py +++ b/spreg/twosls_sp.py @@ -8,14 +8,14 @@ from . import twosls as TSLS from . import user_output as USER from .utils import set_endog, sp_att, set_warn +from .sputils import _spmultiplier import pandas as pd -from .output import output, _spat_diag_out, _spat_pseudo_r2 +from .output import output, _spat_diag_out, _spat_pseudo_r2, _summary_impacts __all__ = ["GM_Lag"] class BaseGM_Lag(TSLS.BaseTSLS): - """ Spatial two stage least squares (S2SLS) (note: no consistency checks, diagnostics or constant added); Anselin (1988) [Anselin1988]_ @@ -169,18 +169,18 @@ class BaseGM_Lag(TSLS.BaseTSLS): """ def __init__( - self, - y, - x, - yend=None, - q=None, - w=None, - w_lags=1, - slx_lags=0, - lag_q=True, - robust=None, - gwk=None, - sig2n_k=False, + self, + y, + x, + yend=None, + q=None, + w=None, + w_lags=1, + slx_lags=0, + lag_q=True, + robust=None, + gwk=None, + sig2n_k=False, ): if slx_lags > 0: @@ -195,7 +195,6 @@ def __init__( class GM_Lag(BaseGM_Lag): - """ Spatial two stage least squares (S2SLS) with results and diagnostics; Anselin (1988) :cite:`Anselin1988` @@ -238,7 +237,10 @@ class GM_Lag(BaseGM_Lag): sig2n_k : boolean If True, then use n-k to estimate sigma^2. If False, use n. spat_diag : boolean - If True, then compute Anselin-Kelejian test + If True, then compute Anselin-Kelejian test and Common Factor Hypothesis test (if applicable) + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results vm : boolean If True, include variance-covariance matrix in summary results @@ -258,7 +260,9 @@ class GM_Lag(BaseGM_Lag): Name of dataset for use in output latex : boolean Specifies if summary is to be printed in latex format - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the bounds of -1 and 1. Attributes ---------- output : dataframe @@ -323,6 +327,9 @@ class GM_Lag(BaseGM_Lag): ak_test : tuple Anselin-Kelejian test; tuple contains the pair (statistic, p-value) + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). name_y : string Name of dependent variable for use in output name_x : list of strings @@ -485,28 +492,30 @@ class GM_Lag(BaseGM_Lag): """ def __init__( - self, - y, - x, - yend=None, - q=None, - w=None, - w_lags=1, - lag_q=True, - slx_lags=0, - robust=None, - gwk=None, - sig2n_k=False, - spat_diag=False, - vm=False, - name_y=None, - name_x=None, - name_yend=None, - name_q=None, - name_w=None, - name_gwk=None, - name_ds=None, - latex=False, + self, + y, + x, + yend=None, + q=None, + w=None, + w_lags=1, + lag_q=True, + slx_lags=0, + robust=None, + gwk=None, + sig2n_k=False, + spat_diag=True, + spat_impacts=True, + vm=False, + name_y=None, + name_x=None, + name_yend=None, + name_q=None, + name_w=None, + name_gwk=None, + name_ds=None, + latex=False, + hard_bound=False, ): n = USER.check_arrays(x, yend, q) @@ -514,15 +523,16 @@ def __init__( USER.check_weights(w, y, w_required=True) USER.check_robust(robust, gwk) if robust == "hac" and spat_diag: - set_warn( - self, - "Spatial diagnostics are not available for HAC estimation. The spatial diagnostics have been disabled for this model.", - ) - spat_diag = False + set_warn( + self, + "Spatial diagnostics are not available for HAC estimation. The spatial diagnostics have been disabled for this model.", + ) + spat_diag = False x_constant, name_x, warn = USER.check_constant(x, name_x) + name_x = USER.set_name_x(name_x, x_constant) # need to check for None and set defaults if slx_lags > 0: - name_x += USER.set_name_spatial_lags(name_x, slx_lags) + name_x += USER.set_name_spatial_lags(name_x[1:], slx_lags) # exclude constant set_warn(self, warn) BaseGM_Lag.__init__( @@ -541,7 +551,7 @@ def __init__( ) self.rho = self.betas[-1] self.predy_e, self.e_pred, warn = sp_att( - w, self.y, self.predy, self.yend[:, -1].reshape(self.n, 1), self.rho + w, self.y, self.predy, self.yend[:, -1].reshape(self.n, 1), self.rho, hard_bound=hard_bound ) set_warn(self, warn) self.title = "SPATIAL TWO STAGE LEAST SQUARES" @@ -549,35 +559,43 @@ def __init__( self.title += " WITH SLX (SPATIAL DURBIN MODEL)" self.name_ds = USER.set_name_ds(name_ds) self.name_y = USER.set_name_y(name_y) - self.name_x = USER.set_name_x(name_x, x_constant) # name_x contains SLX terms for slx_lags > 0 + # self.name_x = USER.set_name_x(name_x, x_constant) # name_x contains SLX terms for slx_lags > 0 + self.name_x = name_x # already contains constant in new setup self.name_yend = USER.set_name_yend(name_yend, yend) self.name_yend.append(USER.set_name_yend_sp(self.name_y)) self.name_z = self.name_x + self.name_yend self.name_q = USER.set_name_q(name_q, q) if slx_lags > 0: # need to remove all but last SLX variables from name_x self.name_x0 = [] - self.name_x0.append(self.name_x[0]) # constant -# print(f"x0 first {self.name_x0}") - kx = int((self.k -self.kstar -1)/(slx_lags +1) ) # number of original exogenous vars + self.name_x0.append(self.name_x[0]) # constant + kx = int((self.k - self.kstar - 1) / (slx_lags + 1)) # number of original exogenous vars self.name_x0.extend(self.name_x[-kx:]) -# print(f"in here {self.name_x0}") self.name_q.extend(USER.set_name_q_sp(self.name_x0, w_lags, self.name_q, lag_q)) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['yend'] * (len(self.name_yend) - 1) + ['rho'] else: self.name_q.extend(USER.set_name_q_sp(self.name_x, w_lags, self.name_q, lag_q)) + var_types = ['x'] * len(self.name_x) + ['yend'] * (len(self.name_yend) - 1) + ['rho'] self.name_h = USER.set_name_h(self.name_x, self.name_q) self.robust = USER.set_robust(robust) self.name_w = USER.set_name_w(name_w, w) self.name_gwk = USER.set_name_w(name_gwk, gwk) + self.slx_lags = slx_lags self.output = pd.DataFrame(self.name_x + self.name_yend, columns=['var_names']) - self.output['var_type'] = ['x'] * len(self.name_x) + ['yend'] * (len(self.name_yend)-1) + ['rho'] + self.output['var_type'] = var_types self.output['regime'], self.output['equation'] = (0, 0) self.other_top = _spat_pseudo_r2(self) + diag_out = None if spat_diag: diag_out = _spat_diag_out(self, w, 'yend') - else: - diag_out = None + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts output(reg=self, vm=vm, robust=robust, other_end=diag_out, latex=latex) + def _test(): import doctest diff --git a/spreg/twosls_sp_regimes.py b/spreg/twosls_sp_regimes.py old mode 100644 new mode 100755 index db0b23c3..185a0878 --- a/spreg/twosls_sp_regimes.py +++ b/spreg/twosls_sp_regimes.py @@ -5,16 +5,18 @@ __author__ = "Luc Anselin luc.anselin@asu.edu, Pedro V. Amaral pedro.amaral@asu.edu, David C. Folch david.folch@asu.edu" import numpy as np +import pandas as pd +import multiprocessing as mp from . import regimes as REGI from . import user_output as USER -import multiprocessing as mp from .twosls_regimes import TSLS_Regimes, _optimal_weight from .twosls import BaseTSLS -from .utils import set_endog, set_endog_sparse, sp_att, set_warn, sphstack, spdot +from .utils import set_endog, set_endog_sparse, sp_att, set_warn, sphstack, spdot, optim_k from .robust import hac_multi -import pandas as pd -from .output import output, _spat_diag_out, _spat_pseudo_r2 - +from .sputils import _spmultiplier +from .output import output, _spat_diag_out, _spat_pseudo_r2, _summary_impacts +from .skater_reg import Skater_reg +from .twosls_sp import BaseGM_Lag class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): @@ -86,8 +88,11 @@ class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): matrix must have ones along the main diagonal. sig2n_k : boolean If True, then use n-k to estimate sigma^2. If False, use n. + spat_impacts : boolean + If True, include average direct impact (ADI), average indirect impact (AII), + and average total impact (ATI) in summary results spat_diag : boolean - If True, then compute Anselin-Kelejian test + If True, then compute Anselin-Kelejian test and Common Factor Hypothesis test (if applicable) vm : boolean If True, include variance-covariance matrix in summary results @@ -206,6 +211,9 @@ class GM_Lag_Regimes(TSLS_Regimes, REGI.Regimes_Frame): p-value) Only available in dictionary 'multi' when multiple regressions (see 'multi' below for details) + cfh_test : tuple + Common Factor Hypothesis test; tuple contains the pair (statistic, + p-value). Only when it applies (see specific documentation). name_y : string Name of dependent variable for use in output name_x : list of strings @@ -443,14 +451,15 @@ def __init__( w_lags=1, slx_lags=0, lag_q=True, - robust=None, + robust='white', gwk=None, sig2n_k=False, - spat_diag=False, + spat_diag=True, + spat_impacts=True, constant_regi="many", cols2regi="all", - regime_lag_sep=True, - regime_err_sep=True, + regime_lag_sep=False, + regime_err_sep=False, cores=False, vm=False, name_y=None, @@ -462,6 +471,7 @@ def __init__( name_gwk=None, name_ds=None, latex=False, + hard_bound=False, ): n = USER.check_arrays(y, x) @@ -501,14 +511,19 @@ def __init__( x_constant = np.hstack((x_constant, wx)) name_slx = USER.set_name_spatial_lags(name_x, slx_lags) name_q.extend(USER.set_name_q_sp(name_slx[-len(name_x):], w_lags, name_q, lag_q, force_all=True)) - name_x += name_slx + name_x += name_slx + if cols2regi == 'all': + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False)[0:-1] + else: + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False) else: name_q.extend(USER.set_name_q_sp(name_x, w_lags, name_q, lag_q, force_all=True)) yend2, q2 = yend, q + cols2regi = REGI.check_cols2regi( + constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False) self.n = x_constant.shape[0] - cols2regi = REGI.check_cols2regi( - constant_regi, cols2regi, x_constant, yend=yend2, add_cons=False - ) self.cols2regi = cols2regi self.regimes_set = REGI._get_regimes_set(regimes) self.regimes = regimes @@ -522,6 +537,7 @@ def __init__( regime_lag_sep = False self.regime_err_sep = regime_err_sep self.regime_lag_sep = regime_lag_sep + if regime_lag_sep == True: cols2regi += [True] w_i, regi_ids, warn = REGI.w_regimes( @@ -533,7 +549,6 @@ def __init__( min_n=len(cols2regi) + 1, ) set_warn(self, warn) - else: cols2regi += [False] @@ -559,6 +574,7 @@ def __init__( gwk=gwk, sig2n_k=sig2n_k, cols2regi=cols2regi, + spat_impacts=spat_impacts, spat_diag=spat_diag, vm=vm, name_y=name_y, @@ -570,6 +586,7 @@ def __init__( name_gwk=name_gwk, name_ds=name_ds, latex=latex, + hard_bound=hard_bound, ) else: if regime_lag_sep == True: @@ -610,18 +627,27 @@ def __init__( self.rho = self.betas[-1] self.output.iat[-1, self.output.columns.get_loc('var_type')] = 'rho' self.predy_e, self.e_pred, warn = sp_att( - w, self.y, self.predy, yend2[:, -1].reshape(self.n, 1), self.rho - ) + w, self.y, self.predy, yend2[:, -1].reshape(self.n, 1), self.rho, hard_bound=hard_bound) set_warn(self, warn) self.regime_lag_sep = regime_lag_sep self.title = "SPATIAL " + self.title if slx_lags > 0: + for m in self.regimes_set: + r_output = self.output[(self.output['regime'] == str(m)) & (self.output['var_type'] == 'x')] + wx_index = r_output.index[-((len(r_output)-1)//(slx_lags+1)) * slx_lags:] + self.output.loc[wx_index, 'var_type'] = 'wx' self.title = " SPATIAL 2SLS WITH SLX (SPATIAL DURBIN MODEL) - REGIMES" self.other_top = _spat_pseudo_r2(self) + self.slx_lags = slx_lags + diag_out = None if spat_diag: diag_out = _spat_diag_out(self, w, 'yend') - else: - diag_out = None + if spat_impacts and slx_lags == 0: + impacts = _summary_impacts(self, _spmultiplier(w, self.rho)) + try: + diag_out += impacts + except TypeError: + diag_out = impacts output(reg=self, vm=vm, robust=robust, other_end=diag_out, latex=latex) def GM_Lag_Regimes_Multi( @@ -641,6 +667,7 @@ def GM_Lag_Regimes_Multi( gwk=None, sig2n_k=False, cols2regi="all", + spat_impacts=False, spat_diag=False, vm=False, name_y=None, @@ -652,6 +679,7 @@ def GM_Lag_Regimes_Multi( name_gwk=None, name_ds=None, latex=False, + hard_bound=False, ): # pool = mp.Pool(cores) self.name_ds = USER.set_name_ds(name_ds) @@ -771,7 +799,7 @@ def GM_Lag_Regimes_Multi( results[r].y, results[r].predy, results[r].yend[:, -1].reshape(results[r].n, 1), - results[r].rho, + results[r].rho, hard_bound=hard_bound ) set_warn(results[r], warn) results[r].w = w_i[r] @@ -807,12 +835,19 @@ def GM_Lag_Regimes_Multi( ] = results[r].h results[r].other_top = _spat_pseudo_r2(results[r]) results[r].other_mid = "" + if slx_lags > 0: + kx = (results[r].k - results[r].kstar - 1) // (slx_lags + 1) + var_types = ['x'] * (kx + 1) + ['wx'] * kx * slx_lags + ['yend'] * (len(results[r].name_yend) - 1) + ['rho'] + else: + var_types = ['x'] * len(results[r].name_x) + ['yend'] * (len(results[r].name_yend)-1) + ['rho'] + results[r].output = pd.DataFrame({'var_names': results[r].name_x + results[r].name_yend, + 'var_type': var_types, + 'regime': r, 'equation': r}) + self.output = pd.concat([self.output, results[r].output], ignore_index=True) if spat_diag: results[r].other_mid += _spat_diag_out(results[r], results[r].w, 'yend') - self.output = pd.concat([self.output, pd.DataFrame({'var_names': results[r].name_x + results[r].name_yend, - 'var_type': ['x'] * len(results[r].name_x) + [ - 'yend'] * (len(results[r].name_yend)-1) + ['rho'], - 'regime': r, 'equation': r})], ignore_index=True) + if spat_impacts and slx_lags == 0: + results[r].other_mid += _summary_impacts(results[r], _spmultiplier(results[r].w, results[r].rho)) counter += 1 self.multi = results if robust == "hac": @@ -823,9 +858,8 @@ def GM_Lag_Regimes_Multi( "Residuals treated as homoskedastic for the purpose of diagnostics.", ) self.chow = REGI.Chow(self) - if spat_diag: - # self._get_spat_diag_props(y, x, w, yend, q, w_lags, lag_q) - pass + #if spat_diag: + # self._get_spat_diag_props(y, x, w, yend, q, w_lags, lag_q) output(reg=self, vm=vm, robust=robust, other_end=False, latex=latex) def sp_att_reg(self, w_i, regi_ids, wy): @@ -929,9 +963,48 @@ def _work( model.name_h = model.name_x + model.name_q model.name_w = name_w model.name_regimes = name_regimes + model.slx_lags = slx_lags return model +class GM_Lag_Endog_Regimes(GM_Lag_Regimes): + def __init__( + self, y, x, w, n_clusters=None, quorum=-np.inf, trace=True, **kwargs): + + n = USER.check_arrays(y, x) + y = USER.check_y(y, n) + USER.check_weights(w, y, w_required=True) + + # Standardize the variables + x_std = (x - np.mean(x, axis=0)) / np.std(x, axis=0) + + if not n_clusters: + if quorum < 0: + quorum = np.max([(x.shape[1]+1)*10, 30]) + n_clusters_opt = x.shape[0]*0.70//quorum + if n_clusters_opt < 2: + raise ValueError( + "The combination of the values of `N` and `quorum` is not compatible with regimes estimation.") + sk_reg_results = Skater_reg().fit(n_clusters_opt, w, x_std, {'reg':BaseGM_Lag,'y':y,'x':x,'w':w}, quorum=quorum, trace=True) + n_clusters = optim_k([sk_reg_results._trace[i][1][2] for i in range(1, len(sk_reg_results._trace))]) + self.clusters = sk_reg_results._trace[n_clusters-1][0] + else: + try: + # Call the Skater_reg method based on GM_Lag + sk_reg_results = Skater_reg().fit(n_clusters, w, x_std, {'reg':BaseGM_Lag,'y':y,'x':x,'w':w}, quorum=quorum, trace=trace) + self.clusters = sk_reg_results.current_labels_ + except Exception as e: + if str(e) == "one or more input arrays have more columns than rows": + raise ValueError("One or more input ended up with more variables than observations. Please check your setting for `quorum`.") + else: + print("An error occurred:", e) + + self._trace = sk_reg_results._trace + self.SSR = [self._trace[i][1][2] for i in range(1, len(self._trace))] + + GM_Lag_Regimes.__init__(self, y, x, regimes=self.clusters, w=w, name_regimes='Skater_reg', **kwargs) + + def _test(): import doctest diff --git a/spreg/user_output.py b/spreg/user_output.py index cf7f9e68..376fd3dd 100755 --- a/spreg/user_output.py +++ b/spreg/user_output.py @@ -189,12 +189,29 @@ def set_name_q_sp(name_x, w_lags, name_q, lag_q, force_all=False): if lag_q: names = names + name_q sp_inst_names = [] - for j in names: - sp_inst_names.append("W_" + j) - if w_lags > 1: - for i in range(2, w_lags + 1): - for j in names: - sp_inst_names.append("W" + str(i) + "_" + j) + existing_names = set(names) + name_count = {} # Dictionary to store the count of each name + + for name in names: + if not name.startswith("W_"): + name_count[name] = 2 + else: + if name[2:] not in name_count: + name_count[name[2:]] = 2 + + for i in range(w_lags): + for name in names: + if name.startswith("W_"): + lag_name = f"W{name_count[name[2:]]}_{name[2:]}" + name_count[name[2:]] += 1 + else: + if ("W_" + name) not in existing_names: + lag_name = f"W_{name}" + else: + lag_name = f"W{name_count[name]}_{name}" + name_count[name] += 1 + sp_inst_names.append(lag_name) + existing_names.add(lag_name) return sp_inst_names diff --git a/spreg/utils.py b/spreg/utils.py index c369c74d..ccd74926 100755 --- a/spreg/utils.py +++ b/spreg/utils.py @@ -322,7 +322,7 @@ def _moments2eqs(A1, s, u): return [G, g] -def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): +def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None, hard_bound=False): """ Optimization of moments ... @@ -341,7 +341,10 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): solution or just the 1st. Default is 1st only. start : list List with initial values for the optimization - + hard_bound : boolean + If true, raises an exception if the estimated spatial + autoregressive parameter is outside the maximum/minimum bounds. + Returns ------- x, f, d : tuple @@ -367,13 +370,13 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): np.array([[float(par[0]), float(par[0]) ** 2.0]]).T, moments ) start = [0.0] - bounds = [(-1.0, 1.0)] + bounds = [(-0.99, 0.99)] if moments[0].shape[0] == 3: optim_par = lambda par: foptim_par( np.array([[float(par[0]), float(par[0]) ** 2.0, float(par[1])]]).T, moments ) start = [0.0, 1.0] - bounds = [(-1.0, 1.0), (0.0, None)] + bounds = [(-0.99, 0.99), (0.0, None)] if moments[0].shape[1] == 4: optim_par = lambda par: foptim_par( np.array( @@ -383,8 +386,13 @@ def optim_moments(moments_in, vcX=np.array([0]), all_par=False, start=None): ) if not start: start = [0.0, 1.0, 1.0] - bounds = [(-1.0, 1.0), (0.0, None), (0.0, None)] + bounds = [(-0.99, 0.99), (0.0, None), (0.0, None)] lambdaX = op.fmin_l_bfgs_b(optim_par, start, approx_grad=True, bounds=bounds) + + if hard_bound: + if abs(lambdaX[0][0]) >= 0.99: + raise Exception("Spatial parameter was outside the bounds of -0.99 and 0.99") + if all_par: return lambdaX[0] return lambdaX[0][0] @@ -734,7 +742,7 @@ def iter_msg(iteration, max_iter): return iter_stop -def sp_att(w, y, predy, w_y, rho): +def sp_att(w, y, predy, w_y, rho, hard_bound=False): xb = predy - rho * w_y if np.abs(rho) < 1: predy_sp = inverse_prod(w, xb, rho) @@ -742,11 +750,16 @@ def sp_att(w, y, predy, w_y, rho): # Note 1: Here if omitting pseudo-R2; If not, see Note 2. resid_sp = y - predy_sp else: - # warn = "Warning: Estimate for rho is outside the boundary (-1, 1). Computation of true inverse of W was required (slow)." - # predy_sp = inverse_prod(w, xb, rho, inv_method="true_inv") - warn = "*** WARNING: Estimate for spatial lag coefficient is outside the boundary (-1, 1). ***" - predy_sp = np.zeros(y.shape, float) - resid_sp = np.zeros(y.shape, float) + if hard_bound: + raise Exception( + "Spatial autoregressive parameter is outside the maximum/minimum bounds." + ) + else: + # warn = "Warning: Estimate for rho is outside the boundary (-1, 1). Computation of true inverse of W was required (slow)." + # predy_sp = inverse_prod(w, xb, rho, inv_method="true_inv") + warn = "*** WARNING: Estimate for spatial lag coefficient is outside the boundary (-1, 1). ***" + predy_sp = np.zeros(y.shape, float) + resid_sp = np.zeros(y.shape, float) # resid_sp = y - predy_sp #Note 2: Here if computing true inverse; If not, # see Note 1. return predy_sp, resid_sp, warn @@ -789,6 +802,56 @@ def RegressionProps_basic( if vm is not None: reg.vm = vm +def optim_k(trace, window_size=None): + """ + Finds optimal number of regimes for the endogenous spatial regimes model + using a method adapted from Mojena (1977)'s Rule Two. + + Parameters + ---------- + trace : list + List of SSR values for different number of regimes + window_size : integer + Size of the window to be used in the moving average + (Defaults to N//4) + Returns + ------- + i+window_size : integer + Optimal number of regimes + + Examples + -------- + >>> import libpysal as ps + >>> import numpy as np + >>> import spreg + >>> data = ps.io.open(ps.examples.get_path('NAT.dbf')) + >>> y = np.array(data.by_col('HR90')).reshape((-1,1)) + >>> x_var = ['PS90','UE90'] + >>> x = np.array([data.by_col(name) for name in x_var]).T + >>> w = ps.weights.Queen.from_shapefile(ps.examples.get_path("NAT.shp")) + >>> x_std = (x - np.mean(x,axis=0)) / np.std(x,axis=0) + >>> reg = spreg.Skater_reg().fit(20, w, x_std, {'reg':spreg.OLS,'y':y,'x':x}, quorum=100, trace=True) + >>> spreg.utils.optim_k([reg._trace[i][1][2] for i in range(1, len(reg._trace))]) + 9 + + + """ + + N = len(trace) + if not window_size: + window_size = N//4 # Mojena suggests from 70% to 90% + std_dev = [np.std(trace[i:i+window_size]) for i in range(N - window_size + 1)] + ma = np.convolve(trace, np.ones(window_size)/window_size, mode='valid') + treshold = [True] + i = 0 + while treshold[-1] and i < (N - window_size): + b = (6/(window_size*(window_size*window_size-1)) + )*((2*np.sum(np.arange(1, i+2)*trace[window_size-1:i+window_size]) + )-((window_size+1)*np.sum(trace[window_size-1:i+window_size]))) + l = (window_size-1)*b/2 + treshold.append(trace[i+window_size] < ma[i] - b - l - 2.75*std_dev[i]) + i += 1 + return i+window_size def _test(): import doctest