Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Julian Easter, LjungBox #475

Merged
merged 2 commits into from
Nov 6, 2024
Merged

Julian Easter, LjungBox #475

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cb630c4
Correction for the refreshing of models with Julian Easter (issue 51)
Oct 24, 2024
e468d1b
Correction in modified ljung-box Qs)
palatej Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

### Fixed

- ![STAT] Correct the refreshing of models with Julian Easter [#465](https://github.com/jdemetra/jdplus-main/issues/465)

## [3.3.0] - 2024-10-21

### Fixed
Expand Down
21 changes: 13 additions & 8 deletions ...src/main/java/jdplus/toolkit/base/api/information/formatters/RegressionItemFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,24 +66,29 @@ public String format(Object obj, int item, Locale locale) {
++item;
}
switch (Math.abs(item)) {
case 1:
case 1 -> {
return StringFormatter.cleanup(reg.getDescription());
case 2:
}
case 2 -> {
return newFormat6(locale).format(reg.getCoefficient());
case 3:
}
case 3 -> {
if (reg.getStdError() == 0) {
return null;
} else {
return newFormat4(locale).format(reg.getCoefficient() / reg.getStdError());
}
case 4:
}
case 4 -> {
return newFormat4(locale).format(reg.getPvalue());
// case 5:
// return fmt.format(reg.getStdError());
default:
}
default -> {
return null;
}
}
}
// case 5:
// return fmt.format(reg.getStdError());
}

private String format(RegressionItem reg, Locale locale) {
StringBuilder builder = new StringBuilder();
Expand Down
18 changes: 6 additions & 12 deletions ...rc/main/java/jdplus/toolkit/base/api/information/formatters/StatisticalTestFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,17 +48,11 @@ public String format(Object obj, int item, Locale locale) {
StatisticalTest test = (StatisticalTest)obj;
if (item == 0)
return newFormat6(locale).format(test.getValue());
if (test.getDescription() != null)
++item;
switch (Math.abs(item)) {
case 1:
return test.getDescription();
case 2:
return newFormat6(locale).format(test.getValue());
case 3:
return newFormat4(locale).format(test.getPvalue());
default:
return null;
}
return switch (Math.abs(item)) {
case 1 -> newFormat6(locale).format(test.getValue());
case 2 -> newFormat4(locale).format(test.getPvalue());
case 3 -> test.getDescription();
default -> null;
};
}
}
95 changes: 64 additions & 31 deletions ...e-core/src/main/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplines.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,16 @@
*/
package jdplus.toolkit.base.core.math.splines;

import java.util.Arrays;
import jdplus.toolkit.base.api.data.DoubleSeq;
import jdplus.toolkit.base.core.data.DataBlock;
import jdplus.toolkit.base.core.math.matrices.FastMatrix;
import jdplus.toolkit.base.core.math.matrices.LowerTriangularMatrix;
import jdplus.toolkit.base.core.math.matrices.decomposition.ElementaryTransformations;
import jdplus.toolkit.base.core.math.polynomials.UnitRoots;
import jdplus.toolkit.base.core.stats.linearmodel.LeastSquaresResults;
import jdplus.toolkit.base.core.stats.linearmodel.LinearModel;
import jdplus.toolkit.base.core.stats.linearmodel.Ols;

/**
*
Expand All @@ -33,12 +37,19 @@ public class AdaptativePeriodicSplines {
@lombok.Builder(builderClassName = "Builder")
public static class Specification {

// Points
DoubleSeq x, y;
// Order of the B-Splines
int splineOrder;
// Period of the splines
double period;
// Knots on which the splines are built
double[] knots;
// Fixed knots (can't be removed)
int[] fixedKnots;

double precision;
// Selection threshold
double selectionThreshold;
int maxIter, minKnots;

Expand Down Expand Up @@ -69,11 +80,13 @@ public static Builder builder() {
double[] knots = spec.getKnots();
double P = spec.getPeriod();
BSplines.BSpline bs = BSplines.periodic(k, knots, P);
// B is the matrix of the regression variables corresponding to the splines
B = BSplines.splines(bs, spec.getX());
FastMatrix Bt = B.transpose();
ElementaryTransformations.fastGivensTriangularize(Bt);
int q = knots.length;
DoubleSeq coeff = UnitRoots.D(1, k).coefficients();
// Differencing matrix
D = FastMatrix.square(q);
for (int i = 0; i < coeff.length(); ++i) {
D.subDiagonal(-i).set(coeff.get(i));
Expand Down Expand Up @@ -118,8 +131,8 @@ public static AdaptativePeriodicSplines of(Specification spec) {

public boolean process(double lambda) {
int q = w.length;
int n = 0;
int[] fixedKnots = spec.getFixedKnots();
int nfixed = fixedKnots == null ? 0 : fixedKnots.length;
for (; niter < spec.getMaxIter(); ++niter) {
FastMatrix LBp = B2.extract(0, q, 0, q);
LBp.copy(LB);
Expand All @@ -133,47 +146,67 @@ public boolean process(double lambda) {
LowerTriangularMatrix.solvexL(LBp, A);
// New w
for (int i = 0; i < q; ++i) {

double da = D.row(i).dot(A);
double wcur = 1 / (da * da + 1e-10);
w[i] = wcur;
z[i] = da * da * wcur;
}
// e=y-Xb
DataBlock e = DataBlock.of(spec.y);
e.addAProduct(-1, B.rowsIterator(), A);
res = e;
boolean stop = A.distance(a) < spec.getPrecision();
a = A;
if (fixedKnots != null) {
for (int i = 0; i < fixedKnots.length; ++i) {
w[fixedKnots[i]] = 0;
z[fixedKnots[i]] = 1;
if (w[i] != 0) {
double da = D.row(i).dot(A);
double wcur = 1 / (da * da + 1e-10);
w[i] = wcur;
// z[i] = 0 (if da = 0) or 1 (if da >> 1.e5)
z[i] = da * da * wcur;
}
}
double ll = -0.5 * e.ssq() / sigma2;
n = 0;
for (int i = 0; i < q; ++i) {
if (z[i] >= spec.getSelectionThreshold()) {
++n;
}
}
if (n < spec.minKnots || (fixedKnots!= null && n == fixedKnots.length)) {
break;
}
aic = -2 * (ll - n);
bic = -2 * ll + Math.log(B.getRowsCount()) * n;
double da = A.distance(a);
a = A;
boolean stop = da < spec.getPrecision();
if (stop) {
break;
}
}
selectedKnots = new int[n];
int n = 0;
for (int i = 0; i < q; ++i) {
if (z[i] >= spec.getSelectionThreshold()) {
++n;
}
}
selectedKnots = new int[n + nfixed];
for (int i = 0, j = 0; i < q; ++i) {
if (z[i] >= spec.getSelectionThreshold()) {
selectedKnots[j++] = i;
}
}

for (int i = 0; i < nfixed; ++i) {
selectedKnots[n + i] = fixedKnots[i];
}
n+=nfixed;
Arrays.sort(selectedKnots);
double[] ksel = new double[n];
double[] knots = spec.getKnots();
double P = spec.getPeriod();
for (int i = 0; i < ksel.length; ++i) {
ksel[i] = knots[selectedKnots[i]];
}
int k = spec.getSplineOrder();
BSplines.BSpline bs = BSplines.periodic(k, ksel, P);
FastMatrix Bnew = BSplines.splines(bs, spec.getX());
LinearModel lm = LinearModel.builder()
.y(spec.getY())
.addX(Bnew)
.build();
LeastSquaresResults rslt = Ols.compute(lm);
double ll = -0.5*rslt.getErrorSumOfSquares()/sigma2;

aic = -2 * (ll - n);
bic = -2 * ll + Math.log(B.getRowsCount()) * n;

// if (n+nfixed < spec.minKnots || n == 0) {
// break;
// }
// double ll = -0.5 * e.ssq() / sigma2;
// // e=y-Xb
// DataBlock e = DataBlock.of(spec.y);
// e.addAProduct(-1, B.rowsIterator(), A);
// res = e;
// aic = -2 * (ll - n);
// bic = -2 * ll + Math.log(B.getRowsCount()) * n;
return niter < spec.maxIter;
}

Expand Down
18 changes: 13 additions & 5 deletions ...jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/stats/tests/LjungBox.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,15 @@ public static int defaultAutoCorrelationsCount(int period) {
return 4 * period;
}
}

public static int defaultAutoCorrelationsCount(int period, int nobs) {
int lbdf;

switch (period) {
case 12 -> lbdf = 24;
case 1 -> lbdf = 8;
case 12 ->
lbdf = 24;
case 1 ->
lbdf = 8;
default -> {
lbdf = 4 * period;
if (nobs <= 22 && period == 4) {
Expand All @@ -61,7 +63,6 @@ public static int defaultAutoCorrelationsCount(int period, int nobs) {
}
return lbdf;
}


private int lag = 1;
private int k = 12;
Expand Down Expand Up @@ -146,14 +147,21 @@ public LjungBox sign(int s) {
return this;
}

public StatisticalTest build() {
private double value() {
double res = 0.0;
for (int i = 1; i <= k; i++) {
double ai = autoCorrelations.applyAsDouble(i * lag);
if (sign == 0 || (sign == 1 && ai > 0) || (sign == -1 && ai < 0)) {
res += ai * ai / (n - i * lag);
} else if (i == 1) {
return 0;
}
}
return res;
}

public StatisticalTest build() {
double res = value();
double val = res * n * (n + 2);
Chi2 chi = new Chi2(lag == 1 ? (k - nhp) : k);
return TestsUtility.testOf(val, chi, TestType.Upper);
Expand Down
11 changes: 6 additions & 5 deletions ...re/src/test/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplinesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@ public static void main(String[] arg) {
knots[i] = i * c;
}

int nyears = 20;
int ny = (int) (nyears * P);
int nyears = 5;
int ny = (int) (nyears * P + 1);
int jump = 4;
int nq = q / jump;
int[] fixedKnots = new int[nq];
for (int i = 0; i < nq; ++i) {
fixedKnots[i] = i * jump;
}

DoubleSeq m = DoubleSeq.onMapping(ny, i -> i * c - P * (int) ((i * c) / P));
DoubleSeq m = DoubleSeq.onMapping(ny, i -> i - P * (int) (i / P));
SymmetricFilter sf = LocalPolynomialFilters.of(26, 1, DiscreteKernel.uniform(26));
IFiniteFilter[] afilters = AsymmetricFiltersFactory.mmsreFilters(sf, 0, new double[]{1}, null);
IFiniteFilter[] lfilters = afilters.clone();
Expand All @@ -76,8 +76,9 @@ public static void main(String[] arg) {
.y(Y)
.period(P)
.knots(knots)
.minKnots(15)
.fixedKnots(fixedKnots)
.minKnots(10)
.splineOrder(4)
// .fixedKnots(fixedKnots)
.build();

AdaptativePeriodicSplines kernel = AdaptativePeriodicSplines.of(spec);
Expand Down
25 changes: 18 additions & 7 deletions ...us-tramoseats-base-core/src/main/java/jdplus/tramoseats/base/core/tramo/TramoFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
import jdplus.toolkit.base.api.timeseries.regression.JulianEasterVariable;

/**
*
Expand Down Expand Up @@ -254,12 +255,22 @@ private void update(EasterSpec espec, Variable[] vars, CalendarSpec.Builder buil
.filter(v -> ModellingUtility.isEaster(v)).findFirst();
if (fe.isPresent()) {
Variable ev = fe.orElseThrow();
EasterVariable evar = (EasterVariable) ev.getCore();
espec = espec.toBuilder()
.test(false)
.duration(evar.getDuration())
.coefficient(ev.getCoefficient(0))
.build();
if (ev.getCore() instanceof EasterVariable evar) {
espec = espec.toBuilder()
.test(false)
.duration(evar.getDuration())
.coefficient(ev.getCoefficient(0))
.build();
} else if (ev.getCore() instanceof JulianEasterVariable evar) {
espec = espec.toBuilder()
.test(false)
.duration(evar.getDuration())
.coefficient(ev.getCoefficient(0))
.julian(true)
.build();
} else {
espec = EasterSpec.none();
}
} else {
espec = EasterSpec.none();
}
Expand Down Expand Up @@ -339,7 +350,7 @@ private void freeVariables(RegressionSpec reg, TramoSpec domainSpec, TramoSpec.B
RegressionSpec.Builder rbuilder = reg.toBuilder();
MeanSpec mean = reg.getMean();
if (mean.hasFixedCoefficient()) {
if (! dreg.getMean().hasFixedCoefficient()){
if (!dreg.getMean().hasFixedCoefficient()) {
mean = MeanSpec.mean(Parameter.initial(mean.getCoefficient().getValue()));
}
}
Expand Down
13 changes: 12 additions & 1 deletion ...jdplus-x13-base-core/src/main/java/jdplus/x13/base/core/x13/regarima/RegArimaFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.Map;
import jdplus.toolkit.base.api.timeseries.regression.JulianEasterVariable;

/**
*
Expand Down Expand Up @@ -217,13 +218,23 @@ private void update(EasterSpec espec, Variable[] vars, RegressionSpec.Builder bu
.filter(v -> ModellingUtility.isEaster(v)).findFirst();
if (fe.isPresent()) {
Variable ev = fe.orElseThrow();
EasterVariable evar = (EasterVariable) ev.getCore();
if (ev.getCore() instanceof EasterVariable evar) {
espec = espec.toBuilder()
.type(EasterSpec.Type.Easter)
.test(RegressionTestSpec.None)
.duration(evar.getDuration())
.coefficient(ev.getCoefficient(0))
.build();
} else if (ev.getCore() instanceof JulianEasterVariable evar) {
espec = espec.toBuilder()
.type(EasterSpec.Type.Easter)
.test(RegressionTestSpec.None)
.duration(evar.getDuration())
.coefficient(ev.getCoefficient(0))
.build();
} else {
espec = EasterSpec.none();
}
} else {
espec = EasterSpec.none();
}
Expand Down
Loading