From cb630c46da6d76056706bc0faa99f24db127dabc Mon Sep 17 00:00:00 2001 From: palatej Date: Thu, 24 Oct 2024 19:33:31 +0200 Subject: [PATCH 1/2] Correction for the refreshing of models with Julian Easter (issue 51) --- CHANGELOG.md | 4 +++ .../base/core/tramo/TramoFactory.java | 25 +++++++++++++------ .../core/x13/regarima/RegArimaFactory.java | 13 +++++++++- .../netbeans/core/startup/Bundle.properties | 2 +- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 748e6784..5f45783f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed + +- ![STAT] Correct the refreshing of models with Julian Easter [#465](https://github.com/jdemetra/jdplus-main/issues/465) + ## [3.3.0] - 2024-10-21 ### Fixed diff --git a/jdplus-main-base/jdplus-tramoseats-base-parent/jdplus-tramoseats-base-core/src/main/java/jdplus/tramoseats/base/core/tramo/TramoFactory.java b/jdplus-main-base/jdplus-tramoseats-base-parent/jdplus-tramoseats-base-core/src/main/java/jdplus/tramoseats/base/core/tramo/TramoFactory.java index e56b96ef..a11c7f13 100644 --- a/jdplus-main-base/jdplus-tramoseats-base-parent/jdplus-tramoseats-base-core/src/main/java/jdplus/tramoseats/base/core/tramo/TramoFactory.java +++ b/jdplus-main-base/jdplus-tramoseats-base-parent/jdplus-tramoseats-base-core/src/main/java/jdplus/tramoseats/base/core/tramo/TramoFactory.java @@ -44,6 +44,7 @@ import java.time.LocalDateTime; import java.util.HashMap; import java.util.Map; +import jdplus.toolkit.base.api.timeseries.regression.JulianEasterVariable; /** * @@ -254,12 +255,22 @@ private void update(EasterSpec espec, Variable[] vars, CalendarSpec.Builder buil .filter(v -> ModellingUtility.isEaster(v)).findFirst(); if (fe.isPresent()) { Variable ev = fe.orElseThrow(); - EasterVariable evar = (EasterVariable) ev.getCore(); - espec = espec.toBuilder() - .test(false) - .duration(evar.getDuration()) - .coefficient(ev.getCoefficient(0)) - .build(); + if (ev.getCore() instanceof EasterVariable evar) { + espec = espec.toBuilder() + .test(false) + .duration(evar.getDuration()) + .coefficient(ev.getCoefficient(0)) + .build(); + } else if (ev.getCore() instanceof JulianEasterVariable evar) { + espec = espec.toBuilder() + .test(false) + .duration(evar.getDuration()) + .coefficient(ev.getCoefficient(0)) + .julian(true) + .build(); + } else { + espec = EasterSpec.none(); + } } else { espec = EasterSpec.none(); } @@ -339,7 +350,7 @@ private void freeVariables(RegressionSpec reg, TramoSpec domainSpec, TramoSpec.B RegressionSpec.Builder rbuilder = reg.toBuilder(); MeanSpec mean = reg.getMean(); if (mean.hasFixedCoefficient()) { - if (! dreg.getMean().hasFixedCoefficient()){ + if (!dreg.getMean().hasFixedCoefficient()) { mean = MeanSpec.mean(Parameter.initial(mean.getCoefficient().getValue())); } } diff --git a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-core/src/main/java/jdplus/x13/base/core/x13/regarima/RegArimaFactory.java b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-core/src/main/java/jdplus/x13/base/core/x13/regarima/RegArimaFactory.java index 9eb5a576..a37fa2a5 100644 --- a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-core/src/main/java/jdplus/x13/base/core/x13/regarima/RegArimaFactory.java +++ b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-core/src/main/java/jdplus/x13/base/core/x13/regarima/RegArimaFactory.java @@ -44,6 +44,7 @@ import java.time.LocalDateTime; import java.util.HashMap; import java.util.Map; +import jdplus.toolkit.base.api.timeseries.regression.JulianEasterVariable; /** * @@ -217,13 +218,23 @@ private void update(EasterSpec espec, Variable[] vars, RegressionSpec.Builder bu .filter(v -> ModellingUtility.isEaster(v)).findFirst(); if (fe.isPresent()) { Variable ev = fe.orElseThrow(); - EasterVariable evar = (EasterVariable) ev.getCore(); + if (ev.getCore() instanceof EasterVariable evar) { espec = espec.toBuilder() .type(EasterSpec.Type.Easter) .test(RegressionTestSpec.None) .duration(evar.getDuration()) .coefficient(ev.getCoefficient(0)) .build(); + } else if (ev.getCore() instanceof JulianEasterVariable evar) { + espec = espec.toBuilder() + .type(EasterSpec.Type.Easter) + .test(RegressionTestSpec.None) + .duration(evar.getDuration()) + .coefficient(ev.getCoefficient(0)) + .build(); + } else { + espec = EasterSpec.none(); + } } else { espec = EasterSpec.none(); } diff --git a/jdplus-main-desktop/jdplus-main-desktop-branding/src/main/nbm-branding/core/core.jar/org/netbeans/core/startup/Bundle.properties b/jdplus-main-desktop/jdplus-main-desktop-branding/src/main/nbm-branding/core/core.jar/org/netbeans/core/startup/Bundle.properties index 2b23e22c..9ac80be6 100644 --- a/jdplus-main-desktop/jdplus-main-desktop-branding/src/main/nbm-branding/core/core.jar/org/netbeans/core/startup/Bundle.properties +++ b/jdplus-main-desktop/jdplus-main-desktop-branding/src/main/nbm-branding/core/core.jar/org/netbeans/core/startup/Bundle.properties @@ -1,7 +1,7 @@ # BEWARE that this file might have been copied and filtered by maven-resources-plugin # Original file is in src/main/resources -currentVersion=v3.2.5-SNAPSHOT +currentVersion=v3.3.1-SNAPSHOT # The following properties customize splash screen # See https://github.com/apache/netbeans/blob/ac3936a4bdc7828721f45bba2feece190bc99a43/nb/ide.branding/core.startup/src/org/netbeans/core/startup/Bundle_nb.properties#L22 From e468d1b6af1156176df82198d93bff70e8024683 Mon Sep 17 00:00:00 2001 From: palatej Date: Wed, 6 Nov 2024 12:04:56 +0100 Subject: [PATCH 2/2] Correction in modified ljung-box Qs) Adaptative splines Bug in protobuf (X13 results) --- .../formatters/RegressionItemFormatter.java | 21 ++-- .../formatters/StatisticalTestFormatter.java | 18 ++-- .../splines/AdaptativePeriodicSplines.java | 95 +++++++++++++------ .../base/core/stats/tests/LjungBox.java | 18 +++- .../AdaptativePeriodicSplinesTest.java | 11 ++- .../x13/base/protobuf/X13ProtosUtility.java | 2 +- .../test/java/jdplus/x13/base/r/X13Test.java | 28 +++++- 7 files changed, 128 insertions(+), 65 deletions(-) diff --git a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/RegressionItemFormatter.java b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/RegressionItemFormatter.java index 12e24fd6..ee4fa992 100644 --- a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/RegressionItemFormatter.java +++ b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/RegressionItemFormatter.java @@ -66,24 +66,29 @@ public String format(Object obj, int item, Locale locale) { ++item; } switch (Math.abs(item)) { - case 1: + case 1 -> { return StringFormatter.cleanup(reg.getDescription()); - case 2: + } + case 2 -> { return newFormat6(locale).format(reg.getCoefficient()); - case 3: + } + case 3 -> { if (reg.getStdError() == 0) { return null; } else { return newFormat4(locale).format(reg.getCoefficient() / reg.getStdError()); } - case 4: + } + case 4 -> { return newFormat4(locale).format(reg.getPvalue()); -// case 5: -// return fmt.format(reg.getStdError()); - default: + } + default -> { return null; + } } - } +// case 5: +// return fmt.format(reg.getStdError()); + } private String format(RegressionItem reg, Locale locale) { StringBuilder builder = new StringBuilder(); diff --git a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/StatisticalTestFormatter.java b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/StatisticalTestFormatter.java index 805dcf38..330d83da 100644 --- a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/StatisticalTestFormatter.java +++ b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-api/src/main/java/jdplus/toolkit/base/api/information/formatters/StatisticalTestFormatter.java @@ -48,17 +48,11 @@ public String format(Object obj, int item, Locale locale) { StatisticalTest test = (StatisticalTest)obj; if (item == 0) return newFormat6(locale).format(test.getValue()); - if (test.getDescription() != null) - ++item; - switch (Math.abs(item)) { - case 1: - return test.getDescription(); - case 2: - return newFormat6(locale).format(test.getValue()); - case 3: - return newFormat4(locale).format(test.getPvalue()); - default: - return null; - } + return switch (Math.abs(item)) { + case 1 -> newFormat6(locale).format(test.getValue()); + case 2 -> newFormat4(locale).format(test.getPvalue()); + case 3 -> test.getDescription(); + default -> null; + }; } } diff --git a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplines.java b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplines.java index 054b07a9..f5bbc8c8 100644 --- a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplines.java +++ b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplines.java @@ -16,12 +16,16 @@ */ package jdplus.toolkit.base.core.math.splines; +import java.util.Arrays; import jdplus.toolkit.base.api.data.DoubleSeq; import jdplus.toolkit.base.core.data.DataBlock; import jdplus.toolkit.base.core.math.matrices.FastMatrix; import jdplus.toolkit.base.core.math.matrices.LowerTriangularMatrix; import jdplus.toolkit.base.core.math.matrices.decomposition.ElementaryTransformations; import jdplus.toolkit.base.core.math.polynomials.UnitRoots; +import jdplus.toolkit.base.core.stats.linearmodel.LeastSquaresResults; +import jdplus.toolkit.base.core.stats.linearmodel.LinearModel; +import jdplus.toolkit.base.core.stats.linearmodel.Ols; /** * @@ -33,12 +37,19 @@ public class AdaptativePeriodicSplines { @lombok.Builder(builderClassName = "Builder") public static class Specification { + // Points DoubleSeq x, y; + // Order of the B-Splines int splineOrder; + // Period of the splines double period; + // Knots on which the splines are built double[] knots; + // Fixed knots (can't be removed) int[] fixedKnots; + double precision; + // Selection threshold double selectionThreshold; int maxIter, minKnots; @@ -69,11 +80,13 @@ public static Builder builder() { double[] knots = spec.getKnots(); double P = spec.getPeriod(); BSplines.BSpline bs = BSplines.periodic(k, knots, P); + // B is the matrix of the regression variables corresponding to the splines B = BSplines.splines(bs, spec.getX()); FastMatrix Bt = B.transpose(); ElementaryTransformations.fastGivensTriangularize(Bt); int q = knots.length; DoubleSeq coeff = UnitRoots.D(1, k).coefficients(); + // Differencing matrix D = FastMatrix.square(q); for (int i = 0; i < coeff.length(); ++i) { D.subDiagonal(-i).set(coeff.get(i)); @@ -118,8 +131,8 @@ public static AdaptativePeriodicSplines of(Specification spec) { public boolean process(double lambda) { int q = w.length; - int n = 0; int[] fixedKnots = spec.getFixedKnots(); + int nfixed = fixedKnots == null ? 0 : fixedKnots.length; for (; niter < spec.getMaxIter(); ++niter) { FastMatrix LBp = B2.extract(0, q, 0, q); LBp.copy(LB); @@ -133,47 +146,67 @@ public boolean process(double lambda) { LowerTriangularMatrix.solvexL(LBp, A); // New w for (int i = 0; i < q; ++i) { - - double da = D.row(i).dot(A); - double wcur = 1 / (da * da + 1e-10); - w[i] = wcur; - z[i] = da * da * wcur; - } - // e=y-Xb - DataBlock e = DataBlock.of(spec.y); - e.addAProduct(-1, B.rowsIterator(), A); - res = e; - boolean stop = A.distance(a) < spec.getPrecision(); - a = A; - if (fixedKnots != null) { - for (int i = 0; i < fixedKnots.length; ++i) { - w[fixedKnots[i]] = 0; - z[fixedKnots[i]] = 1; + if (w[i] != 0) { + double da = D.row(i).dot(A); + double wcur = 1 / (da * da + 1e-10); + w[i] = wcur; + // z[i] = 0 (if da = 0) or 1 (if da >> 1.e5) + z[i] = da * da * wcur; } } - double ll = -0.5 * e.ssq() / sigma2; - n = 0; - for (int i = 0; i < q; ++i) { - if (z[i] >= spec.getSelectionThreshold()) { - ++n; - } - } - if (n < spec.minKnots || (fixedKnots!= null && n == fixedKnots.length)) { - break; - } - aic = -2 * (ll - n); - bic = -2 * ll + Math.log(B.getRowsCount()) * n; + double da = A.distance(a); + a = A; + boolean stop = da < spec.getPrecision(); if (stop) { break; } } - selectedKnots = new int[n]; + int n = 0; + for (int i = 0; i < q; ++i) { + if (z[i] >= spec.getSelectionThreshold()) { + ++n; + } + } + selectedKnots = new int[n + nfixed]; for (int i = 0, j = 0; i < q; ++i) { if (z[i] >= spec.getSelectionThreshold()) { selectedKnots[j++] = i; } } - + for (int i = 0; i < nfixed; ++i) { + selectedKnots[n + i] = fixedKnots[i]; + } + n+=nfixed; + Arrays.sort(selectedKnots); + double[] ksel = new double[n]; + double[] knots = spec.getKnots(); + double P = spec.getPeriod(); + for (int i = 0; i < ksel.length; ++i) { + ksel[i] = knots[selectedKnots[i]]; + } + int k = spec.getSplineOrder(); + BSplines.BSpline bs = BSplines.periodic(k, ksel, P); + FastMatrix Bnew = BSplines.splines(bs, spec.getX()); + LinearModel lm = LinearModel.builder() + .y(spec.getY()) + .addX(Bnew) + .build(); + LeastSquaresResults rslt = Ols.compute(lm); + double ll = -0.5*rslt.getErrorSumOfSquares()/sigma2; + + aic = -2 * (ll - n); + bic = -2 * ll + Math.log(B.getRowsCount()) * n; + +// if (n+nfixed < spec.minKnots || n == 0) { +// break; +// } +// double ll = -0.5 * e.ssq() / sigma2; +// // e=y-Xb +// DataBlock e = DataBlock.of(spec.y); +// e.addAProduct(-1, B.rowsIterator(), A); +// res = e; +// aic = -2 * (ll - n); +// bic = -2 * ll + Math.log(B.getRowsCount()) * n; return niter < spec.maxIter; } diff --git a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/stats/tests/LjungBox.java b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/stats/tests/LjungBox.java index 19449b70..135c1ba6 100644 --- a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/stats/tests/LjungBox.java +++ b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/main/java/jdplus/toolkit/base/core/stats/tests/LjungBox.java @@ -42,13 +42,15 @@ public static int defaultAutoCorrelationsCount(int period) { return 4 * period; } } - + public static int defaultAutoCorrelationsCount(int period, int nobs) { int lbdf; switch (period) { - case 12 -> lbdf = 24; - case 1 -> lbdf = 8; + case 12 -> + lbdf = 24; + case 1 -> + lbdf = 8; default -> { lbdf = 4 * period; if (nobs <= 22 && period == 4) { @@ -61,7 +63,6 @@ public static int defaultAutoCorrelationsCount(int period, int nobs) { } return lbdf; } - private int lag = 1; private int k = 12; @@ -146,14 +147,21 @@ public LjungBox sign(int s) { return this; } - public StatisticalTest build() { + private double value() { double res = 0.0; for (int i = 1; i <= k; i++) { double ai = autoCorrelations.applyAsDouble(i * lag); if (sign == 0 || (sign == 1 && ai > 0) || (sign == -1 && ai < 0)) { res += ai * ai / (n - i * lag); + } else if (i == 1) { + return 0; } } + return res; + } + + public StatisticalTest build() { + double res = value(); double val = res * n * (n + 2); Chi2 chi = new Chi2(lag == 1 ? (k - nhp) : k); return TestsUtility.testOf(val, chi, TestType.Upper); diff --git a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/test/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplinesTest.java b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/test/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplinesTest.java index 89455157..ff5aa7af 100644 --- a/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/test/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplinesTest.java +++ b/jdplus-main-base/jdplus-toolkit-base-parent/jdplus-toolkit-base-core/src/test/java/jdplus/toolkit/base/core/math/splines/AdaptativePeriodicSplinesTest.java @@ -47,8 +47,8 @@ public static void main(String[] arg) { knots[i] = i * c; } - int nyears = 20; - int ny = (int) (nyears * P); + int nyears = 5; + int ny = (int) (nyears * P + 1); int jump = 4; int nq = q / jump; int[] fixedKnots = new int[nq]; @@ -56,7 +56,7 @@ public static void main(String[] arg) { fixedKnots[i] = i * jump; } - DoubleSeq m = DoubleSeq.onMapping(ny, i -> i * c - P * (int) ((i * c) / P)); + DoubleSeq m = DoubleSeq.onMapping(ny, i -> i - P * (int) (i / P)); SymmetricFilter sf = LocalPolynomialFilters.of(26, 1, DiscreteKernel.uniform(26)); IFiniteFilter[] afilters = AsymmetricFiltersFactory.mmsreFilters(sf, 0, new double[]{1}, null); IFiniteFilter[] lfilters = afilters.clone(); @@ -76,8 +76,9 @@ public static void main(String[] arg) { .y(Y) .period(P) .knots(knots) - .minKnots(15) - .fixedKnots(fixedKnots) + .minKnots(10) + .splineOrder(4) + // .fixedKnots(fixedKnots) .build(); AdaptativePeriodicSplines kernel = AdaptativePeriodicSplines.of(spec); diff --git a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-protobuf/src/main/java/jdplus/x13/base/protobuf/X13ProtosUtility.java b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-protobuf/src/main/java/jdplus/x13/base/protobuf/X13ProtosUtility.java index 1e936903..c89b250d 100644 --- a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-protobuf/src/main/java/jdplus/x13/base/protobuf/X13ProtosUtility.java +++ b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-protobuf/src/main/java/jdplus/x13/base/protobuf/X13ProtosUtility.java @@ -107,7 +107,7 @@ public SeasonalFilterOption convert(SeasonalFilter sf) { public List convert(SeasonalFilterOption[] sf) { if (sf == null || sf.length == 0) { - return null; + return List.of(); } SeasonalFilterOption sf0 = sf[0]; boolean same = true; diff --git a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-r/src/test/java/jdplus/x13/base/r/X13Test.java b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-r/src/test/java/jdplus/x13/base/r/X13Test.java index 6d48b636..1a3f3790 100644 --- a/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-r/src/test/java/jdplus/x13/base/r/X13Test.java +++ b/jdplus-main-base/jdplus-x13-base-parent/jdplus-x13-base-r/src/test/java/jdplus/x13/base/r/X13Test.java @@ -5,6 +5,7 @@ */ package jdplus.x13.base.r; +import jdplus.x13.base.api.x11.X11Spec; import tck.demetra.data.Data; import jdplus.x13.base.api.x13.X13Spec; import jdplus.x13.base.core.x13.X13Output; @@ -17,7 +18,7 @@ * @author PALATEJ */ public class X13Test { - + public X13Test() { } @@ -32,8 +33,29 @@ public void testProd() { // } byte[] sbytes = X13.toBuffer(rslt.getEstimationSpec()); X13Spec spec = X13.specOf(sbytes); - + assertTrue(spec != null); } - + + @Test + public void testProd_nosa() { + + X11Spec x11 = X13Spec.RSA0.getX11().toBuilder() + .seasonal(false) + .build(); + + X13Spec spec = X13Spec.RSA0.toBuilder() + .x11(x11) + .build(); + + X13Output rslt = X13.fullProcess(Data.TS_PROD, spec, null); +// if (outliers != null) { +// for (int i = 0; i < outliers.length; ++i) { +// System.out.println(outliers[i]); +// } +// } + byte[] brslt = X13.toBuffer(rslt); + + assertTrue(brslt != null); + } }